mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-03-05 20:59:13 -08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Resolve conflicts.

Browse source 
Signed-off-by: Harkishen-Singh <harkishensingh@hotmail.com>
This commit is contained in:
Harkishen-Singh 2020-08-18 12:36:03 +05:30
parent 74a9422147 10363c442a
commit cadf8f11bc
3223 changed files with 402210 additions and 194875 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
.circleci/config.yml
View file

@ -2,14 +2,16 @@
version: 2.1
orbs:
prometheus: prometheus/prometheus@0.1.0
prometheus: prometheus/prometheus@0.4.0
go: circleci/go@0.2.0
win: circleci/windows@2.3.0
executors:
# Whenever the Go version is updated here, .travis.yml and .promu.yml
# Whenever the Go version is updated here, .promu.yml
# should also be updated.
golang:
docker:
- image: circleci/golang:1.12
- image: circleci/golang:1.15-node
fuzzit:
docker:
@ -21,21 +23,55 @@ jobs:
steps:
- prometheus/setup_environment
- go/load-cache:
key: v1
- restore_cache:
keys:
- v3-npm-deps-{{ checksum "web/ui/react-app/yarn.lock" }}
- v3-npm-deps-
- run:
command: make
environment:
# Run garbage collection more aggresively to avoid getting OOMed during the lint phase.
# Run garbage collection more aggressively to avoid getting OOMed during the lint phase.
GOGC: "20"
# By default Go uses GOMAXPROCS but a Circle CI executor has many
# cores (> 30) while the CPU and RAM resources are throttled. If we
# don't limit this to the number of allocated cores, the job is
# likely to get OOMed and killed.
GOOPTS: "-p 2"
- prometheus/check_proto
GOMAXPROCS: "2"
- prometheus/check_proto:
version: "3.12.3"
- prometheus/store_artifact:
file: prometheus
- prometheus/store_artifact:
file: promtool
- go/save-cache:
key: v1
- save_cache:
key: v3-npm-deps-{{ checksum "web/ui/react-app/yarn.lock" }}
paths:
- /home/circleci/.cache/yarn
- store_test_results:
path: test-results
test_windows:
executor: win/default
working_directory: /go/src/github.com/prometheus/prometheus
steps:
- checkout
- run:
# Temporary workaround until circleci updates go.
shell: bash
command: |
choco upgrade -y golang
- run:
shell: bash
command: |
(cd web/ui && GOOS= GOARCH= go generate -mod=vendor)
go test -mod=vendor -vet=off -test.v `go list ./...|grep -Exv "(github.com/prometheus/prometheus/discovery.*|github.com/prometheus/prometheus/config|github.com/prometheus/prometheus/web)"`
environment:
GOGC: "20"
GOOPTS: "-p 2 -mod=vendor"
fuzzit_regression:
executor: fuzzit
working_directory: /go/src/github.com/prometheus/prometheus
@ -51,11 +87,11 @@ jobs:
- setup_remote_docker
- run: ./fuzzit.sh fuzzing
makefile_sync:
repo_sync:
executor: golang
steps:
- checkout
- run: ./scripts/sync_makefiles.sh
- run: ./scripts/sync_repo_files.sh
workflows:
version: 2
@ -65,6 +101,10 @@ workflows:
filters:
tags:
only: /.*/
- test_windows:
filters:
tags:
only: /.*/
- fuzzit_regression:
filters:
tags:
@ -82,6 +122,7 @@ workflows:
filters:
branches:
only: master
image: circleci/golang:1-node
- prometheus/publish_release:
context: org-context
requires:
@ -92,6 +133,7 @@ workflows:
only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
branches:
ignore: /.*/
image: circleci/golang:1-node
nightly:
triggers:
- schedule:
@ -101,7 +143,7 @@ workflows:
only:
- master
jobs:
- makefile_sync:
- repo_sync:
context: org-context
- fuzzit_fuzzing:
context: org-context

2
.dockerignore
View file

@ -5,3 +5,5 @@ data/
!.build/linux-amd64/
!.build/linux-armv7/
!.build/linux-arm64/
!.build/linux-ppc64le/
!.build/linux-s390x/

14
.github/ISSUE_TEMPLATE.md → .github/ISSUE_TEMPLATE/bug_report.md vendored
View file

@ -1,3 +1,10 @@
---
name: Bug report
about: Create a report to help us improve.
title: ''
assignees: ''
---
<!--
Please do *NOT* ask usage questions in Github issues.
@ -10,12 +17,7 @@
You can find more information at: https://prometheus.io/community/
-->
## Proposal
**Use case. Why is this important?**
*“Nice to have” is not a good use case. :)*
## Bug Report
**What did you do?**
**What did you expect to see?**
@ -47,8 +49,6 @@ insert configuration here (if relevant to the issue)
```
* Logs:
```
insert Prometheus and Alertmanager logs relevant to the issue here

8
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View file

@ -0,0 +1,8 @@
blank_issues_enabled: false
contact_links:
- name: Users mailing list
url: https://groups.google.com/forum/#!forum/prometheus-users
about: Please ask and answer usage questions here.
- name: Prometheus community
url: https://prometheus.io/community/
about: List of communication channels for the Prometheus community.

24
.github/ISSUE_TEMPLATE/feature_request.md vendored Normal file
View file

@ -0,0 +1,24 @@
---
name: Feature request
about: Suggest an idea for this project.
title: ''
labels: ''
assignees: ''
---
<!--
Please do *NOT* ask usage questions in Github issues.
If your issue is not a feature request or bug report use:
https://groups.google.com/forum/#!forum/prometheus-users. If
you are unsure whether you hit a bug, search and ask in the
mailing list first.
You can find more information at: https://prometheus.io/community/
-->
## Proposal
**Use case. Why is this important?**
*“Nice to have” is not a good use case. :)*

56
.github/stale.yml vendored Normal file
View file

@ -0,0 +1,56 @@
# Configuration for probot-stale - https://github.com/probot/stale
# Number of days of inactivity before an Issue or Pull Request becomes stale
daysUntilStale: 60
# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
daysUntilClose: false
# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
onlyLabels: []
# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
exemptLabels:
- keepalive
# Set to true to ignore issues in a project (defaults to false)
exemptProjects: false
# Set to true to ignore issues in a milestone (defaults to false)
exemptMilestones: false
# Set to true to ignore issues with an assignee (defaults to false)
exemptAssignees: false
# Label to use when marking as stale
staleLabel: stale
# Comment to post when marking as stale. Set to `false` to disable
markComment: false
# Comment to post when removing the stale label.
# unmarkComment: >
# Your comment here.
# Comment to post when closing a stale Issue or Pull Request.
# closeComment: >
# Your comment here.
# Limit the number of actions per hour, from 1-30. Default is 30
limitPerRun: 30
# Limit to only `issues` or `pulls`
only: pulls
# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
# pulls:
# daysUntilStale: 30
# markComment: >
# This pull request has been automatically marked as stale because it has not had
# recent activity. It will be closed if no further activity occurs. Thank you
# for your contributions.
# issues:
# exemptLabels:
# - confirmed

57
.github/workflows/funcbench.yml vendored Normal file
View file

@ -0,0 +1,57 @@
on:
repository_dispatch:
types: [funcbench_start]
name: Funcbench Workflow
jobs:
run_funcbench:
name: Running funcbench
if: github.event.action == 'funcbench_start'
runs-on: ubuntu-latest
env:
AUTH_FILE: ${{ secrets.TEST_INFRA_GKE_AUTH }}
BRANCH: ${{ github.event.client_payload.BRANCH }}
BENCH_FUNC_REGEX: ${{ github.event.client_payload.BENCH_FUNC_REGEX }}
PACKAGE_PATH: ${{ github.event.client_payload.PACKAGE_PATH }}
GITHUB_TOKEN: ${{ secrets.PROMBOT_TOKEN }}
GITHUB_ORG: prometheus
GITHUB_REPO: prometheus
GITHUB_STATUS_TARGET_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
LAST_COMMIT_SHA: ${{ github.event.client_payload.LAST_COMMIT_SHA }}
PROJECT_ID: macro-mile-203600
PR_NUMBER: ${{ github.event.client_payload.PR_NUMBER }}
ZONE: europe-west3-a
steps:
- name: Update status to pending
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"pending","context":"funcbench-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Prepare nodepool
uses: docker://prominfra/funcbench:master
with:
entrypoint: 'docker_entrypoint'
args: make deploy
- name: Delete all resources
if: always()
uses: docker://prominfra/funcbench:master
with:
entrypoint: 'docker_entrypoint'
args: make clean
- name: Update status to failure
if: failure()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"failure","context":"funcbench-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Update status to success
if: success()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"success","context":"funcbench-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"

125
.github/workflows/prombench.yml vendored Normal file
View file

@ -0,0 +1,125 @@
on:
repository_dispatch:
types: [prombench_start,prombench_restart,prombench_stop]
name: Prombench Workflow
env:
AUTH_FILE: ${{ secrets.TEST_INFRA_GKE_AUTH }}
CLUSTER_NAME: test-infra
DOMAIN_NAME: prombench.prometheus.io
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_ORG: prometheus
GITHUB_REPO: prometheus
GITHUB_STATUS_TARGET_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
LAST_COMMIT_SHA: ${{ github.event.client_payload.LAST_COMMIT_SHA }}
PROJECT_ID: macro-mile-203600
PR_NUMBER: ${{ github.event.client_payload.PR_NUMBER }}
RELEASE: ${{ github.event.client_payload.RELEASE }}
ZONE: europe-west3-a
jobs:
benchmark_start:
name: Benchmark Start
if: github.event.action == 'prombench_start'
runs-on: ubuntu-latest
steps:
- name: Update status to pending
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"pending", "context": "prombench-status-update-start", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Run make deploy to start test
id: make_deploy
uses: docker://prominfra/prombench:master
with:
args: >-
until make all_nodepools_deleted; do echo "waiting for nodepools to be deleted"; sleep 10; done;
make deploy;
- name: Update status to failure
if: failure()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"failure", "context": "prombench-status-update-start", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Update status to success
if: success()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"success", "context": "prombench-status-update-start", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
benchmark_cancel:
name: Benchmark Cancel
if: github.event.action == 'prombench_stop'
runs-on: ubuntu-latest
steps:
- name: Update status to pending
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"pending", "context": "prombench-status-update-cancel", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Run make clean to stop test
id: make_clean
uses: docker://prominfra/prombench:master
with:
args: >-
until make all_nodepools_running; do echo "waiting for nodepools to be created"; sleep 10; done;
make clean;
- name: Update status to failure
if: failure()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"failure", "context": "prombench-status-update-cancel", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Update status to success
if: success()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"success", "context": "prombench-status-update-cancel", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
benchmark_restart:
name: Benchmark Restart
if: github.event.action == 'prombench_restart'
runs-on: ubuntu-latest
steps:
- name: Update status to pending
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"pending", "context": "prombench-status-update-restart", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Run make clean then make deploy to restart test
id: make_restart
uses: docker://prominfra/prombench:master
with:
args: >-
until make all_nodepools_running; do echo "waiting for nodepools to be created"; sleep 10; done;
make clean;
until make all_nodepools_deleted; do echo "waiting for nodepools to be deleted"; sleep 10; done;
make deploy;
- name: Update status to failure
if: failure()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"failure", "context": "prombench-status-update-restart", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Update status to success
if: success()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"success", "context": "prombench-status-update-restart", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"

5
.gitignore vendored
View file

@ -10,9 +10,14 @@ benchmark.txt
/data
/cmd/prometheus/data
/cmd/prometheus/debug
/benchout
!/.travis.yml
!/.promu.yml
!/.golangci.yml
/documentation/examples/remote_storage/remote_storage_adapter/remote_storage_adapter
/documentation/examples/remote_storage/example_write_adapter/example_writer_adapter
npm_licenses.tar.bz2
/web/ui/static/react
/web/ui/assets_vfsdata.go

18
.gitpod.yml Normal file
View file

@ -0,0 +1,18 @@
tasks:
- init:
make build
command: |
gp sync-done build
./prometheus --config.file=documentation/examples/prometheus.yml
- command: |
cd web/ui/react-app
gp sync-await build
unset BROWSER
export DANGEROUSLY_DISABLE_HOST_CHECK=true
yarn start
openMode: split-right
ports:
- port: 3000
onOpen: open-preview
- port: 9090
onOpen: ignore

4
.golangci.yml
View file

@ -2,6 +2,10 @@ run:
modules-download-mode: vendor
deadline: 5m
linters:
enable:
- golint
issues:
exclude-rules:
- path: _test.go

8
.promu.yml
View file

@ -1,7 +1,7 @@
go:
# Whenever the Go version is updated here, .travis.yml and
# Whenever the Go version is updated here,
# .circle/config.yml should also be updated.
version: 1.12
version: 1.15
repository:
path: github.com/prometheus/prometheus
build:
@ -10,7 +10,7 @@ build:
path: ./cmd/prometheus
- name: promtool
path: ./cmd/promtool
flags: -mod=vendor -a -tags netgo
flags: -mod=vendor -a -tags netgo,builtinassets
ldflags: |
-X github.com/prometheus/common/version.Version={{.Version}}
-X github.com/prometheus/common/version.Revision={{.Revision}}
@ -24,12 +24,12 @@ tarball:
- documentation/examples/prometheus.yml
- LICENSE
- NOTICE
- npm_licenses.tar.bz2
crossbuild:
platforms:
- linux/amd64
- linux/386
- darwin/amd64
- darwin/386
- windows/amd64
- windows/386
- freebsd/amd64

19
.travis.yml
View file

@ -1,19 +0,0 @@
language: go
# Whenever the Go version is updated here, .circleci/config.yml and .promu.yml
# should also be updated.
go:
- 1.12.x
go_import_path: github.com/prometheus/prometheus
# This ensures that the local cache is filled before running the CI.
# travis_retry retries the command 3 times if it fails as we've experienced
# random issues on Travis.
before_install:
- travis_retry make deps
- if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then choco install make; fi
script:
- make check_license style unused test lint check_assets
- git diff --exit-code

236
CHANGELOG.md
View file

@ -1,3 +1,227 @@
## 2.20.1 / 2020-08-05
* [BUGFIX] SD: Reduce the Consul watch timeout to 2m and adjust the request timeout accordingly. #7724
## 2.20.0 / 2020-07-22
This release changes WAL compression from opt-in to default. WAL compression will prevent a downgrade to v2.10 or earlier without deleting the WAL. Disable WAL compression explicitly by setting the command line flag `--no-storage.tsdb.wal-compression` if you require downgrading to v2.10 or earlier.
* [CHANGE] promtool: Changed rule numbering from 0-based to 1-based when reporting rule errors. #7495
* [CHANGE] Remote read: Added `prometheus_remote_storage_read_queries_total` counter and `prometheus_remote_storage_read_request_duration_seconds` histogram, removed `prometheus_remote_storage_remote_read_queries_total` counter.
* [CHANGE] Remote write: Added buckets for longer durations to `prometheus_remote_storage_sent_batch_duration_seconds` histogram.
* [CHANGE] TSDB: WAL compression is enabled by default. #7410
* [FEATURE] PromQL: Added `group()` aggregator. #7480
* [FEATURE] SD: Added Docker Swarm SD. #7420
* [FEATURE] SD: Added DigitalOcean SD. #7407
* [FEATURE] SD: Added Openstack config option to query alternative endpoints. #7494
* [ENHANCEMENT] Configuration: Exit early on invalid config file and signal it with exit code 2. #7399
* [ENHANCEMENT] PromQL: `without` is now a valid metric identifier. #7533
* [ENHANCEMENT] PromQL: Optimized regex label matching for literals within the pattern or as prefix/suffix. #7453 #7503
* [ENHANCEMENT] promtool: Added time range parameters for labels API in promtool. #7463
* [ENHANCEMENT] Remote write: Include samples waiting in channel in pending samples metric. Log number of dropped samples on hard shutdown. #7335
* [ENHANCEMENT] Scrape: Ingest synthetic scrape report metrics atomically with the corresponding scraped metrics. #7562
* [ENHANCEMENT] SD: Reduce timeouts for Openstack SD. #7507
* [ENHANCEMENT] SD: Use 10m timeout for Consul watches. #7423
* [ENHANCEMENT] SD: Added AMI meta label for EC2 SD. #7386
* [ENHANCEMENT] TSDB: Increment WAL corruption metric also on WAL corruption during checkpointing. #7491
* [ENHANCEMENT] TSDB: Improved query performance for high-cardinality labels. #7448
* [ENHANCEMENT] UI: Display dates as well as timestamps in status page. #7544
* [ENHANCEMENT] UI: Improved scrolling when following hash-fragment links. #7456
* [ENHANCEMENT] UI: React UI renders numbers in alerts in a more human-readable way. #7426
* [BUGFIX] API: Fixed error status code in the query API. #7435
* [BUGFIX] PromQL: Fixed `avg` and `avg_over_time` for NaN, Inf, and float64 overflows. #7346
* [BUGFIX] PromQL: Fixed off-by-one error in `histogram_quantile`. #7393
* [BUGFIX] promtool: Support extended durations in rules unit tests. #6297
* [BUGFIX] Scrape: Fix undercounting for `scrape_samples_post_metric_relabeling` in case of errors. #7342
* [BUGFIX] TSDB: Don't panic on WAL corruptions. #7550
* [BUGFIX] TSDB: Avoid leaving behind empty files in `chunks_head`, causing startup failures. #7573
* [BUGFIX] TSDB: Fixed race between compact (gc, populate) and head append causing unknown symbol error. #7560
* [BUGFIX] TSDB: Fixed unknown symbol error during head compaction. #7526
* [BUGFIX] TSDB: Fixed panic during TSDB metric registration. #7501
* [BUGFIX] TSDB: Fixed `--limit` command line flag in `tsdb` tool. #7430
## 2.19.2 / 2020-06-26
* [BUGFIX] Remote Write: Fix panic when reloading config with modified queue parameters. #7452
## 2.19.1 / 2020-06-18
* [BUGFIX] TSDB: Fix m-map file truncation leading to unsequential files. #7414
## 2.19.0 / 2020-06-09
* [FEATURE] TSDB: Memory-map full chunks of Head (in-memory) block from disk. This reduces memory footprint and makes restarts faster. #6679
* [ENHANCEMENT] Discovery: Added discovery support for Triton global zones. #7250
* [ENHANCEMENT] Increased alert resend delay to be more tolerant towards failures. #7228
* [ENHANCEMENT] Remote Read: Added `prometheus_remote_storage_remote_read_queries_total` counter to count total number of remote read queries. #7328
* [ENHANCEMEMT] Added time range parameters for label names and label values API. #7288
* [ENHANCEMENT] TSDB: Reduced contention in isolation for high load. #7332
* [BUGFIX] PromQL: Eliminated collision while checking for duplicate labels. #7058
* [BUGFIX] React UI: Don't null out data when clicking on the current tab. #7243
* [BUGFIX] PromQL: Correctly track number of samples for a query. #7307
* [BUGFIX] PromQL: Return NaN when histogram buckets have 0 observations. #7318
## 2.18.2 / 2020-06-09
* [BUGFIX] TSDB: Fix incorrect query results when using Prometheus with remote reads configured #7361
## 2.18.1 / 2020-05-07
* [BUGFIX] TSDB: Fixed snapshot API. #7217
## 2.18.0 / 2020-05-05
* [CHANGE] Federation: Only use local TSDB for federation (ignore remote read). #7096
* [CHANGE] Rules: `rule_evaluations_total` and `rule_evaluation_failures_total` have a `rule_group` label now. #7094
* [FEATURE] Tracing: Added experimental Jaeger support #7148
* [ENHANCEMENT] TSDB: Significantly reduce WAL size kept around after a block cut. #7098
* [ENHANCEMENT] Discovery: Add `architecture` meta label for EC2. #7000
* [BUGFIX] UI: Fixed wrong MinTime reported by /status. #7182
* [BUGFIX] React UI: Fixed multiselect legend on OSX. #6880
* [BUGFIX] Remote Write: Fixed blocked resharding edge case. #7122
* [BUGFIX] Remote Write: Fixed remote write not updating on relabel configs change. #7073
## 2.17.2 / 2020-04-20
* [BUGFIX] Federation: Register federation metrics #7081
* [BUGFIX] PromQL: Fix panic in parser error handling #7132
* [BUGFIX] Rules: Fix reloads hanging when deleting a rule group that is being evaluated #7138
* [BUGFIX] TSDB: Fix a memory leak when prometheus starts with an empty TSDB WAL #7135
* [BUGFIX] TSDB: Make isolation more robust to panics in web handlers #7129 #7136
## 2.17.1 / 2020-03-26
* [BUGFIX] TSDB: Fix query performance regression that increased memory and CPU usage #7051
## 2.17.0 / 2020-03-24
This release implements isolation in TSDB. API queries and recording rules are
guaranteed to only see full scrapes and full recording rules. This comes with a
certain overhead in resource usage. Depending on the situation, there might be
some increase in memory usage, CPU usage, or query latency.
* [FEATURE] TSDB: Support isolation #6841
* [ENHANCEMENT] PromQL: Allow more keywords as metric names #6933
* [ENHANCEMENT] React UI: Add normalization of localhost URLs in targets page #6794
* [ENHANCEMENT] Remote read: Read from remote storage concurrently #6770
* [ENHANCEMENT] Rules: Mark deleted rule series as stale after a reload #6745
* [ENHANCEMENT] Scrape: Log scrape append failures as debug rather than warn #6852
* [ENHANCEMENT] TSDB: Improve query performance for queries that partially hit the head #6676
* [ENHANCEMENT] Consul SD: Expose service health as meta label #5313
* [ENHANCEMENT] EC2 SD: Expose EC2 instance lifecycle as meta label #6914
* [ENHANCEMENT] Kubernetes SD: Expose service type as meta label for K8s service role #6684
* [ENHANCEMENT] Kubernetes SD: Expose label_selector and field_selector #6807
* [ENHANCEMENT] Openstack SD: Expose hypervisor id as meta label #6962
* [BUGFIX] PromQL: Do not escape HTML-like chars in query log #6834 #6795
* [BUGFIX] React UI: Fix data table matrix values #6896
* [BUGFIX] React UI: Fix new targets page not loading when using non-ASCII characters #6892
* [BUGFIX] Remote read: Fix duplication of metrics read from remote storage with external labels #6967 #7018
* [BUGFIX] Remote write: Register WAL watcher and live reader metrics for all remotes, not just the first one #6998
* [BUGFIX] Scrape: Prevent removal of metric names upon relabeling #6891
* [BUGFIX] Scrape: Fix 'superfluous response.WriteHeader call' errors when scrape fails under some circonstances #6986
* [BUGFIX] Scrape: Fix crash when reloads are separated by two scrape intervals #7011
## 2.16.0 / 2020-02-13
* [FEATURE] React UI: Support local timezone on /graph #6692
* [FEATURE] PromQL: add absent_over_time query function #6490
* [FEATURE] Adding optional logging of queries to their own file #6520
* [ENHANCEMENT] React UI: Add support for rules page and "Xs ago" duration displays #6503
* [ENHANCEMENT] React UI: alerts page, replace filtering togglers tabs with checkboxes #6543
* [ENHANCEMENT] TSDB: Export metric for WAL write errors #6647
* [ENHANCEMENT] TSDB: Improve query performance for queries that only touch the most recent 2h of data. #6651
* [ENHANCEMENT] PromQL: Refactoring in parser errors to improve error messages #6634
* [ENHANCEMENT] PromQL: Support trailing commas in grouping opts #6480
* [ENHANCEMENT] Scrape: Reduce memory usage on reloads by reusing scrape cache #6670
* [ENHANCEMENT] Scrape: Add metrics to track bytes and entries in the metadata cache #6675
* [ENHANCEMENT] promtool: Add support for line-column numbers for invalid rules output #6533
* [ENHANCEMENT] Avoid restarting rule groups when it is unnecessary #6450
* [BUGFIX] React UI: Send cookies on fetch() on older browsers #6553
* [BUGFIX] React UI: adopt grafana flot fix for stacked graphs #6603
* [BUFGIX] React UI: broken graph page browser history so that back button works as expected #6659
* [BUGFIX] TSDB: ensure compactionsSkipped metric is registered, and log proper error if one is returned from head.Init #6616
* [BUGFIX] TSDB: return an error on ingesting series with duplicate labels #6664
* [BUGFIX] PromQL: Fix unary operator precedence #6579
* [BUGFIX] PromQL: Respect query.timeout even when we reach query.max-concurrency #6712
* [BUGFIX] PromQL: Fix string and parentheses handling in engine, which affected React UI #6612
* [BUGFIX] PromQL: Remove output labels returned by absent() if they are produced by multiple identical label matchers #6493
* [BUGFIX] Scrape: Validate that OpenMetrics input ends with `# EOF` #6505
* [BUGFIX] Remote read: return the correct error if configs can't be marshal'd to JSON #6622
* [BUGFIX] Remote write: Make remote client `Store` use passed context, which can affect shutdown timing #6673
* [BUGFIX] Remote write: Improve sharding calculation in cases where we would always be consistently behind by tracking pendingSamples #6511
* [BUGFIX] Ensure prometheus_rule_group metrics are deleted when a rule group is removed #6693
## 2.15.2 / 2020-01-06
* [BUGFIX] TSDB: Fixed support for TSDB blocks built with Prometheus before 2.1.0. #6564
* [BUGFIX] TSDB: Fixed block compaction issues on Windows. #6547
## 2.15.1 / 2019-12-25
* [BUGFIX] TSDB: Fixed race on concurrent queries against same data. #6512
## 2.15.0 / 2019-12-23
* [CHANGE] Discovery: Removed `prometheus_sd_kubernetes_cache_*` metrics. Additionally `prometheus_sd_kubernetes_workqueue_latency_seconds` and `prometheus_sd_kubernetes_workqueue_work_duration_seconds` metrics now show correct values in seconds. #6393
* [CHANGE] Remote write: Changed `query` label on `prometheus_remote_storage_*` metrics to `remote_name` and `url`. #6043
* [FEATURE] API: Added new endpoint for exposing per metric metadata `/metadata`. #6420 #6442
* [ENHANCEMENT] TSDB: Significantly reduced memory footprint of loaded TSDB blocks. #6418 #6461
* [ENHANCEMENT] TSDB: Significantly optimized what we buffer during compaction which should result in lower memory footprint during compaction. #6422 #6452 #6468 #6475
* [ENHANCEMENT] TSDB: Improve replay latency. #6230
* [ENHANCEMENT] TSDB: WAL size is now used for size based retention calculation. #5886
* [ENHANCEMENT] Remote read: Added query grouping and range hints to the remote read request #6401
* [ENHANCEMENT] Remote write: Added `prometheus_remote_storage_sent_bytes_total` counter per queue. #6344
* [ENHANCEMENT] promql: Improved PromQL parser performance. #6356
* [ENHANCEMENT] React UI: Implemented missing pages like `/targets` #6276, TSDB status page #6281 #6267 and many other fixes and performance improvements.
* [ENHANCEMENT] promql: Prometheus now accepts spaces between time range and square bracket. e.g `[ 5m]` #6065
* [BUGFIX] Config: Fixed alertmanager configuration to not miss targets when configurations are similar. #6455
* [BUGFIX] Remote write: Value of `prometheus_remote_storage_shards_desired` gauge shows raw value of desired shards and it's updated correctly. #6378
* [BUGFIX] Rules: Prometheus now fails the evaluation of rules and alerts where metric results collide with labels specified in `labels` field. #6469
* [BUGFIX] API: Targets Metadata API `/targets/metadata` now accepts empty `match_targets` parameter as in the spec. #6303
## 2.14.0 / 2019-11-11
* [SECURITY/BUGFIX] UI: Ensure warnings from the API are escaped. #6279
* [FEATURE] API: `/api/v1/status/runtimeinfo` and `/api/v1/status/buildinfo` endpoints added for use by the React UI. #6243
* [FEATURE] React UI: implement the new experimental React based UI. #5694 and many more
* Can be found by under `/new`.
* Not all pages are implemented yet.
* [FEATURE] Status: Cardinality statistics added to the Runtime & Build Information page. #6125
* [ENHANCEMENT/BUGFIX] Remote write: fix delays in remote write after a compaction. #6021
* [ENHANCEMENT] UI: Alerts can be filtered by state. #5758
* [BUGFIX] API: lifecycle endpoints return 403 when not enabled. #6057
* [BUGFIX] Build: Fix Solaris build. #6149
* [BUGFIX] Promtool: Remove false duplicate rule warnings when checking rule files with alerts. #6270
* [BUGFIX] Remote write: restore use of deduplicating logger in remote write. #6113
* [BUGFIX] Remote write: do not reshard when unable to send samples. #6111
* [BUGFIX] Service discovery: errors are no longer logged on context cancellation. #6116, #6133
* [BUGFIX] UI: handle null response from API properly. #6071
## 2.13.1 / 2019-10-16
* [BUGFIX] Fix panic in ARM builds of Prometheus. #6110
* [BUGFIX] promql: fix potential panic in the query logger. #6094
* [BUGFIX] Multiple errors of http: superfluous response.WriteHeader call in the logs. #6145
## 2.13.0 / 2019-10-04
* [SECURITY/BUGFIX] UI: Fix a Stored DOM XSS vulnerability with query history [CVE-2019-10215](http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-10215). #6098
* [CHANGE] Metrics: renamed prometheus_sd_configs_failed_total to prometheus_sd_failed_configs and changed to Gauge #5254
* [ENHANCEMENT] Include the tsdb tool in builds. #6089
* [ENHANCEMENT] Service discovery: add new node address types for kubernetes. #5902
* [ENHANCEMENT] UI: show warnings if query have returned some warnings. #5964
* [ENHANCEMENT] Remote write: reduce memory usage of the series cache. #5849
* [ENHANCEMENT] Remote read: use remote read streaming to reduce memory usage. #5703
* [ENHANCEMENT] Metrics: added metrics for remote write max/min/desired shards to queue manager. #5787
* [ENHANCEMENT] Promtool: show the warnings during label query. #5924
* [ENHANCEMENT] Promtool: improve error messages when parsing bad rules. #5965
* [ENHANCEMENT] Promtool: more promlint rules. #5515
* [BUGFIX] Promtool: fix recording inconsistency due to duplicate labels. #6026
* [BUGFIX] UI: fixes service-discovery view when accessed from unhealthy targets. #5915
* [BUGFIX] Metrics format: OpenMetrics parser crashes on short input. #5939
* [BUGFIX] UI: avoid truncated Y-axis values. #6014
## 2.12.0 / 2019-08-17
* [FEATURE] Track currently active PromQL queries in a log file. #5794
@ -16,6 +240,10 @@
* [BUGFIX] Fix error propagation in WAL watcher helper functions. #5741
* [BUGFIX] Correctly handle empty labels from alert templates. #5845
## 2.11.2 / 2019-08-14
* [BUGFIX/SECURITY] Fix a Stored DOM XSS vulnerability with query history. #5888
## 2.11.1 / 2019-07-10
* [BUGFIX] Fix potential panic when prometheus is watching multiple zookeeper paths. #5749
@ -110,7 +338,7 @@ and the memory is available to the kernel when it needs it.
* [BUGFIX] Check if label value is valid when unmarshaling external labels from YAML. #5316
* [BUGFIX] Promparse: sort all labels when parsing. #5372
* [BUGFIX] Reload rules: copy state on both name and labels. #5368
* [BUGFIX] Exponentation operator to drop metric name in result of operation. #5329
* [BUGFIX] Exponentiation operator to drop metric name in result of operation. #5329
* [BUGFIX] Config: resolve more file paths. #5284
* [BUGFIX] Promtool: resolve relative paths in alert test files. #5336
* [BUGFIX] Set TLSHandshakeTimeout in HTTP transport. common#179
@ -403,7 +631,7 @@ This release includes multiple bugfixes and features. Further, the WAL implement
* [FEATURE] New Service Discovery UI showing labels before and after relabelling.
* [FEATURE] New Admin APIs added to v1 to delete, snapshot and remove tombstones.
* [ENHANCEMENT] The graph UI autcomplete now includes your previous queries.
* [ENHANCEMENT] The graph UI autocomplete now includes your previous queries.
* [ENHANCEMENT] Federation is now much faster for large numbers of series.
* [ENHANCEMENT] Added new metrics to measure rule timings.
* [ENHANCEMENT] Rule evaluation times added to the rules UI.
@ -522,7 +750,7 @@ https://prometheus.io/docs/prometheus/2.0/migration/
## 1.6.3 / 2017-05-18
* [BUGFIX] Fix disappearing Alertmanger targets in Alertmanager discovery.
* [BUGFIX] Fix disappearing Alertmanager targets in Alertmanager discovery.
* [BUGFIX] Fix panic with remote_write on ARMv7.
* [BUGFIX] Fix stacked graphs to adapt min/max values.
@ -698,7 +926,7 @@ This is a breaking change to the Kubernetes service discovery.
## 1.2.1 / 2016-10-10
* [BUGFIX] Count chunk evictions properly so that the server doesn't
assume it runs out of memory and subsequencly throttles ingestion.
assume it runs out of memory and subsequently throttles ingestion.
* [BUGFIX] Use Go1.7.1 for prebuilt binaries to fix issues on MacOS Sierra.
## 1.2.0 / 2016-10-07

0
code-of-conduct.md → CODE_OF_CONDUCT.md
View file

4
CONTRIBUTING.md
View file

@ -28,6 +28,8 @@ Should you wish to work on an issue, please claim it first by commenting on the
Please check the [`low-hanging-fruit`](https://github.com/prometheus/prometheus/issues?q=is%3Aissue+is%3Aopen+label%3A%22low+hanging+fruit%22) label to find issues that are good for getting started. If you have questions about one of the issues, with or without the tag, please comment on them and one of the maintainers will clarify it. For a quicker response, contact us over [IRC](https://prometheus.io/community).
You can [spin up a prebuilt dev environment](https://gitpod.io/#https://github.com/prometheus/prometheus) using Gitpod.io.
For complete instructions on how to compile see: [Building From Source](https://github.com/prometheus/prometheus#building-from-source)
For quickly compiling and testing your changes do:
@ -56,7 +58,7 @@ All our issues are regularly tagged so that you can also filter down the issues
## Dependency management
The Prometheus project uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies on external packages. This requires a working Go environment with version 1.12 or greater installed.
The Prometheus project uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies on external packages.
All dependencies are vendored in the `vendor/` directory.

3
Dockerfile
View file

@ -10,6 +10,9 @@ COPY .build/${OS}-${ARCH}/promtool /bin/promtool
COPY documentation/examples/prometheus.yml /etc/prometheus/prometheus.yml
COPY console_libraries/ /usr/share/prometheus/console_libraries/
COPY consoles/ /usr/share/prometheus/consoles/
COPY LICENSE /LICENSE
COPY NOTICE /NOTICE
COPY npm_licenses.tar.bz2 /npm_licenses.tar.bz2
RUN ln -s /usr/share/prometheus/console_libraries /usr/share/prometheus/consoles/ /etc/prometheus/
RUN mkdir -p /prometheus && \

23
MAINTAINERS.md
View file

@ -1,6 +1,21 @@
Maintainers of this repository with their focus areas:
@brian-brazil is the main/default maintainer, some parts of the codebase have other maintainers:
* Brian Brazil <brian.brazil@robustperception.io> @brian-brazil: Console templates; semantics of PromQL, service discovery, and relabeling.
* Fabian Reinartz <freinartz@google.com> @fabxc: PromQL parsing and evaluation; implementation of retrieval, alert notification, and service discovery.
* Julius Volz <julius.volz@gmail.com> @juliusv: Remote storage integrations; web UI.
* `cmd`
* `promtool`: @simonpasquier
* `discovery`
* `k8s`: @brancz
* `documentation`
* `prometheus-mixin`: @beorn7
* `storage`
* `remote`: @csmarchbanks, @cstyan, @bwplotka
* `tsdb`: @codesome, @krasi-georgiev, @bwplotka
* `web`
* `ui`: @juliusv
* `Makefile` and related build configuration: @simonpasquier, @SuperQ
For the sake of brevity all subtrees are not explicitly listed. Due to the size
of this repository, the natural changes in focus of maintainers over time, and
nuances of where particular features live, this list will always be incomplete
and out of date. However the listed maintainer(s) should be able to direct a
PR/question to the right person.

87
Makefile
View file

@ -12,41 +12,80 @@
# limitations under the License.
# Needs to be defined before including Makefile.common to auto-generate targets
DOCKER_ARCHS ?= amd64 armv7 arm64
DOCKER_ARCHS ?= amd64 armv7 arm64 ppc64le s390x
TSDB_PROJECT_DIR = "./tsdb"
TSDB_CLI_DIR="$(TSDB_PROJECT_DIR)/cmd/tsdb"
TSDB_BIN = "$(TSDB_CLI_DIR)/tsdb"
REACT_APP_PATH = web/ui/react-app
REACT_APP_SOURCE_FILES = $(wildcard $(REACT_APP_PATH)/public/* $(REACT_APP_PATH)/src/* $(REACT_APP_PATH)/tsconfig.json)
REACT_APP_OUTPUT_DIR = web/ui/static/react
REACT_APP_NODE_MODULES_PATH = $(REACT_APP_PATH)/node_modules
REACT_APP_NPM_LICENSES_TARBALL = "npm_licenses.tar.bz2"
PROMTOOL = ./promtool
TSDB_BENCHMARK_NUM_METRICS ?= 1000
TSDB_BENCHMARK_DATASET ?= "$(TSDB_PROJECT_DIR)/testdata/20kseries.json"
TSDB_BENCHMARK_OUTPUT_DIR ?= "$(TSDB_CLI_DIR)/benchout"
TSDB_BENCHMARK_DATASET ?= ./tsdb/testdata/20kseries.json
TSDB_BENCHMARK_OUTPUT_DIR ?= ./benchout
include Makefile.common
DOCKER_IMAGE_NAME ?= prometheus
$(REACT_APP_NODE_MODULES_PATH): $(REACT_APP_PATH)/package.json $(REACT_APP_PATH)/yarn.lock
cd $(REACT_APP_PATH) && yarn --frozen-lockfile
$(REACT_APP_OUTPUT_DIR): $(REACT_APP_NODE_MODULES_PATH) $(REACT_APP_SOURCE_FILES)
@echo ">> building React app"
@./scripts/build_react_app.sh
.PHONY: assets
assets:
assets: $(REACT_APP_OUTPUT_DIR)
@echo ">> writing assets"
cd $(PREFIX)/web/ui && GO111MODULE=$(GO111MODULE) $(GO) generate -x -v $(GOOPTS)
# Un-setting GOOS and GOARCH here because the generated Go code is always the same,
# but the cached object code is incompatible between architectures and OSes (which
# breaks cross-building for different combinations on CI in the same container).
cd web/ui && GO111MODULE=$(GO111MODULE) GOOS= GOARCH= $(GO) generate -x -v $(GOOPTS)
@$(GOFMT) -w ./web/ui
.PHONY: check_assets
check_assets: assets
@echo ">> checking that assets are up-to-date"
@if ! (cd $(PREFIX)/web/ui && git diff --exit-code); then \
echo "Run 'make assets' and commit the changes to fix the error."; \
exit 1; \
fi
.PHONY: react-app-lint
react-app-lint:
@echo ">> running React app linting"
cd $(REACT_APP_PATH) && yarn lint:ci
build_tsdb:
GO111MODULE=$(GO111MODULE) $(GO) build -o $(TSDB_BIN) $(TSDB_CLI_DIR)
.PHONY: react-app-lint-fix
react-app-lint-fix:
@echo ">> running React app linting and fixing errors where possible"
cd $(REACT_APP_PATH) && yarn lint
bench_tsdb: build_tsdb
.PHONY: react-app-test
react-app-test: | $(REACT_APP_NODE_MODULES_PATH) react-app-lint
@echo ">> running React app tests"
cd $(REACT_APP_PATH) && yarn test --no-watch --coverage
.PHONY: test
test: common-test react-app-test
.PHONY: npm_licenses
npm_licenses: $(REACT_APP_NODE_MODULES_PATH)
@echo ">> bundling npm licenses"
rm -f $(REACT_APP_NPM_LICENSES_TARBALL)
find $(REACT_APP_NODE_MODULES_PATH) -iname "license*" | tar cfj $(REACT_APP_NPM_LICENSES_TARBALL) --transform 's/^/npm_licenses\//' --files-from=-
.PHONY: tarball
tarball: npm_licenses common-tarball
.PHONY: docker
docker: npm_licenses common-docker
.PHONY: build
build: assets common-build
.PHONY: bench_tsdb
bench_tsdb: $(PROMU)
@echo ">> building promtool"
@GO111MODULE=$(GO111MODULE) $(PROMU) build --prefix $(PREFIX) promtool
@echo ">> running benchmark, writing result to $(TSDB_BENCHMARK_OUTPUT_DIR)"
@$(TSDB_BIN) bench write --metrics=$(TSDB_BENCHMARK_NUM_METRICS) --out=$(TSDB_BENCHMARK_OUTPUT_DIR) $(TSDB_BENCHMARK_DATASET)
@$(GO) tool pprof -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/cpu.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/cpuprof.svg
@$(GO) tool pprof --inuse_space -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/mem.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/memprof.inuse.svg
@$(GO) tool pprof --alloc_space -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/mem.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/memprof.alloc.svg
@$(GO) tool pprof -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/block.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/blockprof.svg
@$(GO) tool pprof -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/mutex.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/mutexprof.svg
@$(PROMTOOL) tsdb bench write --metrics=$(TSDB_BENCHMARK_NUM_METRICS) --out=$(TSDB_BENCHMARK_OUTPUT_DIR) $(TSDB_BENCHMARK_DATASET)
@$(GO) tool pprof -svg $(PROMTOOL) $(TSDB_BENCHMARK_OUTPUT_DIR)/cpu.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/cpuprof.svg
@$(GO) tool pprof --inuse_space -svg $(PROMTOOL) $(TSDB_BENCHMARK_OUTPUT_DIR)/mem.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/memprof.inuse.svg
@$(GO) tool pprof --alloc_space -svg $(PROMTOOL) $(TSDB_BENCHMARK_OUTPUT_DIR)/mem.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/memprof.alloc.svg
@$(GO) tool pprof -svg $(PROMTOOL) $(TSDB_BENCHMARK_OUTPUT_DIR)/block.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/blockprof.svg
@$(GO) tool pprof -svg $(PROMTOOL) $(TSDB_BENCHMARK_OUTPUT_DIR)/mutex.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/mutexprof.svg

33
Makefile.common
View file

@ -69,12 +69,21 @@ else
GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)
endif
GOTEST := $(GO) test
GOTEST_DIR :=
ifneq ($(CIRCLE_JOB),)
ifneq ($(shell which gotestsum),)
GOTEST_DIR := test-results
GOTEST := gotestsum --junitfile $(GOTEST_DIR)/unit-tests.xml --
endif
endif
PROMU_VERSION ?= 0.5.0
PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
GOLANGCI_LINT :=
GOLANGCI_LINT_OPTS ?=
GOLANGCI_LINT_VERSION ?= v1.17.1
GOLANGCI_LINT_VERSION ?= v1.18.0
# golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
# windows isn't included here because of the path separator being different.
ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
@ -141,15 +150,29 @@ else
$(GO) get $(GOOPTS) -t ./...
endif
.PHONY: update-go-deps
update-go-deps:
@echo ">> updating Go dependencies"
@for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \
$(GO) get $$m; \
done
GO111MODULE=$(GO111MODULE) $(GO) mod tidy
ifneq (,$(wildcard vendor))
GO111MODULE=$(GO111MODULE) $(GO) mod vendor
endif
.PHONY: common-test-short
common-test-short:
common-test-short: $(GOTEST_DIR)
@echo ">> running short tests"
GO111MODULE=$(GO111MODULE) $(GO) test -short $(GOOPTS) $(pkgs)
GO111MODULE=$(GO111MODULE) $(GOTEST) -short $(GOOPTS) $(pkgs)
.PHONY: common-test
common-test:
common-test: $(GOTEST_DIR)
@echo ">> running all tests"
GO111MODULE=$(GO111MODULE) $(GO) test $(test-flags) $(GOOPTS) $(pkgs)
GO111MODULE=$(GO111MODULE) $(GOTEST) $(test-flags) $(GOOPTS) $(pkgs)
$(GOTEST_DIR):
@mkdir -p $@
.PHONY: common-format
common-format:

11
NOTICE
View file

@ -85,3 +85,14 @@ go-zookeeper - Native ZooKeeper client for Go
https://github.com/samuel/go-zookeeper
Copyright (c) 2013, Samuel Stauffer <samuel@descolada.com>
See https://github.com/samuel/go-zookeeper/blob/master/LICENSE for license details.
Time series compression algorithm from Facebook's Gorilla paper
https://github.com/dgryski/go-tsz
Copyright (c) 2015,2016 Damian Gryski <damian@gryski.com>
See https://github.com/dgryski/go-tsz/blob/master/LICENSE for license details.
We also use code from a large number of npm packages. For details, see:
- https://github.com/prometheus/prometheus/blob/master/web/ui/react-app/package.json
- https://github.com/prometheus/prometheus/blob/master/web/ui/react-app/package-lock.json
- The individual package licenses as copied from the node_modules directory can be found in
the npm_licenses.tar.bz2 archive in release tarballs and Docker images.

35
README.md
View file

@ -1,12 +1,12 @@
# Prometheus
# Prometheus
[![Build Status](https://travis-ci.org/prometheus/prometheus.svg)][travis]
[![CircleCI](https://circleci.com/gh/prometheus/prometheus/tree/master.svg?style=shield)][circleci]
[![Docker Repository on Quay](https://quay.io/repository/prometheus/prometheus/status)][quay]
[![Docker Pulls](https://img.shields.io/docker/pulls/prom/prometheus.svg?maxAge=604800)][hub]
[![Go Report Card](https://goreportcard.com/badge/github.com/prometheus/prometheus)](https://goreportcard.com/report/github.com/prometheus/prometheus)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/486/badge)](https://bestpractices.coreinfrastructure.org/projects/486)
[![fuzzit](https://app.fuzzit.dev/badge?org_id=prometheus&branch=master)](https://fuzzit.dev)
[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/prometheus/prometheus)
Visit [prometheus.io](https://prometheus.io) for the full documentation,
examples and guides.
@ -16,10 +16,10 @@ from configured targets at given intervals, evaluates rule expressions,
displays the results, and can trigger alerts if some condition is observed
to be true.
Prometheus' main distinguishing features as compared to other monitoring systems are:
Prometheus's main distinguishing features as compared to other monitoring systems are:
- a **multi-dimensional** data model (timeseries defined by metric name and set of key/value dimensions)
- a **flexible query language** to leverage this dimensionality
- PromQL, a **powerful and flexible query language** to leverage this dimensionality
- no dependency on distributed storage; **single server nodes are autonomous**
- timeseries collection happens via a **pull model** over HTTP
- **pushing timeseries** is supported via an intermediary gateway
@ -44,8 +44,6 @@ is the recommended way of installing Prometheus.
See the [Installing](https://prometheus.io/docs/introduction/install/)
chapter in the documentation for all the details.
Debian packages [are available](https://packages.debian.org/sid/net/prometheus).
### Docker images
Docker images are available on [Quay.io](https://quay.io/repository/prometheus/prometheus) or [Docker Hub](https://hub.docker.com/r/prom/prometheus/).
@ -59,7 +57,9 @@ Prometheus will now be reachable at http://localhost:9090/.
### Building from source
To build Prometheus from the source code yourself you need to have a working
Go environment with [version 1.12 or greater installed](https://golang.org/doc/install).
Go environment with [version 1.13 or greater installed](https://golang.org/doc/install).
You will also need to have [Node.js](https://nodejs.org/) and [Yarn](https://yarnpkg.com/)
installed in order to build the frontend assets.
You can directly use the `go` tool to download and install the `prometheus`
and `promtool` binaries into your `GOPATH`:
@ -67,7 +67,16 @@ and `promtool` binaries into your `GOPATH`:
$ go get github.com/prometheus/prometheus/cmd/...
$ prometheus --config.file=your_config.yml
You can also clone the repository yourself and build using `make`:
*However*, when using `go get` to build Prometheus, Prometheus will expect to be able to
read its web assets from local filesystem directories under `web/ui/static` and
`web/ui/templates`. In order for these assets to be found, you will have to run Prometheus
from the root of the cloned repository. Note also that these directories do not include the
new experimental React UI unless it has been built explicitly using `make assets` or `make build`.
An example of the above configuration file can be found [here.](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus.yml)
You can also clone the repository yourself and build using `make build`, which will compile in
the web assets so that Prometheus can be run from anywhere:
$ mkdir -p $GOPATH/src/github.com/prometheus
$ cd $GOPATH/src/github.com/prometheus
@ -78,18 +87,21 @@ You can also clone the repository yourself and build using `make`:
The Makefile provides several targets:
* *build*: build the `prometheus` and `promtool` binaries
* *build*: build the `prometheus` and `promtool` binaries (includes building and compiling in web assets)
* *test*: run the tests
* *test-short*: run the short tests
* *format*: format the source code
* *vet*: check the source code for common errors
* *assets*: rebuild the static assets
* *docker*: build a docker container for the current `HEAD`
## React UI Development
For more information on building, running, and developing on the new React-based UI, see the React app's [README.md](https://github.com/prometheus/prometheus/blob/master/web/ui/react-app/README.md).
## More information
* The source code is periodically indexed: [Prometheus Core](https://godoc.org/github.com/prometheus/prometheus).
* You will find a Travis CI configuration in `.travis.yml`.
* You will find a CircleCI configuration in `.circleci/config.yml`.
* See the [Community page](https://prometheus.io/community) for how to reach the Prometheus developers and users on various communication channels.
## Contributing
@ -101,7 +113,6 @@ Refer to [CONTRIBUTING.md](https://github.com/prometheus/prometheus/blob/master/
Apache License 2.0, see [LICENSE](https://github.com/prometheus/prometheus/blob/master/LICENSE).
[travis]: https://travis-ci.org/prometheus/prometheus
[hub]: https://hub.docker.com/r/prom/prometheus/
[circleci]: https://circleci.com/gh/prometheus/prometheus
[quay]: https://quay.io/repository/prometheus/prometheus

81
RELEASE.md
View file

@ -19,7 +19,15 @@ Release cadence of first pre-releases being cut is 6 weeks.
| v2.12 | 2019-08-14 | Julius Volz (GitHub: @juliusv) |
| v2.13 | 2019-09-25 | Krasi Georgiev (GitHub: @krasi-georgiev) |
| v2.14 | 2019-11-06 | Chris Marchbanks (GitHub: @csmarchbanks) |
| v2.15 | 2019-12-18 | **searching for volunteer** |
| v2.15 | 2019-12-18 | Bartek Plotka (GitHub: @bwplotka) |
| v2.16 | 2020-01-29 | Callum Styan (GitHub: @cstyan) |
| v2.17 | 2020-03-11 | Julien Pivotto (GitHub: @roidelapluie) |
| v2.18 | 2020-04-22 | Bartek Plotka (GitHub: @bwplotka) |
| v2.19 | 2020-06-03 | Ganesh Vernekar (GitHub: @codesome) |
| v2.20 | 2020-07-15 | Björn Rabenstein (GitHub: @beorn7) |
| v2.21 | 2020-08-26 | Julien Pivotto (GitHub: @roidelapluie) |
| v2.22 | 2020-10-07 | Frederic Branczyk (GitHub: @brancz) |
| v2.23 | 2020-11-18 | **searching for volunteer** |
If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice.
@ -44,22 +52,47 @@ We use [Semantic Versioning](https://semver.org/).
We maintain a separate branch for each minor release, named `release-<major>.<minor>`, e.g. `release-1.1`, `release-2.0`.
Note that branch protection kicks in automatically for any branches whose name starts with `release-`. Never use names starting with `release-` for branches that are not release branches.
The usual flow is to merge new features and changes into the master branch and to merge bug fixes into the latest release branch. Bug fixes are then merged into master from the latest release branch. The master branch should always contain all commits from the latest release branch. As long as master hasn't deviated from the release branch, new commits can also go to master, followed by merging master back into the release branch.
If a bug fix got accidentally merged into master after non-bug-fix changes in master, the bug-fix commits have to be cherry-picked into the release branch, which then have to be merged back into master. Try to avoid that situation.
Maintaining the release branches for older minor releases happens on a best effort basis.
### Prepare your release
### 0. Updating dependencies
For a patch release, work in the branch of the minor release you want to patch.
A few days before a major or minor release, consider updating the dependencies:
For a new major or minor release, create the corresponding release branch based on the master branch.
```
make update-go-deps
git add go.mod go.sum vendor
git commit -m "Update dependencies"
```
Bump the version in the `VERSION` file and update `CHANGELOG.md`. Do this in a proper PR as this gives others the opportunity to chime in on the release in general and on the addition to the changelog in particular.
Then create a pull request against the master branch.
Note that after a dependency update, you should look out for any weirdness that
might have happened. Such weirdnesses include but are not limited to: flaky
tests, differences in resource usage, panic.
In case of doubt or issues that can't be solved in a reasonable amount of time,
you can skip the dependency update or only update select dependencies. In such a
case, you have to create an issue or pull request in the GitHub project for
later follow-up.
### 1. Prepare your release
At the start of a new major or minor release cycle create the corresponding release branch based on the master branch. For example if we're releasing `2.17.0` and the previous stable release is `2.16.0` we need to create a `release-2.17` branch. Note that all releases are handled in protected release branches, see the above `Branch management and versioning` section. Release candidates and patch releases for any given major or minor release happen in the same `release-<major>.<minor>` branch. Do not create `release-<version>` for patch or release candidate releases.
Changes for a patch release or release candidate should be merged into the previously mentioned release branch via pull request.
Bump the version in the `VERSION` file and update `CHANGELOG.md`. Do this in a proper PR pointing to the release branch as this gives others the opportunity to chime in on the release in general and on the addition to the changelog in particular. For a release candidate, append something like `-rc.0` to the version (with the corresponding changes to the tag name, the release name etc.).
Note that `CHANGELOG.md` should only document changes relevant to users of Prometheus, including external API changes, performance improvements, and new features. Do not document changes of internal interfaces, code refactorings and clean-ups, changes to the build process, etc. People interested in these are asked to refer to the git history.
For release candidates still update `CHANGELOG.md`, but when you cut the final release later, merge all the changes from the pre-releases into the one final update.
Entries in the `CHANGELOG.md` are meant to be in this order:
* `[CHANGE]`
@ -67,38 +100,38 @@ Entries in the `CHANGELOG.md` are meant to be in this order:
* `[ENHANCEMENT]`
* `[BUGFIX]`
### Draft the new release
### 2. Draft the new release
Tag the new release with a tag named `v<major>.<minor>.<patch>`, e.g. `v2.1.3`. Note the `v` prefix.
You can do the tagging on the commandline:
Tag the new release via the following commands:
```bash
$ tag=$(< VERSION)
$ git tag -s "v${tag}" -m "v${tag}"
$ git push --tags
$ tag="v$(< VERSION)"
$ git tag -s "${tag}" -m "${tag}"
$ git push origin "${tag}"
```
Optionally, you can use this handy `.gitconfig` alias.
```ini
[alias]
tag-release = "!f() { tag=v${1:-$(cat VERSION)} ; git tag -s ${tag} -m ${tag} && git push origin ${tag}; }; f"
```
Then release with `git tag-release`.
Signing a tag with a GPG key is appreciated, but in case you can't add a GPG key to your Github account using the following [procedure](https://help.github.com/articles/generating-a-gpg-key/), you can replace the `-s` flag by `-a` flag of the `git tag` command to only annotate the tag without signing.
Once a tag is created, the release process through CircleCI will be triggered for this tag and Circle CI will draft the GitHub release using the `prombot` account.
Now all you can do is to wait for tarballs to be uploaded to the Github release and the container images to be pushed to the Docker Hub and Quay.io. Once that has happened, click _Publish release_, which will make the release publicly visible and create a GitHub notification.
Finally, wait for the build step for the tag to finish. The point here is to wait for tarballs to be uploaded to the Github release and the container images to be pushed to the Docker Hub and Quay.io. Once that has happened, click _Publish release_, which will make the release publicly visible and create a GitHub notification.
** Note: ** for a release candidate version ensure the _This is a pre-release_ box is checked when drafting the release in the Github UI. The CI job should take care of this but it's a good idea to double check before clicking _Publish release_.`
### Wrapping up
### 3. Wrapping up
For release candidate versions (`v2.16.0-rc.0`), run the benchmark for 3 days using the `/prombench vX.Y.Z` command, `vX.Y.Z` being the latest stable patch release's tag of the previous minor release series, such as `v2.15.2`.
If the release has happened in the latest release branch, merge the changes into master.
To update the docs, a PR needs to be created to `prometheus/docs`. See [this PR](https://github.com/prometheus/docs/pull/952/files) for inspiration (note: only actually merge this for final releases, not for pre-releases like a release candidate).
Once the binaries have been uploaded, announce the release on `prometheus-announce@googlegroups.com`. (Please do not use `prometheus-users@googlegroups.com` for announcements anymore.) Check out previous announcement mails for inspiration.
### Pre-releases
The following changes to the above procedures apply:
* In line with [Semantic Versioning](https://semver.org/), append something like `-rc.0` to the version (with the corresponding changes to the tag name, the release name etc.).
* Tick the _This is a pre-release_ box when drafting the release in the Github UI.
* Still update `CHANGELOG.md`, but when you cut the final release later, merge all the changes from the pre-releases into the one final update.
* Run the benchmark for 3 days using the `/benchmark x.y.z` command, `x.y.z` being the latest stable patch release of the previous minor release series.

2
VERSION
View file

@ -1 +1 @@
2.12.0
2.20.1

454
cmd/prometheus/main.go
View file

@ -16,9 +16,9 @@ package main
import (
"context"
"crypto/md5"
"encoding/json"
"fmt"
"io"
"math"
"net"
"net/http"
_ "net/http/pprof" // Comment this line to disable pprof endpoint.
@ -33,15 +33,20 @@ import (
"syscall"
"time"
"github.com/alecthomas/units"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
conntrack "github.com/mwitkow/go-conntrack"
"github.com/oklog/run"
"github.com/opentracing/opentracing-go"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
"github.com/prometheus/common/promlog"
"github.com/prometheus/common/version"
jcfg "github.com/uber/jaeger-client-go/config"
jprom "github.com/uber/jaeger-lib/metrics/prometheus"
"go.uber.org/atomic"
kingpin "gopkg.in/alecthomas/kingpin.v2"
"k8s.io/klog"
@ -50,6 +55,8 @@ import (
"github.com/prometheus/prometheus/discovery"
sd_config "github.com/prometheus/prometheus/discovery/config"
"github.com/prometheus/prometheus/notifier"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/logging"
"github.com/prometheus/prometheus/pkg/relabel"
prom_runtime "github.com/prometheus/prometheus/pkg/runtime"
"github.com/prometheus/prometheus/promql"
@ -57,7 +64,7 @@ import (
"github.com/prometheus/prometheus/scrape"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/storage/remote"
"github.com/prometheus/prometheus/storage/tsdb"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/util/strutil"
"github.com/prometheus/prometheus/web"
)
@ -107,7 +114,7 @@ func main() {
outageTolerance model.Duration
resendDelay model.Duration
web web.Options
tsdb tsdb.Options
tsdb tsdbOptions
lookbackDelta model.Duration
webTimeout model.Duration
queryTimeout model.Duration
@ -195,10 +202,10 @@ func main() {
a.Flag("storage.tsdb.retention", "[DEPRECATED] How long to retain samples in storage. This flag has been deprecated, use \"storage.tsdb.retention.time\" instead.").
SetValue(&oldFlagRetentionDuration)
a.Flag("storage.tsdb.retention.time", "How long to retain samples in storage. When this flag is set it overrides \"storage.tsdb.retention\". If neither this flag nor \"storage.tsdb.retention\" nor \"storage.tsdb.retention.size\" is set, the retention time defaults to "+defaultRetentionString+".").
a.Flag("storage.tsdb.retention.time", "How long to retain samples in storage. When this flag is set it overrides \"storage.tsdb.retention\". If neither this flag nor \"storage.tsdb.retention\" nor \"storage.tsdb.retention.size\" is set, the retention time defaults to "+defaultRetentionString+". Units Supported: y, w, d, h, m, s, ms.").
SetValue(&newFlagRetentionDuration)
a.Flag("storage.tsdb.retention.size", "[EXPERIMENTAL] Maximum number of bytes that can be stored for blocks. Units supported: KB, MB, GB, TB, PB. This flag is experimental and can be changed in future releases.").
a.Flag("storage.tsdb.retention.size", "[EXPERIMENTAL] Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\". This flag is experimental and can be changed in future releases.").
BytesVar(&cfg.tsdb.MaxBytes)
a.Flag("storage.tsdb.no-lockfile", "Do not create lockfile in data directory.").
@ -208,7 +215,7 @@ func main() {
Default("false").BoolVar(&cfg.tsdb.AllowOverlappingBlocks)
a.Flag("storage.tsdb.wal-compression", "Compress the tsdb WAL.").
Default("false").BoolVar(&cfg.tsdb.WALCompression)
Default("true").BoolVar(&cfg.tsdb.WALCompression)
a.Flag("storage.remote.flush-deadline", "How long to wait flushing sample on shutdown or config reload.").
Default("1m").PlaceHolder("<duration>").SetValue(&cfg.RemoteFlushDeadline)
@ -237,7 +244,7 @@ func main() {
a.Flag("alertmanager.timeout", "Timeout for sending alerts to Alertmanager.").
Default("10s").SetValue(&cfg.notifierTimeout)
a.Flag("query.lookback-delta", "The maximum lookback duration for retrieving metrics during expression evaluations.").
a.Flag("query.lookback-delta", "The maximum lookback duration for retrieving metrics during expression evaluations and federation.").
Default("5m").SetValue(&cfg.lookbackDelta)
a.Flag("query.timeout", "Maximum time a query may take before being aborted.").
@ -272,6 +279,12 @@ func main() {
os.Exit(2)
}
// Throw error for invalid config before starting other components.
if _, err := config.LoadFile(cfg.configFile); err != nil {
level.Error(logger).Log("msg", fmt.Sprintf("Error loading config (--config.file=%s)", cfg.configFile), "err", err)
os.Exit(2)
}
cfg.web.ReadTimeout = time.Duration(cfg.webTimeout)
// Default -web.route-prefix to path of -web.external-url.
if cfg.web.RoutePrefix == "" {
@ -293,7 +306,7 @@ func main() {
if cfg.tsdb.RetentionDuration == 0 && cfg.tsdb.MaxBytes == 0 {
cfg.tsdb.RetentionDuration = defaultRetentionDuration
level.Info(logger).Log("msg", "no time or size retention was set so using the default time retention", "duration", defaultRetentionDuration)
level.Info(logger).Log("msg", "No time or size retention was set so using the default time retention", "duration", defaultRetentionDuration)
}
// Check for overflows. This limits our max retention to 100y.
@ -303,7 +316,7 @@ func main() {
panic(err)
}
cfg.tsdb.RetentionDuration = y
level.Warn(logger).Log("msg", "time retention value is too high. Limiting to: "+y.String())
level.Warn(logger).Log("msg", "Time retention value is too high. Limiting to: "+y.String())
}
}
@ -322,8 +335,8 @@ func main() {
}
}
promql.LookbackDelta = time.Duration(cfg.lookbackDelta)
promql.SetDefaultEvaluationInterval(time.Duration(config.DefaultGlobalConfig.EvaluationInterval))
noStepSubqueryInterval := &safePromQLNoStepSubqueryInterval{}
noStepSubqueryInterval.Set(config.DefaultGlobalConfig.EvaluationInterval)
// Above level 6, the k8s client would log bearer tokens in clear-text.
klog.ClampLevel(6)
@ -333,10 +346,10 @@ func main() {
level.Info(logger).Log("build_context", version.BuildContext())
level.Info(logger).Log("host_details", prom_runtime.Uname())
level.Info(logger).Log("fd_limits", prom_runtime.FdLimits())
level.Info(logger).Log("vm_limits", prom_runtime.VmLimits())
level.Info(logger).Log("vm_limits", prom_runtime.VMLimits())
var (
localStorage = &tsdb.ReadyStorage{}
localStorage = &readyStorage{}
remoteStorage = remote.NewStorage(log.With(logger, "component", "remote"), prometheus.DefaultRegisterer, localStorage.StartTime, cfg.localStoragePath, time.Duration(cfg.RemoteFlushDeadline))
fanoutStorage = storage.NewFanout(logger, localStorage, remoteStorage)
)
@ -356,19 +369,20 @@ func main() {
scrapeManager = scrape.NewManager(log.With(logger, "component", "scrape manager"), fanoutStorage)
opts = promql.EngineOpts{
Logger: log.With(logger, "component", "query engine"),
Reg: prometheus.DefaultRegisterer,
MaxConcurrent: cfg.queryConcurrency,
MaxSamples: cfg.queryMaxSamples,
Timeout: time.Duration(cfg.queryTimeout),
ActiveQueryTracker: promql.NewActiveQueryTracker(cfg.localStoragePath, cfg.queryConcurrency, log.With(logger, "component", "activeQueryTracker")),
Logger: log.With(logger, "component", "query engine"),
Reg: prometheus.DefaultRegisterer,
MaxSamples: cfg.queryMaxSamples,
Timeout: time.Duration(cfg.queryTimeout),
ActiveQueryTracker: promql.NewActiveQueryTracker(cfg.localStoragePath, cfg.queryConcurrency, log.With(logger, "component", "activeQueryTracker")),
LookbackDelta: time.Duration(cfg.lookbackDelta),
NoStepSubqueryIntervalFn: noStepSubqueryInterval.Get,
}
queryEngine = promql.NewEngine(opts)
ruleManager = rules.NewManager(&rules.ManagerOptions{
Appendable: fanoutStorage,
TSDB: localStorage,
Queryable: localStorage,
QueryFunc: rules.EngineQueryFunc(queryEngine, fanoutStorage),
NotifyFunc: sendAlerts(notifierManager, cfg.web.ExternalURL.String()),
Context: ctxRule,
@ -382,13 +396,16 @@ func main() {
)
cfg.web.Context = ctxWeb
cfg.web.TSDB = localStorage.Get
cfg.web.TSDBRetentionDuration = cfg.tsdb.RetentionDuration
cfg.web.TSDBMaxBytes = cfg.tsdb.MaxBytes
cfg.web.TSDBDir = cfg.localStoragePath
cfg.web.LocalStorage = localStorage
cfg.web.Storage = fanoutStorage
cfg.web.QueryEngine = queryEngine
cfg.web.ScrapeManager = scrapeManager
cfg.web.RuleManager = ruleManager
cfg.web.Notifier = notifierManager
cfg.web.TSDBCfg = cfg.tsdb
cfg.web.LookbackDelta = time.Duration(cfg.lookbackDelta)
cfg.web.Version = &web.PrometheusVersion{
Version: version.Version,
@ -419,48 +436,73 @@ func main() {
conntrack.DialWithTracing(),
)
reloaders := []func(cfg *config.Config) error{
remoteStorage.ApplyConfig,
webHandler.ApplyConfig,
// The Scrape and notifier managers need to reload before the Discovery manager as
// they need to read the most updated config when receiving the new targets list.
scrapeManager.ApplyConfig,
func(cfg *config.Config) error {
c := make(map[string]sd_config.ServiceDiscoveryConfig)
for _, v := range cfg.ScrapeConfigs {
c[v.JobName] = v.ServiceDiscoveryConfig
}
return discoveryManagerScrape.ApplyConfig(c)
},
notifierManager.ApplyConfig,
func(cfg *config.Config) error {
c := make(map[string]sd_config.ServiceDiscoveryConfig)
for _, v := range cfg.AlertingConfig.AlertmanagerConfigs {
// AlertmanagerConfigs doesn't hold an unique identifier so we use the config hash as the identifier.
b, err := json.Marshal(v)
reloaders := []reloader{
{
name: "remote_storage",
reloader: remoteStorage.ApplyConfig,
}, {
name: "web_handler",
reloader: webHandler.ApplyConfig,
}, {
name: "query_engine",
reloader: func(cfg *config.Config) error {
if cfg.GlobalConfig.QueryLogFile == "" {
queryEngine.SetQueryLogger(nil)
return nil
}
l, err := logging.NewJSONFileLogger(cfg.GlobalConfig.QueryLogFile)
if err != nil {
return err
}
c[fmt.Sprintf("%x", md5.Sum(b))] = v.ServiceDiscoveryConfig
}
return discoveryManagerNotify.ApplyConfig(c)
},
func(cfg *config.Config) error {
// Get all rule files matching the configuration paths.
var files []string
for _, pat := range cfg.RuleFiles {
fs, err := filepath.Glob(pat)
if err != nil {
// The only error can be a bad pattern.
return errors.Wrapf(err, "error retrieving rule files for %s", pat)
queryEngine.SetQueryLogger(l)
return nil
},
}, {
// The Scrape and notifier managers need to reload before the Discovery manager as
// they need to read the most updated config when receiving the new targets list.
name: "scrape",
reloader: scrapeManager.ApplyConfig,
}, {
name: "scrape_sd",
reloader: func(cfg *config.Config) error {
c := make(map[string]sd_config.ServiceDiscoveryConfig)
for _, v := range cfg.ScrapeConfigs {
c[v.JobName] = v.ServiceDiscoveryConfig
}
files = append(files, fs...)
}
return ruleManager.Update(
time.Duration(cfg.GlobalConfig.EvaluationInterval),
files,
cfg.GlobalConfig.ExternalLabels,
)
return discoveryManagerScrape.ApplyConfig(c)
},
}, {
name: "notify",
reloader: notifierManager.ApplyConfig,
}, {
name: "notify_sd",
reloader: func(cfg *config.Config) error {
c := make(map[string]sd_config.ServiceDiscoveryConfig)
for k, v := range cfg.AlertingConfig.AlertmanagerConfigs.ToMap() {
c[k] = v.ServiceDiscoveryConfig
}
return discoveryManagerNotify.ApplyConfig(c)
},
}, {
name: "rules",
reloader: func(cfg *config.Config) error {
// Get all rule files matching the configuration paths.
var files []string
for _, pat := range cfg.RuleFiles {
fs, err := filepath.Glob(pat)
if err != nil {
// The only error can be a bad pattern.
return errors.Wrapf(err, "error retrieving rule files for %s", pat)
}
files = append(files, fs...)
}
return ruleManager.Update(
time.Duration(cfg.GlobalConfig.EvaluationInterval),
files,
cfg.GlobalConfig.ExternalLabels,
)
},
},
}
@ -487,6 +529,13 @@ func main() {
})
}
closer, err := initTracing(logger)
if err != nil {
level.Error(logger).Log("msg", "Unable to init tracing", "err", err)
os.Exit(2)
}
defer closer.Close()
var g run.Group
{
// Termination handler.
@ -500,12 +549,10 @@ func main() {
case <-term:
level.Warn(logger).Log("msg", "Received SIGTERM, exiting gracefully...")
reloadReady.Close()
case <-webHandler.Quit():
level.Warn(logger).Log("msg", "Received termination request via web service, exiting gracefully...")
case <-cancel:
reloadReady.Close()
break
}
return nil
},
@ -579,11 +626,11 @@ func main() {
for {
select {
case <-hup:
if err := reloadConfig(cfg.configFile, logger, reloaders...); err != nil {
if err := reloadConfig(cfg.configFile, logger, noStepSubqueryInterval, reloaders...); err != nil {
level.Error(logger).Log("msg", "Error reloading config", "err", err)
}
case rc := <-webHandler.Reload():
if err := reloadConfig(cfg.configFile, logger, reloaders...); err != nil {
if err := reloadConfig(cfg.configFile, logger, noStepSubqueryInterval, reloaders...); err != nil {
level.Error(logger).Log("msg", "Error reloading config", "err", err)
rc <- err
} else {
@ -609,14 +656,13 @@ func main() {
func() error {
select {
case <-dbOpen:
break
// In case a shutdown is initiated before the dbOpen is released
case <-cancel:
reloadReady.Close()
return nil
}
if err := reloadConfig(cfg.configFile, logger, reloaders...); err != nil {
if err := reloadConfig(cfg.configFile, logger, noStepSubqueryInterval, reloaders...); err != nil {
return errors.Wrapf(err, "error loading config from %q", cfg.configFile)
}
@ -634,23 +680,20 @@ func main() {
}
{
// Rule manager.
// TODO(krasi) refactor ruleManager.Run() to be blocking to avoid using an extra blocking channel.
cancel := make(chan struct{})
g.Add(
func() error {
<-reloadReady.C
ruleManager.Run()
<-cancel
return nil
},
func(err error) {
ruleManager.Stop()
close(cancel)
},
)
}
{
// TSDB.
opts := cfg.tsdb.ToTSDBOptions()
cancel := make(chan struct{})
g.Add(
func() error {
@ -660,16 +703,23 @@ func main() {
return errors.New("flag 'storage.tsdb.wal-segment-size' must be set between 10MB and 256MB")
}
}
db, err := tsdb.Open(
db, err := openDBWithMetrics(
cfg.localStoragePath,
log.With(logger, "component", "tsdb"),
logger,
prometheus.DefaultRegisterer,
&cfg.tsdb,
&opts,
)
if err != nil {
return errors.Wrapf(err, "opening storage failed")
}
level.Info(logger).Log("fs_type", prom_runtime.Statfs(cfg.localStoragePath))
switch fsType := prom_runtime.Statfs(cfg.localStoragePath); fsType {
case "NFS_SUPER_MAGIC":
level.Warn(logger).Log("fs_type", fsType, "msg", "This filesystem is not supported and may lead to data corruption and data loss. Please carefully read https://prometheus.io/docs/prometheus/latest/storage/ to learn more about supported filesystems.")
default:
level.Info(logger).Log("fs_type", fsType)
}
level.Info(logger).Log("msg", "TSDB started")
level.Debug(logger).Log("msg", "TSDB options",
"MinBlockDuration", cfg.tsdb.MinBlockDuration,
@ -739,7 +789,63 @@ func main() {
level.Info(logger).Log("msg", "See you next time!")
}
func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config) error) (err error) {
func openDBWithMetrics(dir string, logger log.Logger, reg prometheus.Registerer, opts *tsdb.Options) (*tsdb.DB, error) {
db, err := tsdb.Open(
dir,
log.With(logger, "component", "tsdb"),
reg,
opts,
)
if err != nil {
return nil, err
}
reg.MustRegister(
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "prometheus_tsdb_lowest_timestamp_seconds",
Help: "Lowest timestamp value stored in the database.",
}, func() float64 {
bb := db.Blocks()
if len(bb) == 0 {
return float64(db.Head().MinTime() / 1000)
}
return float64(db.Blocks()[0].Meta().MinTime / 1000)
}), prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "prometheus_tsdb_head_min_time_seconds",
Help: "Minimum time bound of the head block.",
}, func() float64 { return float64(db.Head().MinTime() / 1000) }),
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "prometheus_tsdb_head_max_time_seconds",
Help: "Maximum timestamp of the head block.",
}, func() float64 { return float64(db.Head().MaxTime() / 1000) }),
)
return db, nil
}
type safePromQLNoStepSubqueryInterval struct {
value atomic.Int64
}
func durationToInt64Millis(d time.Duration) int64 {
return int64(d / time.Millisecond)
}
func (i *safePromQLNoStepSubqueryInterval) Set(ev model.Duration) {
i.value.Store(durationToInt64Millis(time.Duration(ev)))
}
func (i *safePromQLNoStepSubqueryInterval) Get(int64) int64 {
return i.value.Load()
}
type reloader struct {
name string
reloader func(*config.Config) error
}
func reloadConfig(filename string, logger log.Logger, noStepSuqueryInterval *safePromQLNoStepSubqueryInterval, rls ...reloader) (err error) {
start := time.Now()
timings := []interface{}{}
level.Info(logger).Log("msg", "Loading configuration file", "filename", filename)
defer func() {
@ -758,17 +864,20 @@ func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config
failed := false
for _, rl := range rls {
if err := rl(conf); err != nil {
rstart := time.Now()
if err := rl.reloader(conf); err != nil {
level.Error(logger).Log("msg", "Failed to apply configuration", "err", err)
failed = true
}
timings = append(timings, rl.name, time.Since(rstart))
}
if failed {
return errors.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename)
}
promql.SetDefaultEvaluationInterval(time.Duration(conf.GlobalConfig.EvaluationInterval))
level.Info(logger).Log("msg", "Completed loading of configuration file", "filename", filename)
noStepSuqueryInterval.Set(conf.GlobalConfig.EvaluationInterval)
l := []interface{}{"msg", "Completed loading of configuration file", "filename", filename, "totalDuration", time.Since(start)}
level.Info(logger).Log(append(l, timings...)...)
return nil
}
@ -848,3 +957,190 @@ func sendAlerts(s sender, externalURL string) rules.NotifyFunc {
}
}
}
// readyStorage implements the Storage interface while allowing to set the actual
// storage at a later point in time.
type readyStorage struct {
mtx sync.RWMutex
db *tsdb.DB
startTimeMargin int64
}
// Set the storage.
func (s *readyStorage) Set(db *tsdb.DB, startTimeMargin int64) {
s.mtx.Lock()
defer s.mtx.Unlock()
s.db = db
s.startTimeMargin = startTimeMargin
}
// get is internal, you should use readyStorage as the front implementation layer.
func (s *readyStorage) get() *tsdb.DB {
s.mtx.RLock()
x := s.db
s.mtx.RUnlock()
return x
}
// StartTime implements the Storage interface.
func (s *readyStorage) StartTime() (int64, error) {
if x := s.get(); x != nil {
var startTime int64
if len(x.Blocks()) > 0 {
startTime = x.Blocks()[0].Meta().MinTime
} else {
startTime = time.Now().Unix() * 1000
}
// Add a safety margin as it may take a few minutes for everything to spin up.
return startTime + s.startTimeMargin, nil
}
return math.MaxInt64, tsdb.ErrNotReady
}
// Querier implements the Storage interface.
func (s *readyStorage) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) {
if x := s.get(); x != nil {
return x.Querier(ctx, mint, maxt)
}
return nil, tsdb.ErrNotReady
}
// ChunkQuerier implements the Storage interface.
func (s *readyStorage) ChunkQuerier(ctx context.Context, mint, maxt int64) (storage.ChunkQuerier, error) {
if x := s.get(); x != nil {
return x.ChunkQuerier(ctx, mint, maxt)
}
return nil, tsdb.ErrNotReady
}
// Appender implements the Storage interface.
func (s *readyStorage) Appender(ctx context.Context) storage.Appender {
if x := s.get(); x != nil {
return x.Appender(ctx)
}
return notReadyAppender{}
}
type notReadyAppender struct{}
func (n notReadyAppender) Add(l labels.Labels, t int64, v float64) (uint64, error) {
return 0, tsdb.ErrNotReady
}
func (n notReadyAppender) AddFast(ref uint64, t int64, v float64) error { return tsdb.ErrNotReady }
func (n notReadyAppender) Commit() error { return tsdb.ErrNotReady }
func (n notReadyAppender) Rollback() error { return tsdb.ErrNotReady }
// Close implements the Storage interface.
func (s *readyStorage) Close() error {
if x := s.get(); x != nil {
return x.Close()
}
return nil
}
// CleanTombstones implements the api_v1.TSDBAdminStats and api_v2.TSDBAdmin interfaces.
func (s *readyStorage) CleanTombstones() error {
if x := s.get(); x != nil {
return x.CleanTombstones()
}
return tsdb.ErrNotReady
}
// Delete implements the api_v1.TSDBAdminStats and api_v2.TSDBAdmin interfaces.
func (s *readyStorage) Delete(mint, maxt int64, ms ...*labels.Matcher) error {
if x := s.get(); x != nil {
return x.Delete(mint, maxt, ms...)
}
return tsdb.ErrNotReady
}
// Snapshot implements the api_v1.TSDBAdminStats and api_v2.TSDBAdmin interfaces.
func (s *readyStorage) Snapshot(dir string, withHead bool) error {
if x := s.get(); x != nil {
return x.Snapshot(dir, withHead)
}
return tsdb.ErrNotReady
}
// Stats implements the api_v1.TSDBAdminStats interface.
func (s *readyStorage) Stats(statsByLabelName string) (*tsdb.Stats, error) {
if x := s.get(); x != nil {
return x.Head().Stats(statsByLabelName), nil
}
return nil, tsdb.ErrNotReady
}
// tsdbOptions is tsdb.Option version with defined units.
// This is required as tsdb.Option fields are unit agnostic (time).
type tsdbOptions struct {
WALSegmentSize units.Base2Bytes
RetentionDuration model.Duration
MaxBytes units.Base2Bytes
NoLockfile bool
AllowOverlappingBlocks bool
WALCompression bool
StripeSize int
MinBlockDuration model.Duration
MaxBlockDuration model.Duration
}
func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
return tsdb.Options{
WALSegmentSize: int(opts.WALSegmentSize),
RetentionDuration: int64(time.Duration(opts.RetentionDuration) / time.Millisecond),
MaxBytes: int64(opts.MaxBytes),
NoLockfile: opts.NoLockfile,
AllowOverlappingBlocks: opts.AllowOverlappingBlocks,
WALCompression: opts.WALCompression,
StripeSize: opts.StripeSize,
MinBlockDuration: int64(time.Duration(opts.MinBlockDuration) / time.Millisecond),
MaxBlockDuration: int64(time.Duration(opts.MaxBlockDuration) / time.Millisecond),
}
}
func initTracing(logger log.Logger) (io.Closer, error) {
// Set tracing configuration defaults.
cfg := &jcfg.Configuration{
ServiceName: "prometheus",
Disabled: true,
}
// Available options can be seen here:
// https://github.com/jaegertracing/jaeger-client-go#environment-variables
cfg, err := cfg.FromEnv()
if err != nil {
return nil, errors.Wrap(err, "unable to get tracing config from environment")
}
jLogger := jaegerLogger{logger: log.With(logger, "component", "tracing")}
tracer, closer, err := cfg.NewTracer(
jcfg.Logger(jLogger),
jcfg.Metrics(jprom.New()),
)
if err != nil {
return nil, errors.Wrap(err, "unable to init tracing")
}
opentracing.SetGlobalTracer(tracer)
return closer, nil
}
type jaegerLogger struct {
logger log.Logger
}
func (l jaegerLogger) Error(msg string) {
level.Error(l.logger).Log("msg", msg)
}
func (l jaegerLogger) Infof(msg string, args ...interface{}) {
keyvals := []interface{}{"msg", fmt.Sprintf(msg, args...)}
level.Info(l.logger).Log(keyvals...)
}

170
cmd/prometheus/main_test.go
View file

@ -15,9 +15,9 @@ package main
import (
"context"
"flag"
"fmt"
"net/http"
"io/ioutil"
"math"
"os"
"os/exec"
"path/filepath"
@ -25,93 +25,36 @@ import (
"testing"
"time"
"github.com/go-kit/kit/log"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/notifier"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/rules"
"github.com/prometheus/prometheus/util/testutil"
)
var promPath string
var promPath = os.Args[0]
var promConfig = filepath.Join("..", "..", "documentation", "examples", "prometheus.yml")
var promData = filepath.Join(os.TempDir(), "data")
func TestMain(m *testing.M) {
flag.Parse()
if testing.Short() {
os.Exit(m.Run())
for i, arg := range os.Args {
if arg == "-test.main" {
os.Args = append(os.Args[:i], os.Args[i+1:]...)
main()
return
}
}
// On linux with a global proxy the tests will fail as the go client(http,grpc) tries to connect through the proxy.
os.Setenv("no_proxy", "localhost,127.0.0.1,0.0.0.0,:")
var err error
promPath, err = os.Getwd()
if err != nil {
fmt.Printf("can't get current dir :%s \n", err)
os.Exit(1)
}
promPath = filepath.Join(promPath, "prometheus")
build := exec.Command("go", "build", "-o", promPath)
output, err := build.CombinedOutput()
if err != nil {
fmt.Printf("compilation error :%s \n", output)
os.Exit(1)
}
exitCode := m.Run()
os.Remove(promPath)
os.RemoveAll(promData)
os.Exit(exitCode)
}
// As soon as prometheus starts responding to http request should be able to accept Interrupt signals for a graceful shutdown.
func TestStartupInterrupt(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
prom := exec.Command(promPath, "--config.file="+promConfig, "--storage.tsdb.path="+promData)
err := prom.Start()
if err != nil {
t.Errorf("execution error: %v", err)
return
}
done := make(chan error)
go func() {
done <- prom.Wait()
}()
var startedOk bool
var stoppedErr error
Loop:
for x := 0; x < 10; x++ {
// error=nil means prometheus has started so can send the interrupt signal and wait for the grace shutdown.
if _, err := http.Get("http://localhost:9090/graph"); err == nil {
startedOk = true
prom.Process.Signal(os.Interrupt)
select {
case stoppedErr = <-done:
break Loop
case <-time.After(10 * time.Second):
}
break Loop
}
time.Sleep(500 * time.Millisecond)
}
if !startedOk {
t.Errorf("prometheus didn't start in the specified timeout")
return
}
if err := prom.Process.Kill(); err == nil {
t.Errorf("prometheus didn't shutdown gracefully after sending the Interrupt signal")
} else if stoppedErr != nil && stoppedErr.Error() != "signal: interrupt" { // TODO - find a better way to detect when the process didn't exit as expected!
t.Errorf("prometheus exited with an unexpected error:%v", stoppedErr)
}
}
func TestComputeExternalURL(t *testing.T) {
tests := []struct {
input string
@ -168,9 +111,9 @@ func TestFailedStartupExitCode(t *testing.T) {
}
fakeInputFile := "fake-input-file"
expectedExitStatus := 1
expectedExitStatus := 2
prom := exec.Command(promPath, "--config.file="+fakeInputFile)
prom := exec.Command(promPath, "-test.main", "--config.file="+fakeInputFile)
err := prom.Run()
testutil.NotOk(t, err)
@ -258,8 +201,17 @@ func TestWALSegmentSizeBounds(t *testing.T) {
}
for size, expectedExitStatus := range map[string]int{"9MB": 1, "257MB": 1, "10": 2, "1GB": 1, "12MB": 0} {
prom := exec.Command(promPath, "--storage.tsdb.wal-segment-size="+size, "--config.file="+promConfig)
err := prom.Start()
prom := exec.Command(promPath, "-test.main", "--storage.tsdb.wal-segment-size="+size, "--config.file="+promConfig)
// Log stderr in case of failure.
stderr, err := prom.StderrPipe()
testutil.Ok(t, err)
go func() {
slurp, _ := ioutil.ReadAll(stderr)
t.Log(string(slurp))
}()
err = prom.Start()
testutil.Ok(t, err)
if expectedExitStatus == 0 {
@ -269,7 +221,7 @@ func TestWALSegmentSizeBounds(t *testing.T) {
case err := <-done:
t.Errorf("prometheus should be still running: %v", err)
case <-time.After(5 * time.Second):
prom.Process.Signal(os.Interrupt)
prom.Process.Kill()
}
continue
}
@ -284,3 +236,71 @@ func TestWALSegmentSizeBounds(t *testing.T) {
}
}
}
func TestTimeMetrics(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "time_metrics_e2e")
testutil.Ok(t, err)
defer func() {
testutil.Ok(t, os.RemoveAll(tmpDir))
}()
reg := prometheus.NewRegistry()
db, err := openDBWithMetrics(tmpDir, log.NewNopLogger(), reg, nil)
testutil.Ok(t, err)
defer func() {
testutil.Ok(t, db.Close())
}()
// Check initial values.
testutil.Equals(t, map[string]float64{
"prometheus_tsdb_lowest_timestamp_seconds": float64(math.MaxInt64) / 1000,
"prometheus_tsdb_head_min_time_seconds": float64(math.MaxInt64) / 1000,
"prometheus_tsdb_head_max_time_seconds": float64(math.MinInt64) / 1000,
}, getCurrentGaugeValuesFor(t, reg,
"prometheus_tsdb_lowest_timestamp_seconds",
"prometheus_tsdb_head_min_time_seconds",
"prometheus_tsdb_head_max_time_seconds",
))
app := db.Appender(context.Background())
_, err = app.Add(labels.FromStrings(model.MetricNameLabel, "a"), 1000, 1)
testutil.Ok(t, err)
_, err = app.Add(labels.FromStrings(model.MetricNameLabel, "a"), 2000, 1)
testutil.Ok(t, err)
_, err = app.Add(labels.FromStrings(model.MetricNameLabel, "a"), 3000, 1)
testutil.Ok(t, err)
testutil.Ok(t, app.Commit())
testutil.Equals(t, map[string]float64{
"prometheus_tsdb_lowest_timestamp_seconds": 1.0,
"prometheus_tsdb_head_min_time_seconds": 1.0,
"prometheus_tsdb_head_max_time_seconds": 3.0,
}, getCurrentGaugeValuesFor(t, reg,
"prometheus_tsdb_lowest_timestamp_seconds",
"prometheus_tsdb_head_min_time_seconds",
"prometheus_tsdb_head_max_time_seconds",
))
}
func getCurrentGaugeValuesFor(t *testing.T, reg prometheus.Gatherer, metricNames ...string) map[string]float64 {
f, err := reg.Gather()
testutil.Ok(t, err)
res := make(map[string]float64, len(metricNames))
for _, g := range f {
for _, m := range metricNames {
if g.GetName() != m {
continue
}
testutil.Equals(t, 1, len(g.GetMetric()))
if _, ok := res[m]; ok {
t.Error("expected only one metric family for", m)
t.FailNow()
}
res[m] = *g.GetMetric()[0].GetGauge().Value
}
}
return res
}

74
cmd/prometheus/main_unix_test.go Normal file
View file

@ -0,0 +1,74 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// +build !windows
package main
import (
"net/http"
"os"
"os/exec"
"testing"
"time"
)
// As soon as prometheus starts responding to http request it should be able to
// accept Interrupt signals for a graceful shutdown.
func TestStartupInterrupt(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
prom := exec.Command(promPath, "-test.main", "--config.file="+promConfig, "--storage.tsdb.path="+promData)
err := prom.Start()
if err != nil {
t.Errorf("execution error: %v", err)
return
}
done := make(chan error)
go func() {
done <- prom.Wait()
}()
var startedOk bool
var stoppedErr error
Loop:
for x := 0; x < 10; x++ {
// error=nil means prometheus has started so we can send the interrupt
// signal and wait for the graceful shutdown.
if _, err := http.Get("http://localhost:9090/graph"); err == nil {
startedOk = true
prom.Process.Signal(os.Interrupt)
select {
case stoppedErr = <-done:
break Loop
case <-time.After(10 * time.Second):
}
break Loop
}
time.Sleep(500 * time.Millisecond)
}
if !startedOk {
t.Errorf("prometheus didn't start in the specified timeout")
return
}
if err := prom.Process.Kill(); err == nil {
t.Errorf("prometheus didn't shutdown gracefully after sending the Interrupt signal")
} else if stoppedErr != nil && stoppedErr.Error() != "signal: interrupt" { // TODO - find a better way to detect when the process didn't exit as expected!
t.Errorf("prometheus exited with an unexpected error:%v", stoppedErr)
}
}

436
cmd/prometheus/query_log_test.go Normal file
View file

@ -0,0 +1,436 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"sync"
"testing"
"time"
"github.com/prometheus/prometheus/util/testutil"
)
type origin int
const (
apiOrigin origin = iota
consoleOrigin
ruleOrigin
)
// queryLogTest defines a query log test.
type queryLogTest struct {
origin origin // Kind of queries tested: api, console, rules.
prefix string // Set as --web.route-prefix.
host string // Used in --web.listen-address. Used with 127.0.0.1 and ::1.
port int // Used in --web.listen-address.
cwd string // Directory where the test is running. Required to find the rules in testdata.
configFile *os.File // The configuration file.
enabledAtStart bool // Whether query log is enabled at startup.
}
// skip checks if the test is needed and the prerequisites are met.
func (p *queryLogTest) skip(t *testing.T) {
if p.prefix != "" && p.origin == ruleOrigin {
t.Skip("changing prefix has no effect on rules")
}
// Some systems don't support IPv4 or IPv6.
l, err := net.Listen("tcp", fmt.Sprintf("%s:0", p.host))
if err != nil {
t.Skip("ip version not supported")
}
l.Close()
}
// waitForPrometheus waits for Prometheus to be ready.
func (p *queryLogTest) waitForPrometheus() error {
var err error
for x := 0; x < 20; x++ {
var r *http.Response
if r, err = http.Get(fmt.Sprintf("http://%s:%d%s/-/ready", p.host, p.port, p.prefix)); err == nil && r.StatusCode == 200 {
break
}
time.Sleep(500 * time.Millisecond)
}
return err
}
// setQueryLog alters the configuration file to enable or disable the query log,
// then reloads the configuration if needed.
func (p *queryLogTest) setQueryLog(t *testing.T, queryLogFile string) {
err := p.configFile.Truncate(0)
testutil.Ok(t, err)
_, err = p.configFile.Seek(0, 0)
testutil.Ok(t, err)
if queryLogFile != "" {
_, err = p.configFile.Write([]byte(fmt.Sprintf("global:\n query_log_file: %s\n", queryLogFile)))
testutil.Ok(t, err)
}
_, err = p.configFile.Write([]byte(p.configuration()))
testutil.Ok(t, err)
}
// reloadConfig reloads the configuration using POST.
func (p *queryLogTest) reloadConfig(t *testing.T) {
r, err := http.Post(fmt.Sprintf("http://%s:%d%s/-/reload", p.host, p.port, p.prefix), "text/plain", nil)
testutil.Ok(t, err)
testutil.Equals(t, 200, r.StatusCode)
}
// query runs a query according to the test origin.
func (p *queryLogTest) query(t *testing.T) {
switch p.origin {
case apiOrigin:
r, err := http.Get(fmt.Sprintf(
"http://%s:%d%s/api/v1/query_range?step=5&start=0&end=3600&query=%s",
p.host,
p.port,
p.prefix,
url.QueryEscape("query_with_api"),
))
testutil.Ok(t, err)
testutil.Equals(t, 200, r.StatusCode)
case consoleOrigin:
r, err := http.Get(fmt.Sprintf(
"http://%s:%d%s/consoles/test.html",
p.host,
p.port,
p.prefix,
))
testutil.Ok(t, err)
testutil.Equals(t, 200, r.StatusCode)
case ruleOrigin:
time.Sleep(2 * time.Second)
default:
panic("can't query this origin")
}
}
// queryString returns the expected queryString of a this test.
func (p *queryLogTest) queryString() string {
switch p.origin {
case apiOrigin:
return "query_with_api"
case ruleOrigin:
return "query_in_rule"
case consoleOrigin:
return "query_in_console"
default:
panic("unknown origin")
}
}
// validateLastQuery checks that the last query in the query log matches the
// test parameters.
func (p *queryLogTest) validateLastQuery(t *testing.T, ql []queryLogLine) {
q := ql[len(ql)-1]
testutil.Equals(t, p.queryString(), q.Params.Query)
switch p.origin {
case apiOrigin:
testutil.Equals(t, 5, q.Params.Step)
testutil.Equals(t, "1970-01-01T00:00:00.000Z", q.Params.Start)
testutil.Equals(t, "1970-01-01T01:00:00.000Z", q.Params.End)
default:
testutil.Equals(t, 0, q.Params.Step)
}
if p.origin != ruleOrigin {
host := p.host
if host == "[::1]" {
host = "::1"
}
testutil.Equals(t, host, q.Request.ClientIP)
}
switch p.origin {
case apiOrigin:
testutil.Equals(t, p.prefix+"/api/v1/query_range", q.Request.Path)
case consoleOrigin:
testutil.Equals(t, p.prefix+"/consoles/test.html", q.Request.Path)
case ruleOrigin:
testutil.Equals(t, "querylogtest", q.RuleGroup.Name)
testutil.Equals(t, filepath.Join(p.cwd, "testdata", "rules", "test.yml"), q.RuleGroup.File)
default:
panic("unknown origin")
}
}
func (p *queryLogTest) String() string {
var name string
switch p.origin {
case apiOrigin:
name = "api queries"
case consoleOrigin:
name = "console queries"
case ruleOrigin:
name = "rule queries"
}
name = name + ", " + p.host + ":" + strconv.Itoa(p.port)
if p.enabledAtStart {
name = name + ", enabled at start"
}
if p.prefix != "" {
name = name + ", with prefix " + p.prefix
}
return name
}
// params returns the specific command line parameters of this test.
func (p *queryLogTest) params() []string {
s := []string{}
if p.prefix != "" {
s = append(s, "--web.route-prefix="+p.prefix)
}
if p.origin == consoleOrigin {
s = append(s, "--web.console.templates="+filepath.Join("testdata", "consoles"))
}
return s
}
// configuration returns the specific configuration lines required for this
// test.
func (p *queryLogTest) configuration() string {
switch p.origin {
case ruleOrigin:
return "\nrule_files:\n- " + filepath.Join(p.cwd, "testdata", "rules", "test.yml") + "\n"
default:
return "\n"
}
}
// exactQueryCount returns whether we can match an exact query count. False on
// recording rules are they are regular time intervals.
func (p *queryLogTest) exactQueryCount() bool {
return p.origin != ruleOrigin
}
// run launches the scenario of this query log test.
func (p *queryLogTest) run(t *testing.T) {
p.skip(t)
// Setup temporary files for this test.
queryLogFile, err := ioutil.TempFile("", "query")
testutil.Ok(t, err)
defer os.Remove(queryLogFile.Name())
p.configFile, err = ioutil.TempFile("", "config")
testutil.Ok(t, err)
defer os.Remove(p.configFile.Name())
if p.enabledAtStart {
p.setQueryLog(t, queryLogFile.Name())
} else {
p.setQueryLog(t, "")
}
dir, err := ioutil.TempDir("", "query_log_test")
testutil.Ok(t, err)
defer func() {
testutil.Ok(t, os.RemoveAll(dir))
}()
params := append([]string{
"-test.main",
"--config.file=" + p.configFile.Name(),
"--web.enable-lifecycle",
fmt.Sprintf("--web.listen-address=%s:%d", p.host, p.port),
"--storage.tsdb.path=" + dir,
}, p.params()...)
prom := exec.Command(promPath, params...)
// Log stderr in case of failure.
stderr, err := prom.StderrPipe()
testutil.Ok(t, err)
// We use a WaitGroup to avoid calling t.Log after the test is done.
var wg sync.WaitGroup
wg.Add(1)
defer wg.Wait()
go func() {
slurp, _ := ioutil.ReadAll(stderr)
t.Log(string(slurp))
wg.Done()
}()
testutil.Ok(t, prom.Start())
defer func() {
prom.Process.Kill()
prom.Wait()
}()
testutil.Ok(t, p.waitForPrometheus())
if !p.enabledAtStart {
p.query(t)
testutil.Equals(t, 0, len(readQueryLog(t, queryLogFile.Name())))
p.setQueryLog(t, queryLogFile.Name())
p.reloadConfig(t)
}
p.query(t)
ql := readQueryLog(t, queryLogFile.Name())
qc := len(ql)
if p.exactQueryCount() {
testutil.Equals(t, 1, qc)
} else {
testutil.Assert(t, qc > 0, "no queries logged")
}
p.validateLastQuery(t, ql)
p.setQueryLog(t, "")
p.reloadConfig(t)
if !p.exactQueryCount() {
qc = len(readQueryLog(t, queryLogFile.Name()))
}
p.query(t)
ql = readQueryLog(t, queryLogFile.Name())
testutil.Equals(t, qc, len(ql))
qc = len(ql)
p.setQueryLog(t, queryLogFile.Name())
p.reloadConfig(t)
p.query(t)
qc++
ql = readQueryLog(t, queryLogFile.Name())
if p.exactQueryCount() {
testutil.Equals(t, qc, len(ql))
} else {
testutil.Assert(t, len(ql) > qc, "no queries logged")
}
p.validateLastQuery(t, ql)
qc = len(ql)
// The last part of the test can not succeed on Windows because you can't
// rename files used by other processes.
if runtime.GOOS == "windows" {
return
}
// Move the file, Prometheus should still write to the old file.
newFile, err := ioutil.TempFile("", "newLoc")
testutil.Ok(t, err)
testutil.Ok(t, newFile.Close())
defer os.Remove(newFile.Name())
testutil.Ok(t, os.Rename(queryLogFile.Name(), newFile.Name()))
ql = readQueryLog(t, newFile.Name())
if p.exactQueryCount() {
testutil.Equals(t, qc, len(ql))
}
p.validateLastQuery(t, ql)
qc = len(ql)
p.query(t)
qc++
ql = readQueryLog(t, newFile.Name())
if p.exactQueryCount() {
testutil.Equals(t, qc, len(ql))
} else {
testutil.Assert(t, len(ql) > qc, "no queries logged")
}
p.validateLastQuery(t, ql)
p.reloadConfig(t)
p.query(t)
ql = readQueryLog(t, queryLogFile.Name())
qc = len(ql)
if p.exactQueryCount() {
testutil.Equals(t, 1, qc)
} else {
testutil.Assert(t, qc > 0, "no queries logged")
}
}
type queryLogLine struct {
Params struct {
Query string `json:"query"`
Step int `json:"step"`
Start string `json:"start"`
End string `json:"end"`
} `json:"params"`
Request struct {
Path string `json:"path"`
ClientIP string `json:"clientIP"`
} `json:"httpRequest"`
RuleGroup struct {
File string `json:"file"`
Name string `json:"name"`
} `json:"ruleGroup"`
}
// readQueryLog unmarshal a json-formatted query log into query log lines.
func readQueryLog(t *testing.T, path string) []queryLogLine {
ql := []queryLogLine{}
file, err := os.Open(path)
testutil.Ok(t, err)
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
var q queryLogLine
testutil.Ok(t, json.Unmarshal(scanner.Bytes(), &q))
ql = append(ql, q)
}
return ql
}
func TestQueryLog(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
cwd, err := os.Getwd()
testutil.Ok(t, err)
port := 15000
for _, host := range []string{"127.0.0.1", "[::1]"} {
for _, prefix := range []string{"", "/foobar"} {
for _, enabledAtStart := range []bool{true, false} {
for _, origin := range []origin{apiOrigin, consoleOrigin, ruleOrigin} {
p := &queryLogTest{
origin: origin,
host: host,
enabledAtStart: enabledAtStart,
prefix: prefix,
port: port,
cwd: cwd,
}
t.Run(p.String(), func(t *testing.T) {
p.run(t)
})
}
}
}
}
}

1
cmd/prometheus/testdata/consoles/test.html vendored Normal file
View file

@ -0,0 +1 @@
{{ query "query_in_console" }}

6
cmd/prometheus/testdata/rules/test.yml vendored Normal file
View file

@ -0,0 +1,6 @@
groups:
- name: querylogtest
interval: 1s
rules:
- record: test
expr: query_in_rule

2
cmd/promtool/archive.go
View file

@ -21,7 +21,7 @@ import (
"github.com/pkg/errors"
)
const filePerm = 0644
const filePerm = 0666
type tarGzFileWriter struct {
tarWriter *tar.Writer

159
cmd/promtool/main.go
View file

@ -23,6 +23,7 @@ import (
"net/url"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"time"
@ -32,14 +33,14 @@ import (
"github.com/prometheus/client_golang/api"
v1 "github.com/prometheus/client_golang/api/prometheus/v1"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/prometheus/client_golang/prometheus/testutil/promlint"
config_util "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/version"
kingpin "gopkg.in/alecthomas/kingpin.v2"
"gopkg.in/alecthomas/kingpin.v2"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/pkg/rulefmt"
"github.com/prometheus/prometheus/util/promlint"
)
func main() {
@ -94,6 +95,8 @@ func main() {
queryLabelsCmd := queryCmd.Command("labels", "Run labels query.")
queryLabelsServer := queryLabelsCmd.Arg("server", "Prometheus server to query.").Required().URL()
queryLabelsName := queryLabelsCmd.Arg("name", "Label name to provide label values for.").Required().String()
queryLabelsBegin := queryLabelsCmd.Flag("start", "Start time (RFC3339 or Unix timestamp).").String()
queryLabelsEnd := queryLabelsCmd.Flag("end", "End time (RFC3339 or Unix timestamp).").String()
testCmd := app.Command("test", "Unit testing.")
testRulesCmd := testCmd.Command("rules", "Unit tests for rules.")
@ -102,6 +105,29 @@ func main() {
"The unit test file.",
).Required().ExistingFiles()
defaultDBPath := "data/"
tsdbCmd := app.Command("tsdb", "Run tsdb commands.")
tsdbBenchCmd := tsdbCmd.Command("bench", "Run benchmarks.")
tsdbBenchWriteCmd := tsdbBenchCmd.Command("write", "Run a write performance benchmark.")
benchWriteOutPath := tsdbBenchWriteCmd.Flag("out", "Set the output path.").Default("benchout").String()
benchWriteNumMetrics := tsdbBenchWriteCmd.Flag("metrics", "Number of metrics to read.").Default("10000").Int()
benchSamplesFile := tsdbBenchWriteCmd.Arg("file", "Input file with samples data, default is ("+filepath.Join("..", "..", "tsdb", "testdata", "20kseries.json")+").").Default(filepath.Join("..", "..", "tsdb", "testdata", "20kseries.json")).String()
tsdbAnalyzeCmd := tsdbCmd.Command("analyze", "Analyze churn, label pair cardinality.")
analyzePath := tsdbAnalyzeCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
analyzeBlockID := tsdbAnalyzeCmd.Arg("block id", "Block to analyze (default is the last block).").String()
analyzeLimit := tsdbAnalyzeCmd.Flag("limit", "How many items to show in each list.").Default("20").Int()
tsdbListCmd := tsdbCmd.Command("list", "List tsdb blocks.")
listHumanReadable := tsdbListCmd.Flag("human-readable", "Print human readable values.").Short('r').Bool()
listPath := tsdbListCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
tsdbDumpCmd := tsdbCmd.Command("dump", "Dump samples from a TSDB.")
dumpPath := tsdbDumpCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
dumpMinTime := tsdbDumpCmd.Flag("min-time", "Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64()
dumpMaxTime := tsdbDumpCmd.Flag("max-time", "Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64()
parsedCmd := kingpin.MustParse(app.Parse(os.Args[1:]))
var p printer
@ -141,10 +167,22 @@ func main() {
os.Exit(debugAll(*debugAllServer))
case queryLabelsCmd.FullCommand():
os.Exit(QueryLabels(*queryLabelsServer, *queryLabelsName, p))
os.Exit(QueryLabels(*queryLabelsServer, *queryLabelsName, *queryLabelsBegin, *queryLabelsEnd, p))
case testRulesCmd.FullCommand():
os.Exit(RulesUnitTest(*testRulesFiles...))
case tsdbBenchWriteCmd.FullCommand():
os.Exit(checkErr(benchmarkWrite(*benchWriteOutPath, *benchSamplesFile, *benchWriteNumMetrics)))
case tsdbAnalyzeCmd.FullCommand():
os.Exit(checkErr(analyzeBlock(*analyzePath, *analyzeBlockID, *analyzeLimit)))
case tsdbListCmd.FullCommand():
os.Exit(checkErr(listBlocks(*listPath, *listHumanReadable)))
case tsdbDumpCmd.FullCommand():
os.Exit(checkErr(dumpSamples(*dumpPath, *dumpMinTime, *dumpMaxTime)))
}
}
@ -303,9 +341,56 @@ func checkRules(filename string) (int, []error) {
numRules += len(rg.Rules)
}
dRules := checkDuplicates(rgs.Groups)
if len(dRules) != 0 {
fmt.Printf("%d duplicate rules(s) found.\n", len(dRules))
for _, n := range dRules {
fmt.Printf("Metric: %s\nLabel(s):\n", n.metric)
for i, l := range n.label {
fmt.Printf("\t%s: %s\n", i, l)
}
}
fmt.Println("Might cause inconsistency while recording expressions.")
}
return numRules, nil
}
type compareRuleType struct {
metric string
label map[string]string
}
func checkDuplicates(groups []rulefmt.RuleGroup) []compareRuleType {
var duplicates []compareRuleType
for _, group := range groups {
for index, rule := range group.Rules {
inst := compareRuleType{
metric: ruleMetric(rule),
label: rule.Labels,
}
for i := 0; i < index; i++ {
t := compareRuleType{
metric: ruleMetric(group.Rules[i]),
label: group.Rules[i].Labels,
}
if reflect.DeepEqual(t, inst) {
duplicates = append(duplicates, t)
}
}
}
}
return duplicates
}
func ruleMetric(rule rulefmt.RuleNode) string {
if rule.Alert.Value != "" {
return rule.Alert.Value
}
return rule.Record.Value
}
var checkMetricsUsage = strings.TrimSpace(`
Pass Prometheus metrics over stdin to lint them for consistency and correctness.
@ -405,11 +490,13 @@ func QueryRange(url string, headers map[string]string, query, start, end string,
stime, err = parseTime(start)
if err != nil {
fmt.Fprintln(os.Stderr, "error parsing start time:", err)
return 1
}
}
if !stime.Before(etime) {
fmt.Fprintln(os.Stderr, "start time is not before end time")
return 1
}
if step == 0 {
@ -447,30 +534,10 @@ func QuerySeries(url *url.URL, matchers []string, start, end string, p printer)
return 1
}
// TODO: clean up timestamps
var (
minTime = time.Now().Add(-9999 * time.Hour)
maxTime = time.Now().Add(9999 * time.Hour)
)
var stime, etime time.Time
if start == "" {
stime = minTime
} else {
stime, err = parseTime(start)
if err != nil {
fmt.Fprintln(os.Stderr, "error parsing start time:", err)
}
}
if end == "" {
etime = maxTime
} else {
etime, err = parseTime(end)
if err != nil {
fmt.Fprintln(os.Stderr, "error parsing end time:", err)
}
stime, etime, err := parseStartTimeAndEndTime(start, end)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
// Run query against client.
@ -489,7 +556,7 @@ func QuerySeries(url *url.URL, matchers []string, start, end string, p printer)
}
// QueryLabels queries for label values against a Prometheus server.
func QueryLabels(url *url.URL, name string, p printer) int {
func QueryLabels(url *url.URL, name string, start, end string, p printer) int {
config := api.Config{
Address: url.String(),
}
@ -501,10 +568,16 @@ func QueryLabels(url *url.URL, name string, p printer) int {
return 1
}
stime, etime, err := parseStartTimeAndEndTime(start, end)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
// Run query against client.
api := v1.NewAPI(c)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
val, warn, err := api.LabelValues(ctx, name)
val, warn, err := api.LabelValues(ctx, name, stime, etime)
cancel()
for _, v := range warn {
@ -520,10 +593,36 @@ func QueryLabels(url *url.URL, name string, p printer) int {
return 0
}
func parseStartTimeAndEndTime(start, end string) (time.Time, time.Time, error) {
var (
minTime = time.Now().Add(-9999 * time.Hour)
maxTime = time.Now().Add(9999 * time.Hour)
err error
)
stime := minTime
etime := maxTime
if start != "" {
stime, err = parseTime(start)
if err != nil {
return stime, etime, errors.Wrap(err, "error parsing start time")
}
}
if end != "" {
etime, err = parseTime(end)
if err != nil {
return stime, etime, errors.Wrap(err, "error parsing end time")
}
}
return stime, etime, nil
}
func parseTime(s string) (time.Time, error) {
if t, err := strconv.ParseFloat(s, 64); err == nil {
s, ns := math.Modf(t)
return time.Unix(int64(s), int64(ns*float64(time.Second))), nil
return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
}
if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
return t, nil

13
cmd/promtool/testdata/alerts.yml vendored Normal file
View file

@ -0,0 +1,13 @@
# This is the rules file.
groups:
- name: example
rules:
- alert: InstanceDown
expr: up == 0
for: 5m
labels:
severity: page
annotations:
summary: "Instance {{ $labels.instance }} down"
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."

21
cmd/promtool/testdata/unittest.yml vendored Normal file
View file

@ -0,0 +1,21 @@
rule_files:
- alerts.yml
evaluation_interval: 1m
tests:
- interval: 1m
input_series:
- series: 'up{job="prometheus", instance="localhost:9090"}'
values: "0+0x1440"
alert_rule_test:
- eval_time: 1d
alertname: InstanceDown
exp_alerts:
- exp_labels:
severity: page
instance: localhost:9090
job: prometheus
exp_annotations:
summary: "Instance localhost:9090 down"
description: "localhost:9090 of job prometheus has been down for more than 5 minutes."

306
tsdb/cmd/tsdb/main.go → cmd/promtool/tsdb.go
View file

@ -15,10 +15,10 @@ package main
import (
"bufio"
"context"
"fmt"
"io"
"io/ioutil"
"math"
"os"
"path/filepath"
"runtime"
@ -32,113 +32,16 @@ import (
"github.com/go-kit/kit/log"
"github.com/pkg/errors"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/tsdb/chunks"
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
"github.com/prometheus/prometheus/tsdb/labels"
kingpin "gopkg.in/alecthomas/kingpin.v2"
)
func main() {
if err := execute(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
var merr tsdb_errors.MultiError
func execute() (err error) {
var (
defaultDBPath = filepath.Join("benchout", "storage")
cli = kingpin.New(filepath.Base(os.Args[0]), "CLI tool for tsdb")
benchCmd = cli.Command("bench", "run benchmarks")
benchWriteCmd = benchCmd.Command("write", "run a write performance benchmark")
benchWriteOutPath = benchWriteCmd.Flag("out", "set the output path").Default("benchout").String()
benchWriteNumMetrics = benchWriteCmd.Flag("metrics", "number of metrics to read").Default("10000").Int()
benchSamplesFile = benchWriteCmd.Arg("file", "input file with samples data, default is ("+filepath.Join("..", "..", "testdata", "20kseries.json")+")").Default(filepath.Join("..", "..", "testdata", "20kseries.json")).String()
listCmd = cli.Command("ls", "list db blocks")
listCmdHumanReadable = listCmd.Flag("human-readable", "print human readable values").Short('h').Bool()
listPath = listCmd.Arg("db path", "database path (default is "+defaultDBPath+")").Default(defaultDBPath).String()
analyzeCmd = cli.Command("analyze", "analyze churn, label pair cardinality.")
analyzePath = analyzeCmd.Arg("db path", "database path (default is "+defaultDBPath+")").Default(defaultDBPath).String()
analyzeBlockID = analyzeCmd.Arg("block id", "block to analyze (default is the last block)").String()
analyzeLimit = analyzeCmd.Flag("limit", "how many items to show in each list").Default("20").Int()
dumpCmd = cli.Command("dump", "dump samples from a TSDB")
dumpPath = dumpCmd.Arg("db path", "database path (default is "+defaultDBPath+")").Default(defaultDBPath).String()
dumpMinTime = dumpCmd.Flag("min-time", "minimum timestamp to dump").Default(strconv.FormatInt(math.MinInt64, 10)).Int64()
dumpMaxTime = dumpCmd.Flag("max-time", "maximum timestamp to dump").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64()
)
logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
var merr tsdb_errors.MultiError
switch kingpin.MustParse(cli.Parse(os.Args[1:])) {
case benchWriteCmd.FullCommand():
wb := &writeBenchmark{
outPath: *benchWriteOutPath,
numMetrics: *benchWriteNumMetrics,
samplesFile: *benchSamplesFile,
logger: logger,
}
return wb.run()
case listCmd.FullCommand():
db, err := tsdb.OpenDBReadOnly(*listPath, nil)
if err != nil {
return err
}
defer func() {
merr.Add(err)
merr.Add(db.Close())
err = merr.Err()
}()
blocks, err := db.Blocks()
if err != nil {
return err
}
printBlocks(blocks, listCmdHumanReadable)
case analyzeCmd.FullCommand():
db, err := tsdb.OpenDBReadOnly(*analyzePath, nil)
if err != nil {
return err
}
defer func() {
merr.Add(err)
merr.Add(db.Close())
err = merr.Err()
}()
blocks, err := db.Blocks()
if err != nil {
return err
}
var block tsdb.BlockReader
if *analyzeBlockID != "" {
for _, b := range blocks {
if b.Meta().ULID.String() == *analyzeBlockID {
block = b
break
}
}
} else if len(blocks) > 0 {
block = blocks[len(blocks)-1]
}
if block == nil {
return fmt.Errorf("block not found")
}
return analyzeBlock(block, *analyzeLimit)
case dumpCmd.FullCommand():
db, err := tsdb.OpenDBReadOnly(*dumpPath, nil)
if err != nil {
return err
}
defer func() {
merr.Add(err)
merr.Add(db.Close())
err = merr.Err()
}()
return dumpSamples(db, *dumpMinTime, *dumpMaxTime)
}
return nil
}
const timeDelta = 30000
type writeBenchmark struct {
outPath string
@ -155,7 +58,13 @@ type writeBenchmark struct {
logger log.Logger
}
func (b *writeBenchmark) run() error {
func benchmarkWrite(outPath, samplesFile string, numMetrics int) error {
b := &writeBenchmark{
outPath: outPath,
samplesFile: samplesFile,
numMetrics: numMetrics,
logger: log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)),
}
if b.outPath == "" {
dir, err := ioutil.TempDir("", "tsdb_bench")
if err != nil {
@ -176,30 +85,30 @@ func (b *writeBenchmark) run() error {
l := log.With(b.logger, "ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller)
st, err := tsdb.Open(dir, l, nil, &tsdb.Options{
RetentionDuration: 15 * 24 * 60 * 60 * 1000, // 15 days in milliseconds
BlockRanges: tsdb.ExponentialBlockRanges(2*60*60*1000, 5, 3),
RetentionDuration: int64(15 * 24 * time.Hour / time.Millisecond),
MinBlockDuration: int64(2 * time.Hour / time.Millisecond),
})
if err != nil {
return err
}
st.DisableCompactions()
b.storage = st
var labels []labels.Labels
var lbs []labels.Labels
_, err = measureTime("readData", func() error {
if _, err = measureTime("readData", func() error {
f, err := os.Open(b.samplesFile)
if err != nil {
return err
}
defer f.Close()
labels, err = readPrometheusLabels(f, b.numMetrics)
lbs, err = readPrometheusLabels(f, b.numMetrics)
if err != nil {
return err
}
return nil
})
if err != nil {
}); err != nil {
return err
}
@ -209,7 +118,7 @@ func (b *writeBenchmark) run() error {
if err := b.startProfiling(); err != nil {
return err
}
total, err = b.ingestScrapes(labels, 3000)
total, err = b.ingestScrapes(lbs, 3000)
if err != nil {
return err
}
@ -222,23 +131,19 @@ func (b *writeBenchmark) run() error {
fmt.Println(" > total samples:", total)
fmt.Println(" > samples/sec:", float64(total)/dur.Seconds())
_, err = measureTime("stopStorage", func() error {
if _, err = measureTime("stopStorage", func() error {
if err := b.storage.Close(); err != nil {
return err
}
if err := b.stopProfiling(); err != nil {
return err
}
return nil
})
if err != nil {
return b.stopProfiling()
}); err != nil {
return err
}
return nil
}
const timeDelta = 30000
func (b *writeBenchmark) ingestScrapes(lbls []labels.Labels, scrapeCount int) (uint64, error) {
var mu sync.Mutex
var total uint64
@ -294,7 +199,7 @@ func (b *writeBenchmark) ingestScrapesShard(lbls []labels.Labels, scrapeCount in
total := uint64(0)
for i := 0; i < scrapeCount; i++ {
app := b.storage.Appender()
app := b.storage.Appender(context.TODO())
ts += timeDelta
for _, s := range scrape {
@ -308,7 +213,7 @@ func (b *writeBenchmark) ingestScrapesShard(lbls []labels.Labels, scrapeCount in
s.ref = &ref
} else if err := app.AddFast(*s.ref, ts, float64(s.value)); err != nil {
if errors.Cause(err) != tsdb.ErrNotFound {
if errors.Cause(err) != storage.ErrNotFound {
panic(err)
}
@ -397,10 +302,10 @@ func (b *writeBenchmark) stopProfiling() error {
func measureTime(stage string, f func() error) (time.Duration, error) {
fmt.Printf(">> start stage=%s\n", stage)
start := time.Now()
err := f()
if err != nil {
if err := f(); err != nil {
return 0, err
}
fmt.Printf(">> completed stage=%s duration=%s\n", stage, time.Since(start))
return time.Since(start), nil
}
@ -436,19 +341,38 @@ func readPrometheusLabels(r io.Reader, n int) ([]labels.Labels, error) {
return mets, nil
}
func printBlocks(blocks []tsdb.BlockReader, humanReadable *bool) {
func listBlocks(path string, humanReadable bool) error {
db, err := tsdb.OpenDBReadOnly(path, nil)
if err != nil {
return err
}
defer func() {
merr.Add(err)
merr.Add(db.Close())
err = merr.Err()
}()
blocks, err := db.Blocks()
if err != nil {
return err
}
printBlocks(blocks, humanReadable)
return nil
}
func printBlocks(blocks []tsdb.BlockReader, humanReadable bool) {
tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
defer tw.Flush()
fmt.Fprintln(tw, "BLOCK ULID\tMIN TIME\tMAX TIME\tNUM SAMPLES\tNUM CHUNKS\tNUM SERIES")
fmt.Fprintln(tw, "BLOCK ULID\tMIN TIME\tMAX TIME\tDURATION\tNUM SAMPLES\tNUM CHUNKS\tNUM SERIES")
for _, b := range blocks {
meta := b.Meta()
fmt.Fprintf(tw,
"%v\t%v\t%v\t%v\t%v\t%v\n",
"%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
meta.ULID,
getFormatedTime(meta.MinTime, humanReadable),
getFormatedTime(meta.MaxTime, humanReadable),
time.Duration(meta.MaxTime-meta.MinTime)*time.Millisecond,
meta.Stats.NumSamples,
meta.Stats.NumChunks,
meta.Stats.NumSeries,
@ -456,20 +380,56 @@ func printBlocks(blocks []tsdb.BlockReader, humanReadable *bool) {
}
}
func getFormatedTime(timestamp int64, humanReadable *bool) string {
if *humanReadable {
return time.Unix(timestamp/1000, 0).String()
func getFormatedTime(timestamp int64, humanReadable bool) string {
if humanReadable {
return time.Unix(timestamp/1000, 0).UTC().String()
}
return strconv.FormatInt(timestamp, 10)
}
func analyzeBlock(b tsdb.BlockReader, limit int) error {
meta := b.Meta()
func openBlock(path, blockID string) (*tsdb.DBReadOnly, tsdb.BlockReader, error) {
db, err := tsdb.OpenDBReadOnly(path, nil)
if err != nil {
return nil, nil, err
}
blocks, err := db.Blocks()
if err != nil {
return nil, nil, err
}
var block tsdb.BlockReader
if blockID != "" {
for _, b := range blocks {
if b.Meta().ULID.String() == blockID {
block = b
break
}
}
} else if len(blocks) > 0 {
block = blocks[len(blocks)-1]
}
if block == nil {
return nil, nil, fmt.Errorf("block %s not found", blockID)
}
return db, block, nil
}
func analyzeBlock(path, blockID string, limit int) error {
db, block, err := openBlock(path, blockID)
if err != nil {
return err
}
defer func() {
merr.Add(err)
merr.Add(db.Close())
err = merr.Err()
}()
meta := block.Meta()
fmt.Printf("Block ID: %s\n", meta.ULID)
// Presume 1ms resolution that Prometheus uses.
fmt.Printf("Duration: %s\n", (time.Duration(meta.MaxTime-meta.MinTime) * 1e6).String())
fmt.Printf("Series: %d\n", meta.Stats.NumSeries)
ir, err := b.Index()
ir, err := block.Index()
if err != nil {
return err
}
@ -491,10 +451,10 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error {
sort.Slice(postingInfos, func(i, j int) bool { return postingInfos[i].metric > postingInfos[j].metric })
for i, pc := range postingInfos {
fmt.Printf("%d %s\n", pc.metric, pc.key)
if i >= limit {
break
}
fmt.Printf("%d %s\n", pc.metric, pc.key)
}
}
@ -554,22 +514,14 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error {
postingInfos = postingInfos[:0]
for _, n := range allLabelNames {
values, err := ir.LabelValues(n)
values, err := ir.SortedLabelValues(n)
if err != nil {
return err
}
var cumulativeLength uint64
for i := 0; i < values.Len(); i++ {
value, err := values.At(i)
if err != nil {
return err
}
for _, str := range value {
cumulativeLength += uint64(len(str))
}
for _, str := range values {
cumulativeLength += uint64(len(str))
}
postingInfos = append(postingInfos, postingInfo{n, cumulativeLength})
}
@ -578,78 +530,88 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error {
postingInfos = postingInfos[:0]
for _, n := range allLabelNames {
lv, err := ir.LabelValues(n)
lv, err := ir.SortedLabelValues(n)
if err != nil {
return err
}
postingInfos = append(postingInfos, postingInfo{n, uint64(lv.Len())})
postingInfos = append(postingInfos, postingInfo{n, uint64(len(lv))})
}
fmt.Printf("\nHighest cardinality labels:\n")
printInfo(postingInfos)
postingInfos = postingInfos[:0]
lv, err := ir.LabelValues("__name__")
lv, err := ir.SortedLabelValues("__name__")
if err != nil {
return err
}
for i := 0; i < lv.Len(); i++ {
names, err := lv.At(i)
for _, n := range lv {
postings, err := ir.Postings("__name__", n)
if err != nil {
return err
}
for _, n := range names {
postings, err := ir.Postings("__name__", n)
if err != nil {
return err
}
count := 0
for postings.Next() {
count++
}
if postings.Err() != nil {
return postings.Err()
}
postingInfos = append(postingInfos, postingInfo{n, uint64(count)})
count := 0
for postings.Next() {
count++
}
if postings.Err() != nil {
return postings.Err()
}
postingInfos = append(postingInfos, postingInfo{n, uint64(count)})
}
fmt.Printf("\nHighest cardinality metric names:\n")
printInfo(postingInfos)
return nil
}
func dumpSamples(db *tsdb.DBReadOnly, mint, maxt int64) (err error) {
q, err := db.Querier(mint, maxt)
func dumpSamples(path string, mint, maxt int64) (err error) {
db, err := tsdb.OpenDBReadOnly(path, nil)
if err != nil {
return err
}
defer func() {
var merr tsdb_errors.MultiError
merr.Add(err)
merr.Add(q.Close())
merr.Add(db.Close())
err = merr.Err()
}()
ss, err := q.Select(labels.NewMustRegexpMatcher("", ".*"))
q, err := db.Querier(context.TODO(), mint, maxt)
if err != nil {
return err
}
defer q.Close()
ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "", ".*"))
for ss.Next() {
series := ss.At()
labels := series.Labels()
lbs := series.Labels()
it := series.Iterator()
for it.Next() {
ts, val := it.At()
fmt.Printf("%s %g %d\n", labels, val, ts)
fmt.Printf("%s %g %d\n", lbs, val, ts)
}
if it.Err() != nil {
return ss.Err()
}
}
if ws := ss.Warnings(); len(ws) > 0 {
var merr tsdb_errors.MultiError
for _, w := range ws {
merr.Add(w)
}
return merr.Err()
}
if ss.Err() != nil {
return ss.Err()
}
return nil
}
func checkErr(err error) int {
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
return 0
}

77
cmd/promtool/unittest.go
View file

@ -29,8 +29,10 @@ import (
"github.com/pkg/errors"
yaml "gopkg.in/yaml.v2"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/promql/parser"
"github.com/prometheus/prometheus/rules"
"github.com/prometheus/prometheus/storage"
)
@ -75,15 +77,16 @@ func ruleUnitTest(filename string) []error {
}
if unitTestInp.EvaluationInterval == 0 {
unitTestInp.EvaluationInterval = 1 * time.Minute
unitTestInp.EvaluationInterval = model.Duration(1 * time.Minute)
}
// Bounds for evaluating the rules.
mint := time.Unix(0, 0)
mint := time.Unix(0, 0).UTC()
maxd := unitTestInp.maxEvalTime()
maxt := mint.Add(maxd)
evalInterval := time.Duration(unitTestInp.EvaluationInterval)
// Rounding off to nearest Eval time (> maxt).
maxt = maxt.Add(unitTestInp.EvaluationInterval / 2).Round(unitTestInp.EvaluationInterval)
maxt = maxt.Add(evalInterval / 2).Round(evalInterval)
// Giving number for groups mentioned in the file for ordering.
// Lower number group should be evaluated before higher number group.
@ -98,7 +101,7 @@ func ruleUnitTest(filename string) []error {
// Testing.
var errs []error
for _, t := range unitTestInp.Tests {
ers := t.test(mint, maxt, unitTestInp.EvaluationInterval, groupOrderMap,
ers := t.test(mint, maxt, evalInterval, groupOrderMap,
unitTestInp.RuleFiles...)
if ers != nil {
errs = append(errs, ers...)
@ -113,10 +116,10 @@ func ruleUnitTest(filename string) []error {
// unitTestFile holds the contents of a single unit test file.
type unitTestFile struct {
RuleFiles []string `yaml:"rule_files"`
EvaluationInterval time.Duration `yaml:"evaluation_interval,omitempty"`
GroupEvalOrder []string `yaml:"group_eval_order"`
Tests []testGroup `yaml:"tests"`
RuleFiles []string `yaml:"rule_files"`
EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"`
GroupEvalOrder []string `yaml:"group_eval_order"`
Tests []testGroup `yaml:"tests"`
}
func (utf *unitTestFile) maxEvalTime() time.Duration {
@ -145,6 +148,9 @@ func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {
if err != nil {
return err
}
if len(m) <= 0 {
fmt.Fprintln(os.Stderr, " WARNING: no file match pattern", rf)
}
globbedFiles = append(globbedFiles, m...)
}
utf.RuleFiles = globbedFiles
@ -153,7 +159,7 @@ func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {
// testGroup is a group of input series and tests associated with it.
type testGroup struct {
Interval time.Duration `yaml:"interval"`
Interval model.Duration `yaml:"interval"`
InputSeries []series `yaml:"input_series"`
AlertRuleTests []alertTestCase `yaml:"alert_rule_test,omitempty"`
PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"`
@ -178,7 +184,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
Logger: log.NewNopLogger(),
}
m := rules.NewManager(opts)
groupsMap, ers := m.LoadGroups(tg.Interval, tg.ExternalLabels, ruleFiles...)
groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, ruleFiles...)
if ers != nil {
return ers
}
@ -189,11 +195,11 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
// This avoids storing them in memory, as the number of evals might be high.
// All the `eval_time` for which we have unit tests for alerts.
alertEvalTimesMap := map[time.Duration]struct{}{}
alertEvalTimesMap := map[model.Duration]struct{}{}
// Map of all the eval_time+alertname combination present in the unit tests.
alertsInTest := make(map[time.Duration]map[string]struct{})
alertsInTest := make(map[model.Duration]map[string]struct{})
// Map of all the unit tests for given eval_time.
alertTests := make(map[time.Duration][]alertTestCase)
alertTests := make(map[model.Duration][]alertTestCase)
for _, alert := range tg.AlertRuleTests {
alertEvalTimesMap[alert.EvalTime] = struct{}{}
@ -204,7 +210,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
alertTests[alert.EvalTime] = append(alertTests[alert.EvalTime], alert)
}
alertEvalTimes := make([]time.Duration, 0, len(alertEvalTimesMap))
alertEvalTimes := make([]model.Duration, 0, len(alertEvalTimesMap))
for k := range alertEvalTimesMap {
alertEvalTimes = append(alertEvalTimes, k)
}
@ -228,7 +234,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
for _, r := range g.Rules() {
if r.LastError() != nil {
errs = append(errs, errors.Errorf(" rule: %s, time: %s, err: %v",
r.Name(), ts.Sub(time.Unix(0, 0)), r.LastError()))
r.Name(), ts.Sub(time.Unix(0, 0).UTC()), r.LastError()))
}
}
}
@ -238,8 +244,8 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
}
for {
if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= alertEvalTimes[curr] &&
alertEvalTimes[curr] < ts.Add(evalInterval).Sub(mint)) {
if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= time.Duration(alertEvalTimes[curr]) &&
time.Duration(alertEvalTimes[curr]) < ts.Add(time.Duration(evalInterval)).Sub(mint)) {
break
}
@ -318,10 +324,10 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
// Checking promql expressions.
Outer:
for _, testCase := range tg.PromqlExprTests {
got, err := query(suite.Context(), testCase.Expr, mint.Add(testCase.EvalTime),
got, err := query(suite.Context(), testCase.Expr, mint.Add(time.Duration(testCase.EvalTime)),
suite.QueryEngine(), suite.Queryable())
if err != nil {
errs = append(errs, errors.Errorf(" expr:'%s', time:%s, err:%s", testCase.Expr,
errs = append(errs, errors.Errorf(" expr: %q, time: %s, err: %s", testCase.Expr,
testCase.EvalTime.String(), err.Error()))
continue
}
@ -336,9 +342,10 @@ Outer:
var expSamples []parsedSample
for _, s := range testCase.ExpSamples {
lb, err := promql.ParseMetric(s.Labels)
lb, err := parser.ParseMetric(s.Labels)
if err != nil {
errs = append(errs, errors.Errorf(" expr:'%s', time:%s, err:%s", testCase.Expr,
err = errors.Wrapf(err, "labels %q", s.Labels)
errs = append(errs, errors.Errorf(" expr: %q, time: %s, err: %s", testCase.Expr,
testCase.EvalTime.String(), err.Error()))
continue Outer
}
@ -355,7 +362,7 @@ Outer:
return labels.Compare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
})
if !reflect.DeepEqual(expSamples, gotSamples) {
errs = append(errs, errors.Errorf(" expr:'%s', time:%s, \n exp:%#v, \n got:%#v", testCase.Expr,
errs = append(errs, errors.Errorf(" expr: %q, time: %s,\n exp:%#v\n got:%#v", testCase.Expr,
testCase.EvalTime.String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
}
}
@ -368,15 +375,15 @@ Outer:
// seriesLoadingString returns the input series in PromQL notation.
func (tg *testGroup) seriesLoadingString() string {
result := ""
result += "load " + shortDuration(tg.Interval) + "\n"
result := fmt.Sprintf("load %v\n", shortDuration(tg.Interval))
for _, is := range tg.InputSeries {
result += " " + is.Series + " " + is.Values + "\n"
result += fmt.Sprintf(" %v %v\n", is.Series, is.Values)
}
return result
}
func shortDuration(d time.Duration) string {
func shortDuration(d model.Duration) string {
s := d.String()
if strings.HasSuffix(s, "m0s") {
s = s[:len(s)-2]
@ -402,7 +409,7 @@ func orderedGroups(groupsMap map[string]*rules.Group, groupOrderMap map[string]i
// maxEvalTime returns the max eval time among all alert and promql unit tests.
func (tg *testGroup) maxEvalTime() time.Duration {
var maxd time.Duration
var maxd model.Duration
for _, alert := range tg.AlertRuleTests {
if alert.EvalTime > maxd {
maxd = alert.EvalTime
@ -413,7 +420,7 @@ func (tg *testGroup) maxEvalTime() time.Duration {
maxd = pet.EvalTime
}
}
return maxd
return time.Duration(maxd)
}
func query(ctx context.Context, qs string, t time.Time, engine *promql.Engine, qu storage.Queryable) (promql.Vector, error) {
@ -478,9 +485,9 @@ type series struct {
}
type alertTestCase struct {
EvalTime time.Duration `yaml:"eval_time"`
Alertname string `yaml:"alertname"`
ExpAlerts []alert `yaml:"exp_alerts"`
EvalTime model.Duration `yaml:"eval_time"`
Alertname string `yaml:"alertname"`
ExpAlerts []alert `yaml:"exp_alerts"`
}
type alert struct {
@ -489,9 +496,9 @@ type alert struct {
}
type promqlTestCase struct {
Expr string `yaml:"expr"`
EvalTime time.Duration `yaml:"eval_time"`
ExpSamples []sample `yaml:"exp_samples"`
Expr string `yaml:"expr"`
EvalTime model.Duration `yaml:"eval_time"`
ExpSamples []sample `yaml:"exp_samples"`
}
type sample struct {
@ -510,7 +517,7 @@ func parsedSamplesString(pss []parsedSample) string {
return "nil"
}
s := pss[0].String()
for _, ps := range pss[0:] {
for _, ps := range pss[1:] {
s += ", " + ps.String()
}
return s

42
cmd/promtool/unittest_test.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import "testing"
func TestRulesUnitTest(t *testing.T) {
type args struct {
files []string
}
tests := []struct {
name string
args args
want int
}{
{
name: "Passing Unit Tests",
args: args{
files: []string{"./testdata/unittest.yml"},
},
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := RulesUnitTest(tt.args.files...); got != tt.want {
t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
}
})
}
}

47
config/config.go
View file

@ -179,6 +179,12 @@ func resolveFilepaths(baseDir string, cfg *Config) {
for _, consulcfg := range cfg.ConsulSDConfigs {
tlsPaths(&consulcfg.TLSConfig)
}
for _, digitaloceancfg := range cfg.DigitalOceanSDConfigs {
clientPaths(&digitaloceancfg.HTTPClientConfig)
}
for _, dockerswarmcfg := range cfg.DockerSwarmSDConfigs {
clientPaths(&dockerswarmcfg.HTTPClientConfig)
}
for _, cfg := range cfg.OpenstackSDConfigs {
tlsPaths(&cfg.TLSConfig)
}
@ -265,15 +271,27 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
}
jobNames[scfg.JobName] = struct{}{}
}
rwNames := map[string]struct{}{}
for _, rwcfg := range c.RemoteWriteConfigs {
if rwcfg == nil {
return errors.New("empty or null remote write config section")
}
// Skip empty names, we fill their name with their config hash in remote write code.
if _, ok := rwNames[rwcfg.Name]; ok && rwcfg.Name != "" {
return errors.Errorf("found multiple remote write configs with job name %q", rwcfg.Name)
}
rwNames[rwcfg.Name] = struct{}{}
}
rrNames := map[string]struct{}{}
for _, rrcfg := range c.RemoteReadConfigs {
if rrcfg == nil {
return errors.New("empty or null remote read config section")
}
// Skip empty names, we fill their name with their config hash in remote read code.
if _, ok := rrNames[rrcfg.Name]; ok && rrcfg.Name != "" {
return errors.Errorf("found multiple remote read configs with job name %q", rrcfg.Name)
}
rrNames[rrcfg.Name] = struct{}{}
}
return nil
}
@ -287,6 +305,8 @@ type GlobalConfig struct {
ScrapeTimeout model.Duration `yaml:"scrape_timeout,omitempty"`
// How frequently to evaluate rules by default.
EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"`
// File to which PromQL queries are logged.
QueryLogFile string `yaml:"query_log_file,omitempty"`
// The labels to add to any timeseries that this Prometheus instance scrapes.
ExternalLabels labels.Labels `yaml:"external_labels,omitempty"`
}
@ -337,7 +357,8 @@ func (c *GlobalConfig) isZero() bool {
return c.ExternalLabels == nil &&
c.ScrapeInterval == 0 &&
c.ScrapeTimeout == 0 &&
c.EvaluationInterval == 0
c.EvaluationInterval == 0 &&
c.QueryLogFile == ""
}
// ScrapeConfig configures a scraping unit for Prometheus.
@ -358,8 +379,11 @@ type ScrapeConfig struct {
MetricsPath string `yaml:"metrics_path,omitempty"`
// The URL scheme with which to fetch metrics from targets.
Scheme string `yaml:"scheme,omitempty"`
// More than this many samples post metric-relabelling will cause the scrape to fail.
// More than this many samples post metric-relabeling will cause the scrape to fail.
SampleLimit uint `yaml:"sample_limit,omitempty"`
// More than this many targets after the target relabeling will cause the
// scrapes to fail.
TargetLimit uint `yaml:"target_limit,omitempty"`
// We cannot do proper Go type embedding below as the parser will then parse
// values arbitrarily into the overflow maps of further-down types.
@ -432,8 +456,8 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
// AlertingConfig configures alerting and alertmanager related configs.
type AlertingConfig struct {
AlertRelabelConfigs []*relabel.Config `yaml:"alert_relabel_configs,omitempty"`
AlertmanagerConfigs []*AlertmanagerConfig `yaml:"alertmanagers,omitempty"`
AlertRelabelConfigs []*relabel.Config `yaml:"alert_relabel_configs,omitempty"`
AlertmanagerConfigs AlertmanagerConfigs `yaml:"alertmanagers,omitempty"`
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
@ -454,6 +478,18 @@ func (c *AlertingConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
return nil
}
// AlertmanagerConfigs is a slice of *AlertmanagerConfig.
type AlertmanagerConfigs []*AlertmanagerConfig
// ToMap converts a slice of *AlertmanagerConfig to a map.
func (a AlertmanagerConfigs) ToMap() map[string]*AlertmanagerConfig {
ret := make(map[string]*AlertmanagerConfig)
for i := range a {
ret[fmt.Sprintf("config-%d", i)] = a[i]
}
return ret
}
// AlertmanagerAPIVersion represents a version of the
// github.com/prometheus/alertmanager/api, e.g. 'v1' or 'v2'.
type AlertmanagerAPIVersion string
@ -584,6 +620,7 @@ type RemoteWriteConfig struct {
URL *config_util.URL `yaml:"url"`
RemoteTimeout model.Duration `yaml:"remote_timeout,omitempty"`
WriteRelabelConfigs []*relabel.Config `yaml:"write_relabel_configs,omitempty"`
Name string `yaml:"name,omitempty"`
// We cannot do proper Go type embedding below as the parser will then parse
// values arbitrarily into the overflow maps of further-down types.
@ -642,6 +679,8 @@ type RemoteReadConfig struct {
URL *config_util.URL `yaml:"url"`
RemoteTimeout model.Duration `yaml:"remote_timeout,omitempty"`
ReadRecent bool `yaml:"read_recent,omitempty"`
Name string `yaml:"name,omitempty"`
// We cannot do proper Go type embedding below as the parser will then parse
// values arbitrarily into the overflow maps of further-down types.
HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`

107
config/config_test.go
View file

@ -30,7 +30,9 @@ import (
"github.com/prometheus/prometheus/discovery/azure"
sd_config "github.com/prometheus/prometheus/discovery/config"
"github.com/prometheus/prometheus/discovery/consul"
"github.com/prometheus/prometheus/discovery/digitalocean"
"github.com/prometheus/prometheus/discovery/dns"
"github.com/prometheus/prometheus/discovery/dockerswarm"
"github.com/prometheus/prometheus/discovery/ec2"
"github.com/prometheus/prometheus/discovery/file"
"github.com/prometheus/prometheus/discovery/kubernetes"
@ -57,6 +59,7 @@ var expectedConf = &Config{
ScrapeInterval: model.Duration(15 * time.Second),
ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout,
EvaluationInterval: model.Duration(30 * time.Second),
QueryLogFile: "",
ExternalLabels: labels.Labels{
{Name: "foo", Value: "bar"},
@ -73,6 +76,7 @@ var expectedConf = &Config{
{
URL: mustParseURL("http://remote1/push"),
RemoteTimeout: model.Duration(30 * time.Second),
Name: "drop_expensive",
WriteRelabelConfigs: []*relabel.Config{
{
SourceLabels: model.LabelNames{"__name__"},
@ -88,6 +92,7 @@ var expectedConf = &Config{
URL: mustParseURL("http://remote2/push"),
RemoteTimeout: model.Duration(30 * time.Second),
QueueConfig: DefaultQueueConfig,
Name: "rw_tls",
HTTPClientConfig: config_util.HTTPClientConfig{
TLSConfig: config_util.TLSConfig{
CertFile: filepath.FromSlash("testdata/valid_cert_file"),
@ -102,11 +107,13 @@ var expectedConf = &Config{
URL: mustParseURL("http://remote1/read"),
RemoteTimeout: model.Duration(1 * time.Minute),
ReadRecent: true,
Name: "default",
},
{
URL: mustParseURL("http://remote3/read"),
RemoteTimeout: model.Duration(1 * time.Minute),
ReadRecent: false,
Name: "read_special",
RequiredMatchers: model.LabelSet{"job": "special"},
HTTPClientConfig: config_util.HTTPClientConfig{
TLSConfig: config_util.TLSConfig{
@ -585,8 +592,8 @@ var expectedConf = &Config{
ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
TritonSDConfigs: []*triton.SDConfig{
{
Account: "testAccount",
Role: "container",
DNSSuffix: "triton.example.com",
Endpoint: "triton.example.com",
Port: 9163,
@ -600,6 +607,49 @@ var expectedConf = &Config{
},
},
},
{
JobName: "digitalocean-droplets",
HonorTimestamps: true,
ScrapeInterval: model.Duration(15 * time.Second),
ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
DigitalOceanSDConfigs: []*digitalocean.SDConfig{
{
HTTPClientConfig: config_util.HTTPClientConfig{
BearerToken: "abcdef",
},
Port: 80,
RefreshInterval: model.Duration(60 * time.Second),
},
},
},
},
{
JobName: "dockerswarm",
HonorTimestamps: true,
ScrapeInterval: model.Duration(15 * time.Second),
ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
DockerSwarmSDConfigs: []*dockerswarm.SDConfig{
{
Host: "http://127.0.0.1:2375",
Role: "nodes",
Port: 80,
RefreshInterval: model.Duration(60 * time.Second),
},
},
},
},
{
JobName: "service-openstack",
@ -616,6 +666,7 @@ var expectedConf = &Config{
Role: "instance",
Region: "RegionOne",
Port: 80,
Availability: "public",
RefreshInterval: model.Duration(60 * time.Second),
TLSConfig: config_util.TLSConfig{
CAFile: "testdata/valid_ca_file",
@ -664,6 +715,15 @@ func TestLoadConfig(t *testing.T) {
testutil.Equals(t, expectedConf, c)
}
func TestScrapeIntervalLarger(t *testing.T) {
c, err := LoadFile("testdata/scrape_interval_larger.good.yml")
testutil.Ok(t, err)
testutil.Equals(t, 1, len(c.ScrapeConfigs))
for _, sc := range c.ScrapeConfigs {
testutil.Equals(t, true, sc.ScrapeInterval >= sc.ScrapeTimeout)
}
}
// YAML marshaling must not reveal authentication credentials.
func TestElideSecrets(t *testing.T) {
c, err := LoadFile("testdata/conf.good.yml")
@ -676,7 +736,7 @@ func TestElideSecrets(t *testing.T) {
yamlConfig := string(config)
matches := secretRe.FindAllStringIndex(yamlConfig, -1)
testutil.Assert(t, len(matches) == 7, "wrong number of secret matches found")
testutil.Assert(t, len(matches) == 8, "wrong number of secret matches found")
testutil.Assert(t, !strings.Contains(yamlConfig, "mysecret"),
"yaml marshal reveals authentication credentials.")
}
@ -695,6 +755,19 @@ func TestKubernetesEmptyAPIServer(t *testing.T) {
testutil.Ok(t, err)
}
func TestKubernetesSelectors(t *testing.T) {
_, err := LoadFile("testdata/kubernetes_selectors_endpoints.good.yml")
testutil.Ok(t, err)
_, err = LoadFile("testdata/kubernetes_selectors_node.good.yml")
testutil.Ok(t, err)
_, err = LoadFile("testdata/kubernetes_selectors_ingress.good.yml")
testutil.Ok(t, err)
_, err = LoadFile("testdata/kubernetes_selectors_pod.good.yml")
testutil.Ok(t, err)
_, err = LoadFile("testdata/kubernetes_selectors_service.good.yml")
testutil.Ok(t, err)
}
var expectedErrors = []struct {
filename string
errMsg string
@ -777,9 +850,30 @@ var expectedErrors = []struct {
}, {
filename: "kubernetes_role.bad.yml",
errMsg: "role",
}, {
filename: "kubernetes_selectors_endpoints.bad.yml",
errMsg: "endpoints role supports only pod, service, endpoints selectors",
}, {
filename: "kubernetes_selectors_ingress.bad.yml",
errMsg: "ingress role supports only ingress selectors",
}, {
filename: "kubernetes_selectors_node.bad.yml",
errMsg: "node role supports only node selectors",
}, {
filename: "kubernetes_selectors_pod.bad.yml",
errMsg: "pod role supports only pod selectors",
}, {
filename: "kubernetes_selectors_service.bad.yml",
errMsg: "service role supports only service selectors",
}, {
filename: "kubernetes_namespace_discovery.bad.yml",
errMsg: "field foo not found in type kubernetes.plain",
}, {
filename: "kubernetes_selectors_duplicated_role.bad.yml",
errMsg: "duplicated selector role: pod",
}, {
filename: "kubernetes_selectors_incorrect_selector.bad.yml",
errMsg: "invalid selector: 'metadata.status-Running'; can't understand 'metadata.status-Running'",
}, {
filename: "kubernetes_bearertoken_basicauth.bad.yml",
errMsg: "at most one of basic_auth, bearer_token & bearer_token_file must be configured",
@ -798,6 +892,9 @@ var expectedErrors = []struct {
}, {
filename: "openstack_role.bad.yml",
errMsg: "unknown OpenStack SD role",
}, {
filename: "openstack_availability.bad.yml",
errMsg: "unknown availability invalid, must be one of admin, internal or public",
}, {
filename: "url_in_targetgroup.bad.yml",
errMsg: "\"http://bad\" is not a valid hostname",
@ -816,6 +913,12 @@ var expectedErrors = []struct {
}, {
filename: "remote_write_url_missing.bad.yml",
errMsg: `url for remote_write is empty`,
}, {
filename: "remote_write_dup.bad.yml",
errMsg: `found multiple remote write configs with job name "queue1"`,
}, {
filename: "remote_read_dup.bad.yml",
errMsg: `found multiple remote read configs with job name "queue1"`,
},
{
filename: "ec2_filters_empty_values.bad.yml",

13
config/testdata/conf.good.yml vendored
View file

@ -14,11 +14,13 @@ rule_files:
remote_write:
- url: http://remote1/push
name: drop_expensive
write_relabel_configs:
- source_labels: [__name__]
regex: expensive.*
action: drop
- url: http://remote2/push
name: rw_tls
tls_config:
cert_file: valid_cert_file
key_file: valid_key_file
@ -26,8 +28,10 @@ remote_write:
remote_read:
- url: http://remote1/read
read_recent: true
name: default
- url: http://remote3/read
read_recent: false
name: read_special
required_matchers:
job: special
tls_config:
@ -255,6 +259,15 @@ scrape_configs:
cert_file: valid_cert_file
key_file: valid_key_file
- job_name: digitalocean-droplets
digitalocean_sd_configs:
- bearer_token: abcdef
- job_name: dockerswarm
dockerswarm_sd_configs:
- host: http://127.0.0.1:2375
role: nodes
- job_name: service-openstack
openstack_sd_configs:
- role: instance

11
config/testdata/kubernetes_selectors_duplicated_role.bad.yml vendored Normal file
View file

@ -0,0 +1,11 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: endpoints
selectors:
- role: "pod"
label: "foo=bar"
field: "metadata.status=Running"
- role: "pod"
label: "foo=bar"
field: "metadata.status=Running"

14
config/testdata/kubernetes_selectors_endpoints.bad.yml vendored Normal file
View file

@ -0,0 +1,14 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: endpoints
selectors:
- role: "node"
label: "foo=bar"
field: "metadata.status=Running"
- role: "service"
label: "foo=bar"
field: "metadata.status=Running"
- role: "endpoints"
label: "foo=bar"
field: "metadata.status=Running"

14
config/testdata/kubernetes_selectors_endpoints.good.yml vendored Normal file
View file

@ -0,0 +1,14 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: endpoints
selectors:
- role: "pod"
label: "foo=bar"
field: "metadata.status=Running"
- role: "service"
label: "foo=bar"
field: "metadata.status=Running"
- role: "endpoints"
label: "foo=bar"
field: "metadata.status=Running"

7
config/testdata/kubernetes_selectors_incorrect_selector.bad.yml vendored Normal file
View file

@ -0,0 +1,7 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: endpoints
selectors:
- role: "pod"
field: "metadata.status-Running"

8
config/testdata/kubernetes_selectors_ingress.bad.yml vendored Normal file
View file

@ -0,0 +1,8 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: ingress
selectors:
- role: "node"
label: "foo=bar"
field: "metadata.status=Running"

8
config/testdata/kubernetes_selectors_ingress.good.yml vendored Normal file
View file

@ -0,0 +1,8 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: ingress
selectors:
- role: "ingress"
label: "foo=bar"
field: "metadata.status=Running"

8
config/testdata/kubernetes_selectors_node.bad.yml vendored Normal file
View file

@ -0,0 +1,8 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: node
selectors:
- role: "pod"
label: "foo=bar"
field: "metadata.status=Running"

8
config/testdata/kubernetes_selectors_node.good.yml vendored Normal file
View file

@ -0,0 +1,8 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: node
selectors:
- role: "node"
label: "foo=bar"
field: "metadata.status=Running"

8
config/testdata/kubernetes_selectors_pod.bad.yml vendored Normal file
View file

@ -0,0 +1,8 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: pod
selectors:
- role: "node"
label: "foo=bar"
field: "metadata.status=Running"

8
config/testdata/kubernetes_selectors_pod.good.yml vendored Normal file
View file

@ -0,0 +1,8 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: pod
selectors:
- role: "pod"
label: "foo=bar"
field: "metadata.status=Running"

8
config/testdata/kubernetes_selectors_service.bad.yml vendored Normal file
View file

@ -0,0 +1,8 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: service
selectors:
- role: "pod"
label: "foo=bar"
field: "metadata.status=Running"

8
config/testdata/kubernetes_selectors_service.good.yml vendored Normal file
View file

@ -0,0 +1,8 @@
scrape_configs:
- job_name: prometheus
kubernetes_sd_configs:
- role: service
selectors:
- role: "service"
label: "foo=bar"
field: "metadata.status=Running"

4
config/testdata/openstack_availability.bad.yml vendored Normal file
View file

@ -0,0 +1,4 @@
scrape_configs:
- openstack_sd_configs:
- availability: invalid

5
config/testdata/remote_read_dup.bad.yml vendored Normal file
View file

@ -0,0 +1,5 @@
remote_read:
- url: http://localhost:9090
name: queue1
- url: localhost:9091
name: queue1

6
config/testdata/remote_write_dup.bad.yml vendored Normal file
View file

@ -0,0 +1,6 @@
remote_write:
- url: localhost:9090
name: queue1
- url: localhost:9091
name: queue1

16
config/testdata/scrape_interval_larger.good.yml vendored Normal file
View file

@ -0,0 +1,16 @@
global:
scrape_interval: 15s
scrape_timeout: 15s
scrape_configs:
- job_name: prometheus
scrape_interval: 5s
dns_sd_configs:
- refresh_interval: 15s
names:
- first.dns.address.domain.com
- second.dns.address.domain.com
- names:
- first.dns.address.domain.com

6
console_libraries/prom.lib
View file

@ -2,15 +2,15 @@
{{/* Load Prometheus console library JS/CSS. Should go in <head> */}}
{{ define "prom_console_head" }}
<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/static/vendor/rickshaw/rickshaw.min.css">
<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/static/vendor/bootstrap-4.3.1/css/bootstrap.min.css">
<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/static/vendor/bootstrap-4.5.2/css/bootstrap.min.css">
<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/static/css/prom_console.css">
<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css">
<script src="{{ pathPrefix }}/static/vendor/rickshaw/vendor/d3.v3.js"></script>
<script src="{{ pathPrefix }}/static/vendor/rickshaw/vendor/d3.layout.min.js"></script>
<script src="{{ pathPrefix }}/static/vendor/rickshaw/rickshaw.min.js"></script>
<script src="{{ pathPrefix }}/static/vendor/js/jquery-3.3.1.min.js"></script>
<script src="{{ pathPrefix }}/static/vendor/js/jquery-3.5.1.min.js"></script>
<script src="{{ pathPrefix }}/static/vendor/js/popper.min.js"></script>
<script src="{{ pathPrefix }}/static/vendor/bootstrap-4.3.1/js/bootstrap.min.js"></script>
<script src="{{ pathPrefix }}/static/vendor/bootstrap-4.5.2/js/bootstrap.min.js"></script>
<script>
var PATH_PREFIX = "{{ pathPrefix }}";

20
discovery/README.md
View file

@ -2,10 +2,6 @@
This directory contains the service discovery (SD) component of Prometheus.
There is currently a moratorium on new service discovery mechanisms being added
to Prometheus due to a lack of developer capacity. In the meantime `file_sd`
remains available.
## Design of a Prometheus SD
There are many requests to add new SDs to Prometheus, this section looks at
@ -19,6 +15,10 @@ use across multiple organizations. It should allow discovering of machines
and/or services running somewhere. When exactly an SD is popular enough to
justify being added to Prometheus natively is an open question.
Note: As part of lifting the past moratorium on new SD implementations it was
agreed that, in addition to the existing requirements, new service discovery
implementations will be required to have a committed maintainer with push access (i.e., on -team).
It should not be a brand new SD mechanism, or a variant of an established
mechanism. We want to integrate Prometheus with the SD that's already there in
your infrastructure, not invent yet more ways to do service discovery. We also
@ -216,4 +216,16 @@ If all the targets in a group go away, we need to send the target groups with em
```
down the channel.
### New Service Discovery Check List
Here are some non-obvious parts of adding service discoveries that need to be verified:
- Check for `nil` SDConfigs in `discovery/config/config.go`.
- Validate that discovery configs can be DeepEqualled by adding them to
`config/testdata/conf.good.yml` and to the associated tests.
- If there is a TLSConfig or HTTPClientConfig, add them to
`resolveFilepaths` in `config/config.go`.
- List the service discovery in both `<scrape_config>` and
`<alertmanager_config>` in `docs/configuration/configuration.md`.
<!-- TODO: Add best-practices -->

8
discovery/azure/azure.go
View file

@ -302,11 +302,9 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
labels[azureLabelMachineScaleSet] = model.LabelValue(vm.ScaleSet)
}
if vm.Tags != nil {
for k, v := range vm.Tags {
name := strutil.SanitizeLabelName(k)
labels[azureLabelMachineTag+model.LabelName(name)] = model.LabelValue(*v)
}
for k, v := range vm.Tags {
name := strutil.SanitizeLabelName(k)
labels[azureLabelMachineTag+model.LabelName(name)] = model.LabelValue(*v)
}
// Get the IP address information via separate call to the network provider.

26
discovery/azure/azure_test.go
View file

@ -18,8 +18,13 @@ import (
"testing"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
"go.uber.org/goleak"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
func TestMapFromVMWithEmptyTags(t *testing.T) {
id := "test"
name := "name"
@ -201,3 +206,24 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
t.Errorf("Expected %v got %v", expectedVM, actualVM)
}
}
func TestNewAzureResourceFromID(t *testing.T) {
for _, tc := range []struct {
id string
expected azureResource
}{
{
id: "/a/b/c/group/d/e/f/name",
expected: azureResource{"name", "group"},
},
{
id: "/a/b/c/group/d/e/f/name/g/h",
expected: azureResource{"name", "group"},
},
} {
actual, _ := newAzureResourceFromID(tc.id, nil)
if !reflect.DeepEqual(tc.expected, actual) {
t.Errorf("Expected %v got %v", tc.expected, actual)
}
}
}

21
discovery/config/config.go
View file

@ -18,7 +18,9 @@ import (
"github.com/prometheus/prometheus/discovery/azure"
"github.com/prometheus/prometheus/discovery/consul"
"github.com/prometheus/prometheus/discovery/digitalocean"
"github.com/prometheus/prometheus/discovery/dns"
"github.com/prometheus/prometheus/discovery/dockerswarm"
"github.com/prometheus/prometheus/discovery/ec2"
"github.com/prometheus/prometheus/discovery/file"
"github.com/prometheus/prometheus/discovery/gce"
@ -40,6 +42,10 @@ type ServiceDiscoveryConfig struct {
FileSDConfigs []*file.SDConfig `yaml:"file_sd_configs,omitempty"`
// List of Consul service discovery configurations.
ConsulSDConfigs []*consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
// List of DigitalOcean service discovery configurations.
DigitalOceanSDConfigs []*digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
// List of Docker Swarm service discovery configurations.
DockerSwarmSDConfigs []*dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
// List of Serverset service discovery configurations.
ServersetSDConfigs []*zookeeper.ServersetSDConfig `yaml:"serverset_sd_configs,omitempty"`
// NerveSDConfigs is a list of Nerve service discovery configurations.
@ -72,6 +78,16 @@ func (c *ServiceDiscoveryConfig) Validate() error {
return errors.New("empty or null section in consul_sd_configs")
}
}
for _, cfg := range c.DigitalOceanSDConfigs {
if cfg == nil {
return errors.New("empty or null section in digitalocean_sd_configs")
}
}
for _, cfg := range c.DockerSwarmSDConfigs {
if cfg == nil {
return errors.New("empty or null section in dockerswarm_sd_configs")
}
}
for _, cfg := range c.DNSSDConfigs {
if cfg == nil {
return errors.New("empty or null section in dns_sd_configs")
@ -122,5 +138,10 @@ func (c *ServiceDiscoveryConfig) Validate() error {
return errors.New("empty or null section in static_configs")
}
}
for _, cfg := range c.TritonSDConfigs {
if cfg == nil {
return errors.New("empty or null section in triton_sd_configs")
}
}
return nil
}

58
discovery/config/config_test.go Normal file
View file

@ -0,0 +1,58 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"fmt"
"reflect"
"strings"
"testing"
"github.com/prometheus/prometheus/util/testutil"
"gopkg.in/yaml.v2"
)
func TestForNilSDConfig(t *testing.T) {
// Get all the yaml fields names of the ServiceDiscoveryConfig struct.
s := reflect.ValueOf(ServiceDiscoveryConfig{})
configType := s.Type()
n := s.NumField()
fieldsSlice := make([]string, n)
for i := 0; i < n; i++ {
field := configType.Field(i)
tag := field.Tag.Get("yaml")
tag = strings.Split(tag, ",")[0]
fieldsSlice = append(fieldsSlice, tag)
}
// Unmarshall all possible yaml keys and validate errors check upon nil
// SD config.
for _, f := range fieldsSlice {
if f == "" {
continue
}
t.Run(f, func(t *testing.T) {
c := &ServiceDiscoveryConfig{}
err := yaml.Unmarshal([]byte(fmt.Sprintf(`
---
%s:
-
`, f)), c)
testutil.Ok(t, err)
err = c.Validate()
testutil.NotOk(t, err)
testutil.Equals(t, fmt.Sprintf("empty or null section in %s", f), err.Error())
})
}
}

74
discovery/consul/consul.go
View file

@ -36,7 +36,7 @@ import (
)
const (
watchTimeout = 30 * time.Second
watchTimeout = 2 * time.Minute
retryInterval = 15 * time.Second
// addressLabel is the name for the label containing a target's address.
@ -51,6 +51,8 @@ const (
tagsLabel = model.MetaLabelPrefix + "consul_tags"
// serviceLabel is the name of the label containing the service name.
serviceLabel = model.MetaLabelPrefix + "consul_service"
// healthLabel is the name of the label containing the health of the service instance
healthLabel = model.MetaLabelPrefix + "consul_health"
// serviceAddressLabel is the name of the label containing the (optional) service address.
serviceAddressLabel = model.MetaLabelPrefix + "consul_service_address"
//servicePortLabel is the name of the label containing the service port.
@ -83,13 +85,17 @@ var (
[]string{"endpoint", "call"},
)
// Initialize metric vectors.
servicesRPCDuration = rpcDuration.WithLabelValues("catalog", "services")
serviceRPCDuration = rpcDuration.WithLabelValues("catalog", "service")
// DefaultSDConfig is the default Consul SD configuration.
DefaultSDConfig = SDConfig{
TagSeparator: ",",
Scheme: "http",
Server: "localhost:8500",
AllowStale: true,
RefreshInterval: model.Duration(watchTimeout),
RefreshInterval: model.Duration(30 * time.Second),
}
)
@ -141,10 +147,6 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
func init() {
prometheus.MustRegister(rpcFailuresCount)
prometheus.MustRegister(rpcDuration)
// Initialize metric vectors.
rpcDuration.WithLabelValues("catalog", "service")
rpcDuration.WithLabelValues("catalog", "services")
}
// Discovery retrieves target information from a Consul server
@ -173,7 +175,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
return nil, err
}
transport := &http.Transport{
IdleConnTimeout: 5 * time.Duration(conf.RefreshInterval),
IdleConnTimeout: 2 * time.Duration(watchTimeout),
TLSClientConfig: tls,
DialContext: conntrack.NewDialContextFunc(
conntrack.DialWithTracing(),
@ -182,7 +184,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
}
wrapper := &http.Client{
Transport: transport,
Timeout: 35 * time.Second,
Timeout: time.Duration(watchTimeout) + 15*time.Second,
}
clientConf := &consul.Config{
@ -344,7 +346,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
// entire list of services.
func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.Group, lastIndex *uint64, services map[string]func()) {
catalog := d.client.Catalog()
level.Debug(d.logger).Log("msg", "Watching services", "tags", d.watchedTags)
level.Debug(d.logger).Log("msg", "Watching services", "tags", strings.Join(d.watchedTags, ","))
t0 := time.Now()
opts := &consul.QueryOptions{
@ -355,7 +357,14 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
}
srvs, meta, err := catalog.Services(opts.WithContext(ctx))
elapsed := time.Since(t0)
rpcDuration.WithLabelValues("catalog", "services").Observe(elapsed.Seconds())
servicesRPCDuration.Observe(elapsed.Seconds())
// Check the context before in order to exit early.
select {
case <-ctx.Done():
return
default:
}
if err != nil {
level.Error(d.logger).Log("msg", "Error refreshing service list", "err", err)
@ -433,14 +442,14 @@ func (d *Discovery) watchService(ctx context.Context, ch chan<- []*targetgroup.G
go func() {
ticker := time.NewTicker(d.refreshInterval)
var lastIndex uint64
catalog := srv.client.Catalog()
health := srv.client.Health()
for {
select {
case <-ctx.Done():
ticker.Stop()
return
default:
srv.watch(ctx, ch, catalog, &lastIndex)
srv.watch(ctx, ch, health, &lastIndex)
select {
case <-ticker.C:
case <-ctx.Done():
@ -451,8 +460,8 @@ func (d *Discovery) watchService(ctx context.Context, ch chan<- []*targetgroup.G
}
// Get updates for a service.
func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Group, catalog *consul.Catalog, lastIndex *uint64) {
level.Debug(srv.logger).Log("msg", "Watching service", "service", srv.name, "tags", srv.tags)
func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Group, health *consul.Health, lastIndex *uint64) {
level.Debug(srv.logger).Log("msg", "Watching service", "service", srv.name, "tags", strings.Join(srv.tags, ","))
t0 := time.Now()
opts := &consul.QueryOptions{
@ -461,9 +470,10 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
AllowStale: srv.discovery.allowStale,
NodeMeta: srv.discovery.watchedNodeMeta,
}
nodes, meta, err := catalog.ServiceMultipleTags(srv.name, srv.tags, opts.WithContext(ctx))
serviceNodes, meta, err := health.ServiceMultipleTags(srv.name, srv.tags, false, opts.WithContext(ctx))
elapsed := time.Since(t0)
rpcDuration.WithLabelValues("catalog", "service").Observe(elapsed.Seconds())
serviceRPCDuration.Observe(elapsed.Seconds())
// Check the context before in order to exit early.
select {
@ -474,7 +484,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
}
if err != nil {
level.Error(srv.logger).Log("msg", "Error refreshing service", "service", srv.name, "tags", srv.tags, "err", err)
level.Error(srv.logger).Log("msg", "Error refreshing service", "service", srv.name, "tags", strings.Join(srv.tags, ","), "err", err)
rpcFailuresCount.Inc()
time.Sleep(retryInterval)
return
@ -488,48 +498,48 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
tgroup := targetgroup.Group{
Source: srv.name,
Labels: srv.labels,
Targets: make([]model.LabelSet, 0, len(nodes)),
Targets: make([]model.LabelSet, 0, len(serviceNodes)),
}
for _, node := range nodes {
for _, serviceNode := range serviceNodes {
// We surround the separated list with the separator as well. This way regular expressions
// in relabeling rules don't have to consider tag positions.
var tags = srv.tagSeparator + strings.Join(node.ServiceTags, srv.tagSeparator) + srv.tagSeparator
var tags = srv.tagSeparator + strings.Join(serviceNode.Service.Tags, srv.tagSeparator) + srv.tagSeparator
// If the service address is not empty it should be used instead of the node address
// since the service may be registered remotely through a different node.
var addr string
if node.ServiceAddress != "" {
addr = net.JoinHostPort(node.ServiceAddress, fmt.Sprintf("%d", node.ServicePort))
if serviceNode.Service.Address != "" {
addr = net.JoinHostPort(serviceNode.Service.Address, fmt.Sprintf("%d", serviceNode.Service.Port))
} else {
addr = net.JoinHostPort(node.Address, fmt.Sprintf("%d", node.ServicePort))
addr = net.JoinHostPort(serviceNode.Node.Address, fmt.Sprintf("%d", serviceNode.Service.Port))
}
labels := model.LabelSet{
model.AddressLabel: model.LabelValue(addr),
addressLabel: model.LabelValue(node.Address),
nodeLabel: model.LabelValue(node.Node),
addressLabel: model.LabelValue(serviceNode.Node.Address),
nodeLabel: model.LabelValue(serviceNode.Node.Node),
tagsLabel: model.LabelValue(tags),
serviceAddressLabel: model.LabelValue(node.ServiceAddress),
servicePortLabel: model.LabelValue(strconv.Itoa(node.ServicePort)),
serviceIDLabel: model.LabelValue(node.ServiceID),
serviceAddressLabel: model.LabelValue(serviceNode.Service.Address),
servicePortLabel: model.LabelValue(strconv.Itoa(serviceNode.Service.Port)),
serviceIDLabel: model.LabelValue(serviceNode.Service.ID),
healthLabel: model.LabelValue(serviceNode.Checks.AggregatedStatus()),
}
// Add all key/value pairs from the node's metadata as their own labels.
for k, v := range node.NodeMeta {
for k, v := range serviceNode.Node.Meta {
name := strutil.SanitizeLabelName(k)
labels[metaDataLabel+model.LabelName(name)] = model.LabelValue(v)
}
// Add all key/value pairs from the service's metadata as their own labels.
for k, v := range node.ServiceMeta {
for k, v := range serviceNode.Service.Meta {
name := strutil.SanitizeLabelName(k)
labels[serviceMetaDataLabel+model.LabelName(name)] = model.LabelValue(v)
}
// Add all key/value pairs from the service's tagged addresses as their own labels.
for k, v := range node.TaggedAddresses {
for k, v := range serviceNode.Node.TaggedAddresses {
name := strutil.SanitizeLabelName(k)
labels[taggedAddressesLabel+model.LabelName(name)] = model.LabelValue(v)
}

129
discovery/consul/consul_test.go
View file

@ -26,8 +26,13 @@ import (
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/testutil"
"go.uber.org/goleak"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
func TestConfiguredService(t *testing.T) {
conf := &SDConfig{
Services: []string{"configuredServiceName"}}
@ -172,21 +177,47 @@ func TestNonConfiguredService(t *testing.T) {
const (
AgentAnswer = `{"Config": {"Datacenter": "test-dc"}}`
ServiceTestAnswer = `[{
"ID": "b78c2e48-5ef3-1814-31b8-0d880f50471e",
"Node": "node1",
"Address": "1.1.1.1",
"Datacenter": "test-dc",
"TaggedAddresses": {"lan":"192.168.10.10","wan":"10.0.10.10"},
"NodeMeta": {"rack_name": "2304"},
"ServiceID": "test",
"ServiceName": "test",
"ServiceMeta": {"version":"1.0.0","environment":"stagging"},
"ServiceTags": ["tag1"],
"ServicePort": 3341,
"CreateIndex": 1,
"ModifyIndex": 1
ServiceTestAnswer = `
[{
"Node": {
"ID": "b78c2e48-5ef3-1814-31b8-0d880f50471e",
"Node": "node1",
"Address": "1.1.1.1",
"Datacenter": "test-dc",
"TaggedAddresses": {
"lan": "192.168.10.10",
"wan": "10.0.10.10"
},
"Meta": {"rack_name": "2304"},
"CreateIndex": 1,
"ModifyIndex": 1
},
"Service": {
"ID": "test",
"Service": "test",
"Tags": ["tag1"],
"Address": "",
"Meta": {"version":"1.0.0","environment":"staging"},
"Port": 3341,
"Weights": {
"Passing": 1,
"Warning": 1
},
"EnableTagOverride": false,
"ProxyDestination": "",
"Proxy": {},
"Connect": {},
"CreateIndex": 1,
"ModifyIndex": 1
},
"Checks": [{
"Node": "node1",
"CheckID": "serfHealth",
"Name": "Serf Health Status",
"Status": "passing"
}]
}]`
ServicesTestAnswer = `{"test": ["tag1"], "other": ["tag2"]}`
)
@ -197,20 +228,20 @@ func newServer(t *testing.T) (*httptest.Server, *SDConfig) {
switch r.URL.String() {
case "/v1/agent/self":
response = AgentAnswer
case "/v1/catalog/service/test?node-meta=rack_name%3A2304&stale=&tag=tag1&wait=30000ms":
case "/v1/health/service/test?node-meta=rack_name%3A2304&stale=&tag=tag1&wait=120000ms":
response = ServiceTestAnswer
case "/v1/catalog/service/test?wait=30000ms":
case "/v1/health/service/test?wait=120000ms":
response = ServiceTestAnswer
case "/v1/catalog/service/other?wait=30000ms":
case "/v1/health/service/other?wait=120000ms":
response = `[]`
case "/v1/catalog/services?node-meta=rack_name%3A2304&stale=&wait=30000ms":
case "/v1/catalog/services?node-meta=rack_name%3A2304&stale=&wait=120000ms":
response = ServicesTestAnswer
case "/v1/catalog/services?wait=30000ms":
case "/v1/catalog/services?wait=120000ms":
response = ServicesTestAnswer
case "/v1/catalog/services?index=1&node-meta=rack_name%3A2304&stale=&wait=30000ms":
case "/v1/catalog/services?index=1&node-meta=rack_name%3A2304&stale=&wait=120000ms":
time.Sleep(5 * time.Second)
response = ServicesTestAnswer
case "/v1/catalog/services?index=1&wait=30000ms":
case "/v1/catalog/services?index=1&wait=120000ms":
time.Sleep(5 * time.Second)
response = ServicesTestAnswer
default:
@ -257,10 +288,14 @@ func TestAllServices(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
ch := make(chan []*targetgroup.Group)
go d.Run(ctx, ch)
go func() {
d.Run(ctx, ch)
close(ch)
}()
checkOneTarget(t, <-ch)
checkOneTarget(t, <-ch)
cancel()
<-ch
}
// Watch only the test service.
@ -293,7 +328,55 @@ func TestAllOptions(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
ch := make(chan []*targetgroup.Group)
go d.Run(ctx, ch)
go func() {
d.Run(ctx, ch)
close(ch)
}()
checkOneTarget(t, <-ch)
cancel()
<-ch
}
func TestGetDatacenterShouldReturnError(t *testing.T) {
for _, tc := range []struct {
handler func(http.ResponseWriter, *http.Request)
errMessage string
}{
{
// Define a handler that will return status 500.
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(500)
},
errMessage: "Unexpected response code: 500 ()",
},
{
// Define a handler that will return incorrect response.
handler: func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"Config": {"Not-Datacenter": "test-dc"}}`))
},
errMessage: "invalid value '<nil>' for Config.Datacenter",
},
} {
stub := httptest.NewServer(http.HandlerFunc(tc.handler))
stuburl, err := url.Parse(stub.URL)
testutil.Ok(t, err)
config := &SDConfig{
Server: stuburl.Host,
Token: "fake-token",
RefreshInterval: model.Duration(1 * time.Second),
}
defer stub.Close()
d := newDiscovery(t, config)
// Should be empty if not initialized.
testutil.Equals(t, "", d.clientDatacenter)
err = d.getDatacenter()
// An error should be returned.
testutil.Equals(t, tc.errMessage, err.Error())
// Should still be empty.
testutil.Equals(t, "", d.clientDatacenter)
}
}

199
discovery/digitalocean/digitalocean.go Normal file
View file

@ -0,0 +1,199 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package digitalocean
import (
"context"
"fmt"
"net"
"net/http"
"strconv"
"strings"
"time"
"github.com/digitalocean/godo"
"github.com/go-kit/kit/log"
config_util "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/version"
"github.com/prometheus/prometheus/discovery/refresh"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
const (
doLabel = model.MetaLabelPrefix + "digitalocean_"
doLabelID = doLabel + "droplet_id"
doLabelName = doLabel + "droplet_name"
doLabelImage = doLabel + "image"
doLabelPrivateIPv4 = doLabel + "private_ipv4"
doLabelPublicIPv4 = doLabel + "public_ipv4"
doLabelPublicIPv6 = doLabel + "public_ipv6"
doLabelRegion = doLabel + "region"
doLabelSize = doLabel + "size"
doLabelStatus = doLabel + "status"
doLabelFeatures = doLabel + "features"
doLabelTags = doLabel + "tags"
separator = ","
)
// DefaultSDConfig is the default DigitalOcean SD configuration.
var DefaultSDConfig = SDConfig{
Port: 80,
RefreshInterval: model.Duration(60 * time.Second),
}
// SDConfig is the configuration for DigitalOcean based service discovery.
type SDConfig struct {
HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
RefreshInterval model.Duration `yaml:"refresh_interval"`
Port int `yaml:"port"`
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
*c = DefaultSDConfig
type plain SDConfig
err := unmarshal((*plain)(c))
if err != nil {
return err
}
return nil
}
// Discovery periodically performs DigitalOcean requests. It implements
// the Discoverer interface.
type Discovery struct {
*refresh.Discovery
client *godo.Client
port int
}
// NewDiscovery returns a new Discovery which periodically refreshes its targets.
func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
d := &Discovery{
port: conf.Port,
}
rt, err := config_util.NewRoundTripperFromConfig(conf.HTTPClientConfig, "digitalocean_sd", false)
if err != nil {
return nil, err
}
d.client, err = godo.New(
&http.Client{
Transport: rt,
Timeout: time.Duration(conf.RefreshInterval),
},
godo.SetUserAgent(fmt.Sprintf("Prometheus/%s", version.Version)),
)
if err != nil {
return nil, fmt.Errorf("error setting up digital ocean agent: %w", err)
}
d.Discovery = refresh.NewDiscovery(
logger,
"digitalocean",
time.Duration(conf.RefreshInterval),
d.refresh,
)
return d, nil
}
func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
tg := &targetgroup.Group{
Source: "DigitalOcean",
}
droplets, err := d.listDroplets()
if err != nil {
return nil, err
}
for _, droplet := range droplets {
if droplet.Networks == nil || len(droplet.Networks.V4) == 0 {
continue
}
privateIPv4, err := droplet.PrivateIPv4()
if err != nil {
return nil, fmt.Errorf("error while reading private IPv4 of droplet %d: %w", droplet.ID, err)
}
publicIPv4, err := droplet.PublicIPv4()
if err != nil {
return nil, fmt.Errorf("error while reading public IPv4 of droplet %d: %w", droplet.ID, err)
}
publicIPv6, err := droplet.PublicIPv6()
if err != nil {
return nil, fmt.Errorf("error while reading public IPv6 of droplet %d: %w", droplet.ID, err)
}
labels := model.LabelSet{
doLabelID: model.LabelValue(fmt.Sprintf("%d", droplet.ID)),
doLabelName: model.LabelValue(droplet.Name),
doLabelImage: model.LabelValue(droplet.Image.Slug),
doLabelPrivateIPv4: model.LabelValue(privateIPv4),
doLabelPublicIPv4: model.LabelValue(publicIPv4),
doLabelPublicIPv6: model.LabelValue(publicIPv6),
doLabelRegion: model.LabelValue(droplet.Region.Slug),
doLabelSize: model.LabelValue(droplet.SizeSlug),
doLabelStatus: model.LabelValue(droplet.Status),
}
addr := net.JoinHostPort(publicIPv4, strconv.FormatUint(uint64(d.port), 10))
labels[model.AddressLabel] = model.LabelValue(addr)
if len(droplet.Features) > 0 {
// We surround the separated list with the separator as well. This way regular expressions
// in relabeling rules don't have to consider feature positions.
features := separator + strings.Join(droplet.Features, separator) + separator
labels[doLabelFeatures] = model.LabelValue(features)
}
if len(droplet.Tags) > 0 {
// We surround the separated list with the separator as well. This way regular expressions
// in relabeling rules don't have to consider tag positions.
tags := separator + strings.Join(droplet.Tags, separator) + separator
labels[doLabelTags] = model.LabelValue(tags)
}
tg.Targets = append(tg.Targets, labels)
}
return []*targetgroup.Group{tg}, nil
}
func (d *Discovery) listDroplets() ([]godo.Droplet, error) {
var (
droplets []godo.Droplet
opts = &godo.ListOptions{}
)
for {
paginatedDroplets, resp, err := d.client.Droplets.List(context.Background(), opts)
if err != nil {
return nil, fmt.Errorf("error while listing droplets page %d: %w", opts.Page, err)
}
droplets = append(droplets, paginatedDroplets...)
if resp.Links == nil || resp.Links.IsLastPage() {
break
}
page, err := resp.Links.CurrentPage()
if err != nil {
return nil, err
}
opts.Page = page + 1
}
return droplets, nil
}

125
discovery/digitalocean/digitalocean_test.go Normal file
View file

@ -0,0 +1,125 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package digitalocean
import (
"context"
"fmt"
"net/url"
"testing"
"github.com/go-kit/kit/log"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/util/testutil"
)
type DigitalOceanSDTestSuite struct {
Mock *SDMock
}
func (s *DigitalOceanSDTestSuite) TearDownSuite() {
s.Mock.ShutdownServer()
}
func (s *DigitalOceanSDTestSuite) SetupTest(t *testing.T) {
s.Mock = NewSDMock(t)
s.Mock.Setup()
s.Mock.HandleDropletsList()
}
func TestDigitalOceanSDRefresh(t *testing.T) {
sdmock := &DigitalOceanSDTestSuite{}
sdmock.SetupTest(t)
t.Cleanup(sdmock.TearDownSuite)
cfg := DefaultSDConfig
cfg.HTTPClientConfig.BearerToken = tokenID
d, err := NewDiscovery(&cfg, log.NewNopLogger())
testutil.Ok(t, err)
endpoint, err := url.Parse(sdmock.Mock.Endpoint())
testutil.Ok(t, err)
d.client.BaseURL = endpoint
ctx := context.Background()
tgs, err := d.refresh(ctx)
testutil.Ok(t, err)
testutil.Equals(t, 1, len(tgs))
tg := tgs[0]
testutil.Assert(t, tg != nil, "tg should not be nil")
testutil.Assert(t, tg.Targets != nil, "tg.targets should not be nil")
testutil.Equals(t, 4, len(tg.Targets))
for i, lbls := range []model.LabelSet{
{
"__address__": model.LabelValue("104.236.32.182:80"),
"__meta_digitalocean_droplet_id": model.LabelValue("3164444"),
"__meta_digitalocean_droplet_name": model.LabelValue("example.com"),
"__meta_digitalocean_image": model.LabelValue("ubuntu-16-04-x64"),
"__meta_digitalocean_private_ipv4": model.LabelValue(""),
"__meta_digitalocean_public_ipv4": model.LabelValue("104.236.32.182"),
"__meta_digitalocean_public_ipv6": model.LabelValue("2604:A880:0800:0010:0000:0000:02DD:4001"),
"__meta_digitalocean_region": model.LabelValue("nyc3"),
"__meta_digitalocean_size": model.LabelValue("s-1vcpu-1gb"),
"__meta_digitalocean_status": model.LabelValue("active"),
"__meta_digitalocean_features": model.LabelValue(",backups,ipv6,virtio,"),
},
{
"__address__": model.LabelValue("104.131.186.241:80"),
"__meta_digitalocean_droplet_id": model.LabelValue("3164494"),
"__meta_digitalocean_droplet_name": model.LabelValue("prometheus"),
"__meta_digitalocean_image": model.LabelValue("ubuntu-16-04-x64"),
"__meta_digitalocean_private_ipv4": model.LabelValue(""),
"__meta_digitalocean_public_ipv4": model.LabelValue("104.131.186.241"),
"__meta_digitalocean_public_ipv6": model.LabelValue(""),
"__meta_digitalocean_region": model.LabelValue("nyc3"),
"__meta_digitalocean_size": model.LabelValue("s-1vcpu-1gb"),
"__meta_digitalocean_status": model.LabelValue("active"),
"__meta_digitalocean_tags": model.LabelValue(",monitor,"),
"__meta_digitalocean_features": model.LabelValue(",virtio,"),
},
{
"__address__": model.LabelValue("167.172.111.118:80"),
"__meta_digitalocean_droplet_id": model.LabelValue("175072239"),
"__meta_digitalocean_droplet_name": model.LabelValue("prometheus-demo-old"),
"__meta_digitalocean_image": model.LabelValue("ubuntu-18-04-x64"),
"__meta_digitalocean_private_ipv4": model.LabelValue("10.135.64.211"),
"__meta_digitalocean_public_ipv4": model.LabelValue("167.172.111.118"),
"__meta_digitalocean_public_ipv6": model.LabelValue(""),
"__meta_digitalocean_region": model.LabelValue("fra1"),
"__meta_digitalocean_size": model.LabelValue("s-1vcpu-1gb"),
"__meta_digitalocean_status": model.LabelValue("off"),
"__meta_digitalocean_features": model.LabelValue(",ipv6,private_networking,"),
},
{
"__address__": model.LabelValue("138.65.56.69:80"),
"__meta_digitalocean_droplet_id": model.LabelValue("176011507"),
"__meta_digitalocean_droplet_name": model.LabelValue("prometheus-demo"),
"__meta_digitalocean_image": model.LabelValue("ubuntu-18-04-x64"),
"__meta_digitalocean_private_ipv4": model.LabelValue("10.135.64.212"),
"__meta_digitalocean_public_ipv4": model.LabelValue("138.65.56.69"),
"__meta_digitalocean_public_ipv6": model.LabelValue("2a03:b0c0:3:f0::cf2:4"),
"__meta_digitalocean_region": model.LabelValue("fra1"),
"__meta_digitalocean_size": model.LabelValue("s-1vcpu-1gb"),
"__meta_digitalocean_status": model.LabelValue("active"),
"__meta_digitalocean_features": model.LabelValue(",ipv6,private_networking,"),
},
} {
t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) {
testutil.Equals(t, lbls, tg.Targets[i])
})
}
}

644
discovery/digitalocean/mock_test.go Normal file
View file

@ -0,0 +1,644 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package digitalocean
import (
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"testing"
)
// SDMock is the interface for the DigitalOcean mock
type SDMock struct {
t *testing.T
Server *httptest.Server
Mux *http.ServeMux
}
// NewSDMock returns a new SDMock.
func NewSDMock(t *testing.T) *SDMock {
return &SDMock{
t: t,
}
}
// Endpoint returns the URI to the mock server
func (m *SDMock) Endpoint() string {
return m.Server.URL + "/"
}
// Setup creates the mock server
func (m *SDMock) Setup() {
m.Mux = http.NewServeMux()
m.Server = httptest.NewServer(m.Mux)
}
// ShutdownServer creates the mock server
func (m *SDMock) ShutdownServer() {
m.Server.Close()
}
const tokenID = "3c9a75a2-24fd-4508-b4f2-11f18aa97411"
// HandleDropletsList mocks droplet list.
func (m *SDMock) HandleDropletsList() {
m.Mux.HandleFunc("/v2/droplets", func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", tokenID) {
w.WriteHeader(http.StatusForbidden)
return
}
w.Header().Add("content-type", "application/json; charset=utf-8")
w.Header().Add("ratelimit-limit", "1200")
w.Header().Add("ratelimit-remaining", "965")
w.Header().Add("ratelimit-reset", "1415984218")
w.WriteHeader(http.StatusAccepted)
page := 1
if pageQuery, ok := r.URL.Query()["page"]; ok {
var err error
page, err = strconv.Atoi(pageQuery[0])
if err != nil {
panic(err)
}
}
fmt.Fprint(w, []string{`
{
"droplets": [
{
"id": 3164444,
"name": "example.com",
"memory": 1024,
"vcpus": 1,
"disk": 25,
"locked": false,
"status": "active",
"kernel": {
"id": 2233,
"name": "Ubuntu 14.04 x64 vmlinuz-3.13.0-37-generic",
"version": "3.13.0-37-generic"
},
"created_at": "2014-11-14T16:29:21Z",
"features": [
"backups",
"ipv6",
"virtio"
],
"backup_ids": [
7938002
],
"snapshot_ids": [
],
"image": {
"id": 6918990,
"name": "14.04 x64",
"distribution": "Ubuntu",
"slug": "ubuntu-16-04-x64",
"public": true,
"regions": [
"nyc1",
"ams1",
"sfo1",
"nyc2",
"ams2",
"sgp1",
"lon1",
"nyc3",
"ams3",
"nyc3"
],
"created_at": "2014-10-17T20:24:33Z",
"type": "snapshot",
"min_disk_size": 20,
"size_gigabytes": 2.34
},
"volume_ids": [
],
"size": {
},
"size_slug": "s-1vcpu-1gb",
"networks": {
"v4": [
{
"ip_address": "104.236.32.182",
"netmask": "255.255.192.0",
"gateway": "104.236.0.1",
"type": "public"
}
],
"v6": [
{
"ip_address": "2604:A880:0800:0010:0000:0000:02DD:4001",
"netmask": 64,
"gateway": "2604:A880:0800:0010:0000:0000:0000:0001",
"type": "public"
}
]
},
"region": {
"name": "New York 3",
"slug": "nyc3",
"sizes": [
],
"features": [
"virtio",
"private_networking",
"backups",
"ipv6",
"metadata"
],
"available": null
},
"tags": [
],
"vpc_uuid": "f9b0769c-e118-42fb-a0c4-fed15ef69662"
},
{
"id": 3164494,
"name": "prometheus",
"memory": 1024,
"vcpus": 1,
"disk": 25,
"locked": false,
"status": "active",
"kernel": {
"id": 2233,
"name": "Ubuntu 14.04 x64 vmlinuz-3.13.0-37-generic",
"version": "3.13.0-37-generic"
},
"created_at": "2014-11-14T16:36:31Z",
"features": [
"virtio"
],
"backup_ids": [
],
"snapshot_ids": [
7938206
],
"image": {
"id": 6918990,
"name": "14.04 x64",
"distribution": "Ubuntu",
"slug": "ubuntu-16-04-x64",
"public": true,
"regions": [
"nyc1",
"ams1",
"sfo1",
"nyc2",
"ams2",
"sgp1",
"lon1",
"nyc3",
"ams3",
"nyc3"
],
"created_at": "2014-10-17T20:24:33Z",
"type": "snapshot",
"min_disk_size": 20,
"size_gigabytes": 2.34
},
"volume_ids": [
],
"size": {
},
"size_slug": "s-1vcpu-1gb",
"networks": {
"v4": [
{
"ip_address": "104.131.186.241",
"netmask": "255.255.240.0",
"gateway": "104.131.176.1",
"type": "public"
}
]
},
"region": {
"name": "New York 3",
"slug": "nyc3",
"sizes": [
"s-1vcpu-1gb",
"s-1vcpu-2gb",
"s-1vcpu-3gb",
"s-2vcpu-2gb",
"s-3vcpu-1gb",
"s-2vcpu-4gb",
"s-4vcpu-8gb",
"s-6vcpu-16gb",
"s-8vcpu-32gb",
"s-12vcpu-48gb",
"s-16vcpu-64gb",
"s-20vcpu-96gb",
"s-24vcpu-128gb",
"s-32vcpu-192gb"
],
"features": [
"virtio",
"private_networking",
"backups",
"ipv6",
"metadata"
],
"available": true
},
"tags": [
"monitor"
],
"vpc_uuid": "f9b0769c-e118-42fb-a0c4-fed15ef69662"
}
],
"links": {
"pages": {
"next": "https://api.digitalocean.com/v2/droplets?page=2&per_page=2",
"last": "https://api.digitalocean.com/v2/droplets?page=2&per_page=2"
}
},
"meta": {
"total": 4
}
}
`,
`
{
"droplets": [
{
"id": 175072239,
"name": "prometheus-demo-old",
"memory": 1024,
"vcpus": 1,
"disk": 25,
"locked": false,
"status": "off",
"kernel": null,
"created_at": "2020-01-10T16:47:39Z",
"features": [
"ipv6",
"private_networking"
],
"backup_ids": [],
"next_backup_window": null,
"snapshot_ids": [],
"image": {
"id": 53893572,
"name": "18.04.3 (LTS) x64",
"distribution": "Ubuntu",
"slug": "ubuntu-18-04-x64",
"public": true,
"regions": [
"nyc3",
"nyc1",
"sfo1",
"nyc2",
"ams2",
"sgp1",
"lon1",
"nyc3",
"ams3",
"fra1",
"tor1",
"sfo2",
"blr1",
"sfo3"
],
"created_at": "2019-10-22T01:38:19Z",
"min_disk_size": 20,
"type": "base",
"size_gigabytes": 2.36,
"description": "Ubuntu 18.04 x64 20191022",
"tags": [],
"status": "available"
},
"volume_ids": [],
"size": {
"slug": "s-1vcpu-1gb",
"memory": 1024,
"vcpus": 1,
"disk": 25,
"transfer": 1,
"price_monthly": 5,
"price_hourly": 0.00744,
"regions": [
"ams2",
"ams3",
"blr1",
"fra1",
"lon1",
"nyc1",
"nyc2",
"nyc3",
"sfo1",
"sfo2",
"sfo3",
"sgp1",
"tor1"
],
"available": true
},
"size_slug": "s-1vcpu-1gb",
"networks": {
"v4": [
{
"ip_address": "167.172.111.118",
"netmask": "255.255.240.0",
"gateway": "167.172.176.1",
"type": "public"
},
{
"ip_address": "10.135.64.211",
"netmask": "255.255.0.0",
"gateway": "10.135.0.1",
"type": "private"
}
],
"v6": [
]
},
"region": {
"name": "Frankfurt 1",
"slug": "fra1",
"features": [
"private_networking",
"backups",
"ipv6",
"metadata",
"install_agent",
"storage",
"image_transfer"
],
"available": true,
"sizes": [
"s-1vcpu-1gb",
"512mb",
"s-1vcpu-2gb",
"1gb",
"s-3vcpu-1gb",
"s-2vcpu-2gb",
"s-1vcpu-3gb",
"s-2vcpu-4gb",
"2gb",
"s-4vcpu-8gb",
"m-1vcpu-8gb",
"c-2",
"4gb",
"g-2vcpu-8gb",
"gd-2vcpu-8gb",
"m-16gb",
"s-6vcpu-16gb",
"c-4",
"8gb",
"m-2vcpu-16gb",
"m3-2vcpu-16gb",
"g-4vcpu-16gb",
"gd-4vcpu-16gb",
"m6-2vcpu-16gb",
"m-32gb",
"s-8vcpu-32gb",
"c-8",
"16gb",
"m-4vcpu-32gb",
"m3-4vcpu-32gb",
"g-8vcpu-32gb",
"s-12vcpu-48gb",
"gd-8vcpu-32gb",
"m6-4vcpu-32gb",
"m-64gb",
"s-16vcpu-64gb",
"c-16",
"32gb",
"m-8vcpu-64gb",
"m3-8vcpu-64gb",
"g-16vcpu-64gb",
"s-20vcpu-96gb",
"48gb",
"gd-16vcpu-64gb",
"m6-8vcpu-64gb",
"m-128gb",
"s-24vcpu-128gb",
"c-32",
"64gb",
"m-16vcpu-128gb",
"m3-16vcpu-128gb",
"s-32vcpu-192gb",
"m-24vcpu-192gb",
"m-224gb",
"m6-16vcpu-128gb",
"m3-24vcpu-192gb",
"m6-24vcpu-192gb"
]
},
"tags": []
},
{
"id": 176011507,
"name": "prometheus-demo",
"memory": 1024,
"vcpus": 1,
"disk": 25,
"locked": false,
"status": "active",
"kernel": null,
"created_at": "2020-01-17T12:06:26Z",
"features": [
"ipv6",
"private_networking"
],
"backup_ids": [],
"next_backup_window": null,
"snapshot_ids": [],
"image": {
"id": 53893572,
"name": "18.04.3 (LTS) x64",
"distribution": "Ubuntu",
"slug": "ubuntu-18-04-x64",
"public": true,
"regions": [
"nyc3",
"nyc1",
"sfo1",
"nyc2",
"ams2",
"sgp1",
"lon1",
"nyc3",
"ams3",
"fra1",
"tor1",
"sfo2",
"blr1",
"sfo3"
],
"created_at": "2019-10-22T01:38:19Z",
"min_disk_size": 20,
"type": "base",
"size_gigabytes": 2.36,
"description": "Ubuntu 18.04 x64 20191022",
"tags": [],
"status": "available"
},
"volume_ids": [],
"size": {
"slug": "s-1vcpu-1gb",
"memory": 1024,
"vcpus": 1,
"disk": 25,
"transfer": 1,
"price_monthly": 5,
"price_hourly": 0.00744,
"regions": [
"ams2",
"ams3",
"blr1",
"fra1",
"lon1",
"nyc1",
"nyc2",
"nyc3",
"sfo1",
"sfo2",
"sfo3",
"sgp1",
"tor1"
],
"available": true
},
"size_slug": "s-1vcpu-1gb",
"networks": {
"v4": [
{
"ip_address": "138.65.56.69",
"netmask": "255.255.240.0",
"gateway": "138.65.64.1",
"type": "public"
},
{
"ip_address": "154.245.26.111",
"netmask": "255.255.252.0",
"gateway": "154.245.24.1",
"type": "public"
},
{
"ip_address": "10.135.64.212",
"netmask": "255.255.0.0",
"gateway": "10.135.0.1",
"type": "private"
}
],
"v6": [
{
"ip_address": "2a03:b0c0:3:f0::cf2:4",
"netmask": 64,
"gateway": "2a03:b0c0:3:f0::1",
"type": "public"
}
]
},
"region": {
"name": "Frankfurt 1",
"slug": "fra1",
"features": [
"private_networking",
"backups",
"ipv6",
"metadata",
"install_agent",
"storage",
"image_transfer"
],
"available": true,
"sizes": [
"s-1vcpu-1gb",
"512mb",
"s-1vcpu-2gb",
"1gb",
"s-3vcpu-1gb",
"s-2vcpu-2gb",
"s-1vcpu-3gb",
"s-2vcpu-4gb",
"2gb",
"s-4vcpu-8gb",
"m-1vcpu-8gb",
"c-2",
"4gb",
"g-2vcpu-8gb",
"gd-2vcpu-8gb",
"m-16gb",
"s-6vcpu-16gb",
"c-4",
"8gb",
"m-2vcpu-16gb",
"m3-2vcpu-16gb",
"g-4vcpu-16gb",
"gd-4vcpu-16gb",
"m6-2vcpu-16gb",
"m-32gb",
"s-8vcpu-32gb",
"c-8",
"16gb",
"m-4vcpu-32gb",
"m3-4vcpu-32gb",
"g-8vcpu-32gb",
"s-12vcpu-48gb",
"gd-8vcpu-32gb",
"m6-4vcpu-32gb",
"m-64gb",
"s-16vcpu-64gb",
"c-16",
"32gb",
"m-8vcpu-64gb",
"m3-8vcpu-64gb",
"g-16vcpu-64gb",
"s-20vcpu-96gb",
"48gb",
"gd-16vcpu-64gb",
"m6-8vcpu-64gb",
"m-128gb",
"s-24vcpu-128gb",
"c-32",
"64gb",
"m-16vcpu-128gb",
"m3-16vcpu-128gb",
"s-32vcpu-192gb",
"m-24vcpu-192gb",
"m-224gb",
"m6-16vcpu-128gb",
"m3-24vcpu-192gb",
"m6-24vcpu-192gb"
]
},
"tags": []
}
],
"links": {
"pages": {
"first": "https://api.digitalocean.com/v2/droplets?page=1&per_page=2",
"prev": "https://api.digitalocean.com/v2/droplets?page=1&per_page=2"
}
},
"meta": {
"total": 4
}
}
`,
}[page-1],
)
})
}

19
discovery/dns/dns.go
View file

@ -35,7 +35,10 @@ import (
const (
resolvConf = "/etc/resolv.conf"
dnsNameLabel = model.MetaLabelPrefix + "dns_name"
dnsNameLabel = model.MetaLabelPrefix + "dns_name"
dnsSrvRecordPrefix = model.MetaLabelPrefix + "dns_srv_record_"
dnsSrvRecordTargetLabel = dnsSrvRecordPrefix + "target"
dnsSrvRecordPortLabel = dnsSrvRecordPrefix + "port"
// Constants for instrumentation.
namespace = "prometheus"
@ -151,7 +154,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
wg.Add(len(d.names))
for _, name := range d.names {
go func(n string) {
if err := d.refreshOne(ctx, n, ch); err != nil {
if err := d.refreshOne(ctx, n, ch); err != nil && err != context.Canceled {
level.Error(d.logger).Log("msg", "Error refreshing DNS targets", "err", err)
}
wg.Done()
@ -183,9 +186,13 @@ func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targ
}
for _, record := range response.Answer {
var target model.LabelValue
var target, dnsSrvRecordTarget, dnsSrvRecordPort model.LabelValue
switch addr := record.(type) {
case *dns.SRV:
dnsSrvRecordTarget = model.LabelValue(addr.Target)
dnsSrvRecordPort = model.LabelValue(fmt.Sprintf("%d", addr.Port))
// Remove the final dot from rooted DNS names to make them look more usual.
addr.Target = strings.TrimRight(addr.Target, ".")
@ -199,8 +206,10 @@ func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targ
continue
}
tg.Targets = append(tg.Targets, model.LabelSet{
model.AddressLabel: target,
dnsNameLabel: model.LabelValue(name),
model.AddressLabel: target,
dnsNameLabel: model.LabelValue(name),
dnsSrvRecordTargetLabel: dnsSrvRecordTarget,
dnsSrvRecordPortLabel: dnsSrvRecordPort,
})
}

134
discovery/dns/dns_test.go
View file

@ -22,12 +22,18 @@ import (
"github.com/go-kit/kit/log"
"github.com/miekg/dns"
"github.com/prometheus/common/model"
"go.uber.org/goleak"
"gopkg.in/yaml.v2"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/testutil"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
func TestDNS(t *testing.T) {
testCases := []struct {
name string
@ -69,7 +75,12 @@ func TestDNS(t *testing.T) {
{
Source: "web.example.com.",
Targets: []model.LabelSet{
{"__address__": "192.0.2.2:80", "__meta_dns_name": "web.example.com."},
{
"__address__": "192.0.2.2:80",
"__meta_dns_name": "web.example.com.",
"__meta_dns_srv_record_target": "",
"__meta_dns_srv_record_port": "",
},
},
},
},
@ -94,7 +105,12 @@ func TestDNS(t *testing.T) {
{
Source: "web.example.com.",
Targets: []model.LabelSet{
{"__address__": "[::1]:80", "__meta_dns_name": "web.example.com."},
{
"__address__": "[::1]:80",
"__meta_dns_name": "web.example.com.",
"__meta_dns_srv_record_target": "",
"__meta_dns_srv_record_port": "",
},
},
},
},
@ -103,6 +119,7 @@ func TestDNS(t *testing.T) {
name: "SRV record query",
config: SDConfig{
Names: []string{"_mysql._tcp.db.example.com."},
Type: "SRV",
RefreshInterval: model.Duration(time.Minute),
},
lookup: func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) {
@ -118,8 +135,18 @@ func TestDNS(t *testing.T) {
{
Source: "_mysql._tcp.db.example.com.",
Targets: []model.LabelSet{
{"__address__": "db1.example.com:3306", "__meta_dns_name": "_mysql._tcp.db.example.com."},
{"__address__": "db2.example.com:3306", "__meta_dns_name": "_mysql._tcp.db.example.com."},
{
"__address__": "db1.example.com:3306",
"__meta_dns_name": "_mysql._tcp.db.example.com.",
"__meta_dns_srv_record_target": "db1.example.com.",
"__meta_dns_srv_record_port": "3306",
},
{
"__address__": "db2.example.com:3306",
"__meta_dns_name": "_mysql._tcp.db.example.com.",
"__meta_dns_srv_record_target": "db2.example.com.",
"__meta_dns_srv_record_port": "3306",
},
},
},
},
@ -143,7 +170,12 @@ func TestDNS(t *testing.T) {
{
Source: "_mysql._tcp.db.example.com.",
Targets: []model.LabelSet{
{"__address__": "db1.example.com:3306", "__meta_dns_name": "_mysql._tcp.db.example.com."},
{
"__address__": "db1.example.com:3306",
"__meta_dns_name": "_mysql._tcp.db.example.com.",
"__meta_dns_srv_record_target": "db1.example.com.",
"__meta_dns_srv_record_port": "3306",
},
},
},
},
@ -178,3 +210,93 @@ func TestDNS(t *testing.T) {
})
}
}
func TestSDConfigUnmarshalYAML(t *testing.T) {
marshal := func(c SDConfig) []byte {
d, err := yaml.Marshal(c)
if err != nil {
panic(err)
}
return d
}
unmarshal := func(d []byte) func(interface{}) error {
return func(o interface{}) error {
return yaml.Unmarshal(d, o)
}
}
cases := []struct {
name string
input SDConfig
expectErr bool
}{
{
name: "valid srv",
input: SDConfig{
Names: []string{"a.example.com", "b.example.com"},
Type: "SRV",
},
expectErr: false,
},
{
name: "valid a",
input: SDConfig{
Names: []string{"a.example.com", "b.example.com"},
Type: "A",
Port: 5300,
},
expectErr: false,
},
{
name: "valid aaaa",
input: SDConfig{
Names: []string{"a.example.com", "b.example.com"},
Type: "AAAA",
Port: 5300,
},
expectErr: false,
},
{
name: "invalid a without port",
input: SDConfig{
Names: []string{"a.example.com", "b.example.com"},
Type: "A",
},
expectErr: true,
},
{
name: "invalid aaaa without port",
input: SDConfig{
Names: []string{"a.example.com", "b.example.com"},
Type: "AAAA",
},
expectErr: true,
},
{
name: "invalid empty names",
input: SDConfig{
Names: []string{},
Type: "AAAA",
},
expectErr: true,
},
{
name: "invalid unknown dns type",
input: SDConfig{
Names: []string{"a.example.com", "b.example.com"},
Type: "PTR",
},
expectErr: true,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
var config SDConfig
d := marshal(c.input)
err := config.UnmarshalYAML(unmarshal(d))
testutil.Equals(t, c.expectErr, err != nil)
})
}
}

153
discovery/dockerswarm/dockerswarm.go Normal file
View file

@ -0,0 +1,153 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dockerswarm
import (
"context"
"fmt"
"net/http"
"net/url"
"time"
"github.com/docker/docker/client"
"github.com/go-kit/kit/log"
config_util "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/version"
"github.com/prometheus/prometheus/discovery/refresh"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
const (
swarmLabel = model.MetaLabelPrefix + "dockerswarm_"
)
var userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
// DefaultSDConfig is the default Docker Swarm SD configuration.
var DefaultSDConfig = SDConfig{
RefreshInterval: model.Duration(60 * time.Second),
Port: 80,
}
// SDConfig is the configuration for Docker Swarm based service discovery.
type SDConfig struct {
HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
Host string `yaml:"host"`
Role string `yaml:"role"`
Port int `yaml:"port"`
RefreshInterval model.Duration `yaml:"refresh_interval"`
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
*c = DefaultSDConfig
type plain SDConfig
err := unmarshal((*plain)(c))
if err != nil {
return err
}
if c.Host == "" {
return fmt.Errorf("host missing")
}
if _, err = url.Parse(c.Host); err != nil {
return err
}
switch c.Role {
case "services", "nodes", "tasks":
case "":
return fmt.Errorf("role missing (one of: tasks, services, nodes)")
default:
return fmt.Errorf("invalid role %s, expected tasks, services, or nodes", c.Role)
}
return nil
}
// Discovery periodically performs Docker Swarm requests. It implements
// the Discoverer interface.
type Discovery struct {
*refresh.Discovery
client *client.Client
role string
port int
}
// NewDiscovery returns a new Discovery which periodically refreshes its targets.
func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
var err error
d := &Discovery{
port: conf.Port,
role: conf.Role,
}
hostURL, err := url.Parse(conf.Host)
if err != nil {
return nil, err
}
opts := []client.Opt{
client.WithHost(conf.Host),
client.WithAPIVersionNegotiation(),
}
// There are other protocols than HTTP supported by the Docker daemon, like
// unix, which are not supported by the HTTP client. Passing HTTP client
// options to the Docker client makes those non-HTTP requests fail.
if hostURL.Scheme == "http" || hostURL.Scheme == "https" {
rt, err := config_util.NewRoundTripperFromConfig(conf.HTTPClientConfig, "dockerswarm_sd", false)
if err != nil {
return nil, err
}
opts = append(opts,
client.WithHTTPClient(&http.Client{
Transport: rt,
Timeout: time.Duration(conf.RefreshInterval),
}),
client.WithScheme(hostURL.Scheme),
client.WithHTTPHeaders(map[string]string{
"User-Agent": userAgent,
}),
)
}
d.client, err = client.NewClientWithOpts(opts...)
if err != nil {
return nil, fmt.Errorf("error setting up docker swarm client: %w", err)
}
d.Discovery = refresh.NewDiscovery(
logger,
"dockerswarm",
time.Duration(conf.RefreshInterval),
d.refresh,
)
return d, nil
}
func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
switch d.role {
case "services":
return d.refreshServices(ctx)
case "nodes":
return d.refreshNodes(ctx)
case "tasks":
return d.refreshTasks(ctx)
default:
panic(fmt.Errorf("unexpected role %s", d.role))
}
}

118
discovery/dockerswarm/mock_test.go Normal file
View file

@ -0,0 +1,118 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dockerswarm
import (
"crypto/sha1"
"encoding/base64"
"io/ioutil"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"github.com/prometheus/prometheus/util/strutil"
"github.com/prometheus/prometheus/util/testutil"
"gopkg.in/yaml.v2"
)
// SDMock is the interface for the DigitalOcean mock
type SDMock struct {
t *testing.T
Server *httptest.Server
Mux *http.ServeMux
directory string
calls map[string]int
}
// NewSDMock returns a new SDMock.
func NewSDMock(t *testing.T, directory string) *SDMock {
return &SDMock{
t: t,
directory: directory,
calls: make(map[string]int),
}
}
// Endpoint returns the URI to the mock server
func (m *SDMock) Endpoint() string {
return m.Server.URL + "/"
}
// Setup creates the mock server
func (m *SDMock) Setup() {
m.Mux = http.NewServeMux()
m.Server = httptest.NewServer(m.Mux)
m.t.Cleanup(m.Server.Close)
m.SetupHandlers()
}
// HandleNodesList mocks nodes list.
func (m *SDMock) SetupHandlers() {
headers := make(map[string]string)
rawHeaders, err := ioutil.ReadFile(filepath.Join("testdata", m.directory, "headers.yml"))
testutil.Ok(m.t, err)
yaml.Unmarshal(rawHeaders, &headers)
prefix := "/"
if v, ok := headers["Api-Version"]; ok {
prefix += "v" + v + "/"
}
for _, path := range []string{"_ping", "networks/", "services/", "nodes/", "nodes", "services", "tasks"} {
p := path
handler := prefix + p
if p == "_ping" {
handler = "/" + p
}
m.Mux.HandleFunc(handler, func(w http.ResponseWriter, r *http.Request) {
// The discovery should only call each API endpoint once.
m.calls[r.RequestURI]++
if m.calls[r.RequestURI] != 1 {
w.WriteHeader(http.StatusTooManyRequests)
return
}
for k, v := range headers {
w.Header().Add(k, v)
}
parts := strings.Split(r.RequestURI, "/")
var f string
if strings.HasSuffix(p, "/") {
f = filepath.Join(p[:len(p)-1], strutil.SanitizeLabelName(parts[len(parts)-1]))
} else {
query := strings.Split(parts[len(parts)-1], "?")
f = query[0]
if len(query) == 2 {
h := sha1.New()
h.Write([]byte(query[1]))
// Avoing long filenames for Windows.
f += "__" + base64.URLEncoding.EncodeToString(h.Sum(nil))[:10]
}
}
if response, err := ioutil.ReadFile(filepath.Join("testdata", m.directory, f+".json")); err == nil {
w.Header().Add("content-type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(response)
return
}
if response, err := ioutil.ReadFile(filepath.Join("testdata", m.directory, f)); err == nil {
w.WriteHeader(http.StatusOK)
w.Write(response)
return
}
w.WriteHeader(http.StatusInternalServerError)
})
}
}

55
discovery/dockerswarm/network.go Normal file
View file

@ -0,0 +1,55 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dockerswarm
import (
"context"
"fmt"
"github.com/docker/docker/api/types"
"github.com/prometheus/prometheus/util/strutil"
)
const (
swarmLabelNetworkPrefix = swarmLabel + "network_"
swarmLabelNetworkID = swarmLabelNetworkPrefix + "id"
swarmLabelNetworkName = swarmLabelNetworkPrefix + "name"
swarmLabelNetworkScope = swarmLabelNetworkPrefix + "scope"
swarmLabelNetworkInternal = swarmLabelNetworkPrefix + "internal"
swarmLabelNetworkIngress = swarmLabelNetworkPrefix + "ingress"
swarmLabelNetworkLabelPrefix = swarmLabelNetworkPrefix + "label_"
)
func (d *Discovery) getNetworksLabels(ctx context.Context) (map[string]map[string]string, error) {
networks, err := d.client.NetworkList(ctx, types.NetworkListOptions{})
if err != nil {
return nil, err
}
labels := make(map[string]map[string]string, len(networks))
for _, network := range networks {
labels[network.ID] = map[string]string{
swarmLabelNetworkID: network.ID,
swarmLabelNetworkName: network.Name,
swarmLabelNetworkScope: network.Scope,
swarmLabelNetworkInternal: fmt.Sprintf("%t", network.Internal),
swarmLabelNetworkIngress: fmt.Sprintf("%t", network.Ingress),
}
for k, v := range network.Labels {
ln := strutil.SanitizeLabelName(k)
labels[network.ID][swarmLabelNetworkLabelPrefix+ln] = v
}
}
return labels, nil
}

111
discovery/dockerswarm/nodes.go Normal file
View file

@ -0,0 +1,111 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dockerswarm
import (
"context"
"fmt"
"net"
"strconv"
"github.com/docker/docker/api/types"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/strutil"
)
const (
swarmLabelNodePrefix = swarmLabel + "node_"
swarmLabelNodeAddress = swarmLabelNodePrefix + "address"
swarmLabelNodeAvailability = swarmLabelNodePrefix + "availability"
swarmLabelNodeEngineVersion = swarmLabelNodePrefix + "engine_version"
swarmLabelNodeHostname = swarmLabelNodePrefix + "hostname"
swarmLabelNodeID = swarmLabelNodePrefix + "id"
swarmLabelNodeLabelPrefix = swarmLabelNodePrefix + "label_"
swarmLabelNodeManagerAddr = swarmLabelNodePrefix + "manager_address"
swarmLabelNodeManagerLeader = swarmLabelNodePrefix + "manager_leader"
swarmLabelNodeManagerReachability = swarmLabelNodePrefix + "manager_reachability"
swarmLabelNodePlatformArchitecture = swarmLabelNodePrefix + "platform_architecture"
swarmLabelNodePlatformOS = swarmLabelNodePrefix + "platform_os"
swarmLabelNodeRole = swarmLabelNodePrefix + "role"
swarmLabelNodeStatus = swarmLabelNodePrefix + "status"
)
func (d *Discovery) refreshNodes(ctx context.Context) ([]*targetgroup.Group, error) {
tg := &targetgroup.Group{
Source: "DockerSwarm",
}
nodes, err := d.client.NodeList(ctx, types.NodeListOptions{})
if err != nil {
return nil, fmt.Errorf("error while listing swarm nodes: %w", err)
}
for _, n := range nodes {
labels := model.LabelSet{
swarmLabelNodeID: model.LabelValue(n.ID),
swarmLabelNodeRole: model.LabelValue(n.Spec.Role),
swarmLabelNodeAvailability: model.LabelValue(n.Spec.Availability),
swarmLabelNodeHostname: model.LabelValue(n.Description.Hostname),
swarmLabelNodePlatformArchitecture: model.LabelValue(n.Description.Platform.Architecture),
swarmLabelNodePlatformOS: model.LabelValue(n.Description.Platform.OS),
swarmLabelNodeEngineVersion: model.LabelValue(n.Description.Engine.EngineVersion),
swarmLabelNodeStatus: model.LabelValue(n.Status.State),
swarmLabelNodeAddress: model.LabelValue(n.Status.Addr),
}
if n.ManagerStatus != nil {
labels[swarmLabelNodeManagerLeader] = model.LabelValue(fmt.Sprintf("%t", n.ManagerStatus.Leader))
labels[swarmLabelNodeManagerReachability] = model.LabelValue(n.ManagerStatus.Reachability)
labels[swarmLabelNodeManagerAddr] = model.LabelValue(n.ManagerStatus.Addr)
}
for k, v := range n.Spec.Labels {
ln := strutil.SanitizeLabelName(k)
labels[model.LabelName(swarmLabelNodeLabelPrefix+ln)] = model.LabelValue(v)
}
addr := net.JoinHostPort(n.Status.Addr, strconv.FormatUint(uint64(d.port), 10))
labels[model.AddressLabel] = model.LabelValue(addr)
tg.Targets = append(tg.Targets, labels)
}
return []*targetgroup.Group{tg}, nil
}
func (d *Discovery) getNodesLabels(ctx context.Context) (map[string]map[string]string, error) {
nodes, err := d.client.NodeList(ctx, types.NodeListOptions{})
if err != nil {
return nil, fmt.Errorf("error while listing swarm nodes: %w", err)
}
labels := make(map[string]map[string]string, len(nodes))
for _, n := range nodes {
labels[n.ID] = map[string]string{
swarmLabelNodeID: n.ID,
swarmLabelNodeRole: string(n.Spec.Role),
swarmLabelNodeAddress: n.Status.Addr,
swarmLabelNodeAvailability: string(n.Spec.Availability),
swarmLabelNodeHostname: n.Description.Hostname,
swarmLabelNodePlatformArchitecture: n.Description.Platform.Architecture,
swarmLabelNodePlatformOS: n.Description.Platform.OS,
swarmLabelNodeStatus: string(n.Status.State),
}
for k, v := range n.Spec.Labels {
ln := strutil.SanitizeLabelName(k)
labels[n.ID][swarmLabelNodeLabelPrefix+ln] = v
}
}
return labels, nil
}

130
discovery/dockerswarm/nodes_test.go Normal file
View file

@ -0,0 +1,130 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dockerswarm
import (
"context"
"fmt"
"testing"
"github.com/go-kit/kit/log"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/util/testutil"
"gopkg.in/yaml.v2"
)
func TestDockerSwarmNodesSDRefresh(t *testing.T) {
sdmock := NewSDMock(t, "swarmprom")
sdmock.Setup()
e := sdmock.Endpoint()
url := e[:len(e)-1]
cfgString := fmt.Sprintf(`
---
role: nodes
host: %s
`, url)
var cfg SDConfig
testutil.Ok(t, yaml.Unmarshal([]byte(cfgString), &cfg))
d, err := NewDiscovery(&cfg, log.NewNopLogger())
testutil.Ok(t, err)
ctx := context.Background()
tgs, err := d.refresh(ctx)
testutil.Ok(t, err)
testutil.Equals(t, 1, len(tgs))
tg := tgs[0]
testutil.Assert(t, tg != nil, "tg should not be nil")
testutil.Assert(t, tg.Targets != nil, "tg.targets should not be nil")
testutil.Equals(t, 5, len(tg.Targets))
for i, lbls := range []model.LabelSet{
{
"__address__": model.LabelValue("10.0.232.3:80"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.3"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_engine_version": model.LabelValue("19.03.11"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-3"),
"__meta_dockerswarm_node_id": model.LabelValue("bvtjl7pnrtg0k88ywialsldpd"),
"__meta_dockerswarm_node_manager_address": model.LabelValue("10.0.232.3:2377"),
"__meta_dockerswarm_node_manager_leader": model.LabelValue("false"),
"__meta_dockerswarm_node_manager_reachability": model.LabelValue("reachable"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
},
{
"__address__": model.LabelValue("10.0.232.1:80"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.1"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_engine_version": model.LabelValue("19.03.5-ce"),
"__meta_dockerswarm_node_hostname": model.LabelValue("oxygen"),
"__meta_dockerswarm_node_id": model.LabelValue("d3cw2msquo0d71yn42qrnb0tu"),
"__meta_dockerswarm_node_manager_address": model.LabelValue("10.0.232.1:2377"),
"__meta_dockerswarm_node_manager_leader": model.LabelValue("true"),
"__meta_dockerswarm_node_manager_reachability": model.LabelValue("reachable"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
},
{
"__address__": model.LabelValue("10.0.232.2:80"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.2"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_engine_version": model.LabelValue("19.03.11"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-2"),
"__meta_dockerswarm_node_id": model.LabelValue("i9woemzxymn1n98o9ufebclgm"),
"__meta_dockerswarm_node_manager_address": model.LabelValue("10.0.232.2:2377"),
"__meta_dockerswarm_node_manager_leader": model.LabelValue("false"),
"__meta_dockerswarm_node_manager_reachability": model.LabelValue("reachable"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
},
{
"__address__": model.LabelValue("10.0.232.4:80"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.4"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_engine_version": model.LabelValue("19.03.11"),
"__meta_dockerswarm_node_hostname": model.LabelValue("worker-1"),
"__meta_dockerswarm_node_id": model.LabelValue("jkc2xd7p3xdazf0u6sh1m9dmr"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("worker"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
},
{
"__address__": model.LabelValue("10.0.232.5:80"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.5"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_engine_version": model.LabelValue("19.03.11"),
"__meta_dockerswarm_node_hostname": model.LabelValue("worker-2"),
"__meta_dockerswarm_node_id": model.LabelValue("ldawcom10uqi6owysgi28n4ve"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("worker"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
},
} {
t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) {
testutil.Equals(t, lbls, tg.Targets[i])
})
}
}

162
discovery/dockerswarm/services.go Normal file
View file

@ -0,0 +1,162 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dockerswarm
import (
"context"
"fmt"
"net"
"strconv"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/swarm"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/strutil"
)
const (
swarmLabelServicePrefix = swarmLabel + "service_"
swarmLabelServiceEndpointPortName = swarmLabelServicePrefix + "endpoint_port_name"
swarmLabelServiceEndpointPortPublishMode = swarmLabelServicePrefix + "endpoint_port_publish_mode"
swarmLabelServiceID = swarmLabelServicePrefix + "id"
swarmLabelServiceLabelPrefix = swarmLabelServicePrefix + "label_"
swarmLabelServiceName = swarmLabelServicePrefix + "name"
swarmLabelServiceMode = swarmLabelServicePrefix + "mode"
swarmLabelServiceUpdatingStatus = swarmLabelServicePrefix + "updating_status"
swarmLabelServiceTaskPrefix = swarmLabelServicePrefix + "task_"
swarmLabelServiceTaskContainerImage = swarmLabelServiceTaskPrefix + "container_image"
swarmLabelServiceTaskContainerHostname = swarmLabelServiceTaskPrefix + "container_hostname"
)
func (d *Discovery) refreshServices(ctx context.Context) ([]*targetgroup.Group, error) {
tg := &targetgroup.Group{
Source: "DockerSwarm",
}
services, err := d.client.ServiceList(ctx, types.ServiceListOptions{})
if err != nil {
return nil, fmt.Errorf("error while listing swarm services: %w", err)
}
networkLabels, err := d.getNetworksLabels(ctx)
if err != nil {
return nil, fmt.Errorf("error while computing swarm network labels: %w", err)
}
for _, s := range services {
commonLabels := map[string]string{
swarmLabelServiceID: s.ID,
swarmLabelServiceName: s.Spec.Name,
swarmLabelServiceTaskContainerHostname: s.Spec.TaskTemplate.ContainerSpec.Hostname,
swarmLabelServiceTaskContainerImage: s.Spec.TaskTemplate.ContainerSpec.Image,
}
commonLabels[swarmLabelServiceMode] = getServiceValueMode(s)
if s.UpdateStatus != nil {
commonLabels[swarmLabelServiceUpdatingStatus] = string(s.UpdateStatus.State)
}
for k, v := range s.Spec.Labels {
ln := strutil.SanitizeLabelName(k)
commonLabels[swarmLabelServiceLabelPrefix+ln] = v
}
for _, p := range s.Endpoint.VirtualIPs {
var added bool
ip, _, err := net.ParseCIDR(p.Addr)
if err != nil {
return nil, fmt.Errorf("error while parsing address %s: %w", p.Addr, err)
}
for _, e := range s.Endpoint.Ports {
if e.Protocol != swarm.PortConfigProtocolTCP {
continue
}
labels := model.LabelSet{
swarmLabelServiceEndpointPortName: model.LabelValue(e.Name),
swarmLabelServiceEndpointPortPublishMode: model.LabelValue(e.PublishMode),
}
for k, v := range commonLabels {
labels[model.LabelName(k)] = model.LabelValue(v)
}
for k, v := range networkLabels[p.NetworkID] {
labels[model.LabelName(k)] = model.LabelValue(v)
}
addr := net.JoinHostPort(ip.String(), strconv.FormatUint(uint64(e.PublishedPort), 10))
labels[model.AddressLabel] = model.LabelValue(addr)
tg.Targets = append(tg.Targets, labels)
added = true
}
if !added {
labels := model.LabelSet{}
for k, v := range commonLabels {
labels[model.LabelName(k)] = model.LabelValue(v)
}
for k, v := range networkLabels[p.NetworkID] {
labels[model.LabelName(k)] = model.LabelValue(v)
}
addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", d.port))
labels[model.AddressLabel] = model.LabelValue(addr)
tg.Targets = append(tg.Targets, labels)
}
}
}
return []*targetgroup.Group{tg}, nil
}
func (d *Discovery) getServicesLabelsAndPorts(ctx context.Context) (map[string]map[string]string, map[string][]swarm.PortConfig, error) {
services, err := d.client.ServiceList(ctx, types.ServiceListOptions{})
if err != nil {
return nil, nil, err
}
servicesLabels := make(map[string]map[string]string, len(services))
servicesPorts := make(map[string][]swarm.PortConfig, len(services))
for _, s := range services {
servicesLabels[s.ID] = map[string]string{
swarmLabelServiceID: s.ID,
swarmLabelServiceName: s.Spec.Name,
}
servicesLabels[s.ID][swarmLabelServiceMode] = getServiceValueMode(s)
for k, v := range s.Spec.Labels {
ln := strutil.SanitizeLabelName(k)
servicesLabels[s.ID][swarmLabelServiceLabelPrefix+ln] = v
}
servicesPorts[s.ID] = s.Endpoint.Ports
}
return servicesLabels, servicesPorts, nil
}
func getServiceValueMode(s swarm.Service) string {
if s.Spec.Mode.Global != nil {
return "global"
}
if s.Spec.Mode.Replicated != nil {
return "replicated"
}
return ""
}

316
discovery/dockerswarm/services_test.go Normal file
View file

@ -0,0 +1,316 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dockerswarm
import (
"context"
"fmt"
"testing"
"github.com/go-kit/kit/log"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/util/testutil"
"gopkg.in/yaml.v2"
)
func TestDockerSwarmSDServicesRefresh(t *testing.T) {
sdmock := NewSDMock(t, "swarmprom")
sdmock.Setup()
e := sdmock.Endpoint()
url := e[:len(e)-1]
cfgString := fmt.Sprintf(`
---
role: services
host: %s
`, url)
var cfg SDConfig
testutil.Ok(t, yaml.Unmarshal([]byte(cfgString), &cfg))
d, err := NewDiscovery(&cfg, log.NewNopLogger())
testutil.Ok(t, err)
ctx := context.Background()
tgs, err := d.refresh(ctx)
testutil.Ok(t, err)
testutil.Equals(t, 1, len(tgs))
tg := tgs[0]
testutil.Assert(t, tg != nil, "tg should not be nil")
testutil.Assert(t, tg.Targets != nil, "tg.targets should not be nil")
testutil.Equals(t, 15, len(tg.Targets))
for i, lbls := range []model.LabelSet{
{
"__address__": model.LabelValue("10.0.0.7:9100"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_endpoint_port_name": model.LabelValue(""),
"__meta_dockerswarm_service_endpoint_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_service_id": model.LabelValue("3qvd7bwfptmoht16t1f7jllb6"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_node-exporter"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0@sha256:1d2518ec0501dd848e718bf008df37852b1448c862d6f256f2d39244975758d6"),
},
{
"__address__": model.LabelValue("10.0.1.2:9100"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_endpoint_port_name": model.LabelValue(""),
"__meta_dockerswarm_service_endpoint_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_service_id": model.LabelValue("3qvd7bwfptmoht16t1f7jllb6"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_node-exporter"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0@sha256:1d2518ec0501dd848e718bf008df37852b1448c862d6f256f2d39244975758d6"),
},
{
"__address__": model.LabelValue("10.0.1.34:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_id": model.LabelValue("9bbq7j55tzzz85k2gg52x73rg"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("cloudflare/unsee:v0.8.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_unsee"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("cloudflare/unsee:v0.8.0@sha256:28398f47f63feb1647887999701fa730da351fc6d3cc7228e5da44b40a663ada"),
},
{
"__address__": model.LabelValue("10.0.1.13:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_id": model.LabelValue("hv645udwaaewyw7wuc6nspm68"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-prometheus:v2.5.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_prometheus"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/swarmprom-prometheus:v2.5.0@sha256:f1a3781c4785637ba088dcf54889f77d9b6b69f21b2c0a167c1347473f4e2587"),
},
{
"__address__": model.LabelValue("10.0.1.23:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_id": model.LabelValue("iml8vdd2dudvq457sbirpf66z"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("google/cadvisor"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_cadvisor"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("google/cadvisor:latest@sha256:815386ebbe9a3490f38785ab11bda34ec8dacf4634af77b8912832d4f85dca04"),
},
{
"__address__": model.LabelValue("10.0.1.31:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_id": model.LabelValue("rl4jhdws3r4lkfvq8kx6c4xnr"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-alertmanager:v0.14.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_alertmanager"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/swarmprom-alertmanager:v0.14.0@sha256:7069b656bd5df0606ff1db81a7553a25dc72be51d3aca6a7e08d776566cefbd8"),
},
{
"__address__": model.LabelValue("10.0.0.13:9090"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_endpoint_port_name": model.LabelValue(""),
"__meta_dockerswarm_service_endpoint_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/caddy:latest@sha256:44541cfacb66f4799f81f17fcfb3cb757ccc8f327592745549f5930c42d115c9"),
},
{
"__address__": model.LabelValue("10.0.0.13:9093"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_endpoint_port_name": model.LabelValue(""),
"__meta_dockerswarm_service_endpoint_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/caddy:latest@sha256:44541cfacb66f4799f81f17fcfb3cb757ccc8f327592745549f5930c42d115c9"),
},
{
"__address__": model.LabelValue("10.0.0.13:9094"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_endpoint_port_name": model.LabelValue(""),
"__meta_dockerswarm_service_endpoint_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/caddy:latest@sha256:44541cfacb66f4799f81f17fcfb3cb757ccc8f327592745549f5930c42d115c9"),
},
{
"__address__": model.LabelValue("10.0.1.15:9090"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_endpoint_port_name": model.LabelValue(""),
"__meta_dockerswarm_service_endpoint_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/caddy:latest@sha256:44541cfacb66f4799f81f17fcfb3cb757ccc8f327592745549f5930c42d115c9"),
},
{
"__address__": model.LabelValue("10.0.1.15:9093"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_endpoint_port_name": model.LabelValue(""),
"__meta_dockerswarm_service_endpoint_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/caddy:latest@sha256:44541cfacb66f4799f81f17fcfb3cb757ccc8f327592745549f5930c42d115c9"),
},
{
"__address__": model.LabelValue("10.0.1.15:9094"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_endpoint_port_name": model.LabelValue(""),
"__meta_dockerswarm_service_endpoint_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/caddy:latest@sha256:44541cfacb66f4799f81f17fcfb3cb757ccc8f327592745549f5930c42d115c9"),
},
{
"__address__": model.LabelValue("10.0.0.15:3000"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_endpoint_port_name": model.LabelValue(""),
"__meta_dockerswarm_service_endpoint_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_service_id": model.LabelValue("uk9su5tb9ykfzew3qtmp14uzh"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-grafana:5.3.4"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_grafana"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/swarmprom-grafana:5.3.4@sha256:2aca8aa5716e6e0eed3fcdc88fec256a0a1828c491a8cf240486ae7cc473092d"),
},
{
"__address__": model.LabelValue("10.0.1.29:3000"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_endpoint_port_name": model.LabelValue(""),
"__meta_dockerswarm_service_endpoint_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_service_id": model.LabelValue("uk9su5tb9ykfzew3qtmp14uzh"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-grafana:5.3.4"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_grafana"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/swarmprom-grafana:5.3.4@sha256:2aca8aa5716e6e0eed3fcdc88fec256a0a1828c491a8cf240486ae7cc473092d"),
},
{
"__address__": model.LabelValue("10.0.1.17:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_service_id": model.LabelValue("ul5qawv4s7f7msm7dtyfarw80"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_dockerd-exporter"),
"__meta_dockerswarm_service_task_container_hostname": model.LabelValue(""),
"__meta_dockerswarm_service_task_container_image": model.LabelValue("stefanprodan/caddy:latest@sha256:44541cfacb66f4799f81f17fcfb3cb757ccc8f327592745549f5930c42d115c9"),
},
} {
t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) {
testutil.Equals(t, lbls, tg.Targets[i])
})
}
}

160
discovery/dockerswarm/tasks.go Normal file
View file

@ -0,0 +1,160 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dockerswarm
import (
"context"
"fmt"
"net"
"strconv"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/swarm"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/strutil"
)
const (
swarmLabelTaskPrefix = swarmLabel + "task_"
swarmLabelTaskID = swarmLabelTaskPrefix + "id"
swarmLabelTaskLabelPrefix = swarmLabelTaskPrefix + "label_"
swarmLabelTaskDesiredState = swarmLabelTaskPrefix + "desired_state"
swarmLabelTaskStatus = swarmLabelTaskPrefix + "state"
swarmLabelTaskContainerID = swarmLabelTaskPrefix + "container_id"
swarmLabelTaskSlot = swarmLabelTaskPrefix + "slot"
swarmLabelTaskPortMode = swarmLabelTaskPrefix + "port_publish_mode"
)
func (d *Discovery) refreshTasks(ctx context.Context) ([]*targetgroup.Group, error) {
tg := &targetgroup.Group{
Source: "DockerSwarm",
}
tasks, err := d.client.TaskList(ctx, types.TaskListOptions{})
if err != nil {
return nil, fmt.Errorf("error while listing swarm services: %w", err)
}
serviceLabels, servicePorts, err := d.getServicesLabelsAndPorts(ctx)
if err != nil {
return nil, fmt.Errorf("error while computing services labels and ports: %w", err)
}
nodeLabels, err := d.getNodesLabels(ctx)
if err != nil {
return nil, fmt.Errorf("error while computing nodes labels and ports: %w", err)
}
networkLabels, err := d.getNetworksLabels(ctx)
if err != nil {
return nil, fmt.Errorf("error while computing swarm network labels: %w", err)
}
for _, s := range tasks {
commonLabels := map[string]string{
swarmLabelTaskID: s.ID,
swarmLabelTaskDesiredState: string(s.DesiredState),
swarmLabelTaskStatus: string(s.Status.State),
swarmLabelTaskSlot: strconv.FormatInt(int64(s.Slot), 10),
}
if s.Status.ContainerStatus != nil {
commonLabels[swarmLabelTaskContainerID] = s.Status.ContainerStatus.ContainerID
}
for k, v := range s.Labels {
ln := strutil.SanitizeLabelName(k)
commonLabels[swarmLabelTaskLabelPrefix+ln] = v
}
for k, v := range serviceLabels[s.ServiceID] {
commonLabels[k] = v
}
for k, v := range nodeLabels[s.NodeID] {
commonLabels[k] = v
}
for _, p := range s.Status.PortStatus.Ports {
if p.Protocol != swarm.PortConfigProtocolTCP {
continue
}
labels := model.LabelSet{
swarmLabelTaskPortMode: model.LabelValue(p.PublishMode),
}
for k, v := range commonLabels {
labels[model.LabelName(k)] = model.LabelValue(v)
}
addr := net.JoinHostPort(string(labels[swarmLabelNodeAddress]), strconv.FormatUint(uint64(p.PublishedPort), 10))
labels[model.AddressLabel] = model.LabelValue(addr)
tg.Targets = append(tg.Targets, labels)
}
for _, network := range s.NetworksAttachments {
for _, address := range network.Addresses {
var added bool
ip, _, err := net.ParseCIDR(address)
if err != nil {
return nil, fmt.Errorf("error while parsing address %s: %w", address, err)
}
for _, p := range servicePorts[s.ServiceID] {
if p.Protocol != swarm.PortConfigProtocolTCP {
continue
}
labels := model.LabelSet{
swarmLabelTaskPortMode: model.LabelValue(p.PublishMode),
}
for k, v := range commonLabels {
labels[model.LabelName(k)] = model.LabelValue(v)
}
for k, v := range networkLabels[network.Network.ID] {
labels[model.LabelName(k)] = model.LabelValue(v)
}
addr := net.JoinHostPort(ip.String(), strconv.FormatUint(uint64(p.PublishedPort), 10))
labels[model.AddressLabel] = model.LabelValue(addr)
tg.Targets = append(tg.Targets, labels)
added = true
}
if !added {
labels := model.LabelSet{}
for k, v := range commonLabels {
labels[model.LabelName(k)] = model.LabelValue(v)
}
for k, v := range networkLabels[network.Network.ID] {
labels[model.LabelName(k)] = model.LabelValue(v)
}
addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", d.port))
labels[model.AddressLabel] = model.LabelValue(addr)
tg.Targets = append(tg.Targets, labels)
}
}
}
}
return []*targetgroup.Group{tg}, nil
}

793
discovery/dockerswarm/tasks_test.go Normal file
View file

@ -0,0 +1,793 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dockerswarm
import (
"context"
"fmt"
"testing"
"github.com/go-kit/kit/log"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/util/testutil"
"gopkg.in/yaml.v2"
)
func TestDockerSwarmTasksSDRefresh(t *testing.T) {
sdmock := NewSDMock(t, "swarmprom")
sdmock.Setup()
e := sdmock.Endpoint()
url := e[:len(e)-1]
cfgString := fmt.Sprintf(`
---
role: tasks
host: %s
`, url)
var cfg SDConfig
testutil.Ok(t, yaml.Unmarshal([]byte(cfgString), &cfg))
d, err := NewDiscovery(&cfg, log.NewNopLogger())
testutil.Ok(t, err)
ctx := context.Background()
tgs, err := d.refresh(ctx)
testutil.Ok(t, err)
testutil.Equals(t, 1, len(tgs))
tg := tgs[0]
testutil.Assert(t, tg != nil, "tg should not be nil")
testutil.Assert(t, tg.Targets != nil, "tg.targets should not be nil")
testutil.Equals(t, 27, len(tg.Targets))
for i, lbls := range []model.LabelSet{
{
"__address__": model.LabelValue("10.0.0.8:9100"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.5"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("worker-2"),
"__meta_dockerswarm_node_id": model.LabelValue("ldawcom10uqi6owysgi28n4ve"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("worker"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("3qvd7bwfptmoht16t1f7jllb6"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_node-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("a137690f7070c536470aeae2d40cd4ecabe59842e5c2fce0cc7596f7d3d85c59"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("40wy6ekgchzz38dqjv07qt79w"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.3:9100"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.5"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("worker-2"),
"__meta_dockerswarm_node_id": model.LabelValue("ldawcom10uqi6owysgi28n4ve"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("worker"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("3qvd7bwfptmoht16t1f7jllb6"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_node-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("a137690f7070c536470aeae2d40cd4ecabe59842e5c2fce0cc7596f7d3d85c59"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("40wy6ekgchzz38dqjv07qt79w"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.88:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.2"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-2"),
"__meta_dockerswarm_node_id": model.LabelValue("i9woemzxymn1n98o9ufebclgm"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("iml8vdd2dudvq457sbirpf66z"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("google/cadvisor"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_cadvisor"),
"__meta_dockerswarm_task_container_id": model.LabelValue(""),
"__meta_dockerswarm_task_desired_state": model.LabelValue("ready"),
"__meta_dockerswarm_task_id": model.LabelValue("7ogolpkgw2d2amnht1fbtm9oq"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("rejected"),
},
{
"__address__": model.LabelValue("10.0.0.12:9100"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.4"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("worker-1"),
"__meta_dockerswarm_node_id": model.LabelValue("jkc2xd7p3xdazf0u6sh1m9dmr"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("worker"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("3qvd7bwfptmoht16t1f7jllb6"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_node-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("fe665e092d225bb42da4f9af3439f91c8379f37f1928d74d8f8ae8797f456737"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("b5sk2vnp3gj9j4kmcklzkmkt0"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.7:9100"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.4"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("worker-1"),
"__meta_dockerswarm_node_id": model.LabelValue("jkc2xd7p3xdazf0u6sh1m9dmr"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("worker"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("3qvd7bwfptmoht16t1f7jllb6"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_node-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("fe665e092d225bb42da4f9af3439f91c8379f37f1928d74d8f8ae8797f456737"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("b5sk2vnp3gj9j4kmcklzkmkt0"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.0.11:9100"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.3"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-3"),
"__meta_dockerswarm_node_id": model.LabelValue("bvtjl7pnrtg0k88ywialsldpd"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("3qvd7bwfptmoht16t1f7jllb6"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_node-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("85fcbc74e09fc86532773a757527fca6af60b81bfb4ef6962098c3ae46e0b0b9"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("daxcdhbv3oupklh04l4sxy005"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.6:9100"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.3"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-3"),
"__meta_dockerswarm_node_id": model.LabelValue("bvtjl7pnrtg0k88ywialsldpd"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("3qvd7bwfptmoht16t1f7jllb6"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_node-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("85fcbc74e09fc86532773a757527fca6af60b81bfb4ef6962098c3ae46e0b0b9"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("daxcdhbv3oupklh04l4sxy005"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.0.10:9100"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.2"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-2"),
"__meta_dockerswarm_node_id": model.LabelValue("i9woemzxymn1n98o9ufebclgm"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("3qvd7bwfptmoht16t1f7jllb6"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_node-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("f8e4457895c8ddf4723db4b129b55f904cf2ecde18dbf4a744c85470798813f9"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("fwifn031bokjisimj5q76jwdc"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.5:9100"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.2"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-2"),
"__meta_dockerswarm_node_id": model.LabelValue("i9woemzxymn1n98o9ufebclgm"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("3qvd7bwfptmoht16t1f7jllb6"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-node-exporter:v0.16.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_node-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("f8e4457895c8ddf4723db4b129b55f904cf2ecde18dbf4a744c85470798813f9"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("fwifn031bokjisimj5q76jwdc"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.35:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.4"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("worker-1"),
"__meta_dockerswarm_node_id": model.LabelValue("jkc2xd7p3xdazf0u6sh1m9dmr"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("worker"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("9bbq7j55tzzz85k2gg52x73rg"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("cloudflare/unsee:v0.8.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_unsee"),
"__meta_dockerswarm_task_container_id": model.LabelValue("a0734f9421e710b654ca9e67010cbb55c1d5f92c17cc9e6590ab61130a3587e0"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("i9skiw2n5jkjoq7gix2t9uzhy"),
"__meta_dockerswarm_task_slot": model.LabelValue("1"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.14:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.1"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("oxygen"),
"__meta_dockerswarm_node_id": model.LabelValue("d3cw2msquo0d71yn42qrnb0tu"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("hv645udwaaewyw7wuc6nspm68"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-prometheus:v2.5.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_prometheus"),
"__meta_dockerswarm_task_container_id": model.LabelValue("f35e071f1251951a09566a2231cb318a1a29e158a3c3d15ad68fd937184c56b6"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("irj35fr4j59f1kvi0xdin9oly"),
"__meta_dockerswarm_task_slot": model.LabelValue("1"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.20:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.1"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("oxygen"),
"__meta_dockerswarm_node_id": model.LabelValue("d3cw2msquo0d71yn42qrnb0tu"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("ul5qawv4s7f7msm7dtyfarw80"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_dockerd-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("3cee48a5b2b363576b255207aae2d1bd0d8872aa61c3ff3b6d180d78d672a943"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("k8x2806pevvxigagsxovjmur1"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.19:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.5"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("worker-2"),
"__meta_dockerswarm_node_id": model.LabelValue("ldawcom10uqi6owysgi28n4ve"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("worker"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("ul5qawv4s7f7msm7dtyfarw80"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_dockerd-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("d5fd335a33c1fca8fffa1ec17bb5010a6bb7fba7aa72fd2209c25be8f5d1b975"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("mita4y262u04g4c7zz7ns032a"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.18:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.4"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("worker-1"),
"__meta_dockerswarm_node_id": model.LabelValue("jkc2xd7p3xdazf0u6sh1m9dmr"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("worker"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("ul5qawv4s7f7msm7dtyfarw80"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_dockerd-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("ca72094e1ec0547e84aba227dd44446588172fa9ee273de9bf4e6040ff6937bd"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("nm3v4jifzhfkw3rz33gmnaq2z"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.75:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.5"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("worker-2"),
"__meta_dockerswarm_node_id": model.LabelValue("ldawcom10uqi6owysgi28n4ve"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("worker"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("iml8vdd2dudvq457sbirpf66z"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("google/cadvisor"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_cadvisor"),
"__meta_dockerswarm_task_container_id": model.LabelValue(""),
"__meta_dockerswarm_task_desired_state": model.LabelValue("shutdown"),
"__meta_dockerswarm_task_id": model.LabelValue("pxdv57mpvquv2zbt7faz3xux9"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("rejected"),
},
{
"__address__": model.LabelValue("10.0.0.19:9090"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.2"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-2"),
"__meta_dockerswarm_node_id": model.LabelValue("i9woemzxymn1n98o9ufebclgm"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_task_container_id": model.LabelValue("5d00e145527f14d1bb2b13601000f1de9c95fc8ec1ccc1ff8291cadc31232957"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("qtr16op0cz5pdbx4rylm7up4g"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("1"),
"__meta_dockerswarm_task_state": model.LabelValue("starting"),
},
{
"__address__": model.LabelValue("10.0.0.19:9093"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.2"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-2"),
"__meta_dockerswarm_node_id": model.LabelValue("i9woemzxymn1n98o9ufebclgm"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_task_container_id": model.LabelValue("5d00e145527f14d1bb2b13601000f1de9c95fc8ec1ccc1ff8291cadc31232957"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("qtr16op0cz5pdbx4rylm7up4g"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("1"),
"__meta_dockerswarm_task_state": model.LabelValue("starting"),
},
{
"__address__": model.LabelValue("10.0.0.19:9094"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.2"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-2"),
"__meta_dockerswarm_node_id": model.LabelValue("i9woemzxymn1n98o9ufebclgm"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_task_container_id": model.LabelValue("5d00e145527f14d1bb2b13601000f1de9c95fc8ec1ccc1ff8291cadc31232957"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("qtr16op0cz5pdbx4rylm7up4g"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("1"),
"__meta_dockerswarm_task_state": model.LabelValue("starting"),
},
{
"__address__": model.LabelValue("10.0.1.81:9090"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.2"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-2"),
"__meta_dockerswarm_node_id": model.LabelValue("i9woemzxymn1n98o9ufebclgm"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_task_container_id": model.LabelValue("5d00e145527f14d1bb2b13601000f1de9c95fc8ec1ccc1ff8291cadc31232957"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("qtr16op0cz5pdbx4rylm7up4g"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("1"),
"__meta_dockerswarm_task_state": model.LabelValue("starting"),
},
{
"__address__": model.LabelValue("10.0.1.81:9093"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.2"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-2"),
"__meta_dockerswarm_node_id": model.LabelValue("i9woemzxymn1n98o9ufebclgm"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_task_container_id": model.LabelValue("5d00e145527f14d1bb2b13601000f1de9c95fc8ec1ccc1ff8291cadc31232957"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("qtr16op0cz5pdbx4rylm7up4g"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("1"),
"__meta_dockerswarm_task_state": model.LabelValue("starting"),
},
{
"__address__": model.LabelValue("10.0.1.81:9094"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.2"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-2"),
"__meta_dockerswarm_node_id": model.LabelValue("i9woemzxymn1n98o9ufebclgm"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("tkv91uso46cck763aaqcb4rek"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_caddy"),
"__meta_dockerswarm_task_container_id": model.LabelValue("5d00e145527f14d1bb2b13601000f1de9c95fc8ec1ccc1ff8291cadc31232957"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("qtr16op0cz5pdbx4rylm7up4g"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("1"),
"__meta_dockerswarm_task_state": model.LabelValue("starting"),
},
{
"__address__": model.LabelValue("10.0.1.24:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.1"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("oxygen"),
"__meta_dockerswarm_node_id": model.LabelValue("d3cw2msquo0d71yn42qrnb0tu"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("iml8vdd2dudvq457sbirpf66z"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("google/cadvisor"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_cadvisor"),
"__meta_dockerswarm_task_container_id": model.LabelValue("81dd017b6d9f36d779a3217021111ffc77dcdbd5f26da86c2ae8fada86f33d17"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("s0rh3k9l7ii9vb62lsfp4wc93"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.32:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.3"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-3"),
"__meta_dockerswarm_node_id": model.LabelValue("bvtjl7pnrtg0k88ywialsldpd"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("rl4jhdws3r4lkfvq8kx6c4xnr"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-alertmanager:v0.14.0"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_alertmanager"),
"__meta_dockerswarm_task_container_id": model.LabelValue("2c753737f2ac7b01b577324513d9bd2a1754e65ab2a44714090eb5d83d2961b2"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("sxwkumr37wjaqzir89uepm4nc"),
"__meta_dockerswarm_task_slot": model.LabelValue("1"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.0.16:3000"),
"__meta_dockerswarm_network_id": model.LabelValue("qvwhwd6p61k4o0ulsknqb066z"),
"__meta_dockerswarm_network_ingress": model.LabelValue("true"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_name": model.LabelValue("ingress"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.3"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-3"),
"__meta_dockerswarm_node_id": model.LabelValue("bvtjl7pnrtg0k88ywialsldpd"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("uk9su5tb9ykfzew3qtmp14uzh"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-grafana:5.3.4"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_grafana"),
"__meta_dockerswarm_task_container_id": model.LabelValue("5b4fc187b14eeba86381b50fc2ae652e74d6cbea5acae0bda72eaf3db0f59f0c"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("tj683w0alx5hqaewks4rg0452"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("1"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.30:3000"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.3"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-3"),
"__meta_dockerswarm_node_id": model.LabelValue("bvtjl7pnrtg0k88ywialsldpd"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("uk9su5tb9ykfzew3qtmp14uzh"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/swarmprom-grafana:5.3.4"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("replicated"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_grafana"),
"__meta_dockerswarm_task_container_id": model.LabelValue("5b4fc187b14eeba86381b50fc2ae652e74d6cbea5acae0bda72eaf3db0f59f0c"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("tj683w0alx5hqaewks4rg0452"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("ingress"),
"__meta_dockerswarm_task_slot": model.LabelValue("1"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.232.3:82"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.3"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-3"),
"__meta_dockerswarm_node_id": model.LabelValue("bvtjl7pnrtg0k88ywialsldpd"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("ul5qawv4s7f7msm7dtyfarw80"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_dockerd-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("dd50c428ebc7a7bd786c7b0db873bf53ed5cbe5312995a76eb8fd325dcf16032"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("zhoo9feqxfyi9zg6acnt1eam2"),
"__meta_dockerswarm_task_port_publish_mode": model.LabelValue("host"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
{
"__address__": model.LabelValue("10.0.1.22:80"),
"__meta_dockerswarm_network_id": model.LabelValue("npq2closzy836m07eaq1425k3"),
"__meta_dockerswarm_network_ingress": model.LabelValue("false"),
"__meta_dockerswarm_network_internal": model.LabelValue("false"),
"__meta_dockerswarm_network_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_network_name": model.LabelValue("mon_net"),
"__meta_dockerswarm_network_scope": model.LabelValue("swarm"),
"__meta_dockerswarm_node_address": model.LabelValue("10.0.232.3"),
"__meta_dockerswarm_node_availability": model.LabelValue("active"),
"__meta_dockerswarm_node_hostname": model.LabelValue("master-3"),
"__meta_dockerswarm_node_id": model.LabelValue("bvtjl7pnrtg0k88ywialsldpd"),
"__meta_dockerswarm_node_platform_architecture": model.LabelValue("x86_64"),
"__meta_dockerswarm_node_platform_os": model.LabelValue("linux"),
"__meta_dockerswarm_node_role": model.LabelValue("manager"),
"__meta_dockerswarm_node_status": model.LabelValue("ready"),
"__meta_dockerswarm_service_id": model.LabelValue("ul5qawv4s7f7msm7dtyfarw80"),
"__meta_dockerswarm_service_label_com_docker_stack_image": model.LabelValue("stefanprodan/caddy"),
"__meta_dockerswarm_service_label_com_docker_stack_namespace": model.LabelValue("mon"),
"__meta_dockerswarm_service_mode": model.LabelValue("global"),
"__meta_dockerswarm_service_name": model.LabelValue("mon_dockerd-exporter"),
"__meta_dockerswarm_task_container_id": model.LabelValue("dd50c428ebc7a7bd786c7b0db873bf53ed5cbe5312995a76eb8fd325dcf16032"),
"__meta_dockerswarm_task_desired_state": model.LabelValue("running"),
"__meta_dockerswarm_task_id": model.LabelValue("zhoo9feqxfyi9zg6acnt1eam2"),
"__meta_dockerswarm_task_slot": model.LabelValue("0"),
"__meta_dockerswarm_task_state": model.LabelValue("running"),
},
} {
t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) {
testutil.Equals(t, lbls, tg.Targets[i])
})
}
}

1
discovery/dockerswarm/testdata/swarmprom/_ping vendored Normal file
View file

@ -0,0 +1 @@
OK

5
discovery/dockerswarm/testdata/swarmprom/headers.yml vendored Normal file
View file

@ -0,0 +1,5 @@
Ostype: linux
Server: Docker/19.03.5-ce (linux)
Docker-Experimental: "false"
Api-Version: "1.40"
Cache-Control: no-cache, no-store, must-revalidat

64
discovery/dockerswarm/testdata/swarmprom/networks.json vendored Normal file
View file

@ -0,0 +1,64 @@
[
{
"Name": "mon_net",
"Id": "npq2closzy836m07eaq1425k3",
"Created": "2020-06-20T22:08:52.666829786Z",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "10.0.1.0/24",
"Gateway": "10.0.1.1"
}
]
},
"Internal": false,
"Attachable": true,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": null,
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4097"
},
"Labels": {
"com.docker.stack.namespace": "mon"
}
},
{
"Name": "ingress",
"Id": "qvwhwd6p61k4o0ulsknqb066z",
"Created": "2020-06-20T20:50:07.413991571Z",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "10.0.0.0/24",
"Gateway": "10.0.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": true,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": null,
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4096"
},
"Labels": null
}
]

545
discovery/dockerswarm/testdata/swarmprom/nodes.json vendored Normal file
View file

@ -0,0 +1,545 @@
[
{
"ID": "bvtjl7pnrtg0k88ywialsldpd",
"Version": {
"Index": 46
},
"CreatedAt": "2020-06-19T19:14:12.712080369Z",
"UpdatedAt": "2020-06-19T19:14:29.328553007Z",
"Spec": {
"Labels": {},
"Role": "manager",
"Availability": "active"
},
"Description": {
"Hostname": "master-3",
"Platform": {
"Architecture": "x86_64",
"OS": "linux"
},
"Resources": {
"NanoCPUs": 8000000000,
"MemoryBytes": 33369206784
},
"Engine": {
"EngineVersion": "19.03.11",
"Plugins": [
{
"Type": "Log",
"Name": "awslogs"
},
{
"Type": "Log",
"Name": "fluentd"
},
{
"Type": "Log",
"Name": "gcplogs"
},
{
"Type": "Log",
"Name": "gelf"
},
{
"Type": "Log",
"Name": "journald"
},
{
"Type": "Log",
"Name": "json-file"
},
{
"Type": "Log",
"Name": "local"
},
{
"Type": "Log",
"Name": "logentries"
},
{
"Type": "Log",
"Name": "splunk"
},
{
"Type": "Log",
"Name": "syslog"
},
{
"Type": "Network",
"Name": "bridge"
},
{
"Type": "Network",
"Name": "host"
},
{
"Type": "Network",
"Name": "ipvlan"
},
{
"Type": "Network",
"Name": "macvlan"
},
{
"Type": "Network",
"Name": "null"
},
{
"Type": "Network",
"Name": "overlay"
},
{
"Type": "Volume",
"Name": "local"
}
]
},
"TLSInfo": {
"TrustRoot": "-----BEGIN CERTIFICATE-----\nMIIBazCCARCgAwIBAgIUCEGCo7PvZiKIKJ3Gun/cP+3jIygwCgYIKoZIzj0EAwIw\nEzERMA8GA1UEAxMIc3dhcm0tY2EwHhcNMjAwNjE5MTkwOTAwWhcNNDAwNjE0MTkw\nOTAwWjATMREwDwYDVQQDEwhzd2FybS1jYTBZMBMGByqGSM49AgEGCCqGSM49AwEH\nA0IABF6kgxvCNpsdGvv8CcrDzo13VByoCM+PeOFnxrFNHHEpD/shSBaTP1qvESnZ\n+F/GtPK0O06kJcd9HZSUoTblgfijQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB\nAf8EBTADAQH/MB0GA1UdDgQWBBTw+O5fRh6rhOfP94q0Rp0eJongYzAKBggqhkjO\nPQQDAgNJADBGAiEAxdjqkW94D2ZewZrid2g72aBhlX7XKsa86hx8/PmPt4wCIQDD\nNnFTPgihf1I71vI7JcnEIKePlodqH+w701jl/GvYnw==\n-----END CERTIFICATE-----\n",
"CertIssuerSubject": "MBMxETAPBgNVBAMTCHN3YXJtLWNh",
"CertIssuerPublicKey": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEXqSDG8I2mx0a+/wJysPOjXdUHKgIz4944WfGsU0ccSkP+yFIFpM/Wq8RKdn4X8a08rQ7TqQlx30dlJShNuWB+A=="
}
},
"Status": {
"State": "ready",
"Addr": "10.0.232.3"
},
"ManagerStatus": {
"Reachability": "reachable",
"Addr": "10.0.232.3:2377"
}
},
{
"ID": "d3cw2msquo0d71yn42qrnb0tu",
"Version": {
"Index": 44
},
"CreatedAt": "2020-06-19T19:14:11.450271647Z",
"UpdatedAt": "2020-06-19T19:14:29.314880869Z",
"Spec": {
"Labels": {},
"Role": "manager",
"Availability": "active"
},
"Description": {
"Hostname": "oxygen",
"Platform": {
"Architecture": "x86_64",
"OS": "linux"
},
"Resources": {
"NanoCPUs": 8000000000,
"MemoryBytes": 33369206784
},
"Engine": {
"EngineVersion": "19.03.5-ce",
"Plugins": [
{
"Type": "Log",
"Name": "awslogs"
},
{
"Type": "Log",
"Name": "fluentd"
},
{
"Type": "Log",
"Name": "gcplogs"
},
{
"Type": "Log",
"Name": "gelf"
},
{
"Type": "Log",
"Name": "journald"
},
{
"Type": "Log",
"Name": "json-file"
},
{
"Type": "Log",
"Name": "local"
},
{
"Type": "Log",
"Name": "logentries"
},
{
"Type": "Log",
"Name": "splunk"
},
{
"Type": "Log",
"Name": "syslog"
},
{
"Type": "Network",
"Name": "bridge"
},
{
"Type": "Network",
"Name": "host"
},
{
"Type": "Network",
"Name": "ipvlan"
},
{
"Type": "Network",
"Name": "macvlan"
},
{
"Type": "Network",
"Name": "null"
},
{
"Type": "Network",
"Name": "overlay"
},
{
"Type": "Volume",
"Name": "local"
}
]
},
"TLSInfo": {
"TrustRoot": "-----BEGIN CERTIFICATE-----\nMIIBazCCARCgAwIBAgIUCEGCo7PvZiKIKJ3Gun/cP+3jIygwCgYIKoZIzj0EAwIw\nEzERMA8GA1UEAxMIc3dhcm0tY2EwHhcNMjAwNjE5MTkwOTAwWhcNNDAwNjE0MTkw\nOTAwWjATMREwDwYDVQQDEwhzd2FybS1jYTBZMBMGByqGSM49AgEGCCqGSM49AwEH\nA0IABF6kgxvCNpsdGvv8CcrDzo13VByoCM+PeOFnxrFNHHEpD/shSBaTP1qvESnZ\n+F/GtPK0O06kJcd9HZSUoTblgfijQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB\nAf8EBTADAQH/MB0GA1UdDgQWBBTw+O5fRh6rhOfP94q0Rp0eJongYzAKBggqhkjO\nPQQDAgNJADBGAiEAxdjqkW94D2ZewZrid2g72aBhlX7XKsa86hx8/PmPt4wCIQDD\nNnFTPgihf1I71vI7JcnEIKePlodqH+w701jl/GvYnw==\n-----END CERTIFICATE-----\n",
"CertIssuerSubject": "MBMxETAPBgNVBAMTCHN3YXJtLWNh",
"CertIssuerPublicKey": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEXqSDG8I2mx0a+/wJysPOjXdUHKgIz4944WfGsU0ccSkP+yFIFpM/Wq8RKdn4X8a08rQ7TqQlx30dlJShNuWB+A=="
}
},
"Status": {
"State": "ready",
"Addr": "10.0.232.1"
},
"ManagerStatus": {
"Leader": true,
"Reachability": "reachable",
"Addr": "10.0.232.1:2377"
}
},
{
"ID": "i9woemzxymn1n98o9ufebclgm",
"Version": {
"Index": 45
},
"CreatedAt": "2020-06-19T19:14:12.087516843Z",
"UpdatedAt": "2020-06-19T19:14:29.320036062Z",
"Spec": {
"Labels": {},
"Role": "manager",
"Availability": "active"
},
"Description": {
"Hostname": "master-2",
"Platform": {
"Architecture": "x86_64",
"OS": "linux"
},
"Resources": {
"NanoCPUs": 8000000000,
"MemoryBytes": 33369206784
},
"Engine": {
"EngineVersion": "19.03.11",
"Plugins": [
{
"Type": "Log",
"Name": "awslogs"
},
{
"Type": "Log",
"Name": "fluentd"
},
{
"Type": "Log",
"Name": "gcplogs"
},
{
"Type": "Log",
"Name": "gelf"
},
{
"Type": "Log",
"Name": "journald"
},
{
"Type": "Log",
"Name": "json-file"
},
{
"Type": "Log",
"Name": "local"
},
{
"Type": "Log",
"Name": "logentries"
},
{
"Type": "Log",
"Name": "splunk"
},
{
"Type": "Log",
"Name": "syslog"
},
{
"Type": "Network",
"Name": "bridge"
},
{
"Type": "Network",
"Name": "host"
},
{
"Type": "Network",
"Name": "ipvlan"
},
{
"Type": "Network",
"Name": "macvlan"
},
{
"Type": "Network",
"Name": "null"
},
{
"Type": "Network",
"Name": "overlay"
},
{
"Type": "Volume",
"Name": "local"
}
]
},
"TLSInfo": {
"TrustRoot": "-----BEGIN CERTIFICATE-----\nMIIBazCCARCgAwIBAgIUCEGCo7PvZiKIKJ3Gun/cP+3jIygwCgYIKoZIzj0EAwIw\nEzERMA8GA1UEAxMIc3dhcm0tY2EwHhcNMjAwNjE5MTkwOTAwWhcNNDAwNjE0MTkw\nOTAwWjATMREwDwYDVQQDEwhzd2FybS1jYTBZMBMGByqGSM49AgEGCCqGSM49AwEH\nA0IABF6kgxvCNpsdGvv8CcrDzo13VByoCM+PeOFnxrFNHHEpD/shSBaTP1qvESnZ\n+F/GtPK0O06kJcd9HZSUoTblgfijQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB\nAf8EBTADAQH/MB0GA1UdDgQWBBTw+O5fRh6rhOfP94q0Rp0eJongYzAKBggqhkjO\nPQQDAgNJADBGAiEAxdjqkW94D2ZewZrid2g72aBhlX7XKsa86hx8/PmPt4wCIQDD\nNnFTPgihf1I71vI7JcnEIKePlodqH+w701jl/GvYnw==\n-----END CERTIFICATE-----\n",
"CertIssuerSubject": "MBMxETAPBgNVBAMTCHN3YXJtLWNh",
"CertIssuerPublicKey": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEXqSDG8I2mx0a+/wJysPOjXdUHKgIz4944WfGsU0ccSkP+yFIFpM/Wq8RKdn4X8a08rQ7TqQlx30dlJShNuWB+A=="
}
},
"Status": {
"State": "ready",
"Addr": "10.0.232.2"
},
"ManagerStatus": {
"Reachability": "reachable",
"Addr": "10.0.232.2:2377"
}
},
{
"ID": "jkc2xd7p3xdazf0u6sh1m9dmr",
"Version": {
"Index": 47
},
"CreatedAt": "2020-06-19T19:14:13.329602795Z",
"UpdatedAt": "2020-06-19T19:14:29.337031695Z",
"Spec": {
"Labels": {},
"Role": "worker",
"Availability": "active"
},
"Description": {
"Hostname": "worker-1",
"Platform": {
"Architecture": "x86_64",
"OS": "linux"
},
"Resources": {
"NanoCPUs": 8000000000,
"MemoryBytes": 33369206784
},
"Engine": {
"EngineVersion": "19.03.11",
"Plugins": [
{
"Type": "Log",
"Name": "awslogs"
},
{
"Type": "Log",
"Name": "fluentd"
},
{
"Type": "Log",
"Name": "gcplogs"
},
{
"Type": "Log",
"Name": "gelf"
},
{
"Type": "Log",
"Name": "journald"
},
{
"Type": "Log",
"Name": "json-file"
},
{
"Type": "Log",
"Name": "local"
},
{
"Type": "Log",
"Name": "logentries"
},
{
"Type": "Log",
"Name": "splunk"
},
{
"Type": "Log",
"Name": "syslog"
},
{
"Type": "Network",
"Name": "bridge"
},
{
"Type": "Network",
"Name": "host"
},
{
"Type": "Network",
"Name": "ipvlan"
},
{
"Type": "Network",
"Name": "macvlan"
},
{
"Type": "Network",
"Name": "null"
},
{
"Type": "Network",
"Name": "overlay"
},
{
"Type": "Volume",
"Name": "local"
}
]
},
"TLSInfo": {
"TrustRoot": "-----BEGIN CERTIFICATE-----\nMIIBazCCARCgAwIBAgIUCEGCo7PvZiKIKJ3Gun/cP+3jIygwCgYIKoZIzj0EAwIw\nEzERMA8GA1UEAxMIc3dhcm0tY2EwHhcNMjAwNjE5MTkwOTAwWhcNNDAwNjE0MTkw\nOTAwWjATMREwDwYDVQQDEwhzd2FybS1jYTBZMBMGByqGSM49AgEGCCqGSM49AwEH\nA0IABF6kgxvCNpsdGvv8CcrDzo13VByoCM+PeOFnxrFNHHEpD/shSBaTP1qvESnZ\n+F/GtPK0O06kJcd9HZSUoTblgfijQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB\nAf8EBTADAQH/MB0GA1UdDgQWBBTw+O5fRh6rhOfP94q0Rp0eJongYzAKBggqhkjO\nPQQDAgNJADBGAiEAxdjqkW94D2ZewZrid2g72aBhlX7XKsa86hx8/PmPt4wCIQDD\nNnFTPgihf1I71vI7JcnEIKePlodqH+w701jl/GvYnw==\n-----END CERTIFICATE-----\n",
"CertIssuerSubject": "MBMxETAPBgNVBAMTCHN3YXJtLWNh",
"CertIssuerPublicKey": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEXqSDG8I2mx0a+/wJysPOjXdUHKgIz4944WfGsU0ccSkP+yFIFpM/Wq8RKdn4X8a08rQ7TqQlx30dlJShNuWB+A=="
}
},
"Status": {
"State": "ready",
"Addr": "10.0.232.4"
}
},
{
"ID": "ldawcom10uqi6owysgi28n4ve",
"Version": {
"Index": 43
},
"CreatedAt": "2020-06-19T19:14:13.444872098Z",
"UpdatedAt": "2020-06-19T19:14:29.306716244Z",
"Spec": {
"Labels": {},
"Role": "worker",
"Availability": "active"
},
"Description": {
"Hostname": "worker-2",
"Platform": {
"Architecture": "x86_64",
"OS": "linux"
},
"Resources": {
"NanoCPUs": 8000000000,
"MemoryBytes": 33369206784
},
"Engine": {
"EngineVersion": "19.03.11",
"Plugins": [
{
"Type": "Log",
"Name": "awslogs"
},
{
"Type": "Log",
"Name": "fluentd"
},
{
"Type": "Log",
"Name": "gcplogs"
},
{
"Type": "Log",
"Name": "gelf"
},
{
"Type": "Log",
"Name": "journald"
},
{
"Type": "Log",
"Name": "json-file"
},
{
"Type": "Log",
"Name": "local"
},
{
"Type": "Log",
"Name": "logentries"
},
{
"Type": "Log",
"Name": "splunk"
},
{
"Type": "Log",
"Name": "syslog"
},
{
"Type": "Network",
"Name": "bridge"
},
{
"Type": "Network",
"Name": "host"
},
{
"Type": "Network",
"Name": "ipvlan"
},
{
"Type": "Network",
"Name": "macvlan"
},
{
"Type": "Network",
"Name": "null"
},
{
"Type": "Network",
"Name": "overlay"
},
{
"Type": "Volume",
"Name": "local"
}
]
},
"TLSInfo": {
"TrustRoot": "-----BEGIN CERTIFICATE-----\nMIIBazCCARCgAwIBAgIUCEGCo7PvZiKIKJ3Gun/cP+3jIygwCgYIKoZIzj0EAwIw\nEzERMA8GA1UEAxMIc3dhcm0tY2EwHhcNMjAwNjE5MTkwOTAwWhcNNDAwNjE0MTkw\nOTAwWjATMREwDwYDVQQDEwhzd2FybS1jYTBZMBMGByqGSM49AgEGCCqGSM49AwEH\nA0IABF6kgxvCNpsdGvv8CcrDzo13VByoCM+PeOFnxrFNHHEpD/shSBaTP1qvESnZ\n+F/GtPK0O06kJcd9HZSUoTblgfijQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB\nAf8EBTADAQH/MB0GA1UdDgQWBBTw+O5fRh6rhOfP94q0Rp0eJongYzAKBggqhkjO\nPQQDAgNJADBGAiEAxdjqkW94D2ZewZrid2g72aBhlX7XKsa86hx8/PmPt4wCIQDD\nNnFTPgihf1I71vI7JcnEIKePlodqH+w701jl/GvYnw==\n-----END CERTIFICATE-----\n",
"CertIssuerSubject": "MBMxETAPBgNVBAMTCHN3YXJtLWNh",
"CertIssuerPublicKey": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEXqSDG8I2mx0a+/wJysPOjXdUHKgIz4944WfGsU0ccSkP+yFIFpM/Wq8RKdn4X8a08rQ7TqQlx30dlJShNuWB+A=="
}
},
"Status": {
"State": "ready",
"Addr": "10.0.232.5"
}
}
]

891
discovery/dockerswarm/testdata/swarmprom/services.json vendored Normal file
View file

@ -0,0 +1,891 @@
[
{
"ID": "3qvd7bwfptmoht16t1f7jllb6",
"Version": {
"Index": 41
},
"CreatedAt": "2020-06-19T19:14:29.269637662Z",
"UpdatedAt": "2020-06-19T19:14:29.288101671Z",
"Spec": {
"Name": "mon_node-exporter",
"Labels": {
"com.docker.stack.image": "stefanprodan/swarmprom-node-exporter:v0.16.0",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "stefanprodan/swarmprom-node-exporter:v0.16.0@sha256:1d2518ec0501dd848e718bf008df37852b1448c862d6f256f2d39244975758d6",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Args": [
"--path.sysfs=/host/sys",
"--path.procfs=/host/proc",
"--collector.textfile.directory=/etc/node-exporter/",
"--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($|/)",
"--no-collector.ipvs"
],
"Env": [
"NODE_ID={{.Node.ID}}"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Mounts": [
{
"Type": "bind",
"Source": "/proc",
"Target": "/host/proc",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/sys",
"Target": "/host/sys",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/",
"Target": "/rootfs",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/etc/hostname",
"Target": "/etc/nodename"
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 134217728
},
"Reservations": {
"MemoryBytes": 67108864
}
},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"node-exporter"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Global": {}
},
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 9100,
"PublishedPort": 9100,
"PublishMode": "ingress"
}
]
}
},
"Endpoint": {
"Spec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 9100,
"PublishedPort": 9100,
"PublishMode": "ingress"
}
]
},
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 9100,
"PublishedPort": 9100,
"PublishMode": "ingress"
}
],
"VirtualIPs": [
{
"NetworkID": "qvwhwd6p61k4o0ulsknqb066z",
"Addr": "10.0.0.7/24"
},
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.2/24"
}
]
}
},
{
"ID": "9bbq7j55tzzz85k2gg52x73rg",
"Version": {
"Index": 116
},
"CreatedAt": "2020-06-19T19:15:16.676178796Z",
"UpdatedAt": "2020-06-19T19:15:16.690701679Z",
"Spec": {
"Name": "mon_unsee",
"Labels": {
"com.docker.stack.image": "cloudflare/unsee:v0.8.0",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "cloudflare/unsee:v0.8.0@sha256:28398f47f63feb1647887999701fa730da351fc6d3cc7228e5da44b40a663ada",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Env": [
"ALERTMANAGER_URIS=default:http://alertmanager:9093"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Isolation": "default"
},
"Resources": {},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"unsee"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"EndpointSpec": {
"Mode": "vip"
}
},
"Endpoint": {
"Spec": {
"Mode": "vip"
},
"VirtualIPs": [
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.34/24"
}
]
}
},
{
"ID": "hv645udwaaewyw7wuc6nspm68",
"Version": {
"Index": 54
},
"CreatedAt": "2020-06-19T19:14:31.707250955Z",
"UpdatedAt": "2020-06-19T19:14:31.713627185Z",
"Spec": {
"Name": "mon_prometheus",
"Labels": {
"com.docker.stack.image": "stefanprodan/swarmprom-prometheus:v2.5.0",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "stefanprodan/swarmprom-prometheus:v2.5.0@sha256:f1a3781c4785637ba088dcf54889f77d9b6b69f21b2c0a167c1347473f4e2587",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Args": [
"--config.file=/etc/prometheus/prometheus.yml",
"--storage.tsdb.path=/prometheus",
"--storage.tsdb.retention=24h"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Mounts": [
{
"Type": "volume",
"Source": "mon_prometheus",
"Target": "/prometheus",
"VolumeOptions": {
"Labels": {
"com.docker.stack.namespace": "mon"
}
}
}
],
"Configs": [
{
"File": {
"Name": "/etc/prometheus/swarm_node.rules.yml",
"UID": "0",
"GID": "0",
"Mode": 292
},
"ConfigID": "zffindunvquarrlgdrklzczbz",
"ConfigName": "mon_node_rules"
},
{
"File": {
"Name": "/etc/prometheus/swarm_task.rules.yml",
"UID": "0",
"GID": "0",
"Mode": 292
},
"ConfigID": "fbhk5cmleeicn5fkzujm909k7",
"ConfigName": "mon_task_rules"
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 2147483648
},
"Reservations": {
"MemoryBytes": 134217728
}
},
"Placement": {
"Constraints": [
"node.role == manager"
],
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"prometheus"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"EndpointSpec": {
"Mode": "vip"
}
},
"Endpoint": {
"Spec": {
"Mode": "vip"
},
"VirtualIPs": [
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.13/24"
}
]
}
},
{
"ID": "iml8vdd2dudvq457sbirpf66z",
"Version": {
"Index": 83
},
"CreatedAt": "2020-06-19T19:14:49.165966453Z",
"UpdatedAt": "2020-06-19T19:14:49.1722461Z",
"Spec": {
"Name": "mon_cadvisor",
"Labels": {
"com.docker.stack.image": "google/cadvisor",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "google/cadvisor:latest@sha256:815386ebbe9a3490f38785ab11bda34ec8dacf4634af77b8912832d4f85dca04",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Args": [
"-logtostderr",
"-docker_only"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Mounts": [
{
"Type": "bind",
"Source": "/var/run/docker.sock",
"Target": "/var/run/docker.sock",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/",
"Target": "/rootfs",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/var/run",
"Target": "/var/run"
},
{
"Type": "bind",
"Source": "/sys",
"Target": "/sys",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/var/lib/docker/",
"Target": "/var/lib/docker",
"ReadOnly": true
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 134217728
},
"Reservations": {
"MemoryBytes": 67108864
}
},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"cadvisor"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Global": {}
},
"EndpointSpec": {
"Mode": "vip"
}
},
"Endpoint": {
"Spec": {
"Mode": "vip"
},
"VirtualIPs": [
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.23/24"
}
]
}
},
{
"ID": "rl4jhdws3r4lkfvq8kx6c4xnr",
"Version": {
"Index": 104
},
"CreatedAt": "2020-06-19T19:15:09.840775581Z",
"UpdatedAt": "2020-06-19T19:15:09.854119631Z",
"Spec": {
"Name": "mon_alertmanager",
"Labels": {
"com.docker.stack.image": "stefanprodan/swarmprom-alertmanager:v0.14.0",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "stefanprodan/swarmprom-alertmanager:v0.14.0@sha256:7069b656bd5df0606ff1db81a7553a25dc72be51d3aca6a7e08d776566cefbd8",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Args": [
"--config.file=/etc/alertmanager/alertmanager.yml",
"--storage.path=/alertmanager"
],
"Env": [
"SLACK_CHANNEL=devops-alerts",
"SLACK_URL=https://hooks.slack.com/services/TOKEN",
"SLACK_USER=alertmanager"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Mounts": [
{
"Type": "volume",
"Source": "mon_alertmanager",
"Target": "/alertmanager",
"VolumeOptions": {
"Labels": {
"com.docker.stack.namespace": "mon"
}
}
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 134217728
},
"Reservations": {
"MemoryBytes": 67108864
}
},
"Placement": {
"Constraints": [
"node.role == manager"
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"alertmanager"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"EndpointSpec": {
"Mode": "vip"
}
},
"Endpoint": {
"Spec": {
"Mode": "vip"
},
"VirtualIPs": [
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.31/24"
}
]
}
},
{
"ID": "tkv91uso46cck763aaqcb4rek",
"Version": {
"Index": 68
},
"CreatedAt": "2020-06-19T19:14:41.685703164Z",
"UpdatedAt": "2020-06-19T19:14:41.696259297Z",
"Spec": {
"Name": "mon_caddy",
"Labels": {
"com.docker.stack.image": "stefanprodan/caddy",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "stefanprodan/caddy:latest@sha256:44541cfacb66f4799f81f17fcfb3cb757ccc8f327592745549f5930c42d115c9",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Env": [
"ADMIN_PASSWORD=admin",
"ADMIN_USER=admin"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Healthcheck": {
"Test": [
"CMD",
"curl",
"-f",
"http://localhost:3000"
],
"Interval": 5000000000,
"Timeout": 1000000000,
"Retries": 5
},
"Configs": [
{
"File": {
"Name": "/etc/caddy/Caddyfile",
"UID": "0",
"GID": "0",
"Mode": 292
},
"ConfigID": "pkjteflcny7cc633glg3m6ck2",
"ConfigName": "mon_caddy_config"
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 134217728
},
"Reservations": {
"MemoryBytes": 67108864
}
},
"Placement": {
"Constraints": [
"node.role == manager"
],
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"caddy"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 9090,
"PublishedPort": 9090,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 9093,
"PublishedPort": 9093,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 9094,
"PublishedPort": 9094,
"PublishMode": "ingress"
}
]
}
},
"Endpoint": {
"Spec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 9090,
"PublishedPort": 9090,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 9093,
"PublishedPort": 9093,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 9094,
"PublishedPort": 9094,
"PublishMode": "ingress"
}
]
},
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 9090,
"PublishedPort": 9090,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 9093,
"PublishedPort": 9093,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 9094,
"PublishedPort": 9094,
"PublishMode": "ingress"
}
],
"VirtualIPs": [
{
"NetworkID": "qvwhwd6p61k4o0ulsknqb066z",
"Addr": "10.0.0.13/24"
},
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.15/24"
}
]
}
},
{
"ID": "uk9su5tb9ykfzew3qtmp14uzh",
"Version": {
"Index": 92
},
"CreatedAt": "2020-06-19T19:14:54.28743674Z",
"UpdatedAt": "2020-06-19T19:14:54.301718496Z",
"Spec": {
"Name": "mon_grafana",
"Labels": {
"com.docker.stack.image": "stefanprodan/swarmprom-grafana:5.3.4",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "stefanprodan/swarmprom-grafana:5.3.4@sha256:2aca8aa5716e6e0eed3fcdc88fec256a0a1828c491a8cf240486ae7cc473092d",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Env": [
"GF_SECURITY_ADMIN_PASSWORD=admin",
"GF_SECURITY_ADMIN_USER=admin",
"GF_USERS_ALLOW_SIGN_UP=false"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Mounts": [
{
"Type": "volume",
"Source": "mon_grafana",
"Target": "/var/lib/grafana",
"VolumeOptions": {
"Labels": {
"com.docker.stack.namespace": "mon"
}
}
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 134217728
},
"Reservations": {
"MemoryBytes": 67108864
}
},
"Placement": {
"Constraints": [
"node.role == manager"
],
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"grafana"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 3000,
"PublishedPort": 3000,
"PublishMode": "ingress"
}
]
}
},
"Endpoint": {
"Spec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 3000,
"PublishedPort": 3000,
"PublishMode": "ingress"
}
]
},
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 3000,
"PublishedPort": 3000,
"PublishMode": "ingress"
}
],
"VirtualIPs": [
{
"NetworkID": "qvwhwd6p61k4o0ulsknqb066z",
"Addr": "10.0.0.15/24"
},
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.29/24"
}
]
}
},
{
"ID": "ul5qawv4s7f7msm7dtyfarw80",
"Version": {
"Index": 74
},
"CreatedAt": "2020-06-19T19:14:45.311963014Z",
"UpdatedAt": "2020-06-19T19:14:45.327315101Z",
"Spec": {
"Name": "mon_dockerd-exporter",
"Labels": {
"com.docker.stack.image": "stefanprodan/caddy",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "stefanprodan/caddy:latest@sha256:44541cfacb66f4799f81f17fcfb3cb757ccc8f327592745549f5930c42d115c9",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Env": [
"DOCKER_GWBRIDGE_IP=172.18.0.1"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Configs": [
{
"File": {
"Name": "/etc/caddy/Caddyfile",
"UID": "0",
"GID": "0",
"Mode": 292
},
"ConfigID": "4glhzmw6fndnkjn2cok90a9gu",
"ConfigName": "mon_dockerd_config"
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 134217728
},
"Reservations": {
"MemoryBytes": 67108864
}
},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"dockerd-exporter"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Global": {}
},
"EndpointSpec": {
"Mode": "vip"
}
},
"Endpoint": {
"Spec": {
"Mode": "vip"
},
"VirtualIPs": [
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.17/24"
}
]
}
}
]

724
discovery/dockerswarm/testdata/swarmprom/services__wvKVCQ4HhD vendored Normal file
View file

@ -0,0 +1,724 @@
[
{
"ID": "3qvd7bwfptmoht16t1f7jllb6",
"Version": {
"Index": 41
},
"CreatedAt": "2020-06-19T19:14:29.269637662Z",
"UpdatedAt": "2020-06-19T19:14:29.288101671Z",
"Spec": {
"Name": "mon_node-exporter",
"Labels": {
"com.docker.stack.image": "stefanprodan/swarmprom-node-exporter:v0.16.0",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "stefanprodan/swarmprom-node-exporter:v0.16.0@sha256:1d2518ec0501dd848e718bf008df37852b1448c862d6f256f2d39244975758d6",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Args": [
"--path.sysfs=/host/sys",
"--path.procfs=/host/proc",
"--collector.textfile.directory=/etc/node-exporter/",
"--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($|/)",
"--no-collector.ipvs"
],
"Env": [
"NODE_ID={{.Node.ID}}"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Mounts": [
{
"Type": "bind",
"Source": "/proc",
"Target": "/host/proc",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/sys",
"Target": "/host/sys",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/",
"Target": "/rootfs",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/etc/hostname",
"Target": "/etc/nodename"
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 134217728
},
"Reservations": {
"MemoryBytes": 67108864
}
},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"node-exporter"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Global": {}
},
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 9100,
"PublishedPort": 9100,
"PublishMode": "ingress"
}
]
}
},
"Endpoint": {
"Spec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 9100,
"PublishedPort": 9100,
"PublishMode": "ingress"
}
]
},
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 9100,
"PublishedPort": 9100,
"PublishMode": "ingress"
}
],
"VirtualIPs": [
{
"NetworkID": "qvwhwd6p61k4o0ulsknqb066z",
"Addr": "10.0.0.7/24"
},
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.2/24"
}
]
}
},
{
"ID": "9bbq7j55tzzz85k2gg52x73rg",
"Version": {
"Index": 116
},
"CreatedAt": "2020-06-19T19:15:16.676178796Z",
"UpdatedAt": "2020-06-19T19:15:16.690701679Z",
"Spec": {
"Name": "mon_unsee",
"Labels": {
"com.docker.stack.image": "cloudflare/unsee:v0.8.0",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "cloudflare/unsee:v0.8.0@sha256:28398f47f63feb1647887999701fa730da351fc6d3cc7228e5da44b40a663ada",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Env": [
"ALERTMANAGER_URIS=default:http://alertmanager:9093"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Isolation": "default"
},
"Resources": {},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"unsee"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"EndpointSpec": {
"Mode": "vip"
}
},
"Endpoint": {
"Spec": {
"Mode": "vip"
},
"VirtualIPs": [
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.34/24"
}
]
}
},
{
"ID": "hv645udwaaewyw7wuc6nspm68",
"Version": {
"Index": 54
},
"CreatedAt": "2020-06-19T19:14:31.707250955Z",
"UpdatedAt": "2020-06-19T19:14:31.713627185Z",
"Spec": {
"Name": "mon_prometheus",
"Labels": {
"com.docker.stack.image": "stefanprodan/swarmprom-prometheus:v2.5.0",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "stefanprodan/swarmprom-prometheus:v2.5.0@sha256:f1a3781c4785637ba088dcf54889f77d9b6b69f21b2c0a167c1347473f4e2587",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Args": [
"--config.file=/etc/prometheus/prometheus.yml",
"--storage.tsdb.path=/prometheus",
"--storage.tsdb.retention=24h"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Mounts": [
{
"Type": "volume",
"Source": "mon_prometheus",
"Target": "/prometheus",
"VolumeOptions": {
"Labels": {
"com.docker.stack.namespace": "mon"
}
}
}
],
"Configs": [
{
"File": {
"Name": "/etc/prometheus/swarm_node.rules.yml",
"UID": "0",
"GID": "0",
"Mode": 292
},
"ConfigID": "zffindunvquarrlgdrklzczbz",
"ConfigName": "mon_node_rules"
},
{
"File": {
"Name": "/etc/prometheus/swarm_task.rules.yml",
"UID": "0",
"GID": "0",
"Mode": 292
},
"ConfigID": "fbhk5cmleeicn5fkzujm909k7",
"ConfigName": "mon_task_rules"
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 2147483648
},
"Reservations": {
"MemoryBytes": 134217728
}
},
"Placement": {
"Constraints": [
"node.role == manager"
],
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"prometheus"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"EndpointSpec": {
"Mode": "vip"
}
},
"Endpoint": {
"Spec": {
"Mode": "vip"
},
"VirtualIPs": [
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.13/24"
}
]
}
},
{
"ID": "iml8vdd2dudvq457sbirpf66z",
"Version": {
"Index": 83
},
"CreatedAt": "2020-06-19T19:14:49.165966453Z",
"UpdatedAt": "2020-06-19T19:14:49.1722461Z",
"Spec": {
"Name": "mon_cadvisor",
"Labels": {
"com.docker.stack.image": "google/cadvisor",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "google/cadvisor:latest@sha256:815386ebbe9a3490f38785ab11bda34ec8dacf4634af77b8912832d4f85dca04",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Args": [
"-logtostderr",
"-docker_only"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Mounts": [
{
"Type": "bind",
"Source": "/var/run/docker.sock",
"Target": "/var/run/docker.sock",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/",
"Target": "/rootfs",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/var/run",
"Target": "/var/run"
},
{
"Type": "bind",
"Source": "/sys",
"Target": "/sys",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/var/lib/docker/",
"Target": "/var/lib/docker",
"ReadOnly": true
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 134217728
},
"Reservations": {
"MemoryBytes": 67108864
}
},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"cadvisor"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Global": {}
},
"EndpointSpec": {
"Mode": "vip"
}
},
"Endpoint": {
"Spec": {
"Mode": "vip"
},
"VirtualIPs": [
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.23/24"
}
]
}
},
{
"ID": "rl4jhdws3r4lkfvq8kx6c4xnr",
"Version": {
"Index": 104
},
"CreatedAt": "2020-06-19T19:15:09.840775581Z",
"UpdatedAt": "2020-06-19T19:15:09.854119631Z",
"Spec": {
"Name": "mon_alertmanager",
"Labels": {
"com.docker.stack.image": "stefanprodan/swarmprom-alertmanager:v0.14.0",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "stefanprodan/swarmprom-alertmanager:v0.14.0@sha256:7069b656bd5df0606ff1db81a7553a25dc72be51d3aca6a7e08d776566cefbd8",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Args": [
"--config.file=/etc/alertmanager/alertmanager.yml",
"--storage.path=/alertmanager"
],
"Env": [
"SLACK_CHANNEL=devops-alerts",
"SLACK_URL=https://hooks.slack.com/services/TOKEN",
"SLACK_USER=alertmanager"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Mounts": [
{
"Type": "volume",
"Source": "mon_alertmanager",
"Target": "/alertmanager",
"VolumeOptions": {
"Labels": {
"com.docker.stack.namespace": "mon"
}
}
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 134217728
},
"Reservations": {
"MemoryBytes": 67108864
}
},
"Placement": {
"Constraints": [
"node.role == manager"
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"alertmanager"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"EndpointSpec": {
"Mode": "vip"
}
},
"Endpoint": {
"Spec": {
"Mode": "vip"
},
"VirtualIPs": [
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.31/24"
}
]
}
},
{
"ID": "uk9su5tb9ykfzew3qtmp14uzh",
"Version": {
"Index": 92
},
"CreatedAt": "2020-06-19T19:14:54.28743674Z",
"UpdatedAt": "2020-06-19T19:14:54.301718496Z",
"Spec": {
"Name": "mon_grafana",
"Labels": {
"com.docker.stack.image": "stefanprodan/swarmprom-grafana:5.3.4",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "stefanprodan/swarmprom-grafana:5.3.4@sha256:2aca8aa5716e6e0eed3fcdc88fec256a0a1828c491a8cf240486ae7cc473092d",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Env": [
"GF_SECURITY_ADMIN_PASSWORD=admin",
"GF_SECURITY_ADMIN_USER=admin",
"GF_USERS_ALLOW_SIGN_UP=false"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Mounts": [
{
"Type": "volume",
"Source": "mon_grafana",
"Target": "/var/lib/grafana",
"VolumeOptions": {
"Labels": {
"com.docker.stack.namespace": "mon"
}
}
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 134217728
},
"Reservations": {
"MemoryBytes": 67108864
}
},
"Placement": {
"Constraints": [
"node.role == manager"
],
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"grafana"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 3000,
"PublishedPort": 3000,
"PublishMode": "ingress"
}
]
}
},
"Endpoint": {
"Spec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 3000,
"PublishedPort": 3000,
"PublishMode": "ingress"
}
]
},
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 3000,
"PublishedPort": 3000,
"PublishMode": "ingress"
}
],
"VirtualIPs": [
{
"NetworkID": "qvwhwd6p61k4o0ulsknqb066z",
"Addr": "10.0.0.15/24"
},
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.29/24"
}
]
}
},
{
"ID": "ul5qawv4s7f7msm7dtyfarw80",
"Version": {
"Index": 74
},
"CreatedAt": "2020-06-19T19:14:45.311963014Z",
"UpdatedAt": "2020-06-19T19:14:45.327315101Z",
"Spec": {
"Name": "mon_dockerd-exporter",
"Labels": {
"com.docker.stack.image": "stefanprodan/caddy",
"com.docker.stack.namespace": "mon"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "stefanprodan/caddy:latest@sha256:44541cfacb66f4799f81f17fcfb3cb757ccc8f327592745549f5930c42d115c9",
"Labels": {
"com.docker.stack.namespace": "mon"
},
"Env": [
"DOCKER_GWBRIDGE_IP=172.18.0.1"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Configs": [
{
"File": {
"Name": "/etc/caddy/Caddyfile",
"UID": "0",
"GID": "0",
"Mode": 292
},
"ConfigID": "4glhzmw6fndnkjn2cok90a9gu",
"ConfigName": "mon_dockerd_config"
}
],
"Isolation": "default"
},
"Resources": {
"Limits": {
"MemoryBytes": 134217728
},
"Reservations": {
"MemoryBytes": 67108864
}
},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "npq2closzy836m07eaq1425k3",
"Aliases": [
"dockerd-exporter"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Global": {}
},
"EndpointSpec": {
"Mode": "vip"
}
},
"Endpoint": {
"Spec": {
"Mode": "vip"
},
"VirtualIPs": [
{
"NetworkID": "npq2closzy836m07eaq1425k3",
"Addr": "10.0.1.17/24"
}
]
}
}
]

2333
discovery/dockerswarm/testdata/swarmprom/tasks.json vendored Normal file
View file

File diff suppressed because it is too large Load diff

44
discovery/ec2/ec2.go
View file

@ -37,22 +37,25 @@ import (
)
const (
ec2Label = model.MetaLabelPrefix + "ec2_"
ec2LabelAZ = ec2Label + "availability_zone"
ec2LabelInstanceID = ec2Label + "instance_id"
ec2LabelInstanceState = ec2Label + "instance_state"
ec2LabelInstanceType = ec2Label + "instance_type"
ec2LabelOwnerID = ec2Label + "owner_id"
ec2LabelPlatform = ec2Label + "platform"
ec2LabelPublicDNS = ec2Label + "public_dns_name"
ec2LabelPublicIP = ec2Label + "public_ip"
ec2LabelPrivateDNS = ec2Label + "private_dns_name"
ec2LabelPrivateIP = ec2Label + "private_ip"
ec2LabelPrimarySubnetID = ec2Label + "primary_subnet_id"
ec2LabelSubnetID = ec2Label + "subnet_id"
ec2LabelTag = ec2Label + "tag_"
ec2LabelVPCID = ec2Label + "vpc_id"
subnetSeparator = ","
ec2Label = model.MetaLabelPrefix + "ec2_"
ec2LabelAMI = ec2Label + "ami"
ec2LabelAZ = ec2Label + "availability_zone"
ec2LabelArch = ec2Label + "architecture"
ec2LabelInstanceID = ec2Label + "instance_id"
ec2LabelInstanceState = ec2Label + "instance_state"
ec2LabelInstanceType = ec2Label + "instance_type"
ec2LabelInstanceLifecycle = ec2Label + "instance_lifecycle"
ec2LabelOwnerID = ec2Label + "owner_id"
ec2LabelPlatform = ec2Label + "platform"
ec2LabelPublicDNS = ec2Label + "public_dns_name"
ec2LabelPublicIP = ec2Label + "public_ip"
ec2LabelPrivateDNS = ec2Label + "private_dns_name"
ec2LabelPrivateIP = ec2Label + "private_ip"
ec2LabelPrimarySubnetID = ec2Label + "primary_subnet_id"
ec2LabelSubnetID = ec2Label + "subnet_id"
ec2LabelTag = ec2Label + "tag_"
ec2LabelVPCID = ec2Label + "vpc_id"
subnetSeparator = ","
)
// DefaultSDConfig is the default EC2 SD configuration.
@ -210,10 +213,19 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
labels[ec2LabelPublicDNS] = model.LabelValue(*inst.PublicDnsName)
}
labels[ec2LabelAMI] = model.LabelValue(*inst.ImageId)
labels[ec2LabelAZ] = model.LabelValue(*inst.Placement.AvailabilityZone)
labels[ec2LabelInstanceState] = model.LabelValue(*inst.State.Name)
labels[ec2LabelInstanceType] = model.LabelValue(*inst.InstanceType)
if inst.InstanceLifecycle != nil {
labels[ec2LabelInstanceLifecycle] = model.LabelValue(*inst.InstanceLifecycle)
}
if inst.Architecture != nil {
labels[ec2LabelArch] = model.LabelValue(*inst.Architecture)
}
if inst.VpcId != nil {
labels[ec2LabelVPCID] = model.LabelValue(*inst.VpcId)
labels[ec2LabelPrimarySubnetID] = model.LabelValue(*inst.SubnetId)

540
discovery/file/file_test.go
View file

@ -15,158 +15,468 @@ package file
import (
"context"
"encoding/json"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"sync"
"testing"
"time"
"github.com/prometheus/common/model"
"go.uber.org/goleak"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/testutil"
)
const testDir = "fixtures"
func TestFileSD(t *testing.T) {
defer os.Remove(filepath.Join(testDir, "_test_valid.yml"))
defer os.Remove(filepath.Join(testDir, "_test_valid.json"))
defer os.Remove(filepath.Join(testDir, "_test_invalid_nil.json"))
defer os.Remove(filepath.Join(testDir, "_test_invalid_nil.yml"))
testFileSD(t, "valid", ".yml", true)
testFileSD(t, "valid", ".json", true)
testFileSD(t, "invalid_nil", ".json", false)
testFileSD(t, "invalid_nil", ".yml", false)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
func testFileSD(t *testing.T, prefix, ext string, expect bool) {
// As interval refreshing is more of a fallback, we only want to test
// whether file watches work as expected.
var conf SDConfig
conf.Files = []string{filepath.Join(testDir, "_*"+ext)}
conf.RefreshInterval = model.Duration(1 * time.Hour)
const defaultWait = time.Second
var (
fsd = NewDiscovery(&conf, nil)
ch = make(chan []*targetgroup.Group)
ctx, cancel = context.WithCancel(context.Background())
)
go fsd.Run(ctx, ch)
type testRunner struct {
*testing.T
dir string
ch chan []*targetgroup.Group
done, stopped chan struct{}
cancelSD context.CancelFunc
select {
case <-time.After(25 * time.Millisecond):
// Expected.
case tgs := <-ch:
t.Fatalf("Unexpected target groups in file discovery: %s", tgs)
mtx sync.Mutex
tgs map[string]*targetgroup.Group
receivedAt time.Time
}
func newTestRunner(t *testing.T) *testRunner {
t.Helper()
tmpDir, err := ioutil.TempDir("", "prometheus-file-sd")
testutil.Ok(t, err)
return &testRunner{
T: t,
dir: tmpDir,
ch: make(chan []*targetgroup.Group),
done: make(chan struct{}),
stopped: make(chan struct{}),
tgs: make(map[string]*targetgroup.Group),
}
}
// To avoid empty group struct sent from the discovery caused by invalid fsnotify updates,
// drain the channel until we are ready with the test files.
fileReady := make(chan struct{})
drainReady := make(chan struct{})
// copyFile atomically copies a file to the runner's directory.
func (t *testRunner) copyFile(src string) string {
t.Helper()
return t.copyFileTo(src, filepath.Base(src))
}
// copyFileTo atomically copies a file with a different name to the runner's directory.
func (t *testRunner) copyFileTo(src string, name string) string {
t.Helper()
newf, err := ioutil.TempFile(t.dir, "")
testutil.Ok(t, err)
f, err := os.Open(src)
testutil.Ok(t, err)
_, err = io.Copy(newf, f)
testutil.Ok(t, err)
testutil.Ok(t, f.Close())
testutil.Ok(t, newf.Close())
dst := filepath.Join(t.dir, name)
err = os.Rename(newf.Name(), dst)
testutil.Ok(t, err)
return dst
}
// writeString writes atomically a string to a file.
func (t *testRunner) writeString(file string, data string) {
t.Helper()
newf, err := ioutil.TempFile(t.dir, "")
testutil.Ok(t, err)
_, err = newf.WriteString(data)
testutil.Ok(t, err)
testutil.Ok(t, newf.Close())
err = os.Rename(newf.Name(), file)
testutil.Ok(t, err)
}
// appendString appends a string to a file.
func (t *testRunner) appendString(file, data string) {
t.Helper()
f, err := os.OpenFile(file, os.O_WRONLY|os.O_APPEND, 0)
testutil.Ok(t, err)
defer f.Close()
_, err = f.WriteString(data)
testutil.Ok(t, err)
}
// run starts the file SD and the loop receiving target groups updates.
func (t *testRunner) run(files ...string) {
go func() {
defer close(t.stopped)
for {
select {
case <-ch:
case <-fileReady:
close(drainReady)
case <-t.done:
os.RemoveAll(t.dir)
return
case tgs := <-t.ch:
t.mtx.Lock()
t.receivedAt = time.Now()
for _, tg := range tgs {
t.tgs[tg.Source] = tg
}
t.mtx.Unlock()
}
}
}()
newf, err := os.Create(filepath.Join(testDir, "_test_"+prefix+ext))
if err != nil {
t.Fatal(err)
for i := range files {
files[i] = filepath.Join(t.dir, files[i])
}
defer newf.Close()
ctx, cancel := context.WithCancel(context.Background())
t.cancelSD = cancel
go func() {
NewDiscovery(
&SDConfig{
Files: files,
// Setting a high refresh interval to make sure that the tests only
// rely on file watches.
RefreshInterval: model.Duration(1 * time.Hour),
},
nil,
).Run(ctx, t.ch)
}()
}
f, err := os.Open(filepath.Join(testDir, prefix+ext))
if err != nil {
t.Fatal(err)
func (t *testRunner) stop() {
t.cancelSD()
close(t.done)
<-t.stopped
}
func (t *testRunner) lastReceive() time.Time {
t.mtx.Lock()
defer t.mtx.Unlock()
return t.receivedAt
}
func (t *testRunner) targets() []*targetgroup.Group {
t.mtx.Lock()
defer t.mtx.Unlock()
var (
keys []string
tgs []*targetgroup.Group
)
for k := range t.tgs {
keys = append(keys, k)
}
defer f.Close()
_, err = io.Copy(newf, f)
if err != nil {
t.Fatal(err)
sort.Strings(keys)
for _, k := range keys {
tgs = append(tgs, t.tgs[k])
}
return tgs
}
// Test file is ready so stop draining the discovery channel.
// It contains two target groups.
close(fileReady)
<-drainReady
newf.WriteString(" ") // One last meaningless write to trigger fsnotify and a new loop of the discovery service.
func (t *testRunner) requireUpdate(ref time.Time, expected []*targetgroup.Group) {
t.Helper()
timeout := time.After(15 * time.Second)
retry:
for {
select {
case <-timeout:
if expect {
t.Fatalf("Expected new target group but got none")
} else {
// Invalid type fsd should always break down.
break retry
}
case tgs := <-ch:
if !expect {
t.Fatalf("Unexpected target groups %s, we expected a failure here.", tgs)
case <-time.After(defaultWait):
t.Fatalf("Expected update but got none")
return
case <-time.After(defaultWait / 10):
if ref.Equal(t.lastReceive()) {
// No update received.
break
}
if len(tgs) != 2 {
continue retry // Potentially a partial write, just retry.
// We can receive partial updates so only check the result when the
// expected number of groups is reached.
tgs := t.targets()
if len(tgs) != len(expected) {
t.Logf("skipping update: expected %d targets, got %d", len(expected), len(tgs))
break
}
tg := tgs[0]
if _, ok := tg.Labels["foo"]; !ok {
t.Fatalf("Label not parsed")
t.requireTargetGroups(expected, tgs)
if ref.After(time.Time{}) {
t.Logf("update received after %v", t.lastReceive().Sub(ref))
}
if tg.String() != filepath.Join(testDir, "_test_"+prefix+ext+":0") {
t.Fatalf("Unexpected target group %s", tg)
}
tg = tgs[1]
if tg.String() != filepath.Join(testDir, "_test_"+prefix+ext+":1") {
t.Fatalf("Unexpected target groups %s", tg)
}
break retry
return
}
}
// Based on unknown circumstances, sometimes fsnotify will trigger more events in
// some runs (which might be empty, chains of different operations etc.).
// We have to drain those (as the target manager would) to avoid deadlocking and must
// not try to make sense of it all...
drained := make(chan struct{})
go func() {
for {
select {
case tgs := <-ch:
// Below we will change the file to a bad syntax. Previously extracted target
// groups must not be deleted via sending an empty target group.
if len(tgs[0].Targets) == 0 {
t.Errorf("Unexpected empty target groups received: %s", tgs)
}
case <-time.After(500 * time.Millisecond):
close(drained)
return
}
}
}()
newf, err = os.Create(filepath.Join(testDir, "_test.new"))
if err != nil {
t.Fatal(err)
}
defer os.Remove(newf.Name())
if _, err := newf.Write([]byte("]gibberish\n][")); err != nil {
t.Fatal(err)
}
newf.Close()
os.Rename(newf.Name(), filepath.Join(testDir, "_test_"+prefix+ext))
cancel()
<-drained
}
func (t *testRunner) requireTargetGroups(expected, got []*targetgroup.Group) {
t.Helper()
b1, err := json.Marshal(expected)
if err != nil {
panic(err)
}
b2, err := json.Marshal(got)
if err != nil {
panic(err)
}
testutil.Equals(t, string(b1), string(b2))
}
// validTg() maps to fixtures/valid.{json,yml}.
func validTg(file string) []*targetgroup.Group {
return []*targetgroup.Group{
{
Targets: []model.LabelSet{
{
model.AddressLabel: model.LabelValue("localhost:9090"),
},
{
model.AddressLabel: model.LabelValue("example.org:443"),
},
},
Labels: model.LabelSet{
model.LabelName("foo"): model.LabelValue("bar"),
fileSDFilepathLabel: model.LabelValue(file),
},
Source: fileSource(file, 0),
},
{
Targets: []model.LabelSet{
{
model.AddressLabel: model.LabelValue("my.domain"),
},
},
Labels: model.LabelSet{
fileSDFilepathLabel: model.LabelValue(file),
},
Source: fileSource(file, 1),
},
}
}
// valid2Tg() maps to fixtures/valid2.{json,yml}.
func valid2Tg(file string) []*targetgroup.Group {
return []*targetgroup.Group{
{
Targets: []model.LabelSet{
{
model.AddressLabel: model.LabelValue("my.domain"),
},
},
Labels: model.LabelSet{
fileSDFilepathLabel: model.LabelValue(file),
},
Source: fileSource(file, 0),
},
{
Targets: []model.LabelSet{
{
model.AddressLabel: model.LabelValue("localhost:9090"),
},
},
Labels: model.LabelSet{
model.LabelName("foo"): model.LabelValue("bar"),
model.LabelName("fred"): model.LabelValue("baz"),
fileSDFilepathLabel: model.LabelValue(file),
},
Source: fileSource(file, 1),
},
{
Targets: []model.LabelSet{
{
model.AddressLabel: model.LabelValue("example.org:443"),
},
},
Labels: model.LabelSet{
model.LabelName("scheme"): model.LabelValue("https"),
fileSDFilepathLabel: model.LabelValue(file),
},
Source: fileSource(file, 2),
},
}
}
func TestInitialUpdate(t *testing.T) {
for _, tc := range []string{
"fixtures/valid.yml",
"fixtures/valid.json",
} {
t.Run(tc, func(t *testing.T) {
t.Parallel()
runner := newTestRunner(t)
sdFile := runner.copyFile(tc)
runner.run("*" + filepath.Ext(tc))
defer runner.stop()
// Verify that we receive the initial target groups.
runner.requireUpdate(time.Time{}, validTg(sdFile))
})
}
}
func TestInvalidFile(t *testing.T) {
for _, tc := range []string{
"fixtures/invalid_nil.yml",
"fixtures/invalid_nil.json",
} {
tc := tc
t.Run(tc, func(t *testing.T) {
t.Parallel()
now := time.Now()
runner := newTestRunner(t)
runner.copyFile(tc)
runner.run("*" + filepath.Ext(tc))
defer runner.stop()
// Verify that we've received nothing.
time.Sleep(defaultWait)
if runner.lastReceive().After(now) {
t.Fatalf("unexpected targets received: %v", runner.targets())
}
})
}
}
func TestNoopFileUpdate(t *testing.T) {
t.Parallel()
runner := newTestRunner(t)
sdFile := runner.copyFile("fixtures/valid.yml")
runner.run("*.yml")
defer runner.stop()
// Verify that we receive the initial target groups.
runner.requireUpdate(time.Time{}, validTg(sdFile))
// Verify that we receive an update with the same target groups.
ref := runner.lastReceive()
runner.copyFileTo("fixtures/valid3.yml", "valid.yml")
runner.requireUpdate(ref, validTg(sdFile))
}
func TestFileUpdate(t *testing.T) {
t.Parallel()
runner := newTestRunner(t)
sdFile := runner.copyFile("fixtures/valid.yml")
runner.run("*.yml")
defer runner.stop()
// Verify that we receive the initial target groups.
runner.requireUpdate(time.Time{}, validTg(sdFile))
// Verify that we receive an update with the new target groups.
ref := runner.lastReceive()
runner.copyFileTo("fixtures/valid2.yml", "valid.yml")
runner.requireUpdate(ref, valid2Tg(sdFile))
}
func TestInvalidFileUpdate(t *testing.T) {
t.Parallel()
runner := newTestRunner(t)
sdFile := runner.copyFile("fixtures/valid.yml")
runner.run("*.yml")
defer runner.stop()
// Verify that we receive the initial target groups.
runner.requireUpdate(time.Time{}, validTg(sdFile))
ref := runner.lastReceive()
runner.writeString(sdFile, "]gibberish\n][")
// Verify that we receive nothing or the same targets as before.
time.Sleep(defaultWait)
if runner.lastReceive().After(ref) {
runner.requireTargetGroups(validTg(sdFile), runner.targets())
}
}
func TestUpdateFileWithPartialWrites(t *testing.T) {
t.Parallel()
runner := newTestRunner(t)
sdFile := runner.copyFile("fixtures/valid.yml")
runner.run("*.yml")
defer runner.stop()
// Verify that we receive the initial target groups.
runner.requireUpdate(time.Time{}, validTg(sdFile))
// Do a partial write operation.
ref := runner.lastReceive()
runner.writeString(sdFile, "- targets")
time.Sleep(defaultWait)
// Verify that we receive nothing or the same target groups as before.
if runner.lastReceive().After(ref) {
runner.requireTargetGroups(validTg(sdFile), runner.targets())
}
// Verify that we receive the update target groups once the file is a valid YAML payload.
ref = runner.lastReceive()
runner.appendString(sdFile, `: ["localhost:9091"]`)
runner.requireUpdate(ref,
[]*targetgroup.Group{
{
Targets: []model.LabelSet{
{
model.AddressLabel: model.LabelValue("localhost:9091"),
},
},
Labels: model.LabelSet{
fileSDFilepathLabel: model.LabelValue(sdFile),
},
Source: fileSource(sdFile, 0),
},
{
Source: fileSource(sdFile, 1),
},
},
)
}
func TestRemoveFile(t *testing.T) {
t.Parallel()
runner := newTestRunner(t)
sdFile := runner.copyFile("fixtures/valid.yml")
runner.run("*.yml")
defer runner.stop()
// Verify that we receive the initial target groups.
runner.requireUpdate(time.Time{}, validTg(sdFile))
// Verify that we receive the update about the target groups being removed.
ref := runner.lastReceive()
testutil.Ok(t, os.Remove(sdFile))
runner.requireUpdate(
ref,
[]*targetgroup.Group{
{
Source: fileSource(sdFile, 0),
},
{
Source: fileSource(sdFile, 1),
}},
)
}

8
discovery/file/fixtures/valid2.yml Normal file
View file

@ -0,0 +1,8 @@
- targets: ['my.domain']
- targets: ['localhost:9090']
labels:
foo: bar
fred: baz
- targets: ['example.org:443']
labels:
scheme: https

7
discovery/file/fixtures/valid3.yml Normal file
View file

@ -0,0 +1,7 @@
# the YAML structure is identical to valid.yml but the raw data is different.
- targets: ['localhost:9090', 'example.org:443']
labels:
foo: bar
- targets: ['my.domain']

150
discovery/kubernetes/client_metrics.go
View file

@ -18,16 +18,11 @@ import (
"time"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/metrics"
"k8s.io/client-go/util/workqueue"
)
const (
cacheMetricsNamespace = metricsNamespace + "_cache"
workqueueMetricsNamespace = metricsNamespace + "_workqueue"
)
const workqueueMetricsNamespace = metricsNamespace + "_workqueue"
var (
// Metrics for client-go's HTTP requests.
@ -49,68 +44,6 @@ var (
[]string{"endpoint"},
)
// Definition of metrics for client-go cache metrics provider.
clientGoCacheListTotalMetric = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: cacheMetricsNamespace,
Name: "list_total",
Help: "Total number of list operations.",
},
)
clientGoCacheListDurationMetric = prometheus.NewSummary(
prometheus.SummaryOpts{
Namespace: cacheMetricsNamespace,
Name: "list_duration_seconds",
Help: "Duration of a Kubernetes API call in seconds.",
Objectives: map[float64]float64{},
},
)
clientGoCacheItemsInListCountMetric = prometheus.NewSummary(
prometheus.SummaryOpts{
Namespace: cacheMetricsNamespace,
Name: "list_items",
Help: "Count of items in a list from the Kubernetes API.",
Objectives: map[float64]float64{},
},
)
clientGoCacheWatchesCountMetric = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: cacheMetricsNamespace,
Name: "watches_total",
Help: "Total number of watch operations.",
},
)
clientGoCacheShortWatchesCountMetric = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: cacheMetricsNamespace,
Name: "short_watches_total",
Help: "Total number of short watch operations.",
},
)
clientGoCacheWatchesDurationMetric = prometheus.NewSummary(
prometheus.SummaryOpts{
Namespace: cacheMetricsNamespace,
Name: "watch_duration_seconds",
Help: "Duration of watches on the Kubernetes API.",
Objectives: map[float64]float64{},
},
)
clientGoCacheItemsInWatchesCountMetric = prometheus.NewSummary(
prometheus.SummaryOpts{
Namespace: cacheMetricsNamespace,
Name: "watch_events",
Help: "Number of items in watches on the Kubernetes API.",
Objectives: map[float64]float64{},
},
)
clientGoCacheLastResourceVersionMetric = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: cacheMetricsNamespace,
Name: "last_resource_version",
Help: "Last resource version from the Kubernetes API.",
},
)
// Definition of metrics for client-go workflow metrics provider
clientGoWorkqueueDepthMetricVec = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
@ -176,7 +109,12 @@ func (noopMetric) Set(float64) {}
type clientGoRequestMetricAdapter struct{}
func (f *clientGoRequestMetricAdapter) Register(registerer prometheus.Registerer) {
metrics.Register(f, f)
metrics.Register(
metrics.RegisterOpts{
RequestLatency: f,
RequestResult: f,
},
)
registerer.MustRegister(
clientGoRequestResultMetricVec,
clientGoRequestLatencyMetricVec,
@ -189,47 +127,6 @@ func (clientGoRequestMetricAdapter) Observe(verb string, u url.URL, latency time
clientGoRequestLatencyMetricVec.WithLabelValues(u.EscapedPath()).Observe(latency.Seconds())
}
// Definition of client-go cache metrics provider definition
type clientGoCacheMetricsProvider struct{}
func (f *clientGoCacheMetricsProvider) Register(registerer prometheus.Registerer) {
cache.SetReflectorMetricsProvider(f)
registerer.MustRegister(
clientGoCacheWatchesDurationMetric,
clientGoCacheWatchesCountMetric,
clientGoCacheListDurationMetric,
clientGoCacheListTotalMetric,
clientGoCacheLastResourceVersionMetric,
clientGoCacheShortWatchesCountMetric,
clientGoCacheItemsInWatchesCountMetric,
clientGoCacheItemsInListCountMetric,
)
}
func (clientGoCacheMetricsProvider) NewListsMetric(name string) cache.CounterMetric {
return clientGoCacheListTotalMetric
}
func (clientGoCacheMetricsProvider) NewListDurationMetric(name string) cache.SummaryMetric {
return clientGoCacheListDurationMetric
}
func (clientGoCacheMetricsProvider) NewItemsInListMetric(name string) cache.SummaryMetric {
return clientGoCacheItemsInListCountMetric
}
func (clientGoCacheMetricsProvider) NewWatchesMetric(name string) cache.CounterMetric {
return clientGoCacheWatchesCountMetric
}
func (clientGoCacheMetricsProvider) NewShortWatchesMetric(name string) cache.CounterMetric {
return clientGoCacheShortWatchesCountMetric
}
func (clientGoCacheMetricsProvider) NewWatchDurationMetric(name string) cache.SummaryMetric {
return clientGoCacheWatchesDurationMetric
}
func (clientGoCacheMetricsProvider) NewItemsInWatchMetric(name string) cache.SummaryMetric {
return clientGoCacheItemsInWatchesCountMetric
}
func (clientGoCacheMetricsProvider) NewLastResourceVersionMetric(name string) cache.GaugeMetric {
return clientGoCacheLastResourceVersionMetric
}
// Definition of client-go workqueue metrics provider definition
type clientGoWorkqueueMetricsProvider struct{}
@ -252,18 +149,10 @@ func (f *clientGoWorkqueueMetricsProvider) NewAddsMetric(name string) workqueue.
return clientGoWorkqueueAddsMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric {
metric := clientGoWorkqueueLatencyMetricVec.WithLabelValues(name)
// Convert microseconds to seconds for consistency across metrics.
return prometheus.ObserverFunc(func(v float64) {
metric.Observe(v / 1e6)
})
return clientGoWorkqueueLatencyMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric {
metric := clientGoWorkqueueWorkDurationMetricVec.WithLabelValues(name)
// Convert microseconds to seconds for consistency across metrics.
return prometheus.ObserverFunc(func(v float64) {
metric.Observe(v / 1e6)
})
return clientGoWorkqueueWorkDurationMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric {
return clientGoWorkqueueUnfinishedWorkSecondsMetricVec.WithLabelValues(name)
@ -275,24 +164,3 @@ func (clientGoWorkqueueMetricsProvider) NewRetriesMetric(name string) workqueue.
// Retries are not used so the metric is omitted.
return noopMetric{}
}
func (clientGoWorkqueueMetricsProvider) NewDeprecatedDepthMetric(name string) workqueue.GaugeMetric {
return noopMetric{}
}
func (clientGoWorkqueueMetricsProvider) NewDeprecatedAddsMetric(name string) workqueue.CounterMetric {
return noopMetric{}
}
func (clientGoWorkqueueMetricsProvider) NewDeprecatedLatencyMetric(name string) workqueue.SummaryMetric {
return noopMetric{}
}
func (f *clientGoWorkqueueMetricsProvider) NewDeprecatedWorkDurationMetric(name string) workqueue.SummaryMetric {
return noopMetric{}
}
func (f *clientGoWorkqueueMetricsProvider) NewDeprecatedUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric {
return noopMetric{}
}
func (f *clientGoWorkqueueMetricsProvider) NewDeprecatedLongestRunningProcessorMicrosecondsMetric(name string) workqueue.SettableGaugeMetric {
return noopMetric{}
}
func (clientGoWorkqueueMetricsProvider) NewDeprecatedRetriesMetric(name string) workqueue.CounterMetric {
return noopMetric{}
}

26
discovery/kubernetes/endpoints.go
View file

@ -29,6 +29,12 @@ import (
"github.com/prometheus/prometheus/discovery/targetgroup"
)
var (
epAddCount = eventCount.WithLabelValues("endpoints", "add")
epUpdateCount = eventCount.WithLabelValues("endpoints", "update")
epDeleteCount = eventCount.WithLabelValues("endpoints", "delete")
)
// Endpoints discovers new endpoint targets.
type Endpoints struct {
logger log.Logger
@ -62,15 +68,15 @@ func NewEndpoints(l log.Logger, svc, eps, pod cache.SharedInformer) *Endpoints {
e.endpointsInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(o interface{}) {
eventCount.WithLabelValues("endpoints", "add").Inc()
epAddCount.Inc()
e.enqueue(o)
},
UpdateFunc: func(_, o interface{}) {
eventCount.WithLabelValues("endpoints", "update").Inc()
epUpdateCount.Inc()
e.enqueue(o)
},
DeleteFunc: func(o interface{}) {
eventCount.WithLabelValues("endpoints", "delete").Inc()
epDeleteCount.Inc()
e.enqueue(o)
},
})
@ -98,15 +104,15 @@ func NewEndpoints(l log.Logger, svc, eps, pod cache.SharedInformer) *Endpoints {
// TODO(fabxc): potentially remove add and delete event handlers. Those should
// be triggered via the endpoint handlers already.
AddFunc: func(o interface{}) {
eventCount.WithLabelValues("service", "add").Inc()
svcAddCount.Inc()
serviceUpdate(o)
},
UpdateFunc: func(_, o interface{}) {
eventCount.WithLabelValues("service", "update").Inc()
svcUpdateCount.Inc()
serviceUpdate(o)
},
DeleteFunc: func(o interface{}) {
eventCount.WithLabelValues("service", "delete").Inc()
svcDeleteCount.Inc()
serviceUpdate(o)
},
})
@ -128,7 +134,9 @@ func (e *Endpoints) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
defer e.queue.ShutDown()
if !cache.WaitForCacheSync(ctx.Done(), e.endpointsInf.HasSynced, e.serviceInf.HasSynced, e.podInf.HasSynced) {
level.Error(e.logger).Log("msg", "endpoints informer unable to sync cache")
if ctx.Err() != context.Canceled {
level.Error(e.logger).Log("msg", "endpoints informer unable to sync cache")
}
return
}
@ -161,7 +169,7 @@ func (e *Endpoints) process(ctx context.Context, ch chan<- []*targetgroup.Group)
return true
}
if !exists {
send(ctx, e.logger, RoleEndpoint, ch, &targetgroup.Group{Source: endpointsSourceFromNamespaceAndName(namespace, name)})
send(ctx, ch, &targetgroup.Group{Source: endpointsSourceFromNamespaceAndName(namespace, name)})
return true
}
eps, err := convertToEndpoints(o)
@ -169,7 +177,7 @@ func (e *Endpoints) process(ctx context.Context, ch chan<- []*targetgroup.Group)
level.Error(e.logger).Log("msg", "converting to Endpoints object failed", "err", err)
return true
}
send(ctx, e.logger, RoleEndpoint, ch, e.buildEndpoints(eps))
send(ctx, ch, e.buildEndpoints(eps))
return true
}

20
discovery/kubernetes/endpoints_test.go
View file

@ -14,14 +14,16 @@
package kubernetes
import (
"context"
"testing"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery/targetgroup"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
func makeEndpoints() *v1.Endpoints {
@ -78,7 +80,7 @@ func TestEndpointsDiscoveryBeforeRun(t *testing.T) {
discovery: n,
beforeRun: func() {
obj := makeEndpoints()
c.CoreV1().Endpoints(obj.Namespace).Create(obj)
c.CoreV1().Endpoints(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{})
},
expectedMaxItems: 1,
expectedRes: map[string]*targetgroup.Group{
@ -184,7 +186,7 @@ func TestEndpointsDiscoveryAdd(t *testing.T) {
},
},
}
c.CoreV1().Endpoints(obj.Namespace).Create(obj)
c.CoreV1().Endpoints(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{})
},
expectedMaxItems: 1,
expectedRes: map[string]*targetgroup.Group{
@ -241,7 +243,7 @@ func TestEndpointsDiscoveryDelete(t *testing.T) {
discovery: n,
afterStart: func() {
obj := makeEndpoints()
c.CoreV1().Endpoints(obj.Namespace).Delete(obj.Name, &metav1.DeleteOptions{})
c.CoreV1().Endpoints(obj.Namespace).Delete(context.Background(), obj.Name, metav1.DeleteOptions{})
},
expectedMaxItems: 2,
expectedRes: map[string]*targetgroup.Group{
@ -294,7 +296,7 @@ func TestEndpointsDiscoveryUpdate(t *testing.T) {
},
},
}
c.CoreV1().Endpoints(obj.Namespace).Update(obj)
c.CoreV1().Endpoints(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{})
},
expectedMaxItems: 2,
expectedRes: map[string]*targetgroup.Group{
@ -336,7 +338,7 @@ func TestEndpointsDiscoveryEmptySubsets(t *testing.T) {
},
Subsets: []v1.EndpointSubset{},
}
c.CoreV1().Endpoints(obj.Namespace).Update(obj)
c.CoreV1().Endpoints(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{})
},
expectedMaxItems: 2,
expectedRes: map[string]*targetgroup.Group{
@ -366,7 +368,7 @@ func TestEndpointsDiscoveryWithService(t *testing.T) {
},
},
}
c.CoreV1().Services(obj.Namespace).Create(obj)
c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{})
},
expectedMaxItems: 1,
expectedRes: map[string]*targetgroup.Group{
@ -421,7 +423,7 @@ func TestEndpointsDiscoveryWithServiceUpdate(t *testing.T) {
},
},
}
c.CoreV1().Services(obj.Namespace).Create(obj)
c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{})
},
afterStart: func() {
obj := &v1.Service{
@ -434,7 +436,7 @@ func TestEndpointsDiscoveryWithServiceUpdate(t *testing.T) {
},
},
}
c.CoreV1().Services(obj.Namespace).Update(obj)
c.CoreV1().Services(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{})
},
expectedMaxItems: 2,
expectedRes: map[string]*targetgroup.Group{

407
discovery/kubernetes/endpointslice.go Normal file
View file

@ -0,0 +1,407 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kubernetes
import (
"context"
"net"
"strconv"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/pkg/errors"
"github.com/prometheus/common/model"
apiv1 "k8s.io/api/core/v1"
disv1beta1 "k8s.io/api/discovery/v1beta1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/strutil"
)
var (
epslAddCount = eventCount.WithLabelValues("endpointslice", "add")
epslUpdateCount = eventCount.WithLabelValues("endpointslice", "update")
epslDeleteCount = eventCount.WithLabelValues("endpointslice", "delete")
)
// EndpointSlice discovers new endpoint targets.
type EndpointSlice struct {
logger log.Logger
endpointSliceInf cache.SharedInformer
serviceInf cache.SharedInformer
podInf cache.SharedInformer
podStore cache.Store
endpointSliceStore cache.Store
serviceStore cache.Store
queue *workqueue.Type
}
// NewEndpointSlice returns a new endpointslice discovery.
func NewEndpointSlice(l log.Logger, svc, eps, pod cache.SharedInformer) *EndpointSlice {
if l == nil {
l = log.NewNopLogger()
}
e := &EndpointSlice{
logger: l,
endpointSliceInf: eps,
endpointSliceStore: eps.GetStore(),
serviceInf: svc,
serviceStore: svc.GetStore(),
podInf: pod,
podStore: pod.GetStore(),
queue: workqueue.NewNamed("endpointSlice"),
}
e.endpointSliceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(o interface{}) {
epslAddCount.Inc()
e.enqueue(o)
},
UpdateFunc: func(_, o interface{}) {
epslUpdateCount.Inc()
e.enqueue(o)
},
DeleteFunc: func(o interface{}) {
epslDeleteCount.Inc()
e.enqueue(o)
},
})
serviceUpdate := func(o interface{}) {
svc, err := convertToService(o)
if err != nil {
level.Error(e.logger).Log("msg", "converting to Service object failed", "err", err)
return
}
// TODO(brancz): use cache.Indexer to index endpoints by
// disv1beta1.LabelServiceName so this operation doesn't have to
// iterate over all endpoint objects.
for _, obj := range e.endpointSliceStore.List() {
ep := obj.(*disv1beta1.EndpointSlice)
if lv, exists := ep.Labels[disv1beta1.LabelServiceName]; exists && lv == svc.Name {
e.enqueue(ep)
}
}
}
e.serviceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(o interface{}) {
svcAddCount.Inc()
serviceUpdate(o)
},
UpdateFunc: func(_, o interface{}) {
svcUpdateCount.Inc()
serviceUpdate(o)
},
DeleteFunc: func(o interface{}) {
svcDeleteCount.Inc()
serviceUpdate(o)
},
})
return e
}
func (e *EndpointSlice) enqueue(obj interface{}) {
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
if err != nil {
return
}
e.queue.Add(key)
}
// Run implements the Discoverer interface.
func (e *EndpointSlice) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
defer e.queue.ShutDown()
if !cache.WaitForCacheSync(ctx.Done(), e.endpointSliceInf.HasSynced, e.serviceInf.HasSynced, e.podInf.HasSynced) {
if ctx.Err() != context.Canceled {
level.Error(e.logger).Log("msg", "endpointslice informer unable to sync cache")
}
return
}
go func() {
for e.process(ctx, ch) {
}
}()
// Block until the target provider is explicitly canceled.
<-ctx.Done()
}
func (e *EndpointSlice) process(ctx context.Context, ch chan<- []*targetgroup.Group) bool {
keyObj, quit := e.queue.Get()
if quit {
return false
}
defer e.queue.Done(keyObj)
key := keyObj.(string)
namespace, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
level.Error(e.logger).Log("msg", "splitting key failed", "key", key)
return true
}
o, exists, err := e.endpointSliceStore.GetByKey(key)
if err != nil {
level.Error(e.logger).Log("msg", "getting object from store failed", "key", key)
return true
}
if !exists {
send(ctx, ch, &targetgroup.Group{Source: endpointSliceSourceFromNamespaceAndName(namespace, name)})
return true
}
eps, err := convertToEndpointSlice(o)
if err != nil {
level.Error(e.logger).Log("msg", "converting to EndpointSlice object failed", "err", err)
return true
}
send(ctx, ch, e.buildEndpointSlice(eps))
return true
}
func convertToEndpointSlice(o interface{}) (*disv1beta1.EndpointSlice, error) {
endpoints, ok := o.(*disv1beta1.EndpointSlice)
if ok {
return endpoints, nil
}
return nil, errors.Errorf("received unexpected object: %v", o)
}
func endpointSliceSource(ep *disv1beta1.EndpointSlice) string {
return endpointSliceSourceFromNamespaceAndName(ep.Namespace, ep.Name)
}
func endpointSliceSourceFromNamespaceAndName(namespace, name string) string {
return "endpointslice/" + namespace + "/" + name
}
const (
endpointSliceNameLabel = metaLabelPrefix + "endpointslice_name"
endpointSliceAddressTypeLabel = metaLabelPrefix + "endpointslice_address_type"
endpointSlicePortNameLabel = metaLabelPrefix + "endpointslice_port_name"
endpointSlicePortProtocolLabel = metaLabelPrefix + "endpointslice_port_protocol"
endpointSlicePortLabel = metaLabelPrefix + "endpointslice_port"
endpointSlicePortAppProtocol = metaLabelPrefix + "endpointslice_port_app_protocol"
endpointSliceEndpointConditionsReadyLabel = metaLabelPrefix + "endpointslice_endpoint_conditions_ready"
endpointSliceEndpointHostnameLabel = metaLabelPrefix + "endpointslice_endpoint_hostname"
endpointSliceAddressTargetKindLabel = metaLabelPrefix + "endpointslice_address_target_kind"
endpointSliceAddressTargetNameLabel = metaLabelPrefix + "endpointslice_address_target_name"
endpointSliceEndpointTopologyLabelPrefix = metaLabelPrefix + "endpointslice_endpoint_topology_"
endpointSliceEndpointTopologyLabelPresentPrefix = metaLabelPrefix + "endpointslice_endpoint_topology_present_"
)
func (e *EndpointSlice) buildEndpointSlice(eps *disv1beta1.EndpointSlice) *targetgroup.Group {
tg := &targetgroup.Group{
Source: endpointSliceSource(eps),
}
tg.Labels = model.LabelSet{
namespaceLabel: lv(eps.Namespace),
endpointSliceNameLabel: lv(eps.Name),
endpointSliceAddressTypeLabel: lv(string(eps.AddressType)),
}
e.addServiceLabels(eps, tg)
type podEntry struct {
pod *apiv1.Pod
servicePorts []disv1beta1.EndpointPort
}
seenPods := map[string]*podEntry{}
add := func(addr string, ep disv1beta1.Endpoint, port disv1beta1.EndpointPort) {
a := addr
if port.Port != nil {
a = net.JoinHostPort(addr, strconv.FormatUint(uint64(*port.Port), 10))
}
target := model.LabelSet{
model.AddressLabel: lv(a),
}
if port.Name != nil {
target[endpointSlicePortNameLabel] = lv(*port.Name)
}
if port.Protocol != nil {
target[endpointSlicePortProtocolLabel] = lv(string(*port.Protocol))
}
if port.Port != nil {
target[endpointSlicePortLabel] = lv(strconv.FormatUint(uint64(*port.Port), 10))
}
if port.AppProtocol != nil {
target[endpointSlicePortAppProtocol] = lv(*port.AppProtocol)
}
if ep.Conditions.Ready != nil {
target[endpointSliceEndpointConditionsReadyLabel] = lv(strconv.FormatBool(*ep.Conditions.Ready))
}
if ep.Hostname != nil {
target[endpointSliceEndpointHostnameLabel] = lv(*ep.Hostname)
}
if ep.TargetRef != nil {
target[model.LabelName(endpointSliceAddressTargetKindLabel)] = lv(ep.TargetRef.Kind)
target[model.LabelName(endpointSliceAddressTargetNameLabel)] = lv(ep.TargetRef.Name)
}
for k, v := range ep.Topology {
ln := strutil.SanitizeLabelName(k)
target[model.LabelName(endpointSliceEndpointTopologyLabelPrefix+ln)] = lv(v)
target[model.LabelName(endpointSliceEndpointTopologyLabelPresentPrefix+ln)] = presentValue
}
pod := e.resolvePodRef(ep.TargetRef)
if pod == nil {
// This target is not a Pod, so don't continue with Pod specific logic.
tg.Targets = append(tg.Targets, target)
return
}
s := pod.Namespace + "/" + pod.Name
sp, ok := seenPods[s]
if !ok {
sp = &podEntry{pod: pod}
seenPods[s] = sp
}
// Attach standard pod labels.
target = target.Merge(podLabels(pod))
// Attach potential container port labels matching the endpoint port.
for _, c := range pod.Spec.Containers {
for _, cport := range c.Ports {
if port.Port == nil {
continue
}
if *port.Port == cport.ContainerPort {
ports := strconv.FormatUint(uint64(*port.Port), 10)
target[podContainerNameLabel] = lv(c.Name)
target[podContainerPortNameLabel] = lv(cport.Name)
target[podContainerPortNumberLabel] = lv(ports)
target[podContainerPortProtocolLabel] = lv(string(cport.Protocol))
break
}
}
}
// Add service port so we know that we have already generated a target
// for it.
sp.servicePorts = append(sp.servicePorts, port)
tg.Targets = append(tg.Targets, target)
}
for _, ep := range eps.Endpoints {
for _, port := range eps.Ports {
for _, addr := range ep.Addresses {
add(addr, ep, port)
}
}
}
// For all seen pods, check all container ports. If they were not covered
// by one of the service endpoints, generate targets for them.
for _, pe := range seenPods {
for _, c := range pe.pod.Spec.Containers {
for _, cport := range c.Ports {
hasSeenPort := func() bool {
for _, eport := range pe.servicePorts {
if eport.Port == nil {
continue
}
if cport.ContainerPort == *eport.Port {
return true
}
}
return false
}
if hasSeenPort() {
continue
}
a := net.JoinHostPort(pe.pod.Status.PodIP, strconv.FormatUint(uint64(cport.ContainerPort), 10))
ports := strconv.FormatUint(uint64(cport.ContainerPort), 10)
target := model.LabelSet{
model.AddressLabel: lv(a),
podContainerNameLabel: lv(c.Name),
podContainerPortNameLabel: lv(cport.Name),
podContainerPortNumberLabel: lv(ports),
podContainerPortProtocolLabel: lv(string(cport.Protocol)),
}
tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod)))
}
}
}
return tg
}
func (e *EndpointSlice) resolvePodRef(ref *apiv1.ObjectReference) *apiv1.Pod {
if ref == nil || ref.Kind != "Pod" {
return nil
}
p := &apiv1.Pod{}
p.Namespace = ref.Namespace
p.Name = ref.Name
obj, exists, err := e.podStore.Get(p)
if err != nil {
level.Error(e.logger).Log("msg", "resolving pod ref failed", "err", err)
return nil
}
if !exists {
return nil
}
return obj.(*apiv1.Pod)
}
func (e *EndpointSlice) addServiceLabels(eps *disv1beta1.EndpointSlice, tg *targetgroup.Group) {
var (
svc = &apiv1.Service{}
found bool
)
svc.Namespace = eps.Namespace
// Every EndpointSlice object has the Service they belong to in the
// kubernetes.io/service-name label.
svc.Name, found = eps.Labels[disv1beta1.LabelServiceName]
if !found {
return
}
obj, exists, err := e.serviceStore.Get(svc)
if err != nil {
level.Error(e.logger).Log("msg", "retrieving service failed", "err", err)
return
}
if !exists {
return
}
svc = obj.(*apiv1.Service)
tg.Labels = tg.Labels.Merge(serviceLabels(svc))
}

631
discovery/kubernetes/endpointslice_test.go Normal file
View file

@ -0,0 +1,631 @@
// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kubernetes
import (
"context"
"testing"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery/targetgroup"
v1 "k8s.io/api/core/v1"
disv1beta1 "k8s.io/api/discovery/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
)
func strptr(str string) *string {
return &str
}
func boolptr(b bool) *bool {
return &b
}
func int32ptr(i int32) *int32 {
return &i
}
func protocolptr(p v1.Protocol) *v1.Protocol {
return &p
}
func makeEndpointSlice() *disv1beta1.EndpointSlice {
return &disv1beta1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "testendpoints",
Namespace: "default",
Labels: map[string]string{
disv1beta1.LabelServiceName: "testendpoints",
},
},
AddressType: disv1beta1.AddressTypeIPv4,
Ports: []disv1beta1.EndpointPort{
{
Name: strptr("testport"),
Port: int32ptr(9000),
Protocol: protocolptr(v1.ProtocolTCP),
},
},
Endpoints: []disv1beta1.Endpoint{
{
Addresses: []string{"1.2.3.4"},
Hostname: strptr("testendpoint1"),
}, {
Addresses: []string{"2.3.4.5"},
Conditions: disv1beta1.EndpointConditions{
Ready: boolptr(true),
},
}, {
Addresses: []string{"3.4.5.6"},
Conditions: disv1beta1.EndpointConditions{
Ready: boolptr(false),
},
},
},
}
}
func TestEndpointSliceDiscoveryBeforeRun(t *testing.T) {
n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{})
k8sDiscoveryTest{
discovery: n,
beforeRun: func() {
obj := makeEndpointSlice()
c.DiscoveryV1beta1().EndpointSlices(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{})
},
expectedMaxItems: 1,
expectedRes: map[string]*targetgroup.Group{
"endpointslice/default/testendpoints": {
Targets: []model.LabelSet{
{
"__address__": "1.2.3.4:9000",
"__meta_kubernetes_endpointslice_endpoint_hostname": "testendpoint1",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
{
"__address__": "2.3.4.5:9000",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
{
"__address__": "3.4.5.6:9000",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "false",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
},
Labels: model.LabelSet{
"__meta_kubernetes_endpointslice_address_type": "IPv4",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_endpointslice_name": "testendpoints",
},
Source: "endpointslice/default/testendpoints",
},
},
}.Run(t)
}
func TestEndpointSliceDiscoveryAdd(t *testing.T) {
obj := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "testpod",
Namespace: "default",
UID: types.UID("deadbeef"),
},
Spec: v1.PodSpec{
NodeName: "testnode",
Containers: []v1.Container{
{
Name: "c1",
Ports: []v1.ContainerPort{
{
Name: "mainport",
ContainerPort: 9000,
Protocol: v1.ProtocolTCP,
},
},
},
{
Name: "c2",
Ports: []v1.ContainerPort{
{
Name: "sideport",
ContainerPort: 9001,
Protocol: v1.ProtocolTCP,
},
},
},
},
},
Status: v1.PodStatus{
HostIP: "2.3.4.5",
PodIP: "1.2.3.4",
},
}
n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{}, obj)
k8sDiscoveryTest{
discovery: n,
afterStart: func() {
obj := &disv1beta1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "testendpoints",
Namespace: "default",
},
AddressType: disv1beta1.AddressTypeIPv4,
Ports: []disv1beta1.EndpointPort{
{
Name: strptr("testport"),
Port: int32ptr(9000),
Protocol: protocolptr(v1.ProtocolTCP),
},
},
Endpoints: []disv1beta1.Endpoint{
{
Addresses: []string{"4.3.2.1"},
TargetRef: &v1.ObjectReference{
Kind: "Pod",
Name: "testpod",
Namespace: "default",
},
Conditions: disv1beta1.EndpointConditions{
Ready: boolptr(false),
},
},
},
}
c.DiscoveryV1beta1().EndpointSlices(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{})
},
expectedMaxItems: 1,
expectedRes: map[string]*targetgroup.Group{
"endpointslice/default/testendpoints": {
Targets: []model.LabelSet{
{
"__address__": "4.3.2.1:9000",
"__meta_kubernetes_endpointslice_address_target_kind": "Pod",
"__meta_kubernetes_endpointslice_address_target_name": "testpod",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "false",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
"__meta_kubernetes_pod_container_name": "c1",
"__meta_kubernetes_pod_container_port_name": "mainport",
"__meta_kubernetes_pod_container_port_number": "9000",
"__meta_kubernetes_pod_container_port_protocol": "TCP",
"__meta_kubernetes_pod_host_ip": "2.3.4.5",
"__meta_kubernetes_pod_ip": "1.2.3.4",
"__meta_kubernetes_pod_name": "testpod",
"__meta_kubernetes_pod_node_name": "testnode",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "deadbeef",
},
{
"__address__": "1.2.3.4:9001",
"__meta_kubernetes_pod_container_name": "c2",
"__meta_kubernetes_pod_container_port_name": "sideport",
"__meta_kubernetes_pod_container_port_number": "9001",
"__meta_kubernetes_pod_container_port_protocol": "TCP",
"__meta_kubernetes_pod_host_ip": "2.3.4.5",
"__meta_kubernetes_pod_ip": "1.2.3.4",
"__meta_kubernetes_pod_name": "testpod",
"__meta_kubernetes_pod_node_name": "testnode",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "deadbeef",
},
},
Labels: model.LabelSet{
"__meta_kubernetes_endpointslice_address_type": "IPv4",
"__meta_kubernetes_endpointslice_name": "testendpoints",
"__meta_kubernetes_namespace": "default",
},
Source: "endpointslice/default/testendpoints",
},
},
}.Run(t)
}
func TestEndpointSliceDiscoveryDelete(t *testing.T) {
n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{}, makeEndpointSlice())
k8sDiscoveryTest{
discovery: n,
afterStart: func() {
obj := makeEndpointSlice()
c.DiscoveryV1beta1().EndpointSlices(obj.Namespace).Delete(context.Background(), obj.Name, metav1.DeleteOptions{})
},
expectedMaxItems: 2,
expectedRes: map[string]*targetgroup.Group{
"endpointslice/default/testendpoints": {
Source: "endpointslice/default/testendpoints",
},
},
}.Run(t)
}
func TestEndpointSliceDiscoveryUpdate(t *testing.T) {
n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{}, makeEndpointSlice())
k8sDiscoveryTest{
discovery: n,
afterStart: func() {
obj := &disv1beta1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "testendpoints",
Namespace: "default",
},
AddressType: disv1beta1.AddressTypeIPv4,
Ports: []disv1beta1.EndpointPort{
{
Name: strptr("testport"),
Port: int32ptr(9000),
Protocol: protocolptr(v1.ProtocolTCP),
},
},
Endpoints: []disv1beta1.Endpoint{
{
Addresses: []string{"1.2.3.4"},
Hostname: strptr("testendpoint1"),
}, {
Addresses: []string{"2.3.4.5"},
Conditions: disv1beta1.EndpointConditions{
Ready: boolptr(true),
},
},
},
}
c.DiscoveryV1beta1().EndpointSlices(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{})
},
expectedMaxItems: 2,
expectedRes: map[string]*targetgroup.Group{
"endpointslice/default/testendpoints": {
Targets: []model.LabelSet{
{
"__address__": "1.2.3.4:9000",
"__meta_kubernetes_endpointslice_endpoint_hostname": "testendpoint1",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
{
"__address__": "2.3.4.5:9000",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
},
Labels: model.LabelSet{
"__meta_kubernetes_endpointslice_address_type": "IPv4",
"__meta_kubernetes_endpointslice_name": "testendpoints",
"__meta_kubernetes_namespace": "default",
},
Source: "endpointslice/default/testendpoints",
},
},
}.Run(t)
}
func TestEndpointSliceDiscoveryEmptyEndpoints(t *testing.T) {
n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{}, makeEndpointSlice())
k8sDiscoveryTest{
discovery: n,
afterStart: func() {
obj := &disv1beta1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "testendpoints",
Namespace: "default",
},
AddressType: disv1beta1.AddressTypeIPv4,
Ports: []disv1beta1.EndpointPort{
{
Name: strptr("testport"),
Port: int32ptr(9000),
Protocol: protocolptr(v1.ProtocolTCP),
},
},
Endpoints: []disv1beta1.Endpoint{},
}
c.DiscoveryV1beta1().EndpointSlices(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{})
},
expectedMaxItems: 2,
expectedRes: map[string]*targetgroup.Group{
"endpointslice/default/testendpoints": {
Labels: model.LabelSet{
"__meta_kubernetes_endpointslice_address_type": "IPv4",
"__meta_kubernetes_endpointslice_name": "testendpoints",
"__meta_kubernetes_namespace": "default",
},
Source: "endpointslice/default/testendpoints",
},
},
}.Run(t)
}
func TestEndpointSliceDiscoveryWithService(t *testing.T) {
n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{}, makeEndpointSlice())
k8sDiscoveryTest{
discovery: n,
beforeRun: func() {
obj := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "testendpoints",
Namespace: "default",
Labels: map[string]string{
"app/name": "test",
},
},
}
c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{})
},
expectedMaxItems: 1,
expectedRes: map[string]*targetgroup.Group{
"endpointslice/default/testendpoints": {
Targets: []model.LabelSet{
{
"__address__": "1.2.3.4:9000",
"__meta_kubernetes_endpointslice_endpoint_hostname": "testendpoint1",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
{
"__address__": "2.3.4.5:9000",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
{
"__address__": "3.4.5.6:9000",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "false",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
},
Labels: model.LabelSet{
"__meta_kubernetes_endpointslice_address_type": "IPv4",
"__meta_kubernetes_endpointslice_name": "testendpoints",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_service_label_app_name": "test",
"__meta_kubernetes_service_labelpresent_app_name": "true",
"__meta_kubernetes_service_name": "testendpoints",
},
Source: "endpointslice/default/testendpoints",
},
},
}.Run(t)
}
func TestEndpointSliceDiscoveryWithServiceUpdate(t *testing.T) {
n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{}, makeEndpointSlice())
k8sDiscoveryTest{
discovery: n,
beforeRun: func() {
obj := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "testendpoints",
Namespace: "default",
Labels: map[string]string{
"app/name": "test",
},
},
}
c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{})
},
afterStart: func() {
obj := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "testendpoints",
Namespace: "default",
Labels: map[string]string{
"app/name": "svc",
"component": "testing",
},
},
}
c.CoreV1().Services(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{})
},
expectedMaxItems: 2,
expectedRes: map[string]*targetgroup.Group{
"endpointslice/default/testendpoints": {
Targets: []model.LabelSet{
{
"__address__": "1.2.3.4:9000",
"__meta_kubernetes_endpointslice_endpoint_hostname": "testendpoint1",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
{
"__address__": "2.3.4.5:9000",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
{
"__address__": "3.4.5.6:9000",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "false",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
},
Labels: model.LabelSet{
"__meta_kubernetes_endpointslice_address_type": "IPv4",
"__meta_kubernetes_endpointslice_name": "testendpoints",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_service_label_app_name": "svc",
"__meta_kubernetes_service_label_component": "testing",
"__meta_kubernetes_service_labelpresent_app_name": "true",
"__meta_kubernetes_service_labelpresent_component": "true",
"__meta_kubernetes_service_name": "testendpoints",
},
Source: "endpointslice/default/testendpoints",
},
},
}.Run(t)
}
func TestEndpointSliceDiscoveryNamespaces(t *testing.T) {
epOne := makeEndpointSlice()
epOne.Namespace = "ns1"
objs := []runtime.Object{
epOne,
&disv1beta1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "testendpoints",
Namespace: "ns2",
},
AddressType: disv1beta1.AddressTypeIPv4,
Ports: []disv1beta1.EndpointPort{
{
Name: strptr("testport"),
Port: int32ptr(9000),
Protocol: protocolptr(v1.ProtocolTCP),
},
},
Endpoints: []disv1beta1.Endpoint{
{
Addresses: []string{"4.3.2.1"},
TargetRef: &v1.ObjectReference{
Kind: "Pod",
Name: "testpod",
Namespace: "ns2",
},
},
},
},
&v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "testendpoints",
Namespace: "ns1",
Labels: map[string]string{
"app": "app1",
},
},
},
&v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "testpod",
Namespace: "ns2",
UID: types.UID("deadbeef"),
},
Spec: v1.PodSpec{
NodeName: "testnode",
Containers: []v1.Container{
{
Name: "c1",
Ports: []v1.ContainerPort{
{
Name: "mainport",
ContainerPort: 9000,
Protocol: v1.ProtocolTCP,
},
},
},
},
},
Status: v1.PodStatus{
HostIP: "2.3.4.5",
PodIP: "4.3.2.1",
},
},
}
n, _ := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"ns1", "ns2"}}, objs...)
k8sDiscoveryTest{
discovery: n,
expectedMaxItems: 2,
expectedRes: map[string]*targetgroup.Group{
"endpointslice/ns1/testendpoints": {
Targets: []model.LabelSet{
{
"__address__": "1.2.3.4:9000",
"__meta_kubernetes_endpointslice_endpoint_hostname": "testendpoint1",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
{
"__address__": "2.3.4.5:9000",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
{
"__address__": "3.4.5.6:9000",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "false",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
},
},
Labels: model.LabelSet{
"__meta_kubernetes_endpointslice_address_type": "IPv4",
"__meta_kubernetes_endpointslice_name": "testendpoints",
"__meta_kubernetes_namespace": "ns1",
"__meta_kubernetes_service_label_app": "app1",
"__meta_kubernetes_service_labelpresent_app": "true",
"__meta_kubernetes_service_name": "testendpoints",
},
Source: "endpointslice/ns1/testendpoints",
},
"endpointslice/ns2/testendpoints": {
Targets: []model.LabelSet{
{
"__address__": "4.3.2.1:9000",
"__meta_kubernetes_endpointslice_address_target_kind": "Pod",
"__meta_kubernetes_endpointslice_address_target_name": "testpod",
"__meta_kubernetes_endpointslice_port": "9000",
"__meta_kubernetes_endpointslice_port_name": "testport",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
"__meta_kubernetes_pod_container_name": "c1",
"__meta_kubernetes_pod_container_port_name": "mainport",
"__meta_kubernetes_pod_container_port_number": "9000",
"__meta_kubernetes_pod_container_port_protocol": "TCP",
"__meta_kubernetes_pod_host_ip": "2.3.4.5",
"__meta_kubernetes_pod_ip": "4.3.2.1",
"__meta_kubernetes_pod_name": "testpod",
"__meta_kubernetes_pod_node_name": "testnode",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "deadbeef",
},
},
Labels: model.LabelSet{
"__meta_kubernetes_endpointslice_address_type": "IPv4",
"__meta_kubernetes_endpointslice_name": "testendpoints",
"__meta_kubernetes_namespace": "ns2",
},
Source: "endpointslice/ns2/testendpoints",
},
},
}.Run(t)
}

25
discovery/kubernetes/ingress.go
View file

@ -20,7 +20,7 @@ import (
"github.com/go-kit/kit/log/level"
"github.com/pkg/errors"
"github.com/prometheus/common/model"
"k8s.io/api/extensions/v1beta1"
"k8s.io/api/networking/v1beta1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
@ -28,6 +28,12 @@ import (
"github.com/prometheus/prometheus/util/strutil"
)
var (
ingressAddCount = eventCount.WithLabelValues("ingress", "add")
ingressUpdateCount = eventCount.WithLabelValues("ingress", "update")
ingressDeleteCount = eventCount.WithLabelValues("ingress", "delete")
)
// Ingress implements discovery of Kubernetes ingress.
type Ingress struct {
logger log.Logger
@ -41,15 +47,15 @@ func NewIngress(l log.Logger, inf cache.SharedInformer) *Ingress {
s := &Ingress{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("ingress")}
s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(o interface{}) {
eventCount.WithLabelValues("ingress", "add").Inc()
ingressAddCount.Inc()
s.enqueue(o)
},
DeleteFunc: func(o interface{}) {
eventCount.WithLabelValues("ingress", "delete").Inc()
ingressDeleteCount.Inc()
s.enqueue(o)
},
UpdateFunc: func(_, o interface{}) {
eventCount.WithLabelValues("ingress", "update").Inc()
ingressUpdateCount.Inc()
s.enqueue(o)
},
})
@ -70,7 +76,9 @@ func (i *Ingress) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
defer i.queue.ShutDown()
if !cache.WaitForCacheSync(ctx.Done(), i.informer.HasSynced) {
level.Error(i.logger).Log("msg", "ingress informer unable to sync cache")
if ctx.Err() != context.Canceled {
level.Error(i.logger).Log("msg", "ingress informer unable to sync cache")
}
return
}
@ -101,7 +109,7 @@ func (i *Ingress) process(ctx context.Context, ch chan<- []*targetgroup.Group) b
return true
}
if !exists {
send(ctx, i.logger, RoleIngress, ch, &targetgroup.Group{Source: ingressSourceFromNamespaceAndName(namespace, name)})
send(ctx, ch, &targetgroup.Group{Source: ingressSourceFromNamespaceAndName(namespace, name)})
return true
}
eps, err := convertToIngress(o)
@ -109,7 +117,7 @@ func (i *Ingress) process(ctx context.Context, ch chan<- []*targetgroup.Group) b
level.Error(i.logger).Log("msg", "converting to Ingress object failed", "err", err)
return true
}
send(ctx, i.logger, RoleIngress, ch, i.buildIngress(eps))
send(ctx, ch, i.buildIngress(eps))
return true
}
@ -142,7 +150,8 @@ const (
)
func ingressLabels(ingress *v1beta1.Ingress) model.LabelSet {
ls := make(model.LabelSet, len(ingress.Labels)+len(ingress.Annotations)+2)
// Each label and annotation will create two key-value pairs in the map.
ls := make(model.LabelSet, 2*(len(ingress.Labels)+len(ingress.Annotations))+2)
ls[ingressNameLabel] = lv(ingress.Name)
ls[namespaceLabel] = lv(ingress.Namespace)

14
discovery/kubernetes/ingress_test.go
View file

@ -14,13 +14,15 @@
package kubernetes
import (
"context"
"fmt"
"testing"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery/targetgroup"
"k8s.io/api/extensions/v1beta1"
"k8s.io/api/networking/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
type TLSMode int
@ -137,7 +139,7 @@ func TestIngressDiscoveryAdd(t *testing.T) {
discovery: n,
afterStart: func() {
obj := makeIngress(TLSNo)
c.ExtensionsV1beta1().Ingresses("default").Create(obj)
c.NetworkingV1beta1().Ingresses("default").Create(context.Background(), obj, metav1.CreateOptions{})
},
expectedMaxItems: 1,
expectedRes: expectedTargetGroups("default", TLSNo),
@ -151,7 +153,7 @@ func TestIngressDiscoveryAddTLS(t *testing.T) {
discovery: n,
afterStart: func() {
obj := makeIngress(TLSYes)
c.ExtensionsV1beta1().Ingresses("default").Create(obj)
c.NetworkingV1beta1().Ingresses("default").Create(context.Background(), obj, metav1.CreateOptions{})
},
expectedMaxItems: 1,
expectedRes: expectedTargetGroups("default", TLSYes),
@ -165,7 +167,7 @@ func TestIngressDiscoveryAddMixed(t *testing.T) {
discovery: n,
afterStart: func() {
obj := makeIngress(TLSMixed)
c.ExtensionsV1beta1().Ingresses("default").Create(obj)
c.NetworkingV1beta1().Ingresses("default").Create(context.Background(), obj, metav1.CreateOptions{})
},
expectedMaxItems: 1,
expectedRes: expectedTargetGroups("default", TLSMixed),
@ -185,7 +187,7 @@ func TestIngressDiscoveryNamespaces(t *testing.T) {
for _, ns := range []string{"ns1", "ns2"} {
obj := makeIngress(TLSNo)
obj.Namespace = ns
c.ExtensionsV1beta1().Ingresses(obj.Namespace).Create(obj)
c.NetworkingV1beta1().Ingresses(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{})
}
},
expectedMaxItems: 2,

239
discovery/kubernetes/kubernetes.go
View file

@ -15,7 +15,9 @@ package kubernetes
import (
"context"
"fmt"
"reflect"
"strings"
"sync"
"time"
@ -26,14 +28,17 @@ import (
config_util "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
apiv1 "k8s.io/api/core/v1"
extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
disv1beta1 "k8s.io/api/discovery/v1beta1"
"k8s.io/api/networking/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"github.com/prometheus/common/version"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
@ -47,6 +52,8 @@ const (
)
var (
// Http header
userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
// Custom events metric
eventCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
@ -65,11 +72,12 @@ type Role string
// The valid options for Role.
const (
RoleNode Role = "node"
RolePod Role = "pod"
RoleService Role = "service"
RoleEndpoint Role = "endpoints"
RoleIngress Role = "ingress"
RoleNode Role = "node"
RolePod Role = "pod"
RoleService Role = "service"
RoleEndpoint Role = "endpoints"
RoleEndpointSlice Role = "endpointslice"
RoleIngress Role = "ingress"
)
// UnmarshalYAML implements the yaml.Unmarshaler interface.
@ -78,7 +86,7 @@ func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error {
return err
}
switch *c {
case RoleNode, RolePod, RoleService, RoleEndpoint, RoleIngress:
case RoleNode, RolePod, RoleService, RoleEndpoint, RoleEndpointSlice, RoleIngress:
return nil
default:
return errors.Errorf("unknown Kubernetes SD role %q", *c)
@ -91,6 +99,27 @@ type SDConfig struct {
Role Role `yaml:"role"`
HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
NamespaceDiscovery NamespaceDiscovery `yaml:"namespaces,omitempty"`
Selectors []SelectorConfig `yaml:"selectors,omitempty"`
}
type roleSelector struct {
node resourceSelector
pod resourceSelector
service resourceSelector
endpoints resourceSelector
endpointslice resourceSelector
ingress resourceSelector
}
type SelectorConfig struct {
Role Role `yaml:"role,omitempty"`
Label string `yaml:"label,omitempty"`
Field string `yaml:"field,omitempty"`
}
type resourceSelector struct {
label string
field string
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
@ -102,7 +131,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
return err
}
if c.Role == "" {
return errors.Errorf("role missing (one of: pod, service, endpoints, node, ingress)")
return errors.Errorf("role missing (one of: pod, service, endpoints, endpointslice, node, ingress)")
}
err = c.HTTPClientConfig.Validate()
if err != nil {
@ -111,6 +140,47 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
if c.APIServer.URL == nil && !reflect.DeepEqual(c.HTTPClientConfig, config_util.HTTPClientConfig{}) {
return errors.Errorf("to use custom HTTP client configuration please provide the 'api_server' URL explicitly")
}
foundSelectorRoles := make(map[Role]struct{})
allowedSelectors := map[Role][]string{
RolePod: {string(RolePod)},
RoleService: {string(RoleService)},
RoleEndpointSlice: {string(RolePod), string(RoleService), string(RoleEndpointSlice)},
RoleEndpoint: {string(RolePod), string(RoleService), string(RoleEndpoint)},
RoleNode: {string(RoleNode)},
RoleIngress: {string(RoleIngress)},
}
for _, selector := range c.Selectors {
if _, ok := foundSelectorRoles[selector.Role]; ok {
return errors.Errorf("duplicated selector role: %s", selector.Role)
}
foundSelectorRoles[selector.Role] = struct{}{}
if _, ok := allowedSelectors[c.Role]; !ok {
return errors.Errorf("invalid role: %q, expecting one of: pod, service, endpoints, endpointslice, node or ingress", c.Role)
}
var allowed bool
for _, role := range allowedSelectors[c.Role] {
if role == string(selector.Role) {
allowed = true
break
}
}
if !allowed {
return errors.Errorf("%s role supports only %s selectors", c.Role, strings.Join(allowedSelectors[c.Role], ", "))
}
_, err := fields.ParseSelector(selector.Field)
if err != nil {
return err
}
_, err = fields.ParseSelector(selector.Label)
if err != nil {
return err
}
}
return nil
}
@ -131,7 +201,7 @@ func init() {
prometheus.MustRegister(eventCount)
// Initialize metric vectors.
for _, role := range []string{"endpoints", "node", "pod", "service", "ingress"} {
for _, role := range []string{"endpointslice", "endpoints", "node", "pod", "service", "ingress"} {
for _, evt := range []string{"add", "delete", "update"} {
eventCount.WithLabelValues(role, evt)
}
@ -139,12 +209,10 @@ func init() {
var (
clientGoRequestMetricAdapterInstance = clientGoRequestMetricAdapter{}
clientGoCacheMetricsProviderInstance = clientGoCacheMetricsProvider{}
clientGoWorkqueueMetricsProviderInstance = clientGoWorkqueueMetricsProvider{}
)
clientGoRequestMetricAdapterInstance.Register(prometheus.DefaultRegisterer)
clientGoCacheMetricsProviderInstance.Register(prometheus.DefaultRegisterer)
clientGoWorkqueueMetricsProviderInstance.Register(prometheus.DefaultRegisterer)
}
@ -163,6 +231,7 @@ type Discovery struct {
logger log.Logger
namespaceDiscovery *NamespaceDiscovery
discoverers []discoverer
selectors roleSelector
}
func (d *Discovery) getNamespaces() []string {
@ -201,7 +270,7 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
}
}
kcfg.UserAgent = "Prometheus/discovery"
kcfg.UserAgent = userAgent
c, err := kubernetes.NewForConfig(kcfg)
if err != nil {
@ -213,9 +282,37 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
role: conf.Role,
namespaceDiscovery: &conf.NamespaceDiscovery,
discoverers: make([]discoverer, 0),
selectors: mapSelector(conf.Selectors),
}, nil
}
func mapSelector(rawSelector []SelectorConfig) roleSelector {
rs := roleSelector{}
for _, resourceSelectorRaw := range rawSelector {
switch resourceSelectorRaw.Role {
case RoleEndpointSlice:
rs.endpointslice.field = resourceSelectorRaw.Field
rs.endpointslice.label = resourceSelectorRaw.Label
case RoleEndpoint:
rs.endpoints.field = resourceSelectorRaw.Field
rs.endpoints.label = resourceSelectorRaw.Label
case RoleIngress:
rs.ingress.field = resourceSelectorRaw.Field
rs.ingress.label = resourceSelectorRaw.Label
case RoleNode:
rs.node.field = resourceSelectorRaw.Field
rs.node.label = resourceSelectorRaw.Label
case RolePod:
rs.pod.field = resourceSelectorRaw.Field
rs.pod.label = resourceSelectorRaw.Label
case RoleService:
rs.service.field = resourceSelectorRaw.Field
rs.service.label = resourceSelectorRaw.Label
}
}
return rs
}
const resyncPeriod = 10 * time.Minute
// Run implements the discoverer interface.
@ -224,33 +321,97 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
namespaces := d.getNamespaces()
switch d.role {
case RoleEndpoint:
case RoleEndpointSlice:
for _, namespace := range namespaces {
e := d.client.CoreV1().Endpoints(namespace)
e := d.client.DiscoveryV1beta1().EndpointSlices(namespace)
elw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
return e.List(options)
options.FieldSelector = d.selectors.endpointslice.field
options.LabelSelector = d.selectors.endpointslice.label
return e.List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return e.Watch(options)
options.FieldSelector = d.selectors.endpointslice.field
options.LabelSelector = d.selectors.endpointslice.label
return e.Watch(ctx, options)
},
}
s := d.client.CoreV1().Services(namespace)
slw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
return s.List(options)
options.FieldSelector = d.selectors.service.field
options.LabelSelector = d.selectors.service.label
return s.List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return s.Watch(options)
options.FieldSelector = d.selectors.service.field
options.LabelSelector = d.selectors.service.label
return s.Watch(ctx, options)
},
}
p := d.client.CoreV1().Pods(namespace)
plw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
return p.List(options)
options.FieldSelector = d.selectors.pod.field
options.LabelSelector = d.selectors.pod.label
return p.List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return p.Watch(options)
options.FieldSelector = d.selectors.pod.field
options.LabelSelector = d.selectors.pod.label
return p.Watch(ctx, options)
},
}
eps := NewEndpointSlice(
log.With(d.logger, "role", "endpointslice"),
cache.NewSharedInformer(slw, &apiv1.Service{}, resyncPeriod),
cache.NewSharedInformer(elw, &disv1beta1.EndpointSlice{}, resyncPeriod),
cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncPeriod),
)
d.discoverers = append(d.discoverers, eps)
go eps.endpointSliceInf.Run(ctx.Done())
go eps.serviceInf.Run(ctx.Done())
go eps.podInf.Run(ctx.Done())
}
case RoleEndpoint:
for _, namespace := range namespaces {
e := d.client.CoreV1().Endpoints(namespace)
elw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = d.selectors.endpoints.field
options.LabelSelector = d.selectors.endpoints.label
return e.List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = d.selectors.endpoints.field
options.LabelSelector = d.selectors.endpoints.label
return e.Watch(ctx, options)
},
}
s := d.client.CoreV1().Services(namespace)
slw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = d.selectors.service.field
options.LabelSelector = d.selectors.service.label
return s.List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = d.selectors.service.field
options.LabelSelector = d.selectors.service.label
return s.Watch(ctx, options)
},
}
p := d.client.CoreV1().Pods(namespace)
plw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = d.selectors.pod.field
options.LabelSelector = d.selectors.pod.label
return p.List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = d.selectors.pod.field
options.LabelSelector = d.selectors.pod.label
return p.Watch(ctx, options)
},
}
eps := NewEndpoints(
@ -269,10 +430,14 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
p := d.client.CoreV1().Pods(namespace)
plw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
return p.List(options)
options.FieldSelector = d.selectors.pod.field
options.LabelSelector = d.selectors.pod.label
return p.List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return p.Watch(options)
options.FieldSelector = d.selectors.pod.field
options.LabelSelector = d.selectors.pod.label
return p.Watch(ctx, options)
},
}
pod := NewPod(
@ -287,10 +452,14 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
s := d.client.CoreV1().Services(namespace)
slw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
return s.List(options)
options.FieldSelector = d.selectors.service.field
options.LabelSelector = d.selectors.service.label
return s.List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return s.Watch(options)
options.FieldSelector = d.selectors.service.field
options.LabelSelector = d.selectors.service.label
return s.Watch(ctx, options)
},
}
svc := NewService(
@ -302,18 +471,22 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
}
case RoleIngress:
for _, namespace := range namespaces {
i := d.client.ExtensionsV1beta1().Ingresses(namespace)
i := d.client.NetworkingV1beta1().Ingresses(namespace)
ilw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
return i.List(options)
options.FieldSelector = d.selectors.ingress.field
options.LabelSelector = d.selectors.ingress.label
return i.List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return i.Watch(options)
options.FieldSelector = d.selectors.ingress.field
options.LabelSelector = d.selectors.ingress.label
return i.Watch(ctx, options)
},
}
ingress := NewIngress(
log.With(d.logger, "role", "ingress"),
cache.NewSharedInformer(ilw, &extensionsv1beta1.Ingress{}, resyncPeriod),
cache.NewSharedInformer(ilw, &v1beta1.Ingress{}, resyncPeriod),
)
d.discoverers = append(d.discoverers, ingress)
go ingress.informer.Run(ctx.Done())
@ -321,10 +494,14 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
case RoleNode:
nlw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
return d.client.CoreV1().Nodes().List(options)
options.FieldSelector = d.selectors.node.field
options.LabelSelector = d.selectors.node.label
return d.client.CoreV1().Nodes().List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return d.client.CoreV1().Nodes().Watch(options)
options.FieldSelector = d.selectors.node.field
options.LabelSelector = d.selectors.node.label
return d.client.CoreV1().Nodes().Watch(ctx, options)
},
}
node := NewNode(
@ -356,7 +533,7 @@ func lv(s string) model.LabelValue {
return model.LabelValue(s)
}
func send(ctx context.Context, l log.Logger, role Role, ch chan<- []*targetgroup.Group, tg *targetgroup.Group) {
func send(ctx context.Context, ch chan<- []*targetgroup.Group, tg *targetgroup.Group) {
if tg == nil {
return
}

53
discovery/kubernetes/kubernetes_test.go
View file

@ -20,14 +20,19 @@ import (
"time"
"github.com/go-kit/kit/log"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/testutil"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/tools/cache"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/testutil"
)
func TestMain(m *testing.M) {
testutil.TolerantVerifyLeak(m)
}
// makeDiscovery creates a kubernetes.Discovery instance for testing.
func makeDiscovery(role Role, nsDiscovery NamespaceDiscovery, objects ...runtime.Object) (*Discovery, kubernetes.Interface) {
clientset := fake.NewSimpleClientset(objects...)
@ -65,6 +70,21 @@ func (d k8sDiscoveryTest) Run(t *testing.T) {
// Run discoverer and start a goroutine to read results.
go d.discovery.Run(ctx, ch)
// Ensure that discovery has a discoverer set. This prevents a race
// condition where the above go routine may or may not have set a
// discoverer yet.
for {
dis := d.discovery.(*Discovery)
dis.RLock()
l := len(dis.discoverers)
dis.RUnlock()
if l > 0 {
break
}
time.Sleep(10 * time.Millisecond)
}
resChan := make(chan map[string]*targetgroup.Group)
go readResultWithTimeout(t, ch, d.expectedMaxItems, time.Second, resChan)
@ -91,14 +111,18 @@ func (d k8sDiscoveryTest) Run(t *testing.T) {
// readResultWithTimeout reads all targegroups from channel with timeout.
// It merges targegroups by source and sends the result to result channel.
func readResultWithTimeout(t *testing.T, ch <-chan []*targetgroup.Group, max int, timeout time.Duration, resChan chan<- map[string]*targetgroup.Group) {
allTgs := make([][]*targetgroup.Group, 0)
res := make(map[string]*targetgroup.Group)
Loop:
for {
select {
case tgs := <-ch:
allTgs = append(allTgs, tgs)
if len(allTgs) == max {
for _, tg := range tgs {
if tg == nil {
continue
}
res[tg.Source] = tg
}
if len(res) == max {
// Reached max target groups we may get, break fast.
break Loop
}
@ -106,21 +130,11 @@ Loop:
// Because we use queue, an object that is created then
// deleted or updated may be processed only once.
// So possibly we may skip events, timed out here.
t.Logf("timed out, got %d (max: %d) items, some events are skipped", len(allTgs), max)
t.Logf("timed out, got %d (max: %d) items, some events are skipped", len(res), max)
break Loop
}
}
// Merge by source and sent it to channel.
res := make(map[string]*targetgroup.Group)
for _, tgs := range allTgs {
for _, tg := range tgs {
if tg == nil {
continue
}
res[tg.Source] = tg
}
}
resChan <- res
}
@ -148,6 +162,7 @@ type hasSynced interface {
var _ hasSynced = &Discovery{}
var _ hasSynced = &Node{}
var _ hasSynced = &Endpoints{}
var _ hasSynced = &EndpointSlice{}
var _ hasSynced = &Ingress{}
var _ hasSynced = &Pod{}
var _ hasSynced = &Service{}
@ -173,6 +188,10 @@ func (e *Endpoints) hasSynced() bool {
return e.endpointsInf.HasSynced() && e.serviceInf.HasSynced() && e.podInf.HasSynced()
}
func (e *EndpointSlice) hasSynced() bool {
return e.endpointSliceInf.HasSynced() && e.serviceInf.HasSynced() && e.podInf.HasSynced()
}
func (i *Ingress) hasSynced() bool {
return i.informer.HasSynced()
}

23
discovery/kubernetes/node.go
View file

@ -34,6 +34,12 @@ const (
NodeLegacyHostIP = "LegacyHostIP"
)
var (
nodeAddCount = eventCount.WithLabelValues("node", "add")
nodeUpdateCount = eventCount.WithLabelValues("node", "update")
nodeDeleteCount = eventCount.WithLabelValues("node", "delete")
)
// Node discovers Kubernetes nodes.
type Node struct {
logger log.Logger
@ -50,15 +56,15 @@ func NewNode(l log.Logger, inf cache.SharedInformer) *Node {
n := &Node{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("node")}
n.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(o interface{}) {
eventCount.WithLabelValues("node", "add").Inc()
nodeAddCount.Inc()
n.enqueue(o)
},
DeleteFunc: func(o interface{}) {
eventCount.WithLabelValues("node", "delete").Inc()
nodeDeleteCount.Inc()
n.enqueue(o)
},
UpdateFunc: func(_, o interface{}) {
eventCount.WithLabelValues("node", "update").Inc()
nodeUpdateCount.Inc()
n.enqueue(o)
},
})
@ -79,7 +85,9 @@ func (n *Node) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
defer n.queue.ShutDown()
if !cache.WaitForCacheSync(ctx.Done(), n.informer.HasSynced) {
level.Error(n.logger).Log("msg", "node informer unable to sync cache")
if ctx.Err() != context.Canceled {
level.Error(n.logger).Log("msg", "node informer unable to sync cache")
}
return
}
@ -110,7 +118,7 @@ func (n *Node) process(ctx context.Context, ch chan<- []*targetgroup.Group) bool
return true
}
if !exists {
send(ctx, n.logger, RoleNode, ch, &targetgroup.Group{Source: nodeSourceFromName(name)})
send(ctx, ch, &targetgroup.Group{Source: nodeSourceFromName(name)})
return true
}
node, err := convertToNode(o)
@ -118,7 +126,7 @@ func (n *Node) process(ctx context.Context, ch chan<- []*targetgroup.Group) bool
level.Error(n.logger).Log("msg", "converting to Node object failed", "err", err)
return true
}
send(ctx, n.logger, RoleNode, ch, n.buildNode(node))
send(ctx, ch, n.buildNode(node))
return true
}
@ -149,7 +157,8 @@ const (
)
func nodeLabels(n *apiv1.Node) model.LabelSet {
ls := make(model.LabelSet, len(n.Labels)+len(n.Annotations)+1)
// Each label and annotation will create two key-value pairs in the map.
ls := make(model.LabelSet, 2*(len(n.Labels)+len(n.Annotations))+1)
ls[nodeNameLabel] = lv(n.Name)

14
discovery/kubernetes/node_test.go
View file

@ -14,13 +14,15 @@
package kubernetes
import (
"context"
"fmt"
"testing"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery/targetgroup"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
func makeNode(name, address string, labels map[string]string, annotations map[string]string) *v1.Node {
@ -62,7 +64,7 @@ func TestNodeDiscoveryBeforeStart(t *testing.T) {
map[string]string{"test-label": "testvalue"},
map[string]string{"test-annotation": "testannotationvalue"},
)
c.CoreV1().Nodes().Create(obj)
c.CoreV1().Nodes().Create(context.Background(), obj, metav1.CreateOptions{})
},
expectedMaxItems: 1,
expectedRes: map[string]*targetgroup.Group{
@ -94,7 +96,7 @@ func TestNodeDiscoveryAdd(t *testing.T) {
discovery: n,
afterStart: func() {
obj := makeEnumeratedNode(1)
c.CoreV1().Nodes().Create(obj)
c.CoreV1().Nodes().Create(context.Background(), obj, metav1.CreateOptions{})
},
expectedMaxItems: 1,
expectedRes: map[string]*targetgroup.Group{
@ -122,7 +124,7 @@ func TestNodeDiscoveryDelete(t *testing.T) {
k8sDiscoveryTest{
discovery: n,
afterStart: func() {
c.CoreV1().Nodes().Delete(obj.Name, &metav1.DeleteOptions{})
c.CoreV1().Nodes().Delete(context.Background(), obj.Name, metav1.DeleteOptions{})
},
expectedMaxItems: 2,
expectedRes: map[string]*targetgroup.Group{
@ -140,14 +142,14 @@ func TestNodeDiscoveryUpdate(t *testing.T) {
discovery: n,
afterStart: func() {
obj1 := makeEnumeratedNode(0)
c.CoreV1().Nodes().Create(obj1)
c.CoreV1().Nodes().Create(context.Background(), obj1, metav1.CreateOptions{})
obj2 := makeNode(
"test0",
"1.2.3.4",
map[string]string{"Unschedulable": "true"},
map[string]string{},
)
c.CoreV1().Nodes().Update(obj2)
c.CoreV1().Nodes().Update(context.Background(), obj2, metav1.UpdateOptions{})
},
expectedMaxItems: 2,
expectedRes: map[string]*targetgroup.Group{

Some files were not shown because too many files have changed in this diff Show more