Merge branch 'main' into elide-queriers

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>

.github/CODEOWNERS

@@ -1,7 +1,7 @@
 /web/ui @juliusv
 /web/ui/module @juliusv @nexucis
 /storage/remote @cstyan @bwplotka @tomwilkie
-/storage/remote/otlptranslator @gouthamve @jesusvazquez
+/storage/remote/otlptranslator @aknuds1 @jesusvazquez
 /discovery/kubernetes @brancz
 /tsdb @jesusvazquez
 /promql @roidelapluie

.github/dependabot.yml

@@ -6,11 +6,12 @@ updates:
       interval: "monthly"
     groups:
       k8s.io:
-          patterns:
+        patterns:
-            - "k8s.io/*"
+          - "k8s.io/*"
       go.opentelemetry.io:
-          patterns:
+        patterns:
-            - "go.opentelemetry.io/*"
+          - "go.opentelemetry.io/*"
+    open-pull-requests-limit: 20
   - package-ecosystem: "gomod"
     directory: "/documentation/examples/remote_storage"
     schedule:
@@ -19,6 +20,7 @@ updates:
     directory: "/web/ui"
     schedule:
       interval: "monthly"
+    open-pull-requests-limit: 20
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/workflows/buf-lint.yml

@@ -12,14 +12,14 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
-      - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa # v1.28.1
+      - uses: bufbuild/buf-setup-action@dde0b9351db90fbf78e345f41a57de8514bf1091 # v1.32.2
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: bufbuild/buf-lint-action@bd48f53224baaaf0fc55de9a913e7680ca6dbea4 # v1.0.3
+      - uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1
         with:
           input: 'prompb'
-      - uses: bufbuild/buf-breaking-action@f47418c81c00bfd65394628385593542f64db477 # v1.1.2
+      - uses: bufbuild/buf-breaking-action@c57b3d842a5c3f3b454756ef65305a50a587c5ba # v1.1.4
         with:
           input: 'prompb'
           against: 'https://github.com/prometheus/prometheus.git#branch=main,ref=HEAD,subdir=prompb'

.github/workflows/buf.yml

@@ -12,14 +12,14 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'prometheus'
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
-      - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa # v1.28.1
+      - uses: bufbuild/buf-setup-action@dde0b9351db90fbf78e345f41a57de8514bf1091 # v1.32.2
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: bufbuild/buf-lint-action@bd48f53224baaaf0fc55de9a913e7680ca6dbea4 # v1.0.3
+      - uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1
         with:
           input: 'prompb'
-      - uses: bufbuild/buf-breaking-action@f47418c81c00bfd65394628385593542f64db477 # v1.1.2
+      - uses: bufbuild/buf-breaking-action@c57b3d842a5c3f3b454756ef65305a50a587c5ba # v1.1.4
         with:
           input: 'prompb'
           against: 'https://github.com/prometheus/prometheus.git#branch=main,ref=HEAD~1,subdir=prompb'

.github/workflows/ci.yml

@@ -8,34 +8,56 @@ jobs:
   test_go:
     name: Go tests
     runs-on: ubuntu-latest
-    # Whenever the Go version is updated here, .promu.yml
-    # should also be updated.
     container:
-      image: quay.io/prometheus/golang-builder:1.21-base
+      # Whenever the Go version is updated here, .promu.yml
+      # should also be updated.
+      image: quay.io/prometheus/golang-builder:1.22-base
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
       - uses: ./.github/promci/actions/setup_environment
-      - run: make GO_ONLY=1 SKIP_GOLANGCI_LINT=1
+      - run: make GOOPTS=--tags=stringlabels GO_ONLY=1 SKIP_GOLANGCI_LINT=1
-      - run: go test ./tsdb/ -test.tsdb-isolation=false
+      - run: go test --tags=stringlabels ./tsdb/ -test.tsdb-isolation=false
-      - run: go test --tags=stringlabels ./...
-      - run: GOARCH=386 go test ./cmd/prometheus
       - run: make -C documentation/examples/remote_storage
       - run: make -C documentation/examples
+
+  test_go_more:
+    name: More Go tests
+    runs-on: ubuntu-latest
+    container:
+      image: quay.io/prometheus/golang-builder:1.22-base
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
+      - uses: ./.github/promci/actions/setup_environment
+      - run: go test --tags=dedupelabels ./...
+      - run: GOARCH=386 go test ./cmd/prometheus
       - uses: ./.github/promci/actions/check_proto
         with:
           version: "3.15.8"
 
+  test_go_oldest:
+    name: Go tests with previous Go version
+    runs-on: ubuntu-latest
+    container:
+      # The go version in this image should be N-1 wrt test_go.
+      image: quay.io/prometheus/golang-builder:1.21-base
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - run: make build
+      # Don't run NPM build; don't run race-detector.
+      - run: make test GO_ONLY=1 test-flags=""
+
   test_ui:
     name: UI tests
     runs-on: ubuntu-latest
     # Whenever the Go version is updated here, .promu.yml
     # should also be updated.
     container:
-      image: quay.io/prometheus/golang-builder:1.21-base
+      image: quay.io/prometheus/golang-builder:1.22-base
 
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
       - uses: ./.github/promci/actions/setup_environment
         with:
@@ -52,36 +74,24 @@ jobs:
     name: Go tests on Windows
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
-      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
+      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
         with:
-          go-version: 1.21.x
+          go-version: 1.22.x
       - run: |
           $TestTargets = go list ./... | Where-Object { $_ -NotMatch "(github.com/prometheus/prometheus/discovery.*|github.com/prometheus/prometheus/config|github.com/prometheus/prometheus/web)"}
           go test $TestTargets -vet=off -v
         shell: powershell
 
-  test_golang_oldest:
-    name: Go tests with previous Go version
-    runs-on: ubuntu-latest
-    # The go verson in this image should be N-1 wrt test_go.
-    container:
-      image: quay.io/prometheus/golang-builder:1.20-base
-    steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      - run: make build
-      - run: go test ./tsdb/...
-      - run: go test ./tsdb/ -test.tsdb-isolation=false
-
   test_mixins:
     name: Mixins tests
     runs-on: ubuntu-latest
     # Whenever the Go version is updated here, .promu.yml
     # should also be updated.
     container:
-      image: quay.io/prometheus/golang-builder:1.20-base
+      image: quay.io/prometheus/golang-builder:1.22-base
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       - run: go install ./cmd/promtool/.
       - run: go install github.com/google/go-jsonnet/cmd/jsonnet@latest
       - run: go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest
@@ -104,7 +114,7 @@ jobs:
       matrix:
         thread: [ 0, 1, 2 ]
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
       - uses: ./.github/promci/actions/build
         with:
@@ -127,32 +137,44 @@ jobs:
     # Whenever the Go version is updated here, .promu.yml
     # should also be updated.
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
       - uses: ./.github/promci/actions/build
         with:
           parallelism: 12
           thread: ${{ matrix.thread }}
+  check_generated_parser:
+    name: Check generated parser
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - name: Install Go
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
+        with:
+          cache: false
+          go-version: 1.22.x
+      - name: Run goyacc and check for diff
+        run: make install-goyacc check-generated-parser
   golangci:
     name: golangci-lint
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       - name: Install Go
-        uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
         with:
-          cache: false
+          go-version: 1.22.x
-          go-version: 1.21.x
       - name: Install snmp_exporter/generator dependencies
         run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
         if: github.repository == 'prometheus/snmp_exporter'
       - name: Lint
-        uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
+        uses: golangci/golangci-lint-action@a4f60bb28d35aeee14e6880718e0c85ff1882e64 # v6.0.1
         with:
           args: --verbose
           # Make sure to sync this with Makefile.common and scripts/golangci-lint.yml.
-          version: v1.55.2
+          version: v1.59.0
   fuzzing:
     uses: ./.github/workflows/fuzzing.yml
     if: github.event_name == 'pull_request'
@@ -162,10 +184,10 @@ jobs:
   publish_main:
     name: Publish main branch artifacts
     runs-on: ubuntu-latest
-    needs: [test_ui, test_go, test_windows, golangci, codeql, build_all]
+    needs: [test_ui, test_go, test_go_more, test_go_oldest, test_windows, golangci, codeql, build_all]
     if: github.event_name == 'push' && github.event.ref == 'refs/heads/main'
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
       - uses: ./.github/promci/actions/publish_main
         with:
@@ -176,10 +198,10 @@ jobs:
   publish_release:
     name: Publish release artefacts
     runs-on: ubuntu-latest
-    needs: [test_ui, test_go, test_windows, golangci, codeql, build_all]
+    needs: [test_ui, test_go, test_go_more, test_go_oldest, test_windows, golangci, codeql, build_all]
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
       - uses: ./.github/promci/actions/publish_release
         with:
@@ -194,14 +216,14 @@ jobs:
     needs: [test_ui, codeql]
     steps:
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
       - name: Install nodejs
-        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
+        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
         with:
           node-version-file: "web/ui/.nvmrc"
           registry-url: "https://registry.npmjs.org"
-      - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}

.github/workflows/codeql-analysis.yml

@@ -20,22 +20,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: ["go", "javascript"]
+        language: ["javascript"]
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
-      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
-        with:
-          go-version: 1.21.x
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
+        uses: github/codeql-action/init@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8
         with:
           languages: ${{ matrix.language }}
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
+        uses: github/codeql-action/autobuild@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
+        uses: github/codeql-action/analyze@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8

.github/workflows/container_description.yml

@@ -0,0 +1,57 @@
+---
+name: Push README to Docker Hub
+on:
+  push:
+    paths:
+      - "README.md"
+      - "README-containers.md"
+      - ".github/workflows/container_description.yml"
+    branches: [ main, master ]
+
+permissions:
+  contents: read
+
+jobs:
+  PushDockerHubReadme:
+    runs-on: ubuntu-latest
+    name: Push README to Docker Hub
+    if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks.
+    steps:
+      - name: git checkout
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - name: Set docker hub repo name
+        run: echo "DOCKER_REPO_NAME=$(make docker-repo-name)" >> $GITHUB_ENV
+      - name: Push README to Dockerhub
+        uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_HUB_LOGIN }}
+          DOCKER_PASS: ${{ secrets.DOCKER_HUB_PASSWORD }}
+        with:
+          destination_container_repo: ${{ env.DOCKER_REPO_NAME }}
+          provider: dockerhub
+          short_description: ${{ env.DOCKER_REPO_NAME }}
+          # Empty string results in README-containers.md being pushed if it
+          # exists. Otherwise, README.md is pushed.
+          readme_file: ''
+
+  PushQuayIoReadme:
+    runs-on: ubuntu-latest
+    name: Push README to quay.io
+    if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks.
+    steps:
+      - name: git checkout
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - name: Set quay.io org name
+        run: echo "DOCKER_REPO=$(echo quay.io/${GITHUB_REPOSITORY_OWNER} | tr -d '-')" >> $GITHUB_ENV
+      - name: Set quay.io repo name
+        run: echo "DOCKER_REPO_NAME=$(make docker-repo-name)" >> $GITHUB_ENV
+      - name: Push README to quay.io
+        uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1
+        env:
+          DOCKER_APIKEY: ${{ secrets.QUAY_IO_API_TOKEN }}
+        with:
+          destination_container_repo: ${{ env.DOCKER_REPO_NAME }}
+          provider: quay
+          # Empty string results in README-containers.md being pushed if it
+          # exists. Otherwise, README.md is pushed.
+          readme_file: ''

.github/workflows/fuzzing.yml

@@ -21,7 +21,7 @@ jobs:
           fuzz-seconds: 600
           dry-run: false
       - name: Upload Crash
-        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         if: failure() && steps.build.outcome == 'success'
         with:
           name: artifacts

.github/workflows/repo_sync.yml

@@ -13,7 +13,7 @@ jobs:
     container:
       image: quay.io/prometheus/golang-builder
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       - run: ./scripts/sync_repo_files.sh
         env:
           GITHUB_TOKEN: ${{ secrets.PROMBOT_GITHUB_TOKEN }}

.github/workflows/scorecards.yml

@@ -21,12 +21,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # tag=v4.1.6
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # tag=v2.3.1
+        uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # tag=v2.3.3
         with:
           results_file: results.sarif
           results_format: sarif
@@ -37,7 +37,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # tag=v4.0.0
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # tag=v4.3.3
         with:
           name: SARIF file
           path: results.sarif
@@ -45,6 +45,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # tag=v3.22.12
+        uses: github/codeql-action/upload-sarif@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # tag=v3.25.8
         with:
           sarif_file: results.sarif

.gitpod.Dockerfile

@@ -1,15 +1,33 @@
 FROM gitpod/workspace-full
 
+# Set Node.js version as an environment variable.
 ENV CUSTOM_NODE_VERSION=16
-ENV CUSTOM_GO_VERSION=1.19
-ENV GOPATH=$HOME/go-packages
-ENV GOROOT=$HOME/go
-ENV PATH=$GOROOT/bin:$GOPATH/bin:$PATH
 
+# Install and use the specified Node.js version via nvm.
 RUN bash -c ". .nvm/nvm.sh && nvm install ${CUSTOM_NODE_VERSION} && nvm use ${CUSTOM_NODE_VERSION} && nvm alias default ${CUSTOM_NODE_VERSION}"
 
+# Ensure nvm uses the default Node.js version in all new shells.
 RUN echo "nvm use default &>/dev/null" >> ~/.bashrc.d/51-nvm-fix
-RUN curl -fsSL https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz | tar xzs \
-    && printf '%s\n' 'export GOPATH=/workspace/go' \
-                      'export PATH=$GOPATH/bin:$PATH' > $HOME/.bashrc.d/300-go
 
+# Remove any existing Go installation in $HOME path.
+RUN rm -rf $HOME/go $HOME/go-packages
+
+# Export go environment variables.
+RUN echo "export GOPATH=/workspace/go" >> ~/.bashrc.d/300-go && \
+    echo "export GOBIN=\$GOPATH/bin" >> ~/.bashrc.d/300-go && \
+    echo "export GOROOT=${HOME}/go" >> ~/.bashrc.d/300-go && \
+    echo "export PATH=\$GOROOT/bin:\$GOBIN:\$PATH" >> ~/.bashrc
+
+# Reload the environment variables to ensure go environment variables are
+# available in subsequent commands.
+RUN bash -c "source ~/.bashrc && source ~/.bashrc.d/300-go"
+
+# Fetch the Go version dynamically from the Prometheus go.mod file and Install Go in $HOME path.
+RUN export CUSTOM_GO_VERSION=$(curl -sSL "https://raw.githubusercontent.com/prometheus/prometheus/main/go.mod" | awk '/^go/{print $2".0"}') && \
+    curl -fsSL "https://dl.google.com/go/go${CUSTOM_GO_VERSION}.linux-amd64.tar.gz" | \
+    tar -xz -C $HOME
+
+# Fetch the goyacc parser version dynamically from the Prometheus Makefile
+# and install it globally in $GOBIN path.
+RUN GOYACC_VERSION=$(curl -fsSL "https://raw.githubusercontent.com/prometheus/prometheus/main/Makefile" | awk -F'=' '/GOYACC_VERSION \?=/{gsub(/ /, "", $2); print $2}') && \
+    go install "golang.org/x/tools/cmd/goyacc@${GOYACC_VERSION}"

.golangci.yml

@@ -21,11 +21,15 @@ linters:
     - goimports
     - misspell
     - nolintlint
+    - perfsprint
     - predeclared
     - revive
     - testifylint
     - unconvert
     - unused
+    - usestdlibvars
+    - whitespace
+    - loggercheck
 
 issues:
   max-same-issues: 0
@@ -42,24 +46,32 @@ issues:
     - linters:
         - godot
       source: "^// ==="
-
+    - linters:
+        - perfsprint
+      text: "fmt.Sprintf can be replaced with string concatenation"
 linters-settings:
   depguard:
     rules:
       main:
         deny:
-        - pkg: "sync/atomic"
+          - pkg: "sync/atomic"
-          desc: "Use go.uber.org/atomic instead of sync/atomic"
+            desc: "Use go.uber.org/atomic instead of sync/atomic"
-        - pkg: "github.com/stretchr/testify/assert"
+          - pkg: "github.com/stretchr/testify/assert"
-          desc: "Use github.com/stretchr/testify/require instead of github.com/stretchr/testify/assert"
+            desc: "Use github.com/stretchr/testify/require instead of github.com/stretchr/testify/assert"
-        - pkg: "github.com/go-kit/kit/log"
+          - pkg: "github.com/go-kit/kit/log"
-          desc: "Use github.com/go-kit/log instead of github.com/go-kit/kit/log"
+            desc: "Use github.com/go-kit/log instead of github.com/go-kit/kit/log"
-        - pkg: "io/ioutil"
+          - pkg: "io/ioutil"
-          desc: "Use corresponding 'os' or 'io' functions instead."
+            desc: "Use corresponding 'os' or 'io' functions instead."
-        - pkg: "regexp"
+          - pkg: "regexp"
-          desc: "Use github.com/grafana/regexp instead of regexp"
+            desc: "Use github.com/grafana/regexp instead of regexp"
-        - pkg: "github.com/pkg/errors"
+          - pkg: "github.com/pkg/errors"
-          desc: "Use 'errors' or 'fmt' instead of github.com/pkg/errors"
+            desc: "Use 'errors' or 'fmt' instead of github.com/pkg/errors"
+          - pkg: "gzip"
+            desc: "Use github.com/klauspost/compress instead of gzip"
+          - pkg: "zlib"
+            desc: "Use github.com/klauspost/compress instead of zlib"
+          - pkg: "golang.org/x/exp/slices"
+            desc: "Use 'slices' instead."
   errcheck:
     exclude-functions:
       # Don't flag lines such as "io.Copy(io.Discard, resp.Body)".
@@ -77,6 +89,9 @@ linters-settings:
     local-prefixes: github.com/prometheus/prometheus
   gofumpt:
     extra-rules: true
+  perfsprint:
+     # Optimizes `fmt.Errorf`.
+    errorf: false
   revive:
     # By default, revive will enable only the linting rules that are named in the configuration file.
     # So, it's needed to explicitly set in configuration all required rules.
@@ -129,4 +144,3 @@ linters-settings:
       - require-error
       - suite-dont-use-pkg
       - suite-extra-assert-call
- 

.promu.yml

@@ -1,7 +1,7 @@
 go:
     # Whenever the Go version is updated here,
-    # .circle/config.yml should also be updated.
+    # .github/workflows should also be updated.
-    version: 1.21
+    version: 1.22
 repository:
     path: github.com/prometheus/prometheus
 build:

.yamllint

@@ -1,5 +1,7 @@
 ---
 extends: default
+ignore: |
+  ui/react-app/node_modules
 
 rules:
   braces:

CHANGELOG.md

@@ -1,5 +1,134 @@
 # Changelog
 
+## unreleased
+
+## 2.53.0 / 2024-06-16
+
+This release changes the default for GOGC, the Go runtime control for the trade-off between excess memory use and CPU usage. We have found that Prometheus operates with minimal additional CPU usage, but greatly reduced memory by adjusting the upstream Go default from 100 to 75.
+
+* [CHANGE] Rules: Execute 1 query instead of N (where N is the number of alerts within alert rule) when restoring alerts. #13980 #14048
+* [CHANGE] Runtime: Change GOGC threshold from 100 to 75 #14176 #14285
+* [FEATURE] Rules: Add new option `query_offset` for each rule group via rule group configuration file and `rule_query_offset` as part of the global configuration to have more resilience for remote write delays. #14061 #14216 #14273
+* [ENHANCEMENT] Rules: Add `rule_group_last_restore_duration_seconds` metric to measure the time it takes to restore a rule group. #13974
+* [ENHANCEMENT] OTLP: Improve remote write format translation performance by using label set hashes for metric identifiers instead of string based ones. #14006 #13991
+* [ENHANCEMENT] TSDB: Optimize querying with regexp matchers. #13620
+* [BUGFIX] OTLP: Don't generate target_info unless there are metrics and at least one identifying label is defined. #13991
+* [BUGFIX] Scrape: Do no try to ingest native histograms when the native histograms feature is turned off. This happened when protobuf scrape was enabled by for example the created time feature. #13987
+* [BUGFIX] Scaleway SD: Use the instance's public IP if no private IP is available as the `__address__` meta label. #13941
+* [BUGFIX] Query logger: Do not leak file descriptors on error. #13948
+* [BUGFIX] TSDB: Let queries with heavy regex matches be cancelled and not use up the CPU. #14096 #14103 #14118 #14199
+* [BUGFIX] API: Do not warn if result count is equal to the limit, only when exceeding the limit for the series, label-names and label-values APIs. #14116
+* [BUGFIX] TSDB: Fix head stats and hooks when replaying a corrupted snapshot. #14079
+
+## 2.52.1 / 2024-05-29
+
+* [BUGFIX] Linode SD: Fix partial fetch when discovery would return more than 500 elements. #14141
+
+## 2.52.0 / 2024-05-07
+
+* [CHANGE] TSDB: Fix the predicate checking for blocks which are beyond the retention period to include the ones right at the retention boundary. #9633
+* [FEATURE] Kubernetes SD: Add a new metric `prometheus_sd_kubernetes_failures_total` to track failed requests to Kubernetes API. #13554
+* [FEATURE] Kubernetes SD: Add node and zone metadata labels when using the endpointslice role. #13935
+* [FEATURE] Azure SD/Remote Write: Allow usage of Azure authorization SDK. #13099
+* [FEATURE] Alerting: Support native histogram templating. #13731
+* [FEATURE] Linode SD: Support IPv6 range discovery and region filtering. #13774
+* [ENHANCEMENT] PromQL: Performance improvements for queries with regex matchers. #13461
+* [ENHANCEMENT] PromQL: Performance improvements when using aggregation operators. #13744
+* [ENHANCEMENT] PromQL: Validate label_join destination label. #13803
+* [ENHANCEMENT] Scrape: Increment `prometheus_target_scrapes_sample_duplicate_timestamp_total` metric on duplicated series during one scrape. #12933
+* [ENHANCEMENT] TSDB: Many improvements in performance. #13742 #13673 #13782
+* [ENHANCEMENT] TSDB: Pause regular block compactions if the head needs to be compacted (prioritize head as it increases memory consumption). #13754
+* [ENHANCEMENT] Observability: Improved logging during signal handling termination. #13772
+* [ENHANCEMENT] Observability: All log lines for drop series use "num_dropped" key consistently. #13823
+* [ENHANCEMENT] Observability: Log chunk snapshot and mmaped chunk replay duration during WAL replay. #13838
+* [ENHANCEMENT] Observability: Log if the block is being created from WBL during compaction. #13846
+* [BUGFIX] PromQL: Fix inaccurate sample number statistic when querying histograms. #13667
+* [BUGFIX] PromQL: Fix `histogram_stddev` and `histogram_stdvar` for cases where the histogram has negative buckets. #13852
+* [BUGFIX] PromQL: Fix possible duplicated label name and values in a metric result for specific queries. #13845
+* [BUGFIX] Scrape: Fix setting native histogram schema factor during scrape. #13846
+* [BUGFIX] TSDB: Fix counting of histogram samples when creating WAL checkpoint stats. #13776
+* [BUGFIX] TSDB: Fix cases of compacting empty heads. #13755
+* [BUGFIX] TSDB: Count float histograms in WAL checkpoint. #13844
+* [BUGFIX] Remote Read: Fix memory leak due to broken requests. #13777
+* [BUGFIX] API: Stop building response for `/api/v1/series/` when the API request was cancelled. #13766
+* [BUGFIX] promtool: Fix panic on `promtool tsdb analyze --extended` when no native histograms are present. #13976
+
+## 2.51.2 / 2024-04-09
+
+Bugfix release.
+
+[BUGFIX] Notifier: could hang when using relabeling on alerts #13861
+
+## 2.51.1 / 2024-03-27
+
+Bugfix release.
+
+* [BUGFIX] PromQL: Re-instate validation of label_join destination label #13803
+* [BUGFIX] Scraping (experimental native histograms): Fix handling of the min bucket factor on sync of targets #13846
+* [BUGFIX] PromQL: Some queries could return the same series twice (library use only) #13845
+
+## 2.51.0 / 2024-03-18
+
+This version is built with Go 1.22.1.
+
+There is a new optional build tag "dedupelabels", which should reduce memory consumption (#12304).
+It is off by default; there will be an optional alternative image to try it out.
+
+* [CHANGE] Scraping: Do experimental timestamp alignment even if tolerance is bigger than 1% of scrape interval #13624, #13737
+* [FEATURE] Alerting: Relabel rules for AlertManagerConfig; allows routing alerts to different alertmanagers #12551, #13735
+* [FEATURE] API: add limit param to series, label-names and label-values APIs #13396
+* [FEATURE] UI (experimental native histograms): Add native histogram chart to Table view #13658
+* [FEATURE] Promtool: Add a "tsdb dump-openmetrics" to dump in OpenMetrics format. #13194
+* [FEATURE] PromQL (experimental native histograms): Add histogram_avg function #13467
+* [ENHANCEMENT] Rules: Evaluate independent rules concurrently #12946, #13527
+* [ENHANCEMENT] Scraping (experimental native histograms): Support exemplars #13488
+* [ENHANCEMENT] Remote Write: Disable resharding during active retry backoffs #13562
+* [ENHANCEMENT] Observability: Add native histograms to latency/duration metrics #13681
+* [ENHANCEMENT] Observability: Add 'type' label to prometheus_tsdb_head_out_of_order_samples_appended_total #13607
+* [ENHANCEMENT] API: Faster generation of targets into JSON #13469, #13484
+* [ENHANCEMENT] Scraping, API: Use faster compression library #10782
+* [ENHANCEMENT] OpenTelemetry: Performance improvements in OTLP parsing #13627
+* [ENHANCEMENT] PromQL: Optimisations to reduce CPU and memory #13448, #13536
+* [BUGFIX] PromQL: Constrain extrapolation in rate() to half of sample interval #13725
+* [BUGFIX] Remote Write: Stop slowing down when a new WAL segment is created #13583, #13628
+* [BUGFIX] PromQL: Fix wrongly scoped range vectors with @ modifier #13559
+* [BUGFIX] Kubernetes SD: Pod status changes were not discovered by Endpoints service discovery #13337
+* [BUGFIX] Azure SD: Fix 'error: parameter virtualMachineScaleSetName cannot be empty' (#13702)
+* [BUGFIX] Remote Write: Fix signing for AWS sigv4 transport #13497
+* [BUGFIX] Observability: Exemplars emitted by Prometheus use "trace_id" not "traceID" #13589
+
+## 2.50.1 / 2024-02-26
+
+* [BUGFIX] API: Fix metadata API using wrong field names. #13633
+
+## 2.50.0 / 2024-02-22
+
+* [CHANGE] Remote Write: Error `storage.ErrTooOldSample` is now generating HTTP error 400 instead of HTTP error 500. #13335
+* [FEATURE] Remote Write: Drop old inmemory samples. Activated using the config entry `sample_age_limit`. #13002
+* [FEATURE] **Experimental**: Add support for ingesting zeros as created timestamps. (enabled under the feature-flag `created-timestamp-zero-ingestion`). #12733 #13279
+* [FEATURE] Promtool: Add `analyze` histograms command. #12331
+* [FEATURE] TSDB/compaction: Add a way to enable overlapping compaction. #13282 #13393 #13398
+* [FEATURE] Add automatic memory limit handling. Activated using the feature flag. `auto-gomemlimit` #13395
+* [ENHANCEMENT] Promtool: allow specifying multiple matchers in `promtool tsdb dump`. #13296
+* [ENHANCEMENT] PromQL: Restore more efficient version of `NewPossibleNonCounterInfo` annotation. #13022
+* [ENHANCEMENT] Kuma SD: Extend configuration to allow users to specify client ID. #13278
+* [ENHANCEMENT] PromQL: Use natural sort in `sort_by_label` and `sort_by_label_desc`. This is **experimental**. #13411
+* [ENHANCEMENT] Native Histograms: support `native_histogram_min_bucket_factor` in scrape_config. #13222
+* [ENHANCEMENT] Native Histograms: Issue warning if histogramRate is applied to the wrong kind of histogram. #13392
+* [ENHANCEMENT] TSDB: Make transaction isolation data structures smaller. #13015
+* [ENHANCEMENT] TSDB/postings: Optimize merge using Loser Tree. #12878
+* [ENHANCEMENT] TSDB: Simplify internal series delete function. #13261
+* [ENHANCEMENT] Agent: Performance improvement by making the global hash lookup table smaller. #13262
+* [ENHANCEMENT] PromQL: faster execution of metric functions, e.g. abs(), rate() #13446
+* [ENHANCEMENT] TSDB: Optimize label values with matchers by taking shortcuts. #13426
+* [ENHANCEMENT] Kubernetes SD: Check preconditions earlier and avoid unnecessary checks or iterations in kube_sd. #13408
+* [ENHANCEMENT] Promtool: Improve visibility for `promtool test rules` with JSON colored formatting. #13342
+* [ENHANCEMENT] Consoles: Exclude iowait and steal from CPU Utilisation. #9593
+* [ENHANCEMENT] Various improvements and optimizations on Native Histograms. #13267, #13215, #13276 #13289, #13340
+* [BUGFIX] Scraping: Fix quality value in HTTP Accept header. #13313
+* [BUGFIX] UI: Fix usage of the function `time()` that was crashing. #13371
+* [BUGFIX] Azure SD: Fix SD crashing when it finds a VM scale set. #13578
+
 ## 2.49.1 / 2024-01-15
 
 * [BUGFIX] TSDB: Fixed a wrong `q=` value in scrape accept header #13313

CONTRIBUTING.md

@@ -42,7 +42,12 @@ go build ./cmd/prometheus/
 make test         # Make sure all the tests pass before you commit and push :)
 ```
 
-We use [`golangci-lint`](https://github.com/golangci/golangci-lint) for linting the code. If it reports an issue and you think that the warning needs to be disregarded or is a false-positive, you can add a special comment `//nolint:linter1[,linter2,...]` before the offending line. Use this sparingly though, fixing the code to comply with the linter's recommendation is in general the preferred course of action.
+To run a collection of Go linters through [`golangci-lint`](https://github.com/golangci/golangci-lint), do:
+```bash
+make lint
+```
+
+If it reports an issue and you think that the warning needs to be disregarded or is a false-positive, you can add a special comment `//nolint:linter1[,linter2,...]` before the offending line. Use this sparingly though, fixing the code to comply with the linter's recommendation is in general the preferred course of action. See [this section of the golangci-lint documentation](https://golangci-lint.run/usage/false-positives/#nolint-directive) for more information.
 
 All our issues are regularly tagged so that you can also filter down the issues involving the components you want to work on. For our labeling policy refer [the wiki page](https://github.com/prometheus/prometheus/wiki/Label-Names-and-Descriptions).
 
@@ -90,7 +95,7 @@ can modify the `./promql/parser/generated_parser.y.go` manually.
 ```golang
 // As of writing this was somewhere around line 600.
 var (
-	yyDebug        = 0 // This can be be a number 0 -> 5.
+	yyDebug        = 0 // This can be a number 0 -> 5.
 	yyErrorVerbose = false  // This can be set to true.
 )
 

MAINTAINERS.md

@@ -1,7 +1,12 @@
 # Maintainers
 
-Julien Pivotto (<roidelapluie@prometheus.io> / @roidelapluie) and Levi Harrison (<levi@leviharrison.dev> / @LeviHarrison) are the main/default maintainers, some parts of the codebase have other maintainers:
+General maintainers:
+* Bryan Boreham (bjboreham@gmail.com / @bboreham)
+* Levi Harrison (levi@leviharrison.dev / @LeviHarrison)
+* Ayoub Mrini (ayoubmrini424@gmail.com / @machine424)
+* Julien Pivotto (roidelapluie@prometheus.io / @roidelapluie)
 
+Maintainers for specific parts of the codebase:
 * `cmd`
   * `promtool`: David Leadbeater (<dgl@dgl.cx> / @dgl)
 * `discovery`
@@ -12,6 +17,7 @@ Julien Pivotto (<roidelapluie@prometheus.io> / @roidelapluie) and Levi Harrison
 George Krajcsovits (<gyorgy.krajcsovits@grafana.com> / @krajorama)
 * `storage`
   * `remote`: Callum Styan (<callumstyan@gmail.com> / @cstyan), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Tom Wilkie (<tom.wilkie@gmail.com> / @tomwilkie)
+    * `otlptranslator`: Arve Knudsen (<arve.knudsen@gmail.com> / @aknuds1), Jesús Vázquez (<jesus.vazquez@grafana.com> / @jesusvazquez)
 * `tsdb`: Ganesh Vernekar (<ganesh@grafana.com> / @codesome), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Jesús Vázquez (<jesus.vazquez@grafana.com> / @jesusvazquez)
   * `agent`: Robert Fratto (<robert.fratto@grafana.com> / @rfratto)
 * `web`
@@ -19,9 +25,18 @@ George Krajcsovits (<gyorgy.krajcsovits@grafana.com> / @krajorama)
     * `module`: Augustin Husson (<husson.augustin@gmail.com> @nexucis)
 * `Makefile` and related build configuration: Simon Pasquier (<pasquier.simon@gmail.com> / @simonpasquier), Ben Kochie (<superq@gmail.com> / @SuperQ)
 
-
 For the sake of brevity, not all subtrees are explicitly listed. Due to the
 size of this repository, the natural changes in focus of maintainers over time,
 and nuances of where particular features live, this list will always be
 incomplete and out of date. However the listed maintainer(s) should be able to
 direct a PR/question to the right person.
+
+v3 release coordinators:
+* Alex Greenbank (<alex.greenbank@grafana.com> / @alexgreenbank)
+* Carrie Edwards (<carrie.edwards@grafana.com> / @carrieedwards)
+* Fiona Liao (<fiona.liao@grafana.com> / @fionaliao)
+* Jan Fajerski (<github@fajerski.name> / @jan--f)
+* Jesús Vázquez (<jesus.vazquez@grafana.com> / @jesusvazquez)
+* Nico Pazos (<nicolas.pazos-mendez@grafana.com> / @npazosmendez)
+* Owen Williams (<owen.williams@grafana.com> / @ywwg)
+* Tom Braack (<me@shorez.de> / @sh0rez)

Makefile

@@ -24,6 +24,7 @@ TSDB_BENCHMARK_DATASET ?= ./tsdb/testdata/20kseries.json
 TSDB_BENCHMARK_OUTPUT_DIR ?= ./benchout
 
 GOLANGCI_LINT_OPTS ?= --timeout 4m
+GOYACC_VERSION ?= v0.6.0
 
 include Makefile.common
 
@@ -78,24 +79,42 @@ assets-tarball: assets
 	@echo '>> packaging assets'
 	scripts/package_assets.sh
 
-# We only want to generate the parser when there's changes to the grammar.
 .PHONY: parser
 parser:
 	@echo ">> running goyacc to generate the .go file."
-ifeq (, $(shell command -v goyacc > /dev/null))
+ifeq (, $(shell command -v goyacc 2> /dev/null))
 	@echo "goyacc not installed so skipping"
-	@echo "To install: go install golang.org/x/tools/cmd/goyacc@v0.6.0"
+	@echo "To install: \"go install golang.org/x/tools/cmd/goyacc@$(GOYACC_VERSION)\" or run \"make install-goyacc\""
 else
-	goyacc -o promql/parser/generated_parser.y.go promql/parser/generated_parser.y
+	$(MAKE) promql/parser/generated_parser.y.go
 endif
 
+promql/parser/generated_parser.y.go: promql/parser/generated_parser.y
+	@echo ">> running goyacc to generate the .go file."
+	@$(FIRST_GOPATH)/bin/goyacc -l -o promql/parser/generated_parser.y.go promql/parser/generated_parser.y
+
+.PHONY: clean-parser
+clean-parser:
+	@echo ">> cleaning generated parser"
+	@rm -f promql/parser/generated_parser.y.go
+
+.PHONY: check-generated-parser
+check-generated-parser: clean-parser promql/parser/generated_parser.y.go
+	@echo ">> checking generated parser"
+	@git diff --exit-code -- promql/parser/generated_parser.y.go || (echo "Generated parser is out of date. Please run 'make parser' and commit the changes." && false)
+
+.PHONY: install-goyacc
+install-goyacc:
+	@echo ">> installing goyacc $(GOYACC_VERSION)"
+	@go install golang.org/x/tools/cmd/goyacc@$(GOYACC_VERSION)
+
 .PHONY: test
 # If we only want to only test go code we have to change the test target
 # which is called by all.
 ifeq ($(GO_ONLY),1)
 test: common-test check-go-mod-version
 else
-test: common-test ui-build-module ui-test ui-lint check-go-mod-version
+test: check-generated-parser common-test ui-build-module ui-test ui-lint check-go-mod-version
 endif
 
 .PHONY: npm_licenses

Makefile.common

@@ -49,23 +49,23 @@ endif
 GOTEST := $(GO) test
 GOTEST_DIR :=
 ifneq ($(CIRCLE_JOB),)
-ifneq ($(shell command -v gotestsum > /dev/null),)
+ifneq ($(shell command -v gotestsum 2> /dev/null),)
 	GOTEST_DIR := test-results
 	GOTEST := gotestsum --junitfile $(GOTEST_DIR)/unit-tests.xml --
 endif
 endif
 
-PROMU_VERSION ?= 0.15.0
+PROMU_VERSION ?= 0.17.0
 PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
 
 SKIP_GOLANGCI_LINT :=
 GOLANGCI_LINT :=
 GOLANGCI_LINT_OPTS ?=
-GOLANGCI_LINT_VERSION ?= v1.55.2
+GOLANGCI_LINT_VERSION ?= v1.59.0
-# golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
+# golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64.
 # windows isn't included here because of the path separator being different.
 ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
-	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386))
+	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386 arm64))
 		# If we're in CI and there is an Actions file, that means the linter
 		# is being run in Actions, so we don't need to run it here.
 		ifneq (,$(SKIP_GOLANGCI_LINT))
@@ -169,16 +169,20 @@ common-vet:
 common-lint: $(GOLANGCI_LINT)
 ifdef GOLANGCI_LINT
 	@echo ">> running golangci-lint"
-# 'go list' needs to be executed before staticcheck to prepopulate the modules cache.
-# Otherwise staticcheck might fail randomly for some reason not yet explained.
-	$(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null
 	$(GOLANGCI_LINT) run $(GOLANGCI_LINT_OPTS) $(pkgs)
 endif
 
+.PHONY: common-lint-fix
+common-lint-fix: $(GOLANGCI_LINT)
+ifdef GOLANGCI_LINT
+	@echo ">> running golangci-lint fix"
+	$(GOLANGCI_LINT) run --fix $(GOLANGCI_LINT_OPTS) $(pkgs)
+endif
+
 .PHONY: common-yamllint
 common-yamllint:
 	@echo ">> running yamllint on all YAML files in the repository"
-ifeq (, $(shell command -v yamllint > /dev/null))
+ifeq (, $(shell command -v yamllint 2> /dev/null))
 	@echo "yamllint not installed so skipping"
 else
 	yamllint .
@@ -204,6 +208,10 @@ common-tarball: promu
 	@echo ">> building release tarball"
 	$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR)
 
+.PHONY: common-docker-repo-name
+common-docker-repo-name:
+	@echo "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)"
+
 .PHONY: common-docker $(BUILD_DOCKER_ARCHS)
 common-docker: $(BUILD_DOCKER_ARCHS)
 $(BUILD_DOCKER_ARCHS): common-docker-%:

README.md

@@ -1,8 +1,8 @@
 <h1 align="center" style="border-bottom: none">
-    <a href="//prometheus.io" target="_blank"><img alt="Prometheus" src="/documentation/images/prometheus-logo.svg"></a><br>Prometheus
+    <a href="https://prometheus.io" target="_blank"><img alt="Prometheus" src="/documentation/images/prometheus-logo.svg"></a><br>Prometheus
 </h1>
 
-<p align="center">Visit <a href="//prometheus.io" target="_blank">prometheus.io</a> for the full documentation,
+<p align="center">Visit <a href="https://prometheus.io" target="_blank">prometheus.io</a> for the full documentation,
 examples and guides.</p>
 
 <div align="center">
@@ -149,7 +149,7 @@ We are publishing our Remote Write protobuf independently at
 You can use that as a library:
 
 ```shell
-go get go.buf.build/protocolbuffers/go/prometheus/prometheus
+go get buf.build/gen/go/prometheus/prometheus/protocolbuffers/go@latest
 ```
 
 This is experimental.

RELEASE.md

@@ -55,7 +55,9 @@ Release cadence of first pre-releases being cut is 6 weeks.
 | v2.48          | 2023-10-04                                 | Levi Harrison (GitHub: @LeviHarrison)       |
 | v2.49          | 2023-12-05                                 | Bartek Plotka (GitHub: @bwplotka)           |
 | v2.50          | 2024-01-16                                 | Augustin Husson (GitHub: @nexucis)          |
-| v2.51          | 2024-02-13                                 | **searching for volunteer**                 |
+| v2.51          | 2024-03-07                                 | Bryan Boreham (GitHub: @bboreham)           |
+| v2.52          | 2024-04-22                                 | Arthur Silva Sens (GitHub: @ArthurSens)     |
+| v2.53          | 2024-06-03                                 | George Krajcsovits (GitHub: @krajorama)     |
 
 If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice.
 
@@ -147,6 +149,8 @@ Changes for a patch release or release candidate should be merged into the previ
 
 Bump the version in the `VERSION` file and update `CHANGELOG.md`. Do this in a proper PR pointing to the release branch as this gives others the opportunity to chime in on the release in general and on the addition to the changelog in particular. For a release candidate, append something like `-rc.0` to the version (with the corresponding changes to the tag name, the release name etc.).
 
+When updating the `CHANGELOG.md` look at all PRs included in the release since the last release and verify if they need a changelog entry.
+
 Note that `CHANGELOG.md` should only document changes relevant to users of Prometheus, including external API changes, performance improvements, and new features. Do not document changes of internal interfaces, code refactorings and clean-ups, changes to the build process, etc. People interested in these are asked to refer to the git history.
 
 For release candidates still update `CHANGELOG.md`, but when you cut the final release later, merge all the changes from the pre-releases into the one final update.

VERSION

@@ -1 +1 @@
-2.49.1
+2.53.0

cmd/prometheus/main.go

@@ -28,6 +28,8 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
+	"runtime/debug"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -42,6 +44,8 @@ import (
 	"github.com/mwitkow/go-conntrack"
 	"github.com/oklog/run"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	versioncollector "github.com/prometheus/client_golang/prometheus/collectors/version"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promlog"
 	promlogflag "github.com/prometheus/common/promlog/flag"
@@ -99,7 +103,7 @@ var (
 )
 
 func init() {
-	prometheus.MustRegister(version.NewCollector(strings.ReplaceAll(appName, "-", "_")))
+	prometheus.MustRegister(versioncollector.NewCollector(strings.ReplaceAll(appName, "-", "_")))
 
 	var err error
 	defaultRetentionDuration, err = model.ParseDuration(defaultRetentionString)
@@ -216,6 +220,7 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 				level.Info(logger).Log("msg", "Experimental PromQL functions enabled.")
 			case "native-histograms":
 				c.tsdb.EnableNativeHistograms = true
+				c.scrape.EnableNativeHistogramsIngestion = true
 				// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers.
 				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
 				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
@@ -250,6 +255,18 @@ func main() {
 		newFlagRetentionDuration model.Duration
 	)
 
+	// Unregister the default GoCollector, and reregister with our defaults.
+	if prometheus.Unregister(collectors.NewGoCollector()) {
+		prometheus.MustRegister(
+			collectors.NewGoCollector(
+				collectors.WithGoCollectorRuntimeMetrics(
+					collectors.MetricsGC,
+					collectors.MetricsScheduler,
+				),
+			),
+		)
+	}
+
 	cfg := flagConfig{
 		notifier: notifier.Options{
 			Registerer: prometheus.DefaultRegisterer,
@@ -416,7 +433,7 @@ func main() {
 	serverOnlyFlag(a, "rules.alert.resend-delay", "Minimum amount of time to wait before resending an alert to Alertmanager.").
 		Default("1m").SetValue(&cfg.resendDelay)
 
-	serverOnlyFlag(a, "rules.max-concurrent-evals", "Global concurrency limit for independent rules that can run concurrently.").
+	serverOnlyFlag(a, "rules.max-concurrent-evals", "Global concurrency limit for independent rules that can run concurrently. When set, \"query.max-concurrency\" may need to be adjusted accordingly.").
 		Default("4").Int64Var(&cfg.maxConcurrentEvals)
 
 	a.Flag("scrape.adjust-timestamps", "Adjust scrape timestamps by up to `scrape.timestamp-tolerance` to align them to the intended schedule. See https://github.com/prometheus/prometheus/issues/7846 for more context. Experimental. This flag will be removed in a future release.").
@@ -446,7 +463,7 @@ func main() {
 	a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates.").
 		Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval)
 
-	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
+	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, created-timestamp-zero-ingestion, concurrent-rule-eval. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
 		Default("").StringsVar(&cfg.featureList)
 
 	promlogflag.AddFlags(a, &cfg.promlogConfig)
@@ -770,6 +787,9 @@ func main() {
 			ResendDelay:            time.Duration(cfg.resendDelay),
 			MaxConcurrentEvals:     cfg.maxConcurrentEvals,
 			ConcurrentEvalsEnabled: cfg.enableConcurrentRuleEval,
+			DefaultRuleQueryOffset: func() time.Duration {
+				return time.Duration(cfgFile.GlobalConfig.RuleQueryOffset)
+			},
 		})
 	}
 
@@ -959,8 +979,8 @@ func main() {
 			func() error {
 				// Don't forget to release the reloadReady channel so that waiting blocks can exit normally.
 				select {
-				case <-term:
+				case sig := <-term:
-					level.Warn(logger).Log("msg", "Received SIGTERM, exiting gracefully...")
+					level.Warn(logger).Log("msg", "Received an OS signal, exiting gracefully...", "signal", sig.String())
 					reloadReady.Close()
 				case <-webHandler.Quit():
 					level.Warn(logger).Log("msg", "Received termination request via web service, exiting gracefully...")
@@ -1177,7 +1197,7 @@ func main() {
 	}
 	if agentMode {
 		// WAL storage.
-		opts := cfg.agent.ToAgentOptions()
+		opts := cfg.agent.ToAgentOptions(cfg.tsdb.OutOfOrderTimeWindow)
 		cancel := make(chan struct{})
 		g.Add(
 			func() error {
@@ -1213,6 +1233,7 @@ func main() {
 					"TruncateFrequency", cfg.agent.TruncateFrequency,
 					"MinWALTime", cfg.agent.MinWALTime,
 					"MaxWALTime", cfg.agent.MaxWALTime,
+					"OutOfOrderTimeWindow", cfg.agent.OutOfOrderTimeWindow,
 				)
 
 				localStorage.Set(db, 0)
@@ -1366,6 +1387,17 @@ func reloadConfig(filename string, expandExternalLabels, enableExemplarStorage b
 		return fmt.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename)
 	}
 
+	oldGoGC := debug.SetGCPercent(conf.Runtime.GoGC)
+	if oldGoGC != conf.Runtime.GoGC {
+		level.Info(logger).Log("msg", "updated GOGC", "old", oldGoGC, "new", conf.Runtime.GoGC)
+	}
+	// Write the new setting out to the ENV var for runtime API output.
+	if conf.Runtime.GoGC >= 0 {
+		os.Setenv("GOGC", strconv.Itoa(conf.Runtime.GoGC))
+	} else {
+		os.Setenv("GOGC", "off")
+	}
+
 	noStepSuqueryInterval.Set(conf.GlobalConfig.EvaluationInterval)
 	l := []interface{}{"msg", "Completed loading of configuration file", "filename", filename, "totalDuration", time.Since(start)}
 	level.Info(logger).Log(append(l, timings...)...)
@@ -1705,17 +1737,22 @@ type agentOptions struct {
 	TruncateFrequency      model.Duration
 	MinWALTime, MaxWALTime model.Duration
 	NoLockfile             bool
+	OutOfOrderTimeWindow   int64
 }
 
-func (opts agentOptions) ToAgentOptions() agent.Options {
+func (opts agentOptions) ToAgentOptions(outOfOrderTimeWindow int64) agent.Options {
+	if outOfOrderTimeWindow < 0 {
+		outOfOrderTimeWindow = 0
+	}
 	return agent.Options{
-		WALSegmentSize:    int(opts.WALSegmentSize),
+		WALSegmentSize:       int(opts.WALSegmentSize),
-		WALCompression:    wlog.ParseCompressionType(opts.WALCompression, opts.WALCompressionType),
+		WALCompression:       wlog.ParseCompressionType(opts.WALCompression, opts.WALCompressionType),
-		StripeSize:        opts.StripeSize,
+		StripeSize:           opts.StripeSize,
-		TruncateFrequency: time.Duration(opts.TruncateFrequency),
+		TruncateFrequency:    time.Duration(opts.TruncateFrequency),
-		MinWALTime:        durationToInt64Millis(time.Duration(opts.MinWALTime)),
+		MinWALTime:           durationToInt64Millis(time.Duration(opts.MinWALTime)),
-		MaxWALTime:        durationToInt64Millis(time.Duration(opts.MaxWALTime)),
+		MaxWALTime:           durationToInt64Millis(time.Duration(opts.MaxWALTime)),
-		NoLockfile:        opts.NoLockfile,
+		NoLockfile:           opts.NoLockfile,
+		OutOfOrderTimeWindow: outOfOrderTimeWindow,
 	}
 }
 

cmd/prometheus/main_test.go

@@ -24,6 +24,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"testing"
@@ -126,12 +127,9 @@ func TestFailedStartupExitCode(t *testing.T) {
 	require.Error(t, err)
 
 	var exitError *exec.ExitError
-	if errors.As(err, &exitError) {
+	require.ErrorAs(t, err, &exitError)
-		status := exitError.Sys().(syscall.WaitStatus)
+	status := exitError.Sys().(syscall.WaitStatus)
-		require.Equal(t, expectedExitStatus, status.ExitStatus())
+	require.Equal(t, expectedExitStatus, status.ExitStatus())
-	} else {
-		t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
-	}
 }
 
 type senderFunc func(alerts ...*notifier.Alert)
@@ -192,11 +190,9 @@ func TestSendAlerts(t *testing.T) {
 
 	for i, tc := range testCases {
 		tc := tc
-		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			senderFunc := senderFunc(func(alerts ...*notifier.Alert) {
-				if len(tc.in) == 0 {
+				require.NotEmpty(t, tc.in, "sender called with 0 alert")
-					t.Fatalf("sender called with 0 alert")
-				}
 				require.Equal(t, tc.exp, alerts)
 			})
 			rules.SendAlerts(senderFunc, "http://localhost:9090")(context.TODO(), "up", tc.in...)
@@ -228,7 +224,7 @@ func TestWALSegmentSizeBounds(t *testing.T) {
 			go func() { done <- prom.Wait() }()
 			select {
 			case err := <-done:
-				t.Errorf("prometheus should be still running: %v", err)
+				require.Fail(t, "prometheus should be still running: %v", err)
 			case <-time.After(startupTime):
 				prom.Process.Kill()
 				<-done
@@ -239,12 +235,9 @@ func TestWALSegmentSizeBounds(t *testing.T) {
 		err = prom.Wait()
 		require.Error(t, err)
 		var exitError *exec.ExitError
-		if errors.As(err, &exitError) {
+		require.ErrorAs(t, err, &exitError)
-			status := exitError.Sys().(syscall.WaitStatus)
+		status := exitError.Sys().(syscall.WaitStatus)
-			require.Equal(t, expectedExitStatus, status.ExitStatus())
+		require.Equal(t, expectedExitStatus, status.ExitStatus())
-		} else {
-			t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
-		}
 	}
 }
 
@@ -274,7 +267,7 @@ func TestMaxBlockChunkSegmentSizeBounds(t *testing.T) {
 			go func() { done <- prom.Wait() }()
 			select {
 			case err := <-done:
-				t.Errorf("prometheus should be still running: %v", err)
+				require.Fail(t, "prometheus should be still running: %v", err)
 			case <-time.After(startupTime):
 				prom.Process.Kill()
 				<-done
@@ -285,12 +278,9 @@ func TestMaxBlockChunkSegmentSizeBounds(t *testing.T) {
 		err = prom.Wait()
 		require.Error(t, err)
 		var exitError *exec.ExitError
-		if errors.As(err, &exitError) {
+		require.ErrorAs(t, err, &exitError)
-			status := exitError.Sys().(syscall.WaitStatus)
+		status := exitError.Sys().(syscall.WaitStatus)
-			require.Equal(t, expectedExitStatus, status.ExitStatus())
+		require.Equal(t, expectedExitStatus, status.ExitStatus())
-		} else {
-			t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
-		}
 	}
 }
 
@@ -347,10 +337,8 @@ func getCurrentGaugeValuesFor(t *testing.T, reg prometheus.Gatherer, metricNames
 			}
 
 			require.Len(t, g.GetMetric(), 1)
-			if _, ok := res[m]; ok {
+			_, ok := res[m]
-				t.Error("expected only one metric family for", m)
+			require.False(t, ok, "expected only one metric family for", m)
-				t.FailNow()
-			}
 			res[m] = *g.GetMetric()[0].GetGauge().Value
 		}
 	}

cmd/prometheus/main_unix_test.go

@@ -23,6 +23,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/prometheus/prometheus/util/testutil"
 )
 
@@ -37,9 +39,7 @@ func TestStartupInterrupt(t *testing.T) {
 
 	prom := exec.Command(promPath, "-test.main", "--config.file="+promConfig, "--storage.tsdb.path="+t.TempDir(), "--web.listen-address=0.0.0.0"+port)
 	err := prom.Start()
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("execution error: %v", err)
-	}
 
 	done := make(chan error, 1)
 	go func() {
@@ -68,14 +68,11 @@ Loop:
 		time.Sleep(500 * time.Millisecond)
 	}
 
-	if !startedOk {
+	require.True(t, startedOk, "prometheus didn't start in the specified timeout")
-		t.Fatal("prometheus didn't start in the specified timeout")
+	err = prom.Process.Kill()
-	}
+	require.Error(t, err, "prometheus didn't shutdown gracefully after sending the Interrupt signal")
-	switch err := prom.Process.Kill(); {
+	// TODO - find a better way to detect when the process didn't exit as expected!
-	case err == nil:
+	if stoppedErr != nil {
-		t.Errorf("prometheus didn't shutdown gracefully after sending the Interrupt signal")
+		require.EqualError(t, stoppedErr, "signal: interrupt", "prometheus exit")
-	case stoppedErr != nil && stoppedErr.Error() != "signal: interrupt":
-		// TODO: find a better way to detect when the process didn't exit as expected!
-		t.Errorf("prometheus exited with an unexpected error: %v", stoppedErr)
 	}
 }

cmd/prometheus/query_log_test.go

@@ -296,7 +296,7 @@ func (p *queryLogTest) run(t *testing.T) {
 	if p.exactQueryCount() {
 		require.Equal(t, 1, qc)
 	} else {
-		require.Greater(t, qc, 0, "no queries logged")
+		require.Positive(t, qc, "no queries logged")
 	}
 	p.validateLastQuery(t, ql)
 
@@ -366,7 +366,7 @@ func (p *queryLogTest) run(t *testing.T) {
 	if p.exactQueryCount() {
 		require.Equal(t, 1, qc)
 	} else {
-		require.Greater(t, qc, 0, "no queries logged")
+		require.Positive(t, qc, "no queries logged")
 	}
 }
 

cmd/promtool/archive.go

@@ -15,9 +15,10 @@ package main
 
 import (
 	"archive/tar"
-	"compress/gzip"
 	"fmt"
 	"os"
+
+	"github.com/klauspost/compress/gzip"
 )
 
 const filePerm = 0o666

cmd/promtool/backfill.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/oklog/ulid"
 
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/textparse"
@@ -88,7 +89,7 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
 	blockDuration := getCompatibleBlockDuration(maxBlockDuration)
 	mint = blockDuration * (mint / blockDuration)
 
-	db, err := tsdb.OpenDBReadOnly(outputDir, nil)
+	db, err := tsdb.OpenDBReadOnly(outputDir, "", nil)
 	if err != nil {
 		return err
 	}
@@ -127,7 +128,8 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
 
 			ctx := context.Background()
 			app := w.Appender(ctx)
-			p := textparse.NewOpenMetricsParser(input)
+			symbolTable := labels.NewSymbolTable() // One table per block means it won't grow too large.
+			p := textparse.NewOpenMetricsParser(input, symbolTable)
 			samplesCount := 0
 			for {
 				e, err := p.Next()
@@ -190,6 +192,10 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
 				if quiet {
 					break
 				}
+				// Empty block, don't print.
+				if block.Compare(ulid.ULID{}) == 0 {
+					break
+				}
 				blocks, err := db.Blocks()
 				if err != nil {
 					return fmt.Errorf("get blocks: %w", err)
@@ -216,7 +222,7 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
 }
 
 func backfill(maxSamplesInAppender int, input []byte, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration) (err error) {
-	p := textparse.NewOpenMetricsParser(input)
+	p := textparse.NewOpenMetricsParser(input, nil) // Don't need a SymbolTable to get max and min timestamps.
 	maxt, mint, err := getMinAndMaxTimestamps(p)
 	if err != nil {
 		return fmt.Errorf("getting min and max timestamp: %w", err)

cmd/promtool/backfill_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 type backfillSample struct {
@@ -76,7 +77,7 @@ func testBlocks(t *testing.T, db *tsdb.DB, expectedMinTime, expectedMaxTime, exp
 	allSamples := queryAllSeries(t, q, expectedMinTime, expectedMaxTime)
 	sortSamples(allSamples)
 	sortSamples(expectedSamples)
-	require.Equal(t, expectedSamples, allSamples, "did not create correct samples")
+	testutil.RequireEqual(t, expectedSamples, allSamples, "did not create correct samples")
 
 	if len(allSamples) > 0 {
 		require.Equal(t, expectedMinTime, allSamples[0].Timestamp, "timestamp of first sample is not the expected minimum time")

cmd/promtool/main.go

@@ -56,8 +56,8 @@ import (
 	"github.com/prometheus/prometheus/model/rulefmt"
 	"github.com/prometheus/prometheus/notifier"
 	_ "github.com/prometheus/prometheus/plugins" // Register plugins.
-	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
+	"github.com/prometheus/prometheus/promql/promqltest"
 	"github.com/prometheus/prometheus/scrape"
 	"github.com/prometheus/prometheus/util/documentcli"
 )
@@ -235,10 +235,18 @@ func main() {
 
 	tsdbDumpCmd := tsdbCmd.Command("dump", "Dump samples from a TSDB.")
 	dumpPath := tsdbDumpCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
+	dumpSandboxDirRoot := tsdbDumpCmd.Flag("sandbox-dir-root", "Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end.").Default(defaultDBPath).String()
 	dumpMinTime := tsdbDumpCmd.Flag("min-time", "Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64()
 	dumpMaxTime := tsdbDumpCmd.Flag("max-time", "Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64()
 	dumpMatch := tsdbDumpCmd.Flag("match", "Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings()
 
+	tsdbDumpOpenMetricsCmd := tsdbCmd.Command("dump-openmetrics", "[Experimental] Dump samples from a TSDB into OpenMetrics text format, excluding native histograms and staleness markers, which are not representable in OpenMetrics.")
+	dumpOpenMetricsPath := tsdbDumpOpenMetricsCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
+	dumpOpenMetricsSandboxDirRoot := tsdbDumpOpenMetricsCmd.Flag("sandbox-dir-root", "Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end.").Default(defaultDBPath).String()
+	dumpOpenMetricsMinTime := tsdbDumpOpenMetricsCmd.Flag("min-time", "Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64()
+	dumpOpenMetricsMaxTime := tsdbDumpOpenMetricsCmd.Flag("max-time", "Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64()
+	dumpOpenMetricsMatch := tsdbDumpOpenMetricsCmd.Flag("match", "Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings()
+
 	importCmd := tsdbCmd.Command("create-blocks-from", "[Experimental] Import samples from input and produce TSDB blocks. Please refer to the storage docs for more details.")
 	importHumanReadable := importCmd.Flag("human-readable", "Print human readable values.").Short('r').Bool()
 	importQuiet := importCmd.Flag("quiet", "Do not print created blocks.").Short('q').Bool()
@@ -371,7 +379,7 @@ func main() {
 
 	case testRulesCmd.FullCommand():
 		os.Exit(RulesUnitTest(
-			promql.LazyLoaderOpts{
+			promqltest.LazyLoaderOpts{
 				EnableAtModifier:     true,
 				EnableNegativeOffset: true,
 			},
@@ -390,7 +398,9 @@ func main() {
 		os.Exit(checkErr(listBlocks(*listPath, *listHumanReadable)))
 
 	case tsdbDumpCmd.FullCommand():
-		os.Exit(checkErr(dumpSamples(ctx, *dumpPath, *dumpMinTime, *dumpMaxTime, *dumpMatch)))
+		os.Exit(checkErr(dumpSamples(ctx, *dumpPath, *dumpSandboxDirRoot, *dumpMinTime, *dumpMaxTime, *dumpMatch, formatSeriesSet)))
+	case tsdbDumpOpenMetricsCmd.FullCommand():
+		os.Exit(checkErr(dumpSamples(ctx, *dumpOpenMetricsPath, *dumpOpenMetricsSandboxDirRoot, *dumpOpenMetricsMinTime, *dumpOpenMetricsMaxTime, *dumpOpenMetricsMatch, formatSeriesSetOpenMetrics)))
 	// TODO(aSquare14): Work on adding support for custom block size.
 	case openMetricsImportCmd.FullCommand():
 		os.Exit(backfillOpenMetrics(*importFilePath, *importDBPath, *importHumanReadable, *importQuiet, *maxBlockDuration))
@@ -474,7 +484,7 @@ func CheckServerStatus(serverURL *url.URL, checkEndpoint string, roundTripper ht
 		return err
 	}
 
-	request, err := http.NewRequest("GET", config.Address, nil)
+	request, err := http.NewRequest(http.MethodGet, config.Address, nil)
 	if err != nil {
 		return err
 	}

cmd/promtool/main_test.go

@@ -25,6 +25,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"testing"
@@ -410,7 +411,7 @@ func TestExitCodes(t *testing.T) {
 	} {
 		t.Run(c.file, func(t *testing.T) {
 			for _, lintFatal := range []bool{true, false} {
-				t.Run(fmt.Sprintf("%t", lintFatal), func(t *testing.T) {
+				t.Run(strconv.FormatBool(lintFatal), func(t *testing.T) {
 					args := []string{"-test.main", "check", "config", "testdata/" + c.file}
 					if lintFatal {
 						args = append(args, "--lint-fatal")

cmd/promtool/rules.go

@@ -234,17 +234,3 @@ func (m *multipleAppender) flushAndCommit(ctx context.Context) error {
 	}
 	return nil
 }
-
-func max(x, y int64) int64 {
-	if x > y {
-		return x
-	}
-	return y
-}
-
-func min(x, y int64) int64 {
-	if x < y {
-		return x
-	}
-	return y
-}

cmd/promtool/rules_test.go

@@ -78,7 +78,6 @@ func TestBackfillRuleIntegration(t *testing.T) {
 			// Execute the test more than once to simulate running the rule importer twice with the same data.
 			// We expect duplicate blocks with the same series are created when run more than once.
 			for i := 0; i < tt.runcount; i++ {
-
 				ruleImporter, err := newTestRuleImporter(ctx, start, tmpDir, tt.samples, tt.maxBlockDuration)
 				require.NoError(t, err)
 				path1 := filepath.Join(tmpDir, "test.file")

cmd/promtool/sd.go

@@ -18,10 +18,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"reflect"
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/google/go-cmp/cmp"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/prometheus/prometheus/config"
@@ -153,7 +153,7 @@ func getSDCheckResult(targetGroups []*targetgroup.Group, scrapeConfig *config.Sc
 
 			duplicateRes := false
 			for _, sdCheckRes := range sdCheckResults {
-				if reflect.DeepEqual(sdCheckRes, result) {
+				if cmp.Equal(sdCheckRes, result, cmp.Comparer(labels.Equal)) {
 					duplicateRes = true
 					break
 				}

cmd/promtool/sd_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
+	"github.com/prometheus/prometheus/util/testutil"
 
 	"github.com/stretchr/testify/require"
 )
@@ -69,5 +70,5 @@ func TestSDCheckResult(t *testing.T) {
 		},
 	}
 
-	require.Equal(t, expectedSDCheckResult, getSDCheckResult(targetGroups, scrapeConfig, true))
+	testutil.RequireEqual(t, expectedSDCheckResult, getSDCheckResult(targetGroups, scrapeConfig, true))
 }

cmd/promtool/testdata/dump-openmetrics-roundtrip-test.prom

@@ -0,0 +1,15 @@
+my_histogram_bucket{instance="localhost:8000",job="example2",le="+Inf"} 1.0267820369e+10 1700215884.373
+my_histogram_bucket{instance="localhost:8000",job="example2",le="+Inf"} 1.026872507e+10 1700215889.373
+my_histogram_bucket{instance="localhost:8000",job="example2",le="0.01"} 0 1700215884.373
+my_histogram_bucket{instance="localhost:8000",job="example2",le="0.01"} 0 1700215889.373
+my_histogram_count{instance="localhost:8000",job="example2"} 1.0267820369e+10 1700215884.373
+my_histogram_count{instance="localhost:8000",job="example2"} 1.026872507e+10 1700215889.373
+my_summary_count{instance="localhost:8000",job="example5"} 9.518161497e+09 1700211684.981
+my_summary_count{instance="localhost:8000",job="example5"} 9.519048034e+09 1700211689.984
+my_summary_sum{instance="localhost:8000",job="example5"} 5.2349889185e+10 1700211684.981
+my_summary_sum{instance="localhost:8000",job="example5"} 5.2354761848e+10 1700211689.984
+up{instance="localhost:8000",job="example2"} 1 1700226034.330
+up{instance="localhost:8000",job="example2"} 1 1700226094.329
+up{instance="localhost:8000",job="example3"} 1 1700210681.366
+up{instance="localhost:8000",job="example3"} 1 1700210686.366
+# EOF

cmd/promtool/testdata/dump-openmetrics-test.prom

@@ -0,0 +1,11 @@
+my_counter{baz="abc",foo="bar"} 1 0.000
+my_counter{baz="abc",foo="bar"} 2 60.000
+my_counter{baz="abc",foo="bar"} 3 120.000
+my_counter{baz="abc",foo="bar"} 4 180.000
+my_counter{baz="abc",foo="bar"} 5 240.000
+my_gauge{abc="baz",bar="foo"} 9 0.000
+my_gauge{abc="baz",bar="foo"} 8 60.000
+my_gauge{abc="baz",bar="foo"} 0 120.000
+my_gauge{abc="baz",bar="foo"} 4 180.000
+my_gauge{abc="baz",bar="foo"} 7 240.000
+# EOF

cmd/promtool/tsdb.go

@@ -15,6 +15,7 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -23,6 +24,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"runtime/pprof"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -31,7 +33,7 @@ import (
 
 	"github.com/alecthomas/units"
 	"github.com/go-kit/log"
-	"golang.org/x/exp/slices"
+	"go.uber.org/atomic"
 
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql/parser"
@@ -148,8 +150,7 @@ func benchmarkWrite(outPath, samplesFile string, numMetrics, numScrapes int) err
 }
 
 func (b *writeBenchmark) ingestScrapes(lbls []labels.Labels, scrapeCount int) (uint64, error) {
-	var mu sync.Mutex
+	var total atomic.Uint64
-	var total uint64
 
 	for i := 0; i < scrapeCount; i += 100 {
 		var wg sync.WaitGroup
@@ -164,22 +165,21 @@ func (b *writeBenchmark) ingestScrapes(lbls []labels.Labels, scrapeCount int) (u
 
 			wg.Add(1)
 			go func() {
+				defer wg.Done()
+
 				n, err := b.ingestScrapesShard(batch, 100, int64(timeDelta*i))
 				if err != nil {
 					// exitWithError(err)
 					fmt.Println(" err", err)
 				}
-				mu.Lock()
+				total.Add(n)
-				total += n
-				mu.Unlock()
-				wg.Done()
 			}()
 		}
 		wg.Wait()
 	}
 	fmt.Println("ingestion completed")
 
-	return total, nil
+	return total.Load(), nil
 }
 
 func (b *writeBenchmark) ingestScrapesShard(lbls []labels.Labels, scrapeCount int, baset int64) (uint64, error) {
@@ -338,7 +338,7 @@ func readPrometheusLabels(r io.Reader, n int) ([]labels.Labels, error) {
 }
 
 func listBlocks(path string, humanReadable bool) error {
-	db, err := tsdb.OpenDBReadOnly(path, nil)
+	db, err := tsdb.OpenDBReadOnly(path, "", nil)
 	if err != nil {
 		return err
 	}
@@ -393,7 +393,7 @@ func getFormatedBytes(bytes int64, humanReadable bool) string {
 }
 
 func openBlock(path, blockID string) (*tsdb.DBReadOnly, tsdb.BlockReader, error) {
-	db, err := tsdb.OpenDBReadOnly(path, nil)
+	db, err := tsdb.OpenDBReadOnly(path, "", nil)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -706,8 +706,10 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb.
 	return nil
 }
 
-func dumpSamples(ctx context.Context, path string, mint, maxt int64, match []string) (err error) {
+type SeriesSetFormatter func(series storage.SeriesSet) error
-	db, err := tsdb.OpenDBReadOnly(path, nil)
+
+func dumpSamples(ctx context.Context, dbDir, sandboxDirRoot string, mint, maxt int64, match []string, formatter SeriesSetFormatter) (err error) {
+	db, err := tsdb.OpenDBReadOnly(dbDir, sandboxDirRoot, nil)
 	if err != nil {
 		return err
 	}
@@ -736,6 +738,22 @@ func dumpSamples(ctx context.Context, path string, mint, maxt int64, match []str
 		ss = q.Select(ctx, false, nil, matcherSets[0]...)
 	}
 
+	err = formatter(ss)
+	if err != nil {
+		return err
+	}
+
+	if ws := ss.Warnings(); len(ws) > 0 {
+		return tsdb_errors.NewMulti(ws.AsErrors()...).Err()
+	}
+
+	if ss.Err() != nil {
+		return ss.Err()
+	}
+	return nil
+}
+
+func formatSeriesSet(ss storage.SeriesSet) error {
 	for ss.Next() {
 		series := ss.At()
 		lbs := series.Labels()
@@ -756,14 +774,44 @@ func dumpSamples(ctx context.Context, path string, mint, maxt int64, match []str
 			return ss.Err()
 		}
 	}
+	return nil
+}
 
-	if ws := ss.Warnings(); len(ws) > 0 {
+// CondensedString is labels.Labels.String() without spaces after the commas.
-		return tsdb_errors.NewMulti(ws.AsErrors()...).Err()
+func CondensedString(ls labels.Labels) string {
-	}
+	var b bytes.Buffer
 
-	if ss.Err() != nil {
+	b.WriteByte('{')
-		return ss.Err()
+	i := 0
+	ls.Range(func(l labels.Label) {
+		if i > 0 {
+			b.WriteByte(',')
+		}
+		b.WriteString(l.Name)
+		b.WriteByte('=')
+		b.WriteString(strconv.Quote(l.Value))
+		i++
+	})
+	b.WriteByte('}')
+	return b.String()
+}
+
+func formatSeriesSetOpenMetrics(ss storage.SeriesSet) error {
+	for ss.Next() {
+		series := ss.At()
+		lbs := series.Labels()
+		metricName := lbs.Get(labels.MetricName)
+		lbs = lbs.DropMetricName()
+		it := series.Iterator(nil)
+		for it.Next() == chunkenc.ValFloat {
+			ts, val := it.At()
+			fmt.Printf("%s%s %g %.3f\n", metricName, CondensedString(lbs), val, float64(ts)/1000)
+		}
+		if it.Err() != nil {
+			return ss.Err()
+		}
 	}
+	fmt.Println("# EOF")
 	return nil
 }
 
@@ -790,6 +838,10 @@ func backfillOpenMetrics(path, outputDir string, humanReadable, quiet bool, maxB
 }
 
 func displayHistogram(dataType string, datas []int, total int) {
+	if len(datas) == 0 {
+		fmt.Printf("%s: N/A\n\n", dataType)
+		return
+	}
 	slices.Sort(datas)
 	start, end, step := generateBucket(datas[0], datas[len(datas)-1])
 	sum := 0
@@ -804,9 +856,9 @@ func displayHistogram(dataType string, datas []int, total int) {
 	}
 	avg := sum / len(datas)
 	fmt.Printf("%s (min/avg/max): %d/%d/%d\n", dataType, datas[0], avg, datas[len(datas)-1])
-	maxLeftLen := strconv.Itoa(len(fmt.Sprintf("%d", end)))
+	maxLeftLen := strconv.Itoa(len(strconv.Itoa(end)))
-	maxRightLen := strconv.Itoa(len(fmt.Sprintf("%d", end+step)))
+	maxRightLen := strconv.Itoa(len(strconv.Itoa(end + step)))
-	maxCountLen := strconv.Itoa(len(fmt.Sprintf("%d", maxCount)))
+	maxCountLen := strconv.Itoa(len(strconv.Itoa(maxCount)))
 	for bucket, count := range buckets {
 		percentage := 100.0 * count / total
 		fmt.Printf("[%"+maxLeftLen+"d, %"+maxRightLen+"d]: %"+maxCountLen+"d %s\n", bucket*step+start+1, (bucket+1)*step+start, count, strings.Repeat("#", percentage))

cmd/promtool/tsdb_test.go

@@ -22,10 +22,12 @@ import (
 	"runtime"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/promql/promqltest"
+	"github.com/prometheus/prometheus/tsdb"
 )
 
 func TestGenerateBucket(t *testing.T) {
@@ -52,7 +54,7 @@ func TestGenerateBucket(t *testing.T) {
 }
 
 // getDumpedSamples dumps samples and returns them.
-func getDumpedSamples(t *testing.T, path string, mint, maxt int64, match []string) string {
+func getDumpedSamples(t *testing.T, path string, mint, maxt int64, match []string, formatter SeriesSetFormatter) string {
 	t.Helper()
 
 	oldStdout := os.Stdout
@@ -62,9 +64,11 @@ func getDumpedSamples(t *testing.T, path string, mint, maxt int64, match []strin
 	err := dumpSamples(
 		context.Background(),
 		path,
+		t.TempDir(),
 		mint,
 		maxt,
 		match,
+		formatter,
 	)
 	require.NoError(t, err)
 
@@ -76,8 +80,16 @@ func getDumpedSamples(t *testing.T, path string, mint, maxt int64, match []strin
 	return buf.String()
 }
 
+func normalizeNewLine(b []byte) []byte {
+	if strings.Contains(runtime.GOOS, "windows") {
+		// We use "/n" while dumping on windows as well.
+		return bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n"))
+	}
+	return b
+}
+
 func TestTSDBDump(t *testing.T) {
-	storage := promql.LoadedStorage(t, `
+	storage := promqltest.LoadedStorage(t, `
 		load 1m
 			metric{foo="bar", baz="abc"} 1 2 3 4 5
 			heavy_metric{foo="bar"} 5 4 3 2 1
@@ -136,15 +148,48 @@ func TestTSDBDump(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			dumpedMetrics := getDumpedSamples(t, storage.Dir(), tt.mint, tt.maxt, tt.match)
+			dumpedMetrics := getDumpedSamples(t, storage.Dir(), tt.mint, tt.maxt, tt.match, formatSeriesSet)
 			expectedMetrics, err := os.ReadFile(tt.expectedDump)
 			require.NoError(t, err)
-			if strings.Contains(runtime.GOOS, "windows") {
+			expectedMetrics = normalizeNewLine(expectedMetrics)
-				// We use "/n" while dumping on windows as well.
-				expectedMetrics = bytes.ReplaceAll(expectedMetrics, []byte("\r\n"), []byte("\n"))
-			}
 			// even though in case of one matcher samples are not sorted, the order in the cases above should stay the same.
 			require.Equal(t, string(expectedMetrics), dumpedMetrics)
 		})
 	}
 }
+
+func TestTSDBDumpOpenMetrics(t *testing.T) {
+	storage := promqltest.LoadedStorage(t, `
+		load 1m
+			my_counter{foo="bar", baz="abc"} 1 2 3 4 5
+			my_gauge{bar="foo", abc="baz"} 9 8 0 4 7
+	`)
+
+	expectedMetrics, err := os.ReadFile("testdata/dump-openmetrics-test.prom")
+	require.NoError(t, err)
+	expectedMetrics = normalizeNewLine(expectedMetrics)
+	dumpedMetrics := getDumpedSamples(t, storage.Dir(), math.MinInt64, math.MaxInt64, []string{"{__name__=~'(?s:.*)'}"}, formatSeriesSetOpenMetrics)
+	require.Equal(t, string(expectedMetrics), dumpedMetrics)
+}
+
+func TestTSDBDumpOpenMetricsRoundTrip(t *testing.T) {
+	initialMetrics, err := os.ReadFile("testdata/dump-openmetrics-roundtrip-test.prom")
+	require.NoError(t, err)
+	initialMetrics = normalizeNewLine(initialMetrics)
+
+	dbDir := t.TempDir()
+	// Import samples from OM format
+	err = backfill(5000, initialMetrics, dbDir, false, false, 2*time.Hour)
+	require.NoError(t, err)
+	db, err := tsdb.Open(dbDir, nil, nil, tsdb.DefaultOptions(), nil)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, db.Close())
+	})
+
+	// Dump the blocks into OM format
+	dumpedMetrics := getDumpedSamples(t, dbDir, math.MinInt64, math.MaxInt64, []string{"{__name__=~'(?s:.*)'}"}, formatSeriesSetOpenMetrics)
+
+	// Should get back the initial metrics.
+	require.Equal(t, string(initialMetrics), dumpedMetrics)
+}

cmd/promtool/unittest.go

@@ -20,13 +20,13 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"reflect"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/google/go-cmp/cmp"
 	"github.com/grafana/regexp"
 	"github.com/nsf/jsondiff"
 	"github.com/prometheus/common/model"
@@ -36,13 +36,14 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
+	"github.com/prometheus/prometheus/promql/promqltest"
 	"github.com/prometheus/prometheus/rules"
 	"github.com/prometheus/prometheus/storage"
 )
 
 // RulesUnitTest does unit testing of rules based on the unit testing files provided.
 // More info about the file format can be found in the docs.
-func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
+func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
 	failed := false
 
 	var run *regexp.Regexp
@@ -69,7 +70,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, diffFla
 	return successExitCode
 }
 
-func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error {
+func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error {
 	fmt.Println("Unit Testing: ", filename)
 
 	b, err := os.ReadFile(filename)
@@ -175,13 +176,18 @@ type testGroup struct {
 }
 
 // test performs the unit tests.
-func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, diffFlag bool, ruleFiles ...string) []error {
+func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promqltest.LazyLoaderOpts, diffFlag bool, ruleFiles ...string) (outErr []error) {
 	// Setup testing suite.
-	suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString(), queryOpts)
+	suite, err := promqltest.NewLazyLoader(tg.seriesLoadingString(), queryOpts)
 	if err != nil {
 		return []error{err}
 	}
-	defer suite.Close()
+	defer func() {
+		err := suite.Close()
+		if err != nil {
+			outErr = append(outErr, err)
+		}
+	}()
 	suite.SubqueryInterval = evalInterval
 
 	// Load the rule files.
@@ -340,7 +346,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 				sort.Sort(gotAlerts)
 				sort.Sort(expAlerts)
 
-				if !reflect.DeepEqual(expAlerts, gotAlerts) {
+				if !cmp.Equal(expAlerts, gotAlerts, cmp.Comparer(labels.Equal)) {
 					var testName string
 					if tg.TestGroupName != "" {
 						testName = fmt.Sprintf("    name: %s,\n", tg.TestGroupName)
@@ -408,7 +414,7 @@ Outer:
 			gotSamples = append(gotSamples, parsedSample{
 				Labels:    s.Metric.Copy(),
 				Value:     s.F,
-				Histogram: promql.HistogramTestExpression(s.H),
+				Histogram: promqltest.HistogramTestExpression(s.H),
 			})
 		}
 
@@ -438,7 +444,7 @@ Outer:
 			expSamples = append(expSamples, parsedSample{
 				Labels:    lb,
 				Value:     s.Value,
-				Histogram: promql.HistogramTestExpression(hist),
+				Histogram: promqltest.HistogramTestExpression(hist),
 			})
 		}
 
@@ -448,7 +454,7 @@ Outer:
 		sort.Slice(gotSamples, func(i, j int) bool {
 			return labels.Compare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
 		})
-		if !reflect.DeepEqual(expSamples, gotSamples) {
+		if !cmp.Equal(expSamples, gotSamples, cmp.Comparer(labels.Equal)) {
 			errs = append(errs, fmt.Errorf("    expr: %q, time: %s,\n        exp: %v\n        got: %v", testCase.Expr,
 				testCase.EvalTime.String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
 		}
@@ -567,7 +573,7 @@ func (la labelsAndAnnotations) String() string {
 	}
 	s := "[\n0:" + indentLines("\n"+la[0].String(), "  ")
 	for i, l := range la[1:] {
-		s += ",\n" + fmt.Sprintf("%d", i+1) + ":" + indentLines("\n"+l.String(), "  ")
+		s += ",\n" + strconv.Itoa(i+1) + ":" + indentLines("\n"+l.String(), "  ")
 	}
 	s += "\n]"
 

cmd/promtool/unittest_test.go

@@ -16,7 +16,9 @@ package main
 import (
 	"testing"
 
-	"github.com/prometheus/prometheus/promql"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/promql/promqltest"
 )
 
 func TestRulesUnitTest(t *testing.T) {
@@ -26,7 +28,7 @@ func TestRulesUnitTest(t *testing.T) {
 	tests := []struct {
 		name      string
 		args      args
-		queryOpts promql.LazyLoaderOpts
+		queryOpts promqltest.LazyLoaderOpts
 		want      int
 	}{
 		{
@@ -90,7 +92,7 @@ func TestRulesUnitTest(t *testing.T) {
 			args: args{
 				files: []string{"./testdata/at-modifier-test.yml"},
 			},
-			queryOpts: promql.LazyLoaderOpts{
+			queryOpts: promqltest.LazyLoaderOpts{
 				EnableAtModifier: true,
 			},
 			want: 0,
@@ -107,7 +109,7 @@ func TestRulesUnitTest(t *testing.T) {
 			args: args{
 				files: []string{"./testdata/negative-offset-test.yml"},
 			},
-			queryOpts: promql.LazyLoaderOpts{
+			queryOpts: promqltest.LazyLoaderOpts{
 				EnableNegativeOffset: true,
 			},
 			want: 0,
@@ -117,7 +119,7 @@ func TestRulesUnitTest(t *testing.T) {
 			args: args{
 				files: []string{"./testdata/no-test-group-interval.yml"},
 			},
-			queryOpts: promql.LazyLoaderOpts{
+			queryOpts: promqltest.LazyLoaderOpts{
 				EnableNegativeOffset: true,
 			},
 			want: 0,
@@ -140,7 +142,7 @@ func TestRulesUnitTestRun(t *testing.T) {
 	tests := []struct {
 		name      string
 		args      args
-		queryOpts promql.LazyLoaderOpts
+		queryOpts promqltest.LazyLoaderOpts
 		want      int
 	}{
 		{
@@ -178,9 +180,8 @@ func TestRulesUnitTestRun(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...); got != tt.want {
+			got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...)
-				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
+			require.Equal(t, tt.want, got)
-			}
 		})
 	}
 }

config/config.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -82,7 +83,7 @@ func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, erro
 		return cfg, nil
 	}
 
-	b := labels.ScratchBuilder{}
+	b := labels.NewScratchBuilder(0)
 	cfg.GlobalConfig.ExternalLabels.Range(func(v labels.Label) {
 		newV := os.Expand(v.Value, func(s string) string {
 			if s == "$" {
@@ -97,6 +98,7 @@ func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, erro
 		if newV != v.Value {
 			level.Debug(logger).Log("msg", "External label replaced", "label", v.Name, "input", v.Value, "output", newV)
 		}
+		// Note newV can be blank. https://github.com/prometheus/prometheus/issues/11024
 		b.Add(v.Name, newV)
 	})
 	cfg.GlobalConfig.ExternalLabels = b.Labels()
@@ -144,11 +146,17 @@ var (
 		ScrapeInterval:     model.Duration(1 * time.Minute),
 		ScrapeTimeout:      model.Duration(10 * time.Second),
 		EvaluationInterval: model.Duration(1 * time.Minute),
+		RuleQueryOffset:    model.Duration(0 * time.Minute),
 		// When native histogram feature flag is enabled, ScrapeProtocols default
 		// changes to DefaultNativeHistogramScrapeProtocols.
 		ScrapeProtocols: DefaultScrapeProtocols,
 	}
 
+	DefaultRuntimeConfig = RuntimeConfig{
+		// Go runtime tuning.
+		GoGC: 75,
+	}
+
 	// DefaultScrapeConfig is the default scrape configuration.
 	DefaultScrapeConfig = ScrapeConfig{
 		// ScrapeTimeout, ScrapeInterval and ScrapeProtocols default to the configured globals.
@@ -223,6 +231,7 @@ var (
 // Config is the top-level configuration for Prometheus's config files.
 type Config struct {
 	GlobalConfig      GlobalConfig    `yaml:"global"`
+	Runtime           RuntimeConfig   `yaml:"runtime,omitempty"`
 	AlertingConfig    AlertingConfig  `yaml:"alerting,omitempty"`
 	RuleFiles         []string        `yaml:"rule_files,omitempty"`
 	ScrapeConfigFiles []string        `yaml:"scrape_config_files,omitempty"`
@@ -333,6 +342,14 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		c.GlobalConfig = DefaultGlobalConfig
 	}
 
+	// If a runtime block was open but empty the default runtime config is overwritten.
+	// We have to restore it here.
+	if c.Runtime.isZero() {
+		c.Runtime = DefaultRuntimeConfig
+		// Use the GOGC env var value if the runtime section is empty.
+		c.Runtime.GoGC = getGoGCEnv()
+	}
+
 	for _, rf := range c.RuleFiles {
 		if !patRulePath.MatchString(rf) {
 			return fmt.Errorf("invalid rule file path %q", rf)
@@ -396,6 +413,8 @@ type GlobalConfig struct {
 	ScrapeProtocols []ScrapeProtocol `yaml:"scrape_protocols,omitempty"`
 	// How frequently to evaluate rules by default.
 	EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"`
+	// Offset the rule evaluation timestamp of this particular group by the specified duration into the past to ensure the underlying metrics have been received.
+	RuleQueryOffset model.Duration `yaml:"rule_query_offset,omitempty"`
 	// File to which PromQL queries are logged.
 	QueryLogFile string `yaml:"query_log_file,omitempty"`
 	// The labels to add to any timeseries that this Prometheus instance scrapes.
@@ -555,10 +574,22 @@ func (c *GlobalConfig) isZero() bool {
 		c.ScrapeInterval == 0 &&
 		c.ScrapeTimeout == 0 &&
 		c.EvaluationInterval == 0 &&
+		c.RuleQueryOffset == 0 &&
 		c.QueryLogFile == "" &&
 		c.ScrapeProtocols == nil
 }
 
+// RuntimeConfig configures the values for the process behavior.
+type RuntimeConfig struct {
+	// The Go garbage collection target percentage.
+	GoGC int `yaml:"gogc,omitempty"`
+}
+
+// isZero returns true iff the global config is the zero value.
+func (c *RuntimeConfig) isZero() bool {
+	return c.GoGC == 0
+}
+
 type ScrapeConfigs struct {
 	ScrapeConfigs []*ScrapeConfig `yaml:"scrape_configs,omitempty"`
 }
@@ -940,6 +971,8 @@ type AlertmanagerConfig struct {
 
 	// List of Alertmanager relabel configurations.
 	RelabelConfigs []*relabel.Config `yaml:"relabel_configs,omitempty"`
+	// Relabel alerts before sending to the specific alertmanager.
+	AlertRelabelConfigs []*relabel.Config `yaml:"alert_relabel_configs,omitempty"`
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -982,6 +1015,12 @@ func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) er
 		}
 	}
 
+	for _, rlcfg := range c.AlertRelabelConfigs {
+		if rlcfg == nil {
+			return errors.New("empty or null Alertmanager alert relabeling rule")
+		}
+	}
+
 	return nil
 }
 
@@ -1198,3 +1237,19 @@ func filePath(filename string) string {
 func fileErr(filename string, err error) error {
 	return fmt.Errorf("%q: %w", filePath(filename), err)
 }
+
+func getGoGCEnv() int {
+	goGCEnv := os.Getenv("GOGC")
+	// If the GOGC env var is set, use the same logic as upstream Go.
+	if goGCEnv != "" {
+		// Special case for GOGC=off.
+		if strings.ToLower(goGCEnv) == "off" {
+			return -1
+		}
+		i, err := strconv.Atoi(goGCEnv)
+		if err == nil {
+			return i
+		}
+	}
+	return DefaultRuntimeConfig.GoGC
+}

config/config_default_test.go

@@ -19,6 +19,7 @@ const ruleFilesConfigFile = "testdata/rules_abs_path.good.yml"
 
 var ruleFilesExpectedConf = &Config{
 	GlobalConfig: DefaultGlobalConfig,
+	Runtime:      DefaultRuntimeConfig,
 	RuleFiles: []string{
 		"testdata/first.rules",
 		"testdata/rules/second.rules",

config/config_test.go

@@ -58,6 +58,7 @@ import (
 	"github.com/prometheus/prometheus/discovery/zookeeper"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func mustParseURL(u string) *config.URL {
@@ -75,6 +76,7 @@ const (
 	globLabelLimit            = 30
 	globLabelNameLengthLimit  = 200
 	globLabelValueLengthLimit = 200
+	globalGoGC                = 42
 )
 
 var expectedConf = &Config{
@@ -95,6 +97,10 @@ var expectedConf = &Config{
 		ScrapeProtocols:       DefaultGlobalConfig.ScrapeProtocols,
 	},
 
+	Runtime: RuntimeConfig{
+		GoGC: globalGoGC,
+	},
+
 	RuleFiles: []string{
 		filepath.FromSlash("testdata/first.rules"),
 		filepath.FromSlash("testdata/my/*.rules"),
@@ -1839,7 +1845,7 @@ var expectedErrors = []struct {
 	},
 	{
 		filename: "azure_authentication_method.bad.yml",
-		errMsg:   "unknown authentication_type \"invalid\". Supported types are \"OAuth\" or \"ManagedIdentity\"",
+		errMsg:   "unknown authentication_type \"invalid\". Supported types are \"OAuth\", \"ManagedIdentity\" or \"SDK\"",
 	},
 	{
 		filename: "azure_bearertoken_basicauth.bad.yml",
@@ -2037,16 +2043,16 @@ func TestExpandExternalLabels(t *testing.T) {
 
 	c, err := LoadFile("testdata/external_labels.good.yml", false, false, log.NewNopLogger())
 	require.NoError(t, err)
-	require.Equal(t, labels.FromStrings("bar", "foo", "baz", "foo${TEST}bar", "foo", "${TEST}", "qux", "foo$${TEST}", "xyz", "foo$$bar"), c.GlobalConfig.ExternalLabels)
+	testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "foo${TEST}bar", "foo", "${TEST}", "qux", "foo$${TEST}", "xyz", "foo$$bar"), c.GlobalConfig.ExternalLabels)
 
 	c, err = LoadFile("testdata/external_labels.good.yml", false, true, log.NewNopLogger())
 	require.NoError(t, err)
-	require.Equal(t, labels.FromStrings("bar", "foo", "baz", "foobar", "foo", "", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
+	testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "foobar", "foo", "", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
 
 	os.Setenv("TEST", "TestValue")
 	c, err = LoadFile("testdata/external_labels.good.yml", false, true, log.NewNopLogger())
 	require.NoError(t, err)
-	require.Equal(t, labels.FromStrings("bar", "foo", "baz", "fooTestValuebar", "foo", "TestValue", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
+	testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "fooTestValuebar", "foo", "TestValue", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
 }
 
 func TestAgentMode(t *testing.T) {
@@ -2080,6 +2086,7 @@ func TestEmptyGlobalBlock(t *testing.T) {
 	c, err := Load("global:\n", false, log.NewNopLogger())
 	require.NoError(t, err)
 	exp := DefaultConfig
+	exp.Runtime = DefaultRuntimeConfig
 	require.Equal(t, exp, *c)
 }
 

config/config_windows_test.go

@@ -17,6 +17,7 @@ const ruleFilesConfigFile = "testdata/rules_abs_path_windows.good.yml"
 
 var ruleFilesExpectedConf = &Config{
 	GlobalConfig: DefaultGlobalConfig,
+	Runtime:      DefaultRuntimeConfig,
 	RuleFiles: []string{
 		"testdata\\first.rules",
 		"testdata\\rules\\second.rules",

config/testdata/conf.good.yml

@@ -14,6 +14,9 @@ global:
     monitor: codelab
     foo: bar
 
+runtime:
+  gogc: 42
+
 rule_files:
   - "first.rules"
   - "my/*.rules"

discovery/aws/ec2.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 	"time"
 
@@ -41,28 +42,29 @@ import (
 )
 
 const (
-	ec2Label                  = model.MetaLabelPrefix + "ec2_"
+	ec2Label                     = model.MetaLabelPrefix + "ec2_"
-	ec2LabelAMI               = ec2Label + "ami"
+	ec2LabelAMI                  = ec2Label + "ami"
-	ec2LabelAZ                = ec2Label + "availability_zone"
+	ec2LabelAZ                   = ec2Label + "availability_zone"
-	ec2LabelAZID              = ec2Label + "availability_zone_id"
+	ec2LabelAZID                 = ec2Label + "availability_zone_id"
-	ec2LabelArch              = ec2Label + "architecture"
+	ec2LabelArch                 = ec2Label + "architecture"
-	ec2LabelIPv6Addresses     = ec2Label + "ipv6_addresses"
+	ec2LabelIPv6Addresses        = ec2Label + "ipv6_addresses"
-	ec2LabelInstanceID        = ec2Label + "instance_id"
+	ec2LabelInstanceID           = ec2Label + "instance_id"
-	ec2LabelInstanceLifecycle = ec2Label + "instance_lifecycle"
+	ec2LabelInstanceLifecycle    = ec2Label + "instance_lifecycle"
-	ec2LabelInstanceState     = ec2Label + "instance_state"
+	ec2LabelInstanceState        = ec2Label + "instance_state"
-	ec2LabelInstanceType      = ec2Label + "instance_type"
+	ec2LabelInstanceType         = ec2Label + "instance_type"
-	ec2LabelOwnerID           = ec2Label + "owner_id"
+	ec2LabelOwnerID              = ec2Label + "owner_id"
-	ec2LabelPlatform          = ec2Label + "platform"
+	ec2LabelPlatform             = ec2Label + "platform"
-	ec2LabelPrimarySubnetID   = ec2Label + "primary_subnet_id"
+	ec2LabelPrimaryIPv6Addresses = ec2Label + "primary_ipv6_addresses"
-	ec2LabelPrivateDNS        = ec2Label + "private_dns_name"
+	ec2LabelPrimarySubnetID      = ec2Label + "primary_subnet_id"
-	ec2LabelPrivateIP         = ec2Label + "private_ip"
+	ec2LabelPrivateDNS           = ec2Label + "private_dns_name"
-	ec2LabelPublicDNS         = ec2Label + "public_dns_name"
+	ec2LabelPrivateIP            = ec2Label + "private_ip"
-	ec2LabelPublicIP          = ec2Label + "public_ip"
+	ec2LabelPublicDNS            = ec2Label + "public_dns_name"
-	ec2LabelRegion            = ec2Label + "region"
+	ec2LabelPublicIP             = ec2Label + "public_ip"
-	ec2LabelSubnetID          = ec2Label + "subnet_id"
+	ec2LabelRegion               = ec2Label + "region"
-	ec2LabelTag               = ec2Label + "tag_"
+	ec2LabelSubnetID             = ec2Label + "subnet_id"
-	ec2LabelVPCID             = ec2Label + "vpc_id"
+	ec2LabelTag                  = ec2Label + "tag_"
-	ec2LabelSeparator         = ","
+	ec2LabelVPCID                = ec2Label + "vpc_id"
+	ec2LabelSeparator            = ","
 )
 
 // DefaultEC2SDConfig is the default EC2 SD configuration.
@@ -279,7 +281,7 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error
 				if inst.PrivateDnsName != nil {
 					labels[ec2LabelPrivateDNS] = model.LabelValue(*inst.PrivateDnsName)
 				}
-				addr := net.JoinHostPort(*inst.PrivateIpAddress, fmt.Sprintf("%d", d.cfg.Port))
+				addr := net.JoinHostPort(*inst.PrivateIpAddress, strconv.Itoa(d.cfg.Port))
 				labels[model.AddressLabel] = model.LabelValue(addr)
 
 				if inst.Platform != nil {
@@ -316,6 +318,7 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error
 
 					var subnets []string
 					var ipv6addrs []string
+					var primaryipv6addrs []string
 					subnetsMap := make(map[string]struct{})
 					for _, eni := range inst.NetworkInterfaces {
 						if eni.SubnetId == nil {
@@ -329,6 +332,15 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error
 
 						for _, ipv6addr := range eni.Ipv6Addresses {
 							ipv6addrs = append(ipv6addrs, *ipv6addr.Ipv6Address)
+							if *ipv6addr.IsPrimaryIpv6 {
+								// we might have to extend the slice with more than one element
+								// that could leave empty strings in the list which is intentional
+								// to keep the position/device index information
+								for int64(len(primaryipv6addrs)) <= *eni.Attachment.DeviceIndex {
+									primaryipv6addrs = append(primaryipv6addrs, "")
+								}
+								primaryipv6addrs[*eni.Attachment.DeviceIndex] = *ipv6addr.Ipv6Address
+							}
 						}
 					}
 					labels[ec2LabelSubnetID] = model.LabelValue(
@@ -341,6 +353,12 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error
 								strings.Join(ipv6addrs, ec2LabelSeparator) +
 								ec2LabelSeparator)
 					}
+					if len(primaryipv6addrs) > 0 {
+						labels[ec2LabelPrimaryIPv6Addresses] = model.LabelValue(
+							ec2LabelSeparator +
+								strings.Join(primaryipv6addrs, ec2LabelSeparator) +
+								ec2LabelSeparator)
+					}
 				}
 
 				for _, t := range inst.Tags {

discovery/aws/lightsail.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 	"time"
 
@@ -229,7 +230,7 @@ func (d *LightsailDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group,
 			lightsailLabelRegion:              model.LabelValue(d.cfg.Region),
 		}
 
-		addr := net.JoinHostPort(*inst.PrivateIpAddress, fmt.Sprintf("%d", d.cfg.Port))
+		addr := net.JoinHostPort(*inst.PrivateIpAddress, strconv.Itoa(d.cfg.Port))
 		labels[model.AddressLabel] = model.LabelValue(addr)
 
 		if inst.PublicIpAddress != nil {

discovery/azure/azure.go

@@ -20,6 +20,7 @@ import (
 	"math/rand"
 	"net"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -65,6 +66,7 @@ const (
 	azureLabelMachineSize          = azureLabel + "machine_size"
 
 	authMethodOAuth           = "OAuth"
+	authMethodSDK             = "SDK"
 	authMethodManagedIdentity = "ManagedIdentity"
 )
 
@@ -164,8 +166,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		}
 	}
 
-	if c.AuthenticationMethod != authMethodOAuth && c.AuthenticationMethod != authMethodManagedIdentity {
+	if c.AuthenticationMethod != authMethodOAuth && c.AuthenticationMethod != authMethodManagedIdentity && c.AuthenticationMethod != authMethodSDK {
-		return fmt.Errorf("unknown authentication_type %q. Supported types are %q or %q", c.AuthenticationMethod, authMethodOAuth, authMethodManagedIdentity)
+		return fmt.Errorf("unknown authentication_type %q. Supported types are %q, %q or %q", c.AuthenticationMethod, authMethodOAuth, authMethodManagedIdentity, authMethodSDK)
 	}
 
 	return c.HTTPClientConfig.Validate()
@@ -212,6 +214,14 @@ func NewDiscovery(cfg *SDConfig, logger log.Logger, metrics discovery.Discoverer
 	return d, nil
 }
 
+type client interface {
+	getVMs(ctx context.Context, resourceGroup string) ([]virtualMachine, error)
+	getScaleSets(ctx context.Context, resourceGroup string) ([]armcompute.VirtualMachineScaleSet, error)
+	getScaleSetVMs(ctx context.Context, scaleSet armcompute.VirtualMachineScaleSet) ([]virtualMachine, error)
+	getVMNetworkInterfaceByID(ctx context.Context, networkInterfaceID string) (*armnetwork.Interface, error)
+	getVMScaleSetVMNetworkInterfaceByID(ctx context.Context, networkInterfaceID, scaleSetName, instanceID string) (*armnetwork.Interface, error)
+}
+
 // azureClient represents multiple Azure Resource Manager providers.
 type azureClient struct {
 	nic    *armnetwork.InterfacesClient
@@ -221,14 +231,17 @@ type azureClient struct {
 	logger log.Logger
 }
 
+var _ client = &azureClient{}
+
 // createAzureClient is a helper function for creating an Azure compute client to ARM.
-func createAzureClient(cfg SDConfig) (azureClient, error) {
+func createAzureClient(cfg SDConfig, logger log.Logger) (client, error) {
 	cloudConfiguration, err := CloudConfigurationFromName(cfg.Environment)
 	if err != nil {
-		return azureClient{}, err
+		return &azureClient{}, err
 	}
 
 	var c azureClient
+	c.logger = logger
 
 	telemetry := policy.TelemetryOptions{
 		ApplicationID: userAgent,
@@ -239,12 +252,12 @@ func createAzureClient(cfg SDConfig) (azureClient, error) {
 		Telemetry: telemetry,
 	})
 	if err != nil {
-		return azureClient{}, err
+		return &azureClient{}, err
 	}
 
 	client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, "azure_sd")
 	if err != nil {
-		return azureClient{}, err
+		return &azureClient{}, err
 	}
 	options := &arm.ClientOptions{
 		ClientOptions: policy.ClientOptions{
@@ -256,25 +269,25 @@ func createAzureClient(cfg SDConfig) (azureClient, error) {
 
 	c.vm, err = armcompute.NewVirtualMachinesClient(cfg.SubscriptionID, credential, options)
 	if err != nil {
-		return azureClient{}, err
+		return &azureClient{}, err
 	}
 
 	c.nic, err = armnetwork.NewInterfacesClient(cfg.SubscriptionID, credential, options)
 	if err != nil {
-		return azureClient{}, err
+		return &azureClient{}, err
 	}
 
 	c.vmss, err = armcompute.NewVirtualMachineScaleSetsClient(cfg.SubscriptionID, credential, options)
 	if err != nil {
-		return azureClient{}, err
+		return &azureClient{}, err
 	}
 
 	c.vmssvm, err = armcompute.NewVirtualMachineScaleSetVMsClient(cfg.SubscriptionID, credential, options)
 	if err != nil {
-		return azureClient{}, err
+		return &azureClient{}, err
 	}
 
-	return c, nil
+	return &c, nil
 }
 
 func newCredential(cfg SDConfig, policyClientOptions policy.ClientOptions) (azcore.TokenCredential, error) {
@@ -294,6 +307,16 @@ func newCredential(cfg SDConfig, policyClientOptions policy.ClientOptions) (azco
 			return nil, err
 		}
 		credential = azcore.TokenCredential(secretCredential)
+	case authMethodSDK:
+		options := &azidentity.DefaultAzureCredentialOptions{ClientOptions: policyClientOptions}
+		if len(cfg.TenantID) != 0 {
+			options.TenantID = cfg.TenantID
+		}
+		sdkCredential, err := azidentity.NewDefaultAzureCredential(options)
+		if err != nil {
+			return nil, err
+		}
+		credential = azcore.TokenCredential(sdkCredential)
 	}
 	return credential, nil
 }
@@ -330,12 +353,11 @@ func newAzureResourceFromID(id string, logger log.Logger) (*arm.ResourceID, erro
 func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	defer level.Debug(d.logger).Log("msg", "Azure discovery completed")
 
-	client, err := createAzureClient(*d.cfg)
+	client, err := createAzureClient(*d.cfg, d.logger)
 	if err != nil {
 		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("could not create Azure client: %w", err)
 	}
-	client.logger = d.logger
 
 	machines, err := client.getVMs(ctx, d.cfg.ResourceGroup)
 	if err != nil {
@@ -374,102 +396,8 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	for _, vm := range machines {
 		go func(vm virtualMachine) {
 			defer wg.Done()
-			r, err := newAzureResourceFromID(vm.ID, d.logger)
+			labelSet, err := d.vmToLabelSet(ctx, client, vm)
-			if err != nil {
+			ch <- target{labelSet: labelSet, err: err}
-				ch <- target{labelSet: nil, err: err}
-				return
-			}
-
-			labels := model.LabelSet{
-				azureLabelSubscriptionID:       model.LabelValue(d.cfg.SubscriptionID),
-				azureLabelTenantID:             model.LabelValue(d.cfg.TenantID),
-				azureLabelMachineID:            model.LabelValue(vm.ID),
-				azureLabelMachineName:          model.LabelValue(vm.Name),
-				azureLabelMachineComputerName:  model.LabelValue(vm.ComputerName),
-				azureLabelMachineOSType:        model.LabelValue(vm.OsType),
-				azureLabelMachineLocation:      model.LabelValue(vm.Location),
-				azureLabelMachineResourceGroup: model.LabelValue(r.ResourceGroupName),
-				azureLabelMachineSize:          model.LabelValue(vm.Size),
-			}
-
-			if vm.ScaleSet != "" {
-				labels[azureLabelMachineScaleSet] = model.LabelValue(vm.ScaleSet)
-			}
-
-			for k, v := range vm.Tags {
-				name := strutil.SanitizeLabelName(k)
-				labels[azureLabelMachineTag+model.LabelName(name)] = model.LabelValue(*v)
-			}
-
-			// Get the IP address information via separate call to the network provider.
-			for _, nicID := range vm.NetworkInterfaces {
-				var networkInterface *armnetwork.Interface
-				if v, ok := d.getFromCache(nicID); ok {
-					networkInterface = v
-					d.metrics.cacheHitCount.Add(1)
-				} else {
-					if vm.ScaleSet == "" {
-						networkInterface, err = client.getVMNetworkInterfaceByID(ctx, nicID)
-					} else {
-						networkInterface, err = client.getVMScaleSetVMNetworkInterfaceByID(ctx, nicID, vm.ScaleSet, vm.InstanceID)
-					}
-					if err != nil {
-						if errors.Is(err, errorNotFound) {
-							level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
-						} else {
-							ch <- target{labelSet: nil, err: err}
-						}
-						d.addToCache(nicID, networkInterface)
-					} else {
-						networkInterface, err = client.getVMScaleSetVMNetworkInterfaceByID(ctx, nicID, vm.ScaleSet, vm.InstanceID)
-						if err != nil {
-							if errors.Is(err, errorNotFound) {
-								level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
-							} else {
-								ch <- target{labelSet: nil, err: err}
-							}
-							// Get out of this routine because we cannot continue without a network interface.
-							return
-						}
-						d.addToCache(nicID, networkInterface)
-					}
-				}
-
-				if networkInterface.Properties == nil {
-					continue
-				}
-
-				// Unfortunately Azure does not return information on whether a VM is deallocated.
-				// This information is available via another API call however the Go SDK does not
-				// yet support this. On deallocated machines, this value happens to be nil so it
-				// is a cheap and easy way to determine if a machine is allocated or not.
-				if networkInterface.Properties.Primary == nil {
-					level.Debug(d.logger).Log("msg", "Skipping deallocated virtual machine", "machine", vm.Name)
-					return
-				}
-
-				if *networkInterface.Properties.Primary {
-					for _, ip := range networkInterface.Properties.IPConfigurations {
-						// IPAddress is a field defined in PublicIPAddressPropertiesFormat,
-						// therefore we need to validate that both are not nil.
-						if ip.Properties != nil && ip.Properties.PublicIPAddress != nil && ip.Properties.PublicIPAddress.Properties != nil && ip.Properties.PublicIPAddress.Properties.IPAddress != nil {
-							labels[azureLabelMachinePublicIP] = model.LabelValue(*ip.Properties.PublicIPAddress.Properties.IPAddress)
-						}
-						if ip.Properties != nil && ip.Properties.PrivateIPAddress != nil {
-							labels[azureLabelMachinePrivateIP] = model.LabelValue(*ip.Properties.PrivateIPAddress)
-							address := net.JoinHostPort(*ip.Properties.PrivateIPAddress, fmt.Sprintf("%d", d.port))
-							labels[model.AddressLabel] = model.LabelValue(address)
-							ch <- target{labelSet: labels, err: nil}
-							return
-						}
-						// If we made it here, we don't have a private IP which should be impossible.
-						// Return an empty target and error to ensure an all or nothing situation.
-						err = fmt.Errorf("unable to find a private IP for VM %s", vm.Name)
-						ch <- target{labelSet: nil, err: err}
-						return
-					}
-				}
-			}
 		}(vm)
 	}
 
@@ -490,6 +418,95 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	return []*targetgroup.Group{&tg}, nil
 }
 
+func (d *Discovery) vmToLabelSet(ctx context.Context, client client, vm virtualMachine) (model.LabelSet, error) {
+	r, err := newAzureResourceFromID(vm.ID, d.logger)
+	if err != nil {
+		return nil, err
+	}
+
+	labels := model.LabelSet{
+		azureLabelSubscriptionID:       model.LabelValue(d.cfg.SubscriptionID),
+		azureLabelTenantID:             model.LabelValue(d.cfg.TenantID),
+		azureLabelMachineID:            model.LabelValue(vm.ID),
+		azureLabelMachineName:          model.LabelValue(vm.Name),
+		azureLabelMachineComputerName:  model.LabelValue(vm.ComputerName),
+		azureLabelMachineOSType:        model.LabelValue(vm.OsType),
+		azureLabelMachineLocation:      model.LabelValue(vm.Location),
+		azureLabelMachineResourceGroup: model.LabelValue(r.ResourceGroupName),
+		azureLabelMachineSize:          model.LabelValue(vm.Size),
+	}
+
+	if vm.ScaleSet != "" {
+		labels[azureLabelMachineScaleSet] = model.LabelValue(vm.ScaleSet)
+	}
+
+	for k, v := range vm.Tags {
+		name := strutil.SanitizeLabelName(k)
+		labels[azureLabelMachineTag+model.LabelName(name)] = model.LabelValue(*v)
+	}
+
+	// Get the IP address information via separate call to the network provider.
+	for _, nicID := range vm.NetworkInterfaces {
+		var networkInterface *armnetwork.Interface
+		if v, ok := d.getFromCache(nicID); ok {
+			networkInterface = v
+			d.metrics.cacheHitCount.Add(1)
+		} else {
+			if vm.ScaleSet == "" {
+				networkInterface, err = client.getVMNetworkInterfaceByID(ctx, nicID)
+			} else {
+				networkInterface, err = client.getVMScaleSetVMNetworkInterfaceByID(ctx, nicID, vm.ScaleSet, vm.InstanceID)
+			}
+			if err != nil {
+				if errors.Is(err, errorNotFound) {
+					level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
+				} else {
+					return nil, err
+				}
+				// Get out of this routine because we cannot continue without a network interface.
+				return nil, nil
+			}
+
+			// Continue processing with the network interface
+			d.addToCache(nicID, networkInterface)
+		}
+
+		if networkInterface.Properties == nil {
+			continue
+		}
+
+		// Unfortunately Azure does not return information on whether a VM is deallocated.
+		// This information is available via another API call however the Go SDK does not
+		// yet support this. On deallocated machines, this value happens to be nil so it
+		// is a cheap and easy way to determine if a machine is allocated or not.
+		if networkInterface.Properties.Primary == nil {
+			level.Debug(d.logger).Log("msg", "Skipping deallocated virtual machine", "machine", vm.Name)
+			return nil, nil
+		}
+
+		if *networkInterface.Properties.Primary {
+			for _, ip := range networkInterface.Properties.IPConfigurations {
+				// IPAddress is a field defined in PublicIPAddressPropertiesFormat,
+				// therefore we need to validate that both are not nil.
+				if ip.Properties != nil && ip.Properties.PublicIPAddress != nil && ip.Properties.PublicIPAddress.Properties != nil && ip.Properties.PublicIPAddress.Properties.IPAddress != nil {
+					labels[azureLabelMachinePublicIP] = model.LabelValue(*ip.Properties.PublicIPAddress.Properties.IPAddress)
+				}
+				if ip.Properties != nil && ip.Properties.PrivateIPAddress != nil {
+					labels[azureLabelMachinePrivateIP] = model.LabelValue(*ip.Properties.PrivateIPAddress)
+					address := net.JoinHostPort(*ip.Properties.PrivateIPAddress, strconv.Itoa(d.port))
+					labels[model.AddressLabel] = model.LabelValue(address)
+					return labels, nil
+				}
+				// If we made it here, we don't have a private IP which should be impossible.
+				// Return an empty target and error to ensure an all or nothing situation.
+				return nil, fmt.Errorf("unable to find a private IP for VM %s", vm.Name)
+			}
+		}
+	}
+	// TODO: Should we say something at this point?
+	return nil, nil
+}
+
 func (client *azureClient) getVMs(ctx context.Context, resourceGroup string) ([]virtualMachine, error) {
 	var vms []virtualMachine
 	if len(resourceGroup) == 0 {
@@ -569,7 +586,7 @@ func (client *azureClient) getScaleSetVMs(ctx context.Context, scaleSet armcompu
 }
 
 func mapFromVM(vm armcompute.VirtualMachine) virtualMachine {
-	osType := string(*vm.Properties.StorageProfile.OSDisk.OSType)
+	var osType string
 	tags := map[string]*string{}
 	networkInterfaces := []string{}
 	var computerName string
@@ -580,6 +597,12 @@ func mapFromVM(vm armcompute.VirtualMachine) virtualMachine {
 	}
 
 	if vm.Properties != nil {
+		if vm.Properties.StorageProfile != nil &&
+			vm.Properties.StorageProfile.OSDisk != nil &&
+			vm.Properties.StorageProfile.OSDisk.OSType != nil {
+			osType = string(*vm.Properties.StorageProfile.OSDisk.OSType)
+		}
+
 		if vm.Properties.NetworkProfile != nil {
 			for _, vmNIC := range vm.Properties.NetworkProfile.NetworkInterfaces {
 				networkInterfaces = append(networkInterfaces, *vmNIC.ID)
@@ -608,7 +631,7 @@ func mapFromVM(vm armcompute.VirtualMachine) virtualMachine {
 }
 
 func mapFromVMScaleSetVM(vm armcompute.VirtualMachineScaleSetVM, scaleSetName string) virtualMachine {
-	osType := string(*vm.Properties.StorageProfile.OSDisk.OSType)
+	var osType string
 	tags := map[string]*string{}
 	networkInterfaces := []string{}
 	var computerName string
@@ -619,6 +642,12 @@ func mapFromVMScaleSetVM(vm armcompute.VirtualMachineScaleSetVM, scaleSetName st
 	}
 
 	if vm.Properties != nil {
+		if vm.Properties.StorageProfile != nil &&
+			vm.Properties.StorageProfile.OSDisk != nil &&
+			vm.Properties.StorageProfile.OSDisk.OSType != nil {
+			osType = string(*vm.Properties.StorageProfile.OSDisk.OSType)
+		}
+
 		if vm.Properties.NetworkProfile != nil {
 			for _, vmNIC := range vm.Properties.NetworkProfile.NetworkInterfaces {
 				networkInterfaces = append(networkInterfaces, *vmNIC.ID)

discovery/azure/azure_test.go

@@ -14,16 +14,24 @@
 package azure
 
 import (
+	"context"
+	"fmt"
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4"
+	cache "github.com/Code-Hex/go-generics-cache"
+	"github.com/Code-Hex/go-generics-cache/policy/lru"
+	"github.com/go-kit/log"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 )
 
 func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
+	goleak.VerifyTestMain(m,
+		goleak.IgnoreTopFunction("github.com/Code-Hex/go-generics-cache.(*janitor).run.func1"),
+	)
 }
 
 func TestMapFromVMWithEmptyTags(t *testing.T) {
@@ -79,6 +87,140 @@ func TestMapFromVMWithEmptyTags(t *testing.T) {
 	require.Equal(t, expectedVM, actualVM)
 }
 
+func TestVMToLabelSet(t *testing.T) {
+	id := "/subscriptions/00000000-0000-0000-0000-000000000000/test"
+	name := "name"
+	size := "size"
+	vmSize := armcompute.VirtualMachineSizeTypes(size)
+	osType := armcompute.OperatingSystemTypesLinux
+	vmType := "type"
+	location := "westeurope"
+	computerName := "computer_name"
+	networkID := "/subscriptions/00000000-0000-0000-0000-000000000000/network1"
+	ipAddress := "10.20.30.40"
+	primary := true
+	networkProfile := armcompute.NetworkProfile{
+		NetworkInterfaces: []*armcompute.NetworkInterfaceReference{
+			{
+				ID:         &networkID,
+				Properties: &armcompute.NetworkInterfaceReferenceProperties{Primary: &primary},
+			},
+		},
+	}
+	properties := &armcompute.VirtualMachineProperties{
+		OSProfile: &armcompute.OSProfile{
+			ComputerName: &computerName,
+		},
+		StorageProfile: &armcompute.StorageProfile{
+			OSDisk: &armcompute.OSDisk{
+				OSType: &osType,
+			},
+		},
+		NetworkProfile: &networkProfile,
+		HardwareProfile: &armcompute.HardwareProfile{
+			VMSize: &vmSize,
+		},
+	}
+
+	testVM := armcompute.VirtualMachine{
+		ID:         &id,
+		Name:       &name,
+		Type:       &vmType,
+		Location:   &location,
+		Tags:       nil,
+		Properties: properties,
+	}
+
+	expectedVM := virtualMachine{
+		ID:                id,
+		Name:              name,
+		ComputerName:      computerName,
+		Type:              vmType,
+		Location:          location,
+		OsType:            "Linux",
+		Tags:              map[string]*string{},
+		NetworkInterfaces: []string{networkID},
+		Size:              size,
+	}
+
+	actualVM := mapFromVM(testVM)
+
+	require.Equal(t, expectedVM, actualVM)
+
+	cfg := DefaultSDConfig
+	d := &Discovery{
+		cfg:    &cfg,
+		logger: log.NewNopLogger(),
+		cache:  cache.New(cache.AsLRU[string, *armnetwork.Interface](lru.WithCapacity(5))),
+	}
+	network := armnetwork.Interface{
+		Name: &networkID,
+		Properties: &armnetwork.InterfacePropertiesFormat{
+			Primary: &primary,
+			IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
+				{Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
+					PrivateIPAddress: &ipAddress,
+				}},
+			},
+		},
+	}
+	client := &mockAzureClient{
+		networkInterface: &network,
+	}
+	labelSet, err := d.vmToLabelSet(context.Background(), client, actualVM)
+	require.NoError(t, err)
+	require.Len(t, labelSet, 11)
+}
+
+func TestMapFromVMWithEmptyOSType(t *testing.T) {
+	id := "test"
+	name := "name"
+	size := "size"
+	vmSize := armcompute.VirtualMachineSizeTypes(size)
+	vmType := "type"
+	location := "westeurope"
+	computerName := "computer_name"
+	networkProfile := armcompute.NetworkProfile{
+		NetworkInterfaces: []*armcompute.NetworkInterfaceReference{},
+	}
+	properties := &armcompute.VirtualMachineProperties{
+		OSProfile: &armcompute.OSProfile{
+			ComputerName: &computerName,
+		},
+		StorageProfile: &armcompute.StorageProfile{
+			OSDisk: &armcompute.OSDisk{},
+		},
+		NetworkProfile: &networkProfile,
+		HardwareProfile: &armcompute.HardwareProfile{
+			VMSize: &vmSize,
+		},
+	}
+
+	testVM := armcompute.VirtualMachine{
+		ID:         &id,
+		Name:       &name,
+		Type:       &vmType,
+		Location:   &location,
+		Tags:       nil,
+		Properties: properties,
+	}
+
+	expectedVM := virtualMachine{
+		ID:                id,
+		Name:              name,
+		ComputerName:      computerName,
+		Type:              vmType,
+		Location:          location,
+		Tags:              map[string]*string{},
+		NetworkInterfaces: []string{},
+		Size:              size,
+	}
+
+	actualVM := mapFromVM(testVM)
+
+	require.Equal(t, expectedVM, actualVM)
+}
+
 func TestMapFromVMWithTags(t *testing.T) {
 	id := "test"
 	name := "name"
@@ -193,6 +335,58 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) {
 	require.Equal(t, expectedVM, actualVM)
 }
 
+func TestMapFromVMScaleSetVMWithEmptyOSType(t *testing.T) {
+	id := "test"
+	name := "name"
+	size := "size"
+	vmSize := armcompute.VirtualMachineSizeTypes(size)
+	vmType := "type"
+	instanceID := "123"
+	location := "westeurope"
+	computerName := "computer_name"
+	networkProfile := armcompute.NetworkProfile{
+		NetworkInterfaces: []*armcompute.NetworkInterfaceReference{},
+	}
+	properties := &armcompute.VirtualMachineScaleSetVMProperties{
+		OSProfile: &armcompute.OSProfile{
+			ComputerName: &computerName,
+		},
+		StorageProfile: &armcompute.StorageProfile{},
+		NetworkProfile: &networkProfile,
+		HardwareProfile: &armcompute.HardwareProfile{
+			VMSize: &vmSize,
+		},
+	}
+
+	testVM := armcompute.VirtualMachineScaleSetVM{
+		ID:         &id,
+		Name:       &name,
+		Type:       &vmType,
+		InstanceID: &instanceID,
+		Location:   &location,
+		Tags:       nil,
+		Properties: properties,
+	}
+
+	scaleSet := "testSet"
+	expectedVM := virtualMachine{
+		ID:                id,
+		Name:              name,
+		ComputerName:      computerName,
+		Type:              vmType,
+		Location:          location,
+		Tags:              map[string]*string{},
+		NetworkInterfaces: []string{},
+		ScaleSet:          scaleSet,
+		InstanceID:        instanceID,
+		Size:              size,
+	}
+
+	actualVM := mapFromVMScaleSetVM(testVM, scaleSet)
+
+	require.Equal(t, expectedVM, actualVM)
+}
+
 func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
 	id := "test"
 	name := "name"
@@ -280,3 +474,35 @@ func TestNewAzureResourceFromID(t *testing.T) {
 		require.Equal(t, tc.expected.ResourceGroupName, actual.ResourceGroupName)
 	}
 }
+
+type mockAzureClient struct {
+	networkInterface *armnetwork.Interface
+}
+
+var _ client = &mockAzureClient{}
+
+func (*mockAzureClient) getVMs(ctx context.Context, resourceGroup string) ([]virtualMachine, error) {
+	return nil, nil
+}
+
+func (*mockAzureClient) getScaleSets(ctx context.Context, resourceGroup string) ([]armcompute.VirtualMachineScaleSet, error) {
+	return nil, nil
+}
+
+func (*mockAzureClient) getScaleSetVMs(ctx context.Context, scaleSet armcompute.VirtualMachineScaleSet) ([]virtualMachine, error) {
+	return nil, nil
+}
+
+func (m *mockAzureClient) getVMNetworkInterfaceByID(ctx context.Context, networkInterfaceID string) (*armnetwork.Interface, error) {
+	if networkInterfaceID == "" {
+		return nil, fmt.Errorf("parameter networkInterfaceID cannot be empty")
+	}
+	return m.networkInterface, nil
+}
+
+func (m *mockAzureClient) getVMScaleSetVMNetworkInterfaceByID(ctx context.Context, networkInterfaceID, scaleSetName, instanceID string) (*armnetwork.Interface, error) {
+	if scaleSetName == "" {
+		return nil, fmt.Errorf("parameter virtualMachineScaleSetName cannot be empty")
+	}
+	return m.networkInterface, nil
+}

discovery/consul/consul.go

@@ -539,9 +539,9 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
 		// since the service may be registered remotely through a different node.
 		var addr string
 		if serviceNode.Service.Address != "" {
-			addr = net.JoinHostPort(serviceNode.Service.Address, fmt.Sprintf("%d", serviceNode.Service.Port))
+			addr = net.JoinHostPort(serviceNode.Service.Address, strconv.Itoa(serviceNode.Service.Port))
 		} else {
-			addr = net.JoinHostPort(serviceNode.Node.Address, fmt.Sprintf("%d", serviceNode.Service.Port))
+			addr = net.JoinHostPort(serviceNode.Node.Address, strconv.Itoa(serviceNode.Service.Port))
 		}
 
 		labels := model.LabelSet{

discovery/consul/consul_test.go

@@ -56,15 +56,11 @@ func TestConfiguredService(t *testing.T) {
 	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
 
 	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
-	if err != nil {
+	require.NoError(t, err, "when initializing discovery")
-		t.Errorf("Unexpected error when initializing discovery %v", err)
+	require.True(t, consulDiscovery.shouldWatch("configuredServiceName", []string{""}),
-	}
+		"Expected service %s to be watched", "configuredServiceName")
-	if !consulDiscovery.shouldWatch("configuredServiceName", []string{""}) {
+	require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}),
-		t.Errorf("Expected service %s to be watched", "configuredServiceName")
+		"Expected service %s to not be watched", "nonConfiguredServiceName")
-	}
-	if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
-		t.Errorf("Expected service %s to not be watched", "nonConfiguredServiceName")
-	}
 }
 
 func TestConfiguredServiceWithTag(t *testing.T) {
@@ -76,21 +72,18 @@ func TestConfiguredServiceWithTag(t *testing.T) {
 	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
 
 	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
-	if err != nil {
+	require.NoError(t, err, "when initializing discovery")
-		t.Errorf("Unexpected error when initializing discovery %v", err)
+	require.False(t, consulDiscovery.shouldWatch("configuredServiceName", []string{""}),
-	}
+		"Expected service %s to not be watched without tag", "configuredServiceName")
-	if consulDiscovery.shouldWatch("configuredServiceName", []string{""}) {
+
-		t.Errorf("Expected service %s to not be watched without tag", "configuredServiceName")
+	require.True(t, consulDiscovery.shouldWatch("configuredServiceName", []string{"http"}),
-	}
+		"Expected service %s to be watched with tag %s", "configuredServiceName", "http")
-	if !consulDiscovery.shouldWatch("configuredServiceName", []string{"http"}) {
+
-		t.Errorf("Expected service %s to be watched with tag %s", "configuredServiceName", "http")
+	require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}),
-	}
+		"Expected service %s to not be watched without tag", "nonConfiguredServiceName")
-	if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
+
-		t.Errorf("Expected service %s to not be watched without tag", "nonConfiguredServiceName")
+	require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{"http"}),
-	}
+		"Expected service %s to not be watched with tag %s", "nonConfiguredServiceName", "http")
-	if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{"http"}) {
-		t.Errorf("Expected service %s to not be watched with tag %s", "nonConfiguredServiceName", "http")
-	}
 }
 
 func TestConfiguredServiceWithTags(t *testing.T) {
@@ -173,13 +166,10 @@ func TestConfiguredServiceWithTags(t *testing.T) {
 		metrics := NewTestMetrics(t, tc.conf, prometheus.NewRegistry())
 
 		consulDiscovery, err := NewDiscovery(tc.conf, nil, metrics)
-		if err != nil {
+		require.NoError(t, err, "when initializing discovery")
-			t.Errorf("Unexpected error when initializing discovery %v", err)
-		}
 		ret := consulDiscovery.shouldWatch(tc.serviceName, tc.serviceTags)
-		if ret != tc.shouldWatch {
+		require.Equal(t, tc.shouldWatch, ret, "Watched service and tags: %s %+v, input was %s %+v",
-			t.Errorf("Expected should watch? %t, got %t. Watched service and tags: %s %+v, input was %s %+v", tc.shouldWatch, ret, tc.conf.Services, tc.conf.ServiceTags, tc.serviceName, tc.serviceTags)
+			tc.conf.Services, tc.conf.ServiceTags, tc.serviceName, tc.serviceTags)
-		}
 	}
 }
 
@@ -189,12 +179,8 @@ func TestNonConfiguredService(t *testing.T) {
 	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
 
 	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
-	if err != nil {
+	require.NoError(t, err, "when initializing discovery")
-		t.Errorf("Unexpected error when initializing discovery %v", err)
+	require.True(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}), "Expected service %s to be watched", "nonConfiguredServiceName")
-	}
-	if !consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
-		t.Errorf("Expected service %s to be watched", "nonConfiguredServiceName")
-	}
 }
 
 const (
@@ -502,13 +488,10 @@ oauth2:
 			var config SDConfig
 			err := config.UnmarshalYAML(unmarshal([]byte(test.config)))
 			if err != nil {
-				require.Equalf(t, err.Error(), test.errMessage, "Expected error '%s', got '%v'", test.errMessage, err)
+				require.EqualError(t, err, test.errMessage)
-				return
-			}
-			if test.errMessage != "" {
-				t.Errorf("Expected error %s, got none", test.errMessage)
 				return
 			}
+			require.Empty(t, test.errMessage, "Expected error.")
 
 			require.Equal(t, test.expected, config)
 		})

discovery/digitalocean/digitalocean.go

@@ -177,7 +177,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 		}
 
 		labels := model.LabelSet{
-			doLabelID:          model.LabelValue(fmt.Sprintf("%d", droplet.ID)),
+			doLabelID:          model.LabelValue(strconv.Itoa(droplet.ID)),
 			doLabelName:        model.LabelValue(droplet.Name),
 			doLabelImage:       model.LabelValue(droplet.Image.Slug),
 			doLabelImageName:   model.LabelValue(droplet.Image.Name),

discovery/dns/dns.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -200,7 +201,7 @@ func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targ
 
 	tg := &targetgroup.Group{}
 	hostPort := func(a string, p int) model.LabelValue {
-		return model.LabelValue(net.JoinHostPort(a, fmt.Sprintf("%d", p)))
+		return model.LabelValue(net.JoinHostPort(a, strconv.Itoa(p)))
 	}
 
 	for _, record := range response.Answer {
@@ -209,7 +210,7 @@ func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targ
 		switch addr := record.(type) {
 		case *dns.SRV:
 			dnsSrvRecordTarget = model.LabelValue(addr.Target)
-			dnsSrvRecordPort = model.LabelValue(fmt.Sprintf("%d", addr.Port))
+			dnsSrvRecordPort = model.LabelValue(strconv.Itoa(int(addr.Port)))
 
 			// Remove the final dot from rooted DNS names to make them look more usual.
 			addr.Target = strings.TrimRight(addr.Target, ".")

discovery/eureka/client.go

@@ -81,7 +81,7 @@ const appListPath string = "/apps"
 func fetchApps(ctx context.Context, server string, client *http.Client) (*Applications, error) {
 	url := fmt.Sprintf("%s%s", server, appListPath)
 
-	request, err := http.NewRequest("GET", url, nil)
+	request, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return nil, err
 	}

discovery/eureka/eureka.go

@@ -228,7 +228,6 @@ func targetsForApp(app *Application) []model.LabelSet {
 		}
 
 		targets = append(targets, target)
-
 	}
 	return targets
 }

discovery/file/file_test.go

@@ -208,7 +208,6 @@ func (t *testRunner) requireUpdate(ref time.Time, expected []*targetgroup.Group)
 		select {
 		case <-timeout:
 			t.Fatalf("Expected update but got none")
-			return
 		case <-time.After(defaultWait / 10):
 			if ref.Equal(t.lastReceive()) {
 				// No update received.
@@ -358,9 +357,7 @@ func TestInvalidFile(t *testing.T) {
 
 			// Verify that we've received nothing.
 			time.Sleep(defaultWait)
-			if runner.lastReceive().After(now) {
+			require.False(t, runner.lastReceive().After(now), "unexpected targets received: %v", runner.targets())
-				t.Fatalf("unexpected targets received: %v", runner.targets())
-			}
 		})
 	}
 }

discovery/hetzner/hcloud.go

@@ -15,7 +15,6 @@ package hetzner
 
 import (
 	"context"
-	"fmt"
 	"net"
 	"net/http"
 	"strconv"
@@ -92,7 +91,7 @@ func (d *hcloudDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er
 	for i, server := range servers {
 		labels := model.LabelSet{
 			hetznerLabelRole:              model.LabelValue(HetznerRoleHcloud),
-			hetznerLabelServerID:          model.LabelValue(fmt.Sprintf("%d", server.ID)),
+			hetznerLabelServerID:          model.LabelValue(strconv.FormatInt(server.ID, 10)),
 			hetznerLabelServerName:        model.LabelValue(server.Name),
 			hetznerLabelDatacenter:        model.LabelValue(server.Datacenter.Name),
 			hetznerLabelPublicIPv4:        model.LabelValue(server.PublicNet.IPv4.IP.String()),
@@ -102,10 +101,10 @@ func (d *hcloudDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er
 			hetznerLabelHcloudDatacenterLocation:            model.LabelValue(server.Datacenter.Location.Name),
 			hetznerLabelHcloudDatacenterLocationNetworkZone: model.LabelValue(server.Datacenter.Location.NetworkZone),
 			hetznerLabelHcloudType:                          model.LabelValue(server.ServerType.Name),
-			hetznerLabelHcloudCPUCores:                      model.LabelValue(fmt.Sprintf("%d", server.ServerType.Cores)),
+			hetznerLabelHcloudCPUCores:                      model.LabelValue(strconv.Itoa(server.ServerType.Cores)),
 			hetznerLabelHcloudCPUType:                       model.LabelValue(server.ServerType.CPUType),
-			hetznerLabelHcloudMemoryGB:                      model.LabelValue(fmt.Sprintf("%d", int(server.ServerType.Memory))),
+			hetznerLabelHcloudMemoryGB:                      model.LabelValue(strconv.Itoa(int(server.ServerType.Memory))),
-			hetznerLabelHcloudDiskGB:                        model.LabelValue(fmt.Sprintf("%d", server.ServerType.Disk)),
+			hetznerLabelHcloudDiskGB:                        model.LabelValue(strconv.Itoa(server.ServerType.Disk)),
 
 			model.AddressLabel: model.LabelValue(net.JoinHostPort(server.PublicNet.IPv4.IP.String(), strconv.FormatUint(uint64(d.port), 10))),
 		}

discovery/hetzner/robot.go

@@ -70,7 +70,7 @@ func newRobotDiscovery(conf *SDConfig, _ log.Logger) (*robotDiscovery, error) {
 }
 
 func (d *robotDiscovery) refresh(context.Context) ([]*targetgroup.Group, error) {
-	req, err := http.NewRequest("GET", d.endpoint+"/server", nil)
+	req, err := http.NewRequest(http.MethodGet, d.endpoint+"/server", nil)
 	if err != nil {
 		return nil, err
 	}
@@ -112,7 +112,7 @@ func (d *robotDiscovery) refresh(context.Context) ([]*targetgroup.Group, error)
 			hetznerLabelPublicIPv4:     model.LabelValue(server.Server.ServerIP),
 			hetznerLabelServerStatus:   model.LabelValue(server.Server.Status),
 			hetznerLabelRobotProduct:   model.LabelValue(server.Server.Product),
-			hetznerLabelRobotCancelled: model.LabelValue(fmt.Sprintf("%t", server.Server.Canceled)),
+			hetznerLabelRobotCancelled: model.LabelValue(strconv.FormatBool(server.Server.Canceled)),
 
 			model.AddressLabel: model.LabelValue(net.JoinHostPort(server.Server.ServerIP, strconv.FormatUint(uint64(d.port), 10))),
 		}
@@ -122,7 +122,6 @@ func (d *robotDiscovery) refresh(context.Context) ([]*targetgroup.Group, error)
 				labels[hetznerLabelPublicIPv6Network] = model.LabelValue(fmt.Sprintf("%s/%s", subnet.IP, subnet.Mask))
 				break
 			}
-
 		}
 		targets[i] = labels
 	}

discovery/http/http.go

@@ -150,7 +150,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPCli
 }
 
 func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) {
-	req, err := http.NewRequest("GET", d.url, nil)
+	req, err := http.NewRequest(http.MethodGet, d.url, nil)
 	if err != nil {
 		return nil, err
 	}

discovery/kubernetes/endpoints.go

@@ -62,6 +62,8 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 	svcUpdateCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleUpdate)
 	svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete)
 
+	podUpdateCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleUpdate)
+
 	e := &Endpoints{
 		logger:           l,
 		endpointsInf:     eps,
@@ -131,6 +133,29 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 	if err != nil {
 		level.Error(l).Log("msg", "Error adding services event handler.", "err", err)
 	}
+	_, err = e.podInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		UpdateFunc: func(old, cur interface{}) {
+			podUpdateCount.Inc()
+			oldPod, ok := old.(*apiv1.Pod)
+			if !ok {
+				return
+			}
+
+			curPod, ok := cur.(*apiv1.Pod)
+			if !ok {
+				return
+			}
+
+			// the Pod's phase may change without triggering an update on the Endpoints/Service.
+			// https://github.com/prometheus/prometheus/issues/11305.
+			if curPod.Status.Phase != oldPod.Status.Phase {
+				e.enqueuePod(namespacedName(curPod.Namespace, curPod.Name))
+			}
+		},
+	})
+	if err != nil {
+		level.Error(l).Log("msg", "Error adding pods event handler.", "err", err)
+	}
 	if e.withNodeMetadata {
 		_, err = e.nodeInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 			AddFunc: func(o interface{}) {
@@ -166,6 +191,18 @@ func (e *Endpoints) enqueueNode(nodeName string) {
 	}
 }
 
+func (e *Endpoints) enqueuePod(podNamespacedName string) {
+	endpoints, err := e.endpointsInf.GetIndexer().ByIndex(podIndex, podNamespacedName)
+	if err != nil {
+		level.Error(e.logger).Log("msg", "Error getting endpoints for pod", "pod", podNamespacedName, "err", err)
+		return
+	}
+
+	for _, endpoint := range endpoints {
+		e.enqueue(endpoint)
+	}
+}
+
 func (e *Endpoints) enqueue(obj interface{}) {
 	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
 	if err != nil {
@@ -312,7 +349,7 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group {
 			tg.Targets = append(tg.Targets, target)
 			return
 		}
-		s := pod.Namespace + "/" + pod.Name
+		s := namespacedName(pod.Namespace, pod.Name)
 
 		sp, ok := seenPods[s]
 		if !ok {

discovery/kubernetes/endpoints_test.go

@@ -969,3 +969,123 @@ func TestEndpointsDiscoveryEmptyPodStatus(t *testing.T) {
 		expectedRes:      map[string]*targetgroup.Group{},
 	}.Run(t)
 }
+
+// TestEndpointsUpdatePod makes sure that Endpoints discovery detects underlying Pods changes.
+// See https://github.com/prometheus/prometheus/issues/11305 for more details.
+func TestEndpointsDiscoveryUpdatePod(t *testing.T) {
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "testpod",
+			Namespace: "default",
+			UID:       types.UID("deadbeef"),
+		},
+		Spec: v1.PodSpec{
+			NodeName: "testnode",
+			Containers: []v1.Container{
+				{
+					Name:  "c1",
+					Image: "c1:latest",
+					Ports: []v1.ContainerPort{
+						{
+							Name:          "mainport",
+							ContainerPort: 9000,
+							Protocol:      v1.ProtocolTCP,
+						},
+					},
+				},
+			},
+		},
+		Status: v1.PodStatus{
+			// Pod is in Pending phase when discovered for first time.
+			Phase: "Pending",
+			Conditions: []v1.PodCondition{
+				{
+					Type:   v1.PodReady,
+					Status: v1.ConditionFalse,
+				},
+			},
+			HostIP: "2.3.4.5",
+			PodIP:  "4.3.2.1",
+		},
+	}
+	objs := []runtime.Object{
+		&v1.Endpoints{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "testendpoints",
+				Namespace: "default",
+			},
+			Subsets: []v1.EndpointSubset{
+				{
+					Addresses: []v1.EndpointAddress{
+						{
+							IP: "4.3.2.1",
+							// The Pending Pod may be included because the Endpoints was created manually.
+							// Or because the corresponding service has ".spec.publishNotReadyAddresses: true".
+							TargetRef: &v1.ObjectReference{
+								Kind:      "Pod",
+								Name:      "testpod",
+								Namespace: "default",
+							},
+						},
+					},
+					Ports: []v1.EndpointPort{
+						{
+							Name:     "mainport",
+							Port:     9000,
+							Protocol: v1.ProtocolTCP,
+						},
+					},
+				},
+			},
+		},
+		pod,
+	}
+	n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, objs...)
+
+	k8sDiscoveryTest{
+		discovery: n,
+		afterStart: func() {
+			// the Pod becomes Ready.
+			pod.Status.Phase = "Running"
+			pod.Status.Conditions = []v1.PodCondition{
+				{
+					Type:   v1.PodReady,
+					Status: v1.ConditionTrue,
+				},
+			}
+			c.CoreV1().Pods(pod.Namespace).Update(context.Background(), pod, metav1.UpdateOptions{})
+		},
+		expectedMaxItems: 2,
+		expectedRes: map[string]*targetgroup.Group{
+			"endpoints/default/testendpoints": {
+				Targets: []model.LabelSet{
+					{
+						"__address__":                                    "4.3.2.1:9000",
+						"__meta_kubernetes_endpoint_port_name":           "mainport",
+						"__meta_kubernetes_endpoint_port_protocol":       "TCP",
+						"__meta_kubernetes_endpoint_ready":               "true",
+						"__meta_kubernetes_endpoint_address_target_kind": "Pod",
+						"__meta_kubernetes_endpoint_address_target_name": "testpod",
+						"__meta_kubernetes_pod_name":                     "testpod",
+						"__meta_kubernetes_pod_ip":                       "4.3.2.1",
+						"__meta_kubernetes_pod_ready":                    "true",
+						"__meta_kubernetes_pod_phase":                    "Running",
+						"__meta_kubernetes_pod_node_name":                "testnode",
+						"__meta_kubernetes_pod_host_ip":                  "2.3.4.5",
+						"__meta_kubernetes_pod_container_name":           "c1",
+						"__meta_kubernetes_pod_container_image":          "c1:latest",
+						"__meta_kubernetes_pod_container_port_name":      "mainport",
+						"__meta_kubernetes_pod_container_port_number":    "9000",
+						"__meta_kubernetes_pod_container_port_protocol":  "TCP",
+						"__meta_kubernetes_pod_uid":                      "deadbeef",
+					},
+				},
+				Labels: model.LabelSet{
+					"__meta_kubernetes_namespace":      "default",
+					"__meta_kubernetes_endpoints_name": "testendpoints",
+				},
+				Source: "endpoints/default/testendpoints",
+			},
+		},
+	}.Run(t)
+}

discovery/kubernetes/endpointslice.go

@@ -265,7 +265,9 @@ const (
 	endpointSliceEndpointConditionsReadyLabel       = metaLabelPrefix + "endpointslice_endpoint_conditions_ready"
 	endpointSliceEndpointConditionsServingLabel     = metaLabelPrefix + "endpointslice_endpoint_conditions_serving"
 	endpointSliceEndpointConditionsTerminatingLabel = metaLabelPrefix + "endpointslice_endpoint_conditions_terminating"
+	endpointSliceEndpointZoneLabel                  = metaLabelPrefix + "endpointslice_endpoint_zone"
 	endpointSliceEndpointHostnameLabel              = metaLabelPrefix + "endpointslice_endpoint_hostname"
+	endpointSliceEndpointNodenameLabel              = metaLabelPrefix + "endpointslice_endpoint_node_name"
 	endpointSliceAddressTargetKindLabel             = metaLabelPrefix + "endpointslice_address_target_kind"
 	endpointSliceAddressTargetNameLabel             = metaLabelPrefix + "endpointslice_address_target_name"
 	endpointSliceEndpointTopologyLabelPrefix        = metaLabelPrefix + "endpointslice_endpoint_topology_"
@@ -338,6 +340,14 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou
 			target[model.LabelName(endpointSliceAddressTargetNameLabel)] = lv(ep.targetRef().Name)
 		}
 
+		if ep.nodename() != nil {
+			target[endpointSliceEndpointNodenameLabel] = lv(*ep.nodename())
+		}
+
+		if ep.zone() != nil {
+			target[model.LabelName(endpointSliceEndpointZoneLabel)] = lv(*ep.zone())
+		}
+
 		for k, v := range ep.topology() {
 			ln := strutil.SanitizeLabelName(k)
 			target[model.LabelName(endpointSliceEndpointTopologyLabelPrefix+ln)] = lv(v)
@@ -358,7 +368,7 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou
 			tg.Targets = append(tg.Targets, target)
 			return
 		}
-		s := pod.Namespace + "/" + pod.Name
+		s := namespacedName(pod.Namespace, pod.Name)
 
 		sp, ok := seenPods[s]
 		if !ok {

discovery/kubernetes/endpointslice_adaptor.go

@@ -44,6 +44,7 @@ type endpointSliceEndpointAdaptor interface {
 	addresses() []string
 	hostname() *string
 	nodename() *string
+	zone() *string
 	conditions() endpointSliceEndpointConditionsAdaptor
 	targetRef() *corev1.ObjectReference
 	topology() map[string]string
@@ -181,6 +182,10 @@ func (e *endpointSliceEndpointAdaptorV1) nodename() *string {
 	return e.endpoint.NodeName
 }
 
+func (e *endpointSliceEndpointAdaptorV1) zone() *string {
+	return e.endpoint.Zone
+}
+
 func (e *endpointSliceEndpointAdaptorV1) conditions() endpointSliceEndpointConditionsAdaptor {
 	return newEndpointSliceEndpointConditionsAdaptorFromV1(e.endpoint.Conditions)
 }
@@ -233,6 +238,10 @@ func (e *endpointSliceEndpointAdaptorV1beta1) nodename() *string {
 	return e.endpoint.NodeName
 }
 
+func (e *endpointSliceEndpointAdaptorV1beta1) zone() *string {
+	return nil
+}
+
 func (e *endpointSliceEndpointAdaptorV1beta1) conditions() endpointSliceEndpointConditionsAdaptor {
 	return newEndpointSliceEndpointConditionsAdaptorFromV1beta1(e.endpoint.Conditions)
 }

discovery/kubernetes/endpointslice_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 
 	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/discovery/v1"
 	"k8s.io/api/discovery/v1beta1"
@@ -79,6 +80,7 @@ func makeEndpointSliceV1() *v1.EndpointSlice {
 				DeprecatedTopology: map[string]string{
 					"topology": "value",
 				},
+				Zone: strptr("us-east-1a"),
 			}, {
 				Addresses: []string{"2.3.4.5"},
 				Conditions: v1.EndpointConditions{
@@ -86,6 +88,7 @@ func makeEndpointSliceV1() *v1.EndpointSlice {
 					Serving:     boolptr(true),
 					Terminating: boolptr(false),
 				},
+				Zone: strptr("us-east-1b"),
 			}, {
 				Addresses: []string{"3.4.5.6"},
 				Conditions: v1.EndpointConditions{
@@ -93,6 +96,7 @@ func makeEndpointSliceV1() *v1.EndpointSlice {
 					Serving:     boolptr(true),
 					Terminating: boolptr(true),
 				},
+				Zone: strptr("us-east-1c"),
 			}, {
 				Addresses: []string{"4.5.6.7"},
 				Conditions: v1.EndpointConditions{
@@ -104,6 +108,7 @@ func makeEndpointSliceV1() *v1.EndpointSlice {
 					Kind: "Node",
 					Name: "barbaz",
 				},
+				Zone: strptr("us-east-1a"),
 			},
 		},
 	}
@@ -184,8 +189,10 @@ func TestEndpointSliceDiscoveryBeforeRun(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":        "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating":    "false",
 						"__meta_kubernetes_endpointslice_endpoint_hostname":                  "testendpoint1",
+						"__meta_kubernetes_endpointslice_endpoint_node_name":                 "foobar",
 						"__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true",
 						"__meta_kubernetes_endpointslice_endpoint_topology_topology":         "value",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                      "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                               "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":                  "http",
 						"__meta_kubernetes_endpointslice_port_name":                          "testport",
@@ -196,6 +203,7 @@ func TestEndpointSliceDiscoveryBeforeRun(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1b",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -206,6 +214,7 @@ func TestEndpointSliceDiscoveryBeforeRun(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "false",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1c",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -218,6 +227,7 @@ func TestEndpointSliceDiscoveryBeforeRun(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -451,8 +461,10 @@ func TestEndpointSliceDiscoveryDelete(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":        "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating":    "false",
 						"__meta_kubernetes_endpointslice_endpoint_hostname":                  "testendpoint1",
+						"__meta_kubernetes_endpointslice_endpoint_node_name":                 "foobar",
 						"__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true",
 						"__meta_kubernetes_endpointslice_endpoint_topology_topology":         "value",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                      "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                               "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":                  "http",
 						"__meta_kubernetes_endpointslice_port_name":                          "testport",
@@ -463,6 +475,7 @@ func TestEndpointSliceDiscoveryDelete(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1b",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -473,6 +486,7 @@ func TestEndpointSliceDiscoveryDelete(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "false",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1c",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -485,6 +499,7 @@ func TestEndpointSliceDiscoveryDelete(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -551,8 +566,10 @@ func TestEndpointSliceDiscoveryUpdate(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":        "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating":    "false",
 						"__meta_kubernetes_endpointslice_endpoint_hostname":                  "testendpoint1",
+						"__meta_kubernetes_endpointslice_endpoint_node_name":                 "foobar",
 						"__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true",
 						"__meta_kubernetes_endpointslice_endpoint_topology_topology":         "value",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                      "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                               "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":                  "http",
 						"__meta_kubernetes_endpointslice_port_name":                          "testport",
@@ -563,6 +580,7 @@ func TestEndpointSliceDiscoveryUpdate(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1b",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -573,6 +591,7 @@ func TestEndpointSliceDiscoveryUpdate(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "false",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1c",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -585,6 +604,7 @@ func TestEndpointSliceDiscoveryUpdate(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -640,8 +660,10 @@ func TestEndpointSliceDiscoveryEmptyEndpoints(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":        "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating":    "false",
 						"__meta_kubernetes_endpointslice_endpoint_hostname":                  "testendpoint1",
+						"__meta_kubernetes_endpointslice_endpoint_node_name":                 "foobar",
 						"__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true",
 						"__meta_kubernetes_endpointslice_endpoint_topology_topology":         "value",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                      "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                               "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":                  "http",
 						"__meta_kubernetes_endpointslice_port_name":                          "testport",
@@ -652,6 +674,7 @@ func TestEndpointSliceDiscoveryEmptyEndpoints(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1b",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -662,6 +685,7 @@ func TestEndpointSliceDiscoveryEmptyEndpoints(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "false",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1c",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -674,6 +698,7 @@ func TestEndpointSliceDiscoveryEmptyEndpoints(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -724,8 +749,10 @@ func TestEndpointSliceDiscoveryWithService(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":        "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating":    "false",
 						"__meta_kubernetes_endpointslice_endpoint_hostname":                  "testendpoint1",
+						"__meta_kubernetes_endpointslice_endpoint_node_name":                 "foobar",
 						"__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true",
 						"__meta_kubernetes_endpointslice_endpoint_topology_topology":         "value",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                      "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                               "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":                  "http",
 						"__meta_kubernetes_endpointslice_port_name":                          "testport",
@@ -736,6 +763,7 @@ func TestEndpointSliceDiscoveryWithService(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1b",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -746,6 +774,7 @@ func TestEndpointSliceDiscoveryWithService(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "false",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1c",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -758,6 +787,7 @@ func TestEndpointSliceDiscoveryWithService(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -824,8 +854,10 @@ func TestEndpointSliceDiscoveryWithServiceUpdate(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":        "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating":    "false",
 						"__meta_kubernetes_endpointslice_endpoint_hostname":                  "testendpoint1",
+						"__meta_kubernetes_endpointslice_endpoint_node_name":                 "foobar",
 						"__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true",
 						"__meta_kubernetes_endpointslice_endpoint_topology_topology":         "value",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                      "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                               "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":                  "http",
 						"__meta_kubernetes_endpointslice_port_name":                          "testport",
@@ -836,6 +868,7 @@ func TestEndpointSliceDiscoveryWithServiceUpdate(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1b",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
 						"__meta_kubernetes_endpointslice_port_protocol":                   "TCP",
@@ -846,6 +879,7 @@ func TestEndpointSliceDiscoveryWithServiceUpdate(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "false",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1c",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
 						"__meta_kubernetes_endpointslice_port_protocol":                   "TCP",
@@ -858,6 +892,7 @@ func TestEndpointSliceDiscoveryWithServiceUpdate(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -914,8 +949,10 @@ func TestEndpointsSlicesDiscoveryWithNodeMetadata(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":        "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating":    "false",
 						"__meta_kubernetes_endpointslice_endpoint_hostname":                  "testendpoint1",
+						"__meta_kubernetes_endpointslice_endpoint_node_name":                 "foobar",
 						"__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true",
 						"__meta_kubernetes_endpointslice_endpoint_topology_topology":         "value",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                      "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                               "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":                  "http",
 						"__meta_kubernetes_endpointslice_port_name":                          "testport",
@@ -929,6 +966,7 @@ func TestEndpointsSlicesDiscoveryWithNodeMetadata(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1b",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -939,6 +977,7 @@ func TestEndpointsSlicesDiscoveryWithNodeMetadata(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "false",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1c",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -951,6 +990,7 @@ func TestEndpointsSlicesDiscoveryWithNodeMetadata(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -1014,8 +1054,10 @@ func TestEndpointsSlicesDiscoveryWithUpdatedNodeMetadata(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":        "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating":    "false",
 						"__meta_kubernetes_endpointslice_endpoint_hostname":                  "testendpoint1",
+						"__meta_kubernetes_endpointslice_endpoint_node_name":                 "foobar",
 						"__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true",
 						"__meta_kubernetes_endpointslice_endpoint_topology_topology":         "value",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                      "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                               "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":                  "http",
 						"__meta_kubernetes_endpointslice_port_name":                          "testport",
@@ -1029,6 +1071,7 @@ func TestEndpointsSlicesDiscoveryWithUpdatedNodeMetadata(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1b",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -1039,6 +1082,7 @@ func TestEndpointsSlicesDiscoveryWithUpdatedNodeMetadata(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "false",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1c",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -1051,6 +1095,7 @@ func TestEndpointsSlicesDiscoveryWithUpdatedNodeMetadata(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -1160,8 +1205,10 @@ func TestEndpointSliceDiscoveryNamespaces(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":        "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating":    "false",
 						"__meta_kubernetes_endpointslice_endpoint_hostname":                  "testendpoint1",
+						"__meta_kubernetes_endpointslice_endpoint_node_name":                 "foobar",
 						"__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true",
 						"__meta_kubernetes_endpointslice_endpoint_topology_topology":         "value",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                      "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                               "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":                  "http",
 						"__meta_kubernetes_endpointslice_port_name":                          "testport",
@@ -1172,6 +1219,7 @@ func TestEndpointSliceDiscoveryNamespaces(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1b",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
 						"__meta_kubernetes_endpointslice_port_protocol":                   "TCP",
@@ -1182,6 +1230,7 @@ func TestEndpointSliceDiscoveryNamespaces(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "false",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1c",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
 						"__meta_kubernetes_endpointslice_port_protocol":                   "TCP",
@@ -1194,6 +1243,7 @@ func TestEndpointSliceDiscoveryNamespaces(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -1308,8 +1358,10 @@ func TestEndpointSliceDiscoveryOwnNamespace(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":        "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating":    "false",
 						"__meta_kubernetes_endpointslice_endpoint_hostname":                  "testendpoint1",
+						"__meta_kubernetes_endpointslice_endpoint_node_name":                 "foobar",
 						"__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true",
 						"__meta_kubernetes_endpointslice_endpoint_topology_topology":         "value",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                      "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                               "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":                  "http",
 						"__meta_kubernetes_endpointslice_port_name":                          "testport",
@@ -1320,6 +1372,7 @@ func TestEndpointSliceDiscoveryOwnNamespace(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1b",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
 						"__meta_kubernetes_endpointslice_port_protocol":                   "TCP",
@@ -1330,6 +1383,7 @@ func TestEndpointSliceDiscoveryOwnNamespace(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "false",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1c",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
 						"__meta_kubernetes_endpointslice_port_protocol":                   "TCP",
@@ -1342,6 +1396,7 @@ func TestEndpointSliceDiscoveryOwnNamespace(t *testing.T) {
 						"__meta_kubernetes_endpointslice_endpoint_conditions_ready":       "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_serving":     "true",
 						"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
+						"__meta_kubernetes_endpointslice_endpoint_zone":                   "us-east-1a",
 						"__meta_kubernetes_endpointslice_port":                            "9000",
 						"__meta_kubernetes_endpointslice_port_app_protocol":               "http",
 						"__meta_kubernetes_endpointslice_port_name":                       "testport",
@@ -1405,3 +1460,41 @@ func TestEndpointSliceDiscoveryEmptyPodStatus(t *testing.T) {
 		expectedRes:      map[string]*targetgroup.Group{},
 	}.Run(t)
 }
+
+// TestEndpointSliceInfIndexersCount makes sure that RoleEndpointSlice discovery
+// sets up indexing for the main Kube informer only when needed.
+// See: https://github.com/prometheus/prometheus/pull/13554#discussion_r1490965817
+func TestEndpointSliceInfIndexersCount(t *testing.T) {
+	tests := []struct {
+		name             string
+		withNodeMetadata bool
+	}{
+		{"with node metadata", true},
+		{"without node metadata", false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var (
+				n                    *Discovery
+				mainInfIndexersCount int
+			)
+			if tc.withNodeMetadata {
+				mainInfIndexersCount = 1
+				n, _ = makeDiscoveryWithMetadata(RoleEndpointSlice, NamespaceDiscovery{}, AttachMetadataConfig{Node: true})
+			} else {
+				n, _ = makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{})
+			}
+
+			k8sDiscoveryTest{
+				discovery: n,
+				afterStart: func() {
+					n.RLock()
+					defer n.RUnlock()
+					require.Len(t, n.discoverers, 1)
+					require.Len(t, n.discoverers[0].(*EndpointSlice).endpointSliceInf.GetIndexer().GetIndexers(), mainInfIndexersCount)
+				},
+			}.Run(t)
+		})
+	}
+}

discovery/kubernetes/kubernetes.go

@@ -311,7 +311,7 @@ func New(l log.Logger, metrics discovery.DiscovererMetrics, conf *SDConfig) (*Di
 		}
 	case conf.APIServer.URL == nil:
 		// Use the Kubernetes provided pod service account
-		// as described in https://kubernetes.io/docs/admin/service-accounts-admin/
+		// as described in https://kubernetes.io/docs/tasks/run-application/access-api-from-pod/#using-official-client-libraries
 		kcfg, err = rest.InClusterConfig()
 		if err != nil {
 			return nil, err
@@ -485,8 +485,8 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			eps := NewEndpointSlice(
 				log.With(d.logger, "role", "endpointslice"),
 				informer,
-				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
+				d.mustNewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
-				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
+				d.mustNewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
 				d.metrics.eventCount,
 			)
@@ -545,8 +545,8 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			eps := NewEndpoints(
 				log.With(d.logger, "role", "endpoint"),
 				d.newEndpointsByNodeInformer(elw),
-				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
+				d.mustNewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
-				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
+				d.mustNewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
 				d.metrics.eventCount,
 			)
@@ -602,7 +602,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			}
 			svc := NewService(
 				log.With(d.logger, "role", "service"),
-				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
+				d.mustNewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
 				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, svc)
@@ -641,7 +641,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 						return i.Watch(ctx, options)
 					},
 				}
-				informer = cache.NewSharedInformer(ilw, &networkv1.Ingress{}, resyncDisabled)
+				informer = d.mustNewSharedInformer(ilw, &networkv1.Ingress{}, resyncDisabled)
 			} else {
 				i := d.client.NetworkingV1beta1().Ingresses(namespace)
 				ilw := &cache.ListWatch{
@@ -656,7 +656,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 						return i.Watch(ctx, options)
 					},
 				}
-				informer = cache.NewSharedInformer(ilw, &v1beta1.Ingress{}, resyncDisabled)
+				informer = d.mustNewSharedInformer(ilw, &v1beta1.Ingress{}, resyncDisabled)
 			}
 			ingress := NewIngress(
 				log.With(d.logger, "role", "ingress"),
@@ -747,7 +747,7 @@ func (d *Discovery) newNodeInformer(ctx context.Context) cache.SharedInformer {
 			return d.client.CoreV1().Nodes().Watch(ctx, options)
 		},
 	}
-	return cache.NewSharedInformer(nlw, &apiv1.Node{}, resyncDisabled)
+	return d.mustNewSharedInformer(nlw, &apiv1.Node{}, resyncDisabled)
 }
 
 func (d *Discovery) newPodsByNodeInformer(plw *cache.ListWatch) cache.SharedIndexInformer {
@@ -762,13 +762,28 @@ func (d *Discovery) newPodsByNodeInformer(plw *cache.ListWatch) cache.SharedInde
 		}
 	}
 
-	return cache.NewSharedIndexInformer(plw, &apiv1.Pod{}, resyncDisabled, indexers)
+	return d.mustNewSharedIndexInformer(plw, &apiv1.Pod{}, resyncDisabled, indexers)
 }
 
 func (d *Discovery) newEndpointsByNodeInformer(plw *cache.ListWatch) cache.SharedIndexInformer {
 	indexers := make(map[string]cache.IndexFunc)
+	indexers[podIndex] = func(obj interface{}) ([]string, error) {
+		e, ok := obj.(*apiv1.Endpoints)
+		if !ok {
+			return nil, fmt.Errorf("object is not endpoints")
+		}
+		var pods []string
+		for _, target := range e.Subsets {
+			for _, addr := range target.Addresses {
+				if addr.TargetRef != nil && addr.TargetRef.Kind == "Pod" {
+					pods = append(pods, namespacedName(addr.TargetRef.Namespace, addr.TargetRef.Name))
+				}
+			}
+		}
+		return pods, nil
+	}
 	if !d.attachMetadata.Node {
-		return cache.NewSharedIndexInformer(plw, &apiv1.Endpoints{}, resyncDisabled, indexers)
+		return d.mustNewSharedIndexInformer(plw, &apiv1.Endpoints{}, resyncDisabled, indexers)
 	}
 
 	indexers[nodeIndex] = func(obj interface{}) ([]string, error) {
@@ -794,13 +809,13 @@ func (d *Discovery) newEndpointsByNodeInformer(plw *cache.ListWatch) cache.Share
 		return nodes, nil
 	}
 
-	return cache.NewSharedIndexInformer(plw, &apiv1.Endpoints{}, resyncDisabled, indexers)
+	return d.mustNewSharedIndexInformer(plw, &apiv1.Endpoints{}, resyncDisabled, indexers)
 }
 
 func (d *Discovery) newEndpointSlicesByNodeInformer(plw *cache.ListWatch, object runtime.Object) cache.SharedIndexInformer {
 	indexers := make(map[string]cache.IndexFunc)
 	if !d.attachMetadata.Node {
-		cache.NewSharedIndexInformer(plw, &disv1.EndpointSlice{}, resyncDisabled, indexers)
+		return d.mustNewSharedIndexInformer(plw, object, resyncDisabled, indexers)
 	}
 
 	indexers[nodeIndex] = func(obj interface{}) ([]string, error) {
@@ -839,7 +854,32 @@ func (d *Discovery) newEndpointSlicesByNodeInformer(plw *cache.ListWatch, object
 		return nodes, nil
 	}
 
-	return cache.NewSharedIndexInformer(plw, object, resyncDisabled, indexers)
+	return d.mustNewSharedIndexInformer(plw, object, resyncDisabled, indexers)
+}
+
+func (d *Discovery) informerWatchErrorHandler(r *cache.Reflector, err error) {
+	d.metrics.failuresCount.Inc()
+	cache.DefaultWatchErrorHandler(r, err)
+}
+
+func (d *Discovery) mustNewSharedInformer(lw cache.ListerWatcher, exampleObject runtime.Object, defaultEventHandlerResyncPeriod time.Duration) cache.SharedInformer {
+	informer := cache.NewSharedInformer(lw, exampleObject, defaultEventHandlerResyncPeriod)
+	// Invoking SetWatchErrorHandler should fail only if the informer has been started beforehand.
+	// Such a scenario would suggest an incorrect use of the API, thus the panic.
+	if err := informer.SetWatchErrorHandler(d.informerWatchErrorHandler); err != nil {
+		panic(err)
+	}
+	return informer
+}
+
+func (d *Discovery) mustNewSharedIndexInformer(lw cache.ListerWatcher, exampleObject runtime.Object, defaultEventHandlerResyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+	informer := cache.NewSharedIndexInformer(lw, exampleObject, defaultEventHandlerResyncPeriod, indexers)
+	// Invoking SetWatchErrorHandler should fail only if the informer has been started beforehand.
+	// Such a scenario would suggest an incorrect use of the API, thus the panic.
+	if err := informer.SetWatchErrorHandler(d.informerWatchErrorHandler); err != nil {
+		panic(err)
+	}
+	return informer
 }
 
 func checkDiscoveryV1Supported(client kubernetes.Interface) (bool, error) {
@@ -872,3 +912,7 @@ func addObjectMetaLabels(labelSet model.LabelSet, objectMeta metav1.ObjectMeta,
 		labelSet[model.LabelName(metaLabelPrefix+string(role)+"_annotationpresent_"+ln)] = presentValue
 	}
 }
+
+func namespacedName(namespace, name string) string {
+	return namespace + "/" + name
+}

discovery/kubernetes/kubernetes_test.go

@@ -21,12 +21,16 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/require"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/apimachinery/pkg/watch"
 	fakediscovery "k8s.io/client-go/discovery/fake"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
+	kubetesting "k8s.io/client-go/testing"
 	"k8s.io/client-go/tools/cache"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -128,17 +132,11 @@ func (d k8sDiscoveryTest) Run(t *testing.T) {
 	}
 
 	resChan := make(chan map[string]*targetgroup.Group)
-	go readResultWithTimeout(t, ch, d.expectedMaxItems, time.Second, resChan)
+	go readResultWithTimeout(t, ctx, ch, d.expectedMaxItems, time.Second, resChan)
 
 	dd, ok := d.discovery.(hasSynced)
-	if !ok {
+	require.True(t, ok, "discoverer does not implement hasSynced interface")
-		t.Errorf("discoverer does not implement hasSynced interface")
+	require.True(t, cache.WaitForCacheSync(ctx.Done(), dd.hasSynced), "discoverer failed to sync: %v", dd)
-		return
-	}
-	if !cache.WaitForCacheSync(ctx.Done(), dd.hasSynced) {
-		t.Errorf("discoverer failed to sync: %v", dd)
-		return
-	}
 
 	if d.afterStart != nil {
 		d.afterStart()
@@ -147,13 +145,18 @@ func (d k8sDiscoveryTest) Run(t *testing.T) {
 	if d.expectedRes != nil {
 		res := <-resChan
 		requireTargetGroups(t, d.expectedRes, res)
+	} else {
+		// Stop readResultWithTimeout and wait for it.
+		cancel()
+		<-resChan
 	}
 }
 
 // readResultWithTimeout reads all targetgroups from channel with timeout.
 // It merges targetgroups by source and sends the result to result channel.
-func readResultWithTimeout(t *testing.T, ch <-chan []*targetgroup.Group, max int, timeout time.Duration, resChan chan<- map[string]*targetgroup.Group) {
+func readResultWithTimeout(t *testing.T, ctx context.Context, ch <-chan []*targetgroup.Group, max int, stopAfter time.Duration, resChan chan<- map[string]*targetgroup.Group) {
 	res := make(map[string]*targetgroup.Group)
+	timeout := time.After(stopAfter)
 Loop:
 	for {
 		select {
@@ -168,12 +171,15 @@ Loop:
 				// Reached max target groups we may get, break fast.
 				break Loop
 			}
-		case <-time.After(timeout):
+		case <-timeout:
 			// Because we use queue, an object that is created then
 			// deleted or updated may be processed only once.
 			// So possibly we may skip events, timed out here.
 			t.Logf("timed out, got %d (max: %d) items, some events are skipped", len(res), max)
 			break Loop
+		case <-ctx.Done():
+			t.Logf("stopped, got %d (max: %d) items", len(res), max)
+			break Loop
 		}
 	}
 
@@ -312,3 +318,39 @@ func TestCheckNetworkingV1Supported(t *testing.T) {
 		})
 	}
 }
+
+func TestFailuresCountMetric(t *testing.T) {
+	tests := []struct {
+		role             Role
+		minFailedWatches int
+	}{
+		{RoleNode, 1},
+		{RolePod, 1},
+		{RoleService, 1},
+		{RoleEndpoint, 3},
+		{RoleEndpointSlice, 3},
+		{RoleIngress, 1},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(string(tc.role), func(t *testing.T) {
+			t.Parallel()
+
+			n, c := makeDiscovery(tc.role, NamespaceDiscovery{})
+			// The counter is initialized and no failures at the beginning.
+			require.Equal(t, float64(0), prom_testutil.ToFloat64(n.metrics.failuresCount))
+
+			// Simulate an error on watch requests.
+			c.Discovery().(*fakediscovery.FakeDiscovery).PrependWatchReactor("*", func(action kubetesting.Action) (bool, watch.Interface, error) {
+				return true, nil, apierrors.NewUnauthorized("unauthorized")
+			})
+
+			// Start the discovery.
+			k8sDiscoveryTest{discovery: n}.Run(t)
+
+			// At least the errors of the initial watches should be caught (watches are retried on errors).
+			require.GreaterOrEqual(t, prom_testutil.ToFloat64(n.metrics.failuresCount), float64(tc.minFailedWatches))
+		})
+	}
+}

discovery/kubernetes/metrics.go

@@ -22,7 +22,8 @@ import (
 var _ discovery.DiscovererMetrics = (*kubernetesMetrics)(nil)
 
 type kubernetesMetrics struct {
-	eventCount *prometheus.CounterVec
+	eventCount    *prometheus.CounterVec
+	failuresCount prometheus.Counter
 
 	metricRegisterer discovery.MetricRegisterer
 }
@@ -37,10 +38,18 @@ func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetric
 			},
 			[]string{"role", "event"},
 		),
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: discovery.KubernetesMetricsNamespace,
+				Name:      "failures_total",
+				Help:      "The number of failed WATCH/LIST requests.",
+			},
+		),
 	}
 
 	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
 		m.eventCount,
+		m.failuresCount,
 	})
 
 	// Initialize metric vectors.
@@ -61,6 +70,8 @@ func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetric
 		}
 	}
 
+	m.failuresCount.Add(0)
+
 	return m
 }
 

discovery/kubernetes/pod.go

@@ -33,7 +33,10 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-const nodeIndex = "node"
+const (
+	nodeIndex = "node"
+	podIndex  = "pod"
+)
 
 // Pod discovers new pod targets.
 type Pod struct {
@@ -326,7 +329,7 @@ func podSource(pod *apiv1.Pod) string {
 }
 
 func podSourceFromNamespaceAndName(namespace, name string) string {
-	return "pod/" + namespace + "/" + name
+	return "pod/" + namespacedName(namespace, name)
 }
 
 func podReady(pod *apiv1.Pod) model.LabelValue {

discovery/legacymanager/manager_test.go

@@ -720,7 +720,7 @@ func staticConfig(addrs ...string) discovery.StaticConfig {
 	var cfg discovery.StaticConfig
 	for i, addr := range addrs {
 		cfg = append(cfg, &targetgroup.Group{
-			Source: fmt.Sprint(i),
+			Source: strconv.Itoa(i),
 			Targets: []model.LabelSet{
 				{model.AddressLabel: model.LabelValue(addr)},
 			},
@@ -733,7 +733,6 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
 	t.Helper()
 	if _, ok := tSets[poolKey]; !ok {
 		t.Fatalf("'%s' should be present in Pool keys: %v", poolKey, tSets)
-		return
 	}
 
 	match := false

discovery/linode/linode.go

@@ -59,17 +59,22 @@ const (
 	linodeLabelSpecsVCPUs         = linodeLabel + "specs_vcpus"
 	linodeLabelSpecsTransferBytes = linodeLabel + "specs_transfer_bytes"
 	linodeLabelExtraIPs           = linodeLabel + "extra_ips"
+	linodeLabelIPv6Ranges         = linodeLabel + "ipv6_ranges"
 
 	// This is our events filter; when polling for changes, we care only about
 	// events since our last refresh.
-	// Docs: https://www.linode.com/docs/api/account/#events-list
+	// Docs: https://www.linode.com/docs/api/account/#events-list.
 	filterTemplate = `{"created": {"+gte": "%s"}}`
+
+	// Optional region filtering.
+	regionFilterTemplate = `{"region": "%s"}`
 )
 
 // DefaultSDConfig is the default Linode SD configuration.
 var DefaultSDConfig = SDConfig{
 	TagSeparator:     ",",
 	Port:             80,
+	Region:           "",
 	RefreshInterval:  model.Duration(60 * time.Second),
 	HTTPClientConfig: config.DefaultHTTPClientConfig,
 }
@@ -85,6 +90,7 @@ type SDConfig struct {
 	RefreshInterval model.Duration `yaml:"refresh_interval"`
 	Port            int            `yaml:"port"`
 	TagSeparator    string         `yaml:"tag_separator,omitempty"`
+	Region          string         `yaml:"region,omitempty"`
 }
 
 // NewDiscovererMetrics implements discovery.Config.
@@ -122,6 +128,7 @@ type Discovery struct {
 	*refresh.Discovery
 	client               *linodego.Client
 	port                 int
+	region               string
 	tagSeparator         string
 	lastRefreshTimestamp time.Time
 	pollCount            int
@@ -139,6 +146,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.Discovere
 
 	d := &Discovery{
 		port:                 conf.Port,
+		region:               conf.Region,
 		tagSeparator:         conf.TagSeparator,
 		pollCount:            0,
 		lastRefreshTimestamp: time.Now().UTC(),
@@ -178,12 +186,12 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	if d.lastResults != nil && d.eventPollingEnabled {
 		// Check to see if there have been any events. If so, refresh our data.
-		opts := linodego.ListOptions{
+		eventsOpts := linodego.ListOptions{
 			PageOptions: &linodego.PageOptions{Page: 1},
 			PageSize:    25,
 			Filter:      fmt.Sprintf(filterTemplate, d.lastRefreshTimestamp.Format("2006-01-02T15:04:05")),
 		}
-		events, err := d.client.ListEvents(ctx, &opts)
+		events, err := d.client.ListEvents(ctx, &eventsOpts)
 		if err != nil {
 			var e *linodego.Error
 			if errors.As(err, &e) && e.Code == http.StatusUnauthorized {
@@ -224,16 +232,40 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
 	tg := &targetgroup.Group{
 		Source: "Linode",
 	}
+	// We need 3 of these because Linodego writes into the structure during pagination
+	listInstancesOpts := linodego.ListOptions{
+		PageSize: 500,
+	}
+	listIPAddressesOpts := linodego.ListOptions{
+		PageSize: 500,
+	}
+	listIPv6RangesOpts := linodego.ListOptions{
+		PageSize: 500,
+	}
+
+	// If region filter provided, use it to constrain results.
+	if d.region != "" {
+		listInstancesOpts.Filter = fmt.Sprintf(regionFilterTemplate, d.region)
+		listIPAddressesOpts.Filter = fmt.Sprintf(regionFilterTemplate, d.region)
+		listIPv6RangesOpts.Filter = fmt.Sprintf(regionFilterTemplate, d.region)
+	}
 
 	// Gather all linode instances.
-	instances, err := d.client.ListInstances(ctx, &linodego.ListOptions{PageSize: 500})
+	instances, err := d.client.ListInstances(ctx, &listInstancesOpts)
 	if err != nil {
 		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
 	// Gather detailed IP address info for all IPs on all linode instances.
-	detailedIPs, err := d.client.ListIPAddresses(ctx, &linodego.ListOptions{PageSize: 500})
+	detailedIPs, err := d.client.ListIPAddresses(ctx, &listIPAddressesOpts)
+	if err != nil {
+		d.metrics.failuresCount.Inc()
+		return nil, err
+	}
+
+	// Gather detailed IPv6 Range info for all linode instances.
+	ipv6RangeList, err := d.client.ListIPv6Ranges(ctx, &listIPv6RangesOpts)
 	if err != nil {
 		d.metrics.failuresCount.Inc()
 		return nil, err
@@ -248,7 +280,7 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
 			privateIPv4, publicIPv4, publicIPv6             string
 			privateIPv4RDNS, publicIPv4RDNS, publicIPv6RDNS string
 			backupsStatus                                   string
-			extraIPs                                        []string
+			extraIPs, ipv6Ranges                            []string
 		)
 
 		for _, ip := range instance.IPv4 {
@@ -276,17 +308,23 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
 		}
 
 		if instance.IPv6 != "" {
+			slaac := strings.Split(instance.IPv6, "/")[0]
 			for _, detailedIP := range detailedIPs {
-				if detailedIP.Address != strings.Split(instance.IPv6, "/")[0] {
+				if detailedIP.Address != slaac {
 					continue
 				}
-
 				publicIPv6 = detailedIP.Address
 
 				if detailedIP.RDNS != "" && detailedIP.RDNS != "null" {
 					publicIPv6RDNS = detailedIP.RDNS
 				}
 			}
+			for _, ipv6Range := range ipv6RangeList {
+				if ipv6Range.RouteTarget != slaac {
+					continue
+				}
+				ipv6Ranges = append(ipv6Ranges, fmt.Sprintf("%s/%d", ipv6Range.Range, ipv6Range.Prefix))
+			}
 		}
 
 		if instance.Backups.Enabled {
@@ -296,7 +334,7 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
 		}
 
 		labels := model.LabelSet{
-			linodeLabelID:                 model.LabelValue(fmt.Sprintf("%d", instance.ID)),
+			linodeLabelID:                 model.LabelValue(strconv.Itoa(instance.ID)),
 			linodeLabelName:               model.LabelValue(instance.Label),
 			linodeLabelImage:              model.LabelValue(instance.Image),
 			linodeLabelPrivateIPv4:        model.LabelValue(privateIPv4),
@@ -309,13 +347,13 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
 			linodeLabelType:               model.LabelValue(instance.Type),
 			linodeLabelStatus:             model.LabelValue(instance.Status),
 			linodeLabelGroup:              model.LabelValue(instance.Group),
-			linodeLabelGPUs:               model.LabelValue(fmt.Sprintf("%d", instance.Specs.GPUs)),
+			linodeLabelGPUs:               model.LabelValue(strconv.Itoa(instance.Specs.GPUs)),
 			linodeLabelHypervisor:         model.LabelValue(instance.Hypervisor),
 			linodeLabelBackups:            model.LabelValue(backupsStatus),
-			linodeLabelSpecsDiskBytes:     model.LabelValue(fmt.Sprintf("%d", int64(instance.Specs.Disk)<<20)),
+			linodeLabelSpecsDiskBytes:     model.LabelValue(strconv.FormatInt(int64(instance.Specs.Disk)<<20, 10)),
-			linodeLabelSpecsMemoryBytes:   model.LabelValue(fmt.Sprintf("%d", int64(instance.Specs.Memory)<<20)),
+			linodeLabelSpecsMemoryBytes:   model.LabelValue(strconv.FormatInt(int64(instance.Specs.Memory)<<20, 10)),
-			linodeLabelSpecsVCPUs:         model.LabelValue(fmt.Sprintf("%d", instance.Specs.VCPUs)),
+			linodeLabelSpecsVCPUs:         model.LabelValue(strconv.Itoa(instance.Specs.VCPUs)),
-			linodeLabelSpecsTransferBytes: model.LabelValue(fmt.Sprintf("%d", int64(instance.Specs.Transfer)<<20)),
+			linodeLabelSpecsTransferBytes: model.LabelValue(strconv.FormatInt(int64(instance.Specs.Transfer)<<20, 10)),
 		}
 
 		addr := net.JoinHostPort(publicIPv4, strconv.FormatUint(uint64(d.port), 10))
@@ -330,12 +368,20 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
 
 		if len(extraIPs) > 0 {
 			// This instance has more than one of at least one type of IP address (public, private,
-			// IPv4, IPv6, etc. We provide those extra IPs found here just like we do for instance
+			// IPv4,etc. We provide those extra IPs found here just like we do for instance
 			// tags, we surround a separated list with the tagSeparator config.
 			ips := d.tagSeparator + strings.Join(extraIPs, d.tagSeparator) + d.tagSeparator
 			labels[linodeLabelExtraIPs] = model.LabelValue(ips)
 		}
 
+		if len(ipv6Ranges) > 0 {
+			// This instance has more than one IPv6 Ranges routed to it we provide these
+			// Ranges found here just like we do for instance tags, we surround a separated
+			// list with the tagSeparator config.
+			ips := d.tagSeparator + strings.Join(ipv6Ranges, d.tagSeparator) + d.tagSeparator
+			labels[linodeLabelIPv6Ranges] = model.LabelValue(ips)
+		}
+
 		tg.Targets = append(tg.Targets, labels)
 	}
 	return []*targetgroup.Group{tg}, nil

discovery/linode/linode_test.go

@@ -28,159 +28,236 @@ import (
 	"github.com/prometheus/prometheus/discovery"
 )
 
-type LinodeSDTestSuite struct {
-	Mock *SDMock
-}
-
-func (s *LinodeSDTestSuite) TearDownSuite() {
-	s.Mock.ShutdownServer()
-}
-
-func (s *LinodeSDTestSuite) SetupTest(t *testing.T) {
-	s.Mock = NewSDMock(t)
-	s.Mock.Setup()
-
-	s.Mock.HandleLinodeInstancesList()
-	s.Mock.HandleLinodeNeworkingIPs()
-	s.Mock.HandleLinodeAccountEvents()
-}
-
 func TestLinodeSDRefresh(t *testing.T) {
-	sdmock := &LinodeSDTestSuite{}
+	sdmock := NewSDMock(t)
-	sdmock.SetupTest(t)
+	sdmock.Setup()
-	t.Cleanup(sdmock.TearDownSuite)
 
-	cfg := DefaultSDConfig
+	tests := map[string]struct {
-	cfg.HTTPClientConfig.Authorization = &config.Authorization{
+		region      string
-		Credentials: tokenID,
+		targetCount int
-		Type:        "Bearer",
+		want        []model.LabelSet
+	}{
+		"no_region": {region: "", targetCount: 4, want: []model.LabelSet{
+			{
+				"__address__":                        model.LabelValue("45.33.82.151:80"),
+				"__meta_linode_instance_id":          model.LabelValue("26838044"),
+				"__meta_linode_instance_label":       model.LabelValue("prometheus-linode-sd-exporter-1"),
+				"__meta_linode_image":                model.LabelValue("linode/arch"),
+				"__meta_linode_private_ipv4":         model.LabelValue("192.168.170.51"),
+				"__meta_linode_public_ipv4":          model.LabelValue("45.33.82.151"),
+				"__meta_linode_public_ipv6":          model.LabelValue("2600:3c03::f03c:92ff:fe1a:1382"),
+				"__meta_linode_private_ipv4_rdns":    model.LabelValue(""),
+				"__meta_linode_public_ipv4_rdns":     model.LabelValue("li1028-151.members.linode.com"),
+				"__meta_linode_public_ipv6_rdns":     model.LabelValue(""),
+				"__meta_linode_region":               model.LabelValue("us-east"),
+				"__meta_linode_type":                 model.LabelValue("g6-standard-2"),
+				"__meta_linode_status":               model.LabelValue("running"),
+				"__meta_linode_tags":                 model.LabelValue(",monitoring,"),
+				"__meta_linode_group":                model.LabelValue(""),
+				"__meta_linode_gpus":                 model.LabelValue("0"),
+				"__meta_linode_hypervisor":           model.LabelValue("kvm"),
+				"__meta_linode_backups":              model.LabelValue("disabled"),
+				"__meta_linode_specs_disk_bytes":     model.LabelValue("85899345920"),
+				"__meta_linode_specs_memory_bytes":   model.LabelValue("4294967296"),
+				"__meta_linode_specs_vcpus":          model.LabelValue("2"),
+				"__meta_linode_specs_transfer_bytes": model.LabelValue("4194304000"),
+				"__meta_linode_extra_ips":            model.LabelValue(",96.126.108.16,192.168.201.25,"),
+			},
+			{
+				"__address__":                        model.LabelValue("139.162.196.43:80"),
+				"__meta_linode_instance_id":          model.LabelValue("26848419"),
+				"__meta_linode_instance_label":       model.LabelValue("prometheus-linode-sd-exporter-2"),
+				"__meta_linode_image":                model.LabelValue("linode/debian10"),
+				"__meta_linode_private_ipv4":         model.LabelValue(""),
+				"__meta_linode_public_ipv4":          model.LabelValue("139.162.196.43"),
+				"__meta_linode_public_ipv6":          model.LabelValue("2a01:7e00::f03c:92ff:fe1a:9976"),
+				"__meta_linode_private_ipv4_rdns":    model.LabelValue(""),
+				"__meta_linode_public_ipv4_rdns":     model.LabelValue("li1359-43.members.linode.com"),
+				"__meta_linode_public_ipv6_rdns":     model.LabelValue(""),
+				"__meta_linode_region":               model.LabelValue("eu-west"),
+				"__meta_linode_type":                 model.LabelValue("g6-standard-2"),
+				"__meta_linode_status":               model.LabelValue("running"),
+				"__meta_linode_tags":                 model.LabelValue(",monitoring,"),
+				"__meta_linode_group":                model.LabelValue(""),
+				"__meta_linode_gpus":                 model.LabelValue("0"),
+				"__meta_linode_hypervisor":           model.LabelValue("kvm"),
+				"__meta_linode_backups":              model.LabelValue("disabled"),
+				"__meta_linode_specs_disk_bytes":     model.LabelValue("85899345920"),
+				"__meta_linode_specs_memory_bytes":   model.LabelValue("4294967296"),
+				"__meta_linode_specs_vcpus":          model.LabelValue("2"),
+				"__meta_linode_specs_transfer_bytes": model.LabelValue("4194304000"),
+			},
+			{
+				"__address__":                        model.LabelValue("192.53.120.25:80"),
+				"__meta_linode_instance_id":          model.LabelValue("26837938"),
+				"__meta_linode_instance_label":       model.LabelValue("prometheus-linode-sd-exporter-3"),
+				"__meta_linode_image":                model.LabelValue("linode/ubuntu20.04"),
+				"__meta_linode_private_ipv4":         model.LabelValue(""),
+				"__meta_linode_public_ipv4":          model.LabelValue("192.53.120.25"),
+				"__meta_linode_public_ipv6":          model.LabelValue("2600:3c04::f03c:92ff:fe1a:fb68"),
+				"__meta_linode_private_ipv4_rdns":    model.LabelValue(""),
+				"__meta_linode_public_ipv4_rdns":     model.LabelValue("li2216-25.members.linode.com"),
+				"__meta_linode_public_ipv6_rdns":     model.LabelValue(""),
+				"__meta_linode_region":               model.LabelValue("ca-central"),
+				"__meta_linode_type":                 model.LabelValue("g6-standard-1"),
+				"__meta_linode_status":               model.LabelValue("running"),
+				"__meta_linode_tags":                 model.LabelValue(",monitoring,"),
+				"__meta_linode_group":                model.LabelValue(""),
+				"__meta_linode_gpus":                 model.LabelValue("0"),
+				"__meta_linode_hypervisor":           model.LabelValue("kvm"),
+				"__meta_linode_backups":              model.LabelValue("disabled"),
+				"__meta_linode_specs_disk_bytes":     model.LabelValue("53687091200"),
+				"__meta_linode_specs_memory_bytes":   model.LabelValue("2147483648"),
+				"__meta_linode_specs_vcpus":          model.LabelValue("1"),
+				"__meta_linode_specs_transfer_bytes": model.LabelValue("2097152000"),
+				"__meta_linode_ipv6_ranges":          model.LabelValue(",2600:3c04:e001:456::/64,"),
+			},
+			{
+				"__address__":                        model.LabelValue("66.228.47.103:80"),
+				"__meta_linode_instance_id":          model.LabelValue("26837992"),
+				"__meta_linode_instance_label":       model.LabelValue("prometheus-linode-sd-exporter-4"),
+				"__meta_linode_image":                model.LabelValue("linode/ubuntu20.04"),
+				"__meta_linode_private_ipv4":         model.LabelValue("192.168.148.94"),
+				"__meta_linode_public_ipv4":          model.LabelValue("66.228.47.103"),
+				"__meta_linode_public_ipv6":          model.LabelValue("2600:3c03::f03c:92ff:fe1a:fb4c"),
+				"__meta_linode_private_ipv4_rdns":    model.LabelValue(""),
+				"__meta_linode_public_ipv4_rdns":     model.LabelValue("li328-103.members.linode.com"),
+				"__meta_linode_public_ipv6_rdns":     model.LabelValue(""),
+				"__meta_linode_region":               model.LabelValue("us-east"),
+				"__meta_linode_type":                 model.LabelValue("g6-nanode-1"),
+				"__meta_linode_status":               model.LabelValue("running"),
+				"__meta_linode_tags":                 model.LabelValue(",monitoring,"),
+				"__meta_linode_group":                model.LabelValue(""),
+				"__meta_linode_gpus":                 model.LabelValue("0"),
+				"__meta_linode_hypervisor":           model.LabelValue("kvm"),
+				"__meta_linode_backups":              model.LabelValue("disabled"),
+				"__meta_linode_specs_disk_bytes":     model.LabelValue("26843545600"),
+				"__meta_linode_specs_memory_bytes":   model.LabelValue("1073741824"),
+				"__meta_linode_specs_vcpus":          model.LabelValue("1"),
+				"__meta_linode_specs_transfer_bytes": model.LabelValue("1048576000"),
+				"__meta_linode_extra_ips":            model.LabelValue(",172.104.18.104,"),
+				"__meta_linode_ipv6_ranges":          model.LabelValue(",2600:3c03:e000:123::/64,"),
+			},
+		}},
+		"us-east": {region: "us-east", targetCount: 2, want: []model.LabelSet{
+			{
+				"__address__":                        model.LabelValue("45.33.82.151:80"),
+				"__meta_linode_instance_id":          model.LabelValue("26838044"),
+				"__meta_linode_instance_label":       model.LabelValue("prometheus-linode-sd-exporter-1"),
+				"__meta_linode_image":                model.LabelValue("linode/arch"),
+				"__meta_linode_private_ipv4":         model.LabelValue("192.168.170.51"),
+				"__meta_linode_public_ipv4":          model.LabelValue("45.33.82.151"),
+				"__meta_linode_public_ipv6":          model.LabelValue("2600:3c03::f03c:92ff:fe1a:1382"),
+				"__meta_linode_private_ipv4_rdns":    model.LabelValue(""),
+				"__meta_linode_public_ipv4_rdns":     model.LabelValue("li1028-151.members.linode.com"),
+				"__meta_linode_public_ipv6_rdns":     model.LabelValue(""),
+				"__meta_linode_region":               model.LabelValue("us-east"),
+				"__meta_linode_type":                 model.LabelValue("g6-standard-2"),
+				"__meta_linode_status":               model.LabelValue("running"),
+				"__meta_linode_tags":                 model.LabelValue(",monitoring,"),
+				"__meta_linode_group":                model.LabelValue(""),
+				"__meta_linode_gpus":                 model.LabelValue("0"),
+				"__meta_linode_hypervisor":           model.LabelValue("kvm"),
+				"__meta_linode_backups":              model.LabelValue("disabled"),
+				"__meta_linode_specs_disk_bytes":     model.LabelValue("85899345920"),
+				"__meta_linode_specs_memory_bytes":   model.LabelValue("4294967296"),
+				"__meta_linode_specs_vcpus":          model.LabelValue("2"),
+				"__meta_linode_specs_transfer_bytes": model.LabelValue("4194304000"),
+				"__meta_linode_extra_ips":            model.LabelValue(",96.126.108.16,192.168.201.25,"),
+			},
+			{
+				"__address__":                        model.LabelValue("66.228.47.103:80"),
+				"__meta_linode_instance_id":          model.LabelValue("26837992"),
+				"__meta_linode_instance_label":       model.LabelValue("prometheus-linode-sd-exporter-4"),
+				"__meta_linode_image":                model.LabelValue("linode/ubuntu20.04"),
+				"__meta_linode_private_ipv4":         model.LabelValue("192.168.148.94"),
+				"__meta_linode_public_ipv4":          model.LabelValue("66.228.47.103"),
+				"__meta_linode_public_ipv6":          model.LabelValue("2600:3c03::f03c:92ff:fe1a:fb4c"),
+				"__meta_linode_private_ipv4_rdns":    model.LabelValue(""),
+				"__meta_linode_public_ipv4_rdns":     model.LabelValue("li328-103.members.linode.com"),
+				"__meta_linode_public_ipv6_rdns":     model.LabelValue(""),
+				"__meta_linode_region":               model.LabelValue("us-east"),
+				"__meta_linode_type":                 model.LabelValue("g6-nanode-1"),
+				"__meta_linode_status":               model.LabelValue("running"),
+				"__meta_linode_tags":                 model.LabelValue(",monitoring,"),
+				"__meta_linode_group":                model.LabelValue(""),
+				"__meta_linode_gpus":                 model.LabelValue("0"),
+				"__meta_linode_hypervisor":           model.LabelValue("kvm"),
+				"__meta_linode_backups":              model.LabelValue("disabled"),
+				"__meta_linode_specs_disk_bytes":     model.LabelValue("26843545600"),
+				"__meta_linode_specs_memory_bytes":   model.LabelValue("1073741824"),
+				"__meta_linode_specs_vcpus":          model.LabelValue("1"),
+				"__meta_linode_specs_transfer_bytes": model.LabelValue("1048576000"),
+				"__meta_linode_extra_ips":            model.LabelValue(",172.104.18.104,"),
+				"__meta_linode_ipv6_ranges":          model.LabelValue(",2600:3c03:e000:123::/64,"),
+			},
+		}},
+		"us-central": {region: "ca-central", targetCount: 1, want: []model.LabelSet{
+			{
+				"__address__":                        model.LabelValue("192.53.120.25:80"),
+				"__meta_linode_instance_id":          model.LabelValue("26837938"),
+				"__meta_linode_instance_label":       model.LabelValue("prometheus-linode-sd-exporter-3"),
+				"__meta_linode_image":                model.LabelValue("linode/ubuntu20.04"),
+				"__meta_linode_private_ipv4":         model.LabelValue(""),
+				"__meta_linode_public_ipv4":          model.LabelValue("192.53.120.25"),
+				"__meta_linode_public_ipv6":          model.LabelValue("2600:3c04::f03c:92ff:fe1a:fb68"),
+				"__meta_linode_private_ipv4_rdns":    model.LabelValue(""),
+				"__meta_linode_public_ipv4_rdns":     model.LabelValue("li2216-25.members.linode.com"),
+				"__meta_linode_public_ipv6_rdns":     model.LabelValue(""),
+				"__meta_linode_region":               model.LabelValue("ca-central"),
+				"__meta_linode_type":                 model.LabelValue("g6-standard-1"),
+				"__meta_linode_status":               model.LabelValue("running"),
+				"__meta_linode_tags":                 model.LabelValue(",monitoring,"),
+				"__meta_linode_group":                model.LabelValue(""),
+				"__meta_linode_gpus":                 model.LabelValue("0"),
+				"__meta_linode_hypervisor":           model.LabelValue("kvm"),
+				"__meta_linode_backups":              model.LabelValue("disabled"),
+				"__meta_linode_specs_disk_bytes":     model.LabelValue("53687091200"),
+				"__meta_linode_specs_memory_bytes":   model.LabelValue("2147483648"),
+				"__meta_linode_specs_vcpus":          model.LabelValue("1"),
+				"__meta_linode_specs_transfer_bytes": model.LabelValue("2097152000"),
+				"__meta_linode_ipv6_ranges":          model.LabelValue(",2600:3c04:e001:456::/64,"),
+			},
+		}},
 	}
 
-	reg := prometheus.NewRegistry()
+	for _, tc := range tests {
-	refreshMetrics := discovery.NewRefreshMetrics(reg)
+		cfg := DefaultSDConfig
-	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+		if tc.region != "" {
-	require.NoError(t, metrics.Register())
+			cfg.Region = tc.region
-	defer metrics.Unregister()
+		}
-	defer refreshMetrics.Unregister()
+		cfg.HTTPClientConfig.Authorization = &config.Authorization{
+			Credentials: tokenID,
+			Type:        "Bearer",
+		}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
+		reg := prometheus.NewRegistry()
-	require.NoError(t, err)
+		refreshMetrics := discovery.NewRefreshMetrics(reg)
-	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
+		metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
-	require.NoError(t, err)
+		require.NoError(t, metrics.Register())
-	d.client.SetBaseURL(endpoint.String())
+		defer metrics.Unregister()
+		defer refreshMetrics.Unregister()
 
-	tgs, err := d.refresh(context.Background())
+		d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
-	require.NoError(t, err)
+		require.NoError(t, err)
+		endpoint, err := url.Parse(sdmock.Endpoint())
+		require.NoError(t, err)
+		d.client.SetBaseURL(endpoint.String())
 
-	require.Len(t, tgs, 1)
+		tgs, err := d.refresh(context.Background())
+		require.NoError(t, err)
 
-	tg := tgs[0]
+		require.Len(t, tgs, 1)
-	require.NotNil(t, tg)
-	require.NotNil(t, tg.Targets)
-	require.Len(t, tg.Targets, 4)
 
-	for i, lbls := range []model.LabelSet{
+		tg := tgs[0]
-		{
+		require.NotNil(t, tg)
-			"__address__":                        model.LabelValue("45.33.82.151:80"),
+		require.NotNil(t, tg.Targets)
-			"__meta_linode_instance_id":          model.LabelValue("26838044"),
+		require.Len(t, tg.Targets, tc.targetCount)
-			"__meta_linode_instance_label":       model.LabelValue("prometheus-linode-sd-exporter-1"),
+
-			"__meta_linode_image":                model.LabelValue("linode/arch"),
+		for i, lbls := range tc.want {
-			"__meta_linode_private_ipv4":         model.LabelValue("192.168.170.51"),
+			t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) {
-			"__meta_linode_public_ipv4":          model.LabelValue("45.33.82.151"),
+				require.Equal(t, lbls, tg.Targets[i])
-			"__meta_linode_public_ipv6":          model.LabelValue("2600:3c03::f03c:92ff:fe1a:1382"),
+			})
-			"__meta_linode_private_ipv4_rdns":    model.LabelValue(""),
+		}
-			"__meta_linode_public_ipv4_rdns":     model.LabelValue("li1028-151.members.linode.com"),
-			"__meta_linode_public_ipv6_rdns":     model.LabelValue(""),
-			"__meta_linode_region":               model.LabelValue("us-east"),
-			"__meta_linode_type":                 model.LabelValue("g6-standard-2"),
-			"__meta_linode_status":               model.LabelValue("running"),
-			"__meta_linode_tags":                 model.LabelValue(",monitoring,"),
-			"__meta_linode_group":                model.LabelValue(""),
-			"__meta_linode_gpus":                 model.LabelValue("0"),
-			"__meta_linode_hypervisor":           model.LabelValue("kvm"),
-			"__meta_linode_backups":              model.LabelValue("disabled"),
-			"__meta_linode_specs_disk_bytes":     model.LabelValue("85899345920"),
-			"__meta_linode_specs_memory_bytes":   model.LabelValue("4294967296"),
-			"__meta_linode_specs_vcpus":          model.LabelValue("2"),
-			"__meta_linode_specs_transfer_bytes": model.LabelValue("4194304000"),
-			"__meta_linode_extra_ips":            model.LabelValue(",96.126.108.16,192.168.201.25,"),
-		},
-		{
-			"__address__":                        model.LabelValue("139.162.196.43:80"),
-			"__meta_linode_instance_id":          model.LabelValue("26848419"),
-			"__meta_linode_instance_label":       model.LabelValue("prometheus-linode-sd-exporter-2"),
-			"__meta_linode_image":                model.LabelValue("linode/debian10"),
-			"__meta_linode_private_ipv4":         model.LabelValue(""),
-			"__meta_linode_public_ipv4":          model.LabelValue("139.162.196.43"),
-			"__meta_linode_public_ipv6":          model.LabelValue("2a01:7e00::f03c:92ff:fe1a:9976"),
-			"__meta_linode_private_ipv4_rdns":    model.LabelValue(""),
-			"__meta_linode_public_ipv4_rdns":     model.LabelValue("li1359-43.members.linode.com"),
-			"__meta_linode_public_ipv6_rdns":     model.LabelValue(""),
-			"__meta_linode_region":               model.LabelValue("eu-west"),
-			"__meta_linode_type":                 model.LabelValue("g6-standard-2"),
-			"__meta_linode_status":               model.LabelValue("running"),
-			"__meta_linode_tags":                 model.LabelValue(",monitoring,"),
-			"__meta_linode_group":                model.LabelValue(""),
-			"__meta_linode_gpus":                 model.LabelValue("0"),
-			"__meta_linode_hypervisor":           model.LabelValue("kvm"),
-			"__meta_linode_backups":              model.LabelValue("disabled"),
-			"__meta_linode_specs_disk_bytes":     model.LabelValue("85899345920"),
-			"__meta_linode_specs_memory_bytes":   model.LabelValue("4294967296"),
-			"__meta_linode_specs_vcpus":          model.LabelValue("2"),
-			"__meta_linode_specs_transfer_bytes": model.LabelValue("4194304000"),
-		},
-		{
-			"__address__":                        model.LabelValue("192.53.120.25:80"),
-			"__meta_linode_instance_id":          model.LabelValue("26837938"),
-			"__meta_linode_instance_label":       model.LabelValue("prometheus-linode-sd-exporter-3"),
-			"__meta_linode_image":                model.LabelValue("linode/ubuntu20.04"),
-			"__meta_linode_private_ipv4":         model.LabelValue(""),
-			"__meta_linode_public_ipv4":          model.LabelValue("192.53.120.25"),
-			"__meta_linode_public_ipv6":          model.LabelValue("2600:3c04::f03c:92ff:fe1a:fb68"),
-			"__meta_linode_private_ipv4_rdns":    model.LabelValue(""),
-			"__meta_linode_public_ipv4_rdns":     model.LabelValue("li2216-25.members.linode.com"),
-			"__meta_linode_public_ipv6_rdns":     model.LabelValue(""),
-			"__meta_linode_region":               model.LabelValue("ca-central"),
-			"__meta_linode_type":                 model.LabelValue("g6-standard-1"),
-			"__meta_linode_status":               model.LabelValue("running"),
-			"__meta_linode_tags":                 model.LabelValue(",monitoring,"),
-			"__meta_linode_group":                model.LabelValue(""),
-			"__meta_linode_gpus":                 model.LabelValue("0"),
-			"__meta_linode_hypervisor":           model.LabelValue("kvm"),
-			"__meta_linode_backups":              model.LabelValue("disabled"),
-			"__meta_linode_specs_disk_bytes":     model.LabelValue("53687091200"),
-			"__meta_linode_specs_memory_bytes":   model.LabelValue("2147483648"),
-			"__meta_linode_specs_vcpus":          model.LabelValue("1"),
-			"__meta_linode_specs_transfer_bytes": model.LabelValue("2097152000"),
-		},
-		{
-			"__address__":                        model.LabelValue("66.228.47.103:80"),
-			"__meta_linode_instance_id":          model.LabelValue("26837992"),
-			"__meta_linode_instance_label":       model.LabelValue("prometheus-linode-sd-exporter-4"),
-			"__meta_linode_image":                model.LabelValue("linode/ubuntu20.04"),
-			"__meta_linode_private_ipv4":         model.LabelValue("192.168.148.94"),
-			"__meta_linode_public_ipv4":          model.LabelValue("66.228.47.103"),
-			"__meta_linode_public_ipv6":          model.LabelValue("2600:3c03::f03c:92ff:fe1a:fb4c"),
-			"__meta_linode_private_ipv4_rdns":    model.LabelValue(""),
-			"__meta_linode_public_ipv4_rdns":     model.LabelValue("li328-103.members.linode.com"),
-			"__meta_linode_public_ipv6_rdns":     model.LabelValue(""),
-			"__meta_linode_region":               model.LabelValue("us-east"),
-			"__meta_linode_type":                 model.LabelValue("g6-nanode-1"),
-			"__meta_linode_status":               model.LabelValue("running"),
-			"__meta_linode_tags":                 model.LabelValue(",monitoring,"),
-			"__meta_linode_group":                model.LabelValue(""),
-			"__meta_linode_gpus":                 model.LabelValue("0"),
-			"__meta_linode_hypervisor":           model.LabelValue("kvm"),
-			"__meta_linode_backups":              model.LabelValue("disabled"),
-			"__meta_linode_specs_disk_bytes":     model.LabelValue("26843545600"),
-			"__meta_linode_specs_memory_bytes":   model.LabelValue("1073741824"),
-			"__meta_linode_specs_vcpus":          model.LabelValue("1"),
-			"__meta_linode_specs_transfer_bytes": model.LabelValue("1048576000"),
-			"__meta_linode_extra_ips":            model.LabelValue(",172.104.18.104,"),
-		},
-	} {
-		t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) {
-			require.Equal(t, lbls, tg.Targets[i])
-		})
 	}
 }

discovery/linode/mock_test.go

@@ -14,12 +14,17 @@
 package linode
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"testing"
 )
 
+const tokenID = "7b2c56dd51edd90952c1b94c472b94b176f20c5c777e376849edd8ad1c6c03bb"
+
 // SDMock is the interface for the Linode mock.
 type SDMock struct {
 	t      *testing.T
@@ -43,412 +48,34 @@ func (m *SDMock) Endpoint() string {
 func (m *SDMock) Setup() {
 	m.Mux = http.NewServeMux()
 	m.Server = httptest.NewServer(m.Mux)
+	m.t.Cleanup(m.Server.Close)
+	m.SetupHandlers()
 }
 
-// ShutdownServer creates the mock server.
+// SetupHandlers for endpoints of interest.
-func (m *SDMock) ShutdownServer() {
+func (m *SDMock) SetupHandlers() {
-	m.Server.Close()
+	for _, handler := range []string{"/v4/account/events", "/v4/linode/instances", "/v4/networking/ips", "/v4/networking/ipv6/ranges"} {
-}
+		m.Mux.HandleFunc(handler, func(w http.ResponseWriter, r *http.Request) {
-
+			if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", tokenID) {
-const tokenID = "7b2c56dd51edd90952c1b94c472b94b176f20c5c777e376849edd8ad1c6c03bb"
+				w.WriteHeader(http.StatusUnauthorized)
-
+				return
-// HandleLinodeInstancesList mocks linode instances list.
+			}
-func (m *SDMock) HandleLinodeInstancesList() {
+			xFilter := struct {
-	m.Mux.HandleFunc("/v4/linode/instances", func(w http.ResponseWriter, r *http.Request) {
+				Region string `json:"region"`
-		if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", tokenID) {
+			}{}
-			w.WriteHeader(http.StatusUnauthorized)
+			json.Unmarshal([]byte(r.Header.Get("X-Filter")), &xFilter)
-			return
+
-		}
+			directory := "testdata/no_region_filter"
-
+			if xFilter.Region != "" { // Validate region filter matches test criteria.
-		w.Header().Set("content-type", "application/json; charset=utf-8")
+				directory = "testdata/" + xFilter.Region
-		w.WriteHeader(http.StatusOK)
+			}
-
+			if response, err := os.ReadFile(filepath.Join(directory, r.URL.Path+".json")); err == nil {
-		fmt.Fprint(w, `
+				w.Header().Add("content-type", "application/json; charset=utf-8")
-{
+				w.WriteHeader(http.StatusOK)
-  "data": [
+				w.Write(response)
-    {
+				return
-      "id": 26838044,
+			}
-      "label": "prometheus-linode-sd-exporter-1",
+			w.WriteHeader(http.StatusInternalServerError)
-      "group": "",
+		})
-      "status": "running",
+	}
-      "created": "2021-05-12T04:23:44",
-      "updated": "2021-05-12T04:23:44",
-      "type": "g6-standard-2",
-      "ipv4": [
-        "45.33.82.151",
-        "96.126.108.16",
-        "192.168.170.51",
-        "192.168.201.25"
-      ],
-      "ipv6": "2600:3c03::f03c:92ff:fe1a:1382/128",
-      "image": "linode/arch",
-      "region": "us-east",
-      "specs": {
-        "disk": 81920,
-        "memory": 4096,
-        "vcpus": 2,
-        "gpus": 0,
-        "transfer": 4000
-      },
-      "alerts": {
-        "cpu": 180,
-        "network_in": 10,
-        "network_out": 10,
-        "transfer_quota": 80,
-        "io": 10000
-      },
-      "backups": {
-        "enabled": false,
-        "schedule": {
-          "day": null,
-          "window": null
-        },
-        "last_successful": null
-      },
-      "hypervisor": "kvm",
-      "watchdog_enabled": true,
-      "tags": [
-        "monitoring"
-      ]
-    },
-    {
-      "id": 26848419,
-      "label": "prometheus-linode-sd-exporter-2",
-      "group": "",
-      "status": "running",
-      "created": "2021-05-12T12:41:49",
-      "updated": "2021-05-12T12:41:49",
-      "type": "g6-standard-2",
-      "ipv4": [
-        "139.162.196.43"
-      ],
-      "ipv6": "2a01:7e00::f03c:92ff:fe1a:9976/128",
-      "image": "linode/debian10",
-      "region": "eu-west",
-      "specs": {
-        "disk": 81920,
-        "memory": 4096,
-        "vcpus": 2,
-        "gpus": 0,
-        "transfer": 4000
-      },
-      "alerts": {
-        "cpu": 180,
-        "network_in": 10,
-        "network_out": 10,
-        "transfer_quota": 80,
-        "io": 10000
-      },
-      "backups": {
-        "enabled": false,
-        "schedule": {
-          "day": null,
-          "window": null
-        },
-        "last_successful": null
-      },
-      "hypervisor": "kvm",
-      "watchdog_enabled": true,
-      "tags": [
-        "monitoring"
-      ]
-    },
-    {
-      "id": 26837938,
-      "label": "prometheus-linode-sd-exporter-3",
-      "group": "",
-      "status": "running",
-      "created": "2021-05-12T04:20:11",
-      "updated": "2021-05-12T04:20:11",
-      "type": "g6-standard-1",
-      "ipv4": [
-        "192.53.120.25"
-      ],
-      "ipv6": "2600:3c04::f03c:92ff:fe1a:fb68/128",
-      "image": "linode/ubuntu20.04",
-      "region": "ca-central",
-      "specs": {
-        "disk": 51200,
-        "memory": 2048,
-        "vcpus": 1,
-        "gpus": 0,
-        "transfer": 2000
-      },
-      "alerts": {
-        "cpu": 90,
-        "network_in": 10,
-        "network_out": 10,
-        "transfer_quota": 80,
-        "io": 10000
-      },
-      "backups": {
-        "enabled": false,
-        "schedule": {
-          "day": null,
-          "window": null
-        },
-        "last_successful": null
-      },
-      "hypervisor": "kvm",
-      "watchdog_enabled": true,
-      "tags": [
-        "monitoring"
-      ]
-    },
-    {
-      "id": 26837992,
-      "label": "prometheus-linode-sd-exporter-4",
-      "group": "",
-      "status": "running",
-      "created": "2021-05-12T04:22:06",
-      "updated": "2021-05-12T04:22:06",
-      "type": "g6-nanode-1",
-      "ipv4": [
-        "66.228.47.103",
-        "172.104.18.104",
-        "192.168.148.94"
-      ],
-      "ipv6": "2600:3c03::f03c:92ff:fe1a:fb4c/128",
-      "image": "linode/ubuntu20.04",
-      "region": "us-east",
-      "specs": {
-        "disk": 25600,
-        "memory": 1024,
-        "vcpus": 1,
-        "gpus": 0,
-        "transfer": 1000
-      },
-      "alerts": {
-        "cpu": 90,
-        "network_in": 10,
-        "network_out": 10,
-        "transfer_quota": 80,
-        "io": 10000
-      },
-      "backups": {
-        "enabled": false,
-        "schedule": {
-          "day": null,
-          "window": null
-        },
-        "last_successful": null
-      },
-      "hypervisor": "kvm",
-      "watchdog_enabled": true,
-      "tags": [
-        "monitoring"
-      ]
-    }
-  ],
-  "page": 1,
-  "pages": 1,
-  "results": 4
-}`,
-		)
-	})
-}
-
-// HandleLinodeNeworkingIPs mocks linode networking ips endpoint.
-func (m *SDMock) HandleLinodeNeworkingIPs() {
-	m.Mux.HandleFunc("/v4/networking/ips", func(w http.ResponseWriter, r *http.Request) {
-		if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", tokenID) {
-			w.WriteHeader(http.StatusUnauthorized)
-			return
-		}
-
-		w.Header().Set("content-type", "application/json; charset=utf-8")
-		w.WriteHeader(http.StatusOK)
-
-		fmt.Fprint(w, `
-{
-  "page": 1,
-  "pages": 1,
-  "results": 13,
-  "data": [
-    {
-      "address": "192.53.120.25",
-      "gateway": "192.53.120.1",
-      "subnet_mask": "255.255.255.0",
-      "prefix": 24,
-      "type": "ipv4",
-      "public": true,
-      "rdns": "li2216-25.members.linode.com",
-      "linode_id": 26837938,
-      "region": "ca-central"
-    },
-    {
-      "address": "66.228.47.103",
-      "gateway": "66.228.47.1",
-      "subnet_mask": "255.255.255.0",
-      "prefix": 24,
-      "type": "ipv4",
-      "public": true,
-      "rdns": "li328-103.members.linode.com",
-      "linode_id": 26837992,
-      "region": "us-east"
-    },
-    {
-      "address": "172.104.18.104",
-      "gateway": "172.104.18.1",
-      "subnet_mask": "255.255.255.0",
-      "prefix": 24,
-      "type": "ipv4",
-      "public": true,
-      "rdns": "li1832-104.members.linode.com",
-      "linode_id": 26837992,
-      "region": "us-east"
-    },
-    {
-      "address": "192.168.148.94",
-      "gateway": null,
-      "subnet_mask": "255.255.128.0",
-      "prefix": 17,
-      "type": "ipv4",
-      "public": false,
-      "rdns": null,
-      "linode_id": 26837992,
-      "region": "us-east"
-    },
-    {
-      "address": "192.168.170.51",
-      "gateway": null,
-      "subnet_mask": "255.255.128.0",
-      "prefix": 17,
-      "type": "ipv4",
-      "public": false,
-      "rdns": null,
-      "linode_id": 26838044,
-      "region": "us-east"
-    },
-    {
-      "address": "96.126.108.16",
-      "gateway": "96.126.108.1",
-      "subnet_mask": "255.255.255.0",
-      "prefix": 24,
-      "type": "ipv4",
-      "public": true,
-      "rdns": "li365-16.members.linode.com",
-      "linode_id": 26838044,
-      "region": "us-east"
-    },
-    {
-      "address": "45.33.82.151",
-      "gateway": "45.33.82.1",
-      "subnet_mask": "255.255.255.0",
-      "prefix": 24,
-      "type": "ipv4",
-      "public": true,
-      "rdns": "li1028-151.members.linode.com",
-      "linode_id": 26838044,
-      "region": "us-east"
-    },
-    {
-      "address": "192.168.201.25",
-      "gateway": null,
-      "subnet_mask": "255.255.128.0",
-      "prefix": 17,
-      "type": "ipv4",
-      "public": false,
-      "rdns": null,
-      "linode_id": 26838044,
-      "region": "us-east"
-    },
-    {
-      "address": "139.162.196.43",
-      "gateway": "139.162.196.1",
-      "subnet_mask": "255.255.255.0",
-      "prefix": 24,
-      "type": "ipv4",
-      "public": true,
-      "rdns": "li1359-43.members.linode.com",
-      "linode_id": 26848419,
-      "region": "eu-west"
-    },
-    {
-      "address": "2600:3c04::f03c:92ff:fe1a:fb68",
-      "gateway": "fe80::1",
-      "subnet_mask": "ffff:ffff:ffff:ffff::",
-      "prefix": 64,
-      "type": "ipv6",
-      "rdns": null,
-      "linode_id": 26837938,
-      "region": "ca-central",
-      "public": true
-    },
-    {
-      "address": "2600:3c03::f03c:92ff:fe1a:fb4c",
-      "gateway": "fe80::1",
-      "subnet_mask": "ffff:ffff:ffff:ffff::",
-      "prefix": 64,
-      "type": "ipv6",
-      "rdns": null,
-      "linode_id": 26837992,
-      "region": "us-east",
-      "public": true
-    },
-    {
-      "address": "2600:3c03::f03c:92ff:fe1a:1382",
-      "gateway": "fe80::1",
-      "subnet_mask": "ffff:ffff:ffff:ffff::",
-      "prefix": 64,
-      "type": "ipv6",
-      "rdns": null,
-      "linode_id": 26838044,
-      "region": "us-east",
-      "public": true
-    },
-    {
-      "address": "2a01:7e00::f03c:92ff:fe1a:9976",
-      "gateway": "fe80::1",
-      "subnet_mask": "ffff:ffff:ffff:ffff::",
-      "prefix": 64,
-      "type": "ipv6",
-      "rdns": null,
-      "linode_id": 26848419,
-      "region": "eu-west",
-      "public": true
-    }
-  ]
-}`,
-		)
-	})
-}
-
-// HandleLinodeAccountEvents mocks linode the account/events endpoint.
-func (m *SDMock) HandleLinodeAccountEvents() {
-	m.Mux.HandleFunc("/v4/account/events", func(w http.ResponseWriter, r *http.Request) {
-		if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", tokenID) {
-			w.WriteHeader(http.StatusUnauthorized)
-			return
-		}
-
-		if r.Header.Get("X-Filter") == "" {
-			// This should never happen; if the client sends an events request without
-			// a filter, cause it to fail. The error below is not a real response from
-			// the API, but should aid in debugging failed tests.
-			w.WriteHeader(http.StatusBadRequest)
-			fmt.Fprint(w, `
-{
-	"errors": [
-		{
-			"reason": "Request missing expected X-Filter headers"
-		}
-	]
-}`,
-			)
-			return
-		}
-
-		w.Header().Set("content-type", "application/json; charset=utf-8")
-		w.WriteHeader(http.StatusOK)
-
-		fmt.Fprint(w, `
-{
-	"data": [],
-	"results": 0,
-	"pages": 1,
-	"page": 1
-}`,
-		)
-	})
 }

discovery/linode/testdata/ca-central/v4/account/events.json

@@ -0,0 +1,6 @@
+{
+      "data": [],
+      "results": 0,
+      "pages": 1,
+      "page": 1
+}

discovery/linode/testdata/ca-central/v4/linode/instances.json

@@ -0,0 +1,49 @@
+{
+  "data": [
+    {
+      "id": 26837938,
+      "label": "prometheus-linode-sd-exporter-3",
+      "group": "",
+      "status": "running",
+      "created": "2021-05-12T04:20:11",
+      "updated": "2021-05-12T04:20:11",
+      "type": "g6-standard-1",
+      "ipv4": [
+        "192.53.120.25"
+      ],
+      "ipv6": "2600:3c04::f03c:92ff:fe1a:fb68/128",
+      "image": "linode/ubuntu20.04",
+      "region": "ca-central",
+      "specs": {
+        "disk": 51200,
+        "memory": 2048,
+        "vcpus": 1,
+        "gpus": 0,
+        "transfer": 2000
+      },
+      "alerts": {
+        "cpu": 90,
+        "network_in": 10,
+        "network_out": 10,
+        "transfer_quota": 80,
+        "io": 10000
+      },
+      "backups": {
+        "enabled": false,
+        "schedule": {
+          "day": null,
+          "window": null
+        },
+        "last_successful": null
+      },
+      "hypervisor": "kvm",
+      "watchdog_enabled": true,
+      "tags": [
+        "monitoring"
+      ]
+    }
+  ],
+  "page": 1,
+  "pages": 1,
+  "results": 1
+}

discovery/linode/testdata/ca-central/v4/networking/ips.json

@@ -0,0 +1,29 @@
+{
+  "page": 1,
+  "pages": 1,
+  "results": 2,
+  "data": [
+    {
+      "address": "192.53.120.25",
+      "gateway": "192.53.120.1",
+      "subnet_mask": "255.255.255.0",
+      "prefix": 24,
+      "type": "ipv4",
+      "public": true,
+      "rdns": "li2216-25.members.linode.com",
+      "linode_id": 26837938,
+      "region": "ca-central"
+    },
+    {
+      "address": "2600:3c04::f03c:92ff:fe1a:fb68",
+      "gateway": "fe80::1",
+      "subnet_mask": "ffff:ffff:ffff:ffff::",
+      "prefix": 64,
+      "type": "ipv6",
+      "rdns": null,
+      "linode_id": 26837938,
+      "region": "ca-central",
+      "public": true
+    }
+  ]
+}

discovery/linode/testdata/ca-central/v4/networking/ipv6/ranges.json

@@ -0,0 +1,13 @@
+{
+  "data": [
+    {
+      "range": "2600:3c04:e001:456::",
+      "prefix": 64,
+      "region": "ca-central",
+      "route_target": "2600:3c04::f03c:92ff:fe1a:fb68"
+    }
+  ],
+  "page": 1,
+  "pages": 1,
+  "results": 1
+}

discovery/linode/testdata/no_region_filter/v4/account/events.json

@@ -0,0 +1,6 @@
+{
+      "data": [],
+      "results": 0,
+      "pages": 1,
+      "page": 1
+}

discovery/linode/testdata/no_region_filter/v4/linode/instances.json

@@ -0,0 +1,180 @@
+{
+  "data": [
+    {
+      "id": 26838044,
+      "label": "prometheus-linode-sd-exporter-1",
+      "group": "",
+      "status": "running",
+      "created": "2021-05-12T04:23:44",
+      "updated": "2021-05-12T04:23:44",
+      "type": "g6-standard-2",
+      "ipv4": [
+        "45.33.82.151",
+        "96.126.108.16",
+        "192.168.170.51",
+        "192.168.201.25"
+      ],
+      "ipv6": "2600:3c03::f03c:92ff:fe1a:1382/128",
+      "image": "linode/arch",
+      "region": "us-east",
+      "specs": {
+        "disk": 81920,
+        "memory": 4096,
+        "vcpus": 2,
+        "gpus": 0,
+        "transfer": 4000
+      },
+      "alerts": {
+        "cpu": 180,
+        "network_in": 10,
+        "network_out": 10,
+        "transfer_quota": 80,
+        "io": 10000
+      },
+      "backups": {
+        "enabled": false,
+        "schedule": {
+          "day": null,
+          "window": null
+        },
+        "last_successful": null
+      },
+      "hypervisor": "kvm",
+      "watchdog_enabled": true,
+      "tags": [
+        "monitoring"
+      ]
+    },
+    {
+      "id": 26848419,
+      "label": "prometheus-linode-sd-exporter-2",
+      "group": "",
+      "status": "running",
+      "created": "2021-05-12T12:41:49",
+      "updated": "2021-05-12T12:41:49",
+      "type": "g6-standard-2",
+      "ipv4": [
+        "139.162.196.43"
+      ],
+      "ipv6": "2a01:7e00::f03c:92ff:fe1a:9976/128",
+      "image": "linode/debian10",
+      "region": "eu-west",
+      "specs": {
+        "disk": 81920,
+        "memory": 4096,
+        "vcpus": 2,
+        "gpus": 0,
+        "transfer": 4000
+      },
+      "alerts": {
+        "cpu": 180,
+        "network_in": 10,
+        "network_out": 10,
+        "transfer_quota": 80,
+        "io": 10000
+      },
+      "backups": {
+        "enabled": false,
+        "schedule": {
+          "day": null,
+          "window": null
+        },
+        "last_successful": null
+      },
+      "hypervisor": "kvm",
+      "watchdog_enabled": true,
+      "tags": [
+        "monitoring"
+      ]
+    },
+    {
+      "id": 26837938,
+      "label": "prometheus-linode-sd-exporter-3",
+      "group": "",
+      "status": "running",
+      "created": "2021-05-12T04:20:11",
+      "updated": "2021-05-12T04:20:11",
+      "type": "g6-standard-1",
+      "ipv4": [
+        "192.53.120.25"
+      ],
+      "ipv6": "2600:3c04::f03c:92ff:fe1a:fb68/128",
+      "image": "linode/ubuntu20.04",
+      "region": "ca-central",
+      "specs": {
+        "disk": 51200,
+        "memory": 2048,
+        "vcpus": 1,
+        "gpus": 0,
+        "transfer": 2000
+      },
+      "alerts": {
+        "cpu": 90,
+        "network_in": 10,
+        "network_out": 10,
+        "transfer_quota": 80,
+        "io": 10000
+      },
+      "backups": {
+        "enabled": false,
+        "schedule": {
+          "day": null,
+          "window": null
+        },
+        "last_successful": null
+      },
+      "hypervisor": "kvm",
+      "watchdog_enabled": true,
+      "tags": [
+        "monitoring"
+      ]
+    },
+    {
+      "id": 26837992,
+      "label": "prometheus-linode-sd-exporter-4",
+      "group": "",
+      "status": "running",
+      "created": "2021-05-12T04:22:06",
+      "updated": "2021-05-12T04:22:06",
+      "type": "g6-nanode-1",
+      "ipv4": [
+        "66.228.47.103",
+        "172.104.18.104",
+        "192.168.148.94"
+      ],
+      "ipv6": "2600:3c03::f03c:92ff:fe1a:fb4c/128",
+      "image": "linode/ubuntu20.04",
+      "region": "us-east",
+      "specs": {
+        "disk": 25600,
+        "memory": 1024,
+        "vcpus": 1,
+        "gpus": 0,
+        "transfer": 1000
+      },
+      "alerts": {
+        "cpu": 90,
+        "network_in": 10,
+        "network_out": 10,
+        "transfer_quota": 80,
+        "io": 10000
+      },
+      "backups": {
+        "enabled": false,
+        "schedule": {
+          "day": null,
+          "window": null
+        },
+        "last_successful": null
+      },
+      "hypervisor": "kvm",
+      "watchdog_enabled": true,
+      "tags": [
+        "monitoring"
+      ]
+    }
+  ],
+  "page": 1,
+  "pages": 1,
+  "results": 4
+}

discovery/linode/testdata/no_region_filter/v4/networking/ips.json

@@ -0,0 +1,150 @@
+{
+  "page": 1,
+  "pages": 1,
+  "results": 13,
+  "data": [
+    {
+      "address": "192.53.120.25",
+      "gateway": "192.53.120.1",
+      "subnet_mask": "255.255.255.0",
+      "prefix": 24,
+      "type": "ipv4",
+      "public": true,
+      "rdns": "li2216-25.members.linode.com",
+      "linode_id": 26837938,
+      "region": "ca-central"
+    },
+    {
+      "address": "66.228.47.103",
+      "gateway": "66.228.47.1",
+      "subnet_mask": "255.255.255.0",
+      "prefix": 24,
+      "type": "ipv4",
+      "public": true,
+      "rdns": "li328-103.members.linode.com",
+      "linode_id": 26837992,
+      "region": "us-east"
+    },
+    {
+      "address": "172.104.18.104",
+      "gateway": "172.104.18.1",
+      "subnet_mask": "255.255.255.0",
+      "prefix": 24,
+      "type": "ipv4",
+      "public": true,
+      "rdns": "li1832-104.members.linode.com",
+      "linode_id": 26837992,
+      "region": "us-east"
+    },
+    {
+      "address": "192.168.148.94",
+      "gateway": null,
+      "subnet_mask": "255.255.128.0",
+      "prefix": 17,
+      "type": "ipv4",
+      "public": false,
+      "rdns": null,
+      "linode_id": 26837992,
+      "region": "us-east"
+    },
+    {
+      "address": "192.168.170.51",
+      "gateway": null,
+      "subnet_mask": "255.255.128.0",
+      "prefix": 17,
+      "type": "ipv4",
+      "public": false,
+      "rdns": null,
+      "linode_id": 26838044,
+      "region": "us-east"
+    },
+    {
+      "address": "96.126.108.16",
+      "gateway": "96.126.108.1",
+      "subnet_mask": "255.255.255.0",
+      "prefix": 24,
+      "type": "ipv4",
+      "public": true,
+      "rdns": "li365-16.members.linode.com",
+      "linode_id": 26838044,
+      "region": "us-east"
+    },
+    {
+      "address": "45.33.82.151",
+      "gateway": "45.33.82.1",
+      "subnet_mask": "255.255.255.0",
+      "prefix": 24,
+      "type": "ipv4",
+      "public": true,
+      "rdns": "li1028-151.members.linode.com",
+      "linode_id": 26838044,
+      "region": "us-east"
+    },
+    {
+      "address": "192.168.201.25",
+      "gateway": null,
+      "subnet_mask": "255.255.128.0",
+      "prefix": 17,
+      "type": "ipv4",
+      "public": false,
+      "rdns": null,
+      "linode_id": 26838044,
+      "region": "us-east"
+    },
+    {
+      "address": "139.162.196.43",
+      "gateway": "139.162.196.1",
+      "subnet_mask": "255.255.255.0",
+      "prefix": 24,
+      "type": "ipv4",
+      "public": true,
+      "rdns": "li1359-43.members.linode.com",
+      "linode_id": 26848419,
+      "region": "eu-west"
+    },
+    {
+      "address": "2600:3c04::f03c:92ff:fe1a:fb68",
+      "gateway": "fe80::1",
+      "subnet_mask": "ffff:ffff:ffff:ffff::",
+      "prefix": 64,
+      "type": "ipv6",
+      "rdns": null,
+      "linode_id": 26837938,
+      "region": "ca-central",
+      "public": true
+    },
+    {
+      "address": "2600:3c03::f03c:92ff:fe1a:fb4c",
+      "gateway": "fe80::1",
+      "subnet_mask": "ffff:ffff:ffff:ffff::",
+      "prefix": 64,
+      "type": "ipv6",
+      "rdns": null,
+      "linode_id": 26837992,
+      "region": "us-east",
+      "public": true
+    },
+    {
+      "address": "2600:3c03::f03c:92ff:fe1a:1382",
+      "gateway": "fe80::1",
+      "subnet_mask": "ffff:ffff:ffff:ffff::",
+      "prefix": 64,
+      "type": "ipv6",
+      "rdns": null,
+      "linode_id": 26838044,
+      "region": "us-east",
+      "public": true
+    },
+    {
+      "address": "2a01:7e00::f03c:92ff:fe1a:9976",
+      "gateway": "fe80::1",
+      "subnet_mask": "ffff:ffff:ffff:ffff::",
+      "prefix": 64,
+      "type": "ipv6",
+      "rdns": null,
+      "linode_id": 26848419,
+      "region": "eu-west",
+      "public": true
+    }
+  ]
+}

discovery/linode/testdata/no_region_filter/v4/networking/ipv6/ranges.json

@@ -0,0 +1,19 @@
+{
+  "data": [
+    {
+      "range": "2600:3c03:e000:123::",
+      "prefix": 64,
+      "region": "us-east",
+      "route_target": "2600:3c03::f03c:92ff:fe1a:fb4c"
+    },
+    {
+      "range": "2600:3c04:e001:456::",
+      "prefix": 64,
+      "region": "ca-central",
+      "route_target": "2600:3c04::f03c:92ff:fe1a:fb68"
+    }
+  ],
+  "page": 1,
+  "pages": 1,
+  "results": 2
+}

discovery/linode/testdata/us-east/v4/account/events.json

@@ -0,0 +1,6 @@
+{
+      "data": [],
+      "results": 0,
+      "pages": 1,
+      "page": 1
+}

discovery/linode/testdata/us-east/v4/linode/instances.json

@@ -0,0 +1,97 @@
+{
+  "data": [
+    {
+      "id": 26838044,
+      "label": "prometheus-linode-sd-exporter-1",
+      "group": "",
+      "status": "running",
+      "created": "2021-05-12T04:23:44",
+      "updated": "2021-05-12T04:23:44",
+      "type": "g6-standard-2",
+      "ipv4": [
+        "45.33.82.151",
+        "96.126.108.16",
+        "192.168.170.51",
+        "192.168.201.25"
+      ],
+      "ipv6": "2600:3c03::f03c:92ff:fe1a:1382/128",
+      "image": "linode/arch",
+      "region": "us-east",
+      "specs": {
+        "disk": 81920,
+        "memory": 4096,
+        "vcpus": 2,
+        "gpus": 0,
+        "transfer": 4000
+      },
+      "alerts": {
+        "cpu": 180,
+        "network_in": 10,
+        "network_out": 10,
+        "transfer_quota": 80,
+        "io": 10000
+      },
+      "backups": {
+        "enabled": false,
+        "schedule": {
+          "day": null,
+          "window": null
+        },
+        "last_successful": null
+      },
+      "hypervisor": "kvm",
+      "watchdog_enabled": true,
+      "tags": [
+        "monitoring"
+      ]
+    },
+    {
+      "id": 26837992,
+      "label": "prometheus-linode-sd-exporter-4",
+      "group": "",
+      "status": "running",
+      "created": "2021-05-12T04:22:06",
+      "updated": "2021-05-12T04:22:06",
+      "type": "g6-nanode-1",
+      "ipv4": [
+        "66.228.47.103",
+        "172.104.18.104",
+        "192.168.148.94"
+      ],
+      "ipv6": "2600:3c03::f03c:92ff:fe1a:fb4c/128",
+      "image": "linode/ubuntu20.04",
+      "region": "us-east",
+      "specs": {
+        "disk": 25600,
+        "memory": 1024,
+        "vcpus": 1,
+        "gpus": 0,
+        "transfer": 1000
+      },
+      "alerts": {
+        "cpu": 90,
+        "network_in": 10,
+        "network_out": 10,
+        "transfer_quota": 80,
+        "io": 10000
+      },
+      "backups": {
+        "enabled": false,
+        "schedule": {
+          "day": null,
+          "window": null
+        },
+        "last_successful": null
+      },
+      "hypervisor": "kvm",
+      "watchdog_enabled": true,
+      "tags": [
+        "monitoring"
+      ]
+    }
+  ],
+  "page": 1,
+  "pages": 1,
+  "results": 2
+}
+

discovery/linode/testdata/us-east/v4/networking/ips.json

@@ -0,0 +1,106 @@
+{
+  "page": 1,
+  "pages": 1,
+  "results": 9,
+  "data": [
+    {
+      "address": "66.228.47.103",
+      "gateway": "66.228.47.1",
+      "subnet_mask": "255.255.255.0",
+      "prefix": 24,
+      "type": "ipv4",
+      "public": true,
+      "rdns": "li328-103.members.linode.com",
+      "linode_id": 26837992,
+      "region": "us-east"
+    },
+    {
+      "address": "172.104.18.104",
+      "gateway": "172.104.18.1",
+      "subnet_mask": "255.255.255.0",
+      "prefix": 24,
+      "type": "ipv4",
+      "public": true,
+      "rdns": "li1832-104.members.linode.com",
+      "linode_id": 26837992,
+      "region": "us-east"
+    },
+    {
+      "address": "192.168.148.94",
+      "gateway": null,
+      "subnet_mask": "255.255.128.0",
+      "prefix": 17,
+      "type": "ipv4",
+      "public": false,
+      "rdns": null,
+      "linode_id": 26837992,
+      "region": "us-east"
+    },
+    {
+      "address": "192.168.170.51",
+      "gateway": null,
+      "subnet_mask": "255.255.128.0",
+      "prefix": 17,
+      "type": "ipv4",
+      "public": false,
+      "rdns": null,
+      "linode_id": 26838044,
+      "region": "us-east"
+    },
+    {
+      "address": "96.126.108.16",
+      "gateway": "96.126.108.1",
+      "subnet_mask": "255.255.255.0",
+      "prefix": 24,
+      "type": "ipv4",
+      "public": true,
+      "rdns": "li365-16.members.linode.com",
+      "linode_id": 26838044,
+      "region": "us-east"
+    },
+    {
+      "address": "45.33.82.151",
+      "gateway": "45.33.82.1",
+      "subnet_mask": "255.255.255.0",
+      "prefix": 24,
+      "type": "ipv4",
+      "public": true,
+      "rdns": "li1028-151.members.linode.com",
+      "linode_id": 26838044,
+      "region": "us-east"
+    },
+    {
+      "address": "192.168.201.25",
+      "gateway": null,
+      "subnet_mask": "255.255.128.0",
+      "prefix": 17,
+      "type": "ipv4",
+      "public": false,
+      "rdns": null,
+      "linode_id": 26838044,
+      "region": "us-east"
+    },
+    {
+      "address": "2600:3c03::f03c:92ff:fe1a:fb4c",
+      "gateway": "fe80::1",
+      "subnet_mask": "ffff:ffff:ffff:ffff::",
+      "prefix": 64,
+      "type": "ipv6",
+      "rdns": null,
+      "linode_id": 26837992,
+      "region": "us-east",
+      "public": true
+    },
+    {
+      "address": "2600:3c03::f03c:92ff:fe1a:1382",
+      "gateway": "fe80::1",
+      "subnet_mask": "ffff:ffff:ffff:ffff::",
+      "prefix": 64,
+      "type": "ipv6",
+      "rdns": null,
+      "linode_id": 26838044,
+      "region": "us-east",
+      "public": true
+    }
+  ]
+}

discovery/linode/testdata/us-east/v4/networking/ipv6/ranges.json

@@ -0,0 +1,13 @@
+{
+  "data": [
+    {
+      "range": "2600:3c03:e000:123::",
+      "prefix": 64,
+      "region": "us-east",
+      "route_target": "2600:3c03::f03c:92ff:fe1a:fb4c"
+    }
+  ],
+  "page": 1,
+  "pages": 1,
+  "results": 1
+}

discovery/manager.go

@@ -120,6 +120,16 @@ func Name(n string) func(*Manager) {
 	}
 }
 
+// Updatert sets the updatert of the manager.
+// Used to speed up tests.
+func Updatert(u time.Duration) func(*Manager) {
+	return func(m *Manager) {
+		m.mtx.Lock()
+		defer m.mtx.Unlock()
+		m.updatert = u
+	}
+}
+
 // HTTPClientOptions sets the list of HTTP client options to expose to
 // Discoverers. It is up to Discoverers to choose to use the options provided.
 func HTTPClientOptions(opts ...config.HTTPClientOption) func(*Manager) {
@@ -169,6 +179,13 @@ func (m *Manager) Providers() []*Provider {
 	return m.providers
 }
 
+// UnregisterMetrics unregisters manager metrics. It does not unregister
+// service discovery or refresh metrics, whose lifecycle is managed independent
+// of the discovery Manager.
+func (m *Manager) UnregisterMetrics() {
+	m.metrics.Unregister(m.registerer)
+}
+
 // Run starts the background processing.
 func (m *Manager) Run() error {
 	go m.sender()

discovery/manager_test.go

@@ -36,11 +36,11 @@ func TestMain(m *testing.M) {
 	testutil.TolerantVerifyLeak(m)
 }
 
-func NewTestMetrics(t *testing.T, reg prometheus.Registerer) (*RefreshMetricsManager, map[string]DiscovererMetrics) {
+func NewTestMetrics(t *testing.T, reg prometheus.Registerer) (RefreshMetricsManager, map[string]DiscovererMetrics) {
 	refreshMetrics := NewRefreshMetrics(reg)
 	sdMetrics, err := RegisterSDMetrics(reg, refreshMetrics)
 	require.NoError(t, err)
-	return &refreshMetrics, sdMetrics
+	return refreshMetrics, sdMetrics
 }
 
 // TestTargetUpdatesOrder checks that the target updates are received in the expected order.
@@ -694,7 +694,7 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			for x := 0; x < totalUpdatesCount; x++ {
 				select {
 				case <-ctx.Done():
-					t.Fatalf("%d: no update arrived within the timeout limit", x)
+					require.FailNow(t, "%d: no update arrived within the timeout limit", x)
 				case tgs := <-provUpdates:
 					discoveryManager.updateGroup(poolKey{setName: strconv.Itoa(i), provider: tc.title}, tgs)
 					for _, got := range discoveryManager.allGroups() {
@@ -720,7 +720,7 @@ func staticConfig(addrs ...string) StaticConfig {
 	var cfg StaticConfig
 	for i, addr := range addrs {
 		cfg = append(cfg, &targetgroup.Group{
-			Source: fmt.Sprint(i),
+			Source: strconv.Itoa(i),
 			Targets: []model.LabelSet{
 				{model.AddressLabel: model.LabelValue(addr)},
 			},
@@ -733,7 +733,6 @@ func verifySyncedPresence(t *testing.T, tGroups map[string][]*targetgroup.Group,
 	t.Helper()
 	if _, ok := tGroups[key]; !ok {
 		t.Fatalf("'%s' should be present in Group map keys: %v", key, tGroups)
-		return
 	}
 	match := false
 	var mergedTargets string
@@ -756,10 +755,8 @@ func verifySyncedPresence(t *testing.T, tGroups map[string][]*targetgroup.Group,
 
 func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Group, poolKey poolKey, label string, present bool) {
 	t.Helper()
-	if _, ok := tSets[poolKey]; !ok {
+	_, ok := tSets[poolKey]
-		t.Fatalf("'%s' should be present in Pool keys: %v", poolKey, tSets)
+	require.True(t, ok, "'%s' should be present in Pool keys: %v", poolKey, tSets)
-		return
-	}
 
 	match := false
 	var mergedTargets string
@@ -776,7 +773,7 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
 		if !present {
 			msg = "not"
 		}
-		t.Fatalf("%q should %s be present in Targets labels: %q", label, msg, mergedTargets)
+		require.FailNow(t, "%q should %s be present in Targets labels: %q", label, msg, mergedTargets)
 	}
 }
 
@@ -1088,22 +1085,14 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	syncedTargets = <-discoveryManager.SyncCh()
 	p = pk("static", "prometheus", 1)
 	targetGroups, ok := discoveryManager.targets[p]
-	if !ok {
+	require.True(t, ok, "'%v' should be present in target groups", p)
-		t.Fatalf("'%v' should be present in target groups", p)
-	}
 	group, ok := targetGroups[""]
-	if !ok {
+	require.True(t, ok, "missing '' key in target groups %v", targetGroups)
-		t.Fatalf("missing '' key in target groups %v", targetGroups)
-	}
 
-	if len(group.Targets) != 0 {
+	require.Empty(t, group.Targets, "Invalid number of targets.")
-		t.Fatalf("Invalid number of targets: expected 0, got %d", len(group.Targets))
-	}
 	require.Len(t, syncedTargets, 1)
 	require.Len(t, syncedTargets["prometheus"], 1)
-	if lbls := syncedTargets["prometheus"][0].Labels; lbls != nil {
+	require.Nil(t, syncedTargets["prometheus"][0].Labels)
-		t.Fatalf("Unexpected Group: expected nil Labels, got %v", lbls)
-	}
 }
 
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
@@ -1131,9 +1120,7 @@ func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	syncedTargets := <-discoveryManager.SyncCh()
 	verifyPresence(t, discoveryManager.targets, pk("static", "prometheus", 0), "{__address__=\"foo:9090\"}", true)
 	verifyPresence(t, discoveryManager.targets, pk("static", "prometheus2", 0), "{__address__=\"foo:9090\"}", true)
-	if len(discoveryManager.providers) != 1 {
+	require.Len(t, discoveryManager.providers, 1, "Invalid number of providers.")
-		t.Fatalf("Invalid number of providers: expected 1, got %d", len(discoveryManager.providers))
-	}
 	require.Len(t, syncedTargets, 2)
 	verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
 	require.Len(t, syncedTargets["prometheus"], 1)
@@ -1231,9 +1218,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	<-discoveryManager.SyncCh()
 
 	failedCount := client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
-	if failedCount != 3 {
+	require.Equal(t, 3.0, failedCount, "Expected to have 3 failed configs.")
-		t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
-	}
 
 	c["prometheus"] = Configs{
 		staticConfig("foo:9090"),
@@ -1242,9 +1227,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	<-discoveryManager.SyncCh()
 
 	failedCount = client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
-	if failedCount != 0 {
+	require.Equal(t, 0.0, failedCount, "Expected to get no failed config.")
-		t.Fatalf("Expected to get no failed config, got: %v", failedCount)
-	}
 }
 
 func TestCoordinationWithReceiver(t *testing.T) {
@@ -1388,19 +1371,14 @@ func TestCoordinationWithReceiver(t *testing.T) {
 				time.Sleep(expected.delay)
 				select {
 				case <-ctx.Done():
-					t.Fatalf("step %d: no update received in the expected timeframe", i)
+					require.FailNow(t, "step %d: no update received in the expected timeframe", i)
 				case tgs, ok := <-mgr.SyncCh():
-					if !ok {
+					require.True(t, ok, "step %d: discovery manager channel is closed", i)
-						t.Fatalf("step %d: discovery manager channel is closed", i)
+					require.Equal(t, len(expected.tgs), len(tgs), "step %d: targets mismatch", i)
-					}
+
-					if len(tgs) != len(expected.tgs) {
-						t.Fatalf("step %d: target groups mismatch, got: %d, expected: %d\ngot: %#v\nexpected: %#v",
-							i, len(tgs), len(expected.tgs), tgs, expected.tgs)
-					}
 					for k := range expected.tgs {
-						if _, ok := tgs[k]; !ok {
+						_, ok := tgs[k]
-							t.Fatalf("step %d: target group not found: %s\ngot: %#v", i, k, tgs)
+						require.True(t, ok, "step %d: target group not found: %s", i, k)
-						}
 						assertEqualGroups(t, tgs[k], expected.tgs[k])
 					}
 				}
@@ -1563,3 +1541,24 @@ func (t *testDiscoverer) update(tgs []*targetgroup.Group) {
 	<-t.ready
 	t.up <- tgs
 }
+
+func TestUnregisterMetrics(t *testing.T) {
+	reg := prometheus.NewRegistry()
+	// Check that all metrics can be unregistered, allowing a second manager to be created.
+	for i := 0; i < 2; i++ {
+		ctx, cancel := context.WithCancel(context.Background())
+
+		refreshMetrics, sdMetrics := NewTestMetrics(t, reg)
+
+		discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+		// discoveryManager will be nil if there was an error configuring metrics.
+		require.NotNil(t, discoveryManager)
+		// Unregister all metrics.
+		discoveryManager.UnregisterMetrics()
+		for _, sdMetric := range sdMetrics {
+			sdMetric.Unregister()
+		}
+		refreshMetrics.Unregister()
+		cancel()
+	}
+}

discovery/marathon/marathon.go

@@ -339,7 +339,7 @@ type appListClient func(ctx context.Context, client *http.Client, url string) (*
 
 // fetchApps requests a list of applications from a marathon server.
 func fetchApps(ctx context.Context, client *http.Client, url string) (*appList, error) {
-	request, err := http.NewRequest("GET", url, nil)
+	request, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -453,7 +453,6 @@ func targetsForApp(app *app) []model.LabelSet {
 	// Gather info about the app's 'tasks'. Each instance (container) is considered a task
 	// and can be reachable at one or more host:port endpoints.
 	for _, t := range app.Tasks {
-
 		// There are no labels to gather if only Ports is defined. (eg. with host networking)
 		// Ports can only be gathered from the Task (not from the app) and are guaranteed
 		// to be the same across all tasks. If we haven't gathered any ports by now,
@@ -464,7 +463,6 @@ func targetsForApp(app *app) []model.LabelSet {
 
 		// Iterate over the ports we gathered using one of the methods above.
 		for i, port := range ports {
-
 			// A zero port here means that either the portMapping has a zero port defined,
 			// or there is a portDefinition with requirePorts set to false. This means the port
 			// is auto-generated by Mesos and needs to be looked up in the task.
@@ -507,7 +505,7 @@ func targetEndpoint(task *task, port uint32, containerNet bool) string {
 		host = task.Host
 	}
 
-	return net.JoinHostPort(host, fmt.Sprintf("%d", port))
+	return net.JoinHostPort(host, strconv.Itoa(int(port)))
 }
 
 // Get a list of ports and a list of labels from a PortMapping.
@@ -516,7 +514,6 @@ func extractPortMapping(portMappings []portMapping, containerNet bool) ([]uint32
 	labels := make([]map[string]string, len(portMappings))
 
 	for i := 0; i < len(portMappings); i++ {
-
 		labels[i] = portMappings[i].Labels
 
 		if containerNet {

discovery/marathon/marathon_test.go

@@ -69,23 +69,15 @@ func TestMarathonSDHandleError(t *testing.T) {
 		}
 	)
 	tgs, err := testUpdateServices(client)
-	if !errors.Is(err, errTesting) {
+	require.ErrorIs(t, err, errTesting)
-		t.Fatalf("Expected error: %s", err)
+	require.Empty(t, tgs, "Expected no target groups.")
-	}
-	if len(tgs) != 0 {
-		t.Fatalf("Got group: %s", tgs)
-	}
 }
 
 func TestMarathonSDEmptyList(t *testing.T) {
 	client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { return &appList{}, nil }
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Empty(t, tgs, "Expected no target groups.")
-	}
-	if len(tgs) > 0 {
-		t.Fatalf("Got group: %v", tgs)
-	}
 }
 
 func marathonTestAppList(labels map[string]string, runningTasks int) *appList {
@@ -119,28 +111,16 @@ func TestMarathonSDSendGroup(t *testing.T) {
 		return marathonTestAppList(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 target group, got", len(tgs))
-	}
 
 	tg := tgs[0]
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
+	require.Len(t, tg.Targets, 1, "Expected 1 target.")
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 1 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func TestMarathonSDRemoveApp(t *testing.T) {
@@ -153,40 +133,27 @@ func TestMarathonSDRemoveApp(t *testing.T) {
 	defer refreshMetrics.Unregister()
 
 	md, err := NewDiscovery(cfg, nil, metrics)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("%s", err)
-	}
 
 	md.appsClient = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
 		return marathonTestAppList(marathonValidLabel, 1), nil
 	}
 	tgs, err := md.refresh(context.Background())
-	if err != nil {
+	require.NoError(t, err, "Got error on first update.")
-		t.Fatalf("Got error on first update: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 targetgroup.")
-	}
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 targetgroup, got", len(tgs))
-	}
 	tg1 := tgs[0]
 
 	md.appsClient = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
 		return marathonTestAppList(marathonValidLabel, 0), nil
 	}
 	tgs, err = md.refresh(context.Background())
-	if err != nil {
+	require.NoError(t, err, "Got error on second update.")
-		t.Fatalf("Got error on second update: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 targetgroup.")
-	}
+
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 targetgroup, got", len(tgs))
-	}
 	tg2 := tgs[0]
 
-	if tg2.Source != tg1.Source {
+	require.NotEmpty(t, tg2.Targets, "Got a non-empty target set.")
-		if len(tg2.Targets) > 0 {
+	require.Equal(t, tg1.Source, tg2.Source, "Source is different.")
-			t.Errorf("Got a non-empty target set: %s", tg2.Targets)
-		}
-		t.Fatalf("Source is different: %s != %s", tg1.Source, tg2.Source)
-	}
 }
 
 func marathonTestAppListWithMultiplePorts(labels map[string]string, runningTasks int) *appList {
@@ -221,34 +188,22 @@ func TestMarathonSDSendGroupWithMultiplePort(t *testing.T) {
 		return marathonTestAppListWithMultiplePorts(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
-	}
+		"Wrong portMappings label from the first port: %s", tgt[model.AddressLabel])
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
-	}
+		"Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestZeroTaskPortAppList(labels map[string]string, runningTasks int) *appList {
@@ -278,20 +233,12 @@ func TestMarathonZeroTaskPorts(t *testing.T) {
 		return marathonTestZeroTaskPortAppList(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 target group, got", len(tgs))
-	}
-	tg := tgs[0]
 
-	if tg.Source != "test-service-zero-ports" {
+	tg := tgs[0]
-		t.Fatalf("Wrong target group name: %s", tg.Source)
+	require.Equal(t, "test-service-zero-ports", tg.Source, "Wrong target group name.")
-	}
+	require.Empty(t, tg.Targets, "Wrong number of targets.")
-	if len(tg.Targets) != 0 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 }
 
 func Test500ErrorHttpResponseWithValidJSONBody(t *testing.T) {
@@ -306,9 +253,7 @@ func Test500ErrorHttpResponseWithValidJSONBody(t *testing.T) {
 	defer ts.Close()
 	// Execute test case and validate behavior.
 	_, err := testUpdateServices(nil)
-	if err == nil {
+	require.Error(t, err, "Expected error for 5xx HTTP response from marathon server.")
-		t.Fatalf("Expected error for 5xx HTTP response from marathon server, got nil")
-	}
 }
 
 func marathonTestAppListWithPortDefinitions(labels map[string]string, runningTasks int) *appList {
@@ -346,40 +291,24 @@ func TestMarathonSDSendGroupWithPortDefinitions(t *testing.T) {
 		return marathonTestAppListWithPortDefinitions(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:1234" {
+	require.Equal(t, "mesos-slave1:1234", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
-	}
+		"Wrong portMappings label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]),
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
+		"Wrong portDefinitions label from the first port.")
-	}
+
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:5678" {
+	require.Equal(t, "mesos-slave1:5678", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Empty(t, tgt[model.LabelName(portMappingLabelPrefix+"prometheus")], "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "yes", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithPortDefinitionsRequirePorts(labels map[string]string, runningTasks int) *appList {
@@ -416,40 +345,22 @@ func TestMarathonSDSendGroupWithPortDefinitionsRequirePorts(t *testing.T) {
 		return marathonTestAppListWithPortDefinitionsRequirePorts(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "yes", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithPorts(labels map[string]string, runningTasks int) *appList {
@@ -481,40 +392,22 @@ func TestMarathonSDSendGroupWithPorts(t *testing.T) {
 		return marathonTestAppListWithPorts(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithContainerPortMappings(labels map[string]string, runningTasks int) *appList {
@@ -555,40 +448,22 @@ func TestMarathonSDSendGroupWithContainerPortMappings(t *testing.T) {
 		return marathonTestAppListWithContainerPortMappings(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:12345" {
+	require.Equal(t, "mesos-slave1:12345", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithDockerContainerPortMappings(labels map[string]string, runningTasks int) *appList {
@@ -629,40 +504,22 @@ func TestMarathonSDSendGroupWithDockerContainerPortMappings(t *testing.T) {
 		return marathonTestAppListWithDockerContainerPortMappings(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:12345" {
+	require.Equal(t, "mesos-slave1:12345", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithContainerNetworkAndPortMappings(labels map[string]string, runningTasks int) *appList {
@@ -707,38 +564,20 @@ func TestMarathonSDSendGroupWithContainerNetworkAndPortMapping(t *testing.T) {
 		return marathonTestAppListWithContainerNetworkAndPortMappings(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "1.2.3.4:8080" {
+	require.Equal(t, "1.2.3.4:8080", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "1.2.3.4:1234" {
+	require.Equal(t, "1.2.3.4:1234", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }

discovery/metrics.go

@@ -19,16 +19,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-var (
-	clientGoRequestMetrics  = &clientGoRequestMetricAdapter{}
-	clientGoWorkloadMetrics = &clientGoWorkqueueMetricsProvider{}
-)
-
-func init() {
-	clientGoRequestMetrics.RegisterWithK8sGoClient()
-	clientGoWorkloadMetrics.RegisterWithK8sGoClient()
-}
-
 // Metrics to be used with a discovery manager.
 type Metrics struct {
 	FailedConfigs     prometheus.Gauge
@@ -99,3 +89,12 @@ func NewManagerMetrics(registerer prometheus.Registerer, sdManagerName string) (
 
 	return m, nil
 }
+
+// Unregister unregisters all metrics.
+func (m *Metrics) Unregister(registerer prometheus.Registerer) {
+	registerer.Unregister(m.FailedConfigs)
+	registerer.Unregister(m.DiscoveredTargets)
+	registerer.Unregister(m.ReceivedUpdates)
+	registerer.Unregister(m.DelayedUpdates)
+	registerer.Unregister(m.SentUpdates)
+}

discovery/metrics_k8s_client.go

@@ -35,6 +35,11 @@ const (
 	workqueueMetricsNamespace  = KubernetesMetricsNamespace + "_workqueue"
 )
 
+var (
+	clientGoRequestMetrics  = &clientGoRequestMetricAdapter{}
+	clientGoWorkloadMetrics = &clientGoWorkqueueMetricsProvider{}
+)
+
 var (
 	// Metrics for client-go's HTTP requests.
 	clientGoRequestResultMetricVec = prometheus.NewCounterVec(
@@ -135,6 +140,9 @@ func clientGoMetrics() []prometheus.Collector {
 }
 
 func RegisterK8sClientMetricsWithPrometheus(registerer prometheus.Registerer) error {
+	clientGoRequestMetrics.RegisterWithK8sGoClient()
+	clientGoWorkloadMetrics.RegisterWithK8sGoClient()
+
 	for _, collector := range clientGoMetrics() {
 		err := registerer.Register(collector)
 		if err != nil {

discovery/moby/network.go

@@ -15,7 +15,7 @@ package moby
 
 import (
 	"context"
-	"fmt"
+	"strconv"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/client"
@@ -44,8 +44,8 @@ func getNetworksLabels(ctx context.Context, client *client.Client, labelPrefix s
 			labelPrefix + labelNetworkID:       network.ID,
 			labelPrefix + labelNetworkName:     network.Name,
 			labelPrefix + labelNetworkScope:    network.Scope,
-			labelPrefix + labelNetworkInternal: fmt.Sprintf("%t", network.Internal),
+			labelPrefix + labelNetworkInternal: strconv.FormatBool(network.Internal),
-			labelPrefix + labelNetworkIngress:  fmt.Sprintf("%t", network.Ingress),
+			labelPrefix + labelNetworkIngress:  strconv.FormatBool(network.Ingress),
 		}
 		for k, v := range network.Labels {
 			ln := strutil.SanitizeLabelName(k)

discovery/moby/nodes.go

@@ -66,7 +66,7 @@ func (d *Discovery) refreshNodes(ctx context.Context) ([]*targetgroup.Group, err
 			swarmLabelNodeAddress:              model.LabelValue(n.Status.Addr),
 		}
 		if n.ManagerStatus != nil {
-			labels[swarmLabelNodeManagerLeader] = model.LabelValue(fmt.Sprintf("%t", n.ManagerStatus.Leader))
+			labels[swarmLabelNodeManagerLeader] = model.LabelValue(strconv.FormatBool(n.ManagerStatus.Leader))
 			labels[swarmLabelNodeManagerReachability] = model.LabelValue(n.ManagerStatus.Reachability)
 			labels[swarmLabelNodeManagerAddr] = model.LabelValue(n.ManagerStatus.Addr)
 		}
@@ -80,7 +80,6 @@ func (d *Discovery) refreshNodes(ctx context.Context) ([]*targetgroup.Group, err
 		labels[model.AddressLabel] = model.LabelValue(addr)
 
 		tg.Targets = append(tg.Targets, labels)
-
 	}
 	return []*targetgroup.Group{tg}, nil
 }

discovery/moby/services.go

@@ -116,7 +116,7 @@ func (d *Discovery) refreshServices(ctx context.Context) ([]*targetgroup.Group,
 					labels[model.LabelName(k)] = model.LabelValue(v)
 				}
 
-				addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", d.port))
+				addr := net.JoinHostPort(ip.String(), strconv.Itoa(d.port))
 				labels[model.AddressLabel] = model.LabelValue(addr)
 
 				tg.Targets = append(tg.Targets, labels)

discovery/moby/tasks.go

@@ -150,7 +150,7 @@ func (d *Discovery) refreshTasks(ctx context.Context) ([]*targetgroup.Group, err
 						labels[model.LabelName(k)] = model.LabelValue(v)
 					}
 
-					addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", d.port))
+					addr := net.JoinHostPort(ip.String(), strconv.Itoa(d.port))
 					labels[model.AddressLabel] = model.LabelValue(addr)
 
 					tg.Targets = append(tg.Targets, labels)

discovery/openstack/hypervisor.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strconv"
 
 	"github.com/go-kit/log"
 	"github.com/gophercloud/gophercloud"
@@ -72,7 +73,7 @@ func (h *HypervisorDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group
 	}
 
 	tg := &targetgroup.Group{
-		Source: fmt.Sprintf("OS_" + h.region),
+		Source: "OS_" + h.region,
 	}
 	// OpenStack API reference
 	// https://developer.openstack.org/api-ref/compute/#list-hypervisors-details
@@ -84,7 +85,7 @@ func (h *HypervisorDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group
 		}
 		for _, hypervisor := range hypervisorList {
 			labels := model.LabelSet{}
-			addr := net.JoinHostPort(hypervisor.HostIP, fmt.Sprintf("%d", h.port))
+			addr := net.JoinHostPort(hypervisor.HostIP, strconv.Itoa(h.port))
 			labels[model.AddressLabel] = model.LabelValue(addr)
 			labels[openstackLabelHypervisorID] = model.LabelValue(hypervisor.ID)
 			labels[openstackLabelHypervisorHostName] = model.LabelValue(hypervisor.HypervisorHostname)

discovery/openstack/instance.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strconv"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
@@ -120,7 +121,7 @@ func (i *InstanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group,
 	}
 	pager := servers.List(client, opts)
 	tg := &targetgroup.Group{
-		Source: fmt.Sprintf("OS_" + i.region),
+		Source: "OS_" + i.region,
 	}
 	err = pager.EachPage(func(page pagination.Page) (bool, error) {
 		if ctx.Err() != nil {
@@ -194,7 +195,7 @@ func (i *InstanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group,
 					if val, ok := floatingIPList[floatingIPKey{id: s.ID, fixed: addr}]; ok {
 						lbls[openstackLabelPublicIP] = model.LabelValue(val)
 					}
-					addr = net.JoinHostPort(addr, fmt.Sprintf("%d", i.port))
+					addr = net.JoinHostPort(addr, strconv.Itoa(i.port))
 					lbls[model.AddressLabel] = model.LabelValue(addr)
 
 					tg.Targets = append(tg.Targets, lbls)

discovery/openstack/mock_test.go

@@ -18,6 +18,8 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 // SDMock is the interface for the OpenStack mock.
@@ -49,15 +51,13 @@ func (m *SDMock) Setup() {
 const tokenID = "cbc36478b0bd8e67e89469c7749d4127"
 
 func testMethod(t *testing.T, r *http.Request, expected string) {
-	if expected != r.Method {
+	require.Equal(t, expected, r.Method, "Unexpected request method.")
-		t.Errorf("Request method = %v, expected %v", r.Method, expected)
-	}
 }
 
 func testHeader(t *testing.T, r *http.Request, header, expected string) {
-	if actual := r.Header.Get(header); expected != actual {
+	t.Helper()
-		t.Errorf("Header %s = %s, expected %s", header, actual, expected)
+	actual := r.Header.Get(header)
-	}
+	require.Equal(t, expected, actual, "Unexpected value for request header %s.", header)
 }
 
 // HandleVersionsSuccessfully mocks version call.
@@ -239,7 +239,7 @@ const hypervisorListBody = `
 // HandleHypervisorListSuccessfully mocks os-hypervisors detail call.
 func (m *SDMock) HandleHypervisorListSuccessfully() {
 	m.Mux.HandleFunc("/os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) {
-		testMethod(m.t, r, "GET")
+		testMethod(m.t, r, http.MethodGet)
 		testHeader(m.t, r, "X-Auth-Token", tokenID)
 
 		w.Header().Add("Content-Type", "application/json")
@@ -536,7 +536,7 @@ const serverListBody = `
 // HandleServerListSuccessfully mocks server detail call.
 func (m *SDMock) HandleServerListSuccessfully() {
 	m.Mux.HandleFunc("/servers/detail", func(w http.ResponseWriter, r *http.Request) {
-		testMethod(m.t, r, "GET")
+		testMethod(m.t, r, http.MethodGet)
 		testHeader(m.t, r, "X-Auth-Token", tokenID)
 
 		w.Header().Add("Content-Type", "application/json")
@@ -575,7 +575,7 @@ const listOutput = `
 // HandleFloatingIPListSuccessfully mocks floating ips call.
 func (m *SDMock) HandleFloatingIPListSuccessfully() {
 	m.Mux.HandleFunc("/os-floating-ips", func(w http.ResponseWriter, r *http.Request) {
-		testMethod(m.t, r, "GET")
+		testMethod(m.t, r, http.MethodGet)
 		testHeader(m.t, r, "X-Auth-Token", tokenID)
 
 		w.Header().Add("Content-Type", "application/json")

discovery/ovhcloud/dedicated_server.go

@@ -144,12 +144,12 @@ func (d *dedicatedServerDiscovery) refresh(context.Context) ([]*targetgroup.Grou
 			model.InstanceLabel:                             model.LabelValue(server.Name),
 			dedicatedServerLabelPrefix + "state":            model.LabelValue(server.State),
 			dedicatedServerLabelPrefix + "commercial_range": model.LabelValue(server.CommercialRange),
-			dedicatedServerLabelPrefix + "link_speed":       model.LabelValue(fmt.Sprintf("%d", server.LinkSpeed)),
+			dedicatedServerLabelPrefix + "link_speed":       model.LabelValue(strconv.Itoa(server.LinkSpeed)),
 			dedicatedServerLabelPrefix + "rack":             model.LabelValue(server.Rack),
 			dedicatedServerLabelPrefix + "no_intervention":  model.LabelValue(strconv.FormatBool(server.NoIntervention)),
 			dedicatedServerLabelPrefix + "os":               model.LabelValue(server.Os),
 			dedicatedServerLabelPrefix + "support_level":    model.LabelValue(server.SupportLevel),
-			dedicatedServerLabelPrefix + "server_id":        model.LabelValue(fmt.Sprintf("%d", server.ServerID)),
+			dedicatedServerLabelPrefix + "server_id":        model.LabelValue(strconv.FormatInt(server.ServerID, 10)),
 			dedicatedServerLabelPrefix + "reverse":          model.LabelValue(server.Reverse),
 			dedicatedServerLabelPrefix + "datacenter":       model.LabelValue(server.Datacenter),
 			dedicatedServerLabelPrefix + "name":             model.LabelValue(server.Name),

discovery/ovhcloud/ovhcloud_test.go

@@ -66,7 +66,7 @@ endpoint: %s
 
 	_, err := createClient(&conf)
 
-	require.ErrorContains(t, err, "missing application key")
+	require.ErrorContains(t, err, "missing authentication information")
 }
 
 func TestParseIPs(t *testing.T) {