mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-09-19 23:37:31 -07:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'prometheus/main' into arve/close-engine

Browse source 
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
This commit is contained in:
Arve Knudsen 2024-08-26 11:54:55 +02:00
parent 7c873004c7 349068ad3e
commit c9a460d570
156 changed files with 6294 additions and 2184 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/buf-lint.yml vendored
View file

@ -13,7 +13,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- uses: bufbuild/buf-setup-action@35c243d7f2a909b1d4e40399b348a7fdab27d78d # v1.34.0 - uses: bufbuild/buf-setup-action@aceb106d2419c4cff48863df90161d92decb8591 # v1.35.1
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
- uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1 - uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1

2
.github/workflows/buf.yml vendored
View file

@ -13,7 +13,7 @@ jobs:
if: github.repository_owner == 'prometheus' if: github.repository_owner == 'prometheus'
steps: steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- uses: bufbuild/buf-setup-action@35c243d7f2a909b1d4e40399b348a7fdab27d78d # v1.34.0 - uses: bufbuild/buf-setup-action@aceb106d2419c4cff48863df90161d92decb8591 # v1.35.1
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
- uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1 - uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1

10
.github/workflows/ci.yml vendored
View file

@ -75,7 +75,7 @@ jobs:
runs-on: windows-latest runs-on: windows-latest
steps: steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
with: with:
go-version: 1.22.x go-version: 1.22.x
- run: | - run: |
@ -162,7 +162,7 @@ jobs:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Install Go - name: Install Go
uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
with: with:
cache: false cache: false
go-version: 1.22.x go-version: 1.22.x
@ -175,18 +175,18 @@ jobs:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Install Go - name: Install Go
uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
with: with:
go-version: 1.22.x go-version: 1.22.x
- name: Install snmp_exporter/generator dependencies - name: Install snmp_exporter/generator dependencies
run: sudo apt-get update && sudo apt-get -y install libsnmp-dev run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
if: github.repository == 'prometheus/snmp_exporter' if: github.repository == 'prometheus/snmp_exporter'
- name: Lint - name: Lint
uses: golangci/golangci-lint-action@a4f60bb28d35aeee14e6880718e0c85ff1882e64 # v6.0.1 uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86 # v6.1.0
with: with:
args: --verbose args: --verbose
# Make sure to sync this with Makefile.common and scripts/golangci-lint.yml. # Make sure to sync this with Makefile.common and scripts/golangci-lint.yml.
version: v1.59.1 version: v1.60.2
fuzzing: fuzzing:
uses: ./.github/workflows/fuzzing.yml uses: ./.github/workflows/fuzzing.yml
if: github.event_name == 'pull_request' if: github.event_name == 'pull_request'

6
.github/workflows/codeql-analysis.yml vendored
View file

@ -27,12 +27,12 @@ jobs:
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Initialize CodeQL - name: Initialize CodeQL
uses: github/codeql-action/init@b611370bb5703a7efb587f9d136a52ea24c5c38c # v3.25.11 uses: github/codeql-action/init@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15
with: with:
languages: ${{ matrix.language }} languages: ${{ matrix.language }}
- name: Autobuild - name: Autobuild
uses: github/codeql-action/autobuild@b611370bb5703a7efb587f9d136a52ea24c5c38c # v3.25.11 uses: github/codeql-action/autobuild@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15
- name: Perform CodeQL Analysis - name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@b611370bb5703a7efb587f9d136a52ea24c5c38c # v3.25.11 uses: github/codeql-action/analyze@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15

2
.github/workflows/fuzzing.yml vendored
View file

@ -21,7 +21,7 @@ jobs:
fuzz-seconds: 600 fuzz-seconds: 600
dry-run: false dry-run: false
- name: Upload Crash - name: Upload Crash
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
if: failure() && steps.build.outcome == 'success' if: failure() && steps.build.outcome == 'success'
with: with:
name: artifacts name: artifacts

6
.github/workflows/scorecards.yml vendored
View file

@ -26,7 +26,7 @@ jobs:
persist-credentials: false persist-credentials: false
- name: "Run analysis" - name: "Run analysis"
uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # tag=v2.3.3 uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # tag=v2.4.0
with: with:
results_file: results.sarif results_file: results.sarif
results_format: sarif results_format: sarif
@ -37,7 +37,7 @@ jobs:
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
# format to the repository Actions tab. # format to the repository Actions tab.
- name: "Upload artifact" - name: "Upload artifact"
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # tag=v4.3.3 uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # tag=v4.3.4
with: with:
name: SARIF file name: SARIF file
path: results.sarif path: results.sarif
@ -45,6 +45,6 @@ jobs:
# Upload the results to GitHub's code scanning dashboard. # Upload the results to GitHub's code scanning dashboard.
- name: "Upload to code-scanning" - name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@b611370bb5703a7efb587f9d136a52ea24c5c38c # tag=v3.25.11 uses: github/codeql-action/upload-sarif@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # tag=v3.25.15
with: with:
sarif_file: results.sarif sarif_file: results.sarif

30
.golangci.yml
View file

@ -25,15 +25,34 @@ linters:
- loggercheck - loggercheck
issues: issues:
max-issues-per-linter: 0
max-same-issues: 0 max-same-issues: 0
# The default exclusions are too aggressive. For one, they
# essentially disable any linting on doc comments. We disable
# default exclusions here and add exclusions fitting our codebase
# further down.
exclude-use-default: false
exclude-files: exclude-files:
# Skip autogenerated files. # Skip autogenerated files.
- ^.*\.(pb|y)\.go$ - ^.*\.(pb|y)\.go$
exclude-dirs: exclude-dirs:
# Copied it from a different source # Copied it from a different source.
- storage/remote/otlptranslator/prometheusremotewrite - storage/remote/otlptranslator/prometheusremotewrite
- storage/remote/otlptranslator/prometheus - storage/remote/otlptranslator/prometheus
exclude-rules: exclude-rules:
- linters:
- errcheck
# Taken from the default exclusions (that are otherwise disabled above).
text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked
- linters:
- govet
# We use many Seek methods that do not follow the usual pattern.
text: "stdmethods: method Seek.* should have signature Seek"
- linters:
- revive
# We have stopped at some point to write doc comments on exported symbols.
# TODO(beorn7): Maybe we should enforce this again? There are ~500 offenders right now.
text: exported (.+) should have comment( \(or a comment on this block\))? or be unexported
- linters: - linters:
- gocritic - gocritic
text: "appendAssign" text: "appendAssign"
@ -94,15 +113,14 @@ linters-settings:
errorf: false errorf: false
revive: revive:
# By default, revive will enable only the linting rules that are named in the configuration file. # By default, revive will enable only the linting rules that are named in the configuration file.
# So, it's needed to explicitly set in configuration all required rules. # So, it's needed to explicitly enable all required rules here.
# The following configuration enables all the rules from the defaults.toml
# https://github.com/mgechev/revive/blob/master/defaults.toml
rules: rules:
# https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md
- name: blank-imports - name: blank-imports
- name: comment-spacings
- name: context-as-argument - name: context-as-argument
arguments: arguments:
# allow functions with test or bench signatures # Allow functions with test or bench signatures.
- allowTypesBefore: "*testing.T,testing.TB" - allowTypesBefore: "*testing.T,testing.TB"
- name: context-keys-type - name: context-keys-type
- name: dot-imports - name: dot-imports
@ -118,6 +136,8 @@ linters-settings:
- name: increment-decrement - name: increment-decrement
- name: indent-error-flow - name: indent-error-flow
- name: package-comments - name: package-comments
# TODO(beorn7): Currently, we have a lot of missing package doc comments. Maybe we should have them.
disabled: true
- name: range - name: range
- name: receiver-naming - name: receiver-naming
- name: redefines-builtin-id - name: redefines-builtin-id

46
CHANGELOG.md
View file

@ -3,8 +3,50 @@
## unreleased ## unreleased
* [FEATURE] OTLP receiver: Add new option `otlp.promote_resource_attributes`, for any OTel resource attributes that should be promoted to metric labels. #14200 * [FEATURE] OTLP receiver: Add new option `otlp.promote_resource_attributes`, for any OTel resource attributes that should be promoted to metric labels. #14200
* [FEATURE] Remote-Write: Add sender and receiver support for [Remote Write 2.0-rc.2](https://prometheus.io/docs/specs/remote_write_spec_2_0/) specification #14395 #14427 #14444 * [ENHANCEMENT] OTLP receiver: Warn when encountering exponential histograms with zero count and non-zero sum. #14706
* [ENHANCEMENT] Remote-Write: 1.x messages against Remote Write 2.x Receivers will have now correct values for `prometheus_storage_<samples|histograms|exemplar>_failed_total` in case of partial errors #14444 * [BUGFIX] tsdb/wlog.Watcher.readSegmentForGC: Only count unknown record types against record_decode_failures_total metric. #14042
## 2.54.0-rc.1 / 2024-08-05
Release 2.54 brings a release candidate of a major new version of [Remote Write: 2.0](https://prometheus.io/docs/specs/remote_write_spec_2_0/).
This is experimental at this time and may still change.
Remote-write v2 is enabled by default, but can be disabled via feature-flag `web.remote-write-receiver.accepted-protobuf-messages`.
* [CHANGE] Remote-Write: `highest_timestamp_in_seconds` and `queue_highest_sent_timestamp_seconds` metrics now initialized to 0. #14437
* [CHANGE] API: Split warnings from info annotations in API response. #14327
* [FEATURE] Remote-Write: Version 2.0 experimental, plus metadata in WAL via feature flag `metadata-wal-records` (defaults on). #14395,#14427,#14444
* [FEATURE] PromQL: add limitk() and limit_ratio() aggregation operators. #12503
* [ENHANCEMENT] PromQL: Accept underscores in literal numbers, e.g. 1_000_000 for 1 million. #12821
* [ENHANCEMENT] PromQL: float literal numbers and durations are now interchangeable (experimental). Example: `time() - my_timestamp > 10m`. #9138
* [ENHANCEMENT] PromQL: use Kahan summation for sum(). #14074,#14362
* [ENHANCEMENT] PromQL (experimental native histograms): Optimize `histogram_count` and `histogram_sum` functions. #14097
* [ENHANCEMENT] TSDB: Better support for out-of-order experimental native histogram samples. #14438
* [ENHANCEMENT] TSDB: Optimise seek within index. #14393
* [ENHANCEMENT] TSDB: Optimise deletion of stale series. #14307
* [ENHANCEMENT] TSDB: Reduce locking to optimise adding and removing series. #13286,#14286
* [ENHANCEMENT] TSDB: Small optimisation: streamline special handling for out-of-order data. #14396,#14584
* [ENHANCEMENT] Regexps: Optimize patterns with multiple prefixes. #13843,#14368
* [ENHANCEMENT] Regexps: Optimize patterns containing multiple literal strings. #14173
* [ENHANCEMENT] AWS SD: expose Primary IPv6 addresses as __meta_ec2_primary_ipv6_addresses. #14156
* [ENHANCEMENT] Docker SD: add MatchFirstNetwork for containers with multiple networks. #10490
* [ENHANCEMENT] OpenStack SD: Use `flavor.original_name` if available. #14312
* [ENHANCEMENT] UI (experimental native histograms): more accurate representation. #13680,#14430
* [ENHANCEMENT] Agent: `out_of_order_time_window` config option now applies to agent. #14094
* [ENHANCEMENT] Notifier: Send any outstanding Alertmanager notifications when shutting down. #14290
* [ENHANCEMENT] Rules: Add label-matcher support to Rules API. #10194
* [ENHANCEMENT] HTTP API: Add url to message logged on error while sending response. #14209
* [BUGFIX] CLI: escape `|` characters when generating docs. #14420
* [BUGFIX] PromQL (experimental native histograms): Fix some binary operators between native histogram values. #14454
* [BUGFIX] TSDB: LabelNames API could fail during compaction. #14279
* [BUGFIX] TSDB: Fix rare issue where pending OOO read can be left dangling if creating querier fails. #14341
* [BUGFIX] TSDB: fix check for context cancellation in LabelNamesFor. #14302
* [BUGFIX] Rules: Fix rare panic on reload. #14366
* [BUGFIX] Config: In YAML marshalling, do not output a regexp field if it was never set. #14004
* [BUGFIX] Remote-Write: reject samples with future timestamps. #14304
* [BUGFIX] Remote-Write: Fix data corruption in remote write if max_sample_age is applied. #14078
* [BUGFIX] Notifier: Fix Alertmanager discovery not updating under heavy load. #14174
* [BUGFIX] Regexes: some Unicode characters were not matched by case-insensitive comparison. #14170,#14299
* [BUGFIX] Remote-Read: Resolve occasional segmentation fault on query. #14515
## 2.53.1 / 2024-07-10 ## 2.53.1 / 2024-07-10

2
Makefile.common
View file

@ -61,7 +61,7 @@ PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_
SKIP_GOLANGCI_LINT := SKIP_GOLANGCI_LINT :=
GOLANGCI_LINT := GOLANGCI_LINT :=
GOLANGCI_LINT_OPTS ?= GOLANGCI_LINT_OPTS ?=
GOLANGCI_LINT_VERSION ?= v1.59.1 GOLANGCI_LINT_VERSION ?= v1.60.2
# golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64.
# windows isn't included here because of the path separator being different. # windows isn't included here because of the path separator being different.
ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))

3
README.md
View file

@ -12,9 +12,10 @@ examples and guides.</p>
[![Docker Pulls](https://img.shields.io/docker/pulls/prom/prometheus.svg?maxAge=604800)][hub] [![Docker Pulls](https://img.shields.io/docker/pulls/prom/prometheus.svg?maxAge=604800)][hub]
[![Go Report Card](https://goreportcard.com/badge/github.com/prometheus/prometheus)](https://goreportcard.com/report/github.com/prometheus/prometheus) [![Go Report Card](https://goreportcard.com/badge/github.com/prometheus/prometheus)](https://goreportcard.com/report/github.com/prometheus/prometheus)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/486/badge)](https://bestpractices.coreinfrastructure.org/projects/486) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/486/badge)](https://bestpractices.coreinfrastructure.org/projects/486)
[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://securityscorecards.dev/viewer/?uri=github.com/prometheus/prometheus)
[![CLOMonitor](https://img.shields.io/endpoint?url=https://clomonitor.io/api/projects/cncf/prometheus/badge)](https://clomonitor.io/projects/cncf/prometheus)
[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/prometheus/prometheus) [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/prometheus/prometheus)
[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/prometheus.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:prometheus) [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/prometheus.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:prometheus)
[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://securityscorecards.dev/viewer/?uri=github.com/prometheus/prometheus)
</div> </div>

48
SECURITY-INSIGHTS.yml Normal file
View file

@ -0,0 +1,48 @@
header:
schema-version: '1.0.0'
expiration-date: '2025-07-30T01:00:00.000Z'
last-updated: '2024-07-30'
last-reviewed: '2024-07-30'
project-url: https://github.com/prometheus/prometheus
changelog: https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md
license: https://github.com/prometheus/prometheus/blob/main/LICENSE
project-lifecycle:
status: active
bug-fixes-only: false
core-maintainers:
- https://github.com/prometheus/prometheus/blob/main/MAINTAINERS.md
contribution-policy:
accepts-pull-requests: true
accepts-automated-pull-requests: true
dependencies:
third-party-packages: true
dependencies-lists:
- https://github.com/prometheus/prometheus/blob/main/go.mod
- https://github.com/prometheus/prometheus/blob/main/web/ui/package.json
env-dependencies-policy:
policy-url: https://github.com/prometheus/prometheus/blob/main/CONTRIBUTING.md#dependency-management
distribution-points:
- https://github.com/prometheus/prometheus/releases
documentation:
- https://prometheus.io/docs/introduction/overview/
security-contacts:
- type: email
value: prometheus-team@googlegroups.com
security-testing:
- tool-type: sca
tool-name: Dependabot
tool-version: latest
integration:
ad-hoc: false
ci: true
before-release: true
- tool-type: sast
tool-name: CodeQL
tool-version: latest
integration:
ad-hoc: false
ci: true
before-release: true
vulnerability-reporting:
accepts-vulnerability-reports: true
security-policy: https://github.com/prometheus/prometheus/security/policy

2
VERSION
View file

@ -1 +1 @@
2.53.1 2.54.0-rc.1

42
cmd/prometheus/main.go
View file

@ -152,6 +152,7 @@ type flagConfig struct {
queryConcurrency int queryConcurrency int
queryMaxSamples int queryMaxSamples int
RemoteFlushDeadline model.Duration RemoteFlushDeadline model.Duration
nameEscapingScheme string
featureList []string featureList []string
memlimitRatio float64 memlimitRatio float64
@ -234,6 +235,12 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
level.Info(logger).Log("msg", "Experimental created timestamp zero ingestion enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols)) level.Info(logger).Log("msg", "Experimental created timestamp zero ingestion enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
case "delayed-compaction":
c.tsdb.EnableDelayedCompaction = true
level.Info(logger).Log("msg", "Experimental delayed compaction is enabled.")
case "utf8-names":
model.NameValidationScheme = model.UTF8Validation
level.Info(logger).Log("msg", "Experimental UTF-8 support enabled")
case "": case "":
continue continue
case "promql-at-modifier", "promql-negative-offset": case "promql-at-modifier", "promql-negative-offset":
@ -290,8 +297,8 @@ func main() {
a.Flag("config.file", "Prometheus configuration file path."). a.Flag("config.file", "Prometheus configuration file path.").
Default("prometheus.yml").StringVar(&cfg.configFile) Default("prometheus.yml").StringVar(&cfg.configFile)
a.Flag("web.listen-address", "Address to listen on for UI, API, and telemetry."). a.Flag("web.listen-address", "Address to listen on for UI, API, and telemetry. Can be repeated.").
Default("0.0.0.0:9090").StringVar(&cfg.web.ListenAddress) Default("0.0.0.0:9090").StringsVar(&cfg.web.ListenAddresses)
a.Flag("auto-gomemlimit.ratio", "The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory"). a.Flag("auto-gomemlimit.ratio", "The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory").
Default("0.9").FloatVar(&cfg.memlimitRatio) Default("0.9").FloatVar(&cfg.memlimitRatio)
@ -305,7 +312,7 @@ func main() {
"Maximum duration before timing out read of the request, and closing idle connections."). "Maximum duration before timing out read of the request, and closing idle connections.").
Default("5m").SetValue(&cfg.webTimeout) Default("5m").SetValue(&cfg.webTimeout)
a.Flag("web.max-connections", "Maximum number of simultaneous connections."). a.Flag("web.max-connections", "Maximum number of simultaneous connections across all listeners.").
Default("512").IntVar(&cfg.web.MaxConnections) Default("512").IntVar(&cfg.web.MaxConnections)
a.Flag("web.external-url", a.Flag("web.external-url",
@ -381,6 +388,9 @@ func main() {
serverOnlyFlag(a, "storage.tsdb.allow-overlapping-blocks", "[DEPRECATED] This flag has no effect. Overlapping blocks are enabled by default now."). serverOnlyFlag(a, "storage.tsdb.allow-overlapping-blocks", "[DEPRECATED] This flag has no effect. Overlapping blocks are enabled by default now.").
Default("true").Hidden().BoolVar(&b) Default("true").Hidden().BoolVar(&b)
serverOnlyFlag(a, "storage.tsdb.allow-overlapping-compaction", "Allow compaction of overlapping blocks. If set to false, TSDB stops vertical compaction and leaves overlapping blocks there. The use case is to let another component handle the compaction of overlapping blocks.").
Default("true").Hidden().BoolVar(&cfg.tsdb.EnableOverlappingCompaction)
serverOnlyFlag(a, "storage.tsdb.wal-compression", "Compress the tsdb WAL."). serverOnlyFlag(a, "storage.tsdb.wal-compression", "Compress the tsdb WAL.").
Hidden().Default("true").BoolVar(&cfg.tsdb.WALCompression) Hidden().Default("true").BoolVar(&cfg.tsdb.WALCompression)
@ -475,7 +485,9 @@ func main() {
a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates."). a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates.").
Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval) Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval)
a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, created-timestamp-zero-ingestion, concurrent-rule-eval. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). a.Flag("scrape.name-escaping-scheme", `Method for escaping legacy invalid names when sending to Prometheus that does not support UTF-8. Can be one of "values", "underscores", or "dots".`).Default(scrape.DefaultNameEscapingScheme.String()).StringVar(&cfg.nameEscapingScheme)
a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, utf8-names. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
Default("").StringsVar(&cfg.featureList) Default("").StringsVar(&cfg.featureList)
promlogflag.AddFlags(a, &cfg.promlogConfig) promlogflag.AddFlags(a, &cfg.promlogConfig)
@ -503,6 +515,15 @@ func main() {
os.Exit(1) os.Exit(1)
} }
if cfg.nameEscapingScheme != "" {
scheme, err := model.ToEscapingScheme(cfg.nameEscapingScheme)
if err != nil {
fmt.Fprintf(os.Stderr, `Invalid name escaping scheme: %q; Needs to be one of "values", "underscores", or "dots"`, cfg.nameEscapingScheme)
os.Exit(1)
}
model.NameEscapingScheme = scheme
}
if agentMode && len(serverOnlyFlags) > 0 { if agentMode && len(serverOnlyFlags) > 0 {
fmt.Fprintf(os.Stderr, "The following flag(s) can not be used in agent mode: %q", serverOnlyFlags) fmt.Fprintf(os.Stderr, "The following flag(s) can not be used in agent mode: %q", serverOnlyFlags)
os.Exit(3) os.Exit(3)
@ -523,7 +544,7 @@ func main() {
localStoragePath = cfg.agentStoragePath localStoragePath = cfg.agentStoragePath
} }
cfg.web.ExternalURL, err = computeExternalURL(cfg.prometheusURL, cfg.web.ListenAddress) cfg.web.ExternalURL, err = computeExternalURL(cfg.prometheusURL, cfg.web.ListenAddresses[0])
if err != nil { if err != nil {
fmt.Fprintln(os.Stderr, fmt.Errorf("parse external URL %q: %w", cfg.prometheusURL, err)) fmt.Fprintln(os.Stderr, fmt.Errorf("parse external URL %q: %w", cfg.prometheusURL, err))
os.Exit(2) os.Exit(2)
@ -969,9 +990,9 @@ func main() {
}) })
} }
listener, err := webHandler.Listener() listeners, err := webHandler.Listeners()
if err != nil { if err != nil {
level.Error(logger).Log("msg", "Unable to start web listener", "err", err) level.Error(logger).Log("msg", "Unable to start web listeners", "err", err)
if err := queryEngine.Close(); err != nil { if err := queryEngine.Close(); err != nil {
level.Warn(logger).Log("msg", "Closing query engine failed", "err", err) level.Warn(logger).Log("msg", "Closing query engine failed", "err", err)
} }
@ -1275,7 +1296,7 @@ func main() {
// Web handler. // Web handler.
g.Add( g.Add(
func() error { func() error {
if err := webHandler.Run(ctxWeb, listener, *webConfig); err != nil { if err := webHandler.Run(ctxWeb, listeners, *webConfig); err != nil {
return fmt.Errorf("error starting web server: %w", err) return fmt.Errorf("error starting web server: %w", err)
} }
return nil return nil
@ -1724,6 +1745,8 @@ type tsdbOptions struct {
MaxExemplars int64 MaxExemplars int64
EnableMemorySnapshotOnShutdown bool EnableMemorySnapshotOnShutdown bool
EnableNativeHistograms bool EnableNativeHistograms bool
EnableDelayedCompaction bool
EnableOverlappingCompaction bool
} }
func (opts tsdbOptions) ToTSDBOptions() tsdb.Options { func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
@ -1744,7 +1767,8 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
EnableMemorySnapshotOnShutdown: opts.EnableMemorySnapshotOnShutdown, EnableMemorySnapshotOnShutdown: opts.EnableMemorySnapshotOnShutdown,
EnableNativeHistograms: opts.EnableNativeHistograms, EnableNativeHistograms: opts.EnableNativeHistograms,
OutOfOrderTimeWindow: opts.OutOfOrderTimeWindow, OutOfOrderTimeWindow: opts.OutOfOrderTimeWindow,
EnableOverlappingCompaction: true, EnableDelayedCompaction: opts.EnableDelayedCompaction,
EnableOverlappingCompaction: opts.EnableOverlappingCompaction,
} }
} }

9
cmd/promtool/main.go
View file

@ -204,6 +204,7 @@ func main() {
pushMetricsHeaders := pushMetricsCmd.Flag("header", "Prometheus remote write header.").StringMap() pushMetricsHeaders := pushMetricsCmd.Flag("header", "Prometheus remote write header.").StringMap()
testCmd := app.Command("test", "Unit testing.") testCmd := app.Command("test", "Unit testing.")
junitOutFile := testCmd.Flag("junit", "File path to store JUnit XML test results.").OpenFile(os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
testRulesCmd := testCmd.Command("rules", "Unit tests for rules.") testRulesCmd := testCmd.Command("rules", "Unit tests for rules.")
testRulesRun := testRulesCmd.Flag("run", "If set, will only run test groups whose names match the regular expression. Can be specified multiple times.").Strings() testRulesRun := testRulesCmd.Flag("run", "If set, will only run test groups whose names match the regular expression. Can be specified multiple times.").Strings()
testRulesFiles := testRulesCmd.Arg( testRulesFiles := testRulesCmd.Arg(
@ -378,7 +379,11 @@ func main() {
os.Exit(QueryLabels(serverURL, httpRoundTripper, *queryLabelsMatch, *queryLabelsName, *queryLabelsBegin, *queryLabelsEnd, p)) os.Exit(QueryLabels(serverURL, httpRoundTripper, *queryLabelsMatch, *queryLabelsName, *queryLabelsBegin, *queryLabelsEnd, p))
case testRulesCmd.FullCommand(): case testRulesCmd.FullCommand():
os.Exit(RulesUnitTest( results := io.Discard
if *junitOutFile != nil {
results = *junitOutFile
}
os.Exit(RulesUnitTestResult(results,
promqltest.LazyLoaderOpts{ promqltest.LazyLoaderOpts{
EnableAtModifier: true, EnableAtModifier: true,
EnableNegativeOffset: true, EnableNegativeOffset: true,
@ -466,7 +471,7 @@ func (ls lintConfig) lintDuplicateRules() bool {
return ls.all || ls.duplicateRules return ls.all || ls.duplicateRules
} }
// Check server status - healthy & ready. // CheckServerStatus - healthy & ready.
func CheckServerStatus(serverURL *url.URL, checkEndpoint string, roundTripper http.RoundTripper) error { func CheckServerStatus(serverURL *url.URL, checkEndpoint string, roundTripper http.RoundTripper) error {
if serverURL.Scheme == "" { if serverURL.Scheme == "" {
serverURL.Scheme = "http" serverURL.Scheme = "http"

2
cmd/promtool/metrics.go
View file

@ -31,7 +31,7 @@ import (
"github.com/prometheus/prometheus/util/fmtutil" "github.com/prometheus/prometheus/util/fmtutil"
) )
// Push metrics to a prometheus remote write (for testing purpose only). // PushMetrics to a prometheus remote write (for testing purpose only).
func PushMetrics(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, timeout time.Duration, labels map[string]string, files ...string) int { func PushMetrics(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, timeout time.Duration, labels map[string]string, files ...string) int {
addressURL, err := url.Parse(url.String()) addressURL, err := url.Parse(url.String())
if err != nil { if err != nil {

8
cmd/promtool/tsdb.go
View file

@ -866,16 +866,16 @@ func displayHistogram(dataType string, datas []int, total int) {
fmt.Println() fmt.Println()
} }
func generateBucket(min, max int) (start, end, step int) { func generateBucket(minVal, maxVal int) (start, end, step int) {
s := (max - min) / 10 s := (maxVal - minVal) / 10
step = 10 step = 10
for step < s && step <= 10000 { for step < s && step <= 10000 {
step *= 10 step *= 10
} }
start = min - min%step start = minVal - minVal%step
end = max - max%step + step end = maxVal - maxVal%step + step
return return
} }

13
cmd/promtool/tsdb_test.go
View file

@ -20,6 +20,7 @@ import (
"math" "math"
"os" "os"
"runtime" "runtime"
"slices"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -152,12 +153,18 @@ func TestTSDBDump(t *testing.T) {
expectedMetrics, err := os.ReadFile(tt.expectedDump) expectedMetrics, err := os.ReadFile(tt.expectedDump)
require.NoError(t, err) require.NoError(t, err)
expectedMetrics = normalizeNewLine(expectedMetrics) expectedMetrics = normalizeNewLine(expectedMetrics)
// even though in case of one matcher samples are not sorted, the order in the cases above should stay the same. // Sort both, because Prometheus does not guarantee the output order.
require.Equal(t, string(expectedMetrics), dumpedMetrics) require.Equal(t, sortLines(string(expectedMetrics)), sortLines(dumpedMetrics))
}) })
} }
} }
func sortLines(buf string) string {
lines := strings.Split(buf, "\n")
slices.Sort(lines)
return strings.Join(lines, "\n")
}
func TestTSDBDumpOpenMetrics(t *testing.T) { func TestTSDBDumpOpenMetrics(t *testing.T) {
storage := promqltest.LoadedStorage(t, ` storage := promqltest.LoadedStorage(t, `
load 1m load 1m
@ -169,7 +176,7 @@ func TestTSDBDumpOpenMetrics(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
expectedMetrics = normalizeNewLine(expectedMetrics) expectedMetrics = normalizeNewLine(expectedMetrics)
dumpedMetrics := getDumpedSamples(t, storage.Dir(), math.MinInt64, math.MaxInt64, []string{"{__name__=~'(?s:.*)'}"}, formatSeriesSetOpenMetrics) dumpedMetrics := getDumpedSamples(t, storage.Dir(), math.MinInt64, math.MaxInt64, []string{"{__name__=~'(?s:.*)'}"}, formatSeriesSetOpenMetrics)
require.Equal(t, string(expectedMetrics), dumpedMetrics) require.Equal(t, sortLines(string(expectedMetrics)), sortLines(dumpedMetrics))
} }
func TestTSDBDumpOpenMetricsRoundTrip(t *testing.T) { func TestTSDBDumpOpenMetricsRoundTrip(t *testing.T) {

40
cmd/promtool/unittest.go
View file

@ -18,6 +18,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
@ -29,9 +30,10 @@ import (
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/grafana/regexp" "github.com/grafana/regexp"
"github.com/nsf/jsondiff" "github.com/nsf/jsondiff"
"github.com/prometheus/common/model"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql"
@ -39,12 +41,18 @@ import (
"github.com/prometheus/prometheus/promql/promqltest" "github.com/prometheus/prometheus/promql/promqltest"
"github.com/prometheus/prometheus/rules" "github.com/prometheus/prometheus/rules"
"github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/util/junitxml"
) )
// RulesUnitTest does unit testing of rules based on the unit testing files provided. // RulesUnitTest does unit testing of rules based on the unit testing files provided.
// More info about the file format can be found in the docs. // More info about the file format can be found in the docs.
func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int { func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
return RulesUnitTestResult(io.Discard, queryOpts, runStrings, diffFlag, files...)
}
func RulesUnitTestResult(results io.Writer, queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
failed := false failed := false
junit := &junitxml.JUnitXML{}
var run *regexp.Regexp var run *regexp.Regexp
if runStrings != nil { if runStrings != nil {
@ -52,7 +60,7 @@ func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, dif
} }
for _, f := range files { for _, f := range files {
if errs := ruleUnitTest(f, queryOpts, run, diffFlag); errs != nil { if errs := ruleUnitTest(f, queryOpts, run, diffFlag, junit.Suite(f)); errs != nil {
fmt.Fprintln(os.Stderr, " FAILED:") fmt.Fprintln(os.Stderr, " FAILED:")
for _, e := range errs { for _, e := range errs {
fmt.Fprintln(os.Stderr, e.Error()) fmt.Fprintln(os.Stderr, e.Error())
@ -64,25 +72,30 @@ func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, dif
} }
fmt.Println() fmt.Println()
} }
err := junit.WriteXML(results)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to write JUnit XML: %s\n", err)
}
if failed { if failed {
return failureExitCode return failureExitCode
} }
return successExitCode return successExitCode
} }
func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error { func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool, ts *junitxml.TestSuite) []error {
fmt.Println("Unit Testing: ", filename)
b, err := os.ReadFile(filename) b, err := os.ReadFile(filename)
if err != nil { if err != nil {
ts.Abort(err)
return []error{err} return []error{err}
} }
var unitTestInp unitTestFile var unitTestInp unitTestFile
if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil { if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil {
ts.Abort(err)
return []error{err} return []error{err}
} }
if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil { if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil {
ts.Abort(err)
return []error{err} return []error{err}
} }
@ -91,29 +104,38 @@ func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *reg
} }
evalInterval := time.Duration(unitTestInp.EvaluationInterval) evalInterval := time.Duration(unitTestInp.EvaluationInterval)
ts.Settime(time.Now().Format("2006-01-02T15:04:05"))
// Giving number for groups mentioned in the file for ordering. // Giving number for groups mentioned in the file for ordering.
// Lower number group should be evaluated before higher number group. // Lower number group should be evaluated before higher number group.
groupOrderMap := make(map[string]int) groupOrderMap := make(map[string]int)
for i, gn := range unitTestInp.GroupEvalOrder { for i, gn := range unitTestInp.GroupEvalOrder {
if _, ok := groupOrderMap[gn]; ok { if _, ok := groupOrderMap[gn]; ok {
return []error{fmt.Errorf("group name repeated in evaluation order: %s", gn)} err := fmt.Errorf("group name repeated in evaluation order: %s", gn)
ts.Abort(err)
return []error{err}
} }
groupOrderMap[gn] = i groupOrderMap[gn] = i
} }
// Testing. // Testing.
var errs []error var errs []error
for _, t := range unitTestInp.Tests { for i, t := range unitTestInp.Tests {
if !matchesRun(t.TestGroupName, run) { if !matchesRun(t.TestGroupName, run) {
continue continue
} }
testname := t.TestGroupName
if testname == "" {
testname = fmt.Sprintf("unnamed#%d", i)
}
tc := ts.Case(testname)
if t.Interval == 0 { if t.Interval == 0 {
t.Interval = unitTestInp.EvaluationInterval t.Interval = unitTestInp.EvaluationInterval
} }
ers := t.test(evalInterval, groupOrderMap, queryOpts, diffFlag, unitTestInp.RuleFiles...) ers := t.test(evalInterval, groupOrderMap, queryOpts, diffFlag, unitTestInp.RuleFiles...)
if ers != nil { if ers != nil {
for _, e := range ers {
tc.Fail(e.Error())
}
errs = append(errs, ers...) errs = append(errs, ers...)
} }
} }

50
cmd/promtool/unittest_test.go
View file

@ -14,11 +14,15 @@
package main package main
import ( import (
"bytes"
"encoding/xml"
"fmt"
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/promql/promqltest" "github.com/prometheus/prometheus/promql/promqltest"
"github.com/prometheus/prometheus/util/junitxml"
) )
func TestRulesUnitTest(t *testing.T) { func TestRulesUnitTest(t *testing.T) {
@ -125,13 +129,59 @@ func TestRulesUnitTest(t *testing.T) {
want: 0, want: 0,
}, },
} }
reuseFiles := []string{}
reuseCount := [2]int{}
for _, tt := range tests { for _, tt := range tests {
if (tt.queryOpts == promqltest.LazyLoaderOpts{
EnableNegativeOffset: true,
} || tt.queryOpts == promqltest.LazyLoaderOpts{
EnableAtModifier: true,
}) {
reuseFiles = append(reuseFiles, tt.args.files...)
reuseCount[tt.want] += len(tt.args.files)
}
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
if got := RulesUnitTest(tt.queryOpts, nil, false, tt.args.files...); got != tt.want { if got := RulesUnitTest(tt.queryOpts, nil, false, tt.args.files...); got != tt.want {
t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want) t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
} }
}) })
} }
t.Run("Junit xml output ", func(t *testing.T) {
var buf bytes.Buffer
if got := RulesUnitTestResult(&buf, promqltest.LazyLoaderOpts{}, nil, false, reuseFiles...); got != 1 {
t.Errorf("RulesUnitTestResults() = %v, want 1", got)
}
var test junitxml.JUnitXML
output := buf.Bytes()
err := xml.Unmarshal(output, &test)
if err != nil {
fmt.Println("error in decoding XML:", err)
return
}
var total int
var passes int
var failures int
var cases int
total = len(test.Suites)
if total != len(reuseFiles) {
t.Errorf("JUnit output had %d testsuite elements; expected %d\n", total, len(reuseFiles))
}
for _, i := range test.Suites {
if i.FailureCount == 0 {
passes++
} else {
failures++
}
cases += len(i.Cases)
}
if total != passes+failures {
t.Errorf("JUnit output mismatch: Total testsuites (%d) does not equal the sum of passes (%d) and failures (%d).", total, passes, failures)
}
if cases < total {
t.Errorf("JUnit output had %d suites without test cases\n", total-cases)
}
})
} }
func TestRulesUnitTestRun(t *testing.T) { func TestRulesUnitTestRun(t *testing.T) {

62
config/config.go
View file

@ -37,6 +37,7 @@ import (
"github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel" "github.com/prometheus/prometheus/model/relabel"
"github.com/prometheus/prometheus/storage/remote/azuread" "github.com/prometheus/prometheus/storage/remote/azuread"
"github.com/prometheus/prometheus/storage/remote/googleiam"
) )
var ( var (
@ -66,6 +67,11 @@ var (
} }
) )
const (
LegacyValidationConfig = "legacy"
UTF8ValidationConfig = "utf8"
)
// Load parses the YAML input s into a Config. // Load parses the YAML input s into a Config.
func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, error) { func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, error) {
cfg := &Config{} cfg := &Config{}
@ -445,6 +451,8 @@ type GlobalConfig struct {
// Keep no more than this many dropped targets per job. // Keep no more than this many dropped targets per job.
// 0 means no limit. // 0 means no limit.
KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"` KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"`
// Allow UTF8 Metric and Label Names.
MetricNameValidationScheme string `yaml:"metric_name_validation_scheme,omitempty"`
} }
// ScrapeProtocol represents supported protocol for scraping metrics. // ScrapeProtocol represents supported protocol for scraping metrics.
@ -470,6 +478,7 @@ var (
PrometheusText0_0_4 ScrapeProtocol = "PrometheusText0.0.4" PrometheusText0_0_4 ScrapeProtocol = "PrometheusText0.0.4"
OpenMetricsText0_0_1 ScrapeProtocol = "OpenMetricsText0.0.1" OpenMetricsText0_0_1 ScrapeProtocol = "OpenMetricsText0.0.1"
OpenMetricsText1_0_0 ScrapeProtocol = "OpenMetricsText1.0.0" OpenMetricsText1_0_0 ScrapeProtocol = "OpenMetricsText1.0.0"
UTF8NamesHeader string = model.EscapingKey + "=" + model.AllowUTF8
ScrapeProtocolsHeaders = map[ScrapeProtocol]string{ ScrapeProtocolsHeaders = map[ScrapeProtocol]string{
PrometheusProto: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", PrometheusProto: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited",
@ -655,6 +664,8 @@ type ScrapeConfig struct {
// Keep no more than this many dropped targets per job. // Keep no more than this many dropped targets per job.
// 0 means no limit. // 0 means no limit.
KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"` KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"`
// Allow UTF8 Metric and Label Names.
MetricNameValidationScheme string `yaml:"metric_name_validation_scheme,omitempty"`
// We cannot do proper Go type embedding below as the parser will then parse // We cannot do proper Go type embedding below as the parser will then parse
// values arbitrarily into the overflow maps of further-down types. // values arbitrarily into the overflow maps of further-down types.
@ -761,6 +772,17 @@ func (c *ScrapeConfig) Validate(globalConfig GlobalConfig) error {
return fmt.Errorf("%w for scrape config with job name %q", err, c.JobName) return fmt.Errorf("%w for scrape config with job name %q", err, c.JobName)
} }
switch globalConfig.MetricNameValidationScheme {
case "", LegacyValidationConfig:
case UTF8ValidationConfig:
if model.NameValidationScheme != model.UTF8Validation {
return fmt.Errorf("utf8 name validation requested but feature not enabled via --enable-feature=utf8-names")
}
default:
return fmt.Errorf("unknown name validation method specified, must be either 'legacy' or 'utf8', got %s", globalConfig.MetricNameValidationScheme)
}
c.MetricNameValidationScheme = globalConfig.MetricNameValidationScheme
return nil return nil
} }
@ -1089,8 +1111,9 @@ func (m RemoteWriteProtoMsgs) String() string {
} }
var ( var (
// RemoteWriteProtoMsgV1 represents the deprecated `prometheus.WriteRequest` protobuf // RemoteWriteProtoMsgV1 represents the `prometheus.WriteRequest` protobuf
// message introduced in the https://prometheus.io/docs/specs/remote_write_spec/. // message introduced in the https://prometheus.io/docs/specs/remote_write_spec/,
// which will eventually be deprecated.
// //
// NOTE: This string is used for both HTTP header values and config value, so don't change // NOTE: This string is used for both HTTP header values and config value, so don't change
// this reference. // this reference.
@ -1123,6 +1146,7 @@ type RemoteWriteConfig struct {
MetadataConfig MetadataConfig `yaml:"metadata_config,omitempty"` MetadataConfig MetadataConfig `yaml:"metadata_config,omitempty"`
SigV4Config *sigv4.SigV4Config `yaml:"sigv4,omitempty"` SigV4Config *sigv4.SigV4Config `yaml:"sigv4,omitempty"`
AzureADConfig *azuread.AzureADConfig `yaml:"azuread,omitempty"` AzureADConfig *azuread.AzureADConfig `yaml:"azuread,omitempty"`
GoogleIAMConfig *googleiam.Config `yaml:"google_iam,omitempty"`
} }
// SetDirectory joins any relative file paths with dir. // SetDirectory joins any relative file paths with dir.
@ -1160,17 +1184,33 @@ func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) err
return err return err
} }
httpClientConfigAuthEnabled := c.HTTPClientConfig.BasicAuth != nil || return validateAuthConfigs(c)
c.HTTPClientConfig.Authorization != nil || c.HTTPClientConfig.OAuth2 != nil }
if httpClientConfigAuthEnabled && (c.SigV4Config != nil || c.AzureADConfig != nil) { // validateAuthConfigs validates that at most one of basic_auth, authorization, oauth2, sigv4, azuread or google_iam must be configured.
return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, & azuread must be configured") func validateAuthConfigs(c *RemoteWriteConfig) error {
var authConfigured []string
if c.HTTPClientConfig.BasicAuth != nil {
authConfigured = append(authConfigured, "basic_auth")
} }
if c.HTTPClientConfig.Authorization != nil {
if c.SigV4Config != nil && c.AzureADConfig != nil { authConfigured = append(authConfigured, "authorization")
return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, & azuread must be configured") }
if c.HTTPClientConfig.OAuth2 != nil {
authConfigured = append(authConfigured, "oauth2")
}
if c.SigV4Config != nil {
authConfigured = append(authConfigured, "sigv4")
}
if c.AzureADConfig != nil {
authConfigured = append(authConfigured, "azuread")
}
if c.GoogleIAMConfig != nil {
authConfigured = append(authConfigured, "google_iam")
}
if len(authConfigured) > 1 {
return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, azuread or google_iam must be configured. Currently configured: %v", authConfigured)
} }
return nil return nil
} }
@ -1189,7 +1229,7 @@ func validateHeadersForTracing(headers map[string]string) error {
func validateHeaders(headers map[string]string) error { func validateHeaders(headers map[string]string) error {
for header := range headers { for header := range headers {
if strings.ToLower(header) == "authorization" { if strings.ToLower(header) == "authorization" {
return errors.New("authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, or azuread parameter") return errors.New("authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, azuread or google_iam parameter")
} }
if _, ok := reservedHeaders[strings.ToLower(header)]; ok { if _, ok := reservedHeaders[strings.ToLower(header)]; ok {
return fmt.Errorf("%s is a reserved header. It must not be changed", header) return fmt.Errorf("%s is a reserved header. It must not be changed", header)

2
config/config_test.go
View file

@ -1826,7 +1826,7 @@ var expectedErrors = []struct {
}, },
{ {
filename: "remote_write_authorization_header.bad.yml", filename: "remote_write_authorization_header.bad.yml",
errMsg: `authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, or azuread parameter`, errMsg: `authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, azuread or google_iam parameter`,
}, },
{ {
filename: "remote_write_wrong_msg.bad.yml", filename: "remote_write_wrong_msg.bad.yml",

2
discovery/discoverer_metrics_noop.go
View file

@ -13,7 +13,7 @@
package discovery package discovery
// Create a dummy metrics struct, because this SD doesn't have any metrics. // NoopDiscovererMetrics creates a dummy metrics struct, because this SD doesn't have any metrics.
type NoopDiscovererMetrics struct{} type NoopDiscovererMetrics struct{}
var _ DiscovererMetrics = (*NoopDiscovererMetrics)(nil) var _ DiscovererMetrics = (*NoopDiscovererMetrics)(nil)

22
discovery/discovery.go
View file

@ -39,7 +39,7 @@ type Discoverer interface {
Run(ctx context.Context, up chan<- []*targetgroup.Group) Run(ctx context.Context, up chan<- []*targetgroup.Group)
} }
// Internal metrics of service discovery mechanisms. // DiscovererMetrics are internal metrics of service discovery mechanisms.
type DiscovererMetrics interface { type DiscovererMetrics interface {
Register() error Register() error
Unregister() Unregister()
@ -56,7 +56,7 @@ type DiscovererOptions struct {
HTTPClientOptions []config.HTTPClientOption HTTPClientOptions []config.HTTPClientOption
} }
// Metrics used by the "refresh" package. // RefreshMetrics are used by the "refresh" package.
// We define them here in the "discovery" package in order to avoid a cyclic dependency between // We define them here in the "discovery" package in order to avoid a cyclic dependency between
// "discovery" and "refresh". // "discovery" and "refresh".
type RefreshMetrics struct { type RefreshMetrics struct {
@ -64,17 +64,18 @@ type RefreshMetrics struct {
Duration prometheus.Observer Duration prometheus.Observer
} }
// Instantiate the metrics used by the "refresh" package. // RefreshMetricsInstantiator instantiates the metrics used by the "refresh" package.
type RefreshMetricsInstantiator interface { type RefreshMetricsInstantiator interface {
Instantiate(mech string) *RefreshMetrics Instantiate(mech string) *RefreshMetrics
} }
// An interface for registering, unregistering, and instantiating metrics for the "refresh" package. // RefreshMetricsManager is an interface for registering, unregistering, and
// Refresh metrics are registered and unregistered outside of the service discovery mechanism. // instantiating metrics for the "refresh" package. Refresh metrics are
// This is so that the same metrics can be reused across different service discovery mechanisms. // registered and unregistered outside of the service discovery mechanism. This
// To manage refresh metrics inside the SD mechanism, we'd need to use const labels which are // is so that the same metrics can be reused across different service discovery
// specific to that SD. However, doing so would also expose too many unused metrics on // mechanisms. To manage refresh metrics inside the SD mechanism, we'd need to
// the Prometheus /metrics endpoint. // use const labels which are specific to that SD. However, doing so would also
// expose too many unused metrics on the Prometheus /metrics endpoint.
type RefreshMetricsManager interface { type RefreshMetricsManager interface {
DiscovererMetrics DiscovererMetrics
RefreshMetricsInstantiator RefreshMetricsInstantiator
@ -145,7 +146,8 @@ func (c StaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) {
return staticDiscoverer(c), nil return staticDiscoverer(c), nil
} }
// No metrics are needed for this service discovery mechanism. // NewDiscovererMetrics returns NoopDiscovererMetrics because no metrics are
// needed for this service discovery mechanism.
func (c StaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics { func (c StaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
return &NoopDiscovererMetrics{} return &NoopDiscovererMetrics{}
} }

1
discovery/eureka/client.go
View file

@ -97,7 +97,6 @@ func fetchApps(ctx context.Context, server string, client *http.Client) (*Applic
resp.Body.Close() resp.Body.Close()
}() }()
//nolint:usestdlibvars
if resp.StatusCode/100 != 2 { if resp.StatusCode/100 != 2 {
return nil, fmt.Errorf("non 2xx status '%d' response during eureka service discovery", resp.StatusCode) return nil, fmt.Errorf("non 2xx status '%d' response during eureka service discovery", resp.StatusCode)
} }

1
discovery/hetzner/robot.go
View file

@ -87,7 +87,6 @@ func (d *robotDiscovery) refresh(context.Context) ([]*targetgroup.Group, error)
resp.Body.Close() resp.Body.Close()
}() }()
//nolint:usestdlibvars
if resp.StatusCode/100 != 2 { if resp.StatusCode/100 != 2 {
return nil, fmt.Errorf("non 2xx status '%d' response during hetzner service discovery with role robot", resp.StatusCode) return nil, fmt.Errorf("non 2xx status '%d' response during hetzner service discovery with role robot", resp.StatusCode)
} }

2
discovery/kubernetes/endpoints_test.go
View file

@ -970,7 +970,7 @@ func TestEndpointsDiscoveryEmptyPodStatus(t *testing.T) {
}.Run(t) }.Run(t)
} }
// TestEndpointsUpdatePod makes sure that Endpoints discovery detects underlying Pods changes. // TestEndpointsDiscoveryUpdatePod makes sure that Endpoints discovery detects underlying Pods changes.
// See https://github.com/prometheus/prometheus/issues/11305 for more details. // See https://github.com/prometheus/prometheus/issues/11305 for more details.
func TestEndpointsDiscoveryUpdatePod(t *testing.T) { func TestEndpointsDiscoveryUpdatePod(t *testing.T) {
pod := &v1.Pod{ pod := &v1.Pod{

8
discovery/kubernetes/kubernetes_test.go
View file

@ -154,7 +154,7 @@ func (d k8sDiscoveryTest) Run(t *testing.T) {
// readResultWithTimeout reads all targetgroups from channel with timeout. // readResultWithTimeout reads all targetgroups from channel with timeout.
// It merges targetgroups by source and sends the result to result channel. // It merges targetgroups by source and sends the result to result channel.
func readResultWithTimeout(t *testing.T, ctx context.Context, ch <-chan []*targetgroup.Group, max int, stopAfter time.Duration, resChan chan<- map[string]*targetgroup.Group) { func readResultWithTimeout(t *testing.T, ctx context.Context, ch <-chan []*targetgroup.Group, maxGroups int, stopAfter time.Duration, resChan chan<- map[string]*targetgroup.Group) {
res := make(map[string]*targetgroup.Group) res := make(map[string]*targetgroup.Group)
timeout := time.After(stopAfter) timeout := time.After(stopAfter)
Loop: Loop:
@ -167,7 +167,7 @@ Loop:
} }
res[tg.Source] = tg res[tg.Source] = tg
} }
if len(res) == max { if len(res) == maxGroups {
// Reached max target groups we may get, break fast. // Reached max target groups we may get, break fast.
break Loop break Loop
} }
@ -175,10 +175,10 @@ Loop:
// Because we use queue, an object that is created then // Because we use queue, an object that is created then
// deleted or updated may be processed only once. // deleted or updated may be processed only once.
// So possibly we may skip events, timed out here. // So possibly we may skip events, timed out here.
t.Logf("timed out, got %d (max: %d) items, some events are skipped", len(res), max) t.Logf("timed out, got %d (max: %d) items, some events are skipped", len(res), maxGroups)
break Loop break Loop
case <-ctx.Done(): case <-ctx.Done():
t.Logf("stopped, got %d (max: %d) items", len(res), max) t.Logf("stopped, got %d (max: %d) items", len(res), maxGroups)
break Loop break Loop
} }
} }

1
discovery/legacymanager/manager_test.go
View file

@ -1090,7 +1090,6 @@ func TestCoordinationWithReceiver(t *testing.T) {
} }
for _, tc := range testCases { for _, tc := range testCases {
tc := tc
t.Run(tc.title, func(t *testing.T) { t.Run(tc.title, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel() defer cancel()

2
discovery/manager.go
View file

@ -64,7 +64,7 @@ func (p *Provider) Config() interface{} {
return p.config return p.config
} }
// Registers the metrics needed for SD mechanisms. // CreateAndRegisterSDMetrics registers the metrics needed for SD mechanisms.
// Does not register the metrics for the Discovery Manager. // Does not register the metrics for the Discovery Manager.
// TODO(ptodev): Add ability to unregister the metrics? // TODO(ptodev): Add ability to unregister the metrics?
func CreateAndRegisterSDMetrics(reg prometheus.Registerer) (map[string]DiscovererMetrics, error) { func CreateAndRegisterSDMetrics(reg prometheus.Registerer) (map[string]DiscovererMetrics, error) {

2
discovery/metrics_refresh.go
View file

@ -17,7 +17,7 @@ import (
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
) )
// Metric vectors for the "refresh" package. // RefreshMetricsVecs are metric vectors for the "refresh" package.
// We define them here in the "discovery" package in order to avoid a cyclic dependency between // We define them here in the "discovery" package in order to avoid a cyclic dependency between
// "discovery" and "refresh". // "discovery" and "refresh".
type RefreshMetricsVecs struct { type RefreshMetricsVecs struct {

6
discovery/util.go
View file

@ -19,8 +19,8 @@ import (
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
) )
// A utility to be used by implementations of discovery.Discoverer // MetricRegisterer is used by implementations of discovery.Discoverer that need
// which need to manage the lifetime of their metrics. // to manage the lifetime of their metrics.
type MetricRegisterer interface { type MetricRegisterer interface {
RegisterMetrics() error RegisterMetrics() error
UnregisterMetrics() UnregisterMetrics()
@ -34,7 +34,7 @@ type metricRegistererImpl struct {
var _ MetricRegisterer = &metricRegistererImpl{} var _ MetricRegisterer = &metricRegistererImpl{}
// Creates an instance of a MetricRegisterer. // NewMetricRegisterer creates an instance of a MetricRegisterer.
// Typically called inside the implementation of the NewDiscoverer() method. // Typically called inside the implementation of the NewDiscoverer() method.
func NewMetricRegisterer(reg prometheus.Registerer, metrics []prometheus.Collector) MetricRegisterer { func NewMetricRegisterer(reg prometheus.Registerer, metrics []prometheus.Collector) MetricRegisterer {
return &metricRegistererImpl{ return &metricRegistererImpl{

7
docs/command-line/prometheus.md
View file

@ -15,11 +15,11 @@ The Prometheus monitoring server
| <code class="text-nowrap">-h</code>, <code class="text-nowrap">--help</code> | Show context-sensitive help (also try --help-long and --help-man). | | | <code class="text-nowrap">-h</code>, <code class="text-nowrap">--help</code> | Show context-sensitive help (also try --help-long and --help-man). | |
| <code class="text-nowrap">--version</code> | Show application version. | | | <code class="text-nowrap">--version</code> | Show application version. | |
| <code class="text-nowrap">--config.file</code> | Prometheus configuration file path. | `prometheus.yml` | | <code class="text-nowrap">--config.file</code> | Prometheus configuration file path. | `prometheus.yml` |
| <code class="text-nowrap">--web.listen-address</code> | Address to listen on for UI, API, and telemetry. | `0.0.0.0:9090` | | <code class="text-nowrap">--web.listen-address</code> <code class="text-nowrap">...<code class="text-nowrap"> | Address to listen on for UI, API, and telemetry. Can be repeated. | `0.0.0.0:9090` |
| <code class="text-nowrap">--auto-gomemlimit.ratio</code> | The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory | `0.9` | | <code class="text-nowrap">--auto-gomemlimit.ratio</code> | The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory | `0.9` |
| <code class="text-nowrap">--web.config.file</code> | [EXPERIMENTAL] Path to configuration file that can enable TLS or authentication. | | | <code class="text-nowrap">--web.config.file</code> | [EXPERIMENTAL] Path to configuration file that can enable TLS or authentication. | |
| <code class="text-nowrap">--web.read-timeout</code> | Maximum duration before timing out read of the request, and closing idle connections. | `5m` | | <code class="text-nowrap">--web.read-timeout</code> | Maximum duration before timing out read of the request, and closing idle connections. | `5m` |
| <code class="text-nowrap">--web.max-connections</code> | Maximum number of simultaneous connections. | `512` | | <code class="text-nowrap">--web.max-connections</code> | Maximum number of simultaneous connections across all listeners. | `512` |
| <code class="text-nowrap">--web.external-url</code> | The URL under which Prometheus is externally reachable (for example, if Prometheus is served via a reverse proxy). Used for generating relative and absolute links back to Prometheus itself. If the URL has a path portion, it will be used to prefix all HTTP endpoints served by Prometheus. If omitted, relevant URL components will be derived automatically. | | | <code class="text-nowrap">--web.external-url</code> | The URL under which Prometheus is externally reachable (for example, if Prometheus is served via a reverse proxy). Used for generating relative and absolute links back to Prometheus itself. If the URL has a path portion, it will be used to prefix all HTTP endpoints served by Prometheus. If omitted, relevant URL components will be derived automatically. | |
| <code class="text-nowrap">--web.route-prefix</code> | Prefix for the internal routes of web endpoints. Defaults to path of --web.external-url. | | | <code class="text-nowrap">--web.route-prefix</code> | Prefix for the internal routes of web endpoints. Defaults to path of --web.external-url. | |
| <code class="text-nowrap">--web.user-assets</code> | Path to static asset directory, available at /user. | | | <code class="text-nowrap">--web.user-assets</code> | Path to static asset directory, available at /user. | |
@ -56,7 +56,8 @@ The Prometheus monitoring server
| <code class="text-nowrap">--query.timeout</code> | Maximum time a query may take before being aborted. Use with server mode only. | `2m` | | <code class="text-nowrap">--query.timeout</code> | Maximum time a query may take before being aborted. Use with server mode only. | `2m` |
| <code class="text-nowrap">--query.max-concurrency</code> | Maximum number of queries executed concurrently. Use with server mode only. | `20` | | <code class="text-nowrap">--query.max-concurrency</code> | Maximum number of queries executed concurrently. Use with server mode only. | `20` |
| <code class="text-nowrap">--query.max-samples</code> | Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return. Use with server mode only. | `50000000` | | <code class="text-nowrap">--query.max-samples</code> | Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return. Use with server mode only. | `50000000` |
| <code class="text-nowrap">--enable-feature</code> | Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, created-timestamp-zero-ingestion, concurrent-rule-eval. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | | | <code class="text-nowrap">--scrape.name-escaping-scheme</code> | Method for escaping legacy invalid names when sending to Prometheus that does not support UTF-8. Can be one of "values", "underscores", or "dots". | `values` |
| <code class="text-nowrap">--enable-feature</code> <code class="text-nowrap">...<code class="text-nowrap"> | Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, utf8-names. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | |
| <code class="text-nowrap">--log.level</code> | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` | | <code class="text-nowrap">--log.level</code> | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` |
| <code class="text-nowrap">--log.format</code> | Output format of log messages. One of: [logfmt, json] | `logfmt` | | <code class="text-nowrap">--log.format</code> | Output format of log messages. One of: [logfmt, json] | `logfmt` |

23
docs/command-line/promtool.md
View file

@ -15,7 +15,7 @@ Tooling for the Prometheus monitoring system.
| <code class="text-nowrap">-h</code>, <code class="text-nowrap">--help</code> | Show context-sensitive help (also try --help-long and --help-man). | | <code class="text-nowrap">-h</code>, <code class="text-nowrap">--help</code> | Show context-sensitive help (also try --help-long and --help-man). |
| <code class="text-nowrap">--version</code> | Show application version. | | <code class="text-nowrap">--version</code> | Show application version. |
| <code class="text-nowrap">--experimental</code> | Enable experimental commands. | | <code class="text-nowrap">--experimental</code> | Enable experimental commands. |
| <code class="text-nowrap">--enable-feature</code> | Comma separated feature names to enable (only PromQL related and no-default-scrape-port). See https://prometheus.io/docs/prometheus/latest/feature_flags/ for the options and more details. | | <code class="text-nowrap">--enable-feature</code> <code class="text-nowrap">...<code class="text-nowrap"> | Comma separated feature names to enable (only PromQL related and no-default-scrape-port). See https://prometheus.io/docs/prometheus/latest/feature_flags/ for the options and more details. |
@ -281,7 +281,7 @@ Run series query.
| Flag | Description | | Flag | Description |
| --- | --- | | --- | --- |
| <code class="text-nowrap">--match</code> | Series selector. Can be specified multiple times. | | <code class="text-nowrap">--match</code> <code class="text-nowrap">...<code class="text-nowrap"> | Series selector. Can be specified multiple times. |
| <code class="text-nowrap">--start</code> | Start time (RFC3339 or Unix timestamp). | | <code class="text-nowrap">--start</code> | Start time (RFC3339 or Unix timestamp). |
| <code class="text-nowrap">--end</code> | End time (RFC3339 or Unix timestamp). | | <code class="text-nowrap">--end</code> | End time (RFC3339 or Unix timestamp). |
@ -309,7 +309,7 @@ Run labels query.
| --- | --- | | --- | --- |
| <code class="text-nowrap">--start</code> | Start time (RFC3339 or Unix timestamp). | | <code class="text-nowrap">--start</code> | Start time (RFC3339 or Unix timestamp). |
| <code class="text-nowrap">--end</code> | End time (RFC3339 or Unix timestamp). | | <code class="text-nowrap">--end</code> | End time (RFC3339 or Unix timestamp). |
| <code class="text-nowrap">--match</code> | Series selector. Can be specified multiple times. | | <code class="text-nowrap">--match</code> <code class="text-nowrap">...<code class="text-nowrap"> | Series selector. Can be specified multiple times. |
@ -338,7 +338,7 @@ Run queries against your Prometheus to analyze the usage pattern of certain metr
| <code class="text-nowrap">--type</code> | Type of metric: histogram. | | | <code class="text-nowrap">--type</code> | Type of metric: histogram. | |
| <code class="text-nowrap">--duration</code> | Time frame to analyze. | `1h` | | <code class="text-nowrap">--duration</code> | Time frame to analyze. | `1h` |
| <code class="text-nowrap">--time</code> | Query time (RFC3339 or Unix timestamp), defaults to now. | | | <code class="text-nowrap">--time</code> | Query time (RFC3339 or Unix timestamp), defaults to now. | |
| <code class="text-nowrap">--match</code> | Series selector. Can be specified multiple times. | | | <code class="text-nowrap">--match</code> <code class="text-nowrap">...<code class="text-nowrap"> | Series selector. Can be specified multiple times. | |
@ -442,6 +442,15 @@ Unit testing.
#### Flags
| Flag | Description |
| --- | --- |
| <code class="text-nowrap">--junit</code> | File path to store JUnit XML test results. |
##### `promtool test rules` ##### `promtool test rules`
Unit tests for rules. Unit tests for rules.
@ -452,7 +461,7 @@ Unit tests for rules.
| Flag | Description | Default | | Flag | Description | Default |
| --- | --- | --- | | --- | --- | --- |
| <code class="text-nowrap">--run</code> | If set, will only run test groups whose names match the regular expression. Can be specified multiple times. | | | <code class="text-nowrap">--run</code> <code class="text-nowrap">...<code class="text-nowrap"> | If set, will only run test groups whose names match the regular expression. Can be specified multiple times. | |
| <code class="text-nowrap">--diff</code> | [Experimental] Print colored differential output between expected & received output. | `false` | | <code class="text-nowrap">--diff</code> | [Experimental] Print colored differential output between expected & received output. | `false` |
@ -569,7 +578,7 @@ Dump samples from a TSDB.
| <code class="text-nowrap">--sandbox-dir-root</code> | Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end. | `data/` | | <code class="text-nowrap">--sandbox-dir-root</code> | Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end. | `data/` |
| <code class="text-nowrap">--min-time</code> | Minimum timestamp to dump. | `-9223372036854775808` | | <code class="text-nowrap">--min-time</code> | Minimum timestamp to dump. | `-9223372036854775808` |
| <code class="text-nowrap">--max-time</code> | Maximum timestamp to dump. | `9223372036854775807` | | <code class="text-nowrap">--max-time</code> | Maximum timestamp to dump. | `9223372036854775807` |
| <code class="text-nowrap">--match</code> | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` | | <code class="text-nowrap">--match</code> <code class="text-nowrap">...<code class="text-nowrap"> | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` |
@ -596,7 +605,7 @@ Dump samples from a TSDB.
| <code class="text-nowrap">--sandbox-dir-root</code> | Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end. | `data/` | | <code class="text-nowrap">--sandbox-dir-root</code> | Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end. | `data/` |
| <code class="text-nowrap">--min-time</code> | Minimum timestamp to dump. | `-9223372036854775808` | | <code class="text-nowrap">--min-time</code> | Minimum timestamp to dump. | `-9223372036854775808` |
| <code class="text-nowrap">--max-time</code> | Maximum timestamp to dump. | `9223372036854775807` | | <code class="text-nowrap">--max-time</code> | Maximum timestamp to dump. | `9223372036854775807` |
| <code class="text-nowrap">--match</code> | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` | | <code class="text-nowrap">--match</code> <code class="text-nowrap">...<code class="text-nowrap"> | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` |

38
docs/configuration/configuration.md
View file

@ -121,6 +121,11 @@ global:
# that will be kept in memory. 0 means no limit. # that will be kept in memory. 0 means no limit.
[ keep_dropped_targets: <int> | default = 0 ] [ keep_dropped_targets: <int> | default = 0 ]
# Specifies the validation scheme for metric and label names. Either blank or
# "legacy" for letters, numbers, colons, and underscores; or "utf8" for full
# UTF-8 support.
[ metric_name_validation_scheme <string> | default "legacy" ]
runtime: runtime:
# Configure the Go garbage collector GOGC parameter # Configure the Go garbage collector GOGC parameter
# See: https://tip.golang.org/doc/gc-guide#GOGC # See: https://tip.golang.org/doc/gc-guide#GOGC
@ -461,6 +466,11 @@ metric_relabel_configs:
# that will be kept in memory. 0 means no limit. # that will be kept in memory. 0 means no limit.
[ keep_dropped_targets: <int> | default = 0 ] [ keep_dropped_targets: <int> | default = 0 ]
# Specifies the validation scheme for metric and label names. Either blank or
# "legacy" for letters, numbers, colons, and underscores; or "utf8" for full
# UTF-8 support.
[ metric_name_validation_scheme <string> | default "legacy" ]
# Limit on total number of positive and negative buckets allowed in a single # Limit on total number of positive and negative buckets allowed in a single
# native histogram. The resolution of a histogram with more buckets will be # native histogram. The resolution of a histogram with more buckets will be
# reduced until the number of buckets is within the limit. If the limit cannot # reduced until the number of buckets is within the limit. If the limit cannot
@ -3265,12 +3275,16 @@ Initially, aside from the configured per-target labels, a target's `job`
label is set to the `job_name` value of the respective scrape configuration. label is set to the `job_name` value of the respective scrape configuration.
The `__address__` label is set to the `<host>:<port>` address of the target. The `__address__` label is set to the `<host>:<port>` address of the target.
After relabeling, the `instance` label is set to the value of `__address__` by default if After relabeling, the `instance` label is set to the value of `__address__` by default if
it was not set during relabeling. The `__scheme__` and `__metrics_path__` labels it was not set during relabeling.
are set to the scheme and metrics path of the target respectively. The `__param_<name>`
label is set to the value of the first passed URL parameter called `<name>`. The `__scheme__` and `__metrics_path__` labels
are set to the scheme and metrics path of the target respectively, as specified in `scrape_config`.
The `__param_<name>`
label is set to the value of the first passed URL parameter called `<name>`, as defined in `scrape_config`.
The `__scrape_interval__` and `__scrape_timeout__` labels are set to the target's The `__scrape_interval__` and `__scrape_timeout__` labels are set to the target's
interval and timeout. interval and timeout, as specified in `scrape_config`.
Additional labels prefixed with `__meta_` may be available during the Additional labels prefixed with `__meta_` may be available during the
relabeling phase. They are set by the service discovery mechanism that provided relabeling phase. They are set by the service discovery mechanism that provided
@ -3401,8 +3415,8 @@ authorization:
# It is mutually exclusive with `credentials`. # It is mutually exclusive with `credentials`.
[ credentials_file: <filename> ] [ credentials_file: <filename> ]
# Optionally configures AWS's Signature Verification 4 signing process to # Optionally configures AWS's Signature Verification 4 signing process to sign requests.
# sign requests. Cannot be set at the same time as basic_auth, authorization, or oauth2. # Cannot be set at the same time as basic_auth, authorization, oauth2, azuread or google_iam.
# To use the default credentials from the AWS SDK, use `sigv4: {}`. # To use the default credentials from the AWS SDK, use `sigv4: {}`.
sigv4: sigv4:
# The AWS region. If blank, the region from the default credentials chain # The AWS region. If blank, the region from the default credentials chain
@ -3655,12 +3669,12 @@ sigv4:
[ role_arn: <string> ] [ role_arn: <string> ]
# Optional OAuth 2.0 configuration. # Optional OAuth 2.0 configuration.
# Cannot be used at the same time as basic_auth, authorization, sigv4, or azuread. # Cannot be used at the same time as basic_auth, authorization, sigv4, azuread or google_iam.
oauth2: oauth2:
[ <oauth2> ] [ <oauth2> ]
# Optional AzureAD configuration. # Optional AzureAD configuration.
# Cannot be used at the same time as basic_auth, authorization, oauth2, or sigv4. # Cannot be used at the same time as basic_auth, authorization, oauth2, sigv4 or google_iam.
azuread: azuread:
# The Azure Cloud. Options are 'AzurePublic', 'AzureChina', or 'AzureGovernment'. # The Azure Cloud. Options are 'AzurePublic', 'AzureChina', or 'AzureGovernment'.
[ cloud: <string> | default = AzurePublic ] [ cloud: <string> | default = AzurePublic ]
@ -3680,6 +3694,14 @@ azuread:
[ sdk: [ sdk:
[ tenant_id: <string> ] ] [ tenant_id: <string> ] ]
# WARNING: Remote write is NOT SUPPORTED by Google Cloud. This configuration is reserved for future use.
# Optional Google Cloud Monitoring configuration.
# Cannot be used at the same time as basic_auth, authorization, oauth2, sigv4 or azuread.
# To use the default credentials from the Google Cloud SDK, use `google_iam: {}`.
google_iam:
# Service account key with monitoring write permessions.
credentials_file: <file_name>
# Configures the remote write request's TLS settings. # Configures the remote write request's TLS settings.
tls_config: tls_config:
[ <tls_config> ] [ <tls_config> ]

4
docs/configuration/unit_testing_rules.md
View file

@ -92,7 +92,7 @@ series: <string>
# #
# Native histogram notation: # Native histogram notation:
# Native histograms can be used instead of floating point numbers using the following notation: # Native histograms can be used instead of floating point numbers using the following notation:
# {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5}} # {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5 counter_reset_hint:gauge}}
# Native histograms support the same expanding notation as floating point numbers, i.e. 'axn', 'a+bxn' and 'a-bxn'. # Native histograms support the same expanding notation as floating point numbers, i.e. 'axn', 'a+bxn' and 'a-bxn'.
# All properties are optional and default to 0. The order is not important. The following properties are supported: # All properties are optional and default to 0. The order is not important. The following properties are supported:
# - schema (int): # - schema (int):
@ -119,6 +119,8 @@ series: <string>
# Observation counts in negative buckets. Each represents an absolute count. # Observation counts in negative buckets. Each represents an absolute count.
# - n_offset (int): # - n_offset (int):
# The starting index of the first entry in the negative buckets. # The starting index of the first entry in the negative buckets.
# - counter_reset_hint (one of 'unknown', 'reset', 'not_reset' or 'gauge')
# The counter reset hint associated with this histogram. Defaults to 'unknown' if not set.
values: <string> values: <string>
``` ```

27
docs/feature_flags.md
View file

@ -200,8 +200,9 @@ won't work when you push OTLP metrics.
`--enable-feature=promql-experimental-functions` `--enable-feature=promql-experimental-functions`
Enables PromQL functions that are considered experimental and whose name or Enables PromQL functions that are considered experimental. These functions
semantics could change. might change their name, syntax, or semantics. They might also get removed
entirely.
## Created Timestamps Zero Injection ## Created Timestamps Zero Injection
@ -234,3 +235,25 @@ metadata changes as WAL records on a per-series basis.
This must be used if This must be used if
you are also using remote write 2.0 as it will only gather metadata from the WAL. you are also using remote write 2.0 as it will only gather metadata from the WAL.
## Delay compaction start time
`--enable-feature=delayed-compaction`
A random offset, up to `10%` of the chunk range, is added to the Head compaction start time. This assists Prometheus instances in avoiding simultaneous compactions and reduces the load on shared resources.
Only auto Head compactions and the operations directly resulting from them are subject to this delay.
In the event of multiple consecutive Head compactions being possible, only the first compaction experiences this delay.
Note that during this delay, the Head continues its usual operations, which include serving and appending series.
Despite the delay in compaction, the blocks produced are time-aligned in the same manner as they would be if the delay was not in place.
## UTF-8 Name Support
`--enable-feature=utf8-names`
When enabled, changes the metric and label name validation scheme inside Prometheus to allow the full UTF-8 character set.
By itself, this flag does not enable the request of UTF-8 names via content negotiation.
Users will also have to set `metric_name_validation_scheme` in scrape configs to enable the feature either on the global config or on a per-scrape config basis.

8
docs/querying/functions.md
View file

@ -617,9 +617,9 @@ Like `sort`, `sort_desc` only affects the results of instant queries, as range q
## `sort_by_label()` ## `sort_by_label()`
**This function has to be enabled via the [feature flag](../feature_flags/) `--enable-feature=promql-experimental-functions`.** **This function has to be enabled via the [feature flag](../feature_flags.md#experimental-promql-functions) `--enable-feature=promql-experimental-functions`.**
`sort_by_label(v instant-vector, label string, ...)` returns vector elements sorted by their label values and sample value in case of label values being equal, in ascending order. `sort_by_label(v instant-vector, label string, ...)` returns vector elements sorted by the values of the given labels in ascending order. In case these label values are equal, elements are sorted by their full label sets.
Please note that the sort by label functions only affect the results of instant queries, as range query results always have a fixed output ordering. Please note that the sort by label functions only affect the results of instant queries, as range query results always have a fixed output ordering.
@ -627,7 +627,7 @@ This function uses [natural sort order](https://en.wikipedia.org/wiki/Natural_so
## `sort_by_label_desc()` ## `sort_by_label_desc()`
**This function has to be enabled via the [feature flag](../feature_flags/) `--enable-feature=promql-experimental-functions`.** **This function has to be enabled via the [feature flag](../feature_flags.md#experimental-promql-functions) `--enable-feature=promql-experimental-functions`.**
Same as `sort_by_label`, but sorts in descending order. Same as `sort_by_label`, but sorts in descending order.
@ -676,7 +676,7 @@ over time and return an instant vector with per-series aggregation results:
* `last_over_time(range-vector)`: the most recent point value in the specified interval. * `last_over_time(range-vector)`: the most recent point value in the specified interval.
* `present_over_time(range-vector)`: the value 1 for any series in the specified interval. * `present_over_time(range-vector)`: the value 1 for any series in the specified interval.
If the [feature flag](../feature_flags/) If the [feature flag](../feature_flags.md#experimental-promql-functions)
`--enable-feature=promql-experimental-functions` is set, the following `--enable-feature=promql-experimental-functions` is set, the following
additional functions are available: additional functions are available:

2
documentation/examples/remote_storage/example_write_adapter/README.md
View file

@ -19,7 +19,7 @@ remote_write:
protobuf_message: "io.prometheus.write.v2.Request" protobuf_message: "io.prometheus.write.v2.Request"
``` ```
or for deprecated Remote Write 1.0 message: or for the eventually deprecated Remote Write 1.0 message:
```yaml ```yaml
remote_write: remote_write:

9
documentation/examples/remote_storage/go.mod
View file

@ -16,11 +16,11 @@ require (
require ( require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
github.com/aws/aws-sdk-go v1.53.16 // indirect github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@ -36,7 +36,6 @@ require (
github.com/jpillora/backoff v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/compress v1.17.8 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect
@ -63,7 +62,7 @@ require (
golang.org/x/text v0.16.0 // indirect golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.5.0 // indirect golang.org/x/time v0.5.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
google.golang.org/grpc v1.64.0 // indirect google.golang.org/grpc v1.65.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect

27
documentation/examples/remote_storage/go.sum
View file

@ -1,9 +1,9 @@
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0 h1:sUFnFjzDUie80h24I7mrKtwCKgLY9L8h5Tp2x9+TWqk= github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0/go.mod h1:52JbnQTp15qg5mRkMBHwp0j0ZFwHJ42Sx3zVV5RE9p0= github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA=
@ -26,8 +26,8 @@ github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8V
github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.53.16 h1:8oZjKQO/ml1WLUZw5hvF7pvYjPf8o9f57Wldoy/q9Qc= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
github.com/aws/aws-sdk-go v1.53.16/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps= github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0= github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@ -37,9 +37,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc= github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw=
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@ -50,8 +49,6 @@ github.com/digitalocean/godo v1.117.0 h1:WVlTe09melDYTd7VCVyvHcNWbgB+uI1O115+5LO
github.com/digitalocean/godo v1.117.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo= github.com/digitalocean/godo v1.117.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo=
github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo= github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo=
github.com/docker/docker v26.1.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v26.1.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
@ -279,8 +276,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/prometheus/prometheus v0.53.1-0.20240704074759-c137febfcf8c h1:6GEA48LnonkYZhQ654v7QTIP5uBTbCEVm49oIhif5lc= github.com/prometheus/prometheus v0.53.1-0.20240704074759-c137febfcf8c h1:6GEA48LnonkYZhQ654v7QTIP5uBTbCEVm49oIhif5lc=
github.com/prometheus/prometheus v0.53.1-0.20240704074759-c137febfcf8c/go.mod h1:FcNs5wa7M9yV8IlxlB/05s5oy9vULUIlu/tZsviRIT8= github.com/prometheus/prometheus v0.53.1-0.20240704074759-c137febfcf8c/go.mod h1:FcNs5wa7M9yV8IlxlB/05s5oy9vULUIlu/tZsviRIT8=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.27 h1:yGAraK1uUjlhSXgNMIy8o/J4LFNcy7yeipBqt9N9mVg= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.27 h1:yGAraK1uUjlhSXgNMIy8o/J4LFNcy7yeipBqt9N9mVg=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.27/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.27/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@ -405,8 +402,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

32
go.mod
View file

@ -13,12 +13,12 @@ require (
github.com/KimMachineGun/automemlimit v0.6.1 github.com/KimMachineGun/automemlimit v0.6.1
github.com/alecthomas/kingpin/v2 v2.4.0 github.com/alecthomas/kingpin/v2 v2.4.0
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30
github.com/aws/aws-sdk-go v1.54.19 github.com/aws/aws-sdk-go v1.55.5
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3
github.com/cespare/xxhash/v2 v2.3.0 github.com/cespare/xxhash/v2 v2.3.0
github.com/dennwc/varint v1.0.0 github.com/dennwc/varint v1.0.0
github.com/digitalocean/godo v1.118.0 github.com/digitalocean/godo v1.119.0
github.com/docker/docker v27.0.3+incompatible github.com/docker/docker v27.1.1+incompatible
github.com/edsrzf/mmap-go v1.1.0 github.com/edsrzf/mmap-go v1.1.0
github.com/envoyproxy/go-control-plane v0.12.0 github.com/envoyproxy/go-control-plane v0.12.0
github.com/envoyproxy/protoc-gen-validate v1.0.4 github.com/envoyproxy/protoc-gen-validate v1.0.4
@ -33,17 +33,17 @@ require (
github.com/google/go-cmp v0.6.0 github.com/google/go-cmp v0.6.0
github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da
github.com/google/uuid v1.6.0 github.com/google/uuid v1.6.0
github.com/gophercloud/gophercloud v1.13.0 github.com/gophercloud/gophercloud v1.14.0
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc
github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0
github.com/hashicorp/consul/api v1.29.2 github.com/hashicorp/consul/api v1.29.2
github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3 github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3
github.com/hetznercloud/hcloud-go/v2 v2.10.2 github.com/hetznercloud/hcloud-go/v2 v2.12.0
github.com/ionos-cloud/sdk-go/v6 v6.1.11 github.com/ionos-cloud/sdk-go/v6 v6.2.0
github.com/json-iterator/go v1.1.12 github.com/json-iterator/go v1.1.12
github.com/klauspost/compress v1.17.9 github.com/klauspost/compress v1.17.9
github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b
github.com/linode/linodego v1.37.0 github.com/linode/linodego v1.38.0
github.com/miekg/dns v1.1.61 github.com/miekg/dns v1.1.61
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
@ -75,15 +75,14 @@ require (
go.uber.org/automaxprocs v1.5.3 go.uber.org/automaxprocs v1.5.3
go.uber.org/goleak v1.3.0 go.uber.org/goleak v1.3.0
go.uber.org/multierr v1.11.0 go.uber.org/multierr v1.11.0
golang.org/x/net v0.27.0
golang.org/x/oauth2 v0.21.0 golang.org/x/oauth2 v0.21.0
golang.org/x/sync v0.7.0 golang.org/x/sync v0.7.0
golang.org/x/sys v0.22.0 golang.org/x/sys v0.22.0
golang.org/x/text v0.16.0 golang.org/x/text v0.16.0
golang.org/x/time v0.5.0 golang.org/x/time v0.5.0
golang.org/x/tools v0.23.0 golang.org/x/tools v0.23.0
google.golang.org/api v0.188.0 google.golang.org/api v0.190.0
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f
google.golang.org/grpc v1.65.0 google.golang.org/grpc v1.65.0
google.golang.org/protobuf v1.34.2 google.golang.org/protobuf v1.34.2
gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v2 v2.4.0
@ -96,9 +95,9 @@ require (
) )
require ( require (
cloud.google.com/go/auth v0.7.0 // indirect cloud.google.com/go/auth v0.7.3 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
cloud.google.com/go/compute/metadata v0.4.0 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect
@ -140,9 +139,9 @@ require (
github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-querystring v1.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect github.com/google/s2a-go v0.1.8 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/hashicorp/cronexpr v1.1.2 // indirect github.com/hashicorp/cronexpr v1.1.2 // indirect
@ -190,8 +189,9 @@ require (
golang.org/x/crypto v0.25.0 // indirect golang.org/x/crypto v0.25.0 // indirect
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
golang.org/x/mod v0.19.0 // indirect golang.org/x/mod v0.19.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/term v0.22.0 // indirect golang.org/x/term v0.22.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect
gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect
gotest.tools/v3 v3.0.3 // indirect gotest.tools/v3 v3.0.3 // indirect

60
go.sum
View file

@ -12,18 +12,18 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP
cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
cloud.google.com/go/auth v0.7.0 h1:kf/x9B3WTbBUHkC+1VS8wwwli9TzhSt0vSTVBmMR8Ts= cloud.google.com/go/auth v0.7.3 h1:98Vr+5jMaCZ5NZk6e/uBgf60phTk/XN84r8QEWB9yjY=
cloud.google.com/go/auth v0.7.0/go.mod h1:D+WqdrpcjmiCgWrXmLLxOVq1GACoE36chW6KXoEvuIw= cloud.google.com/go/auth v0.7.3/go.mod h1:HJtWUx1P5eqjy/f6Iq5KeytNpbAcGolPhOgyop2LlzA=
cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI=
cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/compute/metadata v0.4.0 h1:vHzJCWaM4g8XIcm8kopr3XmDA4Gy/lblD3EhhSux05c= cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
cloud.google.com/go/compute/metadata v0.4.0/go.mod h1:SIQh1Kkb4ZJ8zJ874fqVkslA29PRXuleyj6vOzlbK7M= cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
@ -92,8 +92,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:W
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps= github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0= github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
@ -143,14 +143,14 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=
github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/digitalocean/godo v1.118.0 h1:lkzGFQmACrVCp7UqH1sAi4JK/PWwlc5aaxubgorKmC4= github.com/digitalocean/godo v1.119.0 h1:dmFNQwSIAcH3z+FVovHLkazKDC2uA8oOlGvg5+H4vRw=
github.com/digitalocean/godo v1.118.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo= github.com/digitalocean/godo v1.119.0/go.mod h1:WQVH83OHUy6gC4gXpEVQKtxTd4L5oCp+5OialidkPLY=
github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/docker/docker v27.0.3+incompatible h1:aBGI9TeQ4MPlhquTQKq9XbK79rKFVwXNUAYz9aXyEBE= github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY=
github.com/docker/docker v27.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@ -322,8 +322,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da h1:xRmpO92tb8y+Z85iUOMOicpCfaYcv7o3Cg3wKrIpg8g= github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da h1:xRmpO92tb8y+Z85iUOMOicpCfaYcv7o3Cg3wKrIpg8g=
github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM=
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@ -332,10 +332,10 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF
github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s=
github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A=
github.com/gophercloud/gophercloud v1.13.0 h1:8iY9d1DAbzMW6Vok1AxbbK5ZaUjzMp0tdyt4fX9IeJ0= github.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8=
github.com/gophercloud/gophercloud v1.13.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@ -414,8 +414,8 @@ github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3/go.mod h1:svtx
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
github.com/hetznercloud/hcloud-go/v2 v2.10.2 h1:9gyTUPhfNbfbS40Spgij5mV5k37bOZgt8iHKCbfGs5I= github.com/hetznercloud/hcloud-go/v2 v2.12.0 h1:nOgfNTo0gyXZJJdM8mo/XH5MO/e80wAEpldRzdWayhY=
github.com/hetznercloud/hcloud-go/v2 v2.10.2/go.mod h1:xQ+8KhIS62W0D78Dpi57jsufWh844gUw1az5OUvaeq8= github.com/hetznercloud/hcloud-go/v2 v2.12.0/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOFFddpfBm/6R1Zz0IiF0=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@ -423,8 +423,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/ionos-cloud/sdk-go/v6 v6.1.11 h1:J/uRN4UWO3wCyGOeDdMKv8LWRzKu6UIkLEaes38Kzh8= github.com/ionos-cloud/sdk-go/v6 v6.2.0 h1:qX7gachC0wJSmFfVRnd+DHmz9AStvVraKcwQ/JokIB4=
github.com/ionos-cloud/sdk-go/v6 v6.1.11/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k= github.com/ionos-cloud/sdk-go/v6 v6.2.0/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k=
github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@ -472,8 +472,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
github.com/linode/linodego v1.37.0 h1:B/2Spzv9jYXzKA+p+GD8fVCNJ7Wuw6P91ZDD9eCkkso= github.com/linode/linodego v1.38.0 h1:wP3oW9OhGc6vhze8NPf2knbwH4TzSbrjzuCd9okjbTY=
github.com/linode/linodego v1.37.0/go.mod h1:L7GXKFD3PoN2xSEtFc04wIXP5WK65O10jYQx0PQISWQ= github.com/linode/linodego v1.38.0/go.mod h1:L7GXKFD3PoN2xSEtFc04wIXP5WK65O10jYQx0PQISWQ=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
@ -1047,8 +1047,8 @@ google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.188.0 h1:51y8fJ/b1AaaBRJr4yWm96fPcuxSo0JcegXE3DaHQHw= google.golang.org/api v0.190.0 h1:ASM+IhLY1zljNdLu19W1jTmU6A+gMk6M46Wlur61s+Q=
google.golang.org/api v0.188.0/go.mod h1:VR0d+2SIiWOYG3r/jdm7adPW9hI2aRv9ETOSCQ9Beag= google.golang.org/api v0.190.0/go.mod h1:QIr6I9iedBLnfqoD6L6Vze1UvS5Hzj5r2aUBOaZnLHo=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -1085,10 +1085,10 @@ google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1m
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY= google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f h1:b1Ln/PG8orm0SsBbHZWke8dDp2lrCD4jSmfglFpTZbk=
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:AHT0dDg3SoMOgZGnZk29b5xTbPHMoEC8qthmBLJCpys=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b h1:04+jVzTs2XBnOZcPsLnmrTGqltqJbZQ1Ey26hjYdQQ0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf h1:liao9UHurZLtiEwBgT9LMOnKYsHze6eA6w1KQCMVN2Q=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=

6
model/exemplar/exemplar.go
View file

@ -15,7 +15,9 @@ package exemplar
import "github.com/prometheus/prometheus/model/labels" import "github.com/prometheus/prometheus/model/labels"
// The combined length of the label names and values of an Exemplar's LabelSet MUST NOT exceed 128 UTF-8 characters // ExemplarMaxLabelSetLength is defined by OpenMetrics: "The combined length of
// the label names and values of an Exemplar's LabelSet MUST NOT exceed 128
// UTF-8 characters."
// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars // https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars
const ExemplarMaxLabelSetLength = 128 const ExemplarMaxLabelSetLength = 128
@ -49,7 +51,7 @@ func (e Exemplar) Equals(e2 Exemplar) bool {
return e.Value == e2.Value return e.Value == e2.Value
} }
// Sort first by timestamp, then value, then labels. // Compare first timestamps, then values, then labels.
func Compare(a, b Exemplar) int { func Compare(a, b Exemplar) int {
if a.Ts < b.Ts { if a.Ts < b.Ts {
return -1 return -1

11
model/labels/labels.go
View file

@ -315,7 +315,8 @@ func Compare(a, b Labels) int {
return len(a) - len(b) return len(a) - len(b)
} }
// Copy labels from b on top of whatever was in ls previously, reusing memory or expanding if needed. // CopyFrom copies labels from b on top of whatever was in ls previously,
// reusing memory or expanding if needed.
func (ls *Labels) CopyFrom(b Labels) { func (ls *Labels) CopyFrom(b Labels) {
(*ls) = append((*ls)[:0], b...) (*ls) = append((*ls)[:0], b...)
} }
@ -422,7 +423,7 @@ type ScratchBuilder struct {
add Labels add Labels
} }
// Symbol-table is no-op, just for api parity with dedupelabels. // SymbolTable is no-op, just for api parity with dedupelabels.
type SymbolTable struct{} type SymbolTable struct{}
func NewSymbolTable() *SymbolTable { return nil } func NewSymbolTable() *SymbolTable { return nil }
@ -458,7 +459,7 @@ func (b *ScratchBuilder) Add(name, value string) {
b.add = append(b.add, Label{Name: name, Value: value}) b.add = append(b.add, Label{Name: name, Value: value})
} }
// Add a name/value pair, using []byte instead of string. // UnsafeAddBytes adds a name/value pair, using []byte instead of string.
// The '-tags stringlabels' version of this function is unsafe, hence the name. // The '-tags stringlabels' version of this function is unsafe, hence the name.
// This version is safe - it copies the strings immediately - but we keep the same name so everything compiles. // This version is safe - it copies the strings immediately - but we keep the same name so everything compiles.
func (b *ScratchBuilder) UnsafeAddBytes(name, value []byte) { func (b *ScratchBuilder) UnsafeAddBytes(name, value []byte) {
@ -475,14 +476,14 @@ func (b *ScratchBuilder) Assign(ls Labels) {
b.add = append(b.add[:0], ls...) // Copy on top of our slice, so we don't retain the input slice. b.add = append(b.add[:0], ls...) // Copy on top of our slice, so we don't retain the input slice.
} }
// Return the name/value pairs added so far as a Labels object. // Labels returns the name/value pairs added so far as a Labels object.
// Note: if you want them sorted, call Sort() first. // Note: if you want them sorted, call Sort() first.
func (b *ScratchBuilder) Labels() Labels { func (b *ScratchBuilder) Labels() Labels {
// Copy the slice, so the next use of ScratchBuilder doesn't overwrite. // Copy the slice, so the next use of ScratchBuilder doesn't overwrite.
return append([]Label{}, b.add...) return append([]Label{}, b.add...)
} }
// Write the newly-built Labels out to ls. // Overwrite the newly-built Labels out to ls.
// Callers must ensure that there are no other references to ls, or any strings fetched from it. // Callers must ensure that there are no other references to ls, or any strings fetched from it.
func (b *ScratchBuilder) Overwrite(ls *Labels) { func (b *ScratchBuilder) Overwrite(ls *Labels) {
*ls = append((*ls)[:0], b.add...) *ls = append((*ls)[:0], b.add...)

4
model/relabel/relabel.go
View file

@ -213,6 +213,10 @@ func (re Regexp) IsZero() bool {
// String returns the original string used to compile the regular expression. // String returns the original string used to compile the regular expression.
func (re Regexp) String() string { func (re Regexp) String() string {
if re.Regexp == nil {
return ""
}
str := re.Regexp.String() str := re.Regexp.String()
// Trim the anchor `^(?:` prefix and `)$` suffix. // Trim the anchor `^(?:` prefix and `)$` suffix.
return str[4 : len(str)-2] return str[4 : len(str)-2]

13
model/relabel/relabel_test.go
View file

@ -900,3 +900,16 @@ action: replace
}) })
} }
} }
func TestRegexp_ShouldMarshalAndUnmarshalZeroValue(t *testing.T) {
var zero Regexp
marshalled, err := yaml.Marshal(&zero)
require.NoError(t, err)
require.Equal(t, "null\n", string(marshalled))
var unmarshalled Regexp
err = yaml.Unmarshal(marshalled, &unmarshalled)
require.NoError(t, err)
require.Nil(t, unmarshalled.Regexp)
}

4
model/textparse/interface.go
View file

@ -106,8 +106,8 @@ const (
EntryInvalid Entry = -1 EntryInvalid Entry = -1
EntryType Entry = 0 EntryType Entry = 0
EntryHelp Entry = 1 EntryHelp Entry = 1
EntrySeries Entry = 2 // A series with a simple float64 as value. EntrySeries Entry = 2 // EntrySeries marks a series with a simple float64 as value.
EntryComment Entry = 3 EntryComment Entry = 3
EntryUnit Entry = 4 EntryUnit Entry = 4
EntryHistogram Entry = 5 // A series with a native histogram as a value. EntryHistogram Entry = 5 // EntryHistogram marks a series with a native histogram as a value.
) )

173
model/textparse/openmetricsparse.go
View file

@ -94,14 +94,44 @@ type OpenMetricsParser struct {
exemplarVal float64 exemplarVal float64
exemplarTs int64 exemplarTs int64
hasExemplarTs bool hasExemplarTs bool
skipCTSeries bool
} }
// NewOpenMetricsParser returns a new parser of the byte slice. type openMetricsParserOptions struct {
func NewOpenMetricsParser(b []byte, st *labels.SymbolTable) Parser { SkipCTSeries bool
return &OpenMetricsParser{ }
type OpenMetricsOption func(*openMetricsParserOptions)
// WithOMParserCTSeriesSkipped turns off exposing _created lines
// as series, which makes those only used for parsing created timestamp
// for `CreatedTimestamp` method purposes.
//
// It's recommended to use this option to avoid using _created lines for other
// purposes than created timestamp, but leave false by default for the
// best-effort compatibility.
func WithOMParserCTSeriesSkipped() OpenMetricsOption {
return func(o *openMetricsParserOptions) {
o.SkipCTSeries = true
}
}
// NewOpenMetricsParser returns a new parser for the byte slice with option to skip CT series parsing.
func NewOpenMetricsParser(b []byte, st *labels.SymbolTable, opts ...OpenMetricsOption) Parser {
options := &openMetricsParserOptions{}
for _, opt := range opts {
opt(options)
}
parser := &OpenMetricsParser{
l: &openMetricsLexer{b: b}, l: &openMetricsLexer{b: b},
builder: labels.NewScratchBuilderWithSymbolTable(st, 16), builder: labels.NewScratchBuilderWithSymbolTable(st, 16),
skipCTSeries: options.SkipCTSeries,
} }
return parser
} }
// Series returns the bytes of the series, the timestamp if set, and the value // Series returns the bytes of the series, the timestamp if set, and the value
@ -219,10 +249,90 @@ func (p *OpenMetricsParser) Exemplar(e *exemplar.Exemplar) bool {
return true return true
} }
// CreatedTimestamp returns nil as it's not implemented yet. // CreatedTimestamp returns the created timestamp for a current Metric if exists or nil.
// TODO(bwplotka): https://github.com/prometheus/prometheus/issues/12980 // NOTE(Maniktherana): Might use additional CPU/mem resources due to deep copy of parser required for peeking given 1.0 OM specification on _created series.
func (p *OpenMetricsParser) CreatedTimestamp() *int64 { func (p *OpenMetricsParser) CreatedTimestamp() *int64 {
if !TypeRequiresCT(p.mtype) {
// Not a CT supported metric type, fast path.
return nil return nil
}
var (
currLset labels.Labels
buf []byte
peekWithoutNameLsetHash uint64
)
p.Metric(&currLset)
currFamilyLsetHash, buf := currLset.HashWithoutLabels(buf, labels.MetricName, "le", "quantile")
// Search for the _created line for the currFamilyLsetHash using ephemeral parser until
// we see EOF or new metric family. We have to do it as we don't know where (and if)
// that CT line is.
// TODO(bwplotka): Make sure OM 1.1/2.0 pass CT via metadata or exemplar-like to avoid this.
peek := deepCopy(p)
for {
eType, err := peek.Next()
if err != nil {
// This means peek will give error too later on, so def no CT line found.
// This might result in partial scrape with wrong/missing CT, but only
// spec improvement would help.
// TODO(bwplotka): Make sure OM 1.1/2.0 pass CT via metadata or exemplar-like to avoid this.
return nil
}
if eType != EntrySeries {
// Assume we hit different family, no CT line found.
return nil
}
var peekedLset labels.Labels
peek.Metric(&peekedLset)
peekedName := peekedLset.Get(model.MetricNameLabel)
if !strings.HasSuffix(peekedName, "_created") {
// Not a CT line, search more.
continue
}
// We got a CT line here, but let's search if CT line is actually for our series, edge case.
peekWithoutNameLsetHash, _ = peekedLset.HashWithoutLabels(buf, labels.MetricName, "le", "quantile")
if peekWithoutNameLsetHash != currFamilyLsetHash {
// CT line for a different series, for our series no CT.
return nil
}
ct := int64(peek.val)
return &ct
}
}
// TypeRequiresCT returns true if the metric type requires a _created timestamp.
func TypeRequiresCT(t model.MetricType) bool {
switch t {
case model.MetricTypeCounter, model.MetricTypeSummary, model.MetricTypeHistogram:
return true
default:
return false
}
}
// deepCopy creates a copy of a parser without re-using the slices' original memory addresses.
func deepCopy(p *OpenMetricsParser) OpenMetricsParser {
newB := make([]byte, len(p.l.b))
copy(newB, p.l.b)
newLexer := &openMetricsLexer{
b: newB,
i: p.l.i,
start: p.l.start,
err: p.l.err,
state: p.l.state,
}
newParser := OpenMetricsParser{
l: newLexer,
builder: p.builder,
mtype: p.mtype,
val: p.val,
skipCTSeries: false,
}
return newParser
} }
// nextToken returns the next token from the openMetricsLexer. // nextToken returns the next token from the openMetricsLexer.
@ -337,7 +447,13 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
} }
p.series = p.l.b[p.start:p.l.i] p.series = p.l.b[p.start:p.l.i]
return p.parseMetricSuffix(p.nextToken()) if err := p.parseSeriesEndOfLine(p.nextToken()); err != nil {
return EntryInvalid, err
}
if p.skipCTSeries && p.isCreatedSeries() {
return p.Next()
}
return EntrySeries, nil
case tMName: case tMName:
p.offsets = append(p.offsets, p.start, p.l.i) p.offsets = append(p.offsets, p.start, p.l.i)
p.series = p.l.b[p.start:p.l.i] p.series = p.l.b[p.start:p.l.i]
@ -351,8 +467,14 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
p.series = p.l.b[p.start:p.l.i] p.series = p.l.b[p.start:p.l.i]
t2 = p.nextToken() t2 = p.nextToken()
} }
return p.parseMetricSuffix(t2)
if err := p.parseSeriesEndOfLine(t2); err != nil {
return EntryInvalid, err
}
if p.skipCTSeries && p.isCreatedSeries() {
return p.Next()
}
return EntrySeries, nil
default: default:
err = p.parseError("expected a valid start token", t) err = p.parseError("expected a valid start token", t)
} }
@ -467,51 +589,64 @@ func (p *OpenMetricsParser) parseLVals(offsets []int, isExemplar bool) ([]int, e
} }
} }
// parseMetricSuffix parses the end of the line after the metric name and // isCreatedSeries returns true if the current series is a _created series.
// labels. It starts parsing with the provided token. func (p *OpenMetricsParser) isCreatedSeries() bool {
func (p *OpenMetricsParser) parseMetricSuffix(t token) (Entry, error) { var newLbs labels.Labels
p.Metric(&newLbs)
name := newLbs.Get(model.MetricNameLabel)
if TypeRequiresCT(p.mtype) && strings.HasSuffix(name, "_created") {
return true
}
return false
}
// parseSeriesEndOfLine parses the series end of the line (value, optional
// timestamp, commentary, etc.) after the metric name and labels.
// It starts parsing with the provided token.
func (p *OpenMetricsParser) parseSeriesEndOfLine(t token) error {
if p.offsets[0] == -1 { if p.offsets[0] == -1 {
return EntryInvalid, fmt.Errorf("metric name not set while parsing: %q", p.l.b[p.start:p.l.i]) return fmt.Errorf("metric name not set while parsing: %q", p.l.b[p.start:p.l.i])
} }
var err error var err error
p.val, err = p.getFloatValue(t, "metric") p.val, err = p.getFloatValue(t, "metric")
if err != nil { if err != nil {
return EntryInvalid, err return err
} }
p.hasTS = false p.hasTS = false
switch t2 := p.nextToken(); t2 { switch t2 := p.nextToken(); t2 {
case tEOF: case tEOF:
return EntryInvalid, errors.New("data does not end with # EOF") return errors.New("data does not end with # EOF")
case tLinebreak: case tLinebreak:
break break
case tComment: case tComment:
if err := p.parseComment(); err != nil { if err := p.parseComment(); err != nil {
return EntryInvalid, err return err
} }
case tTimestamp: case tTimestamp:
p.hasTS = true p.hasTS = true
var ts float64 var ts float64
// A float is enough to hold what we need for millisecond resolution. // A float is enough to hold what we need for millisecond resolution.
if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil { if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i]) return fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
} }
if math.IsNaN(ts) || math.IsInf(ts, 0) { if math.IsNaN(ts) || math.IsInf(ts, 0) {
return EntryInvalid, fmt.Errorf("invalid timestamp %f", ts) return fmt.Errorf("invalid timestamp %f", ts)
} }
p.ts = int64(ts * 1000) p.ts = int64(ts * 1000)
switch t3 := p.nextToken(); t3 { switch t3 := p.nextToken(); t3 {
case tLinebreak: case tLinebreak:
case tComment: case tComment:
if err := p.parseComment(); err != nil { if err := p.parseComment(); err != nil {
return EntryInvalid, err return err
} }
default: default:
return EntryInvalid, p.parseError("expected next entry after timestamp", t3) return p.parseError("expected next entry after timestamp", t3)
} }
} }
return EntrySeries, nil
return nil
} }
func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error) { func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error) {

330
model/textparse/openmetricsparse_test.go
View file

@ -14,6 +14,7 @@
package textparse package textparse
import ( import (
"errors"
"io" "io"
"testing" "testing"
@ -24,6 +25,8 @@ import (
"github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/labels"
) )
func int64p(x int64) *int64 { return &x }
func TestOpenMetricsParse(t *testing.T) { func TestOpenMetricsParse(t *testing.T) {
input := `# HELP go_gc_duration_seconds A summary of the GC invocation durations. input := `# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary # TYPE go_gc_duration_seconds summary
@ -63,15 +66,34 @@ ss{A="a"} 0
_metric_starting_with_underscore 1 _metric_starting_with_underscore 1
testmetric{_label_starting_with_underscore="foo"} 1 testmetric{_label_starting_with_underscore="foo"} 1
testmetric{label="\"bar\""} 1 testmetric{label="\"bar\""} 1
# HELP foo Counter with and without labels to certify CT is parsed for both cases
# TYPE foo counter # TYPE foo counter
foo_total 17.0 1520879607.789 # {id="counter-test"} 5` foo_total 17.0 1520879607.789 # {id="counter-test"} 5
foo_created 1000
foo_total{a="b"} 17.0 1520879607.789 # {id="counter-test"} 5
foo_created{a="b"} 1000
# HELP bar Summary with CT at the end, making sure we find CT even if it's multiple lines a far
# TYPE bar summary
bar_count 17.0
bar_sum 324789.3
bar{quantile="0.95"} 123.7
bar{quantile="0.99"} 150.0
bar_created 1520430000
# HELP baz Histogram with the same objective as above's summary
# TYPE baz histogram
baz_bucket{le="0.0"} 0
baz_bucket{le="+Inf"} 17
baz_count 17
baz_sum 324789.3
baz_created 1520430000
# HELP fizz_created Gauge which shouldn't be parsed as CT
# TYPE fizz_created gauge
fizz_created 17.0`
input += "\n# HELP metric foo\x00bar" input += "\n# HELP metric foo\x00bar"
input += "\nnull_byte_metric{a=\"abc\x00\"} 1" input += "\nnull_byte_metric{a=\"abc\x00\"} 1"
input += "\n# EOF\n" input += "\n# EOF\n"
int64p := func(x int64) *int64 { return &x }
exp := []expectedParse{ exp := []expectedParse{
{ {
m: "go_gc_duration_seconds", m: "go_gc_duration_seconds",
@ -216,6 +238,9 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
m: "testmetric{label=\"\\\"bar\\\"\"}", m: "testmetric{label=\"\\\"bar\\\"\"}",
v: 1, v: 1,
lset: labels.FromStrings("__name__", "testmetric", "label", `"bar"`), lset: labels.FromStrings("__name__", "testmetric", "label", `"bar"`),
}, {
m: "foo",
help: "Counter with and without labels to certify CT is parsed for both cases",
}, { }, {
m: "foo", m: "foo",
typ: model.MetricTypeCounter, typ: model.MetricTypeCounter,
@ -225,6 +250,76 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
lset: labels.FromStrings("__name__", "foo_total"), lset: labels.FromStrings("__name__", "foo_total"),
t: int64p(1520879607789), t: int64p(1520879607789),
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5}, e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5},
ct: int64p(1000),
}, {
m: `foo_total{a="b"}`,
v: 17.0,
lset: labels.FromStrings("__name__", "foo_total", "a", "b"),
t: int64p(1520879607789),
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5},
ct: int64p(1000),
}, {
m: "bar",
help: "Summary with CT at the end, making sure we find CT even if it's multiple lines a far",
}, {
m: "bar",
typ: model.MetricTypeSummary,
}, {
m: "bar_count",
v: 17.0,
lset: labels.FromStrings("__name__", "bar_count"),
ct: int64p(1520430000),
}, {
m: "bar_sum",
v: 324789.3,
lset: labels.FromStrings("__name__", "bar_sum"),
ct: int64p(1520430000),
}, {
m: `bar{quantile="0.95"}`,
v: 123.7,
lset: labels.FromStrings("__name__", "bar", "quantile", "0.95"),
ct: int64p(1520430000),
}, {
m: `bar{quantile="0.99"}`,
v: 150.0,
lset: labels.FromStrings("__name__", "bar", "quantile", "0.99"),
ct: int64p(1520430000),
}, {
m: "baz",
help: "Histogram with the same objective as above's summary",
}, {
m: "baz",
typ: model.MetricTypeHistogram,
}, {
m: `baz_bucket{le="0.0"}`,
v: 0,
lset: labels.FromStrings("__name__", "baz_bucket", "le", "0.0"),
ct: int64p(1520430000),
}, {
m: `baz_bucket{le="+Inf"}`,
v: 17,
lset: labels.FromStrings("__name__", "baz_bucket", "le", "+Inf"),
ct: int64p(1520430000),
}, {
m: `baz_count`,
v: 17,
lset: labels.FromStrings("__name__", "baz_count"),
ct: int64p(1520430000),
}, {
m: `baz_sum`,
v: 324789.3,
lset: labels.FromStrings("__name__", "baz_sum"),
ct: int64p(1520430000),
}, {
m: "fizz_created",
help: "Gauge which shouldn't be parsed as CT",
}, {
m: "fizz_created",
typ: model.MetricTypeGauge,
}, {
m: `fizz_created`,
v: 17,
lset: labels.FromStrings("__name__", "fizz_created"),
}, { }, {
m: "metric", m: "metric",
help: "foo\x00bar", help: "foo\x00bar",
@ -235,8 +330,8 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
}, },
} }
p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable()) p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable(), WithOMParserCTSeriesSkipped())
checkParseResults(t, p, exp) checkParseResultsWithCT(t, p, exp, true)
} }
func TestUTF8OpenMetricsParse(t *testing.T) { func TestUTF8OpenMetricsParse(t *testing.T) {
@ -251,6 +346,7 @@ func TestUTF8OpenMetricsParse(t *testing.T) {
# UNIT "go.gc_duration_seconds" seconds # UNIT "go.gc_duration_seconds" seconds
{"go.gc_duration_seconds",quantile="0"} 4.9351e-05 {"go.gc_duration_seconds",quantile="0"} 4.9351e-05
{"go.gc_duration_seconds",quantile="0.25"} 7.424100000000001e-05 {"go.gc_duration_seconds",quantile="0.25"} 7.424100000000001e-05
{"go.gc_duration_seconds_created"} 12313
{"go.gc_duration_seconds",quantile="0.5",a="b"} 8.3835e-05 {"go.gc_duration_seconds",quantile="0.5",a="b"} 8.3835e-05
{"http.status",q="0.9",a="b"} 8.3835e-05 {"http.status",q="0.9",a="b"} 8.3835e-05
{"http.status",q="0.9",a="b"} 8.3835e-05 {"http.status",q="0.9",a="b"} 8.3835e-05
@ -274,10 +370,12 @@ func TestUTF8OpenMetricsParse(t *testing.T) {
m: `{"go.gc_duration_seconds",quantile="0"}`, m: `{"go.gc_duration_seconds",quantile="0"}`,
v: 4.9351e-05, v: 4.9351e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0"), lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0"),
ct: int64p(12313),
}, { }, {
m: `{"go.gc_duration_seconds",quantile="0.25"}`, m: `{"go.gc_duration_seconds",quantile="0.25"}`,
v: 7.424100000000001e-05, v: 7.424100000000001e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.25"), lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.25"),
ct: int64p(12313),
}, { }, {
m: `{"go.gc_duration_seconds",quantile="0.5",a="b"}`, m: `{"go.gc_duration_seconds",quantile="0.5",a="b"}`,
v: 8.3835e-05, v: 8.3835e-05,
@ -306,8 +404,8 @@ choices}`, "strange©™\n'quoted' \"name\"", "6"),
}, },
} }
p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable()) p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable(), WithOMParserCTSeriesSkipped())
checkParseResults(t, p, exp) checkParseResultsWithCT(t, p, exp, true)
} }
func TestOpenMetricsParseErrors(t *testing.T) { func TestOpenMetricsParseErrors(t *testing.T) {
@ -598,10 +696,6 @@ func TestOpenMetricsParseErrors(t *testing.T) {
input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 -Inf", input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 -Inf",
err: `invalid exemplar timestamp -Inf`, err: `invalid exemplar timestamp -Inf`,
}, },
{
input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 Inf",
err: `invalid exemplar timestamp +Inf`,
},
} }
for i, c := range cases { for i, c := range cases {
@ -684,3 +778,217 @@ func TestOMNullByteHandling(t *testing.T) {
require.Equal(t, c.err, err.Error(), "test %d", i) require.Equal(t, c.err, err.Error(), "test %d", i)
} }
} }
// While not desirable, there are cases were CT fails to parse and
// these tests show them.
// TODO(maniktherana): Make sure OM 1.1/2.0 pass CT via metadata or exemplar-like to avoid this.
func TestCTParseFailures(t *testing.T) {
input := `# HELP something Histogram with _created between buckets and summary
# TYPE something histogram
something_count 17
something_sum 324789.3
something_created 1520430001
something_bucket{le="0.0"} 0
something_bucket{le="+Inf"} 17
# HELP thing Histogram with _created as first line
# TYPE thing histogram
thing_created 1520430002
thing_count 17
thing_sum 324789.3
thing_bucket{le="0.0"} 0
thing_bucket{le="+Inf"} 17
# HELP yum Summary with _created between sum and quantiles
# TYPE yum summary
yum_count 17.0
yum_sum 324789.3
yum_created 1520430003
yum{quantile="0.95"} 123.7
yum{quantile="0.99"} 150.0
# HELP foobar Summary with _created as the first line
# TYPE foobar summary
foobar_created 1520430004
foobar_count 17.0
foobar_sum 324789.3
foobar{quantile="0.95"} 123.7
foobar{quantile="0.99"} 150.0`
input += "\n# EOF\n"
int64p := func(x int64) *int64 { return &x }
type expectCT struct {
m string
ct *int64
typ model.MetricType
help string
isErr bool
}
exp := []expectCT{
{
m: "something",
help: "Histogram with _created between buckets and summary",
isErr: false,
}, {
m: "something",
typ: model.MetricTypeHistogram,
isErr: false,
}, {
m: `something_count`,
ct: int64p(1520430001),
isErr: false,
}, {
m: `something_sum`,
ct: int64p(1520430001),
isErr: false,
}, {
m: `something_bucket{le="0.0"}`,
ct: int64p(1520430001),
isErr: true,
}, {
m: `something_bucket{le="+Inf"}`,
ct: int64p(1520430001),
isErr: true,
}, {
m: "thing",
help: "Histogram with _created as first line",
isErr: false,
}, {
m: "thing",
typ: model.MetricTypeHistogram,
isErr: false,
}, {
m: `thing_count`,
ct: int64p(1520430002),
isErr: true,
}, {
m: `thing_sum`,
ct: int64p(1520430002),
isErr: true,
}, {
m: `thing_bucket{le="0.0"}`,
ct: int64p(1520430002),
isErr: true,
}, {
m: `thing_bucket{le="+Inf"}`,
ct: int64p(1520430002),
isErr: true,
}, {
m: "yum",
help: "Summary with _created between summary and quantiles",
isErr: false,
}, {
m: "yum",
typ: model.MetricTypeSummary,
isErr: false,
}, {
m: "yum_count",
ct: int64p(1520430003),
isErr: false,
}, {
m: "yum_sum",
ct: int64p(1520430003),
isErr: false,
}, {
m: `yum{quantile="0.95"}`,
ct: int64p(1520430003),
isErr: true,
}, {
m: `yum{quantile="0.99"}`,
ct: int64p(1520430003),
isErr: true,
}, {
m: "foobar",
help: "Summary with _created as the first line",
isErr: false,
}, {
m: "foobar",
typ: model.MetricTypeSummary,
isErr: false,
}, {
m: "foobar_count",
ct: int64p(1520430004),
isErr: true,
}, {
m: "foobar_sum",
ct: int64p(1520430004),
isErr: true,
}, {
m: `foobar{quantile="0.95"}`,
ct: int64p(1520430004),
isErr: true,
}, {
m: `foobar{quantile="0.99"}`,
ct: int64p(1520430004),
isErr: true,
},
}
p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable(), WithOMParserCTSeriesSkipped())
i := 0
var res labels.Labels
for {
et, err := p.Next()
if errors.Is(err, io.EOF) {
break
}
require.NoError(t, err)
switch et {
case EntrySeries:
p.Metric(&res)
if ct := p.CreatedTimestamp(); exp[i].isErr {
require.Nil(t, ct)
} else {
require.Equal(t, *exp[i].ct, *ct)
}
default:
i++
continue
}
i++
}
}
func TestDeepCopy(t *testing.T) {
input := []byte(`# HELP go_goroutines A gauge goroutines.
# TYPE go_goroutines gauge
go_goroutines 33 123.123
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds
go_gc_duration_seconds_created`)
st := labels.NewSymbolTable()
parser := NewOpenMetricsParser(input, st, WithOMParserCTSeriesSkipped()).(*OpenMetricsParser)
// Modify the original parser state
_, err := parser.Next()
require.NoError(t, err)
require.Equal(t, "go_goroutines", string(parser.l.b[parser.offsets[0]:parser.offsets[1]]))
require.True(t, parser.skipCTSeries)
// Create a deep copy of the parser
copyParser := deepCopy(parser)
etype, err := copyParser.Next()
require.NoError(t, err)
require.Equal(t, EntryType, etype)
require.True(t, parser.skipCTSeries)
require.False(t, copyParser.skipCTSeries)
// Modify the original parser further
parser.Next()
parser.Next()
parser.Next()
require.Equal(t, "go_gc_duration_seconds", string(parser.l.b[parser.offsets[0]:parser.offsets[1]]))
require.Equal(t, "summary", string(parser.mtype))
require.False(t, copyParser.skipCTSeries)
require.True(t, parser.skipCTSeries)
// Ensure the copy remains unchanged
copyParser.Next()
copyParser.Next()
require.Equal(t, "go_gc_duration_seconds", string(copyParser.l.b[copyParser.offsets[0]:copyParser.offsets[1]]))
require.False(t, copyParser.skipCTSeries)
}

23
model/textparse/promparse_test.go
View file

@ -18,6 +18,7 @@ import (
"errors" "errors"
"io" "io"
"os" "os"
"strings"
"testing" "testing"
"github.com/klauspost/compress/gzip" "github.com/klauspost/compress/gzip"
@ -41,6 +42,7 @@ type expectedParse struct {
unit string unit string
comment string comment string
e *exemplar.Exemplar e *exemplar.Exemplar
ct *int64
} }
func TestPromParse(t *testing.T) { func TestPromParse(t *testing.T) {
@ -188,6 +190,10 @@ testmetric{label="\"bar\""} 1`
} }
func checkParseResults(t *testing.T, p Parser, exp []expectedParse) { func checkParseResults(t *testing.T, p Parser, exp []expectedParse) {
checkParseResultsWithCT(t, p, exp, false)
}
func checkParseResultsWithCT(t *testing.T, p Parser, exp []expectedParse, ctLinesRemoved bool) {
i := 0 i := 0
var res labels.Labels var res labels.Labels
@ -205,6 +211,14 @@ func checkParseResults(t *testing.T, p Parser, exp []expectedParse) {
p.Metric(&res) p.Metric(&res)
if ctLinesRemoved {
// Are CT series skipped?
_, typ := p.Type()
if TypeRequiresCT(typ) && strings.HasSuffix(res.Get(labels.MetricName), "_created") {
t.Fatalf("we exped created lines skipped")
}
}
require.Equal(t, exp[i].m, string(m)) require.Equal(t, exp[i].m, string(m))
require.Equal(t, exp[i].t, ts) require.Equal(t, exp[i].t, ts)
require.Equal(t, exp[i].v, v) require.Equal(t, exp[i].v, v)
@ -218,6 +232,11 @@ func checkParseResults(t *testing.T, p Parser, exp []expectedParse) {
require.True(t, found) require.True(t, found)
testutil.RequireEqual(t, *exp[i].e, e) testutil.RequireEqual(t, *exp[i].e, e)
} }
if ct := p.CreatedTimestamp(); ct != nil {
require.Equal(t, *exp[i].ct, *ct)
} else {
require.Nil(t, exp[i].ct)
}
case EntryType: case EntryType:
m, typ := p.Type() m, typ := p.Type()
@ -476,7 +495,9 @@ const (
func BenchmarkParse(b *testing.B) { func BenchmarkParse(b *testing.B) {
for parserName, parser := range map[string]func([]byte, *labels.SymbolTable) Parser{ for parserName, parser := range map[string]func([]byte, *labels.SymbolTable) Parser{
"prometheus": NewPromParser, "prometheus": NewPromParser,
"openmetrics": NewOpenMetricsParser, "openmetrics": func(b []byte, st *labels.SymbolTable) Parser {
return NewOpenMetricsParser(b, st)
},
} { } {
for _, fn := range []string{"promtestdata.txt", "promtestdata.nometa.txt"} { for _, fn := range []string{"promtestdata.txt", "promtestdata.nometa.txt"} {
f, err := os.Open(fn) f, err := os.Open(fn)

1
notifier/notifier.go
View file

@ -674,7 +674,6 @@ func (n *Manager) sendOne(ctx context.Context, c *http.Client, url string, b []b
}() }()
// Any HTTP status 2xx is OK. // Any HTTP status 2xx is OK.
//nolint:usestdlibvars
if resp.StatusCode/100 != 2 { if resp.StatusCode/100 != 2 {
return fmt.Errorf("bad response status %s", resp.Status) return fmt.Errorf("bad response status %s", resp.Status)
} }

2
notifier/notifier_test.go
View file

@ -711,7 +711,7 @@ func TestHangingNotifier(t *testing.T) {
) )
var ( var (
sendTimeout = 10 * time.Millisecond sendTimeout = 100 * time.Millisecond
sdUpdatert = sendTimeout / 2 sdUpdatert = sendTimeout / 2
done = make(chan struct{}) done = make(chan struct{})

7
prompb/io/prometheus/write/v2/types.pb.go
View file

@ -302,15 +302,10 @@ type Exemplar struct {
// value represents an exact example value. This can be useful when the exemplar // value represents an exact example value. This can be useful when the exemplar
// is attached to a histogram, which only gives an estimated value through buckets. // is attached to a histogram, which only gives an estimated value through buckets.
Value float64 `protobuf:"fixed64,2,opt,name=value,proto3" json:"value,omitempty"` Value float64 `protobuf:"fixed64,2,opt,name=value,proto3" json:"value,omitempty"`
// timestamp represents an optional timestamp of the sample in ms. // timestamp represents the timestamp of the exemplar in ms.
// //
// For Go, see github.com/prometheus/prometheus/model/timestamp/timestamp.go // For Go, see github.com/prometheus/prometheus/model/timestamp/timestamp.go
// for conversion from/to time.Time to Prometheus timestamp. // for conversion from/to time.Time to Prometheus timestamp.
//
// Note that the "optional" keyword is omitted due to
// https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields
// Zero value means value not set. If you need to use exactly zero value for
// the timestamp, use 1 millisecond before or after.
Timestamp int64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` Timestamp int64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"` XXX_unrecognized []byte `json:"-"`

7
prompb/io/prometheus/write/v2/types.proto
View file

@ -107,15 +107,10 @@ message Exemplar {
// value represents an exact example value. This can be useful when the exemplar // value represents an exact example value. This can be useful when the exemplar
// is attached to a histogram, which only gives an estimated value through buckets. // is attached to a histogram, which only gives an estimated value through buckets.
double value = 2; double value = 2;
// timestamp represents an optional timestamp of the sample in ms. // timestamp represents the timestamp of the exemplar in ms.
// //
// For Go, see github.com/prometheus/prometheus/model/timestamp/timestamp.go // For Go, see github.com/prometheus/prometheus/model/timestamp/timestamp.go
// for conversion from/to time.Time to Prometheus timestamp. // for conversion from/to time.Time to Prometheus timestamp.
//
// Note that the "optional" keyword is omitted due to
// https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields
// Zero value means value not set. If you need to use exactly zero value for
// the timestamp, use 1 millisecond before or after.
int64 timestamp = 3; int64 timestamp = 3;
} }

3
promql/bench_test.go
View file

@ -166,6 +166,9 @@ func rangeQueryCases() []benchCase {
{ {
expr: "sum(a_X)", expr: "sum(a_X)",
}, },
{
expr: "avg(a_X)",
},
{ {
expr: "sum without (l)(h_X)", expr: "sum without (l)(h_X)",
}, },

106
promql/engine.go
View file

@ -584,7 +584,7 @@ func (ng *Engine) validateOpts(expr parser.Expr) error {
return validationErr return validationErr
} }
// NewTestQuery: inject special behaviour into Query for testing. // NewTestQuery injects special behaviour into Query for testing.
func (ng *Engine) NewTestQuery(f func(context.Context) error) Query { func (ng *Engine) NewTestQuery(f func(context.Context) error) Query {
qry := &query{ qry := &query{
q: "test statement", q: "test statement",
@ -1068,7 +1068,7 @@ func (ev *evaluator) recover(expr parser.Expr, ws *annotations.Annotations, errp
buf := make([]byte, 64<<10) buf := make([]byte, 64<<10)
buf = buf[:runtime.Stack(buf, false)] buf = buf[:runtime.Stack(buf, false)]
level.Error(ev.logger).Log("msg", "runtime panic in parser", "expr", expr.String(), "err", e, "stacktrace", string(buf)) level.Error(ev.logger).Log("msg", "runtime panic during query evaluation", "expr", expr.String(), "err", e, "stacktrace", string(buf))
*errp = fmt.Errorf("unexpected error: %w", err) *errp = fmt.Errorf("unexpected error: %w", err)
case errWithWarnings: case errWithWarnings:
*errp = err.err *errp = err.err
@ -2367,6 +2367,11 @@ loop:
} else { } else {
histograms = append(histograms, HPoint{H: &histogram.FloatHistogram{}}) histograms = append(histograms, HPoint{H: &histogram.FloatHistogram{}})
} }
if histograms[n].H == nil {
// Make sure to pass non-nil H to AtFloatHistogram so that it does a deep-copy.
// Not an issue in the loop above since that uses an intermediate buffer.
histograms[n].H = &histogram.FloatHistogram{}
}
histograms[n].T, histograms[n].H = it.AtFloatHistogram(histograms[n].H) histograms[n].T, histograms[n].H = it.AtFloatHistogram(histograms[n].H)
if value.IsStaleNaN(histograms[n].H.Sum) { if value.IsStaleNaN(histograms[n].H.Sum) {
histograms = histograms[:n] histograms = histograms[:n]
@ -2784,15 +2789,20 @@ func vectorElemBinop(op parser.ItemType, lhs, rhs float64, hlhs, hrhs *histogram
} }
type groupedAggregation struct { type groupedAggregation struct {
floatValue float64
histogramValue *histogram.FloatHistogram
floatMean float64
floatKahanC float64 // "Compensating value" for Kahan summation.
groupCount float64
heap vectorByValueHeap
// All bools together for better packing within the struct.
seen bool // Was this output groups seen in the input at this timestamp. seen bool // Was this output groups seen in the input at this timestamp.
hasFloat bool // Has at least 1 float64 sample aggregated. hasFloat bool // Has at least 1 float64 sample aggregated.
hasHistogram bool // Has at least 1 histogram sample aggregated. hasHistogram bool // Has at least 1 histogram sample aggregated.
floatValue float64 incompatibleHistograms bool // If true, group has seen mixed exponential and custom buckets, or incompatible custom buckets.
histogramValue *histogram.FloatHistogram groupAggrComplete bool // Used by LIMITK to short-cut series loop when we've reached K elem on every group.
floatMean float64 // Mean, or "compensating value" for Kahan summation. incrementalMean bool // True after reverting to incremental calculation of the mean value.
groupCount int
groupAggrComplete bool // Used by LIMITK to short-cut series loop when we've reached K elem on every group
heap vectorByValueHeap
} }
// aggregation evaluates sum, avg, count, stdvar, stddev or quantile at one timestep on inputMatrix. // aggregation evaluates sum, avg, count, stdvar, stddev or quantile at one timestep on inputMatrix.
@ -2818,13 +2828,12 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
*group = groupedAggregation{ *group = groupedAggregation{
seen: true, seen: true,
floatValue: f, floatValue: f,
floatMean: f,
incompatibleHistograms: false,
groupCount: 1, groupCount: 1,
} }
switch op { switch op {
case parser.AVG: case parser.AVG, parser.SUM:
group.floatMean = f
fallthrough
case parser.SUM:
if h == nil { if h == nil {
group.hasFloat = true group.hasFloat = true
} else { } else {
@ -2832,7 +2841,6 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
group.hasHistogram = true group.hasHistogram = true
} }
case parser.STDVAR, parser.STDDEV: case parser.STDVAR, parser.STDDEV:
group.floatMean = f
group.floatValue = 0 group.floatValue = 0
case parser.QUANTILE: case parser.QUANTILE:
group.heap = make(vectorByValueHeap, 1) group.heap = make(vectorByValueHeap, 1)
@ -2843,6 +2851,10 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
continue continue
} }
if group.incompatibleHistograms {
continue
}
switch op { switch op {
case parser.SUM: case parser.SUM:
if h != nil { if h != nil {
@ -2851,6 +2863,7 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
_, err := group.histogramValue.Add(h) _, err := group.histogramValue.Add(h)
if err != nil { if err != nil {
handleAggregationError(err, e, inputMatrix[si].Metric.Get(model.MetricNameLabel), &annos) handleAggregationError(err, e, inputMatrix[si].Metric.Get(model.MetricNameLabel), &annos)
group.incompatibleHistograms = true
} }
} }
// Otherwise the aggregation contained floats // Otherwise the aggregation contained floats
@ -2858,7 +2871,7 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
// point in copying the histogram in that case. // point in copying the histogram in that case.
} else { } else {
group.hasFloat = true group.hasFloat = true
group.floatValue, group.floatMean = kahanSumInc(f, group.floatValue, group.floatMean) group.floatValue, group.floatKahanC = kahanSumInc(f, group.floatValue, group.floatKahanC)
} }
case parser.AVG: case parser.AVG:
@ -2866,15 +2879,19 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
if h != nil { if h != nil {
group.hasHistogram = true group.hasHistogram = true
if group.histogramValue != nil { if group.histogramValue != nil {
left := h.Copy().Div(float64(group.groupCount)) left := h.Copy().Div(group.groupCount)
right := group.histogramValue.Copy().Div(float64(group.groupCount)) right := group.histogramValue.Copy().Div(group.groupCount)
toAdd, err := left.Sub(right) toAdd, err := left.Sub(right)
if err != nil { if err != nil {
handleAggregationError(err, e, inputMatrix[si].Metric.Get(model.MetricNameLabel), &annos) handleAggregationError(err, e, inputMatrix[si].Metric.Get(model.MetricNameLabel), &annos)
group.incompatibleHistograms = true
continue
} }
_, err = group.histogramValue.Add(toAdd) _, err = group.histogramValue.Add(toAdd)
if err != nil { if err != nil {
handleAggregationError(err, e, inputMatrix[si].Metric.Get(model.MetricNameLabel), &annos) handleAggregationError(err, e, inputMatrix[si].Metric.Get(model.MetricNameLabel), &annos)
group.incompatibleHistograms = true
continue
} }
} }
// Otherwise the aggregation contained floats // Otherwise the aggregation contained floats
@ -2882,6 +2899,22 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
// point in copying the histogram in that case. // point in copying the histogram in that case.
} else { } else {
group.hasFloat = true group.hasFloat = true
if !group.incrementalMean {
newV, newC := kahanSumInc(f, group.floatValue, group.floatKahanC)
if !math.IsInf(newV, 0) {
// The sum doesn't overflow, so we propagate it to the
// group struct and continue with the regular
// calculation of the mean value.
group.floatValue, group.floatKahanC = newV, newC
break
}
// If we are here, we know that the sum _would_ overflow. So
// instead of continue to sum up, we revert to incremental
// calculation of the mean value from here on.
group.incrementalMean = true
group.floatMean = group.floatValue / (group.groupCount - 1)
group.floatKahanC /= group.groupCount - 1
}
if math.IsInf(group.floatMean, 0) { if math.IsInf(group.floatMean, 0) {
if math.IsInf(f, 0) && (group.floatMean > 0) == (f > 0) { if math.IsInf(f, 0) && (group.floatMean > 0) == (f > 0) {
// The `floatMean` and `s.F` values are `Inf` of the same sign. They // The `floatMean` and `s.F` values are `Inf` of the same sign. They
@ -2899,8 +2932,13 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
break break
} }
} }
currentMean := group.floatMean + group.floatKahanC
group.floatMean, group.floatKahanC = kahanSumInc(
// Divide each side of the `-` by `group.groupCount` to avoid float64 overflows. // Divide each side of the `-` by `group.groupCount` to avoid float64 overflows.
group.floatMean += f/float64(group.groupCount) - group.floatMean/float64(group.groupCount) f/group.groupCount-currentMean/group.groupCount,
group.floatMean,
group.floatKahanC,
)
} }
case parser.GROUP: case parser.GROUP:
@ -2923,7 +2961,7 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
if h == nil { // Ignore native histograms. if h == nil { // Ignore native histograms.
group.groupCount++ group.groupCount++
delta := f - group.floatMean delta := f - group.floatMean
group.floatMean += delta / float64(group.groupCount) group.floatMean += delta / group.groupCount
group.floatValue += delta * (f - group.floatMean) group.floatValue += delta * (f - group.floatMean)
} }
@ -2949,20 +2987,25 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
annos.Add(annotations.NewMixedFloatsHistogramsAggWarning(e.Expr.PositionRange())) annos.Add(annotations.NewMixedFloatsHistogramsAggWarning(e.Expr.PositionRange()))
continue continue
} }
if aggr.hasHistogram { switch {
case aggr.incompatibleHistograms:
continue
case aggr.hasHistogram:
aggr.histogramValue = aggr.histogramValue.Compact(0) aggr.histogramValue = aggr.histogramValue.Compact(0)
} else { case aggr.incrementalMean:
aggr.floatValue = aggr.floatMean aggr.floatValue = aggr.floatMean + aggr.floatKahanC
default:
aggr.floatValue = (aggr.floatValue + aggr.floatKahanC) / aggr.groupCount
} }
case parser.COUNT: case parser.COUNT:
aggr.floatValue = float64(aggr.groupCount) aggr.floatValue = aggr.groupCount
case parser.STDVAR: case parser.STDVAR:
aggr.floatValue /= float64(aggr.groupCount) aggr.floatValue /= aggr.groupCount
case parser.STDDEV: case parser.STDDEV:
aggr.floatValue = math.Sqrt(aggr.floatValue / float64(aggr.groupCount)) aggr.floatValue = math.Sqrt(aggr.floatValue / aggr.groupCount)
case parser.QUANTILE: case parser.QUANTILE:
aggr.floatValue = quantile(q, aggr.heap) aggr.floatValue = quantile(q, aggr.heap)
@ -2973,10 +3016,13 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
annos.Add(annotations.NewMixedFloatsHistogramsAggWarning(e.Expr.PositionRange())) annos.Add(annotations.NewMixedFloatsHistogramsAggWarning(e.Expr.PositionRange()))
continue continue
} }
if aggr.hasHistogram { switch {
case aggr.incompatibleHistograms:
continue
case aggr.hasHistogram:
aggr.histogramValue.Compact(0) aggr.histogramValue.Compact(0)
} else { default:
aggr.floatValue += aggr.floatMean // Add Kahan summation compensating term. aggr.floatValue += aggr.floatKahanC
} }
default: default:
// For other aggregations, we already have the right value. // For other aggregations, we already have the right value.
@ -3142,7 +3188,7 @@ seriesLoop:
return mat, annos return mat, annos
} }
// aggregationK evaluates count_values on vec. // aggregationCountValues evaluates count_values on vec.
// Outputs as many series per group as there are values in the input. // Outputs as many series per group as there are values in the input.
func (ev *evaluator) aggregationCountValues(e *parser.AggregateExpr, grouping []string, valueLabel string, vec Vector, enh *EvalNodeHelper) (Vector, annotations.Annotations) { func (ev *evaluator) aggregationCountValues(e *parser.AggregateExpr, grouping []string, valueLabel string, vec Vector, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
type groupCount struct { type groupCount struct {
@ -3496,14 +3542,14 @@ func makeInt64Pointer(val int64) *int64 {
return valp return valp
} }
// Add RatioSampler interface to allow unit-testing (previously: Randomizer). // RatioSampler allows unit-testing (previously: Randomizer).
type RatioSampler interface { type RatioSampler interface {
// Return this sample "offset" between [0.0, 1.0] // Return this sample "offset" between [0.0, 1.0]
sampleOffset(ts int64, sample *Sample) float64 sampleOffset(ts int64, sample *Sample) float64
AddRatioSample(r float64, sample *Sample) bool AddRatioSample(r float64, sample *Sample) bool
} }
// Use Hash(labels.String()) / maxUint64 as a "deterministic" // HashRatioSampler uses Hash(labels.String()) / maxUint64 as a "deterministic"
// value in [0.0, 1.0]. // value in [0.0, 1.0].
type HashRatioSampler struct{} type HashRatioSampler struct{}

62
promql/engine_test.go
View file

@ -25,7 +25,6 @@ import (
"time" "time"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.uber.org/goleak"
"github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/labels"
@ -50,7 +49,7 @@ const (
func TestMain(m *testing.M) { func TestMain(m *testing.M) {
// Enable experimental functions testing // Enable experimental functions testing
parser.EnableExperimentalFunctions = true parser.EnableExperimentalFunctions = true
goleak.VerifyTestMain(m) testutil.TolerantVerifyLeak(m)
} }
func TestQueryConcurrency(t *testing.T) { func TestQueryConcurrency(t *testing.T) {
@ -3792,3 +3791,62 @@ func makeInt64Pointer(val int64) *int64 {
*valp = val *valp = val
return valp return valp
} }
func TestHistogramCopyFromIteratorRegression(t *testing.T) {
// Loading the following histograms creates two chunks because there's a
// counter reset. Not only the counter is lower in the last histogram
// but also there's missing buckets.
// This in turns means that chunk iterators will have different spans.
load := `load 1m
histogram {{sum:4 count:4 buckets:[2 2]}} {{sum:6 count:6 buckets:[3 3]}} {{sum:1 count:1 buckets:[1]}}
`
storage := promqltest.LoadedStorage(t, load)
t.Cleanup(func() { storage.Close() })
engine := promqltest.NewTestEngine(t, false, 0, promqltest.DefaultMaxSamplesPerQuery)
verify := func(t *testing.T, qry promql.Query, expected []histogram.FloatHistogram) {
res := qry.Exec(context.Background())
require.NoError(t, res.Err)
m, ok := res.Value.(promql.Matrix)
require.True(t, ok)
require.Len(t, m, 1)
series := m[0]
require.Empty(t, series.Floats)
require.Len(t, series.Histograms, len(expected))
for i, e := range expected {
series.Histograms[i].H.CounterResetHint = histogram.UnknownCounterReset // Don't care.
require.Equal(t, &e, series.Histograms[i].H)
}
}
qry, err := engine.NewRangeQuery(context.Background(), storage, nil, "increase(histogram[60s])", time.Unix(0, 0), time.Unix(0, 0).Add(1*time.Minute), time.Minute)
require.NoError(t, err)
verify(t, qry, []histogram.FloatHistogram{
{
Count: 2,
Sum: 2, // Increase from 4 to 6 is 2.
PositiveSpans: []histogram.Span{{Offset: 0, Length: 2}}, // Two buckets changed between the first and second histogram.
PositiveBuckets: []float64{1, 1}, // Increase from 2 to 3 is 1 in both buckets.
},
})
qry, err = engine.NewInstantQuery(context.Background(), storage, nil, "histogram[60s]", time.Unix(0, 0).Add(2*time.Minute))
require.NoError(t, err)
verify(t, qry, []histogram.FloatHistogram{
{
Count: 6,
Sum: 6,
PositiveSpans: []histogram.Span{{Offset: 0, Length: 2}},
PositiveBuckets: []float64{3, 3},
},
{
Count: 1,
Sum: 1,
PositiveSpans: []histogram.Span{{Offset: 0, Length: 1}},
PositiveBuckets: []float64{1},
},
})
}

121
promql/functions.go
View file

@ -97,9 +97,10 @@ func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNod
lastT = samples.Histograms[numSamplesMinusOne].T lastT = samples.Histograms[numSamplesMinusOne].T
var newAnnos annotations.Annotations var newAnnos annotations.Annotations
resultHistogram, newAnnos = histogramRate(samples.Histograms, isCounter, metricName, args[0].PositionRange()) resultHistogram, newAnnos = histogramRate(samples.Histograms, isCounter, metricName, args[0].PositionRange())
annos.Merge(newAnnos)
if resultHistogram == nil { if resultHistogram == nil {
// The histograms are not compatible with each other. // The histograms are not compatible with each other.
return enh.Out, annos.Merge(newAnnos) return enh.Out, annos
} }
case len(samples.Floats) > 1: case len(samples.Floats) > 1:
numSamplesMinusOne = len(samples.Floats) - 1 numSamplesMinusOne = len(samples.Floats) - 1
@ -178,17 +179,29 @@ func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNod
// Otherwise, it returns the calculated histogram and an empty annotation. // Otherwise, it returns the calculated histogram and an empty annotation.
func histogramRate(points []HPoint, isCounter bool, metricName string, pos posrange.PositionRange) (*histogram.FloatHistogram, annotations.Annotations) { func histogramRate(points []HPoint, isCounter bool, metricName string, pos posrange.PositionRange) (*histogram.FloatHistogram, annotations.Annotations) {
prev := points[0].H prev := points[0].H
usingCustomBuckets := prev.UsesCustomBuckets()
last := points[len(points)-1].H last := points[len(points)-1].H
if last == nil { if last == nil {
return nil, annotations.New().Add(annotations.NewMixedFloatsHistogramsWarning(metricName, pos)) return nil, annotations.New().Add(annotations.NewMixedFloatsHistogramsWarning(metricName, pos))
} }
minSchema := prev.Schema minSchema := prev.Schema
if last.Schema < minSchema { if last.Schema < minSchema {
minSchema = last.Schema minSchema = last.Schema
} }
if last.UsesCustomBuckets() != usingCustomBuckets {
return nil, annotations.New().Add(annotations.NewMixedExponentialCustomHistogramsWarning(metricName, pos))
}
var annos annotations.Annotations var annos annotations.Annotations
// We check for gauge type histograms in the loop below, but the loop below does not run on the first and last point,
// so check the first and last point now.
if isCounter && (prev.CounterResetHint == histogram.GaugeType || last.CounterResetHint == histogram.GaugeType) {
annos.Add(annotations.NewNativeHistogramNotCounterWarning(metricName, pos))
}
// First iteration to find out two things: // First iteration to find out two things:
// - What's the smallest relevant schema? // - What's the smallest relevant schema?
// - Are all data points histograms? // - Are all data points histograms?
@ -208,6 +221,9 @@ func histogramRate(points []HPoint, isCounter bool, metricName string, pos posra
if curr.Schema < minSchema { if curr.Schema < minSchema {
minSchema = curr.Schema minSchema = curr.Schema
} }
if curr.UsesCustomBuckets() != usingCustomBuckets {
return nil, annotations.New().Add(annotations.NewMixedExponentialCustomHistogramsWarning(metricName, pos))
}
} }
h := last.CopyToSchema(minSchema) h := last.CopyToSchema(minSchema)
@ -241,7 +257,7 @@ func histogramRate(points []HPoint, isCounter bool, metricName string, pos posra
} }
h.CounterResetHint = histogram.GaugeType h.CounterResetHint = histogram.GaugeType
return h.Compact(0), nil return h.Compact(0), annos
} }
// === delta(Matrix parser.ValueTypeMatrix) (Vector, Annotations) === // === delta(Matrix parser.ValueTypeMatrix) (Vector, Annotations) ===
@ -390,17 +406,22 @@ func funcSortDesc(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel
// === sort_by_label(vector parser.ValueTypeVector, label parser.ValueTypeString...) (Vector, Annotations) === // === sort_by_label(vector parser.ValueTypeVector, label parser.ValueTypeString...) (Vector, Annotations) ===
func funcSortByLabel(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { func funcSortByLabel(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
// In case the labels are the same, NaN should sort to the bottom, so take // First, sort by the full label set. This ensures a consistent ordering in case sorting by the
// ascending sort with NaN first and reverse it. // labels provided as arguments is not conclusive.
var anno annotations.Annotations
vals[0], anno = funcSort(vals, args, enh)
labels := stringSliceFromArgs(args[1:])
slices.SortFunc(vals[0].(Vector), func(a, b Sample) int { slices.SortFunc(vals[0].(Vector), func(a, b Sample) int {
// Iterate over each given label return labels.Compare(a.Metric, b.Metric)
})
labels := stringSliceFromArgs(args[1:])
// Next, sort by the labels provided as arguments.
slices.SortFunc(vals[0].(Vector), func(a, b Sample) int {
// Iterate over each given label.
for _, label := range labels { for _, label := range labels {
lv1 := a.Metric.Get(label) lv1 := a.Metric.Get(label)
lv2 := b.Metric.Get(label) lv2 := b.Metric.Get(label)
// If we encounter multiple samples with the same label values, the sorting which was
// performed in the first step will act as a "tie breaker".
if lv1 == lv2 { if lv1 == lv2 {
continue continue
} }
@ -415,22 +436,27 @@ func funcSortByLabel(vals []parser.Value, args parser.Expressions, enh *EvalNode
return 0 return 0
}) })
return vals[0].(Vector), anno return vals[0].(Vector), nil
} }
// === sort_by_label_desc(vector parser.ValueTypeVector, label parser.ValueTypeString...) (Vector, Annotations) === // === sort_by_label_desc(vector parser.ValueTypeVector, label parser.ValueTypeString...) (Vector, Annotations) ===
func funcSortByLabelDesc(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { func funcSortByLabelDesc(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
// In case the labels are the same, NaN should sort to the bottom, so take // First, sort by the full label set. This ensures a consistent ordering in case sorting by the
// ascending sort with NaN first and reverse it. // labels provided as arguments is not conclusive.
var anno annotations.Annotations
vals[0], anno = funcSortDesc(vals, args, enh)
labels := stringSliceFromArgs(args[1:])
slices.SortFunc(vals[0].(Vector), func(a, b Sample) int { slices.SortFunc(vals[0].(Vector), func(a, b Sample) int {
// Iterate over each given label return labels.Compare(b.Metric, a.Metric)
})
labels := stringSliceFromArgs(args[1:])
// Next, sort by the labels provided as arguments.
slices.SortFunc(vals[0].(Vector), func(a, b Sample) int {
// Iterate over each given label.
for _, label := range labels { for _, label := range labels {
lv1 := a.Metric.Get(label) lv1 := a.Metric.Get(label)
lv2 := b.Metric.Get(label) lv2 := b.Metric.Get(label)
// If we encounter multiple samples with the same label values, the sorting which was
// performed in the first step will act as a "tie breaker".
if lv1 == lv2 { if lv1 == lv2 {
continue continue
} }
@ -445,21 +471,21 @@ func funcSortByLabelDesc(vals []parser.Value, args parser.Expressions, enh *Eval
return 0 return 0
}) })
return vals[0].(Vector), anno return vals[0].(Vector), nil
} }
// === clamp(Vector parser.ValueTypeVector, min, max Scalar) (Vector, Annotations) === // === clamp(Vector parser.ValueTypeVector, min, max Scalar) (Vector, Annotations) ===
func funcClamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { func funcClamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
vec := vals[0].(Vector) vec := vals[0].(Vector)
min := vals[1].(Vector)[0].F minVal := vals[1].(Vector)[0].F
max := vals[2].(Vector)[0].F maxVal := vals[2].(Vector)[0].F
if max < min { if maxVal < minVal {
return enh.Out, nil return enh.Out, nil
} }
for _, el := range vec { for _, el := range vec {
enh.Out = append(enh.Out, Sample{ enh.Out = append(enh.Out, Sample{
Metric: el.Metric.DropMetricName(), Metric: el.Metric.DropMetricName(),
F: math.Max(min, math.Min(max, el.F)), F: math.Max(minVal, math.Min(maxVal, el.F)),
}) })
} }
return enh.Out, nil return enh.Out, nil
@ -468,11 +494,11 @@ func funcClamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper
// === clamp_max(Vector parser.ValueTypeVector, max Scalar) (Vector, Annotations) === // === clamp_max(Vector parser.ValueTypeVector, max Scalar) (Vector, Annotations) ===
func funcClampMax(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { func funcClampMax(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
vec := vals[0].(Vector) vec := vals[0].(Vector)
max := vals[1].(Vector)[0].F maxVal := vals[1].(Vector)[0].F
for _, el := range vec { for _, el := range vec {
enh.Out = append(enh.Out, Sample{ enh.Out = append(enh.Out, Sample{
Metric: el.Metric.DropMetricName(), Metric: el.Metric.DropMetricName(),
F: math.Min(max, el.F), F: math.Min(maxVal, el.F),
}) })
} }
return enh.Out, nil return enh.Out, nil
@ -481,11 +507,11 @@ func funcClampMax(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel
// === clamp_min(Vector parser.ValueTypeVector, min Scalar) (Vector, Annotations) === // === clamp_min(Vector parser.ValueTypeVector, min Scalar) (Vector, Annotations) ===
func funcClampMin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { func funcClampMin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
vec := vals[0].(Vector) vec := vals[0].(Vector)
min := vals[1].(Vector)[0].F minVal := vals[1].(Vector)[0].F
for _, el := range vec { for _, el := range vec {
enh.Out = append(enh.Out, Sample{ enh.Out = append(enh.Out, Sample{
Metric: el.Metric.DropMetricName(), Metric: el.Metric.DropMetricName(),
F: math.Max(min, el.F), F: math.Max(minVal, el.F),
}) })
} }
return enh.Out, nil return enh.Out, nil
@ -573,9 +599,28 @@ func funcAvgOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode
return vec, nil return vec, nil
} }
return aggrOverTime(vals, enh, func(s Series) float64 { return aggrOverTime(vals, enh, func(s Series) float64 {
var mean, count, c float64 var (
sum, mean, count, kahanC float64
incrementalMean bool
)
for _, f := range s.Floats { for _, f := range s.Floats {
count++ count++
if !incrementalMean {
newSum, newC := kahanSumInc(f.F, sum, kahanC)
// Perform regular mean calculation as long as
// the sum doesn't overflow and (in any case)
// for the first iteration (even if we start
// with ±Inf) to not run into division-by-zero
// problems below.
if count == 1 || !math.IsInf(newSum, 0) {
sum, kahanC = newSum, newC
continue
}
// Handle overflow by reverting to incremental calculation of the mean value.
incrementalMean = true
mean = sum / (count - 1)
kahanC /= count - 1
}
if math.IsInf(mean, 0) { if math.IsInf(mean, 0) {
if math.IsInf(f.F, 0) && (mean > 0) == (f.F > 0) { if math.IsInf(f.F, 0) && (mean > 0) == (f.F > 0) {
// The `mean` and `f.F` values are `Inf` of the same sign. They // The `mean` and `f.F` values are `Inf` of the same sign. They
@ -593,13 +638,13 @@ func funcAvgOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode
continue continue
} }
} }
mean, c = kahanSumInc(f.F/count-mean/count, mean, c) correctedMean := mean + kahanC
mean, kahanC = kahanSumInc(f.F/count-correctedMean/count, mean, kahanC)
} }
if incrementalMean {
if math.IsInf(mean, 0) { return mean + kahanC
return mean
} }
return mean + c return (sum + kahanC) / count
}), nil }), nil
} }
@ -665,13 +710,13 @@ func funcMaxOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode
return enh.Out, nil return enh.Out, nil
} }
return aggrOverTime(vals, enh, func(s Series) float64 { return aggrOverTime(vals, enh, func(s Series) float64 {
max := s.Floats[0].F maxVal := s.Floats[0].F
for _, f := range s.Floats { for _, f := range s.Floats {
if f.F > max || math.IsNaN(max) { if f.F > maxVal || math.IsNaN(maxVal) {
max = f.F maxVal = f.F
} }
} }
return max return maxVal
}), nil }), nil
} }
@ -685,13 +730,13 @@ func funcMinOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode
return enh.Out, nil return enh.Out, nil
} }
return aggrOverTime(vals, enh, func(s Series) float64 { return aggrOverTime(vals, enh, func(s Series) float64 {
min := s.Floats[0].F minVal := s.Floats[0].F
for _, f := range s.Floats { for _, f := range s.Floats {
if f.F < min || math.IsNaN(min) { if f.F < minVal || math.IsNaN(minVal) {
min = f.F minVal = f.F
} }
} }
return min return minVal
}), nil }), nil
} }

4
promql/fuzz.go
View file

@ -68,6 +68,10 @@ func fuzzParseMetricWithContentType(in []byte, contentType string) int {
panic(warning) panic(warning)
} }
if contentType == "application/openmetrics-text" {
p = textparse.NewOpenMetricsParser(in, symbolTable)
}
var err error var err error
for { for {
_, err = p.Next() _, err = p.Next()

22
promql/histogram_stats_iterator.go
View file

@ -48,7 +48,6 @@ func (f *histogramStatsIterator) AtHistogram(h *histogram.Histogram) (int64, *hi
var t int64 var t int64
t, f.currentH = f.Iterator.AtHistogram(f.currentH) t, f.currentH = f.Iterator.AtHistogram(f.currentH)
if value.IsStaleNaN(f.currentH.Sum) { if value.IsStaleNaN(f.currentH.Sum) {
f.setLastH(f.currentH)
h = &histogram.Histogram{Sum: f.currentH.Sum} h = &histogram.Histogram{Sum: f.currentH.Sum}
return t, h return t, h
} }
@ -63,9 +62,13 @@ func (f *histogramStatsIterator) AtHistogram(h *histogram.Histogram) (int64, *hi
return t, h return t, h
} }
h.CounterResetHint = f.getResetHint(f.currentH) returnValue := histogram.Histogram{
h.Count = f.currentH.Count CounterResetHint: f.getResetHint(f.currentH),
h.Sum = f.currentH.Sum Count: f.currentH.Count,
Sum: f.currentH.Sum,
}
returnValue.CopyTo(h)
f.setLastH(f.currentH) f.setLastH(f.currentH)
return t, h return t, h
} }
@ -77,7 +80,6 @@ func (f *histogramStatsIterator) AtFloatHistogram(fh *histogram.FloatHistogram)
var t int64 var t int64
t, f.currentFH = f.Iterator.AtFloatHistogram(f.currentFH) t, f.currentFH = f.Iterator.AtFloatHistogram(f.currentFH)
if value.IsStaleNaN(f.currentFH.Sum) { if value.IsStaleNaN(f.currentFH.Sum) {
f.setLastFH(f.currentFH)
return t, &histogram.FloatHistogram{Sum: f.currentFH.Sum} return t, &histogram.FloatHistogram{Sum: f.currentFH.Sum}
} }
@ -91,9 +93,13 @@ func (f *histogramStatsIterator) AtFloatHistogram(fh *histogram.FloatHistogram)
return t, fh return t, fh
} }
fh.CounterResetHint = f.getFloatResetHint(f.currentFH.CounterResetHint) returnValue := histogram.FloatHistogram{
fh.Count = f.currentFH.Count CounterResetHint: f.getFloatResetHint(f.currentFH.CounterResetHint),
fh.Sum = f.currentFH.Sum Count: f.currentFH.Count,
Sum: f.currentFH.Sum,
}
returnValue.CopyTo(fh)
f.setLastFH(f.currentFH) f.setLastFH(f.currentFH)
return t, fh return t, fh
} }

112
promql/histogram_stats_iterator_test.go
View file

@ -14,62 +14,132 @@
package promql package promql
import ( import (
"fmt"
"math"
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/value"
"github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunkenc"
"github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/tsdb/tsdbutil"
) )
func TestHistogramStatsDecoding(t *testing.T) { func TestHistogramStatsDecoding(t *testing.T) {
histograms := []*histogram.Histogram{ cases := []struct {
tsdbutil.GenerateTestHistogram(0), name string
tsdbutil.GenerateTestHistogram(1), histograms []*histogram.Histogram
tsdbutil.GenerateTestHistogram(2), expectedHints []histogram.CounterResetHint
tsdbutil.GenerateTestHistogram(2), }{
} {
histograms[0].CounterResetHint = histogram.NotCounterReset name: "unknown counter reset triggers detection",
histograms[1].CounterResetHint = histogram.UnknownCounterReset histograms: []*histogram.Histogram{
histograms[2].CounterResetHint = histogram.CounterReset tsdbutil.GenerateTestHistogramWithHint(0, histogram.NotCounterReset),
histograms[3].CounterResetHint = histogram.UnknownCounterReset tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
tsdbutil.GenerateTestHistogramWithHint(2, histogram.CounterReset),
expectedHints := []histogram.CounterResetHint{ tsdbutil.GenerateTestHistogramWithHint(2, histogram.UnknownCounterReset),
},
expectedHints: []histogram.CounterResetHint{
histogram.NotCounterReset, histogram.NotCounterReset,
histogram.NotCounterReset, histogram.NotCounterReset,
histogram.CounterReset, histogram.CounterReset,
histogram.NotCounterReset, histogram.NotCounterReset,
},
},
{
name: "stale sample before unknown reset hint",
histograms: []*histogram.Histogram{
tsdbutil.GenerateTestHistogramWithHint(0, histogram.NotCounterReset),
tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
{Sum: math.Float64frombits(value.StaleNaN)},
tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
},
expectedHints: []histogram.CounterResetHint{
histogram.NotCounterReset,
histogram.NotCounterReset,
histogram.UnknownCounterReset,
histogram.NotCounterReset,
},
},
{
name: "unknown counter reset at the beginning",
histograms: []*histogram.Histogram{
tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
},
expectedHints: []histogram.CounterResetHint{
histogram.NotCounterReset,
},
},
{
name: "detect real counter reset",
histograms: []*histogram.Histogram{
tsdbutil.GenerateTestHistogramWithHint(2, histogram.UnknownCounterReset),
tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
},
expectedHints: []histogram.CounterResetHint{
histogram.NotCounterReset,
histogram.CounterReset,
},
},
{
name: "detect real counter reset after stale NaN",
histograms: []*histogram.Histogram{
tsdbutil.GenerateTestHistogramWithHint(2, histogram.UnknownCounterReset),
{Sum: math.Float64frombits(value.StaleNaN)},
tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
},
expectedHints: []histogram.CounterResetHint{
histogram.NotCounterReset,
histogram.UnknownCounterReset,
histogram.CounterReset,
},
},
} }
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Run("histogram_stats", func(t *testing.T) { t.Run("histogram_stats", func(t *testing.T) {
decodedStats := make([]*histogram.Histogram, 0) decodedStats := make([]*histogram.Histogram, 0)
statsIterator := NewHistogramStatsIterator(newHistogramSeries(histograms).Iterator(nil)) statsIterator := NewHistogramStatsIterator(newHistogramSeries(tc.histograms).Iterator(nil))
for statsIterator.Next() != chunkenc.ValNone { for statsIterator.Next() != chunkenc.ValNone {
_, h := statsIterator.AtHistogram(nil) _, h := statsIterator.AtHistogram(nil)
decodedStats = append(decodedStats, h) decodedStats = append(decodedStats, h)
} }
for i := 0; i < len(histograms); i++ { for i := 0; i < len(tc.histograms); i++ {
require.Equal(t, expectedHints[i], decodedStats[i].CounterResetHint) require.Equal(t, tc.expectedHints[i], decodedStats[i].CounterResetHint, fmt.Sprintf("mismatch in counter reset hint for histogram %d", i))
require.Equal(t, histograms[i].Count, decodedStats[i].Count) h := tc.histograms[i]
require.Equal(t, histograms[i].Sum, decodedStats[i].Sum) if value.IsStaleNaN(h.Sum) {
require.True(t, value.IsStaleNaN(decodedStats[i].Sum))
require.Equal(t, uint64(0), decodedStats[i].Count)
} else {
require.Equal(t, tc.histograms[i].Count, decodedStats[i].Count)
require.Equal(t, tc.histograms[i].Sum, decodedStats[i].Sum)
}
} }
}) })
t.Run("float_histogram_stats", func(t *testing.T) { t.Run("float_histogram_stats", func(t *testing.T) {
decodedStats := make([]*histogram.FloatHistogram, 0) decodedStats := make([]*histogram.FloatHistogram, 0)
statsIterator := NewHistogramStatsIterator(newHistogramSeries(histograms).Iterator(nil)) statsIterator := NewHistogramStatsIterator(newHistogramSeries(tc.histograms).Iterator(nil))
for statsIterator.Next() != chunkenc.ValNone { for statsIterator.Next() != chunkenc.ValNone {
_, h := statsIterator.AtFloatHistogram(nil) _, h := statsIterator.AtFloatHistogram(nil)
decodedStats = append(decodedStats, h) decodedStats = append(decodedStats, h)
} }
for i := 0; i < len(histograms); i++ { for i := 0; i < len(tc.histograms); i++ {
fh := histograms[i].ToFloat(nil) require.Equal(t, tc.expectedHints[i], decodedStats[i].CounterResetHint)
require.Equal(t, expectedHints[i], decodedStats[i].CounterResetHint) fh := tc.histograms[i].ToFloat(nil)
if value.IsStaleNaN(fh.Sum) {
require.True(t, value.IsStaleNaN(decodedStats[i].Sum))
require.Equal(t, float64(0), decodedStats[i].Count)
} else {
require.Equal(t, fh.Count, decodedStats[i].Count) require.Equal(t, fh.Count, decodedStats[i].Count)
require.Equal(t, fh.Sum, decodedStats[i].Sum) require.Equal(t, fh.Sum, decodedStats[i].Sum)
} }
}
}) })
})
}
} }
type histogramSeries struct { type histogramSeries struct {

5
promql/parser/ast.go
View file

@ -352,8 +352,7 @@ func (f inspector) Visit(node Node, path []Node) (Visitor, error) {
// f(node, path); node must not be nil. If f returns a nil error, Inspect invokes f // f(node, path); node must not be nil. If f returns a nil error, Inspect invokes f
// for all the non-nil children of node, recursively. // for all the non-nil children of node, recursively.
func Inspect(node Node, f inspector) { func Inspect(node Node, f inspector) {
//nolint: errcheck Walk(f, node, nil) //nolint:errcheck
Walk(f, node, nil)
} }
// Children returns a list of all child nodes of a syntax tree node. // Children returns a list of all child nodes of a syntax tree node.
@ -419,7 +418,7 @@ func mergeRanges(first, last Node) posrange.PositionRange {
} }
} }
// Item implements the Node interface. // PositionRange implements the Node interface.
// This makes it possible to call mergeRanges on them. // This makes it possible to call mergeRanges on them.
func (i *Item) PositionRange() posrange.PositionRange { func (i *Item) PositionRange() posrange.PositionRange {
return posrange.PositionRange{ return posrange.PositionRange{

29
promql/parser/generated_parser.y
View file

@ -23,6 +23,8 @@ import (
"github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/model/value"
"github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/promql/parser/posrange" "github.com/prometheus/prometheus/promql/parser/posrange"
"github.com/prometheus/common/model"
) )
%} %}
@ -84,6 +86,7 @@ NEGATIVE_BUCKETS_DESC
ZERO_BUCKET_DESC ZERO_BUCKET_DESC
ZERO_BUCKET_WIDTH_DESC ZERO_BUCKET_WIDTH_DESC
CUSTOM_VALUES_DESC CUSTOM_VALUES_DESC
COUNTER_RESET_HINT_DESC
%token histogramDescEnd %token histogramDescEnd
// Operators. // Operators.
@ -149,6 +152,14 @@ START
END END
%token preprocessorEnd %token preprocessorEnd
// Counter reset hints.
%token counterResetHintsStart
%token <item>
UNKNOWN_COUNTER_RESET
COUNTER_RESET
NOT_COUNTER_RESET
GAUGE_TYPE
%token counterResetHintsEnd
// Start symbols for the generated parser. // Start symbols for the generated parser.
%token startSymbolsStart %token startSymbolsStart
@ -163,7 +174,7 @@ START_METRIC_SELECTOR
// Type definitions for grammar rules. // Type definitions for grammar rules.
%type <matchers> label_match_list %type <matchers> label_match_list
%type <matcher> label_matcher %type <matcher> label_matcher
%type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op at_modifier_preprocessors string_identifier %type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op at_modifier_preprocessors string_identifier counter_reset_hint
%type <labels> label_set metric %type <labels> label_set metric
%type <lblList> label_set_list %type <lblList> label_set_list
%type <label> label_set_item %type <label> label_set_item
@ -351,11 +362,19 @@ grouping_label_list:
grouping_label : maybe_label grouping_label : maybe_label
{ {
if !isLabel($1.Val) { if !model.LabelName($1.Val).IsValid() {
yylex.(*parser).unexpected("grouping opts", "label") yylex.(*parser).unexpected("grouping opts", "label")
} }
$$ = $1 $$ = $1
} }
| STRING {
if !model.LabelName(yylex.(*parser).unquoteString($1.Val)).IsValid() {
yylex.(*parser).unexpected("grouping opts", "label")
}
$$ = $1
$$.Pos++
$$.Val = yylex.(*parser).unquoteString($$.Val)
}
| error | error
{ yylex.(*parser).unexpected("grouping opts", "label"); $$ = Item{} } { yylex.(*parser).unexpected("grouping opts", "label"); $$ = Item{} }
; ;
@ -839,6 +858,11 @@ histogram_desc_item
$$ = yylex.(*parser).newMap() $$ = yylex.(*parser).newMap()
$$["n_offset"] = $3 $$["n_offset"] = $3
} }
| COUNTER_RESET_HINT_DESC COLON counter_reset_hint
{
$$ = yylex.(*parser).newMap()
$$["counter_reset_hint"] = $3
}
; ;
bucket_set : LEFT_BRACKET bucket_set_list SPACE RIGHT_BRACKET bucket_set : LEFT_BRACKET bucket_set_list SPACE RIGHT_BRACKET
@ -862,6 +886,7 @@ bucket_set_list : bucket_set_list SPACE number
| bucket_set_list error | bucket_set_list error
; ;
counter_reset_hint : UNKNOWN_COUNTER_RESET | COUNTER_RESET | NOT_COUNTER_RESET | GAUGE_TYPE;
/* /*
* Keyword lists. * Keyword lists.

1060
promql/parser/generated_parser.y.go
View file

File diff suppressed because it is too large Load diff

38
promql/parser/lex.go
View file

@ -147,6 +147,14 @@ var histogramDesc = map[string]ItemType{
"z_bucket": ZERO_BUCKET_DESC, "z_bucket": ZERO_BUCKET_DESC,
"z_bucket_w": ZERO_BUCKET_WIDTH_DESC, "z_bucket_w": ZERO_BUCKET_WIDTH_DESC,
"custom_values": CUSTOM_VALUES_DESC, "custom_values": CUSTOM_VALUES_DESC,
"counter_reset_hint": COUNTER_RESET_HINT_DESC,
}
var counterResetHints = map[string]ItemType{
"unknown": UNKNOWN_COUNTER_RESET,
"reset": COUNTER_RESET,
"not_reset": NOT_COUNTER_RESET,
"gauge": GAUGE_TYPE,
} }
// ItemTypeStr is the default string representations for common Items. It does not // ItemTypeStr is the default string representations for common Items. It does not
@ -585,6 +593,11 @@ Loop:
return lexHistogram return lexHistogram
} }
} }
if desc, ok := counterResetHints[strings.ToLower(word)]; ok {
l.emit(desc)
return lexHistogram
}
l.errorf("bad histogram descriptor found: %q", word) l.errorf("bad histogram descriptor found: %q", word)
break Loop break Loop
} }
@ -714,23 +727,23 @@ func lexValueSequence(l *Lexer) stateFn {
// was only modified to integrate with our lexer. // was only modified to integrate with our lexer.
func lexEscape(l *Lexer) stateFn { func lexEscape(l *Lexer) stateFn {
var n int var n int
var base, max uint32 var base, maxVal uint32
ch := l.next() ch := l.next()
switch ch { switch ch {
case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', l.stringOpen: case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', l.stringOpen:
return lexString return lexString
case '0', '1', '2', '3', '4', '5', '6', '7': case '0', '1', '2', '3', '4', '5', '6', '7':
n, base, max = 3, 8, 255 n, base, maxVal = 3, 8, 255
case 'x': case 'x':
ch = l.next() ch = l.next()
n, base, max = 2, 16, 255 n, base, maxVal = 2, 16, 255
case 'u': case 'u':
ch = l.next() ch = l.next()
n, base, max = 4, 16, unicode.MaxRune n, base, maxVal = 4, 16, unicode.MaxRune
case 'U': case 'U':
ch = l.next() ch = l.next()
n, base, max = 8, 16, unicode.MaxRune n, base, maxVal = 8, 16, unicode.MaxRune
case eof: case eof:
l.errorf("escape sequence not terminated") l.errorf("escape sequence not terminated")
return lexString return lexString
@ -759,7 +772,7 @@ func lexEscape(l *Lexer) stateFn {
} }
} }
if x > max || 0xD800 <= x && x < 0xE000 { if x > maxVal || 0xD800 <= x && x < 0xE000 {
l.errorf("escape sequence is an invalid Unicode code point") l.errorf("escape sequence is an invalid Unicode code point")
} }
return lexString return lexString
@ -1046,16 +1059,3 @@ func isDigit(r rune) bool {
func isAlpha(r rune) bool { func isAlpha(r rune) bool {
return r == '_' || ('a' <= r && r <= 'z') || ('A' <= r && r <= 'Z') return r == '_' || ('a' <= r && r <= 'z') || ('A' <= r && r <= 'Z')
} }
// isLabel reports whether the string can be used as label.
func isLabel(s string) bool {
if len(s) == 0 || !isAlpha(rune(s[0])) {
return false
}
for _, c := range s[1:] {
if !isAlphaNumeric(c) {
return false
}
}
return true
}

22
promql/parser/parse.go
View file

@ -580,6 +580,28 @@ func (p *parser) buildHistogramFromMap(desc *map[string]interface{}) *histogram.
} }
} }
val, ok = (*desc)["counter_reset_hint"]
if ok {
resetHint, ok := val.(Item)
if ok {
switch resetHint.Typ {
case UNKNOWN_COUNTER_RESET:
output.CounterResetHint = histogram.UnknownCounterReset
case COUNTER_RESET:
output.CounterResetHint = histogram.CounterReset
case NOT_COUNTER_RESET:
output.CounterResetHint = histogram.NotCounterReset
case GAUGE_TYPE:
output.CounterResetHint = histogram.GaugeType
default:
p.addParseErrf(p.yyParser.lval.item.PositionRange(), "error parsing counter_reset_hint: unknown value %v", resetHint.Typ)
}
} else {
p.addParseErrf(p.yyParser.lval.item.PositionRange(), "error parsing counter_reset_hint: %v", val)
}
}
buckets, spans := p.buildHistogramBucketsAndSpans(desc, "buckets", "offset") buckets, spans := p.buildHistogramBucketsAndSpans(desc, "buckets", "offset")
output.PositiveBuckets = buckets output.PositiveBuckets = buckets
output.PositiveSpans = spans output.PositiveSpans = spans

85
promql/parser/parse_test.go
View file

@ -2397,6 +2397,51 @@ var testExpr = []struct {
}, },
}, },
}, },
{
input: `sum by ("foo")({"some.metric"})`,
expected: &AggregateExpr{
Op: SUM,
Expr: &VectorSelector{
LabelMatchers: []*labels.Matcher{
MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "some.metric"),
},
PosRange: posrange.PositionRange{
Start: 15,
End: 30,
},
},
Grouping: []string{"foo"},
PosRange: posrange.PositionRange{
Start: 0,
End: 31,
},
},
},
{
input: `sum by ("foo)(some_metric{})`,
fail: true,
errMsg: "unterminated quoted string",
},
{
input: `sum by ("foo", bar, 'baz')({"some.metric"})`,
expected: &AggregateExpr{
Op: SUM,
Expr: &VectorSelector{
LabelMatchers: []*labels.Matcher{
MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "some.metric"),
},
PosRange: posrange.PositionRange{
Start: 27,
End: 42,
},
},
Grouping: []string{"foo", "bar", "baz"},
PosRange: posrange.PositionRange{
Start: 0,
End: 43,
},
},
},
{ {
input: "avg by (foo)(some_metric)", input: "avg by (foo)(some_metric)",
expected: &AggregateExpr{ expected: &AggregateExpr{
@ -3844,6 +3889,7 @@ func readable(s string) string {
} }
func TestParseExpressions(t *testing.T) { func TestParseExpressions(t *testing.T) {
model.NameValidationScheme = model.UTF8Validation
for _, test := range testExpr { for _, test := range testExpr {
t.Run(readable(test.input), func(t *testing.T) { t.Run(readable(test.input), func(t *testing.T) {
expr, err := ParseExpr(test.input) expr, err := ParseExpr(test.input)
@ -4038,7 +4084,7 @@ func TestParseHistogramSeries(t *testing.T) {
}, },
{ {
name: "all properties used", name: "all properties used",
input: `{} {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5}}`, input: `{} {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5 counter_reset_hint:gauge}}`,
expected: []histogram.FloatHistogram{{ expected: []histogram.FloatHistogram{{
Schema: 1, Schema: 1,
Sum: -0.3, Sum: -0.3,
@ -4049,11 +4095,12 @@ func TestParseHistogramSeries(t *testing.T) {
PositiveSpans: []histogram.Span{{Offset: -3, Length: 3}}, PositiveSpans: []histogram.Span{{Offset: -3, Length: 3}},
NegativeBuckets: []float64{4.1, 5}, NegativeBuckets: []float64{4.1, 5},
NegativeSpans: []histogram.Span{{Offset: -5, Length: 2}}, NegativeSpans: []histogram.Span{{Offset: -5, Length: 2}},
CounterResetHint: histogram.GaugeType,
}}, }},
}, },
{ {
name: "all properties used - with spaces", name: "all properties used - with spaces",
input: `{} {{schema:1 sum:0.3 count:3 z_bucket:7 z_bucket_w:5 buckets:[5 10 7 ] offset:-3 n_buckets:[4 5] n_offset:5 }}`, input: `{} {{schema:1 sum:0.3 count:3 z_bucket:7 z_bucket_w:5 buckets:[5 10 7 ] offset:-3 n_buckets:[4 5] n_offset:5 counter_reset_hint:gauge }}`,
expected: []histogram.FloatHistogram{{ expected: []histogram.FloatHistogram{{
Schema: 1, Schema: 1,
Sum: 0.3, Sum: 0.3,
@ -4064,6 +4111,7 @@ func TestParseHistogramSeries(t *testing.T) {
PositiveSpans: []histogram.Span{{Offset: -3, Length: 3}}, PositiveSpans: []histogram.Span{{Offset: -3, Length: 3}},
NegativeBuckets: []float64{4, 5}, NegativeBuckets: []float64{4, 5},
NegativeSpans: []histogram.Span{{Offset: 5, Length: 2}}, NegativeSpans: []histogram.Span{{Offset: 5, Length: 2}},
CounterResetHint: histogram.GaugeType,
}}, }},
}, },
{ {
@ -4250,6 +4298,39 @@ func TestParseHistogramSeries(t *testing.T) {
input: `{} {{ schema:1}}`, input: `{} {{ schema:1}}`,
expectedError: `1:7: parse error: unexpected "<Item 57372>" "schema" in series values`, expectedError: `1:7: parse error: unexpected "<Item 57372>" "schema" in series values`,
}, },
{
name: "invalid counter reset hint value",
input: `{} {{counter_reset_hint:foo}}`,
expectedError: `1:25: parse error: bad histogram descriptor found: "foo"`,
},
{
name: "'unknown' counter reset hint value",
input: `{} {{counter_reset_hint:unknown}}`,
expected: []histogram.FloatHistogram{{
CounterResetHint: histogram.UnknownCounterReset,
}},
},
{
name: "'reset' counter reset hint value",
input: `{} {{counter_reset_hint:reset}}`,
expected: []histogram.FloatHistogram{{
CounterResetHint: histogram.CounterReset,
}},
},
{
name: "'not_reset' counter reset hint value",
input: `{} {{counter_reset_hint:not_reset}}`,
expected: []histogram.FloatHistogram{{
CounterResetHint: histogram.NotCounterReset,
}},
},
{
name: "'gauge' counter reset hint value",
input: `{} {{counter_reset_hint:gauge}}`,
expected: []histogram.FloatHistogram{{
CounterResetHint: histogram.GaugeType,
}},
},
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
_, vals, err := ParseSeriesDesc(test.input) _, vals, err := ParseSeriesDesc(test.input)

14
promql/parser/printer.go
View file

@ -77,14 +77,24 @@ func (node *AggregateExpr) getAggOpStr() string {
switch { switch {
case node.Without: case node.Without:
aggrString += fmt.Sprintf(" without (%s) ", strings.Join(node.Grouping, ", ")) aggrString += fmt.Sprintf(" without (%s) ", joinLabels(node.Grouping))
case len(node.Grouping) > 0: case len(node.Grouping) > 0:
aggrString += fmt.Sprintf(" by (%s) ", strings.Join(node.Grouping, ", ")) aggrString += fmt.Sprintf(" by (%s) ", joinLabels(node.Grouping))
} }
return aggrString return aggrString
} }
func joinLabels(ss []string) string {
for i, s := range ss {
// If the label is already quoted, don't quote it again.
if s[0] != '"' && s[0] != '\'' && s[0] != '`' && !model.IsValidLegacyMetricName(model.LabelValue(s)) {
ss[i] = fmt.Sprintf("\"%s\"", s)
}
}
return strings.Join(ss, ", ")
}
func (node *BinaryExpr) String() string { func (node *BinaryExpr) String() string {
returnBool := "" returnBool := ""
if node.ReturnBool { if node.ReturnBool {

11
promql/parser/printer_test.go
View file

@ -16,6 +16,7 @@ package parser
import ( import (
"testing" "testing"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/labels"
@ -44,6 +45,14 @@ func TestExprString(t *testing.T) {
in: `sum without(instance) (task:errors:rate10s{job="s"})`, in: `sum without(instance) (task:errors:rate10s{job="s"})`,
out: `sum without (instance) (task:errors:rate10s{job="s"})`, out: `sum without (instance) (task:errors:rate10s{job="s"})`,
}, },
{
in: `sum by("foo.bar") (task:errors:rate10s{job="s"})`,
out: `sum by ("foo.bar") (task:errors:rate10s{job="s"})`,
},
{
in: `sum without("foo.bar") (task:errors:rate10s{job="s"})`,
out: `sum without ("foo.bar") (task:errors:rate10s{job="s"})`,
},
{ {
in: `topk(5, task:errors:rate10s{job="s"})`, in: `topk(5, task:errors:rate10s{job="s"})`,
}, },
@ -157,6 +166,8 @@ func TestExprString(t *testing.T) {
}, },
} }
model.NameValidationScheme = model.UTF8Validation
for _, test := range inputs { for _, test := range inputs {
expr, err := ParseExpr(test.in) expr, err := ParseExpr(test.in)
require.NoError(t, err) require.NoError(t, err)

28
promql/promqltest/test.go
View file

@ -1014,13 +1014,6 @@ func (t *test) execRangeEval(cmd *evalCmd, engine promql.QueryEngine) error {
return fmt.Errorf("error creating range query for %q (line %d): %w", cmd.expr, cmd.line, err) return fmt.Errorf("error creating range query for %q (line %d): %w", cmd.expr, cmd.line, err)
} }
res := q.Exec(t.context) res := q.Exec(t.context)
countWarnings, _ := res.Warnings.CountWarningsAndInfo()
if !cmd.warn && countWarnings > 0 {
return fmt.Errorf("unexpected warnings evaluating query %q (line %d): %v", cmd.expr, cmd.line, res.Warnings)
}
if cmd.warn && countWarnings == 0 {
return fmt.Errorf("expected warnings evaluating query %q (line %d) but got none", cmd.expr, cmd.line)
}
if res.Err != nil { if res.Err != nil {
if cmd.fail { if cmd.fail {
return cmd.checkExpectedFailure(res.Err) return cmd.checkExpectedFailure(res.Err)
@ -1031,6 +1024,13 @@ func (t *test) execRangeEval(cmd *evalCmd, engine promql.QueryEngine) error {
if res.Err == nil && cmd.fail { if res.Err == nil && cmd.fail {
return fmt.Errorf("expected error evaluating query %q (line %d) but got none", cmd.expr, cmd.line) return fmt.Errorf("expected error evaluating query %q (line %d) but got none", cmd.expr, cmd.line)
} }
countWarnings, _ := res.Warnings.CountWarningsAndInfo()
if !cmd.warn && countWarnings > 0 {
return fmt.Errorf("unexpected warnings evaluating query %q (line %d): %v", cmd.expr, cmd.line, res.Warnings)
}
if cmd.warn && countWarnings == 0 {
return fmt.Errorf("expected warnings evaluating query %q (line %d) but got none", cmd.expr, cmd.line)
}
defer q.Close() defer q.Close()
if err := cmd.compareResult(res.Value); err != nil { if err := cmd.compareResult(res.Value); err != nil {
@ -1061,13 +1061,6 @@ func (t *test) runInstantQuery(iq atModifierTestCase, cmd *evalCmd, engine promq
} }
defer q.Close() defer q.Close()
res := q.Exec(t.context) res := q.Exec(t.context)
countWarnings, _ := res.Warnings.CountWarningsAndInfo()
if !cmd.warn && countWarnings > 0 {
return fmt.Errorf("unexpected warnings evaluating query %q (line %d): %v", iq.expr, cmd.line, res.Warnings)
}
if cmd.warn && countWarnings == 0 {
return fmt.Errorf("expected warnings evaluating query %q (line %d) but got none", iq.expr, cmd.line)
}
if res.Err != nil { if res.Err != nil {
if cmd.fail { if cmd.fail {
if err := cmd.checkExpectedFailure(res.Err); err != nil { if err := cmd.checkExpectedFailure(res.Err); err != nil {
@ -1081,6 +1074,13 @@ func (t *test) runInstantQuery(iq atModifierTestCase, cmd *evalCmd, engine promq
if res.Err == nil && cmd.fail { if res.Err == nil && cmd.fail {
return fmt.Errorf("expected error evaluating query %q (line %d) but got none", iq.expr, cmd.line) return fmt.Errorf("expected error evaluating query %q (line %d) but got none", iq.expr, cmd.line)
} }
countWarnings, _ := res.Warnings.CountWarningsAndInfo()
if !cmd.warn && countWarnings > 0 {
return fmt.Errorf("unexpected warnings evaluating query %q (line %d): %v", iq.expr, cmd.line, res.Warnings)
}
if cmd.warn && countWarnings == 0 {
return fmt.Errorf("expected warnings evaluating query %q (line %d) but got none", iq.expr, cmd.line)
}
err = cmd.compareResult(res.Value) err = cmd.compareResult(res.Value)
if err != nil { if err != nil {
return fmt.Errorf("error in %s %s (line %d): %w", cmd, iq.expr, cmd.line, err) return fmt.Errorf("error in %s %s (line %d): %w", cmd, iq.expr, cmd.line, err)

20
promql/promqltest/testdata/aggregators.test vendored
View file

@ -503,7 +503,7 @@ eval instant at 1m avg(data{test="-big"})
eval instant at 1m avg(data{test="bigzero"}) eval instant at 1m avg(data{test="bigzero"})
{} 0 {} 0
# Test summing extreme values. # Test summing and averaging extreme values.
clear clear
load 10s load 10s
@ -529,21 +529,39 @@ load 10s
eval instant at 1m sum(data{test="ten"}) eval instant at 1m sum(data{test="ten"})
{} 10 {} 10
eval instant at 1m avg(data{test="ten"})
{} 2.5
eval instant at 1m sum by (group) (data{test="pos_inf"}) eval instant at 1m sum by (group) (data{test="pos_inf"})
{group="1"} Inf {group="1"} Inf
{group="2"} Inf {group="2"} Inf
eval instant at 1m avg by (group) (data{test="pos_inf"})
{group="1"} Inf
{group="2"} Inf
eval instant at 1m sum by (group) (data{test="neg_inf"}) eval instant at 1m sum by (group) (data{test="neg_inf"})
{group="1"} -Inf {group="1"} -Inf
{group="2"} -Inf {group="2"} -Inf
eval instant at 1m avg by (group) (data{test="neg_inf"})
{group="1"} -Inf
{group="2"} -Inf
eval instant at 1m sum(data{test="inf_inf"}) eval instant at 1m sum(data{test="inf_inf"})
{} NaN {} NaN
eval instant at 1m avg(data{test="inf_inf"})
{} NaN
eval instant at 1m sum by (group) (data{test="nan"}) eval instant at 1m sum by (group) (data{test="nan"})
{group="1"} NaN {group="1"} NaN
{group="2"} NaN {group="2"} NaN
eval instant at 1m avg by (group) (data{test="nan"})
{group="1"} NaN
{group="2"} NaN
clear clear
# Test that aggregations are deterministic. # Test that aggregations are deterministic.

22
promql/promqltest/testdata/functions.test vendored
View file

@ -523,16 +523,16 @@ load 5m
node_uname_info{job="node_exporter", instance="4m1000", release="1.111.3"} 0+10x10 node_uname_info{job="node_exporter", instance="4m1000", release="1.111.3"} 0+10x10
eval_ordered instant at 50m sort_by_label(http_requests, "instance") eval_ordered instant at 50m sort_by_label(http_requests, "instance")
http_requests{group="production", instance="0", job="api-server"} 100
http_requests{group="canary", instance="0", job="api-server"} 300 http_requests{group="canary", instance="0", job="api-server"} 300
http_requests{group="production", instance="0", job="app-server"} 500
http_requests{group="canary", instance="0", job="app-server"} 700 http_requests{group="canary", instance="0", job="app-server"} 700
http_requests{group="production", instance="1", job="api-server"} 200 http_requests{group="production", instance="0", job="api-server"} 100
http_requests{group="production", instance="0", job="app-server"} 500
http_requests{group="canary", instance="1", job="api-server"} 400 http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="production", instance="1", job="app-server"} 600
http_requests{group="canary", instance="1", job="app-server"} 800 http_requests{group="canary", instance="1", job="app-server"} 800
http_requests{group="production", instance="2", job="api-server"} 100 http_requests{group="production", instance="1", job="api-server"} 200
http_requests{group="production", instance="1", job="app-server"} 600
http_requests{group="canary", instance="2", job="api-server"} NaN http_requests{group="canary", instance="2", job="api-server"} NaN
http_requests{group="production", instance="2", job="api-server"} 100
eval_ordered instant at 50m sort_by_label(http_requests, "instance", "group") eval_ordered instant at 50m sort_by_label(http_requests, "instance", "group")
http_requests{group="canary", instance="0", job="api-server"} 300 http_requests{group="canary", instance="0", job="api-server"} 300
@ -585,14 +585,14 @@ eval_ordered instant at 50m sort_by_label(http_requests, "job", "instance", "gro
eval_ordered instant at 50m sort_by_label_desc(http_requests, "instance") eval_ordered instant at 50m sort_by_label_desc(http_requests, "instance")
http_requests{group="production", instance="2", job="api-server"} 100 http_requests{group="production", instance="2", job="api-server"} 100
http_requests{group="canary", instance="2", job="api-server"} NaN http_requests{group="canary", instance="2", job="api-server"} NaN
http_requests{group="canary", instance="1", job="app-server"} 800
http_requests{group="production", instance="1", job="app-server"} 600 http_requests{group="production", instance="1", job="app-server"} 600
http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="production", instance="1", job="api-server"} 200 http_requests{group="production", instance="1", job="api-server"} 200
http_requests{group="canary", instance="0", job="app-server"} 700 http_requests{group="canary", instance="1", job="app-server"} 800
http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="production", instance="0", job="app-server"} 500 http_requests{group="production", instance="0", job="app-server"} 500
http_requests{group="canary", instance="0", job="api-server"} 300
http_requests{group="production", instance="0", job="api-server"} 100 http_requests{group="production", instance="0", job="api-server"} 100
http_requests{group="canary", instance="0", job="app-server"} 700
http_requests{group="canary", instance="0", job="api-server"} 300
eval_ordered instant at 50m sort_by_label_desc(http_requests, "instance", "group") eval_ordered instant at 50m sort_by_label_desc(http_requests, "instance", "group")
http_requests{group="production", instance="2", job="api-server"} 100 http_requests{group="production", instance="2", job="api-server"} 100
@ -748,7 +748,6 @@ eval instant at 1m avg_over_time(metric6c[1m])
eval instant at 1m sum_over_time(metric6c[1m])/count_over_time(metric6c[1m]) eval instant at 1m sum_over_time(metric6c[1m])/count_over_time(metric6c[1m])
{} NaN {} NaN
eval instant at 1m avg_over_time(metric7[1m]) eval instant at 1m avg_over_time(metric7[1m])
{} NaN {} NaN
@ -783,6 +782,9 @@ load 10s
eval instant at 1m sum_over_time(metric[1m]) eval instant at 1m sum_over_time(metric[1m])
{} 2 {} 2
eval instant at 1m avg_over_time(metric[1m])
{} 0.5
# Tests for stddev_over_time and stdvar_over_time. # Tests for stddev_over_time and stdvar_over_time.
clear clear
load 10s load 10s

26
promql/promqltest/testdata/histograms.test vendored
View file

@ -482,3 +482,29 @@ load_with_nhcb 5m
eval_fail instant at 50m histogram_quantile(0.99, {__name__=~"request_duration_seconds\\d*_bucket"}) eval_fail instant at 50m histogram_quantile(0.99, {__name__=~"request_duration_seconds\\d*_bucket"})
eval_fail instant at 50m histogram_quantile(0.99, {__name__=~"request_duration_seconds\\d*"}) eval_fail instant at 50m histogram_quantile(0.99, {__name__=~"request_duration_seconds\\d*"})
# Histogram with constant buckets.
load_with_nhcb 1m
const_histogram_bucket{le="0.0"} 1 1 1 1 1
const_histogram_bucket{le="1.0"} 1 1 1 1 1
const_histogram_bucket{le="2.0"} 1 1 1 1 1
const_histogram_bucket{le="+Inf"} 1 1 1 1 1
# There is no change to the bucket count over time, thus rate is 0 in each bucket.
eval instant at 5m rate(const_histogram_bucket[5m])
{le="0.0"} 0
{le="1.0"} 0
{le="2.0"} 0
{le="+Inf"} 0
# Native histograms do not represent empty buckets, so here the zeros are implicit.
eval instant at 5m rate(const_histogram[5m])
{} {{schema:-53 sum:0 count:0 custom_values:[0.0 1.0 2.0]}}
# Zero buckets mean no observations, so there is no value that observations fall below,
# which means that any quantile is a NaN.
eval instant at 5m histogram_quantile(1.0, sum by (le) (rate(const_histogram_bucket[5m])))
{} NaN
eval instant at 5m histogram_quantile(1.0, sum(rate(const_histogram[5m])))
{} NaN

138
promql/promqltest/testdata/native_histograms.test vendored
View file

@ -748,3 +748,141 @@ eval instant at 5m histogram_quantile(0.5, custom_buckets_histogram)
eval instant at 5m sum(custom_buckets_histogram) eval instant at 5m sum(custom_buckets_histogram)
{} {{schema:-53 sum:5 count:4 custom_values:[5 10] buckets:[1 2 1]}} {} {{schema:-53 sum:5 count:4 custom_values:[5 10] buckets:[1 2 1]}}
clear
# Test 'this native histogram metric is not a gauge' warning for rate
load 30s
some_metric {{schema:0 sum:1 count:1 buckets:[1] counter_reset_hint:gauge}} {{schema:0 sum:2 count:2 buckets:[2] counter_reset_hint:gauge}} {{schema:0 sum:3 count:3 buckets:[3] counter_reset_hint:gauge}}
# Test the case where we only have two points for rate
eval_warn instant at 30s rate(some_metric[30s])
{} {{count:0.03333333333333333 sum:0.03333333333333333 buckets:[0.03333333333333333]}}
# Test the case where we have more than two points for rate
eval_warn instant at 1m rate(some_metric[1m])
{} {{count:0.03333333333333333 sum:0.03333333333333333 buckets:[0.03333333333333333]}}
clear
# Test rate() over mixed exponential and custom buckets.
load 30s
some_metric {{schema:0 sum:1 count:1 buckets:[1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} {{schema:0 sum:5 count:4 buckets:[1 2 1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}}
# Start and end with exponential, with custom in the middle.
eval_warn instant at 1m rate(some_metric[1m])
# Should produce no results.
# Start and end with custom, with exponential in the middle.
eval_warn instant at 1m30s rate(some_metric[1m])
# Should produce no results.
# Start with custom, end with exponential.
eval_warn instant at 1m rate(some_metric[30s])
# Should produce no results.
# Start with exponential, end with custom.
eval_warn instant at 30s rate(some_metric[30s])
# Should produce no results.
clear
# Histogram with constant buckets.
load 1m
const_histogram {{schema:0 sum:1 count:1 buckets:[1 1 1]}} {{schema:0 sum:1 count:1 buckets:[1 1 1]}} {{schema:0 sum:1 count:1 buckets:[1 1 1]}} {{schema:0 sum:1 count:1 buckets:[1 1 1]}} {{schema:0 sum:1 count:1 buckets:[1 1 1]}}
# There is no change to the bucket count over time, thus rate is 0 in each bucket.
# However native histograms do not represent empty buckets, so here the zeros are implicit.
eval instant at 5m rate(const_histogram[5m])
{} {{schema:0 sum:0 count:0}}
# Zero buckets mean no observations, thus the denominator in the average is 0
# leading to 0/0, which is NaN.
eval instant at 5m histogram_avg(rate(const_histogram[5m]))
{} NaN
# Zero buckets mean no observations, so count is 0.
eval instant at 5m histogram_count(rate(const_histogram[5m]))
{} 0.0
# Zero buckets mean no observations and empty histogram has a sum of 0 by definition.
eval instant at 5m histogram_sum(rate(const_histogram[5m]))
{} 0.0
# Zero buckets mean no observations, thus the denominator in the fraction is 0,
# leading to 0/0, which is NaN.
eval instant at 5m histogram_fraction(0.0, 1.0, rate(const_histogram[5m]))
{} NaN
# Workaround to calculate the observation count corresponding to NaN fraction.
eval instant at 5m histogram_count(rate(const_histogram[5m])) == 0.0 or histogram_fraction(0.0, 1.0, rate(const_histogram[5m])) * histogram_count(rate(const_histogram[5m]))
{} 0.0
# Zero buckets mean no observations, so there is no value that observations fall below,
# which means that any quantile is a NaN.
eval instant at 5m histogram_quantile(1.0, rate(const_histogram[5m]))
{} NaN
# Zero buckets mean no observations, so there is no standard deviation.
eval instant at 5m histogram_stddev(rate(const_histogram[5m]))
{} NaN
# Zero buckets mean no observations, so there is no standard variance.
eval instant at 5m histogram_stdvar(rate(const_histogram[5m]))
{} NaN
clear
# Test mixing exponential and custom buckets.
load 6m
metric{series="exponential"} {{sum:4 count:3 buckets:[1 2 1]}} _ {{sum:4 count:3 buckets:[1 2 1]}}
metric{series="other-exponential"} {{sum:3 count:2 buckets:[1 1 1]}} _ {{sum:3 count:2 buckets:[1 1 1]}}
metric{series="custom"} _ {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}}
metric{series="other-custom"} _ {{schema:-53 sum:15 count:2 custom_values:[5 10] buckets:[0 2]}} {{schema:-53 sum:15 count:2 custom_values:[5 10] buckets:[0 2]}}
# T=0: only exponential
# T=6: only custom
# T=12: mixed, should be ignored and emit a warning
eval_warn range from 0 to 12m step 6m sum(metric)
{} {{sum:7 count:5 buckets:[2 3 2]}} {{schema:-53 sum:16 count:3 custom_values:[5 10] buckets:[1 2]}} _
eval_warn range from 0 to 12m step 6m avg(metric)
{} {{sum:3.5 count:2.5 buckets:[1 1.5 1]}} {{schema:-53 sum:8 count:1.5 custom_values:[5 10] buckets:[0.5 1]}} _
clear
# Test incompatible custom bucket schemas.
load 6m
metric{series="1"} _ {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}}
metric{series="2"} {{schema:-53 sum:1 count:1 custom_values:[2] buckets:[1]}} _ {{schema:-53 sum:1 count:1 custom_values:[2] buckets:[1]}}
metric{series="3"} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}}
# T=0: incompatible, should be ignored and emit a warning
# T=6: compatible
# T=12: incompatible followed by compatible, should be ignored and emit a warning
eval_warn range from 0 to 12m step 6m sum(metric)
{} _ {{schema:-53 sum:2 count:2 custom_values:[5 10] buckets:[2]}} _
eval_warn range from 0 to 12m step 6m avg(metric)
{} _ {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} _
clear
load 1m
metric{group="just-floats", series="1"} 2
metric{group="just-floats", series="2"} 3
metric{group="just-exponential-histograms", series="1"} {{sum:3 count:4 buckets:[1 2 1]}}
metric{group="just-exponential-histograms", series="2"} {{sum:2 count:3 buckets:[1 1 1]}}
metric{group="just-custom-histograms", series="1"} {{schema:-53 sum:1 count:1 custom_values:[2] buckets:[1]}}
metric{group="just-custom-histograms", series="2"} {{schema:-53 sum:3 count:4 custom_values:[2] buckets:[7]}}
metric{group="floats-and-histograms", series="1"} 2
metric{group="floats-and-histograms", series="2"} {{sum:2 count:3 buckets:[1 1 1]}}
metric{group="exponential-and-custom-histograms", series="1"} {{sum:2 count:3 buckets:[1 1 1]}}
metric{group="exponential-and-custom-histograms", series="2"} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}}
metric{group="incompatible-custom-histograms", series="1"} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}}
metric{group="incompatible-custom-histograms", series="2"} {{schema:-53 sum:1 count:1 custom_values:[2] buckets:[1]}}
eval_warn instant at 0 sum by (group) (metric)
{group="just-floats"} 5
{group="just-exponential-histograms"} {{sum:5 count:7 buckets:[2 3 2]}}
{group="just-custom-histograms"} {{schema:-53 sum:4 count:5 custom_values:[2] buckets:[8]}}

3
rules/manager_test.go
View file

@ -32,7 +32,6 @@ import (
"github.com/prometheus/common/model" "github.com/prometheus/common/model"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.uber.org/atomic" "go.uber.org/atomic"
"go.uber.org/goleak"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/labels"
@ -50,7 +49,7 @@ import (
) )
func TestMain(m *testing.M) { func TestMain(m *testing.M) {
goleak.VerifyTestMain(m) prom_testutil.TolerantVerifyLeak(m)
} }
func TestAlertingRule(t *testing.T) { func TestAlertingRule(t *testing.T) {

4
scrape/clientprotobuf.go
View file

@ -23,7 +23,7 @@ import (
dto "github.com/prometheus/client_model/go" dto "github.com/prometheus/client_model/go"
) )
// Write a MetricFamily into a protobuf. // MetricFamilyToProtobuf writes a MetricFamily into a protobuf.
// This function is intended for testing scraping by providing protobuf serialized input. // This function is intended for testing scraping by providing protobuf serialized input.
func MetricFamilyToProtobuf(metricFamily *dto.MetricFamily) ([]byte, error) { func MetricFamilyToProtobuf(metricFamily *dto.MetricFamily) ([]byte, error) {
buffer := &bytes.Buffer{} buffer := &bytes.Buffer{}
@ -34,7 +34,7 @@ func MetricFamilyToProtobuf(metricFamily *dto.MetricFamily) ([]byte, error) {
return buffer.Bytes(), nil return buffer.Bytes(), nil
} }
// Append a MetricFamily protobuf representation to a buffer. // AddMetricFamilyToProtobuf appends a MetricFamily protobuf representation to a buffer.
// This function is intended for testing scraping by providing protobuf serialized input. // This function is intended for testing scraping by providing protobuf serialized input.
func AddMetricFamilyToProtobuf(buffer *bytes.Buffer, metricFamily *dto.MetricFamily) error { func AddMetricFamilyToProtobuf(buffer *bytes.Buffer, metricFamily *dto.MetricFamily) error {
protoBuf, err := proto.Marshal(metricFamily) protoBuf, err := proto.Marshal(metricFamily)

2
scrape/manager.go
View file

@ -93,6 +93,8 @@ type Options struct {
skipOffsetting bool skipOffsetting bool
} }
const DefaultNameEscapingScheme = model.ValueEncodingEscaping
// Manager maintains a set of scrape pools and manages start/stop cycles // Manager maintains a set of scrape pools and manages start/stop cycles
// when receiving new target groups from the discovery manager. // when receiving new target groups from the discovery manager.
type Manager struct { type Manager struct {

23
scrape/scrape.go
View file

@ -303,6 +303,11 @@ func (sp *scrapePool) restartLoops(reuseCache bool) {
mrc = sp.config.MetricRelabelConfigs mrc = sp.config.MetricRelabelConfigs
) )
validationScheme := model.LegacyValidation
if sp.config.MetricNameValidationScheme == config.UTF8ValidationConfig {
validationScheme = model.UTF8Validation
}
sp.targetMtx.Lock() sp.targetMtx.Lock()
forcedErr := sp.refreshTargetLimitErr() forcedErr := sp.refreshTargetLimitErr()
@ -323,7 +328,7 @@ func (sp *scrapePool) restartLoops(reuseCache bool) {
client: sp.client, client: sp.client,
timeout: timeout, timeout: timeout,
bodySizeLimit: bodySizeLimit, bodySizeLimit: bodySizeLimit,
acceptHeader: acceptHeader(sp.config.ScrapeProtocols), acceptHeader: acceptHeader(sp.config.ScrapeProtocols, validationScheme),
acceptEncodingHeader: acceptEncodingHeader(enableCompression), acceptEncodingHeader: acceptEncodingHeader(enableCompression),
} }
newLoop = sp.newLoop(scrapeLoopOptions{ newLoop = sp.newLoop(scrapeLoopOptions{
@ -452,6 +457,11 @@ func (sp *scrapePool) sync(targets []*Target) {
scrapeClassicHistograms = sp.config.ScrapeClassicHistograms scrapeClassicHistograms = sp.config.ScrapeClassicHistograms
) )
validationScheme := model.LegacyValidation
if sp.config.MetricNameValidationScheme == config.UTF8ValidationConfig {
validationScheme = model.UTF8Validation
}
sp.targetMtx.Lock() sp.targetMtx.Lock()
for _, t := range targets { for _, t := range targets {
hash := t.hash() hash := t.hash()
@ -467,7 +477,7 @@ func (sp *scrapePool) sync(targets []*Target) {
client: sp.client, client: sp.client,
timeout: timeout, timeout: timeout,
bodySizeLimit: bodySizeLimit, bodySizeLimit: bodySizeLimit,
acceptHeader: acceptHeader(sp.config.ScrapeProtocols), acceptHeader: acceptHeader(sp.config.ScrapeProtocols, validationScheme),
acceptEncodingHeader: acceptEncodingHeader(enableCompression), acceptEncodingHeader: acceptEncodingHeader(enableCompression),
metrics: sp.metrics, metrics: sp.metrics,
} }
@ -714,11 +724,16 @@ var errBodySizeLimit = errors.New("body size limit exceeded")
// acceptHeader transforms preference from the options into specific header values as // acceptHeader transforms preference from the options into specific header values as
// https://www.rfc-editor.org/rfc/rfc9110.html#name-accept defines. // https://www.rfc-editor.org/rfc/rfc9110.html#name-accept defines.
// No validation is here, we expect scrape protocols to be validated already. // No validation is here, we expect scrape protocols to be validated already.
func acceptHeader(sps []config.ScrapeProtocol) string { func acceptHeader(sps []config.ScrapeProtocol, scheme model.ValidationScheme) string {
var vals []string var vals []string
weight := len(config.ScrapeProtocolsHeaders) + 1 weight := len(config.ScrapeProtocolsHeaders) + 1
for _, sp := range sps { for _, sp := range sps {
vals = append(vals, fmt.Sprintf("%s;q=0.%d", config.ScrapeProtocolsHeaders[sp], weight)) val := config.ScrapeProtocolsHeaders[sp]
if scheme == model.UTF8Validation {
val += ";" + config.UTF8NamesHeader
}
val += fmt.Sprintf(";q=0.%d", weight)
vals = append(vals, val)
weight-- weight--
} }
// Default match anything. // Default match anything.

29
scrape/scrape_test.go
View file

@ -2339,11 +2339,15 @@ func TestTargetScraperScrapeOK(t *testing.T) {
) )
var protobufParsing bool var protobufParsing bool
var allowUTF8 bool
server := httptest.NewServer( server := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if protobufParsing {
accept := r.Header.Get("Accept") accept := r.Header.Get("Accept")
if allowUTF8 {
require.Truef(t, strings.Contains(accept, "escaping=allow-utf-8"), "Expected Accept header to allow utf8, got %q", accept)
}
if protobufParsing {
require.True(t, strings.HasPrefix(accept, "application/vnd.google.protobuf;"), require.True(t, strings.HasPrefix(accept, "application/vnd.google.protobuf;"),
"Expected Accept header to prefer application/vnd.google.protobuf.") "Expected Accept header to prefer application/vnd.google.protobuf.")
} }
@ -2351,7 +2355,11 @@ func TestTargetScraperScrapeOK(t *testing.T) {
timeout := r.Header.Get("X-Prometheus-Scrape-Timeout-Seconds") timeout := r.Header.Get("X-Prometheus-Scrape-Timeout-Seconds")
require.Equal(t, expectedTimeout, timeout, "Expected scrape timeout header.") require.Equal(t, expectedTimeout, timeout, "Expected scrape timeout header.")
if allowUTF8 {
w.Header().Set("Content-Type", `text/plain; version=1.0.0; escaping=allow-utf-8`)
} else {
w.Header().Set("Content-Type", `text/plain; version=0.0.4`) w.Header().Set("Content-Type", `text/plain; version=0.0.4`)
}
w.Write([]byte("metric_a 1\nmetric_b 2\n")) w.Write([]byte("metric_a 1\nmetric_b 2\n"))
}), }),
) )
@ -2380,13 +2388,22 @@ func TestTargetScraperScrapeOK(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
contentType, err := ts.readResponse(context.Background(), resp, &buf) contentType, err := ts.readResponse(context.Background(), resp, &buf)
require.NoError(t, err) require.NoError(t, err)
if allowUTF8 {
require.Equal(t, "text/plain; version=1.0.0; escaping=allow-utf-8", contentType)
} else {
require.Equal(t, "text/plain; version=0.0.4", contentType) require.Equal(t, "text/plain; version=0.0.4", contentType)
}
require.Equal(t, "metric_a 1\nmetric_b 2\n", buf.String()) require.Equal(t, "metric_a 1\nmetric_b 2\n", buf.String())
} }
runTest(acceptHeader(config.DefaultScrapeProtocols)) runTest(acceptHeader(config.DefaultScrapeProtocols, model.LegacyValidation))
protobufParsing = true protobufParsing = true
runTest(acceptHeader(config.DefaultProtoFirstScrapeProtocols)) runTest(acceptHeader(config.DefaultProtoFirstScrapeProtocols, model.LegacyValidation))
protobufParsing = false
allowUTF8 = true
runTest(acceptHeader(config.DefaultScrapeProtocols, model.UTF8Validation))
protobufParsing = true
runTest(acceptHeader(config.DefaultProtoFirstScrapeProtocols, model.UTF8Validation))
} }
func TestTargetScrapeScrapeCancel(t *testing.T) { func TestTargetScrapeScrapeCancel(t *testing.T) {
@ -2412,7 +2429,7 @@ func TestTargetScrapeScrapeCancel(t *testing.T) {
), ),
}, },
client: http.DefaultClient, client: http.DefaultClient,
acceptHeader: acceptHeader(config.DefaultGlobalConfig.ScrapeProtocols), acceptHeader: acceptHeader(config.DefaultGlobalConfig.ScrapeProtocols, model.LegacyValidation),
} }
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@ -2467,7 +2484,7 @@ func TestTargetScrapeScrapeNotFound(t *testing.T) {
), ),
}, },
client: http.DefaultClient, client: http.DefaultClient,
acceptHeader: acceptHeader(config.DefaultGlobalConfig.ScrapeProtocols), acceptHeader: acceptHeader(config.DefaultGlobalConfig.ScrapeProtocols, model.LegacyValidation),
} }
resp, err := ts.scrape(context.Background()) resp, err := ts.scrape(context.Background())
@ -2511,7 +2528,7 @@ func TestTargetScraperBodySizeLimit(t *testing.T) {
}, },
client: http.DefaultClient, client: http.DefaultClient,
bodySizeLimit: bodySizeLimit, bodySizeLimit: bodySizeLimit,
acceptHeader: acceptHeader(config.DefaultGlobalConfig.ScrapeProtocols), acceptHeader: acceptHeader(config.DefaultGlobalConfig.ScrapeProtocols, model.LegacyValidation),
metrics: newTestScrapeMetrics(t), metrics: newTestScrapeMetrics(t),
} }
var buf bytes.Buffer var buf bytes.Buffer

76
scrape/testdata/ca.cer vendored
View file

@ -1,8 +1,66 @@
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
93:6c:9e:29:8d:37:7b:66
Signature Algorithm: sha256WithRSAEncryption
Issuer: C = XX, L = Default City, O = Default Company Ltd, CN = Prometheus Test CA
Validity
Not Before: Aug 20 11:51:23 2024 GMT
Not After : Dec 5 11:51:23 2044 GMT
Subject: C = XX, L = Default City, O = Default Company Ltd, CN = Prometheus Test CA
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
Public-Key: (2048 bit)
Modulus:
00:e9:52:05:4d:f2:5a:95:04:2d:b8:73:8b:3c:e7:
47:48:67:00:be:dd:6c:41:f3:7c:4b:44:73:a3:77:
3e:84:af:30:d7:2a:ad:45:6a:b7:89:23:05:15:b3:
aa:46:79:b8:95:64:cc:13:c4:44:a1:01:a0:e2:3d:
a5:67:2b:aa:d3:13:06:43:33:1c:96:36:12:9e:c6:
1d:36:9b:d7:47:bd:28:2d:88:15:04:fa:14:a3:ff:
8c:26:22:c5:a2:15:c7:76:b3:11:f6:a3:44:9a:28:
0f:ca:fb:f4:51:a8:6a:05:94:7c:77:47:c8:21:56:
25:bf:e2:2e:df:33:f3:e4:bd:d6:47:a5:49:13:12:
c8:1f:4c:d7:2a:56:a2:6c:c1:cf:55:05:5d:9a:75:
a2:23:4e:e6:8e:ff:76:05:d6:e0:c8:0b:51:f0:b6:
a1:b2:7d:8f:9c:6a:a5:ce:86:92:15:8c:5b:86:45:
c0:4a:ff:54:b8:ee:cf:11:bd:07:cb:4b:7d:0b:a1:
9d:72:86:9f:55:bc:f9:6c:d9:55:60:96:30:3f:ec:
2d:f6:5f:9a:32:9a:5a:5c:1c:5f:32:f9:d6:0f:04:
f8:81:08:04:9a:95:c3:9d:5a:30:8e:a5:0e:47:2f:
00:ce:e0:2e:ad:5a:b8:b6:4c:55:7c:8a:59:22:b0:
ed:73
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Subject Key Identifier:
CC:F5:05:99:E5:AB:12:69:D8:78:89:4A:31:CA:F0:8B:0B:AD:66:1B
X509v3 Authority Key Identifier:
CC:F5:05:99:E5:AB:12:69:D8:78:89:4A:31:CA:F0:8B:0B:AD:66:1B
X509v3 Basic Constraints:
CA:TRUE
Signature Algorithm: sha256WithRSAEncryption
Signature Value:
4a:a1:b0:bc:c8:87:4f:7c:96:62:e5:09:29:ae:3a:2e:68:d0:
d2:c5:68:ed:ea:83:36:b1:86:f3:b9:e9:19:2b:b6:73:10:6f:
df:7f:bb:f1:76:81:03:c1:a1:5a:ee:6c:44:b8:7c:10:d1:5a:
d7:c1:92:64:59:35:a6:e0:aa:08:41:37:6e:e7:c8:b6:bd:0c:
4b:47:78:ec:c4:b4:15:a3:62:76:4a:39:8e:6e:19:ff:f0:c0:
8a:7e:1c:cd:87:e5:00:6c:f1:ce:27:26:ff:b8:e9:eb:f7:2f:
bd:c2:4b:9c:d6:57:de:74:74:b3:4f:03:98:9a:b5:08:2d:16:
ca:7f:b6:c8:76:62:86:1b:7c:f2:3e:6c:78:cc:2c:95:9a:bb:
77:25:e8:80:ff:9b:e8:f8:9a:85:3b:85:b7:17:4e:77:a1:cf:
4d:b9:d0:25:e8:5d:8c:e6:7c:f1:d9:52:30:3d:ec:2b:37:91:
bc:e2:e8:39:31:6f:3d:e9:98:70:80:7c:41:dd:19:13:05:21:
94:7b:16:cf:d8:ee:4e:38:34:5e:6a:ff:cd:85:ac:8f:94:9a:
dd:4e:77:05:13:a6:b4:80:52:b2:97:64:76:88:f4:dd:42:0a:
50:1c:80:fd:4b:6e:a9:62:10:aa:ef:2e:c1:2f:be:0e:c2:2e:
b5:28:5f:83
-----BEGIN CERTIFICATE----- -----BEGIN CERTIFICATE-----
MIIDkTCCAnmgAwIBAgIJAJNsnimNN3tmMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV MIIDkTCCAnmgAwIBAgIJAJNsnimNN3tmMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV
BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg
Q29tcGFueSBMdGQxGzAZBgNVBAMMElByb21ldGhldXMgVGVzdCBDQTAeFw0xNTA4 Q29tcGFueSBMdGQxGzAZBgNVBAMMElByb21ldGhldXMgVGVzdCBDQTAeFw0yNDA4
MDQxNDA5MjFaFw0yNTA4MDExNDA5MjFaMF8xCzAJBgNVBAYTAlhYMRUwEwYDVQQH MjAxMTUxMjNaFw00NDEyMDUxMTUxMjNaMF8xCzAJBgNVBAYTAlhYMRUwEwYDVQQH
DAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxGzAZ DAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxGzAZ
BgNVBAMMElByb21ldGhldXMgVGVzdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP BgNVBAMMElByb21ldGhldXMgVGVzdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP
ADCCAQoCggEBAOlSBU3yWpUELbhzizznR0hnAL7dbEHzfEtEc6N3PoSvMNcqrUVq ADCCAQoCggEBAOlSBU3yWpUELbhzizznR0hnAL7dbEHzfEtEc6N3PoSvMNcqrUVq
@ -12,11 +70,11 @@ yB9M1ypWomzBz1UFXZp1oiNO5o7/dgXW4MgLUfC2obJ9j5xqpc6GkhWMW4ZFwEr/
VLjuzxG9B8tLfQuhnXKGn1W8+WzZVWCWMD/sLfZfmjKaWlwcXzL51g8E+IEIBJqV VLjuzxG9B8tLfQuhnXKGn1W8+WzZVWCWMD/sLfZfmjKaWlwcXzL51g8E+IEIBJqV
w51aMI6lDkcvAM7gLq1auLZMVXyKWSKw7XMCAwEAAaNQME4wHQYDVR0OBBYEFMz1 w51aMI6lDkcvAM7gLq1auLZMVXyKWSKw7XMCAwEAAaNQME4wHQYDVR0OBBYEFMz1
BZnlqxJp2HiJSjHK8IsLrWYbMB8GA1UdIwQYMBaAFMz1BZnlqxJp2HiJSjHK8IsL BZnlqxJp2HiJSjHK8IsLrWYbMB8GA1UdIwQYMBaAFMz1BZnlqxJp2HiJSjHK8IsL
rWYbMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAI2iA3w3TK5J15Pu rWYbMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAEqhsLzIh098lmLl
e4fPFB4jxQqsbUwuyXbCCv/jKLeFNCD4BjM181WZEYjPMumeTBVzU3aF45LWQIG1 CSmuOi5o0NLFaO3qgzaxhvO56RkrtnMQb99/u/F2gQPBoVrubES4fBDRWtfBkmRZ
0DJcrCL4mjMz9qgAoGqA7aDDXiJGbukMgYYsn7vrnVmrZH8T3E8ySlltr7+W578k NabgqghBN27nyLa9DEtHeOzEtBWjYnZKOY5uGf/wwIp+HM2H5QBs8c4nJv+46ev3
pJ5FxnbCroQwn0zLyVB3sFbS8E3vpBr3L8oy8PwPHhIScexcNVc3V6/m4vTZsXTH L73CS5zWV950dLNPA5iatQgtFsp/tsh2YoYbfPI+bHjMLJWau3cl6ID/m+j4moU7
U+vUm1XhDgpDcFMTg2QQiJbfpOYUkwIgnRDAT7t282t2KQWtnlqc3zwPQ1F/6Cpx hbcXTnehz0250CXoXYzmfPHZUjA97Cs3kbzi6Dkxbz3pmHCAfEHdGRMFIZR7Fs/Y
j19JeNsaF1DArkD7YlyKj/GhZLtHwFHG5cxznH0mLDJTW7bQvqqh2iQTeXmBk1lU 7k44NF5q/82FrI+Umt1OdwUTprSAUrKXZHaI9N1CClAcgP1LbqliEKrvLsEvvg7C
mM5lH/s= LrUoX4M=
-----END CERTIFICATE----- -----END CERTIFICATE-----

6
scripts/golangci-lint.yml
View file

@ -26,14 +26,14 @@ jobs:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: Install Go - name: Install Go
uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
with: with:
go-version: 1.22.x go-version: 1.22.x
- name: Install snmp_exporter/generator dependencies - name: Install snmp_exporter/generator dependencies
run: sudo apt-get update && sudo apt-get -y install libsnmp-dev run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
if: github.repository == 'prometheus/snmp_exporter' if: github.repository == 'prometheus/snmp_exporter'
- name: Lint - name: Lint
uses: golangci/golangci-lint-action@a4f60bb28d35aeee14e6880718e0c85ff1882e64 # v6.0.1 uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86 # v6.1.0
with: with:
args: --verbose args: --verbose
version: v1.59.1 version: v1.60.2

16
storage/buffer_test.go
View file

@ -96,10 +96,10 @@ func TestSampleRingMixed(t *testing.T) {
// With ValNone as the preferred type, nothing should be initialized. // With ValNone as the preferred type, nothing should be initialized.
r := newSampleRing(10, 2, chunkenc.ValNone) r := newSampleRing(10, 2, chunkenc.ValNone)
require.Zero(t, len(r.fBuf)) require.Empty(t, r.fBuf)
require.Zero(t, len(r.hBuf)) require.Empty(t, r.hBuf)
require.Zero(t, len(r.fhBuf)) require.Empty(t, r.fhBuf)
require.Zero(t, len(r.iBuf)) require.Empty(t, r.iBuf)
// But then mixed adds should work as expected. // But then mixed adds should work as expected.
r.addF(fSample{t: 1, f: 3.14}) r.addF(fSample{t: 1, f: 3.14})
@ -146,10 +146,10 @@ func TestSampleRingAtFloatHistogram(t *testing.T) {
// With ValNone as the preferred type, nothing should be initialized. // With ValNone as the preferred type, nothing should be initialized.
r := newSampleRing(10, 2, chunkenc.ValNone) r := newSampleRing(10, 2, chunkenc.ValNone)
require.Zero(t, len(r.fBuf)) require.Empty(t, r.fBuf)
require.Zero(t, len(r.hBuf)) require.Empty(t, r.hBuf)
require.Zero(t, len(r.fhBuf)) require.Empty(t, r.fhBuf)
require.Zero(t, len(r.iBuf)) require.Empty(t, r.iBuf)
var ( var (
h *histogram.Histogram h *histogram.Histogram

14
storage/errors.go
View file

@ -18,6 +18,7 @@ import "fmt"
type errDuplicateSampleForTimestamp struct { type errDuplicateSampleForTimestamp struct {
timestamp int64 timestamp int64
existing float64 existing float64
existingIsHistogram bool
newValue float64 newValue float64
} }
@ -29,13 +30,26 @@ func NewDuplicateFloatErr(t int64, existing, newValue float64) error {
} }
} }
// NewDuplicateHistogramToFloatErr describes an error where a new float sample is sent for same timestamp as previous histogram.
func NewDuplicateHistogramToFloatErr(t int64, newValue float64) error {
return errDuplicateSampleForTimestamp{
timestamp: t,
existingIsHistogram: true,
newValue: newValue,
}
}
func (e errDuplicateSampleForTimestamp) Error() string { func (e errDuplicateSampleForTimestamp) Error() string {
if e.timestamp == 0 { if e.timestamp == 0 {
return "duplicate sample for timestamp" return "duplicate sample for timestamp"
} }
if e.existingIsHistogram {
return fmt.Sprintf("duplicate sample for timestamp %d; overrides not allowed: existing is a histogram, new value %g", e.timestamp, e.newValue)
}
return fmt.Sprintf("duplicate sample for timestamp %d; overrides not allowed: existing %g, new value %g", e.timestamp, e.existing, e.newValue) return fmt.Sprintf("duplicate sample for timestamp %d; overrides not allowed: existing %g, new value %g", e.timestamp, e.existing, e.newValue)
} }
// Is implements the anonymous interface checked by errors.Is.
// Every errDuplicateSampleForTimestamp compares equal to the global ErrDuplicateSampleForTimestamp. // Every errDuplicateSampleForTimestamp compares equal to the global ErrDuplicateSampleForTimestamp.
func (e errDuplicateSampleForTimestamp) Is(t error) bool { func (e errDuplicateSampleForTimestamp) Is(t error) bool {
if t == ErrDuplicateSampleForTimestamp { if t == ErrDuplicateSampleForTimestamp {

38
storage/errors_test.go Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestErrDuplicateSampleForTimestamp(t *testing.T) {
// All errDuplicateSampleForTimestamp are ErrDuplicateSampleForTimestamp
require.ErrorIs(t, ErrDuplicateSampleForTimestamp, errDuplicateSampleForTimestamp{})
// Same type only is if it has same properties.
err := NewDuplicateFloatErr(1_000, 10, 20)
sameErr := NewDuplicateFloatErr(1_000, 10, 20)
differentErr := NewDuplicateFloatErr(1_001, 30, 40)
require.ErrorIs(t, err, sameErr)
require.NotErrorIs(t, err, differentErr)
// Also works when err is wrapped.
require.ErrorIs(t, fmt.Errorf("failed: %w", err), sameErr)
require.NotErrorIs(t, fmt.Errorf("failed: %w", err), differentErr)
}

2
storage/interface.go
View file

@ -227,9 +227,9 @@ type LabelHints struct {
Limit int Limit int
} }
// TODO(bwplotka): Move to promql/engine_test.go?
// QueryableFunc is an adapter to allow the use of ordinary functions as // QueryableFunc is an adapter to allow the use of ordinary functions as
// Queryables. It follows the idea of http.HandlerFunc. // Queryables. It follows the idea of http.HandlerFunc.
// TODO(bwplotka): Move to promql/engine_test.go?
type QueryableFunc func(mint, maxt int64) (Querier, error) type QueryableFunc func(mint, maxt int64) (Querier, error)
// Querier calls f() with the given parameters. // Querier calls f() with the given parameters.

6
storage/remote/azuread/azuread.go
View file

@ -31,13 +31,15 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
) )
// Clouds.
const ( const (
// Clouds.
AzureChina = "AzureChina" AzureChina = "AzureChina"
AzureGovernment = "AzureGovernment" AzureGovernment = "AzureGovernment"
AzurePublic = "AzurePublic" AzurePublic = "AzurePublic"
)
// Audiences. // Audiences.
const (
IngestionChinaAudience = "https://monitor.azure.cn//.default" IngestionChinaAudience = "https://monitor.azure.cn//.default"
IngestionGovernmentAudience = "https://monitor.azure.us//.default" IngestionGovernmentAudience = "https://monitor.azure.us//.default"
IngestionPublicAudience = "https://monitor.azure.com//.default" IngestionPublicAudience = "https://monitor.azure.com//.default"

12
storage/remote/client.go
View file

@ -37,6 +37,7 @@ import (
"github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/storage/remote/azuread" "github.com/prometheus/prometheus/storage/remote/azuread"
"github.com/prometheus/prometheus/storage/remote/googleiam"
) )
const maxErrMsgLen = 1024 const maxErrMsgLen = 1024
@ -131,6 +132,7 @@ type ClientConfig struct {
HTTPClientConfig config_util.HTTPClientConfig HTTPClientConfig config_util.HTTPClientConfig
SigV4Config *sigv4.SigV4Config SigV4Config *sigv4.SigV4Config
AzureADConfig *azuread.AzureADConfig AzureADConfig *azuread.AzureADConfig
GoogleIAMConfig *googleiam.Config
Headers map[string]string Headers map[string]string
RetryOnRateLimit bool RetryOnRateLimit bool
WriteProtoMsg config.RemoteWriteProtoMsg WriteProtoMsg config.RemoteWriteProtoMsg
@ -192,6 +194,13 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) {
} }
} }
if conf.GoogleIAMConfig != nil {
t, err = googleiam.NewRoundTripper(conf.GoogleIAMConfig, t)
if err != nil {
return nil, err
}
}
writeProtoMsg := config.RemoteWriteProtoMsgV1 writeProtoMsg := config.RemoteWriteProtoMsgV1
if conf.WriteProtoMsg != "" { if conf.WriteProtoMsg != "" {
writeProtoMsg = conf.WriteProtoMsg writeProtoMsg = conf.WriteProtoMsg
@ -278,7 +287,6 @@ func (c *Client) Store(ctx context.Context, req []byte, attempt int) (WriteRespo
// we can continue handling. // we can continue handling.
rs, _ := ParseWriteResponseStats(httpResp) rs, _ := ParseWriteResponseStats(httpResp)
//nolint:usestdlibvars
if httpResp.StatusCode/100 == 2 { if httpResp.StatusCode/100 == 2 {
return rs, nil return rs, nil
} }
@ -288,7 +296,6 @@ func (c *Client) Store(ctx context.Context, req []byte, attempt int) (WriteRespo
body, _ := io.ReadAll(io.LimitReader(httpResp.Body, maxErrMsgLen)) body, _ := io.ReadAll(io.LimitReader(httpResp.Body, maxErrMsgLen))
err = fmt.Errorf("server returned HTTP status %s: %s", httpResp.Status, body) err = fmt.Errorf("server returned HTTP status %s: %s", httpResp.Status, body)
//nolint:usestdlibvars
if httpResp.StatusCode/100 == 5 || if httpResp.StatusCode/100 == 5 ||
(c.retryOnRateLimit && httpResp.StatusCode == http.StatusTooManyRequests) { (c.retryOnRateLimit && httpResp.StatusCode == http.StatusTooManyRequests) {
return rs, RecoverableError{err, retryAfterDuration(httpResp.Header.Get("Retry-After"))} return rs, RecoverableError{err, retryAfterDuration(httpResp.Header.Get("Retry-After"))}
@ -373,7 +380,6 @@ func (c *Client) Read(ctx context.Context, query *prompb.Query) (*prompb.QueryRe
return nil, fmt.Errorf("error reading response. HTTP status code: %s: %w", httpResp.Status, err) return nil, fmt.Errorf("error reading response. HTTP status code: %s: %w", httpResp.Status, err)
} }
//nolint:usestdlibvars
if httpResp.StatusCode/100 != 2 { if httpResp.StatusCode/100 != 2 {
return nil, fmt.Errorf("remote server %s returned HTTP status %s: %s", c.urlString, httpResp.Status, strings.TrimSpace(string(compressed))) return nil, fmt.Errorf("remote server %s returned HTTP status %s: %s", c.urlString, httpResp.Status, strings.TrimSpace(string(compressed)))
} }

54
storage/remote/googleiam/googleiam.go Normal file
View file

@ -0,0 +1,54 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package googleiam provides an http.RoundTripper that attaches an Google Cloud accessToken
// to remote write requests.
package googleiam
import (
"context"
"fmt"
"net/http"
"golang.org/x/oauth2/google"
"google.golang.org/api/option"
apihttp "google.golang.org/api/transport/http"
)
type Config struct {
CredentialsFile string `yaml:"credentials_file,omitempty"`
}
// NewRoundTripper creates a round tripper that adds Google Cloud Monitoring authorization to calls
// using either a credentials file or the default credentials.
func NewRoundTripper(cfg *Config, next http.RoundTripper) (http.RoundTripper, error) {
if next == nil {
next = http.DefaultTransport
}
const scopes = "https://www.googleapis.com/auth/monitoring.write"
ctx := context.Background()
opts := []option.ClientOption{
option.WithScopes(scopes),
}
if cfg.CredentialsFile != "" {
opts = append(opts, option.WithCredentialsFile(cfg.CredentialsFile))
} else {
creds, err := google.FindDefaultCredentials(ctx, scopes)
if err != nil {
return nil, fmt.Errorf("error finding default Google credentials: %w", err)
}
opts = append(opts, option.WithCredentials(creds))
}
return apihttp.NewTransport(ctx, next, opts...)
}

11
storage/remote/otlptranslator/prometheus/normalize_label.go
View file

@ -21,15 +21,14 @@ import (
"unicode" "unicode"
) )
// Normalizes the specified label to follow Prometheus label names standard // Normalizes the specified label to follow Prometheus label names standard.
// //
// See rules at https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels // See rules at https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels.
// //
// Labels that start with non-letter rune will be prefixed with "key_" // Labels that start with non-letter rune will be prefixed with "key_".
// //
// Exception is made for double-underscores which are allowed // An exception is made for double-underscores which are allowed.
func NormalizeLabel(label string) string { func NormalizeLabel(label string) string {
// Trivial case // Trivial case
if len(label) == 0 { if len(label) == 0 {
return label return label
@ -48,7 +47,7 @@ func NormalizeLabel(label string) string {
return label return label
} }
// Return '_' for anything non-alphanumeric // Return '_' for anything non-alphanumeric.
func sanitizeRune(r rune) rune { func sanitizeRune(r rune) rune {
if unicode.IsLetter(r) || unicode.IsDigit(r) { if unicode.IsLetter(r) || unicode.IsDigit(r) {
return r return r

25
storage/remote/otlptranslator/prometheus/normalize_name.go
View file

@ -76,14 +76,15 @@ var perUnitMap = map[string]string{
"y": "year", "y": "year",
} }
// BuildCompliantName builds a Prometheus-compliant metric name for the specified metric // BuildCompliantName builds a Prometheus-compliant metric name for the specified metric.
// //
// Metric name is prefixed with specified namespace and underscore (if any). // Metric name is prefixed with specified namespace and underscore (if any).
// Namespace is not cleaned up. Make sure specified namespace follows Prometheus // Namespace is not cleaned up. Make sure specified namespace follows Prometheus
// naming convention. // naming convention.
// //
// See rules at https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels // See rules at https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels,
// and https://prometheus.io/docs/practices/naming/#metric-and-label-naming // https://prometheus.io/docs/practices/naming/#metric-and-label-naming
// and https://github.com/open-telemetry/opentelemetry-specification/blob/v1.33.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus.
func BuildCompliantName(metric pmetric.Metric, namespace string, addMetricSuffixes bool) string { func BuildCompliantName(metric pmetric.Metric, namespace string, addMetricSuffixes bool) string {
var metricName string var metricName string
@ -110,7 +111,7 @@ func BuildCompliantName(metric pmetric.Metric, namespace string, addMetricSuffix
// Build a normalized name for the specified metric // Build a normalized name for the specified metric
func normalizeName(metric pmetric.Metric, namespace string) string { func normalizeName(metric pmetric.Metric, namespace string) string {
// Split metric name in "tokens" (remove all non-alphanumeric) // Split metric name into "tokens" (remove all non-alphanumerics)
nameTokens := strings.FieldsFunc( nameTokens := strings.FieldsFunc(
metric.Name(), metric.Name(),
func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) },
@ -122,9 +123,9 @@ func normalizeName(metric pmetric.Metric, namespace string) string {
// Main unit // Main unit
// Append if not blank, doesn't contain '{}', and is not present in metric name already // Append if not blank, doesn't contain '{}', and is not present in metric name already
if len(unitTokens) > 0 { if len(unitTokens) > 0 {
mainUnitOtel := strings.TrimSpace(unitTokens[0]) mainUnitOTel := strings.TrimSpace(unitTokens[0])
if mainUnitOtel != "" && !strings.ContainsAny(mainUnitOtel, "{}") { if mainUnitOTel != "" && !strings.ContainsAny(mainUnitOTel, "{}") {
mainUnitProm := CleanUpString(unitMapGetOrDefault(mainUnitOtel)) mainUnitProm := CleanUpString(unitMapGetOrDefault(mainUnitOTel))
if mainUnitProm != "" && !contains(nameTokens, mainUnitProm) { if mainUnitProm != "" && !contains(nameTokens, mainUnitProm) {
nameTokens = append(nameTokens, mainUnitProm) nameTokens = append(nameTokens, mainUnitProm)
} }
@ -133,11 +134,11 @@ func normalizeName(metric pmetric.Metric, namespace string) string {
// Per unit // Per unit
// Append if not blank, doesn't contain '{}', and is not present in metric name already // Append if not blank, doesn't contain '{}', and is not present in metric name already
if len(unitTokens) > 1 && unitTokens[1] != "" { if len(unitTokens) > 1 && unitTokens[1] != "" {
perUnitOtel := strings.TrimSpace(unitTokens[1]) perUnitOTel := strings.TrimSpace(unitTokens[1])
if perUnitOtel != "" && !strings.ContainsAny(perUnitOtel, "{}") { if perUnitOTel != "" && !strings.ContainsAny(perUnitOTel, "{}") {
perUnitProm := CleanUpString(perUnitMapGetOrDefault(perUnitOtel)) perUnitProm := CleanUpString(perUnitMapGetOrDefault(perUnitOTel))
if perUnitProm != "" && !contains(nameTokens, perUnitProm) { if perUnitProm != "" && !contains(nameTokens, perUnitProm) {
nameTokens = append(append(nameTokens, "per"), perUnitProm) nameTokens = append(nameTokens, "per", perUnitProm)
} }
} }
} }
@ -150,7 +151,7 @@ func normalizeName(metric pmetric.Metric, namespace string) string {
} }
// Append _ratio for metrics with unit "1" // Append _ratio for metrics with unit "1"
// Some Otel receivers improperly use unit "1" for counters of objects // Some OTel receivers improperly use unit "1" for counters of objects
// See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aissue+some+metric+units+don%27t+follow+otel+semantic+conventions // See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aissue+some+metric+units+don%27t+follow+otel+semantic+conventions
// Until these issues have been fixed, we're appending `_ratio` for gauges ONLY // Until these issues have been fixed, we're appending `_ratio` for gauges ONLY
// Theoretically, counters could be ratios as well, but it's absurd (for mathematical reasons) // Theoretically, counters could be ratios as well, but it's absurd (for mathematical reasons)

205
storage/remote/otlptranslator/prometheus/normalize_name_test.go Normal file
View file

@ -0,0 +1,205 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Provenance-includes-location: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/95e8f8fdc2a9dc87230406c9a3cf02be4fd68bea/pkg/translator/prometheus/normalize_name_test.go
// Provenance-includes-license: Apache-2.0
// Provenance-includes-copyright: Copyright The OpenTelemetry Authors.
package prometheus
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/collector/pdata/pmetric"
)
func TestByte(t *testing.T) {
require.Equal(t, "system_filesystem_usage_bytes", normalizeName(createGauge("system.filesystem.usage", "By"), ""))
}
func TestByteCounter(t *testing.T) {
require.Equal(t, "system_io_bytes_total", normalizeName(createCounter("system.io", "By"), ""))
require.Equal(t, "network_transmitted_bytes_total", normalizeName(createCounter("network_transmitted_bytes_total", "By"), ""))
}
func TestWhiteSpaces(t *testing.T) {
require.Equal(t, "system_filesystem_usage_bytes", normalizeName(createGauge("\t system.filesystem.usage ", " By\t"), ""))
}
func TestNonStandardUnit(t *testing.T) {
require.Equal(t, "system_network_dropped", normalizeName(createGauge("system.network.dropped", "{packets}"), ""))
}
func TestNonStandardUnitCounter(t *testing.T) {
require.Equal(t, "system_network_dropped_total", normalizeName(createCounter("system.network.dropped", "{packets}"), ""))
}
func TestBrokenUnit(t *testing.T) {
require.Equal(t, "system_network_dropped_packets", normalizeName(createGauge("system.network.dropped", "packets"), ""))
require.Equal(t, "system_network_packets_dropped", normalizeName(createGauge("system.network.packets.dropped", "packets"), ""))
require.Equal(t, "system_network_packets", normalizeName(createGauge("system.network.packets", "packets"), ""))
}
func TestBrokenUnitCounter(t *testing.T) {
require.Equal(t, "system_network_dropped_packets_total", normalizeName(createCounter("system.network.dropped", "packets"), ""))
require.Equal(t, "system_network_packets_dropped_total", normalizeName(createCounter("system.network.packets.dropped", "packets"), ""))
require.Equal(t, "system_network_packets_total", normalizeName(createCounter("system.network.packets", "packets"), ""))
}
func TestRatio(t *testing.T) {
require.Equal(t, "hw_gpu_memory_utilization_ratio", normalizeName(createGauge("hw.gpu.memory.utilization", "1"), ""))
require.Equal(t, "hw_fan_speed_ratio", normalizeName(createGauge("hw.fan.speed_ratio", "1"), ""))
require.Equal(t, "objects_total", normalizeName(createCounter("objects", "1"), ""))
}
func TestHertz(t *testing.T) {
require.Equal(t, "hw_cpu_speed_limit_hertz", normalizeName(createGauge("hw.cpu.speed_limit", "Hz"), ""))
}
func TestPer(t *testing.T) {
require.Equal(t, "broken_metric_speed_km_per_hour", normalizeName(createGauge("broken.metric.speed", "km/h"), ""))
require.Equal(t, "astro_light_speed_limit_meters_per_second", normalizeName(createGauge("astro.light.speed_limit", "m/s"), ""))
}
func TestPercent(t *testing.T) {
require.Equal(t, "broken_metric_success_ratio_percent", normalizeName(createGauge("broken.metric.success_ratio", "%"), ""))
require.Equal(t, "broken_metric_success_percent", normalizeName(createGauge("broken.metric.success_percent", "%"), ""))
}
func TestEmpty(t *testing.T) {
require.Equal(t, "test_metric_no_unit", normalizeName(createGauge("test.metric.no_unit", ""), ""))
require.Equal(t, "test_metric_spaces", normalizeName(createGauge("test.metric.spaces", " \t "), ""))
}
func TestUnsupportedRunes(t *testing.T) {
require.Equal(t, "unsupported_metric_temperature_F", normalizeName(createGauge("unsupported.metric.temperature", "°F"), ""))
require.Equal(t, "unsupported_metric_weird", normalizeName(createGauge("unsupported.metric.weird", "+=.:,!* & #"), ""))
require.Equal(t, "unsupported_metric_redundant_test_per_C", normalizeName(createGauge("unsupported.metric.redundant", "__test $/°C"), ""))
}
func TestOTelReceivers(t *testing.T) {
require.Equal(t, "active_directory_ds_replication_network_io_bytes_total", normalizeName(createCounter("active_directory.ds.replication.network.io", "By"), ""))
require.Equal(t, "active_directory_ds_replication_sync_object_pending_total", normalizeName(createCounter("active_directory.ds.replication.sync.object.pending", "{objects}"), ""))
require.Equal(t, "active_directory_ds_replication_object_rate_per_second", normalizeName(createGauge("active_directory.ds.replication.object.rate", "{objects}/s"), ""))
require.Equal(t, "active_directory_ds_name_cache_hit_rate_percent", normalizeName(createGauge("active_directory.ds.name_cache.hit_rate", "%"), ""))
require.Equal(t, "active_directory_ds_ldap_bind_last_successful_time_milliseconds", normalizeName(createGauge("active_directory.ds.ldap.bind.last_successful.time", "ms"), ""))
require.Equal(t, "apache_current_connections", normalizeName(createGauge("apache.current_connections", "connections"), ""))
require.Equal(t, "apache_workers_connections", normalizeName(createGauge("apache.workers", "connections"), ""))
require.Equal(t, "apache_requests_total", normalizeName(createCounter("apache.requests", "1"), ""))
require.Equal(t, "bigip_virtual_server_request_count_total", normalizeName(createCounter("bigip.virtual_server.request.count", "{requests}"), ""))
require.Equal(t, "system_cpu_utilization_ratio", normalizeName(createGauge("system.cpu.utilization", "1"), ""))
require.Equal(t, "system_disk_operation_time_seconds_total", normalizeName(createCounter("system.disk.operation_time", "s"), ""))
require.Equal(t, "system_cpu_load_average_15m_ratio", normalizeName(createGauge("system.cpu.load_average.15m", "1"), ""))
require.Equal(t, "memcached_operation_hit_ratio_percent", normalizeName(createGauge("memcached.operation_hit_ratio", "%"), ""))
require.Equal(t, "mongodbatlas_process_asserts_per_second", normalizeName(createGauge("mongodbatlas.process.asserts", "{assertions}/s"), ""))
require.Equal(t, "mongodbatlas_process_journaling_data_files_mebibytes", normalizeName(createGauge("mongodbatlas.process.journaling.data_files", "MiBy"), ""))
require.Equal(t, "mongodbatlas_process_network_io_bytes_per_second", normalizeName(createGauge("mongodbatlas.process.network.io", "By/s"), ""))
require.Equal(t, "mongodbatlas_process_oplog_rate_gibibytes_per_hour", normalizeName(createGauge("mongodbatlas.process.oplog.rate", "GiBy/h"), ""))
require.Equal(t, "mongodbatlas_process_db_query_targeting_scanned_per_returned", normalizeName(createGauge("mongodbatlas.process.db.query_targeting.scanned_per_returned", "{scanned}/{returned}"), ""))
require.Equal(t, "nginx_requests", normalizeName(createGauge("nginx.requests", "requests"), ""))
require.Equal(t, "nginx_connections_accepted", normalizeName(createGauge("nginx.connections_accepted", "connections"), ""))
require.Equal(t, "nsxt_node_memory_usage_kilobytes", normalizeName(createGauge("nsxt.node.memory.usage", "KBy"), ""))
require.Equal(t, "redis_latest_fork_microseconds", normalizeName(createGauge("redis.latest_fork", "us"), ""))
}
func TestTrimPromSuffixes(t *testing.T) {
assert.Equal(t, "active_directory_ds_replication_network_io", TrimPromSuffixes("active_directory_ds_replication_network_io_bytes_total", pmetric.MetricTypeSum, "bytes"))
assert.Equal(t, "active_directory_ds_name_cache_hit_rate", TrimPromSuffixes("active_directory_ds_name_cache_hit_rate_percent", pmetric.MetricTypeGauge, "percent"))
assert.Equal(t, "active_directory_ds_ldap_bind_last_successful_time", TrimPromSuffixes("active_directory_ds_ldap_bind_last_successful_time_milliseconds", pmetric.MetricTypeGauge, "milliseconds"))
assert.Equal(t, "apache_requests", TrimPromSuffixes("apache_requests_total", pmetric.MetricTypeSum, "1"))
assert.Equal(t, "system_cpu_utilization", TrimPromSuffixes("system_cpu_utilization_ratio", pmetric.MetricTypeGauge, "ratio"))
assert.Equal(t, "mongodbatlas_process_journaling_data_files", TrimPromSuffixes("mongodbatlas_process_journaling_data_files_mebibytes", pmetric.MetricTypeGauge, "mebibytes"))
assert.Equal(t, "mongodbatlas_process_network_io", TrimPromSuffixes("mongodbatlas_process_network_io_bytes_per_second", pmetric.MetricTypeGauge, "bytes_per_second"))
assert.Equal(t, "mongodbatlas_process_oplog_rate", TrimPromSuffixes("mongodbatlas_process_oplog_rate_gibibytes_per_hour", pmetric.MetricTypeGauge, "gibibytes_per_hour"))
assert.Equal(t, "nsxt_node_memory_usage", TrimPromSuffixes("nsxt_node_memory_usage_kilobytes", pmetric.MetricTypeGauge, "kilobytes"))
assert.Equal(t, "redis_latest_fork", TrimPromSuffixes("redis_latest_fork_microseconds", pmetric.MetricTypeGauge, "microseconds"))
assert.Equal(t, "up", TrimPromSuffixes("up", pmetric.MetricTypeGauge, ""))
// These are not necessarily valid OM units, only tested for the sake of completeness.
assert.Equal(t, "active_directory_ds_replication_sync_object_pending", TrimPromSuffixes("active_directory_ds_replication_sync_object_pending_total", pmetric.MetricTypeSum, "{objects}"))
assert.Equal(t, "apache_current", TrimPromSuffixes("apache_current_connections", pmetric.MetricTypeGauge, "connections"))
assert.Equal(t, "bigip_virtual_server_request_count", TrimPromSuffixes("bigip_virtual_server_request_count_total", pmetric.MetricTypeSum, "{requests}"))
assert.Equal(t, "mongodbatlas_process_db_query_targeting_scanned_per_returned", TrimPromSuffixes("mongodbatlas_process_db_query_targeting_scanned_per_returned", pmetric.MetricTypeGauge, "{scanned}/{returned}"))
assert.Equal(t, "nginx_connections_accepted", TrimPromSuffixes("nginx_connections_accepted", pmetric.MetricTypeGauge, "connections"))
assert.Equal(t, "apache_workers", TrimPromSuffixes("apache_workers_connections", pmetric.MetricTypeGauge, "connections"))
assert.Equal(t, "nginx", TrimPromSuffixes("nginx_requests", pmetric.MetricTypeGauge, "requests"))
// Units shouldn't be trimmed if the unit is not a direct match with the suffix, i.e, a suffix "_seconds" shouldn't be removed if unit is "sec" or "s"
assert.Equal(t, "system_cpu_load_average_15m_ratio", TrimPromSuffixes("system_cpu_load_average_15m_ratio", pmetric.MetricTypeGauge, "1"))
assert.Equal(t, "mongodbatlas_process_asserts_per_second", TrimPromSuffixes("mongodbatlas_process_asserts_per_second", pmetric.MetricTypeGauge, "{assertions}/s"))
assert.Equal(t, "memcached_operation_hit_ratio_percent", TrimPromSuffixes("memcached_operation_hit_ratio_percent", pmetric.MetricTypeGauge, "%"))
assert.Equal(t, "active_directory_ds_replication_object_rate_per_second", TrimPromSuffixes("active_directory_ds_replication_object_rate_per_second", pmetric.MetricTypeGauge, "{objects}/s"))
assert.Equal(t, "system_disk_operation_time_seconds", TrimPromSuffixes("system_disk_operation_time_seconds_total", pmetric.MetricTypeSum, "s"))
}
func TestNamespace(t *testing.T) {
require.Equal(t, "space_test", normalizeName(createGauge("test", ""), "space"))
require.Equal(t, "space_test", normalizeName(createGauge("#test", ""), "space"))
}
func TestCleanUpString(t *testing.T) {
require.Equal(t, "", CleanUpString(""))
require.Equal(t, "a_b", CleanUpString("a b"))
require.Equal(t, "hello_world", CleanUpString("hello, world!"))
require.Equal(t, "hello_you_2", CleanUpString("hello you 2"))
require.Equal(t, "1000", CleanUpString("$1000"))
require.Equal(t, "", CleanUpString("*+$^=)"))
}
func TestUnitMapGetOrDefault(t *testing.T) {
require.Equal(t, "", unitMapGetOrDefault(""))
require.Equal(t, "seconds", unitMapGetOrDefault("s"))
require.Equal(t, "invalid", unitMapGetOrDefault("invalid"))
}
func TestPerUnitMapGetOrDefault(t *testing.T) {
require.Equal(t, "", perUnitMapGetOrDefault(""))
require.Equal(t, "second", perUnitMapGetOrDefault("s"))
require.Equal(t, "invalid", perUnitMapGetOrDefault("invalid"))
}
func TestRemoveItem(t *testing.T) {
require.Equal(t, []string{}, removeItem([]string{}, "test"))
require.Equal(t, []string{}, removeItem([]string{}, ""))
require.Equal(t, []string{"a", "b", "c"}, removeItem([]string{"a", "b", "c"}, "d"))
require.Equal(t, []string{"a", "b", "c"}, removeItem([]string{"a", "b", "c"}, ""))
require.Equal(t, []string{"a", "b"}, removeItem([]string{"a", "b", "c"}, "c"))
require.Equal(t, []string{"a", "c"}, removeItem([]string{"a", "b", "c"}, "b"))
require.Equal(t, []string{"b", "c"}, removeItem([]string{"a", "b", "c"}, "a"))
}
func TestBuildCompliantNameWithNormalize(t *testing.T) {
require.Equal(t, "system_io_bytes_total", BuildCompliantName(createCounter("system.io", "By"), "", true))
require.Equal(t, "system_network_io_bytes_total", BuildCompliantName(createCounter("network.io", "By"), "system", true))
require.Equal(t, "_3_14_digits", BuildCompliantName(createGauge("3.14 digits", ""), "", true))
require.Equal(t, "envoy_rule_engine_zlib_buf_error", BuildCompliantName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "", true))
require.Equal(t, "foo_bar", BuildCompliantName(createGauge(":foo::bar", ""), "", true))
require.Equal(t, "foo_bar_total", BuildCompliantName(createCounter(":foo::bar", ""), "", true))
// Gauges with unit 1 are considered ratios.
require.Equal(t, "foo_bar_ratio", BuildCompliantName(createGauge("foo.bar", "1"), "", true))
// Slashes in units are converted.
require.Equal(t, "system_io_foo_per_bar_total", BuildCompliantName(createCounter("system.io", "foo/bar"), "", true))
}
func TestBuildCompliantNameWithoutSuffixes(t *testing.T) {
require.Equal(t, "system_io", BuildCompliantName(createCounter("system.io", "By"), "", false))
require.Equal(t, "system_network_io", BuildCompliantName(createCounter("network.io", "By"), "system", false))
require.Equal(t, "system_network_I_O", BuildCompliantName(createCounter("network (I/O)", "By"), "system", false))
require.Equal(t, "_3_14_digits", BuildCompliantName(createGauge("3.14 digits", "By"), "", false))
require.Equal(t, "envoy__rule_engine_zlib_buf_error", BuildCompliantName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "", false))
require.Equal(t, ":foo::bar", BuildCompliantName(createGauge(":foo::bar", ""), "", false))
require.Equal(t, ":foo::bar", BuildCompliantName(createCounter(":foo::bar", ""), "", false))
require.Equal(t, "foo_bar", BuildCompliantName(createGauge("foo.bar", "1"), "", false))
require.Equal(t, "system_io", BuildCompliantName(createCounter("system.io", "foo/bar"), "", false))
}

49
storage/remote/otlptranslator/prometheus/testutils_test.go Normal file
View file

@ -0,0 +1,49 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Provenance-includes-location: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/95e8f8fdc2a9dc87230406c9a3cf02be4fd68bea/pkg/translator/prometheus/testutils_test.go
// Provenance-includes-license: Apache-2.0
// Provenance-includes-copyright: Copyright The OpenTelemetry Authors.
package prometheus
import (
"go.opentelemetry.io/collector/pdata/pmetric"
)
var ilm pmetric.ScopeMetrics
func init() {
metrics := pmetric.NewMetrics()
resourceMetrics := metrics.ResourceMetrics().AppendEmpty()
ilm = resourceMetrics.ScopeMetrics().AppendEmpty()
}
// Returns a new Metric of type "Gauge" with specified name and unit
func createGauge(name string, unit string) pmetric.Metric {
gauge := ilm.Metrics().AppendEmpty()
gauge.SetName(name)
gauge.SetUnit(unit)
gauge.SetEmptyGauge()
return gauge
}
// Returns a new Metric of type Monotonic Sum with specified name and unit
func createCounter(name string, unit string) pmetric.Metric {
counter := ilm.Metrics().AppendEmpty()
counter.SetEmptySum().SetIsMonotonic(true)
counter.SetName(name)
counter.SetUnit(unit)
return counter
}

21
storage/remote/otlptranslator/prometheusremotewrite/histograms.go
View file

@ -26,6 +26,7 @@ import (
"github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/model/value"
"github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/util/annotations"
) )
const defaultZeroThreshold = 1e-128 const defaultZeroThreshold = 1e-128
@ -33,13 +34,15 @@ const defaultZeroThreshold = 1e-128
// addExponentialHistogramDataPoints adds OTel exponential histogram data points to the corresponding time series // addExponentialHistogramDataPoints adds OTel exponential histogram data points to the corresponding time series
// as native histogram samples. // as native histogram samples.
func (c *PrometheusConverter) addExponentialHistogramDataPoints(dataPoints pmetric.ExponentialHistogramDataPointSlice, func (c *PrometheusConverter) addExponentialHistogramDataPoints(dataPoints pmetric.ExponentialHistogramDataPointSlice,
resource pcommon.Resource, settings Settings, promName string) error { resource pcommon.Resource, settings Settings, promName string) (annotations.Annotations, error) {
var annots annotations.Annotations
for x := 0; x < dataPoints.Len(); x++ { for x := 0; x < dataPoints.Len(); x++ {
pt := dataPoints.At(x) pt := dataPoints.At(x)
histogram, err := exponentialToNativeHistogram(pt) histogram, ws, err := exponentialToNativeHistogram(pt)
annots.Merge(ws)
if err != nil { if err != nil {
return err return annots, err
} }
lbls := createAttributes( lbls := createAttributes(
@ -58,15 +61,16 @@ func (c *PrometheusConverter) addExponentialHistogramDataPoints(dataPoints pmetr
ts.Exemplars = append(ts.Exemplars, exemplars...) ts.Exemplars = append(ts.Exemplars, exemplars...)
} }
return nil return annots, nil
} }
// exponentialToNativeHistogram translates OTel Exponential Histogram data point // exponentialToNativeHistogram translates OTel Exponential Histogram data point
// to Prometheus Native Histogram. // to Prometheus Native Histogram.
func exponentialToNativeHistogram(p pmetric.ExponentialHistogramDataPoint) (prompb.Histogram, error) { func exponentialToNativeHistogram(p pmetric.ExponentialHistogramDataPoint) (prompb.Histogram, annotations.Annotations, error) {
var annots annotations.Annotations
scale := p.Scale() scale := p.Scale()
if scale < -4 { if scale < -4 {
return prompb.Histogram{}, return prompb.Histogram{}, annots,
fmt.Errorf("cannot convert exponential to native histogram."+ fmt.Errorf("cannot convert exponential to native histogram."+
" Scale must be >= -4, was %d", scale) " Scale must be >= -4, was %d", scale)
} }
@ -114,8 +118,11 @@ func exponentialToNativeHistogram(p pmetric.ExponentialHistogramDataPoint) (prom
h.Sum = p.Sum() h.Sum = p.Sum()
} }
h.Count = &prompb.Histogram_CountInt{CountInt: p.Count()} h.Count = &prompb.Histogram_CountInt{CountInt: p.Count()}
if p.Count() == 0 && h.Sum != 0 {
annots.Add(fmt.Errorf("exponential histogram data point has zero count, but non-zero sum: %f", h.Sum))
} }
return h, nil }
return h, annots, nil
} }
// convertBucketsLayout translates OTel Exponential Histogram dense buckets // convertBucketsLayout translates OTel Exponential Histogram dense buckets

11
storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go
View file

@ -27,6 +27,7 @@ import (
"github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/prompb"
prometheustranslator "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus" prometheustranslator "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus"
"github.com/prometheus/prometheus/util/annotations"
) )
type Settings struct { type Settings struct {
@ -53,7 +54,7 @@ func NewPrometheusConverter() *PrometheusConverter {
} }
// FromMetrics converts pmetric.Metrics to Prometheus remote write format. // FromMetrics converts pmetric.Metrics to Prometheus remote write format.
func (c *PrometheusConverter) FromMetrics(md pmetric.Metrics, settings Settings) (errs error) { func (c *PrometheusConverter) FromMetrics(md pmetric.Metrics, settings Settings) (annots annotations.Annotations, errs error) {
resourceMetricsSlice := md.ResourceMetrics() resourceMetricsSlice := md.ResourceMetrics()
for i := 0; i < resourceMetricsSlice.Len(); i++ { for i := 0; i < resourceMetricsSlice.Len(); i++ {
resourceMetrics := resourceMetricsSlice.At(i) resourceMetrics := resourceMetricsSlice.At(i)
@ -107,12 +108,14 @@ func (c *PrometheusConverter) FromMetrics(md pmetric.Metrics, settings Settings)
errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name())) errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
break break
} }
errs = multierr.Append(errs, c.addExponentialHistogramDataPoints( ws, err := c.addExponentialHistogramDataPoints(
dataPoints, dataPoints,
resource, resource,
settings, settings,
promName, promName,
)) )
annots.Merge(ws)
errs = multierr.Append(errs, err)
case pmetric.MetricTypeSummary: case pmetric.MetricTypeSummary:
dataPoints := metric.Summary().DataPoints() dataPoints := metric.Summary().DataPoints()
if dataPoints.Len() == 0 { if dataPoints.Len() == 0 {
@ -128,7 +131,7 @@ func (c *PrometheusConverter) FromMetrics(md pmetric.Metrics, settings Settings)
addResourceTargetInfo(resource, settings, mostRecentTimestamp, c) addResourceTargetInfo(resource, settings, mostRecentTimestamp, c)
} }
return return annots, errs
} }
func isSameMetric(ts *prompb.TimeSeries, lbls []prompb.Label) bool { func isSameMetric(ts *prompb.TimeSeries, lbls []prompb.Label) bool {

39
storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw_test.go
View file

@ -27,6 +27,41 @@ import (
"go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp" "go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp"
) )
func TestFromMetrics(t *testing.T) {
t.Run("exponential histogram warnings for zero count and non-zero sum", func(t *testing.T) {
request := pmetricotlp.NewExportRequest()
rm := request.Metrics().ResourceMetrics().AppendEmpty()
generateAttributes(rm.Resource().Attributes(), "resource", 10)
metrics := rm.ScopeMetrics().AppendEmpty().Metrics()
ts := pcommon.NewTimestampFromTime(time.Now())
for i := 1; i <= 10; i++ {
m := metrics.AppendEmpty()
m.SetEmptyExponentialHistogram()
m.SetName(fmt.Sprintf("histogram-%d", i))
m.ExponentialHistogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
h := m.ExponentialHistogram().DataPoints().AppendEmpty()
h.SetTimestamp(ts)
h.SetCount(0)
h.SetSum(155)
generateAttributes(h.Attributes(), "series", 10)
}
converter := NewPrometheusConverter()
annots, err := converter.FromMetrics(request.Metrics(), Settings{})
require.NoError(t, err)
require.NotEmpty(t, annots)
ws, infos := annots.AsStrings("", 0, 0)
require.Empty(t, infos)
require.Equal(t, []string{
"exponential histogram data point has zero count, but non-zero sum: 155.000000",
}, ws)
})
}
func BenchmarkPrometheusConverter_FromMetrics(b *testing.B) { func BenchmarkPrometheusConverter_FromMetrics(b *testing.B) {
for _, resourceAttributeCount := range []int{0, 5, 50} { for _, resourceAttributeCount := range []int{0, 5, 50} {
b.Run(fmt.Sprintf("resource attribute count: %v", resourceAttributeCount), func(b *testing.B) { b.Run(fmt.Sprintf("resource attribute count: %v", resourceAttributeCount), func(b *testing.B) {
@ -49,7 +84,9 @@ func BenchmarkPrometheusConverter_FromMetrics(b *testing.B) {
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
converter := NewPrometheusConverter() converter := NewPrometheusConverter()
require.NoError(b, converter.FromMetrics(payload.Metrics(), Settings{})) annots, err := converter.FromMetrics(payload.Metrics(), Settings{})
require.NoError(b, err)
require.Empty(b, annots)
require.NotNil(b, converter.TimeSeries()) require.NotNil(b, converter.TimeSeries())
} }
}) })

14
storage/remote/queue_manager.go
View file

@ -1109,9 +1109,9 @@ func (t *QueueManager) shouldReshard(desiredShards int) bool {
if desiredShards == t.numShards { if desiredShards == t.numShards {
return false return false
} }
// We shouldn't reshard if Prometheus hasn't been able to send to the // We shouldn't reshard if Prometheus hasn't been able to send
// remote endpoint successfully within some period of time. // since the last time it checked if it should reshard.
minSendTimestamp := time.Now().Add(-2 * time.Duration(t.cfg.BatchSendDeadline)).Unix() minSendTimestamp := time.Now().Add(-1 * shardUpdateDuration).Unix()
lsts := t.lastSendTimestamp.Load() lsts := t.lastSendTimestamp.Load()
if lsts < minSendTimestamp { if lsts < minSendTimestamp {
level.Warn(t.logger).Log("msg", "Skipping resharding, last successful send was beyond threshold", "lastSendTimestamp", lsts, "minSendTimestamp", minSendTimestamp) level.Warn(t.logger).Log("msg", "Skipping resharding, last successful send was beyond threshold", "lastSendTimestamp", lsts, "minSendTimestamp", minSendTimestamp)
@ -1522,7 +1522,7 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) {
// Send batches of at most MaxSamplesPerSend samples to the remote storage. // Send batches of at most MaxSamplesPerSend samples to the remote storage.
// If we have fewer samples than that, flush them out after a deadline anyways. // If we have fewer samples than that, flush them out after a deadline anyways.
var ( var (
max = s.qm.cfg.MaxSamplesPerSend maxCount = s.qm.cfg.MaxSamplesPerSend
pBuf = proto.NewBuffer(nil) pBuf = proto.NewBuffer(nil)
pBufRaw []byte pBufRaw []byte
@ -1530,19 +1530,19 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) {
) )
// TODO(@tpaschalis) Should we also raise the max if we have WAL metadata? // TODO(@tpaschalis) Should we also raise the max if we have WAL metadata?
if s.qm.sendExemplars { if s.qm.sendExemplars {
max += int(float64(max) * 0.1) maxCount += int(float64(maxCount) * 0.1)
} }
// TODO: Dry all of this, we should make an interface/generic for the timeseries type. // TODO: Dry all of this, we should make an interface/generic for the timeseries type.
batchQueue := queue.Chan() batchQueue := queue.Chan()
pendingData := make([]prompb.TimeSeries, max) pendingData := make([]prompb.TimeSeries, maxCount)
for i := range pendingData { for i := range pendingData {
pendingData[i].Samples = []prompb.Sample{{}} pendingData[i].Samples = []prompb.Sample{{}}
if s.qm.sendExemplars { if s.qm.sendExemplars {
pendingData[i].Exemplars = []prompb.Exemplar{{}} pendingData[i].Exemplars = []prompb.Exemplar{{}}
} }
} }
pendingDataV2 := make([]writev2.TimeSeries, max) pendingDataV2 := make([]writev2.TimeSeries, maxCount)
for i := range pendingDataV2 { for i := range pendingDataV2 {
pendingDataV2[i].Samples = []writev2.Sample{{}} pendingDataV2[i].Samples = []writev2.Sample{{}}
} }

15
storage/remote/queue_manager_test.go
View file

@ -703,32 +703,35 @@ func TestShouldReshard(t *testing.T) {
startingShards int startingShards int
samplesIn, samplesOut, lastSendTimestamp int64 samplesIn, samplesOut, lastSendTimestamp int64
expectedToReshard bool expectedToReshard bool
sendDeadline model.Duration
} }
cases := []testcase{ cases := []testcase{
{ {
// Resharding shouldn't take place if the last successful send was > batch send deadline*2 seconds ago. // resharding shouldn't take place if we haven't successfully sent
// since the last shardUpdateDuration, even if the send deadline is very low
startingShards: 10, startingShards: 10,
samplesIn: 1000, samplesIn: 1000,
samplesOut: 10, samplesOut: 10,
lastSendTimestamp: time.Now().Unix() - int64(3*time.Duration(config.DefaultQueueConfig.BatchSendDeadline)/time.Second), lastSendTimestamp: time.Now().Unix() - int64(shardUpdateDuration),
expectedToReshard: false, expectedToReshard: false,
sendDeadline: model.Duration(100 * time.Millisecond),
}, },
{ {
startingShards: 5, startingShards: 10,
samplesIn: 1000, samplesIn: 1000,
samplesOut: 10, samplesOut: 10,
lastSendTimestamp: time.Now().Unix(), lastSendTimestamp: time.Now().Unix(),
expectedToReshard: true, expectedToReshard: true,
sendDeadline: config.DefaultQueueConfig.BatchSendDeadline,
}, },
} }
for _, c := range cases { for _, c := range cases {
_, m := newTestClientAndQueueManager(t, defaultFlushDeadline, config.RemoteWriteProtoMsgV1) _, m := newTestClientAndQueueManager(t, time.Duration(c.sendDeadline), config.RemoteWriteProtoMsgV1)
m.numShards = c.startingShards m.numShards = c.startingShards
m.dataIn.incr(c.samplesIn) m.dataIn.incr(c.samplesIn)
m.dataOut.incr(c.samplesOut) m.dataOut.incr(c.samplesOut)
m.lastSendTimestamp.Store(c.lastSendTimestamp) m.lastSendTimestamp.Store(c.lastSendTimestamp)
m.Start() m.Start()
desiredShards := m.calculateDesiredShards() desiredShards := m.calculateDesiredShards()
@ -927,7 +930,7 @@ func createHistograms(numSamples, numSeries int, floatHistogram bool) ([]record.
} }
func createSeriesMetadata(series []record.RefSeries) []record.RefMetadata { func createSeriesMetadata(series []record.RefSeries) []record.RefMetadata {
metas := make([]record.RefMetadata, len(series)) metas := make([]record.RefMetadata, 0, len(series))
for _, s := range series { for _, s := range series {
metas = append(metas, record.RefMetadata{ metas = append(metas, record.RefMetadata{

53
storage/remote/read_handler.go
View file

@ -202,16 +202,34 @@ func (h *readHandler) remoteReadStreamedXORChunks(ctx context.Context, w http.Re
return err return err
} }
chunks := h.getChunkSeriesSet(ctx, query, filteredMatchers) querier, err := h.queryable.ChunkQuerier(query.StartTimestampMs, query.EndTimestampMs)
if err := chunks.Err(); err != nil { if err != nil {
return err return err
} }
defer func() {
if err := querier.Close(); err != nil {
level.Warn(h.logger).Log("msg", "Error on chunk querier close", "err", err.Error())
}
}()
var hints *storage.SelectHints
if query.Hints != nil {
hints = &storage.SelectHints{
Start: query.Hints.StartMs,
End: query.Hints.EndMs,
Step: query.Hints.StepMs,
Func: query.Hints.Func,
Grouping: query.Hints.Grouping,
Range: query.Hints.RangeMs,
By: query.Hints.By,
}
}
ws, err := StreamChunkedReadResponses( ws, err := StreamChunkedReadResponses(
NewChunkedWriter(w, f), NewChunkedWriter(w, f),
int64(i), int64(i),
// The streaming API has to provide the series sorted. // The streaming API has to provide the series sorted.
chunks, querier.Select(ctx, true, hints, filteredMatchers...),
sortedExternalLabels, sortedExternalLabels,
h.remoteReadMaxBytesInFrame, h.remoteReadMaxBytesInFrame,
h.marshalPool, h.marshalPool,
@ -236,35 +254,6 @@ func (h *readHandler) remoteReadStreamedXORChunks(ctx context.Context, w http.Re
} }
} }
// getChunkSeriesSet executes a query to retrieve a ChunkSeriesSet,
// encapsulating the operation in its own function to ensure timely release of
// the querier resources.
func (h *readHandler) getChunkSeriesSet(ctx context.Context, query *prompb.Query, filteredMatchers []*labels.Matcher) storage.ChunkSeriesSet {
querier, err := h.queryable.ChunkQuerier(query.StartTimestampMs, query.EndTimestampMs)
if err != nil {
return storage.ErrChunkSeriesSet(err)
}
defer func() {
if err := querier.Close(); err != nil {
level.Warn(h.logger).Log("msg", "Error on chunk querier close", "err", err.Error())
}
}()
var hints *storage.SelectHints
if query.Hints != nil {
hints = &storage.SelectHints{
Start: query.Hints.StartMs,
End: query.Hints.EndMs,
Step: query.Hints.StepMs,
Func: query.Hints.Func,
Grouping: query.Hints.Grouping,
Range: query.Hints.RangeMs,
By: query.Hints.By,
}
}
return querier.Select(ctx, true, hints, filteredMatchers...)
}
// filterExtLabelsFromMatchers change equality matchers which match external labels // filterExtLabelsFromMatchers change equality matchers which match external labels
// to a matcher that looks for an empty label, // to a matcher that looks for an empty label,
// as that label should not be present in the storage. // as that label should not be present in the storage.

1
storage/remote/write.go
View file

@ -176,6 +176,7 @@ func (rws *WriteStorage) ApplyConfig(conf *config.Config) error {
HTTPClientConfig: rwConf.HTTPClientConfig, HTTPClientConfig: rwConf.HTTPClientConfig,
SigV4Config: rwConf.SigV4Config, SigV4Config: rwConf.SigV4Config,
AzureADConfig: rwConf.AzureADConfig, AzureADConfig: rwConf.AzureADConfig,
GoogleIAMConfig: rwConf.GoogleIAMConfig,
Headers: rwConf.Headers, Headers: rwConf.Headers,
RetryOnRateLimit: rwConf.QueueConfig.RetryOnRateLimit, RetryOnRateLimit: rwConf.QueueConfig.RetryOnRateLimit,
}) })

22
storage/remote/write_handler.go
View file

@ -236,11 +236,16 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err
b := labels.NewScratchBuilder(0) b := labels.NewScratchBuilder(0)
for _, ts := range req.Timeseries { for _, ts := range req.Timeseries {
ls := ts.ToLabels(&b, nil) ls := ts.ToLabels(&b, nil)
// TODO(bwplotka): Even as per 1.0 spec, this should be a 400 error, while other samples are
// potentially written. Perhaps unify with fixed writeV2 implementation a bit.
if !ls.Has(labels.MetricName) || !ls.IsValid() { if !ls.Has(labels.MetricName) || !ls.IsValid() {
level.Warn(h.logger).Log("msg", "Invalid metric names or labels", "got", ls.String()) level.Warn(h.logger).Log("msg", "Invalid metric names or labels", "got", ls.String())
samplesWithInvalidLabels++ samplesWithInvalidLabels++
// TODO(bwplotka): Even as per 1.0 spec, this should be a 400 error, while other samples are continue
// potentially written. Perhaps unify with fixed writeV2 implementation a bit. } else if duplicateLabel, hasDuplicate := ls.HasDuplicateLabelNames(); hasDuplicate {
level.Warn(h.logger).Log("msg", "Invalid labels for series.", "labels", ls.String(), "duplicated_label", duplicateLabel)
samplesWithInvalidLabels++
continue continue
} }
@ -379,6 +384,10 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs *
badRequestErrs = append(badRequestErrs, fmt.Errorf("invalid metric name or labels, got %v", ls.String())) badRequestErrs = append(badRequestErrs, fmt.Errorf("invalid metric name or labels, got %v", ls.String()))
samplesWithInvalidLabels += len(ts.Samples) + len(ts.Histograms) samplesWithInvalidLabels += len(ts.Samples) + len(ts.Histograms)
continue continue
} else if duplicateLabel, hasDuplicate := ls.HasDuplicateLabelNames(); hasDuplicate {
badRequestErrs = append(badRequestErrs, fmt.Errorf("invalid labels for series, labels %v, duplicated label %s", ls.String(), duplicateLabel))
samplesWithInvalidLabels += len(ts.Samples) + len(ts.Histograms)
continue
} }
allSamplesSoFar := rs.AllSamples() allSamplesSoFar := rs.AllSamples()
@ -502,12 +511,17 @@ func (h *otlpWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
otlpCfg := h.configFunc().OTLPConfig otlpCfg := h.configFunc().OTLPConfig
converter := otlptranslator.NewPrometheusConverter() converter := otlptranslator.NewPrometheusConverter()
if err := converter.FromMetrics(req.Metrics(), otlptranslator.Settings{ annots, err := converter.FromMetrics(req.Metrics(), otlptranslator.Settings{
AddMetricSuffixes: true, AddMetricSuffixes: true,
PromoteResourceAttributes: otlpCfg.PromoteResourceAttributes, PromoteResourceAttributes: otlpCfg.PromoteResourceAttributes,
}); err != nil { })
if err != nil {
level.Warn(h.logger).Log("msg", "Error translating OTLP metrics to Prometheus write request", "err", err) level.Warn(h.logger).Log("msg", "Error translating OTLP metrics to Prometheus write request", "err", err)
} }
ws, _ := annots.AsStrings("", 0, 0)
if len(ws) > 0 {
level.Warn(h.logger).Log("msg", "Warnings translating OTLP metrics to Prometheus write request", "warnings", ws)
}
err = h.rwHandler.write(r.Context(), &prompb.WriteRequest{ err = h.rwHandler.write(r.Context(), &prompb.WriteRequest{
Timeseries: converter.TimeSeries(), Timeseries: converter.TimeSeries(),

Some files were not shown because too many files have changed in this diff Show more