mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-24 05:04:05 -08:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Merge branch 'main' into 3.0-main-sync-24-08-01

Browse source 
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
This commit is contained in:
Jan Fajerski 2024-08-01 11:59:55 +02:00
parent 91c05eed84 00ab05c3b9
commit 8a96a75ba8
85 changed files with 3124 additions and 1399 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
.github/workflows/ci.yml vendored
View file

@ -143,6 +143,18 @@ jobs:
with:
parallelism: 12
thread: ${{ matrix.thread }}
build_all_status:
name: Report status of build Prometheus for all architectures
runs-on: ubuntu-latest
needs: [build_all]
if: github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-')
steps:
- name: Successful build
if: ${{ !(contains(needs.*.result, 'failure')) && !(contains(needs.*.result, 'cancelled')) }}
run: exit 0
- name: Failing or cancelled build
if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
run: exit 1
check_generated_parser:
name: Check generated parser
runs-on: ubuntu-latest

4
CHANGELOG.md
View file

@ -8,6 +8,10 @@ _Please add changes here that are only in the release-3.0 branch. These will be
## unreleased
* [FEATURE] OTLP receiver: Add new option `otlp.promote_resource_attributes`, for any OTel resource attributes that should be promoted to metric labels. #14200
* [FEATURE] Remote-Write: Add sender and receiver support for [Remote Write 2.0-rc.2](https://prometheus.io/docs/specs/remote_write_spec_2_0/) specification #14395 #14427 #14444
* [ENHANCEMENT] Remote-Write: 1.x messages against Remote Write 2.x Receivers will have now correct values for `prometheus_storage_<samples|histograms|exemplar>_failed_total` in case of partial errors #14444
## 2.53.1 / 2024-07-10
Fix a bug which would drop samples in remote-write if the sending flow stalled

3
README.md
View file

@ -12,9 +12,10 @@ examples and guides.</p>
[![Docker Pulls](https://img.shields.io/docker/pulls/prom/prometheus.svg?maxAge=604800)][hub]
[![Go Report Card](https://goreportcard.com/badge/github.com/prometheus/prometheus)](https://goreportcard.com/report/github.com/prometheus/prometheus)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/486/badge)](https://bestpractices.coreinfrastructure.org/projects/486)
[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://securityscorecards.dev/viewer/?uri=github.com/prometheus/prometheus)
[![CLOMonitor](https://img.shields.io/endpoint?url=https://clomonitor.io/api/projects/cncf/prometheus/badge)](https://clomonitor.io/projects/cncf/prometheus)
[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/prometheus/prometheus)
[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/prometheus.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:prometheus)
[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://securityscorecards.dev/viewer/?uri=github.com/prometheus/prometheus)
</div>

48
SECURITY-INSIGHTS.yml Normal file
View file

@ -0,0 +1,48 @@
header:
schema-version: '1.0.0'
expiration-date: '2025-07-30T01:00:00.000Z'
last-updated: '2024-07-30'
last-reviewed: '2024-07-30'
project-url: https://github.com/prometheus/prometheus
changelog: https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md
license: https://github.com/prometheus/prometheus/blob/main/LICENSE
project-lifecycle:
status: active
bug-fixes-only: false
core-maintainers:
- https://github.com/prometheus/prometheus/blob/main/MAINTAINERS.md
contribution-policy:
accepts-pull-requests: true
accepts-automated-pull-requests: true
dependencies:
third-party-packages: true
dependencies-lists:
- https://github.com/prometheus/prometheus/blob/main/go.mod
- https://github.com/prometheus/prometheus/blob/main/web/ui/package.json
env-dependencies-policy:
policy-url: https://github.com/prometheus/prometheus/blob/main/CONTRIBUTING.md#dependency-management
distribution-points:
- https://github.com/prometheus/prometheus/releases
documentation:
- https://prometheus.io/docs/introduction/overview/
security-contacts:
- type: email
value: prometheus-team@googlegroups.com
security-testing:
- tool-type: sca
tool-name: Dependabot
tool-version: latest
integration:
ad-hoc: false
ci: true
before-release: true
- tool-type: sast
tool-name: CodeQL
tool-version: latest
integration:
ad-hoc: false
ci: true
before-release: true
vulnerability-reporting:
accepts-vulnerability-reports: true
security-policy: https://github.com/prometheus/prometheus/security/policy

7
cmd/promtool/main.go
View file

@ -204,6 +204,7 @@ func main() {
pushMetricsHeaders := pushMetricsCmd.Flag("header", "Prometheus remote write header.").StringMap()
testCmd := app.Command("test", "Unit testing.")
junitOutFile := testCmd.Flag("junit", "File path to store JUnit XML test results.").OpenFile(os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
testRulesCmd := testCmd.Command("rules", "Unit tests for rules.")
testRulesRun := testRulesCmd.Flag("run", "If set, will only run test groups whose names match the regular expression. Can be specified multiple times.").Strings()
testRulesFiles := testRulesCmd.Arg(
@ -376,7 +377,11 @@ func main() {
os.Exit(QueryLabels(serverURL, httpRoundTripper, *queryLabelsMatch, *queryLabelsName, *queryLabelsBegin, *queryLabelsEnd, p))
case testRulesCmd.FullCommand():
os.Exit(RulesUnitTest(
results := io.Discard
if *junitOutFile != nil {
results = *junitOutFile
}
os.Exit(RulesUnitTestResult(results,
promqltest.LazyLoaderOpts{
EnableAtModifier: true,
EnableNegativeOffset: true,

3
cmd/promtool/metrics.go
View file

@ -101,6 +101,7 @@ func PushMetrics(url *url.URL, roundTripper http.RoundTripper, headers map[strin
return successExitCode
}
// TODO(bwplotka): Add PRW 2.0 support.
func parseAndPushMetrics(client *remote.Client, data []byte, labels map[string]string) bool {
metricsData, err := fmtutil.MetricTextToWriteRequest(bytes.NewReader(data), labels)
if err != nil {
@ -116,7 +117,7 @@ func parseAndPushMetrics(client *remote.Client, data []byte, labels map[string]s
// Encode the request body into snappy encoding.
compressed := snappy.Encode(nil, raw)
err = client.Store(context.Background(), compressed, 0)
_, err = client.Store(context.Background(), compressed, 0)
if err != nil {
fmt.Fprintln(os.Stderr, " FAILED:", err)
return false

40
cmd/promtool/unittest.go
View file

@ -18,6 +18,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
@ -29,9 +30,10 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/grafana/regexp"
"github.com/nsf/jsondiff"
"github.com/prometheus/common/model"
"gopkg.in/yaml.v2"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/promql"
@ -39,12 +41,18 @@ import (
"github.com/prometheus/prometheus/promql/promqltest"
"github.com/prometheus/prometheus/rules"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/util/junitxml"
)
// RulesUnitTest does unit testing of rules based on the unit testing files provided.
// More info about the file format can be found in the docs.
func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
return RulesUnitTestResult(io.Discard, queryOpts, runStrings, diffFlag, files...)
}
func RulesUnitTestResult(results io.Writer, queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
failed := false
junit := &junitxml.JUnitXML{}
var run *regexp.Regexp
if runStrings != nil {
@ -52,7 +60,7 @@ func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, dif
}
for _, f := range files {
if errs := ruleUnitTest(f, queryOpts, run, diffFlag); errs != nil {
if errs := ruleUnitTest(f, queryOpts, run, diffFlag, junit.Suite(f)); errs != nil {
fmt.Fprintln(os.Stderr, " FAILED:")
for _, e := range errs {
fmt.Fprintln(os.Stderr, e.Error())
@ -64,25 +72,30 @@ func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, dif
}
fmt.Println()
}
err := junit.WriteXML(results)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to write JUnit XML: %s\n", err)
}
if failed {
return failureExitCode
}
return successExitCode
}
func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error {
fmt.Println("Unit Testing: ", filename)
func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool, ts *junitxml.TestSuite) []error {
b, err := os.ReadFile(filename)
if err != nil {
ts.Abort(err)
return []error{err}
}
var unitTestInp unitTestFile
if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil {
ts.Abort(err)
return []error{err}
}
if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil {
ts.Abort(err)
return []error{err}
}
@ -91,29 +104,38 @@ func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *reg
}
evalInterval := time.Duration(unitTestInp.EvaluationInterval)
ts.Settime(time.Now().Format("2006-01-02T15:04:05"))
// Giving number for groups mentioned in the file for ordering.
// Lower number group should be evaluated before higher number group.
groupOrderMap := make(map[string]int)
for i, gn := range unitTestInp.GroupEvalOrder {
if _, ok := groupOrderMap[gn]; ok {
return []error{fmt.Errorf("group name repeated in evaluation order: %s", gn)}
err := fmt.Errorf("group name repeated in evaluation order: %s", gn)
ts.Abort(err)
return []error{err}
}
groupOrderMap[gn] = i
}
// Testing.
var errs []error
for _, t := range unitTestInp.Tests {
for i, t := range unitTestInp.Tests {
if !matchesRun(t.TestGroupName, run) {
continue
}
testname := t.TestGroupName
if testname == "" {
testname = fmt.Sprintf("unnamed#%d", i)
}
tc := ts.Case(testname)
if t.Interval == 0 {
t.Interval = unitTestInp.EvaluationInterval
}
ers := t.test(evalInterval, groupOrderMap, queryOpts, diffFlag, unitTestInp.RuleFiles...)
if ers != nil {
for _, e := range ers {
tc.Fail(e.Error())
}
errs = append(errs, ers...)
}
}

50
cmd/promtool/unittest_test.go
View file

@ -14,11 +14,15 @@
package main
import (
"bytes"
"encoding/xml"
"fmt"
"testing"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/promql/promqltest"
"github.com/prometheus/prometheus/util/junitxml"
)
func TestRulesUnitTest(t *testing.T) {
@ -125,13 +129,59 @@ func TestRulesUnitTest(t *testing.T) {
want: 0,
},
}
reuseFiles := []string{}
reuseCount := [2]int{}
for _, tt := range tests {
if (tt.queryOpts == promqltest.LazyLoaderOpts{
EnableNegativeOffset: true,
} || tt.queryOpts == promqltest.LazyLoaderOpts{
EnableAtModifier: true,
}) {
reuseFiles = append(reuseFiles, tt.args.files...)
reuseCount[tt.want] += len(tt.args.files)
}
t.Run(tt.name, func(t *testing.T) {
if got := RulesUnitTest(tt.queryOpts, nil, false, tt.args.files...); got != tt.want {
t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
}
})
}
t.Run("Junit xml output ", func(t *testing.T) {
var buf bytes.Buffer
if got := RulesUnitTestResult(&buf, promqltest.LazyLoaderOpts{}, nil, false, reuseFiles...); got != 1 {
t.Errorf("RulesUnitTestResults() = %v, want 1", got)
}
var test junitxml.JUnitXML
output := buf.Bytes()
err := xml.Unmarshal(output, &test)
if err != nil {
fmt.Println("error in decoding XML:", err)
return
}
var total int
var passes int
var failures int
var cases int
total = len(test.Suites)
if total != len(reuseFiles) {
t.Errorf("JUnit output had %d testsuite elements; expected %d\n", total, len(reuseFiles))
}
for _, i := range test.Suites {
if i.FailureCount == 0 {
passes++
} else {
failures++
}
cases += len(i.Cases)
}
if total != passes+failures {
t.Errorf("JUnit output mismatch: Total testsuites (%d) does not equal the sum of passes (%d) and failures (%d).", total, passes, failures)
}
if cases < total {
t.Errorf("JUnit output had %d suites without test cases\n", total-cases)
}
})
}
func TestRulesUnitTestRun(t *testing.T) {

72
config/config.go
View file

@ -37,6 +37,7 @@ import (
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel"
"github.com/prometheus/prometheus/storage/remote/azuread"
"github.com/prometheus/prometheus/storage/remote/googleiam"
)
var (
@ -227,6 +228,9 @@ var (
DefaultExemplarsConfig = ExemplarsConfig{
MaxExemplars: 100000,
}
// DefaultOTLPConfig is the default OTLP configuration.
DefaultOTLPConfig = OTLPConfig{}
)
// Config is the top-level configuration for Prometheus's config files.
@ -242,6 +246,7 @@ type Config struct {
RemoteWriteConfigs []*RemoteWriteConfig `yaml:"remote_write,omitempty"`
RemoteReadConfigs []*RemoteReadConfig `yaml:"remote_read,omitempty"`
OTLPConfig OTLPConfig `yaml:"otlp,omitempty"`
}
// SetDirectory joins any relative file paths with dir.
@ -1119,6 +1124,7 @@ type RemoteWriteConfig struct {
MetadataConfig MetadataConfig `yaml:"metadata_config,omitempty"`
SigV4Config *sigv4.SigV4Config `yaml:"sigv4,omitempty"`
AzureADConfig *azuread.AzureADConfig `yaml:"azuread,omitempty"`
GoogleIAMConfig *googleiam.Config `yaml:"google_iam,omitempty"`
}
// SetDirectory joins any relative file paths with dir.
@ -1156,17 +1162,33 @@ func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) err
return err
}
httpClientConfigAuthEnabled := c.HTTPClientConfig.BasicAuth != nil ||
c.HTTPClientConfig.Authorization != nil || c.HTTPClientConfig.OAuth2 != nil
return validateAuthConfigs(c)
}
if httpClientConfigAuthEnabled && (c.SigV4Config != nil || c.AzureADConfig != nil) {
return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, & azuread must be configured")
// validateAuthConfigs validates that at most one of basic_auth, authorization, oauth2, sigv4, azuread or google_iam must be configured.
func validateAuthConfigs(c *RemoteWriteConfig) error {
var authConfigured []string
if c.HTTPClientConfig.BasicAuth != nil {
authConfigured = append(authConfigured, "basic_auth")
}
if c.SigV4Config != nil && c.AzureADConfig != nil {
return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, & azuread must be configured")
if c.HTTPClientConfig.Authorization != nil {
authConfigured = append(authConfigured, "authorization")
}
if c.HTTPClientConfig.OAuth2 != nil {
authConfigured = append(authConfigured, "oauth2")
}
if c.SigV4Config != nil {
authConfigured = append(authConfigured, "sigv4")
}
if c.AzureADConfig != nil {
authConfigured = append(authConfigured, "azuread")
}
if c.GoogleIAMConfig != nil {
authConfigured = append(authConfigured, "google_iam")
}
if len(authConfigured) > 1 {
return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, azuread or google_iam must be configured. Currently configured: %v", authConfigured)
}
return nil
}
@ -1185,7 +1207,7 @@ func validateHeadersForTracing(headers map[string]string) error {
func validateHeaders(headers map[string]string) error {
for header := range headers {
if strings.ToLower(header) == "authorization" {
return errors.New("authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, or azuread parameter")
return errors.New("authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, azuread or google_iam parameter")
}
if _, ok := reservedHeaders[strings.ToLower(header)]; ok {
return fmt.Errorf("%s is a reserved header. It must not be changed", header)
@ -1304,3 +1326,35 @@ func getGoGCEnv() int {
}
return DefaultRuntimeConfig.GoGC
}
// OTLPConfig is the configuration for writing to the OTLP endpoint.
type OTLPConfig struct {
PromoteResourceAttributes []string `yaml:"promote_resource_attributes,omitempty"`
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (c *OTLPConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
*c = DefaultOTLPConfig
type plain OTLPConfig
if err := unmarshal((*plain)(c)); err != nil {
return err
}
seen := map[string]struct{}{}
var err error
for i, attr := range c.PromoteResourceAttributes {
attr = strings.TrimSpace(attr)
if attr == "" {
err = errors.Join(err, fmt.Errorf("empty promoted OTel resource attribute"))
continue
}
if _, exists := seen[attr]; exists {
err = errors.Join(err, fmt.Errorf("duplicated promoted OTel resource attribute %q", attr))
continue
}
seen[attr] = struct{}{}
c.PromoteResourceAttributes[i] = attr
}
return err
}

28
config/config_test.go
View file

@ -156,6 +156,12 @@ var expectedConf = &Config{
},
},
OTLPConfig: OTLPConfig{
PromoteResourceAttributes: []string{
"k8s.cluster.name", "k8s.job.name", "k8s.namespace.name",
},
},
RemoteReadConfigs: []*RemoteReadConfig{
{
URL: mustParseURL("http://remote1/read"),
@ -1471,6 +1477,26 @@ func TestRemoteWriteRetryOnRateLimit(t *testing.T) {
require.False(t, got.RemoteWriteConfigs[1].QueueConfig.RetryOnRateLimit)
}
func TestOTLPSanitizeResourceAttributes(t *testing.T) {
t.Run("good config", func(t *testing.T) {
want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes.good.yml"), false, false, log.NewNopLogger())
require.NoError(t, err)
out, err := yaml.Marshal(want)
require.NoError(t, err)
var got Config
require.NoError(t, yaml.UnmarshalStrict(out, &got))
require.Equal(t, []string{"k8s.cluster.name", "k8s.job.name", "k8s.namespace.name"}, got.OTLPConfig.PromoteResourceAttributes)
})
t.Run("bad config", func(t *testing.T) {
_, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes.bad.yml"), false, false, log.NewNopLogger())
require.ErrorContains(t, err, `duplicated promoted OTel resource attribute "k8s.job.name"`)
require.ErrorContains(t, err, `empty promoted OTel resource attribute`)
})
}
func TestLoadConfig(t *testing.T) {
// Parse a valid file that sets a global scrape timeout. This tests whether parsing
// an overwritten default field in the global config permanently changes the default.
@ -1800,7 +1826,7 @@ var expectedErrors = []struct {
},
{
filename: "remote_write_authorization_header.bad.yml",
errMsg: `authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, or azuread parameter`,
errMsg: `authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, azuread or google_iam parameter`,
},
{
filename: "remote_write_wrong_msg.bad.yml",

3
config/testdata/conf.good.yml vendored
View file

@ -45,6 +45,9 @@ remote_write:
headers:
name: value
otlp:
promote_resource_attributes: ["k8s.cluster.name", "k8s.job.name", "k8s.namespace.name"]
remote_read:
- url: http://remote1/read
read_recent: true

2
config/testdata/otlp_sanitize_resource_attributes.bad.yml vendored Normal file
View file

@ -0,0 +1,2 @@
otlp:
promote_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name", "k8s.job.name", ""]

2
config/testdata/otlp_sanitize_resource_attributes.good.yml vendored Normal file
View file

@ -0,0 +1,2 @@
otlp:
promote_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name"]

1
discovery/legacymanager/manager_test.go
View file

@ -1090,7 +1090,6 @@ func TestCoordinationWithReceiver(t *testing.T) {
}
for _, tc := range testCases {
tc := tc
t.Run(tc.title, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()

4
discovery/moby/network.go
View file

@ -17,7 +17,7 @@ import (
"context"
"strconv"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"github.com/prometheus/prometheus/util/strutil"
@ -34,7 +34,7 @@ const (
)
func getNetworksLabels(ctx context.Context, client *client.Client, labelPrefix string) (map[string]map[string]string, error) {
networks, err := client.NetworkList(ctx, types.NetworkListOptions{})
networks, err := client.NetworkList(ctx, network.ListOptions{})
if err != nil {
return nil, err
}

9
docs/command-line/promtool.md
View file

@ -442,6 +442,15 @@ Unit testing.
#### Flags
| Flag | Description |
| --- | --- |
| <code class="text-nowrap">--junit</code> | File path to store JUnit XML test results. |
##### `promtool test rules`
Unit tests for rules.

28
docs/configuration/configuration.md
View file

@ -152,6 +152,10 @@ alerting:
remote_write:
[ - <remote_write> ... ]
# Settings related to the OTLP receiver feature.
otlp:
[ promote_resource_attributes: [<string>, ...] | default = [ ] ]
# Settings related to the remote read feature.
remote_read:
[ - <remote_read> ... ]
@ -458,13 +462,15 @@ metric_relabel_configs:
[ keep_dropped_targets: <int> | default = 0 ]
# Limit on total number of positive and negative buckets allowed in a single
# native histogram. If this is exceeded, the entire scrape will be treated as
# failed. 0 means no limit.
# native histogram. The resolution of a histogram with more buckets will be
# reduced until the number of buckets is within the limit. If the limit cannot
# be reached, the scrape will fail.
# 0 means no limit.
[ native_histogram_bucket_limit: <int> | default = 0 ]
# Lower limit for the growth factor of one bucket to the next in each native
# histogram. The resolution of a histogram with a lower growth factor will be
# reduced until it is within the limit.
# reduced as much as possible until it is within the limit.
# To set an upper limit for the schema (equivalent to "scale" in OTel's
# exponential histograms), use the following factor limits:
#
@ -3395,8 +3401,8 @@ authorization:
# It is mutually exclusive with `credentials`.
[ credentials_file: <filename> ]
# Optionally configures AWS's Signature Verification 4 signing process to
# sign requests. Cannot be set at the same time as basic_auth, authorization, or oauth2.
# Optionally configures AWS's Signature Verification 4 signing process to sign requests.
# Cannot be set at the same time as basic_auth, authorization, oauth2, azuread or google_iam.
# To use the default credentials from the AWS SDK, use `sigv4: {}`.
sigv4:
# The AWS region. If blank, the region from the default credentials chain
@ -3649,12 +3655,12 @@ sigv4:
[ role_arn: <string> ]
# Optional OAuth 2.0 configuration.
# Cannot be used at the same time as basic_auth, authorization, sigv4, or azuread.
# Cannot be used at the same time as basic_auth, authorization, sigv4, azuread or google_iam.
oauth2:
[ <oauth2> ]
# Optional AzureAD configuration.
# Cannot be used at the same time as basic_auth, authorization, oauth2, or sigv4.
# Cannot be used at the same time as basic_auth, authorization, oauth2, sigv4 or google_iam.
azuread:
# The Azure Cloud. Options are 'AzurePublic', 'AzureChina', or 'AzureGovernment'.
[ cloud: <string> | default = AzurePublic ]
@ -3674,6 +3680,14 @@ azuread:
[ sdk:
[ tenant_id: <string> ] ]
# WARNING: Remote write is NOT SUPPORTED by Google Cloud. This configuration is reserved for future use.
# Optional Google Cloud Monitoring configuration.
# Cannot be used at the same time as basic_auth, authorization, oauth2, sigv4 or azuread.
# To use the default credentials from the Google Cloud SDK, use `google_iam: {}`.
google_iam:
# Service account key with monitoring write permessions.
credentials_file: <file_name>
# Configures the remote write request's TLS settings.
tls_config:
[ <tls_config> ]

4
docs/configuration/unit_testing_rules.md
View file

@ -92,7 +92,7 @@ series: <string>
#
# Native histogram notation:
# Native histograms can be used instead of floating point numbers using the following notation:
# {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5}}
# {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5 counter_reset_hint:gauge}}
# Native histograms support the same expanding notation as floating point numbers, i.e. 'axn', 'a+bxn' and 'a-bxn'.
# All properties are optional and default to 0. The order is not important. The following properties are supported:
# - schema (int):
@ -119,6 +119,8 @@ series: <string>
# Observation counts in negative buckets. Each represents an absolute count.
# - n_offset (int):
# The starting index of the first entry in the negative buckets.
# - counter_reset_hint (one of 'unknown', 'reset', 'not_reset' or 'gauge')
# The counter reset hint associated with this histogram. Defaults to 'unknown' if not set.
values: <string>
```

16
docs/querying/basics.md
View file

@ -8,9 +8,15 @@ sort_rank: 1
Prometheus provides a functional query language called PromQL (Prometheus Query
Language) that lets the user select and aggregate time series data in real
time. The result of an expression can either be shown as a graph, viewed as
tabular data in Prometheus's expression browser, or consumed by external
systems via the [HTTP API](api.md).
time.
When you send a query request to Prometheus, it can be an _instant query_, evaluated at one point in time,
or a _range query_ at equally-spaced steps between a start and an end time. PromQL works exactly the same
in each cases; the range query is just like an instant query run multiple times at different timestamps.
In the Prometheus UI, the "Table" tab is for instant queries and the "Graph" tab is for range queries.
Other programs can fetch the result of a PromQL expression via the [HTTP API](api.md).
## Examples
@ -94,9 +100,7 @@ Examples:
## Time series selectors
Time series selectors are responsible for selecting the times series and raw or inferred sample timestamps and values.
Time series *selectors* are not to be confused with higher level concept of instant and range *queries* that can execute the time series *selectors*. A higher level instant query would evaluate the given selector at one point in time, however the range query would evaluate the selector at multiple different times in between a minimum and maximum timestamp at regular steps.
These are the basic building-blocks that instruct PromQL what data to fetch.
### Instant vector selectors

2
documentation/examples/remote_storage/go.mod
View file

@ -10,7 +10,7 @@ require (
github.com/influxdata/influxdb v1.11.5
github.com/prometheus/client_golang v1.19.1
github.com/prometheus/common v0.55.0
github.com/prometheus/prometheus v0.52.1
github.com/prometheus/prometheus v0.53.1
github.com/stretchr/testify v1.9.0
)

96
go.mod
View file

@ -5,20 +5,20 @@ go 1.21.0
toolchain go1.22.5
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0
github.com/Code-Hex/go-generics-cache v1.5.1
github.com/KimMachineGun/automemlimit v0.6.1
github.com/alecthomas/kingpin/v2 v2.4.0
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9
github.com/aws/aws-sdk-go v1.53.16
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30
github.com/aws/aws-sdk-go v1.54.19
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3
github.com/cespare/xxhash/v2 v2.3.0
github.com/dennwc/varint v1.0.0
github.com/digitalocean/godo v1.117.0
github.com/docker/docker v26.1.3+incompatible
github.com/digitalocean/godo v1.119.0
github.com/docker/docker v27.0.3+incompatible
github.com/edsrzf/mmap-go v1.1.0
github.com/envoyproxy/go-control-plane v0.12.0
github.com/envoyproxy/protoc-gen-validate v1.0.4
@ -31,60 +31,60 @@ require (
github.com/gogo/protobuf v1.3.2
github.com/golang/snappy v0.0.4
github.com/google/go-cmp v0.6.0
github.com/google/pprof v0.0.0-20240528025155-186aa0362fba
github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da
github.com/google/uuid v1.6.0
github.com/gophercloud/gophercloud v1.12.0
github.com/gophercloud/gophercloud v1.14.0
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc
github.com/grpc-ecosystem/grpc-gateway v1.16.0
github.com/hashicorp/consul/api v1.29.1
github.com/hashicorp/nomad/api v0.0.0-20240604134157-e73d8bb1140d
github.com/hetznercloud/hcloud-go/v2 v2.9.0
github.com/hashicorp/consul/api v1.29.2
github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3
github.com/hetznercloud/hcloud-go/v2 v2.12.0
github.com/ionos-cloud/sdk-go/v6 v6.1.11
github.com/json-iterator/go v1.1.12
github.com/klauspost/compress v1.17.8
github.com/klauspost/compress v1.17.9
github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b
github.com/linode/linodego v1.35.0
github.com/miekg/dns v1.1.59
github.com/linode/linodego v1.38.0
github.com/miekg/dns v1.1.61
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1
github.com/oklog/run v1.1.0
github.com/oklog/ulid v1.3.1
github.com/ovh/go-ovh v1.5.1
github.com/ovh/go-ovh v1.6.0
github.com/prometheus/alertmanager v0.27.0
github.com/prometheus/client_golang v1.19.1
github.com/prometheus/client_model v0.6.1
github.com/prometheus/common v0.54.0
github.com/prometheus/common v0.55.0
github.com/prometheus/common/assets v0.2.0
github.com/prometheus/common/sigv4 v0.1.0
github.com/prometheus/exporter-toolkit v0.11.0
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.27
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.29
github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c
github.com/stretchr/testify v1.9.0
github.com/vultr/govultr/v2 v2.17.2
go.opentelemetry.io/collector/pdata v1.11.0
go.opentelemetry.io/collector/semconv v0.104.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0
go.opentelemetry.io/otel v1.27.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0
go.opentelemetry.io/otel/sdk v1.27.0
go.opentelemetry.io/otel/trace v1.27.0
go.opentelemetry.io/collector/pdata v1.12.0
go.opentelemetry.io/collector/semconv v0.105.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0
go.opentelemetry.io/otel v1.28.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0
go.opentelemetry.io/otel/sdk v1.28.0
go.opentelemetry.io/otel/trace v1.28.0
go.uber.org/atomic v1.11.0
go.uber.org/automaxprocs v1.5.3
go.uber.org/goleak v1.3.0
go.uber.org/multierr v1.11.0
golang.org/x/net v0.26.0
golang.org/x/net v0.27.0
golang.org/x/oauth2 v0.21.0
golang.org/x/sync v0.7.0
golang.org/x/sys v0.21.0
golang.org/x/sys v0.22.0
golang.org/x/text v0.16.0
golang.org/x/time v0.5.0
golang.org/x/tools v0.22.0
google.golang.org/api v0.183.0
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157
google.golang.org/grpc v1.64.0
golang.org/x/tools v0.23.0
google.golang.org/api v0.189.0
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d
google.golang.org/grpc v1.65.0
google.golang.org/protobuf v1.34.2
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
@ -92,14 +92,14 @@ require (
k8s.io/apimachinery v0.29.3
k8s.io/client-go v0.29.3
k8s.io/klog v1.0.0
k8s.io/klog/v2 v2.120.1
k8s.io/klog/v2 v2.130.1
)
require (
cloud.google.com/go/auth v0.5.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
cloud.google.com/go/compute/metadata v0.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0 // indirect
cloud.google.com/go/auth v0.7.2 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
@ -107,7 +107,7 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cilium/ebpf v0.11.0 // indirect
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 // indirect
github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect
github.com/containerd/cgroups/v3 v3.0.3 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
@ -121,7 +121,7 @@ require (
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-kit/kit v0.12.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/analysis v0.22.2 // indirect
github.com/go-openapi/errors v0.22.0 // indirect
@ -134,7 +134,7 @@ require (
github.com/go-resty/resty/v2 v2.13.1 // indirect
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/glog v1.2.0 // indirect
github.com/golang/glog v1.2.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
@ -142,7 +142,7 @@ require (
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.4 // indirect
github.com/googleapis/gax-go/v2 v2.12.5 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/hashicorp/cronexpr v1.1.2 // indirect
@ -178,20 +178,20 @@ require (
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
go.mongodb.org/mongo-driver v1.14.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/metric v1.27.0 // indirect
go.opentelemetry.io/proto/otlp v1.2.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
golang.org/x/crypto v0.25.0 // indirect
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/term v0.21.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
golang.org/x/mod v0.19.0 // indirect
golang.org/x/term v0.22.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gotest.tools/v3 v3.0.3 // indirect

193
go.sum
View file

@ -12,18 +12,18 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP
cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
cloud.google.com/go/auth v0.5.1 h1:0QNO7VThG54LUzKiQxv8C6x1YX7lUrzlAa1nVLF8CIw=
cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s=
cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4=
cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q=
cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE=
cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs=
cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI=
cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
@ -36,12 +36,12 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0 h1:sUFnFjzDUie80h24I7mrKtwCKgLY9L8h5Tp2x9+TWqk=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0/go.mod h1:52JbnQTp15qg5mRkMBHwp0j0ZFwHJ42Sx3zVV5RE9p0=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 h1:GJHeeA2N7xrG3q30L2UXDyuWRzDM900/65j70wcM4Ww=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do=
@ -75,8 +75,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs=
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 h1:t3eaIm0rUkzbrIewtiFmMK5RXHej2XnoXNhxVsAYUfg=
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@ -92,8 +92,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:W
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.53.16 h1:8oZjKQO/ml1WLUZw5hvF7pvYjPf8o9f57Wldoy/q9Qc=
github.com/aws/aws-sdk-go v1.53.16/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI=
github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
@ -120,8 +120,8 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc=
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM=
github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw=
github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0=
@ -143,14 +143,14 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=
github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/digitalocean/godo v1.117.0 h1:WVlTe09melDYTd7VCVyvHcNWbgB+uI1O115+5LOtdSw=
github.com/digitalocean/godo v1.117.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo=
github.com/digitalocean/godo v1.119.0 h1:dmFNQwSIAcH3z+FVovHLkazKDC2uA8oOlGvg5+H4vRw=
github.com/digitalocean/godo v1.119.0/go.mod h1:WQVH83OHUy6gC4gXpEVQKtxTd4L5oCp+5OialidkPLY=
github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo=
github.com/docker/docker v26.1.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v27.0.3+incompatible h1:aBGI9TeQ4MPlhquTQKq9XbK79rKFVwXNUAYz9aXyEBE=
github.com/docker/docker v27.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@ -210,8 +210,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-openapi/analysis v0.22.2 h1:ZBmNoP2h5omLKr/srIC9bfqrUGzT6g6gNv03HE9Vpj0=
@ -251,8 +251,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68=
github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4=
github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@ -319,8 +319,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g=
github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da h1:xRmpO92tb8y+Z85iUOMOicpCfaYcv7o3Cg3wKrIpg8g=
github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
@ -332,10 +332,10 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF
github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg=
github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI=
github.com/gophercloud/gophercloud v1.12.0 h1:Jrz16vPAL93l80q16fp8NplrTCp93y7rZh2P3Q4Yq7g=
github.com/gophercloud/gophercloud v1.12.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=
github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA=
github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E=
github.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8=
github.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@ -353,10 +353,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
github.com/hashicorp/consul/api v1.29.1 h1:UEwOjYJrd3lG1x5w7HxDRMGiAUPrb3f103EoeKuuEcc=
github.com/hashicorp/consul/api v1.29.1/go.mod h1:lumfRkY/coLuqMICkI7Fh3ylMG31mQSRZyef2c5YvJI=
github.com/hashicorp/consul/proto-public v0.6.1 h1:+uzH3olCrksXYWAYHKqK782CtK9scfqH+Unlw3UHhCg=
github.com/hashicorp/consul/proto-public v0.6.1/go.mod h1:cXXbOg74KBNGajC+o8RlA502Esf0R9prcoJgiOX/2Tg=
github.com/hashicorp/consul/api v1.29.2 h1:aYyRn8EdE2mSfG14S1+L9Qkjtz8RzmaWh6AcNGRNwPw=
github.com/hashicorp/consul/api v1.29.2/go.mod h1:0YObcaLNDSbtlgzIRtmRXI1ZkeuK0trCBxwZQ4MYnIk=
github.com/hashicorp/consul/proto-public v0.6.2 h1:+DA/3g/IiKlJZb88NBn0ZgXrxJp2NlvCZdEyl+qxvL0=
github.com/hashicorp/consul/proto-public v0.6.2/go.mod h1:cXXbOg74KBNGajC+o8RlA502Esf0R9prcoJgiOX/2Tg=
github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/consul/sdk v0.16.1 h1:V8TxTnImoPD5cj0U9Spl0TUxcytjcbbJeADFF07KdHg=
github.com/hashicorp/consul/sdk v0.16.1/go.mod h1:fSXvwxB2hmh1FMZCNl6PwX0Q/1wdWtHJcZ7Ea5tns0s=
@ -409,13 +409,13 @@ github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM=
github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0=
github.com/hashicorp/nomad/api v0.0.0-20240604134157-e73d8bb1140d h1:KHq+mAzWSkumj4PDoXc5VZbycPGcmYu8tohgVLQ6SIc=
github.com/hashicorp/nomad/api v0.0.0-20240604134157-e73d8bb1140d/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE=
github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3 h1:fgVfQ4AC1avVOnu2cfms8VAiD8lUq3vWI8mTocOXN/w=
github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
github.com/hetznercloud/hcloud-go/v2 v2.9.0 h1:s0N6R7Zoi2DPfMtUF5o9VeUBzTtHVY6MIkHOQnfu/AY=
github.com/hetznercloud/hcloud-go/v2 v2.9.0/go.mod h1:qtW/TuU7Bs16ibXl/ktJarWqU2LwHr7eGlwoilHxtgg=
github.com/hetznercloud/hcloud-go/v2 v2.12.0 h1:nOgfNTo0gyXZJJdM8mo/XH5MO/e80wAEpldRzdWayhY=
github.com/hetznercloud/hcloud-go/v2 v2.12.0/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOFFddpfBm/6R1Zz0IiF0=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@ -454,8 +454,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00=
github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@ -472,8 +472,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
github.com/linode/linodego v1.35.0 h1:rIhUeCHBLEDlkoRnOTwzSGzljQ3ksXwLxacmXnrV+Do=
github.com/linode/linodego v1.35.0/go.mod h1:JxuhOEAMfSxun6RU5/MgTKH2GGTmFrhKRj3wL1NFin0=
github.com/linode/linodego v1.38.0 h1:wP3oW9OhGc6vhze8NPf2knbwH4TzSbrjzuCd9okjbTY=
github.com/linode/linodego v1.38.0/go.mod h1:L7GXKFD3PoN2xSEtFc04wIXP5WK65O10jYQx0PQISWQ=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
@ -500,8 +500,8 @@ github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwU
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs=
github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@ -573,8 +573,8 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
github.com/ovh/go-ovh v1.5.1 h1:P8O+7H+NQuFK9P/j4sFW5C0fvSS2DnHYGPwdVCp45wI=
github.com/ovh/go-ovh v1.5.1/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c=
github.com/ovh/go-ovh v1.6.0 h1:ixLOwxQdzYDx296sXcgS35TOPEahJkpjMGtzPadCjQI=
github.com/ovh/go-ovh v1.6.0/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c=
github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
@ -625,8 +625,8 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8=
github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ=
github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
github.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM=
github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI=
github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=
@ -639,8 +639,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@ -650,8 +650,8 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.27 h1:yGAraK1uUjlhSXgNMIy8o/J4LFNcy7yeipBqt9N9mVg=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.27/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.29 h1:BkTk4gynLjguayxrYxZoMZjBnAOh7ntQvUkOFmkMqPU=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.29/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shoenig/test v1.7.1 h1:UJcjSAI3aUKx52kfcfhblgyhZceouhvvs3OYdWgn+PY=
@ -694,6 +694,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@ -723,28 +724,28 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/collector/pdata v1.11.0 h1:rzYyV1zfTQQz1DI9hCiaKyyaczqawN75XO9mdXmR/hE=
go.opentelemetry.io/collector/pdata v1.11.0/go.mod h1:IHxHsp+Jq/xfjORQMDJjSH6jvedOSTOyu3nbxqhWSYE=
go.opentelemetry.io/collector/semconv v0.104.0 h1:dUvajnh+AYJLEW/XOPk0T0BlwltSdi3vrjO7nSOos3k=
go.opentelemetry.io/collector/semconv v0.104.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0=
go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg=
go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY=
go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik=
go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak=
go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI=
go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A=
go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw=
go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4=
go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94=
go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A=
go.opentelemetry.io/collector/pdata v1.12.0 h1:Xx5VK1p4VO0md8MWm2icwC1MnJ7f8EimKItMWw46BmA=
go.opentelemetry.io/collector/pdata v1.12.0/go.mod h1:MYeB0MmMAxeM0hstCFrCqWLzdyeYySim2dG6pDT6nYI=
go.opentelemetry.io/collector/semconv v0.105.0 h1:8p6dZ3JfxFTjbY38d8xlQGB1TQ3nPUvs+D0RERniZ1g=
go.opentelemetry.io/collector/semconv v0.105.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg=
go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk=
go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
@ -773,8 +774,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -809,8 +810,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -856,8 +857,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -946,16 +947,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1025,8 +1026,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1046,8 +1047,8 @@ google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.183.0 h1:PNMeRDwo1pJdgNcFQ9GstuLe/noWKIc89pRWRLMvLwE=
google.golang.org/api v0.183.0/go.mod h1:q43adC5/pHoSZTx5h2mSmdF7NcyfW9JuDyIOJAgS9ZQ=
google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI=
google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -1084,10 +1085,10 @@ google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1m
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw=
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY=
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade h1:oCRSWfwGXQsqlVdErcyTt4A93Y8fo0/9D4b1gnI++qo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
@ -1106,8 +1107,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

24
model/labels/labels.go
View file

@ -38,10 +38,10 @@ func (ls Labels) Bytes(buf []byte) []byte {
b.WriteByte(labelSep)
for i, l := range ls {
if i > 0 {
b.WriteByte(seps[0])
b.WriteByte(sep)
}
b.WriteString(l.Name)
b.WriteByte(seps[0])
b.WriteByte(sep)
b.WriteString(l.Value)
}
return b.Bytes()
@ -86,9 +86,9 @@ func (ls Labels) Hash() uint64 {
}
b = append(b, v.Name...)
b = append(b, seps[0])
b = append(b, sep)
b = append(b, v.Value...)
b = append(b, seps[0])
b = append(b, sep)
}
return xxhash.Sum64(b)
}
@ -106,9 +106,9 @@ func (ls Labels) HashForLabels(b []byte, names ...string) (uint64, []byte) {
i++
default:
b = append(b, ls[i].Name...)
b = append(b, seps[0])
b = append(b, sep)
b = append(b, ls[i].Value...)
b = append(b, seps[0])
b = append(b, sep)
i++
j++
}
@ -130,9 +130,9 @@ func (ls Labels) HashWithoutLabels(b []byte, names ...string) (uint64, []byte) {
continue
}
b = append(b, ls[i].Name...)
b = append(b, seps[0])
b = append(b, sep)
b = append(b, ls[i].Value...)
b = append(b, seps[0])
b = append(b, sep)
}
return xxhash.Sum64(b), b
}
@ -151,10 +151,10 @@ func (ls Labels) BytesWithLabels(buf []byte, names ...string) []byte {
i++
default:
if b.Len() > 1 {
b.WriteByte(seps[0])
b.WriteByte(sep)
}
b.WriteString(ls[i].Name)
b.WriteByte(seps[0])
b.WriteByte(sep)
b.WriteString(ls[i].Value)
i++
j++
@ -177,10 +177,10 @@ func (ls Labels) BytesWithoutLabels(buf []byte, names ...string) []byte {
continue
}
if b.Len() > 1 {
b.WriteByte(seps[0])
b.WriteByte(sep)
}
b.WriteString(ls[i].Name)
b.WriteByte(seps[0])
b.WriteByte(sep)
b.WriteString(ls[i].Value)
}
return b.Bytes()

5
model/labels/labels_common.go
View file

@ -29,10 +29,11 @@ const (
BucketLabel = "le"
InstanceName = "instance"
labelSep = '\xfe'
labelSep = '\xfe' // Used at beginning of `Bytes` return.
sep = '\xff' // Used between labels in `Bytes` and `Hash`.
)
var seps = []byte{'\xff'}
var seps = []byte{sep} // Used with Hash, which has no WriteByte method.
// Label is a key/value pair of strings.
type Label struct {

24
model/labels/labels_dedupelabels.go
View file

@ -146,13 +146,13 @@ func (ls Labels) Bytes(buf []byte) []byte {
b := bytes.NewBuffer(buf[:0])
for i := 0; i < len(ls.data); {
if i > 0 {
b.WriteByte(seps[0])
b.WriteByte(sep)
}
var name, value string
name, i = decodeString(ls.syms, ls.data, i)
value, i = decodeString(ls.syms, ls.data, i)
b.WriteString(name)
b.WriteByte(seps[0])
b.WriteByte(sep)
b.WriteString(value)
}
return b.Bytes()
@ -201,9 +201,9 @@ func (ls Labels) Hash() uint64 {
}
b = append(b, name...)
b = append(b, seps[0])
b = append(b, sep)
b = append(b, value...)
b = append(b, seps[0])
b = append(b, sep)
pos = newPos
}
return xxhash.Sum64(b)
@ -226,9 +226,9 @@ func (ls Labels) HashForLabels(b []byte, names ...string) (uint64, []byte) {
}
if name == names[j] {
b = append(b, name...)
b = append(b, seps[0])
b = append(b, sep)
b = append(b, value...)
b = append(b, seps[0])
b = append(b, sep)
}
}
@ -252,9 +252,9 @@ func (ls Labels) HashWithoutLabels(b []byte, names ...string) (uint64, []byte) {
continue
}
b = append(b, name...)
b = append(b, seps[0])
b = append(b, sep)
b = append(b, value...)
b = append(b, seps[0])
b = append(b, sep)
}
return xxhash.Sum64(b), b
}
@ -275,10 +275,10 @@ func (ls Labels) BytesWithLabels(buf []byte, names ...string) []byte {
}
if lName == names[j] {
if b.Len() > 1 {
b.WriteByte(seps[0])
b.WriteByte(sep)
}
b.WriteString(lName)
b.WriteByte(seps[0])
b.WriteByte(sep)
b.WriteString(lValue)
}
pos = newPos
@ -299,10 +299,10 @@ func (ls Labels) BytesWithoutLabels(buf []byte, names ...string) []byte {
}
if j == len(names) || lName != names[j] {
if b.Len() > 1 {
b.WriteByte(seps[0])
b.WriteByte(sep)
}
b.WriteString(lName)
b.WriteByte(seps[0])
b.WriteByte(sep)
b.WriteString(lValue)
}
pos = newPos

8
model/labels/labels_stringlabels.go
View file

@ -112,9 +112,9 @@ func (ls Labels) HashForLabels(b []byte, names ...string) (uint64, []byte) {
}
if name == names[j] {
b = append(b, name...)
b = append(b, seps[0])
b = append(b, sep)
b = append(b, value...)
b = append(b, seps[0])
b = append(b, sep)
}
}
@ -138,9 +138,9 @@ func (ls Labels) HashWithoutLabels(b []byte, names ...string) (uint64, []byte) {
continue
}
b = append(b, name...)
b = append(b, seps[0])
b = append(b, sep)
b = append(b, value...)
b = append(b, seps[0])
b = append(b, sep)
}
return xxhash.Sum64(b), b
}

4
model/labels/sharding.go
View file

@ -39,9 +39,9 @@ func StableHash(ls Labels) uint64 {
}
b = append(b, v.Name...)
b = append(b, seps[0])
b = append(b, sep)
b = append(b, v.Value...)
b = append(b, seps[0])
b = append(b, sep)
}
return xxhash.Sum64(b)
}

4
model/labels/sharding_dedupelabels.go
View file

@ -43,9 +43,9 @@ func StableHash(ls Labels) uint64 {
}
b = append(b, name...)
b = append(b, seps[0])
b = append(b, sep)
b = append(b, value...)
b = append(b, seps[0])
b = append(b, sep)
pos = newPos
}
return xxhash.Sum64(b)

4
model/labels/sharding_stringlabels.go
View file

@ -43,9 +43,9 @@ func StableHash(ls Labels) uint64 {
}
b = append(b, v.Name...)
b = append(b, seps[0])
b = append(b, sep)
b = append(b, v.Value...)
b = append(b, seps[0])
b = append(b, sep)
}
if h != nil {
return h.Sum64()

4
model/relabel/relabel.go
View file

@ -213,6 +213,10 @@ func (re Regexp) IsZero() bool {
// String returns the original string used to compile the regular expression.
func (re Regexp) String() string {
if re.Regexp == nil {
return ""
}
str := re.Regexp.String()
// Trim the anchor `^(?:` prefix and `)$` suffix.
return str[4 : len(str)-2]

13
model/relabel/relabel_test.go
View file

@ -900,3 +900,16 @@ action: replace
})
}
}
func TestRegexp_ShouldMarshalAndUnmarshalZeroValue(t *testing.T) {
var zero Regexp
marshalled, err := yaml.Marshal(&zero)
require.NoError(t, err)
require.Equal(t, "null\n", string(marshalled))
var unmarshalled Regexp
err = yaml.Unmarshal(marshalled, &unmarshalled)
require.NoError(t, err)
require.Nil(t, unmarshalled.Regexp)
}

3
promql/engine_test.go
View file

@ -26,7 +26,6 @@ import (
"time"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
@ -51,7 +50,7 @@ const (
func TestMain(m *testing.M) {
// Enable experimental functions testing
parser.EnableExperimentalFunctions = true
goleak.VerifyTestMain(m)
testutil.TolerantVerifyLeak(m)
}
func TestQueryConcurrency(t *testing.T) {

22
promql/histogram_stats_iterator.go
View file

@ -48,7 +48,6 @@ func (f *histogramStatsIterator) AtHistogram(h *histogram.Histogram) (int64, *hi
var t int64
t, f.currentH = f.Iterator.AtHistogram(f.currentH)
if value.IsStaleNaN(f.currentH.Sum) {
f.setLastH(f.currentH)
h = &histogram.Histogram{Sum: f.currentH.Sum}
return t, h
}
@ -63,9 +62,13 @@ func (f *histogramStatsIterator) AtHistogram(h *histogram.Histogram) (int64, *hi
return t, h
}
h.CounterResetHint = f.getResetHint(f.currentH)
h.Count = f.currentH.Count
h.Sum = f.currentH.Sum
returnValue := histogram.Histogram{
CounterResetHint: f.getResetHint(f.currentH),
Count: f.currentH.Count,
Sum: f.currentH.Sum,
}
returnValue.CopyTo(h)
f.setLastH(f.currentH)
return t, h
}
@ -77,7 +80,6 @@ func (f *histogramStatsIterator) AtFloatHistogram(fh *histogram.FloatHistogram)
var t int64
t, f.currentFH = f.Iterator.AtFloatHistogram(f.currentFH)
if value.IsStaleNaN(f.currentFH.Sum) {
f.setLastFH(f.currentFH)
return t, &histogram.FloatHistogram{Sum: f.currentFH.Sum}
}
@ -91,9 +93,13 @@ func (f *histogramStatsIterator) AtFloatHistogram(fh *histogram.FloatHistogram)
return t, fh
}
fh.CounterResetHint = f.getFloatResetHint(f.currentFH.CounterResetHint)
fh.Count = f.currentFH.Count
fh.Sum = f.currentFH.Sum
returnValue := histogram.FloatHistogram{
CounterResetHint: f.getFloatResetHint(f.currentFH.CounterResetHint),
Count: f.currentFH.Count,
Sum: f.currentFH.Sum,
}
returnValue.CopyTo(fh)
f.setLastFH(f.currentFH)
return t, fh
}

112
promql/histogram_stats_iterator_test.go
View file

@ -14,62 +14,132 @@
package promql
import (
"fmt"
"math"
"testing"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/value"
"github.com/prometheus/prometheus/tsdb/chunkenc"
"github.com/prometheus/prometheus/tsdb/tsdbutil"
)
func TestHistogramStatsDecoding(t *testing.T) {
histograms := []*histogram.Histogram{
tsdbutil.GenerateTestHistogram(0),
tsdbutil.GenerateTestHistogram(1),
tsdbutil.GenerateTestHistogram(2),
tsdbutil.GenerateTestHistogram(2),
}
histograms[0].CounterResetHint = histogram.NotCounterReset
histograms[1].CounterResetHint = histogram.UnknownCounterReset
histograms[2].CounterResetHint = histogram.CounterReset
histograms[3].CounterResetHint = histogram.UnknownCounterReset
expectedHints := []histogram.CounterResetHint{
cases := []struct {
name string
histograms []*histogram.Histogram
expectedHints []histogram.CounterResetHint
}{
{
name: "unknown counter reset triggers detection",
histograms: []*histogram.Histogram{
tsdbutil.GenerateTestHistogramWithHint(0, histogram.NotCounterReset),
tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
tsdbutil.GenerateTestHistogramWithHint(2, histogram.CounterReset),
tsdbutil.GenerateTestHistogramWithHint(2, histogram.UnknownCounterReset),
},
expectedHints: []histogram.CounterResetHint{
histogram.NotCounterReset,
histogram.NotCounterReset,
histogram.CounterReset,
histogram.NotCounterReset,
},
},
{
name: "stale sample before unknown reset hint",
histograms: []*histogram.Histogram{
tsdbutil.GenerateTestHistogramWithHint(0, histogram.NotCounterReset),
tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
{Sum: math.Float64frombits(value.StaleNaN)},
tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
},
expectedHints: []histogram.CounterResetHint{
histogram.NotCounterReset,
histogram.NotCounterReset,
histogram.UnknownCounterReset,
histogram.NotCounterReset,
},
},
{
name: "unknown counter reset at the beginning",
histograms: []*histogram.Histogram{
tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
},
expectedHints: []histogram.CounterResetHint{
histogram.NotCounterReset,
},
},
{
name: "detect real counter reset",
histograms: []*histogram.Histogram{
tsdbutil.GenerateTestHistogramWithHint(2, histogram.UnknownCounterReset),
tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
},
expectedHints: []histogram.CounterResetHint{
histogram.NotCounterReset,
histogram.CounterReset,
},
},
{
name: "detect real counter reset after stale NaN",
histograms: []*histogram.Histogram{
tsdbutil.GenerateTestHistogramWithHint(2, histogram.UnknownCounterReset),
{Sum: math.Float64frombits(value.StaleNaN)},
tsdbutil.GenerateTestHistogramWithHint(1, histogram.UnknownCounterReset),
},
expectedHints: []histogram.CounterResetHint{
histogram.NotCounterReset,
histogram.UnknownCounterReset,
histogram.CounterReset,
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Run("histogram_stats", func(t *testing.T) {
decodedStats := make([]*histogram.Histogram, 0)
statsIterator := NewHistogramStatsIterator(newHistogramSeries(histograms).Iterator(nil))
statsIterator := NewHistogramStatsIterator(newHistogramSeries(tc.histograms).Iterator(nil))
for statsIterator.Next() != chunkenc.ValNone {
_, h := statsIterator.AtHistogram(nil)
decodedStats = append(decodedStats, h)
}
for i := 0; i < len(histograms); i++ {
require.Equal(t, expectedHints[i], decodedStats[i].CounterResetHint)
require.Equal(t, histograms[i].Count, decodedStats[i].Count)
require.Equal(t, histograms[i].Sum, decodedStats[i].Sum)
for i := 0; i < len(tc.histograms); i++ {
require.Equal(t, tc.expectedHints[i], decodedStats[i].CounterResetHint, fmt.Sprintf("mismatch in counter reset hint for histogram %d", i))
h := tc.histograms[i]
if value.IsStaleNaN(h.Sum) {
require.True(t, value.IsStaleNaN(decodedStats[i].Sum))
require.Equal(t, uint64(0), decodedStats[i].Count)
} else {
require.Equal(t, tc.histograms[i].Count, decodedStats[i].Count)
require.Equal(t, tc.histograms[i].Sum, decodedStats[i].Sum)
}
}
})
t.Run("float_histogram_stats", func(t *testing.T) {
decodedStats := make([]*histogram.FloatHistogram, 0)
statsIterator := NewHistogramStatsIterator(newHistogramSeries(histograms).Iterator(nil))
statsIterator := NewHistogramStatsIterator(newHistogramSeries(tc.histograms).Iterator(nil))
for statsIterator.Next() != chunkenc.ValNone {
_, h := statsIterator.AtFloatHistogram(nil)
decodedStats = append(decodedStats, h)
}
for i := 0; i < len(histograms); i++ {
fh := histograms[i].ToFloat(nil)
require.Equal(t, expectedHints[i], decodedStats[i].CounterResetHint)
for i := 0; i < len(tc.histograms); i++ {
require.Equal(t, tc.expectedHints[i], decodedStats[i].CounterResetHint)
fh := tc.histograms[i].ToFloat(nil)
if value.IsStaleNaN(fh.Sum) {
require.True(t, value.IsStaleNaN(decodedStats[i].Sum))
require.Equal(t, float64(0), decodedStats[i].Count)
} else {
require.Equal(t, fh.Count, decodedStats[i].Count)
require.Equal(t, fh.Sum, decodedStats[i].Sum)
}
}
})
})
}
}
type histogramSeries struct {

17
promql/parser/generated_parser.y
View file

@ -84,6 +84,7 @@ NEGATIVE_BUCKETS_DESC
ZERO_BUCKET_DESC
ZERO_BUCKET_WIDTH_DESC
CUSTOM_VALUES_DESC
COUNTER_RESET_HINT_DESC
%token histogramDescEnd
// Operators.
@ -149,6 +150,14 @@ START
END
%token preprocessorEnd
// Counter reset hints.
%token counterResetHintsStart
%token <item>
UNKNOWN_COUNTER_RESET
COUNTER_RESET
NOT_COUNTER_RESET
GAUGE_TYPE
%token counterResetHintsEnd
// Start symbols for the generated parser.
%token startSymbolsStart
@ -163,7 +172,7 @@ START_METRIC_SELECTOR
// Type definitions for grammar rules.
%type <matchers> label_match_list
%type <matcher> label_matcher
%type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op at_modifier_preprocessors string_identifier
%type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op at_modifier_preprocessors string_identifier counter_reset_hint
%type <labels> label_set metric
%type <lblList> label_set_list
%type <label> label_set_item
@ -839,6 +848,11 @@ histogram_desc_item
$$ = yylex.(*parser).newMap()
$$["n_offset"] = $3
}
| COUNTER_RESET_HINT_DESC COLON counter_reset_hint
{
$$ = yylex.(*parser).newMap()
$$["counter_reset_hint"] = $3
}
;
bucket_set : LEFT_BRACKET bucket_set_list SPACE RIGHT_BRACKET
@ -862,6 +876,7 @@ bucket_set_list : bucket_set_list SPACE number
| bucket_set_list error
;
counter_reset_hint : UNKNOWN_COUNTER_RESET | COUNTER_RESET | NOT_COUNTER_RESET | GAUGE_TYPE;
/*
* Keyword lists.

797
promql/parser/generated_parser.y.go
View file

@ -67,64 +67,71 @@ const NEGATIVE_BUCKETS_DESC = 57376
const ZERO_BUCKET_DESC = 57377
const ZERO_BUCKET_WIDTH_DESC = 57378
const CUSTOM_VALUES_DESC = 57379
const histogramDescEnd = 57380
const operatorsStart = 57381
const ADD = 57382
const DIV = 57383
const EQLC = 57384
const EQL_REGEX = 57385
const GTE = 57386
const GTR = 57387
const LAND = 57388
const LOR = 57389
const LSS = 57390
const LTE = 57391
const LUNLESS = 57392
const MOD = 57393
const MUL = 57394
const NEQ = 57395
const NEQ_REGEX = 57396
const POW = 57397
const SUB = 57398
const AT = 57399
const ATAN2 = 57400
const operatorsEnd = 57401
const aggregatorsStart = 57402
const AVG = 57403
const BOTTOMK = 57404
const COUNT = 57405
const COUNT_VALUES = 57406
const GROUP = 57407
const MAX = 57408
const MIN = 57409
const QUANTILE = 57410
const STDDEV = 57411
const STDVAR = 57412
const SUM = 57413
const TOPK = 57414
const LIMITK = 57415
const LIMIT_RATIO = 57416
const aggregatorsEnd = 57417
const keywordsStart = 57418
const BOOL = 57419
const BY = 57420
const GROUP_LEFT = 57421
const GROUP_RIGHT = 57422
const IGNORING = 57423
const OFFSET = 57424
const ON = 57425
const WITHOUT = 57426
const keywordsEnd = 57427
const preprocessorStart = 57428
const START = 57429
const END = 57430
const preprocessorEnd = 57431
const startSymbolsStart = 57432
const START_METRIC = 57433
const START_SERIES_DESCRIPTION = 57434
const START_EXPRESSION = 57435
const START_METRIC_SELECTOR = 57436
const startSymbolsEnd = 57437
const COUNTER_RESET_HINT_DESC = 57380
const histogramDescEnd = 57381
const operatorsStart = 57382
const ADD = 57383
const DIV = 57384
const EQLC = 57385
const EQL_REGEX = 57386
const GTE = 57387
const GTR = 57388
const LAND = 57389
const LOR = 57390
const LSS = 57391
const LTE = 57392
const LUNLESS = 57393
const MOD = 57394
const MUL = 57395
const NEQ = 57396
const NEQ_REGEX = 57397
const POW = 57398
const SUB = 57399
const AT = 57400
const ATAN2 = 57401
const operatorsEnd = 57402
const aggregatorsStart = 57403
const AVG = 57404
const BOTTOMK = 57405
const COUNT = 57406
const COUNT_VALUES = 57407
const GROUP = 57408
const MAX = 57409
const MIN = 57410
const QUANTILE = 57411
const STDDEV = 57412
const STDVAR = 57413
const SUM = 57414
const TOPK = 57415
const LIMITK = 57416
const LIMIT_RATIO = 57417
const aggregatorsEnd = 57418
const keywordsStart = 57419
const BOOL = 57420
const BY = 57421
const GROUP_LEFT = 57422
const GROUP_RIGHT = 57423
const IGNORING = 57424
const OFFSET = 57425
const ON = 57426
const WITHOUT = 57427
const keywordsEnd = 57428
const preprocessorStart = 57429
const START = 57430
const END = 57431
const preprocessorEnd = 57432
const counterResetHintsStart = 57433
const UNKNOWN_COUNTER_RESET = 57434
const COUNTER_RESET = 57435
const NOT_COUNTER_RESET = 57436
const GAUGE_TYPE = 57437
const counterResetHintsEnd = 57438
const startSymbolsStart = 57439
const START_METRIC = 57440
const START_SERIES_DESCRIPTION = 57441
const START_EXPRESSION = 57442
const START_METRIC_SELECTOR = 57443
const startSymbolsEnd = 57444
var yyToknames = [...]string{
"$end",
@ -164,6 +171,7 @@ var yyToknames = [...]string{
"ZERO_BUCKET_DESC",
"ZERO_BUCKET_WIDTH_DESC",
"CUSTOM_VALUES_DESC",
"COUNTER_RESET_HINT_DESC",
"histogramDescEnd",
"operatorsStart",
"ADD",
@ -216,6 +224,12 @@ var yyToknames = [...]string{
"START",
"END",
"preprocessorEnd",
"counterResetHintsStart",
"UNKNOWN_COUNTER_RESET",
"COUNTER_RESET",
"NOT_COUNTER_RESET",
"GAUGE_TYPE",
"counterResetHintsEnd",
"startSymbolsStart",
"START_METRIC",
"START_SERIES_DESCRIPTION",
@ -240,308 +254,313 @@ var yyExca = [...]int16{
24, 137,
-2, 0,
-1, 61,
2, 175,
15, 175,
78, 175,
84, 175,
-2, 101,
-1, 62,
2, 176,
15, 176,
78, 176,
84, 176,
-2, 102,
-1, 63,
2, 177,
15, 177,
78, 177,
84, 177,
-2, 104,
-1, 64,
2, 178,
15, 178,
78, 178,
84, 178,
-2, 105,
-1, 65,
2, 179,
15, 179,
78, 179,
84, 179,
-2, 106,
-1, 66,
2, 180,
15, 180,
78, 180,
84, 180,
-2, 111,
-1, 67,
79, 180,
85, 180,
-2, 101,
-1, 62,
2, 181,
15, 181,
78, 181,
84, 181,
-2, 113,
-1, 68,
79, 181,
85, 181,
-2, 102,
-1, 63,
2, 182,
15, 182,
78, 182,
84, 182,
-2, 115,
-1, 69,
79, 182,
85, 182,
-2, 104,
-1, 64,
2, 183,
15, 183,
78, 183,
84, 183,
-2, 116,
-1, 70,
79, 183,
85, 183,
-2, 105,
-1, 65,
2, 184,
15, 184,
78, 184,
84, 184,
-2, 117,
-1, 71,
79, 184,
85, 184,
-2, 106,
-1, 66,
2, 185,
15, 185,
78, 185,
84, 185,
-2, 118,
-1, 72,
79, 185,
85, 185,
-2, 111,
-1, 67,
2, 186,
15, 186,
78, 186,
84, 186,
-2, 119,
-1, 73,
79, 186,
85, 186,
-2, 113,
-1, 68,
2, 187,
15, 187,
78, 187,
84, 187,
-2, 123,
-1, 74,
79, 187,
85, 187,
-2, 115,
-1, 69,
2, 188,
15, 188,
78, 188,
84, 188,
79, 188,
85, 188,
-2, 116,
-1, 70,
2, 189,
15, 189,
79, 189,
85, 189,
-2, 117,
-1, 71,
2, 190,
15, 190,
79, 190,
85, 190,
-2, 118,
-1, 72,
2, 191,
15, 191,
79, 191,
85, 191,
-2, 119,
-1, 73,
2, 192,
15, 192,
79, 192,
85, 192,
-2, 123,
-1, 74,
2, 193,
15, 193,
79, 193,
85, 193,
-2, 124,
-1, 200,
9, 237,
12, 237,
13, 237,
18, 237,
19, 237,
25, 237,
40, 237,
46, 237,
47, 237,
50, 237,
56, 237,
61, 237,
62, 237,
63, 237,
64, 237,
65, 237,
66, 237,
67, 237,
68, 237,
69, 237,
70, 237,
71, 237,
72, 237,
73, 237,
74, 237,
78, 237,
82, 237,
84, 237,
87, 237,
88, 237,
9, 242,
12, 242,
13, 242,
18, 242,
19, 242,
25, 242,
41, 242,
47, 242,
48, 242,
51, 242,
57, 242,
62, 242,
63, 242,
64, 242,
65, 242,
66, 242,
67, 242,
68, 242,
69, 242,
70, 242,
71, 242,
72, 242,
73, 242,
74, 242,
75, 242,
79, 242,
83, 242,
85, 242,
88, 242,
89, 242,
-2, 0,
-1, 201,
9, 237,
12, 237,
13, 237,
18, 237,
19, 237,
25, 237,
40, 237,
46, 237,
47, 237,
50, 237,
56, 237,
61, 237,
62, 237,
63, 237,
64, 237,
65, 237,
66, 237,
67, 237,
68, 237,
69, 237,
70, 237,
71, 237,
72, 237,
73, 237,
74, 237,
78, 237,
82, 237,
84, 237,
87, 237,
88, 237,
9, 242,
12, 242,
13, 242,
18, 242,
19, 242,
25, 242,
41, 242,
47, 242,
48, 242,
51, 242,
57, 242,
62, 242,
63, 242,
64, 242,
65, 242,
66, 242,
67, 242,
68, 242,
69, 242,
70, 242,
71, 242,
72, 242,
73, 242,
74, 242,
75, 242,
79, 242,
83, 242,
85, 242,
88, 242,
89, 242,
-2, 0,
}
const yyPrivate = 57344
const yyLast = 728
const yyLast = 763
var yyAct = [...]int16{
155, 331, 329, 275, 336, 152, 226, 39, 192, 44,
289, 288, 156, 118, 82, 178, 55, 106, 6, 53,
77, 109, 56, 133, 108, 22, 54, 110, 107, 172,
300, 198, 57, 199, 200, 201, 60, 111, 326, 151,
325, 302, 321, 308, 266, 154, 55, 75, 128, 105,
291, 300, 160, 18, 19, 309, 54, 20, 307, 218,
105, 320, 159, 76, 113, 306, 114, 330, 61, 62,
63, 64, 65, 66, 67, 68, 69, 70, 71, 72,
73, 74, 112, 161, 180, 13, 87, 89, 265, 24,
101, 30, 104, 150, 31, 32, 115, 98, 99, 162,
109, 101, 102, 104, 88, 349, 110, 2, 3, 4,
5, 264, 196, 149, 111, 163, 160, 103, 337, 173,
167, 170, 84, 182, 348, 166, 159, 347, 103, 194,
157, 158, 83, 181, 183, 165, 184, 197, 77, 186,
185, 195, 202, 203, 204, 205, 206, 207, 208, 209,
210, 211, 212, 213, 214, 215, 216, 129, 269, 263,
217, 160, 219, 220, 55, 38, 35, 53, 77, 267,
56, 159, 270, 22, 54, 121, 297, 188, 7, 259,
57, 296, 262, 161, 319, 119, 318, 317, 271, 179,
261, 180, 161, 260, 258, 75, 295, 84, 122, 162,
187, 18, 19, 316, 268, 20, 315, 83, 162, 286,
287, 76, 314, 290, 313, 81, 61, 62, 63, 64,
65, 66, 67, 68, 69, 70, 71, 72, 73, 74,
182, 86, 292, 13, 55, 10, 312, 24, 311, 30,
181, 183, 31, 32, 54, 79, 134, 135, 136, 137,
138, 139, 140, 141, 142, 143, 144, 145, 146, 147,
148, 310, 127, 36, 126, 1, 121, 298, 299, 301,
164, 303, 49, 48, 190, 294, 119, 55, 160, 304,
305, 193, 55, 160, 117, 196, 223, 54, 159, 122,
222, 228, 54, 159, 293, 350, 50, 47, 46, 169,
132, 238, 78, 323, 324, 221, 45, 244, 43, 161,
328, 322, 168, 333, 334, 335, 130, 332, 171, 177,
339, 338, 341, 340, 176, 162, 125, 342, 343, 42,
59, 124, 344, 9, 9, 240, 241, 175, 346, 242,
131, 8, 41, 40, 123, 37, 51, 255, 351, 191,
229, 231, 233, 234, 235, 243, 245, 248, 249, 250,
251, 252, 256, 257, 345, 272, 230, 232, 236, 237,
239, 246, 247, 85, 189, 55, 253, 254, 53, 77,
224, 56, 80, 120, 22, 54, 153, 58, 227, 52,
116, 57, 0, 0, 0, 0, 0, 0, 0, 0,
0, 228, 0, 0, 0, 0, 75, 0, 0, 0,
0, 238, 18, 19, 0, 0, 20, 244, 0, 0,
0, 225, 76, 0, 0, 0, 0, 61, 62, 63,
155, 333, 331, 275, 338, 152, 226, 39, 192, 44,
290, 289, 156, 118, 82, 178, 106, 55, 109, 105,
53, 77, 133, 56, 110, 108, 22, 54, 356, 6,
172, 107, 60, 57, 345, 346, 347, 348, 111, 198,
328, 199, 200, 201, 327, 154, 303, 355, 266, 75,
354, 151, 160, 128, 259, 18, 19, 160, 55, 20,
301, 101, 159, 104, 113, 76, 114, 159, 54, 258,
61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
71, 72, 73, 74, 161, 112, 269, 13, 103, 161,
292, 24, 115, 30, 309, 265, 31, 32, 332, 267,
162, 270, 109, 223, 323, 162, 150, 222, 110, 308,
301, 263, 310, 149, 161, 163, 307, 271, 264, 173,
167, 170, 221, 322, 166, 2, 3, 4, 5, 194,
162, 157, 158, 179, 262, 180, 184, 197, 165, 186,
196, 195, 202, 203, 204, 205, 206, 207, 208, 209,
210, 211, 212, 213, 214, 215, 216, 129, 188, 121,
217, 121, 219, 220, 55, 38, 218, 53, 77, 119,
56, 119, 339, 22, 54, 182, 169, 260, 298, 117,
57, 187, 122, 297, 122, 181, 183, 160, 295, 168,
261, 180, 111, 77, 164, 55, 75, 159, 296, 357,
7, 55, 18, 19, 268, 54, 20, 294, 35, 287,
288, 54, 76, 291, 321, 320, 319, 61, 62, 63,
64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
74, 0, 0, 0, 13, 240, 241, 0, 24, 242,
30, 0, 0, 31, 32, 0, 0, 255, 105, 0,
229, 231, 233, 234, 235, 243, 245, 248, 249, 250,
251, 252, 256, 257, 0, 0, 230, 232, 236, 237,
239, 246, 247, 17, 77, 89, 253, 254, 0, 22,
0, 0, 327, 0, 0, 98, 99, 0, 0, 101,
0, 104, 88, 277, 278, 276, 283, 285, 282, 284,
279, 280, 281, 17, 35, 0, 0, 18, 19, 22,
0, 20, 0, 0, 0, 0, 103, 0, 0, 0,
0, 0, 11, 12, 14, 15, 16, 21, 23, 25,
26, 27, 28, 29, 33, 34, 0, 18, 19, 13,
0, 20, 0, 24, 0, 30, 0, 0, 31, 32,
0, 0, 11, 12, 14, 15, 16, 21, 23, 25,
26, 27, 28, 29, 33, 34, 105, 0, 0, 13,
0, 0, 0, 24, 174, 30, 0, 0, 31, 32,
0, 0, 0, 0, 0, 105, 0, 0, 0, 0,
0, 0, 87, 89, 90, 0, 91, 92, 93, 94,
95, 96, 97, 98, 99, 100, 0, 101, 102, 104,
88, 87, 89, 90, 0, 91, 92, 93, 94, 95,
96, 97, 98, 99, 100, 274, 101, 102, 104, 88,
105, 0, 273, 0, 103, 0, 277, 278, 276, 283,
285, 282, 284, 279, 280, 281, 0, 0, 0, 105,
0, 0, 0, 103, 0, 0, 87, 89, 90, 0,
91, 92, 93, 0, 95, 96, 97, 98, 99, 100,
74, 182, 293, 318, 13, 160, 317, 316, 24, 315,
30, 181, 183, 31, 32, 159, 134, 135, 136, 137,
138, 139, 140, 141, 142, 143, 144, 145, 146, 147,
148, 314, 313, 55, 105, 84, 84, 299, 300, 302,
86, 304, 177, 54, 190, 83, 83, 176, 160, 305,
306, 193, 125, 185, 81, 196, 10, 124, 159, 312,
175, 311, 89, 50, 8, 36, 79, 228, 37, 78,
123, 1, 98, 99, 325, 326, 101, 238, 104, 88,
161, 330, 49, 244, 335, 336, 337, 324, 334, 48,
47, 341, 340, 343, 342, 127, 162, 126, 59, 349,
350, 9, 9, 103, 351, 46, 132, 45, 43, 130,
353, 171, 240, 241, 42, 131, 242, 41, 40, 51,
191, 352, 272, 85, 255, 358, 189, 229, 231, 233,
234, 235, 243, 245, 248, 249, 250, 251, 252, 256,
257, 224, 80, 230, 232, 236, 237, 239, 246, 247,
344, 120, 55, 253, 254, 53, 77, 153, 56, 274,
58, 22, 54, 227, 52, 116, 273, 0, 57, 0,
277, 278, 276, 283, 285, 282, 284, 279, 280, 281,
286, 0, 0, 0, 75, 0, 0, 0, 0, 0,
18, 19, 0, 0, 20, 0, 0, 0, 0, 0,
76, 0, 0, 0, 0, 61, 62, 63, 64, 65,
66, 67, 68, 69, 70, 71, 72, 73, 74, 228,
0, 0, 13, 0, 0, 0, 24, 0, 30, 238,
329, 31, 32, 0, 0, 244, 0, 0, 0, 225,
0, 277, 278, 276, 283, 285, 282, 284, 279, 280,
281, 286, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 240, 241, 0, 0, 242, 0,
0, 0, 17, 77, 0, 105, 255, 0, 22, 229,
231, 233, 234, 235, 243, 245, 248, 249, 250, 251,
252, 256, 257, 0, 0, 230, 232, 236, 237, 239,
246, 247, 87, 89, 0, 253, 254, 18, 19, 0,
0, 20, 0, 98, 99, 17, 35, 101, 102, 104,
88, 22, 11, 12, 14, 15, 16, 21, 23, 25,
26, 27, 28, 29, 33, 34, 0, 0, 0, 13,
0, 0, 0, 24, 103, 30, 0, 0, 31, 32,
18, 19, 0, 0, 20, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 11, 12, 14, 15, 16,
21, 23, 25, 26, 27, 28, 29, 33, 34, 105,
0, 0, 13, 0, 0, 0, 24, 174, 30, 0,
0, 31, 32, 0, 0, 0, 0, 0, 105, 0,
0, 0, 0, 0, 0, 0, 87, 89, 90, 0,
91, 92, 93, 94, 95, 96, 97, 98, 99, 100,
0, 101, 102, 104, 88, 87, 89, 90, 0, 91,
92, 0, 0, 95, 96, 0, 98, 99, 100, 0,
101, 102, 104, 88, 0, 0, 0, 0, 103, 0,
92, 93, 94, 95, 96, 97, 98, 99, 100, 0,
101, 102, 104, 88, 105, 0, 0, 0, 103, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 103,
0, 0, 0, 105, 0, 0, 0, 103, 0, 0,
0, 87, 89, 90, 0, 91, 92, 93, 0, 95,
96, 97, 98, 99, 100, 0, 101, 102, 104, 88,
87, 89, 90, 0, 91, 92, 0, 0, 95, 96,
0, 98, 99, 100, 0, 101, 102, 104, 88, 0,
0, 0, 0, 103, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 103,
}
var yyPact = [...]int16{
16, 168, 501, 501, 155, 471, -1000, -1000, -1000, 153,
27, 190, 533, 533, 155, 490, -1000, -1000, -1000, 195,
-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-1000, -1000, -1000, -1000, -1000, 195, -1000, 229, -1000, 581,
-1000, -1000, -1000, -1000, -1000, 264, -1000, 268, -1000, 614,
-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-1000, -1000, 22, 99, -1000, -1000, 366, -1000, 366, 125,
-1000, -1000, 23, 177, -1000, -1000, 373, -1000, 373, 180,
-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-1000, -1000, -1000, -1000, -1000, -1000, -1000, 264, -1000, -1000,
324, -1000, -1000, 260, -1000, 24, -1000, -54, -54, -54,
-54, -54, -54, -54, -54, -54, -54, -54, -54, -54,
-54, -54, -54, 37, 43, 268, 99, -57, -1000, 297,
297, 7, -1000, 562, 35, -1000, 317, -1000, -1000, 187,
80, -1000, -1000, -1000, 120, -1000, 175, -1000, 269, 366,
-1000, -50, -45, -1000, 366, 366, 366, 366, 366, 366,
366, 366, 366, 366, 366, 366, 366, 366, 366, -1000,
225, -1000, -1000, 44, -1000, -1000, -1000, -1000, -1000, -1000,
-1000, 107, 107, 284, -1000, -1000, -1000, -1000, 399, -1000,
-1000, 172, -1000, 581, -1000, -1000, 173, -1000, 157, -1000,
-1000, -1000, -1000, -1000, 86, -1000, -1000, -1000, -1000, -1000,
18, 143, 132, -1000, -1000, -1000, 618, 444, 297, 297,
297, 297, 35, 35, 46, 46, 46, 645, 626, 46,
46, 645, 35, 35, 46, 35, 444, -1000, 28, -1000,
-1000, -1000, 273, -1000, 174, -1000, -1000, -1000, -1000, -1000,
-1000, -1000, -1000, -1000, -1000, -1000, -1000, 159, -1000, -1000,
280, -1000, -1000, 323, -1000, 29, -1000, -56, -56, -56,
-56, -56, -56, -56, -56, -56, -56, -56, -56, -56,
-56, -56, -56, 49, 43, 192, 177, -61, -1000, 174,
174, 8, -1000, 595, 5, -1000, 270, -1000, -1000, 131,
187, -1000, -1000, -1000, 263, -1000, 156, -1000, 269, 373,
-1000, -43, -38, -1000, 373, 373, 373, 373, 373, 373,
373, 373, 373, 373, 373, 373, 373, 373, 373, -1000,
254, -1000, -1000, 151, -1000, -1000, -1000, -1000, -1000, -1000,
-1000, 226, 226, 101, -1000, -1000, -1000, -1000, 447, -1000,
-1000, 47, -1000, 614, -1000, -1000, 157, -1000, 109, -1000,
-1000, -1000, -1000, -1000, 93, -1000, -1000, -1000, -1000, -1000,
22, 73, 60, -1000, -1000, -1000, 372, 250, 174, 174,
174, 174, 5, 5, 491, 491, 491, 679, 660, 491,
491, 679, 5, 5, 491, 5, 250, -1000, 68, -1000,
-1000, -1000, 186, -1000, 176, -1000, -1000, -1000, -1000, -1000,
-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 366,
-1000, -1000, -1000, -1000, -1000, -1000, 32, 32, 15, 32,
96, 96, 41, 38, -1000, -1000, 255, 232, 230, 208,
206, 200, 197, 181, 180, 178, -1000, -1000, -1000, -1000,
-1000, -1000, 40, -1000, -1000, -1000, 289, -1000, 581, -1000,
-1000, -1000, 32, -1000, 14, 12, 475, -1000, -1000, -1000,
11, 152, 107, 107, 107, 104, 104, 11, 104, 11,
-1000, -1000, -1000, -1000, -1000, 32, 32, -1000, -1000, -1000,
32, -1000, -1000, -1000, -1000, -1000, -1000, 107, -1000, -1000,
-1000, -1000, -1000, -1000, -1000, 103, -1000, 274, -1000, -1000,
-1000, -1000,
-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 373,
-1000, -1000, -1000, -1000, -1000, -1000, 91, 91, 20, 91,
124, 124, 92, 95, -1000, -1000, 285, 283, 256, 255,
233, 231, 230, 227, 210, 209, 208, -1000, -1000, -1000,
-1000, -1000, -1000, 102, -1000, -1000, -1000, 295, -1000, 614,
-1000, -1000, -1000, 91, -1000, 18, 14, 443, -1000, -1000,
-1000, 41, 48, 226, 226, 226, 158, 158, 41, 158,
41, -58, -1000, -1000, -1000, -1000, -1000, 91, 91, -1000,
-1000, -1000, 91, -1000, -1000, -1000, -1000, -1000, -1000, 226,
-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-1000, -1000, 26, -1000, 178, -1000, -1000, -1000, -1000,
}
var yyPgo = [...]int16{
0, 390, 13, 389, 6, 15, 388, 330, 387, 386,
383, 235, 341, 382, 14, 380, 10, 11, 374, 373,
8, 365, 3, 4, 364, 2, 1, 0, 349, 12,
5, 346, 343, 17, 157, 342, 340, 7, 329, 318,
28, 316, 36, 308, 9, 306, 300, 298, 297, 273,
272, 296, 265, 263,
0, 395, 13, 394, 6, 15, 393, 328, 390, 387,
381, 380, 286, 294, 372, 14, 371, 10, 11, 356,
353, 8, 352, 3, 4, 351, 2, 1, 0, 350,
12, 5, 349, 348, 16, 157, 347, 345, 7, 344,
341, 31, 339, 32, 338, 9, 337, 336, 335, 320,
319, 312, 293, 301, 295,
}
var yyR1 = [...]int8{
0, 52, 52, 52, 52, 52, 52, 52, 37, 37,
37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
32, 32, 32, 32, 33, 33, 35, 35, 35, 35,
35, 35, 35, 35, 35, 35, 35, 35, 35, 35,
35, 35, 34, 36, 36, 46, 46, 41, 41, 41,
41, 16, 16, 16, 16, 15, 15, 15, 4, 4,
38, 40, 40, 39, 39, 39, 47, 45, 45, 45,
31, 31, 31, 9, 9, 43, 49, 49, 49, 49,
49, 49, 50, 51, 51, 51, 42, 42, 42, 1,
1, 1, 2, 2, 2, 2, 2, 2, 2, 12,
12, 7, 7, 7, 7, 7, 7, 7, 7, 7,
0, 53, 53, 53, 53, 53, 53, 53, 38, 38,
38, 38, 38, 38, 38, 38, 38, 38, 38, 38,
33, 33, 33, 33, 34, 34, 36, 36, 36, 36,
36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
36, 36, 35, 37, 37, 47, 47, 42, 42, 42,
42, 17, 17, 17, 17, 16, 16, 16, 4, 4,
39, 41, 41, 40, 40, 40, 48, 46, 46, 46,
32, 32, 32, 9, 9, 44, 50, 50, 50, 50,
50, 50, 51, 52, 52, 52, 43, 43, 43, 1,
1, 1, 2, 2, 2, 2, 2, 2, 2, 13,
13, 7, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 11, 11, 11, 11, 13,
13, 13, 14, 14, 14, 14, 53, 19, 19, 19,
19, 18, 18, 18, 18, 18, 18, 18, 18, 18,
28, 28, 28, 20, 20, 20, 20, 21, 21, 21,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 24, 24, 24, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 6,
7, 7, 7, 7, 7, 12, 12, 12, 12, 14,
14, 14, 15, 15, 15, 15, 54, 20, 20, 20,
20, 19, 19, 19, 19, 19, 19, 19, 19, 19,
29, 29, 29, 21, 21, 21, 21, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 24, 24, 25, 25, 25, 11, 11, 11, 11,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 8, 8,
5, 5, 5, 5, 44, 44, 27, 27, 29, 29,
30, 30, 26, 25, 25, 48, 10, 17, 17,
6, 6, 6, 8, 8, 5, 5, 5, 5, 45,
45, 28, 28, 30, 30, 31, 31, 27, 26, 26,
49, 10, 18, 18,
}
var yyR2 = [...]int8{
@ -562,52 +581,53 @@ var yyR2 = [...]int8{
1, 1, 3, 1, 3, 4, 1, 3, 5, 5,
1, 1, 1, 4, 3, 3, 2, 3, 1, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 3, 3, 1, 2, 1, 1, 1, 1, 1,
3, 4, 3, 3, 1, 2, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 2, 2,
1, 1, 1, 2, 1, 1, 1, 0, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 2, 2, 1, 1, 1, 2, 1,
1, 1, 0, 1,
}
var yyChk = [...]int16{
-1000, -52, 91, 92, 93, 94, 2, 10, -12, -7,
-11, 61, 62, 78, 63, 64, 65, 12, 46, 47,
50, 66, 18, 67, 82, 68, 69, 70, 71, 72,
84, 87, 88, 73, 74, 13, -53, -12, 10, -37,
-32, -35, -38, -43, -44, -45, -47, -48, -49, -50,
-51, -31, -3, 12, 19, 9, 15, 25, -8, -7,
-42, 61, 62, 63, 64, 65, 66, 67, 68, 69,
70, 71, 72, 73, 74, 40, 56, 13, -51, -11,
-13, 20, -14, 12, 2, -19, 2, 40, 58, 41,
42, 44, 45, 46, 47, 48, 49, 50, 51, 52,
53, 55, 56, 82, 57, 14, -33, -40, 2, 78,
84, 15, -40, -37, -37, -42, -1, 20, -2, 12,
-10, 2, 25, 20, 7, 2, 4, 2, 24, -34,
-41, -36, -46, 77, -34, -34, -34, -34, -34, -34,
-34, -34, -34, -34, -34, -34, -34, -34, -34, -44,
56, 2, -30, -9, 2, -27, -29, 87, 88, 19,
9, 40, 56, -44, 2, -40, -33, -16, 15, 2,
-16, -39, 22, -37, 22, 20, 7, 2, -5, 2,
4, 53, 43, 54, -5, 20, -14, 25, 2, -18,
5, -28, -20, 12, -27, -29, 16, -37, 81, 83,
79, 80, -37, -37, -37, -37, -37, -37, -37, -37,
-37, -37, -37, -37, -37, -37, -37, -44, 15, -27,
-27, 21, 6, 2, -15, 22, -4, -6, 2, 61,
77, 62, 78, 63, 64, 65, 79, 80, 12, 81,
46, 47, 50, 66, 18, 67, 82, 83, 68, 69,
70, 71, 72, 87, 88, 58, 73, 74, 22, 7,
20, -2, 25, 2, 25, 2, 26, 26, -29, 26,
40, 56, -21, 24, 17, -22, 30, 28, 29, 35,
36, 37, 33, 31, 34, 32, -16, -16, -17, -16,
-17, 22, -44, 21, 2, 22, 7, 2, -37, -26,
19, -26, 26, -26, -20, -20, 24, 17, 2, 17,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
21, 2, 22, -4, -26, 26, 26, 17, -22, -25,
56, -26, -30, -27, -27, -27, -23, 14, -23, -25,
-23, -25, -26, -26, -26, -24, -27, 24, 21, 2,
21, -27,
-1000, -53, 98, 99, 100, 101, 2, 10, -13, -7,
-12, 62, 63, 79, 64, 65, 66, 12, 47, 48,
51, 67, 18, 68, 83, 69, 70, 71, 72, 73,
85, 88, 89, 74, 75, 13, -54, -13, 10, -38,
-33, -36, -39, -44, -45, -46, -48, -49, -50, -51,
-52, -32, -3, 12, 19, 9, 15, 25, -8, -7,
-43, 62, 63, 64, 65, 66, 67, 68, 69, 70,
71, 72, 73, 74, 75, 41, 57, 13, -52, -12,
-14, 20, -15, 12, 2, -20, 2, 41, 59, 42,
43, 45, 46, 47, 48, 49, 50, 51, 52, 53,
54, 56, 57, 83, 58, 14, -34, -41, 2, 79,
85, 15, -41, -38, -38, -43, -1, 20, -2, 12,
-10, 2, 25, 20, 7, 2, 4, 2, 24, -35,
-42, -37, -47, 78, -35, -35, -35, -35, -35, -35,
-35, -35, -35, -35, -35, -35, -35, -35, -35, -45,
57, 2, -31, -9, 2, -28, -30, 88, 89, 19,
9, 41, 57, -45, 2, -41, -34, -17, 15, 2,
-17, -40, 22, -38, 22, 20, 7, 2, -5, 2,
4, 54, 44, 55, -5, 20, -15, 25, 2, -19,
5, -29, -21, 12, -28, -30, 16, -38, 82, 84,
80, 81, -38, -38, -38, -38, -38, -38, -38, -38,
-38, -38, -38, -38, -38, -38, -38, -45, 15, -28,
-28, 21, 6, 2, -16, 22, -4, -6, 2, 62,
78, 63, 79, 64, 65, 66, 80, 81, 12, 82,
47, 48, 51, 67, 18, 68, 83, 84, 69, 70,
71, 72, 73, 88, 89, 59, 74, 75, 22, 7,
20, -2, 25, 2, 25, 2, 26, 26, -30, 26,
41, 57, -22, 24, 17, -23, 30, 28, 29, 35,
36, 37, 33, 31, 34, 32, 38, -17, -17, -18,
-17, -18, 22, -45, 21, 2, 22, 7, 2, -38,
-27, 19, -27, 26, -27, -21, -21, 24, 17, 2,
17, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 21, 2, 22, -4, -27, 26, 26, 17,
-23, -26, 57, -27, -31, -28, -28, -28, -24, 14,
-24, -26, -24, -26, -11, 92, 93, 94, 95, -27,
-27, -27, -25, -28, 24, 21, 2, 21, -28,
}
var yyDef = [...]int16{
@ -616,37 +636,37 @@ var yyDef = [...]int16{
110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
120, 121, 122, 123, 124, 0, 2, -2, 3, 4,
8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
18, 19, 0, 107, 224, 225, 0, 235, 0, 84,
18, 19, 0, 107, 229, 230, 0, 240, 0, 84,
85, -2, -2, -2, -2, -2, -2, -2, -2, -2,
-2, -2, -2, -2, -2, 218, 219, 0, 5, 99,
-2, -2, -2, -2, -2, 223, 224, 0, 5, 99,
0, 127, 130, 0, 135, 136, 140, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 0, 0, 0, 0, 22, 23, 0,
0, 0, 60, 0, 82, 83, 0, 88, 90, 0,
94, 98, 236, 125, 0, 131, 0, 134, 139, 0,
94, 98, 241, 125, 0, 131, 0, 134, 139, 0,
42, 47, 48, 44, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 67,
0, 69, 70, 0, 72, 230, 231, 73, 74, 226,
227, 0, 0, 0, 81, 20, 21, 24, 0, 54,
0, 69, 70, 0, 72, 235, 236, 73, 74, 231,
232, 0, 0, 0, 81, 20, 21, 24, 0, 54,
25, 0, 62, 64, 66, 86, 0, 91, 0, 97,
220, 221, 222, 223, 0, 126, 129, 132, 133, 138,
225, 226, 227, 228, 0, 126, 129, 132, 133, 138,
141, 143, 146, 150, 151, 152, 0, 26, 0, 0,
-2, -2, 27, 28, 29, 30, 31, 32, 33, 34,
35, 36, 37, 38, 39, 40, 41, 68, 0, 228,
229, 75, 0, 80, 0, 53, 56, 58, 59, 189,
190, 191, 192, 193, 194, 195, 196, 197, 198, 199,
200, 201, 202, 203, 204, 205, 206, 207, 208, 209,
210, 211, 212, 213, 214, 215, 216, 217, 61, 65,
35, 36, 37, 38, 39, 40, 41, 68, 0, 233,
234, 75, 0, 80, 0, 53, 56, 58, 59, 194,
195, 196, 197, 198, 199, 200, 201, 202, 203, 204,
205, 206, 207, 208, 209, 210, 211, 212, 213, 214,
215, 216, 217, 218, 219, 220, 221, 222, 61, 65,
87, 89, 92, 96, 93, 95, 0, 0, 0, 0,
0, 0, 0, 0, 156, 158, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 45, 46, 49, 238,
50, 71, 0, 77, 79, 51, 0, 57, 63, 142,
232, 144, 0, 147, 0, 0, 0, 154, 159, 155,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
76, 78, 52, 55, 145, 0, 0, 153, 157, 160,
0, 234, 161, 162, 163, 164, 165, 0, 166, 167,
168, 169, 148, 149, 233, 0, 173, 0, 171, 174,
170, 172,
0, 0, 0, 0, 0, 0, 0, 45, 46, 49,
243, 50, 71, 0, 77, 79, 51, 0, 57, 63,
142, 237, 144, 0, 147, 0, 0, 0, 154, 159,
155, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 76, 78, 52, 55, 145, 0, 0, 153,
157, 160, 0, 239, 161, 162, 163, 164, 165, 0,
166, 167, 168, 169, 170, 176, 177, 178, 179, 148,
149, 238, 0, 174, 0, 172, 175, 171, 173,
}
var yyTok1 = [...]int8{
@ -663,7 +683,8 @@ var yyTok2 = [...]int8{
62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
72, 73, 74, 75, 76, 77, 78, 79, 80, 81,
82, 83, 84, 85, 86, 87, 88, 89, 90, 91,
92, 93, 94, 95,
92, 93, 94, 95, 96, 97, 98, 99, 100, 101,
102,
}
var yyTok3 = [...]int8{
@ -1800,26 +1821,32 @@ yydefault:
yyVAL.descriptors["n_offset"] = yyDollar[3].int
}
case 170:
yyDollar = yyS[yypt-4 : yypt+1]
yyDollar = yyS[yypt-3 : yypt+1]
{
yyVAL.bucket_set = yyDollar[2].bucket_set
yyVAL.descriptors = yylex.(*parser).newMap()
yyVAL.descriptors["counter_reset_hint"] = yyDollar[3].item
}
case 171:
yyDollar = yyS[yypt-3 : yypt+1]
yyDollar = yyS[yypt-4 : yypt+1]
{
yyVAL.bucket_set = yyDollar[2].bucket_set
}
case 172:
yyDollar = yyS[yypt-3 : yypt+1]
{
yyVAL.bucket_set = append(yyDollar[1].bucket_set, yyDollar[3].float)
yyVAL.bucket_set = yyDollar[2].bucket_set
}
case 173:
yyDollar = yyS[yypt-3 : yypt+1]
{
yyVAL.bucket_set = append(yyDollar[1].bucket_set, yyDollar[3].float)
}
case 174:
yyDollar = yyS[yypt-1 : yypt+1]
{
yyVAL.bucket_set = []float64{yyDollar[1].float}
}
case 224:
case 229:
yyDollar = yyS[yypt-1 : yypt+1]
{
yyVAL.node = &NumberLiteral{
@ -1827,7 +1854,7 @@ yydefault:
PosRange: yyDollar[1].item.PositionRange(),
}
}
case 225:
case 230:
yyDollar = yyS[yypt-1 : yypt+1]
{
var err error
@ -1841,12 +1868,12 @@ yydefault:
PosRange: yyDollar[1].item.PositionRange(),
}
}
case 226:
case 231:
yyDollar = yyS[yypt-1 : yypt+1]
{
yyVAL.float = yylex.(*parser).number(yyDollar[1].item.Val)
}
case 227:
case 232:
yyDollar = yyS[yypt-1 : yypt+1]
{
var err error
@ -1857,17 +1884,17 @@ yydefault:
}
yyVAL.float = dur.Seconds()
}
case 228:
case 233:
yyDollar = yyS[yypt-2 : yypt+1]
{
yyVAL.float = yyDollar[2].float
}
case 229:
case 234:
yyDollar = yyS[yypt-2 : yypt+1]
{
yyVAL.float = -yyDollar[2].float
}
case 232:
case 237:
yyDollar = yyS[yypt-1 : yypt+1]
{
var err error
@ -1876,17 +1903,17 @@ yydefault:
yylex.(*parser).addParseErrf(yyDollar[1].item.PositionRange(), "invalid repetition in series values: %s", err)
}
}
case 233:
case 238:
yyDollar = yyS[yypt-2 : yypt+1]
{
yyVAL.int = -int64(yyDollar[2].uint)
}
case 234:
case 239:
yyDollar = yyS[yypt-1 : yypt+1]
{
yyVAL.int = int64(yyDollar[1].uint)
}
case 235:
case 240:
yyDollar = yyS[yypt-1 : yypt+1]
{
yyVAL.node = &StringLiteral{
@ -1894,7 +1921,7 @@ yydefault:
PosRange: yyDollar[1].item.PositionRange(),
}
}
case 236:
case 241:
yyDollar = yyS[yypt-1 : yypt+1]
{
yyVAL.item = Item{
@ -1903,7 +1930,7 @@ yydefault:
Val: yylex.(*parser).unquoteString(yyDollar[1].item.Val),
}
}
case 237:
case 242:
yyDollar = yyS[yypt-0 : yypt+1]
{
yyVAL.strings = nil

13
promql/parser/lex.go
View file

@ -147,6 +147,14 @@ var histogramDesc = map[string]ItemType{
"z_bucket": ZERO_BUCKET_DESC,
"z_bucket_w": ZERO_BUCKET_WIDTH_DESC,
"custom_values": CUSTOM_VALUES_DESC,
"counter_reset_hint": COUNTER_RESET_HINT_DESC,
}
var counterResetHints = map[string]ItemType{
"unknown": UNKNOWN_COUNTER_RESET,
"reset": COUNTER_RESET,
"not_reset": NOT_COUNTER_RESET,
"gauge": GAUGE_TYPE,
}
// ItemTypeStr is the default string representations for common Items. It does not
@ -585,6 +593,11 @@ Loop:
return lexHistogram
}
}
if desc, ok := counterResetHints[strings.ToLower(word)]; ok {
l.emit(desc)
return lexHistogram
}
l.errorf("bad histogram descriptor found: %q", word)
break Loop
}

22
promql/parser/parse.go
View file

@ -580,6 +580,28 @@ func (p *parser) buildHistogramFromMap(desc *map[string]interface{}) *histogram.
}
}
val, ok = (*desc)["counter_reset_hint"]
if ok {
resetHint, ok := val.(Item)
if ok {
switch resetHint.Typ {
case UNKNOWN_COUNTER_RESET:
output.CounterResetHint = histogram.UnknownCounterReset
case COUNTER_RESET:
output.CounterResetHint = histogram.CounterReset
case NOT_COUNTER_RESET:
output.CounterResetHint = histogram.NotCounterReset
case GAUGE_TYPE:
output.CounterResetHint = histogram.GaugeType
default:
p.addParseErrf(p.yyParser.lval.item.PositionRange(), "error parsing counter_reset_hint: unknown value %v", resetHint.Typ)
}
} else {
p.addParseErrf(p.yyParser.lval.item.PositionRange(), "error parsing counter_reset_hint: %v", val)
}
}
buckets, spans := p.buildHistogramBucketsAndSpans(desc, "buckets", "offset")
output.PositiveBuckets = buckets
output.PositiveSpans = spans

39
promql/parser/parse_test.go
View file

@ -4038,7 +4038,7 @@ func TestParseHistogramSeries(t *testing.T) {
},
{
name: "all properties used",
input: `{} {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5}}`,
input: `{} {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5 counter_reset_hint:gauge}}`,
expected: []histogram.FloatHistogram{{
Schema: 1,
Sum: -0.3,
@ -4049,11 +4049,12 @@ func TestParseHistogramSeries(t *testing.T) {
PositiveSpans: []histogram.Span{{Offset: -3, Length: 3}},
NegativeBuckets: []float64{4.1, 5},
NegativeSpans: []histogram.Span{{Offset: -5, Length: 2}},
CounterResetHint: histogram.GaugeType,
}},
},
{
name: "all properties used - with spaces",
input: `{} {{schema:1 sum:0.3 count:3 z_bucket:7 z_bucket_w:5 buckets:[5 10 7 ] offset:-3 n_buckets:[4 5] n_offset:5 }}`,
input: `{} {{schema:1 sum:0.3 count:3 z_bucket:7 z_bucket_w:5 buckets:[5 10 7 ] offset:-3 n_buckets:[4 5] n_offset:5 counter_reset_hint:gauge }}`,
expected: []histogram.FloatHistogram{{
Schema: 1,
Sum: 0.3,
@ -4064,6 +4065,7 @@ func TestParseHistogramSeries(t *testing.T) {
PositiveSpans: []histogram.Span{{Offset: -3, Length: 3}},
NegativeBuckets: []float64{4, 5},
NegativeSpans: []histogram.Span{{Offset: 5, Length: 2}},
CounterResetHint: histogram.GaugeType,
}},
},
{
@ -4250,6 +4252,39 @@ func TestParseHistogramSeries(t *testing.T) {
input: `{} {{ schema:1}}`,
expectedError: `1:7: parse error: unexpected "<Item 57372>" "schema" in series values`,
},
{
name: "invalid counter reset hint value",
input: `{} {{counter_reset_hint:foo}}`,
expectedError: `1:25: parse error: bad histogram descriptor found: "foo"`,
},
{
name: "'unknown' counter reset hint value",
input: `{} {{counter_reset_hint:unknown}}`,
expected: []histogram.FloatHistogram{{
CounterResetHint: histogram.UnknownCounterReset,
}},
},
{
name: "'reset' counter reset hint value",
input: `{} {{counter_reset_hint:reset}}`,
expected: []histogram.FloatHistogram{{
CounterResetHint: histogram.CounterReset,
}},
},
{
name: "'not_reset' counter reset hint value",
input: `{} {{counter_reset_hint:not_reset}}`,
expected: []histogram.FloatHistogram{{
CounterResetHint: histogram.NotCounterReset,
}},
},
{
name: "'gauge' counter reset hint value",
input: `{} {{counter_reset_hint:gauge}}`,
expected: []histogram.FloatHistogram{{
CounterResetHint: histogram.GaugeType,
}},
},
} {
t.Run(test.name, func(t *testing.T) {
_, vals, err := ParseSeriesDesc(test.input)

10
rules/group.go
View file

@ -621,14 +621,12 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
}
}
// If the rule has no dependencies, it can run concurrently because no other rules in this group depend on its output.
// Try run concurrently if there are slots available.
if ctrl := g.concurrencyController; isRuleEligibleForConcurrentExecution(rule) && ctrl.Allow() {
if ctrl := g.concurrencyController; ctrl.Allow(ctx, g, rule) {
wg.Add(1)
go eval(i, rule, func() {
wg.Done()
ctrl.Done()
ctrl.Done(ctx)
})
} else {
eval(i, rule, nil)
@ -1094,7 +1092,3 @@ func buildDependencyMap(rules []Rule) dependencyMap {
return dependencies
}
func isRuleEligibleForConcurrentExecution(rule Rule) bool {
return rule.NoDependentRules() && rule.NoDependencyRules()
}

54
rules/manager.go
View file

@ -457,67 +457,47 @@ func (c ruleDependencyController) AnalyseRules(rules []Rule) {
// Its purpose is to bound the amount of concurrency in rule evaluations to avoid overwhelming the Prometheus
// server with additional query load. Concurrency is controlled globally, not on a per-group basis.
type RuleConcurrencyController interface {
// Allow determines whether any concurrent evaluation slots are available.
// If Allow() returns true, then Done() must be called to release the acquired slot.
Allow() bool
// Allow determines if the given rule is allowed to be evaluated concurrently.
// If Allow() returns true, then Done() must be called to release the acquired slot and corresponding cleanup is done.
// It is important that both *Group and Rule are not retained and only be used for the duration of the call.
Allow(ctx context.Context, group *Group, rule Rule) bool
// Done releases a concurrent evaluation slot.
Done()
Done(ctx context.Context)
}
// concurrentRuleEvalController holds a weighted semaphore which controls the concurrent evaluation of rules.
type concurrentRuleEvalController struct {
sema *semaphore.Weighted
depMapsMu sync.Mutex
depMaps map[*Group]dependencyMap
}
func newRuleConcurrencyController(maxConcurrency int64) RuleConcurrencyController {
return &concurrentRuleEvalController{
sema: semaphore.NewWeighted(maxConcurrency),
depMaps: map[*Group]dependencyMap{},
}
}
func (c *concurrentRuleEvalController) RuleEligible(g *Group, r Rule) bool {
c.depMapsMu.Lock()
defer c.depMapsMu.Unlock()
depMap, found := c.depMaps[g]
if !found {
depMap = buildDependencyMap(g.rules)
c.depMaps[g] = depMap
}
return depMap.isIndependent(r)
}
func (c *concurrentRuleEvalController) Allow() bool {
func (c *concurrentRuleEvalController) Allow(_ context.Context, _ *Group, rule Rule) bool {
// To allow a rule to be executed concurrently, we need 3 conditions:
// 1. The rule must not have any rules that depend on it.
// 2. The rule itself must not depend on any other rules.
// 3. If 1 & 2 are true, then and only then we should try to acquire the concurrency slot.
if rule.NoDependentRules() && rule.NoDependencyRules() {
return c.sema.TryAcquire(1)
}
return false
}
func (c *concurrentRuleEvalController) Done() {
func (c *concurrentRuleEvalController) Done(_ context.Context) {
c.sema.Release(1)
}
func (c *concurrentRuleEvalController) Invalidate() {
c.depMapsMu.Lock()
defer c.depMapsMu.Unlock()
// Clear out the memoized dependency maps because some or all groups may have been updated.
c.depMaps = map[*Group]dependencyMap{}
}
// sequentialRuleEvalController is a RuleConcurrencyController that runs every rule sequentially.
type sequentialRuleEvalController struct{}
func (c sequentialRuleEvalController) RuleEligible(_ *Group, _ Rule) bool {
func (c sequentialRuleEvalController) Allow(_ context.Context, _ *Group, _ Rule) bool {
return false
}
func (c sequentialRuleEvalController) Allow() bool {
return false
}
func (c sequentialRuleEvalController) Done() {}
func (c sequentialRuleEvalController) Invalidate() {}
func (c sequentialRuleEvalController) Done(_ context.Context) {}

3
rules/manager_test.go
View file

@ -32,7 +32,6 @@ import (
"github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
"go.uber.org/goleak"
"gopkg.in/yaml.v2"
"github.com/prometheus/prometheus/model/labels"
@ -50,7 +49,7 @@ import (
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
prom_testutil.TolerantVerifyLeak(m)
}
func TestAlertingRule(t *testing.T) {

44
storage/merge.go
View file

@ -45,26 +45,25 @@ type mergeGenericQuerier struct {
//
// In case of overlaps between the data given by primaries' and secondaries' Selects, merge function will be used.
func NewMergeQuerier(primaries, secondaries []Querier, mergeFn VerticalSeriesMergeFunc) Querier {
primaries = filterQueriers(primaries)
secondaries = filterQueriers(secondaries)
switch {
case len(primaries)+len(secondaries) == 0:
case len(primaries) == 0 && len(secondaries) == 0:
return noopQuerier{}
case len(primaries) == 1 && len(secondaries) == 0:
return primaries[0]
case len(primaries) == 0 && len(secondaries) == 1:
return secondaries[0]
return &querierAdapter{newSecondaryQuerierFrom(secondaries[0])}
}
queriers := make([]genericQuerier, 0, len(primaries)+len(secondaries))
for _, q := range primaries {
if _, ok := q.(noopQuerier); !ok && q != nil {
queriers = append(queriers, newGenericQuerierFrom(q))
}
}
for _, q := range secondaries {
if _, ok := q.(noopQuerier); !ok && q != nil {
queriers = append(queriers, newSecondaryQuerierFrom(q))
}
}
concurrentSelect := false
if len(secondaries) > 0 {
@ -77,31 +76,40 @@ func NewMergeQuerier(primaries, secondaries []Querier, mergeFn VerticalSeriesMer
}}
}
func filterQueriers(qs []Querier) []Querier {
ret := make([]Querier, 0, len(qs))
for _, q := range qs {
if _, ok := q.(noopQuerier); !ok && q != nil {
ret = append(ret, q)
}
}
return ret
}
// NewMergeChunkQuerier returns a new Chunk Querier that merges results of given primary and secondary chunk queriers.
// See NewFanout commentary to learn more about primary vs secondary differences.
//
// In case of overlaps between the data given by primaries' and secondaries' Selects, merge function will be used.
// TODO(bwplotka): Currently merge will compact overlapping chunks with bigger chunk, without limit. Split it: https://github.com/prometheus/tsdb/issues/670
func NewMergeChunkQuerier(primaries, secondaries []ChunkQuerier, mergeFn VerticalChunkSeriesMergeFunc) ChunkQuerier {
primaries = filterChunkQueriers(primaries)
secondaries = filterChunkQueriers(secondaries)
switch {
case len(primaries) == 0 && len(secondaries) == 0:
return noopChunkQuerier{}
case len(primaries) == 1 && len(secondaries) == 0:
return primaries[0]
case len(primaries) == 0 && len(secondaries) == 1:
return secondaries[0]
return &chunkQuerierAdapter{newSecondaryQuerierFromChunk(secondaries[0])}
}
queriers := make([]genericQuerier, 0, len(primaries)+len(secondaries))
for _, q := range primaries {
if _, ok := q.(noopChunkQuerier); !ok && q != nil {
queriers = append(queriers, newGenericQuerierFromChunk(q))
}
}
for _, querier := range secondaries {
if _, ok := querier.(noopChunkQuerier); !ok && querier != nil {
queriers = append(queriers, newSecondaryQuerierFromChunk(querier))
}
for _, q := range secondaries {
queriers = append(queriers, newSecondaryQuerierFromChunk(q))
}
concurrentSelect := false
@ -115,6 +123,16 @@ func NewMergeChunkQuerier(primaries, secondaries []ChunkQuerier, mergeFn Vertica
}}
}
func filterChunkQueriers(qs []ChunkQuerier) []ChunkQuerier {
ret := make([]ChunkQuerier, 0, len(qs))
for _, q := range qs {
if _, ok := q.(noopChunkQuerier); !ok && q != nil {
ret = append(ret, q)
}
}
return ret
}
// Select returns a set of series that matches the given label matchers.
func (q *mergeGenericQuerier) Select(ctx context.Context, sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) genericSeriesSet {
seriesSets := make([]genericSeriesSet, 0, len(q.queriers))

314
storage/merge_test.go
View file

@ -912,9 +912,23 @@ func TestConcatenatingChunkIterator(t *testing.T) {
}
type mockQuerier struct {
LabelQuerier
mtx sync.Mutex
toReturn []Series
toReturn []Series // Response for Select.
closed bool
labelNamesCalls int
labelNamesRequested []labelNameRequest
sortedSeriesRequested []bool
resp []string // Response for LabelNames and LabelValues; turned into Select response if toReturn is not supplied.
warnings annotations.Annotations
err error
}
type labelNameRequest struct {
name string
matchers []*labels.Matcher
}
type seriesByLabel []Series
@ -924,13 +938,47 @@ func (a seriesByLabel) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a seriesByLabel) Less(i, j int) bool { return labels.Compare(a[i].Labels(), a[j].Labels()) < 0 }
func (m *mockQuerier) Select(_ context.Context, sortSeries bool, _ *SelectHints, _ ...*labels.Matcher) SeriesSet {
cpy := make([]Series, len(m.toReturn))
copy(cpy, m.toReturn)
m.mtx.Lock()
defer m.mtx.Unlock()
m.sortedSeriesRequested = append(m.sortedSeriesRequested, sortSeries)
var ret []Series
if len(m.toReturn) > 0 {
ret = make([]Series, len(m.toReturn))
copy(ret, m.toReturn)
} else if len(m.resp) > 0 {
ret = make([]Series, 0, len(m.resp))
for _, l := range m.resp {
ret = append(ret, NewListSeries(labels.FromStrings("test", l), nil))
}
}
if sortSeries {
sort.Sort(seriesByLabel(cpy))
sort.Sort(seriesByLabel(ret))
}
return NewMockSeriesSet(cpy...)
return &mockSeriesSet{idx: -1, series: ret, warnings: m.warnings, err: m.err}
}
func (m *mockQuerier) LabelValues(_ context.Context, name string, hints *LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
m.mtx.Lock()
m.labelNamesRequested = append(m.labelNamesRequested, labelNameRequest{
name: name,
matchers: matchers,
})
m.mtx.Unlock()
return m.resp, m.warnings, m.err
}
func (m *mockQuerier) LabelNames(context.Context, *LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
m.mtx.Lock()
m.labelNamesCalls++
m.mtx.Unlock()
return m.resp, m.warnings, m.err
}
func (m *mockQuerier) Close() error {
m.closed = true
return nil
}
type mockChunkQuerier struct {
@ -960,6 +1008,9 @@ func (m *mockChunkQuerier) Select(_ context.Context, sortSeries bool, _ *SelectH
type mockSeriesSet struct {
idx int
series []Series
warnings annotations.Annotations
err error
}
func NewMockSeriesSet(series ...Series) SeriesSet {
@ -970,15 +1021,18 @@ func NewMockSeriesSet(series ...Series) SeriesSet {
}
func (m *mockSeriesSet) Next() bool {
if m.err != nil {
return false
}
m.idx++
return m.idx < len(m.series)
}
func (m *mockSeriesSet) At() Series { return m.series[m.idx] }
func (m *mockSeriesSet) Err() error { return nil }
func (m *mockSeriesSet) Err() error { return m.err }
func (m *mockSeriesSet) Warnings() annotations.Annotations { return nil }
func (m *mockSeriesSet) Warnings() annotations.Annotations { return m.warnings }
type mockChunkSeriesSet struct {
idx int
@ -1336,97 +1390,35 @@ func BenchmarkMergeSeriesSet(b *testing.B) {
}
}
type mockGenericQuerier struct {
mtx sync.Mutex
closed bool
labelNamesCalls int
labelNamesRequested []labelNameRequest
sortedSeriesRequested []bool
resp []string
warnings annotations.Annotations
err error
}
type labelNameRequest struct {
name string
matchers []*labels.Matcher
}
func (m *mockGenericQuerier) Select(_ context.Context, b bool, _ *SelectHints, _ ...*labels.Matcher) genericSeriesSet {
m.mtx.Lock()
m.sortedSeriesRequested = append(m.sortedSeriesRequested, b)
m.mtx.Unlock()
return &mockGenericSeriesSet{resp: m.resp, warnings: m.warnings, err: m.err}
}
func (m *mockGenericQuerier) LabelValues(_ context.Context, name string, hints *LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
m.mtx.Lock()
m.labelNamesRequested = append(m.labelNamesRequested, labelNameRequest{
name: name,
matchers: matchers,
})
m.mtx.Unlock()
return m.resp, m.warnings, m.err
}
func (m *mockGenericQuerier) LabelNames(context.Context, *LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
m.mtx.Lock()
m.labelNamesCalls++
m.mtx.Unlock()
return m.resp, m.warnings, m.err
}
func (m *mockGenericQuerier) Close() error {
m.closed = true
return nil
}
type mockGenericSeriesSet struct {
resp []string
warnings annotations.Annotations
err error
curr int
}
func (m *mockGenericSeriesSet) Next() bool {
if m.err != nil {
return false
func visitMockQueriers(t *testing.T, qr Querier, f func(t *testing.T, q *mockQuerier)) int {
count := 0
switch x := qr.(type) {
case *mockQuerier:
count++
f(t, x)
case *querierAdapter:
count += visitMockQueriersInGenericQuerier(t, x.genericQuerier, f)
}
if m.curr >= len(m.resp) {
return false
return count
}
func visitMockQueriersInGenericQuerier(t *testing.T, g genericQuerier, f func(t *testing.T, q *mockQuerier)) int {
count := 0
switch x := g.(type) {
case *mergeGenericQuerier:
for _, q := range x.queriers {
count += visitMockQueriersInGenericQuerier(t, q, f)
}
m.curr++
return true
}
func (m *mockGenericSeriesSet) Err() error { return m.err }
func (m *mockGenericSeriesSet) Warnings() annotations.Annotations { return m.warnings }
func (m *mockGenericSeriesSet) At() Labels {
return mockLabels(m.resp[m.curr-1])
}
type mockLabels string
func (l mockLabels) Labels() labels.Labels {
return labels.FromStrings("test", string(l))
}
func unwrapMockGenericQuerier(t *testing.T, qr genericQuerier) *mockGenericQuerier {
m, ok := qr.(*mockGenericQuerier)
if !ok {
s, ok := qr.(*secondaryQuerier)
require.True(t, ok, "expected secondaryQuerier got something else")
m, ok = s.genericQuerier.(*mockGenericQuerier)
require.True(t, ok, "expected mockGenericQuerier got something else")
case *genericQuerierAdapter:
// Visitor for chunkQuerier not implemented.
count += visitMockQueriers(t, x.q, f)
case *secondaryQuerier:
count += visitMockQueriersInGenericQuerier(t, x.genericQuerier, f)
}
return m
return count
}
func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
func TestMergeQuerierWithSecondaries_ErrorHandling(t *testing.T) {
var (
errStorage = errors.New("storage error")
warnStorage = errors.New("storage warning")
@ -1434,7 +1426,8 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
)
for _, tcase := range []struct {
name string
queriers []genericQuerier
primaries []Querier
secondaries []Querier
expectedSelectsSeries []labels.Labels
expectedLabels []string
@ -1443,10 +1436,8 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
expectedErrs [4]error
}{
{
// NewMergeQuerier will not create a mergeGenericQuerier
// with just one querier inside, but we can test it anyway.
name: "one successful primary querier",
queriers: []genericQuerier{&mockGenericQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil}},
primaries: []Querier{&mockQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil}},
expectedSelectsSeries: []labels.Labels{
labels.FromStrings("test", "a"),
labels.FromStrings("test", "b"),
@ -1455,9 +1446,9 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
},
{
name: "multiple successful primary queriers",
queriers: []genericQuerier{
&mockGenericQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil},
&mockGenericQuerier{resp: []string{"b", "c"}, warnings: nil, err: nil},
primaries: []Querier{
&mockQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil},
&mockQuerier{resp: []string{"b", "c"}, warnings: nil, err: nil},
},
expectedSelectsSeries: []labels.Labels{
labels.FromStrings("test", "a"),
@ -1468,15 +1459,17 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
},
{
name: "one failed primary querier",
queriers: []genericQuerier{&mockGenericQuerier{warnings: nil, err: errStorage}},
primaries: []Querier{&mockQuerier{warnings: nil, err: errStorage}},
expectedErrs: [4]error{errStorage, errStorage, errStorage, errStorage},
},
{
name: "one successful primary querier with successful secondaries",
queriers: []genericQuerier{
&mockGenericQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil},
&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: nil}},
&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: nil}},
primaries: []Querier{
&mockQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil},
},
secondaries: []Querier{
&mockQuerier{resp: []string{"b"}, warnings: nil, err: nil},
&mockQuerier{resp: []string{"c"}, warnings: nil, err: nil},
},
expectedSelectsSeries: []labels.Labels{
labels.FromStrings("test", "a"),
@ -1487,10 +1480,12 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
},
{
name: "one successful primary querier with empty response and successful secondaries",
queriers: []genericQuerier{
&mockGenericQuerier{resp: []string{}, warnings: nil, err: nil},
&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: nil}},
&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: nil}},
primaries: []Querier{
&mockQuerier{resp: []string{}, warnings: nil, err: nil},
},
secondaries: []Querier{
&mockQuerier{resp: []string{"b"}, warnings: nil, err: nil},
&mockQuerier{resp: []string{"c"}, warnings: nil, err: nil},
},
expectedSelectsSeries: []labels.Labels{
labels.FromStrings("test", "b"),
@ -1500,19 +1495,42 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
},
{
name: "one failed primary querier with successful secondaries",
queriers: []genericQuerier{
&mockGenericQuerier{warnings: nil, err: errStorage},
&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: nil}},
&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: nil}},
primaries: []Querier{
&mockQuerier{warnings: nil, err: errStorage},
},
secondaries: []Querier{
&mockQuerier{resp: []string{"b"}, warnings: nil, err: nil},
&mockQuerier{resp: []string{"c"}, warnings: nil, err: nil},
},
expectedErrs: [4]error{errStorage, errStorage, errStorage, errStorage},
},
{
name: "nil primary querier with failed secondary",
primaries: nil,
secondaries: []Querier{
&mockQuerier{resp: []string{"b"}, warnings: nil, err: errStorage},
},
expectedLabels: []string{},
expectedWarnings: annotations.New().Add(errStorage),
},
{
name: "nil primary querier with two failed secondaries",
primaries: nil,
secondaries: []Querier{
&mockQuerier{resp: []string{"b"}, warnings: nil, err: errStorage},
&mockQuerier{resp: []string{"c"}, warnings: nil, err: errStorage},
},
expectedLabels: []string{},
expectedWarnings: annotations.New().Add(errStorage),
},
{
name: "one successful primary querier with failed secondaries",
queriers: []genericQuerier{
&mockGenericQuerier{resp: []string{"a"}, warnings: nil, err: nil},
&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: errStorage}},
&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: errStorage}},
primaries: []Querier{
&mockQuerier{resp: []string{"a"}, warnings: nil, err: nil},
},
secondaries: []Querier{
&mockQuerier{resp: []string{"b"}, warnings: nil, err: errStorage},
&mockQuerier{resp: []string{"c"}, warnings: nil, err: errStorage},
},
expectedSelectsSeries: []labels.Labels{
labels.FromStrings("test", "a"),
@ -1522,9 +1540,11 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
},
{
name: "successful queriers with warnings",
queriers: []genericQuerier{
&mockGenericQuerier{resp: []string{"a"}, warnings: annotations.New().Add(warnStorage), err: nil},
&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: annotations.New().Add(warnStorage), err: nil}},
primaries: []Querier{
&mockQuerier{resp: []string{"a"}, warnings: annotations.New().Add(warnStorage), err: nil},
},
secondaries: []Querier{
&mockQuerier{resp: []string{"b"}, warnings: annotations.New().Add(warnStorage), err: nil},
},
expectedSelectsSeries: []labels.Labels{
labels.FromStrings("test", "a"),
@ -1535,10 +1555,7 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
},
} {
t.Run(tcase.name, func(t *testing.T) {
q := &mergeGenericQuerier{
queriers: tcase.queriers,
mergeFn: func(l ...Labels) Labels { return l[0] },
}
q := NewMergeQuerier(tcase.primaries, tcase.secondaries, func(s ...Series) Series { return s[0] })
t.Run("Select", func(t *testing.T) {
res := q.Select(context.Background(), false, nil)
@ -1551,62 +1568,67 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
require.ErrorIs(t, res.Err(), tcase.expectedErrs[0], "expected error doesn't match")
require.Equal(t, tcase.expectedSelectsSeries, lbls)
for _, qr := range q.queriers {
m := unwrapMockGenericQuerier(t, qr)
// mergeGenericQuerier forces all Selects to be sorted.
require.Equal(t, []bool{true}, m.sortedSeriesRequested)
}
n := visitMockQueriers(t, q, func(t *testing.T, m *mockQuerier) {
// Single queries should be unsorted; merged queries sorted.
exp := len(tcase.primaries)+len(tcase.secondaries) > 1
require.Equal(t, []bool{exp}, m.sortedSeriesRequested)
})
// Check we visited all queriers.
require.Equal(t, len(tcase.primaries)+len(tcase.secondaries), n)
})
t.Run("LabelNames", func(t *testing.T) {
res, w, err := q.LabelNames(ctx, nil)
require.Subset(t, tcase.expectedWarnings, w)
require.ErrorIs(t, err, tcase.expectedErrs[1], "expected error doesn't match")
require.Equal(t, tcase.expectedLabels, res)
requireEqualSlice(t, tcase.expectedLabels, res)
if err != nil {
return
}
for _, qr := range q.queriers {
m := unwrapMockGenericQuerier(t, qr)
visitMockQueriers(t, q, func(t *testing.T, m *mockQuerier) {
require.Equal(t, 1, m.labelNamesCalls)
}
})
})
t.Run("LabelValues", func(t *testing.T) {
res, w, err := q.LabelValues(ctx, "test", nil)
require.Subset(t, tcase.expectedWarnings, w)
require.ErrorIs(t, err, tcase.expectedErrs[2], "expected error doesn't match")
require.Equal(t, tcase.expectedLabels, res)
requireEqualSlice(t, tcase.expectedLabels, res)
if err != nil {
return
}
for _, qr := range q.queriers {
m := unwrapMockGenericQuerier(t, qr)
visitMockQueriers(t, q, func(t *testing.T, m *mockQuerier) {
require.Equal(t, []labelNameRequest{{name: "test"}}, m.labelNamesRequested)
}
})
})
t.Run("LabelValuesWithMatchers", func(t *testing.T) {
matcher := labels.MustNewMatcher(labels.MatchEqual, "otherLabel", "someValue")
res, w, err := q.LabelValues(ctx, "test2", nil, matcher)
require.Subset(t, tcase.expectedWarnings, w)
require.ErrorIs(t, err, tcase.expectedErrs[3], "expected error doesn't match")
require.Equal(t, tcase.expectedLabels, res)
requireEqualSlice(t, tcase.expectedLabels, res)
if err != nil {
return
}
for _, qr := range q.queriers {
m := unwrapMockGenericQuerier(t, qr)
visitMockQueriers(t, q, func(t *testing.T, m *mockQuerier) {
require.Equal(t, []labelNameRequest{
{name: "test"},
{name: "test2", matchers: []*labels.Matcher{matcher}},
}, m.labelNamesRequested)
})
})
})
}
})
})
}
// Check slice but ignore difference between nil and empty.
func requireEqualSlice[T any](t require.TestingT, a, b []T, msgAndArgs ...interface{}) {
if len(a) == 0 {
require.Empty(t, b, msgAndArgs...)
} else {
require.Equal(t, a, b, msgAndArgs...)
}
}

40
storage/remote/client.go
View file

@ -14,7 +14,6 @@
package remote
import (
"bufio"
"bytes"
"context"
"fmt"
@ -38,6 +37,7 @@ import (
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/storage/remote/azuread"
"github.com/prometheus/prometheus/storage/remote/googleiam"
)
const maxErrMsgLen = 1024
@ -132,6 +132,7 @@ type ClientConfig struct {
HTTPClientConfig config_util.HTTPClientConfig
SigV4Config *sigv4.SigV4Config
AzureADConfig *azuread.AzureADConfig
GoogleIAMConfig *googleiam.Config
Headers map[string]string
RetryOnRateLimit bool
WriteProtoMsg config.RemoteWriteProtoMsg
@ -193,6 +194,13 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) {
}
}
if conf.GoogleIAMConfig != nil {
t, err = googleiam.NewRoundTripper(conf.GoogleIAMConfig, t)
if err != nil {
return nil, err
}
}
writeProtoMsg := config.RemoteWriteProtoMsgV1
if conf.WriteProtoMsg != "" {
writeProtoMsg = conf.WriteProtoMsg
@ -235,12 +243,12 @@ type RecoverableError struct {
// Store sends a batch of samples to the HTTP endpoint, the request is the proto marshalled
// and encoded bytes from codec.go.
func (c *Client) Store(ctx context.Context, req []byte, attempt int) error {
func (c *Client) Store(ctx context.Context, req []byte, attempt int) (WriteResponseStats, error) {
httpReq, err := http.NewRequest(http.MethodPost, c.urlString, bytes.NewReader(req))
if err != nil {
// Errors from NewRequest are from unparsable URLs, so are not
// recoverable.
return err
return WriteResponseStats{}, err
}
httpReq.Header.Add("Content-Encoding", string(c.writeCompression))
@ -267,28 +275,34 @@ func (c *Client) Store(ctx context.Context, req []byte, attempt int) error {
if err != nil {
// Errors from Client.Do are from (for example) network errors, so are
// recoverable.
return RecoverableError{err, defaultBackoff}
return WriteResponseStats{}, RecoverableError{err, defaultBackoff}
}
defer func() {
io.Copy(io.Discard, httpResp.Body)
httpResp.Body.Close()
}()
// TODO(bwplotka): Pass logger and emit debug on error?
// Parsing error means there were some response header values we can't parse,
// we can continue handling.
rs, _ := ParseWriteResponseStats(httpResp)
//nolint:usestdlibvars
if httpResp.StatusCode/100 != 2 {
scanner := bufio.NewScanner(io.LimitReader(httpResp.Body, maxErrMsgLen))
line := ""
if scanner.Scan() {
line = scanner.Text()
}
err = fmt.Errorf("server returned HTTP status %s: %s", httpResp.Status, line)
if httpResp.StatusCode/100 == 2 {
return rs, nil
}
// Handling errors e.g. read potential error in the body.
// TODO(bwplotka): Pass logger and emit debug on error?
body, _ := io.ReadAll(io.LimitReader(httpResp.Body, maxErrMsgLen))
err = fmt.Errorf("server returned HTTP status %s: %s", httpResp.Status, body)
//nolint:usestdlibvars
if httpResp.StatusCode/100 == 5 ||
(c.retryOnRateLimit && httpResp.StatusCode == http.StatusTooManyRequests) {
return RecoverableError{err, retryAfterDuration(httpResp.Header.Get("Retry-After"))}
return rs, RecoverableError{err, retryAfterDuration(httpResp.Header.Get("Retry-After"))}
}
return err
return rs, err
}
// retryAfterDuration returns the duration for the Retry-After header. In case of any errors, it

8
storage/remote/client_test.go
View file

@ -73,7 +73,7 @@ func TestStoreHTTPErrorHandling(t *testing.T) {
c, err := NewWriteClient(hash, conf)
require.NoError(t, err)
err = c.Store(context.Background(), []byte{}, 0)
_, err = c.Store(context.Background(), []byte{}, 0)
if test.err != nil {
require.EqualError(t, err, test.err.Error())
} else {
@ -133,7 +133,7 @@ func TestClientRetryAfter(t *testing.T) {
c := getClient(getClientConfig(serverURL, tc.retryOnRateLimit))
var recErr RecoverableError
err = c.Store(context.Background(), []byte{}, 0)
_, err = c.Store(context.Background(), []byte{}, 0)
require.Equal(t, tc.expectedRecoverable, errors.As(err, &recErr), "Mismatch in expected recoverable error status.")
if tc.expectedRecoverable {
require.Equal(t, tc.expectedRetryAfter, recErr.retryAfter)
@ -169,7 +169,7 @@ func TestRetryAfterDuration(t *testing.T) {
}
}
func TestClientHeaders(t *testing.T) {
func TestClientCustomHeaders(t *testing.T) {
headersToSend := map[string]string{"Foo": "Bar", "Baz": "qux"}
var called bool
@ -203,7 +203,7 @@ func TestClientHeaders(t *testing.T) {
c, err := NewWriteClient("c", conf)
require.NoError(t, err)
err = c.Store(context.Background(), []byte{}, 0)
_, err = c.Store(context.Background(), []byte{}, 0)
require.NoError(t, err)
require.True(t, called, "The remote server wasn't called")

54
storage/remote/googleiam/googleiam.go Normal file
View file

@ -0,0 +1,54 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package googleiam provides an http.RoundTripper that attaches an Google Cloud accessToken
// to remote write requests.
package googleiam
import (
"context"
"fmt"
"net/http"
"golang.org/x/oauth2/google"
"google.golang.org/api/option"
apihttp "google.golang.org/api/transport/http"
)
type Config struct {
CredentialsFile string `yaml:"credentials_file,omitempty"`
}
// NewRoundTripper creates a round tripper that adds Google Cloud Monitoring authorization to calls
// using either a credentials file or the default credentials.
func NewRoundTripper(cfg *Config, next http.RoundTripper) (http.RoundTripper, error) {
if next == nil {
next = http.DefaultTransport
}
const scopes = "https://www.googleapis.com/auth/monitoring.write"
ctx := context.Background()
opts := []option.ClientOption{
option.WithScopes(scopes),
}
if cfg.CredentialsFile != "" {
opts = append(opts, option.WithCredentialsFile(cfg.CredentialsFile))
} else {
creds, err := google.FindDefaultCredentials(ctx, scopes)
if err != nil {
return nil, fmt.Errorf("error finding default Google credentials: %w", err)
}
opts = append(opts, option.WithCredentials(creds))
}
return apihttp.NewTransport(ctx, next, opts...)
}

38
storage/remote/otlptranslator/prometheusremotewrite/helper.go
View file

@ -65,14 +65,14 @@ type bucketBoundsData struct {
bound float64
}
// byBucketBoundsData enables the usage of sort.Sort() with a slice of bucket bounds
// byBucketBoundsData enables the usage of sort.Sort() with a slice of bucket bounds.
type byBucketBoundsData []bucketBoundsData
func (m byBucketBoundsData) Len() int { return len(m) }
func (m byBucketBoundsData) Less(i, j int) bool { return m[i].bound < m[j].bound }
func (m byBucketBoundsData) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
// ByLabelName enables the usage of sort.Sort() with a slice of labels
// ByLabelName enables the usage of sort.Sort() with a slice of labels.
type ByLabelName []prompb.Label
func (a ByLabelName) Len() int { return len(a) }
@ -115,14 +115,23 @@ var seps = []byte{'\xff'}
// createAttributes creates a slice of Prometheus Labels with OTLP attributes and pairs of string values.
// Unpaired string values are ignored. String pairs overwrite OTLP labels if collisions happen and
// if logOnOverwrite is true, the overwrite is logged. Resulting label names are sanitized.
func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externalLabels map[string]string,
// If settings.PromoteResourceAttributes is not empty, it's a set of resource attributes that should be promoted to labels.
func createAttributes(resource pcommon.Resource, attributes pcommon.Map, settings Settings,
ignoreAttrs []string, logOnOverwrite bool, extras ...string) []prompb.Label {
resourceAttrs := resource.Attributes()
serviceName, haveServiceName := resourceAttrs.Get(conventions.AttributeServiceName)
instance, haveInstanceID := resourceAttrs.Get(conventions.AttributeServiceInstanceID)
promotedAttrs := make([]prompb.Label, 0, len(settings.PromoteResourceAttributes))
for _, name := range settings.PromoteResourceAttributes {
if value, exists := resourceAttrs.Get(name); exists {
promotedAttrs = append(promotedAttrs, prompb.Label{Name: name, Value: value.AsString()})
}
}
sort.Stable(ByLabelName(promotedAttrs))
// Calculate the maximum possible number of labels we could return so we can preallocate l
maxLabelCount := attributes.Len() + len(externalLabels) + len(extras)/2
maxLabelCount := attributes.Len() + len(settings.ExternalLabels) + len(promotedAttrs) + len(extras)/2
if haveServiceName {
maxLabelCount++
@ -132,9 +141,6 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa
maxLabelCount++
}
// map ensures no duplicate label name
l := make(map[string]string, maxLabelCount)
// Ensure attributes are sorted by key for consistent merging of keys which
// collide when sanitized.
labels := make([]prompb.Label, 0, maxLabelCount)
@ -148,6 +154,8 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa
})
sort.Stable(ByLabelName(labels))
// map ensures no duplicate label names.
l := make(map[string]string, maxLabelCount)
for _, label := range labels {
var finalKey = prometheustranslator.NormalizeLabel(label.Name)
if existingValue, alreadyExists := l[finalKey]; alreadyExists {
@ -157,6 +165,13 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa
}
}
for _, lbl := range promotedAttrs {
normalized := prometheustranslator.NormalizeLabel(lbl.Name)
if _, exists := l[normalized]; !exists {
l[normalized] = lbl.Value
}
}
// Map service.name + service.namespace to job
if haveServiceName {
val := serviceName.AsString()
@ -169,7 +184,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa
if haveInstanceID {
l[model.InstanceLabel] = instance.AsString()
}
for key, value := range externalLabels {
for key, value := range settings.ExternalLabels {
// External labels have already been sanitized
if _, alreadyExists := l[key]; alreadyExists {
// Skip external labels if they are overridden by metric attributes
@ -232,7 +247,7 @@ func (c *PrometheusConverter) addHistogramDataPoints(dataPoints pmetric.Histogra
for x := 0; x < dataPoints.Len(); x++ {
pt := dataPoints.At(x)
timestamp := convertTimeStamp(pt.Timestamp())
baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nil, false)
baseLabels := createAttributes(resource, pt.Attributes(), settings, nil, false)
// If the sum is unset, it indicates the _sum metric point should be
// omitted
@ -408,7 +423,7 @@ func (c *PrometheusConverter) addSummaryDataPoints(dataPoints pmetric.SummaryDat
for x := 0; x < dataPoints.Len(); x++ {
pt := dataPoints.At(x)
timestamp := convertTimeStamp(pt.Timestamp())
baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nil, false)
baseLabels := createAttributes(resource, pt.Attributes(), settings, nil, false)
// treat sum as a sample in an individual TimeSeries
sum := &prompb.Sample{
@ -554,7 +569,8 @@ func addResourceTargetInfo(resource pcommon.Resource, settings Settings, timesta
name = settings.Namespace + "_" + name
}
labels := createAttributes(resource, attributes, settings.ExternalLabels, identifyingAttrs, false, model.MetricNameLabel, name)
settings.PromoteResourceAttributes = nil
labels := createAttributes(resource, attributes, settings, identifyingAttrs, false, model.MetricNameLabel, name)
haveIdentifier := false
for _, l := range labels {
if l.Name == model.JobLabel || l.Name == model.InstanceLabel {

161
storage/remote/otlptranslator/prometheusremotewrite/helper_test.go Normal file
View file

@ -0,0 +1,161 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package prometheusremotewrite
import (
"testing"
"github.com/stretchr/testify/assert"
"go.opentelemetry.io/collector/pdata/pcommon"
"github.com/prometheus/prometheus/prompb"
)
func TestCreateAttributes(t *testing.T) {
resourceAttrs := map[string]string{
"service.name": "service name",
"service.instance.id": "service ID",
"existent-attr": "resource value",
// This one is for testing conflict with metric attribute.
"metric-attr": "resource value",
// This one is for testing conflict with auto-generated job attribute.
"job": "resource value",
// This one is for testing conflict with auto-generated instance attribute.
"instance": "resource value",
}
resource := pcommon.NewResource()
for k, v := range resourceAttrs {
resource.Attributes().PutStr(k, v)
}
attrs := pcommon.NewMap()
attrs.PutStr("__name__", "test_metric")
attrs.PutStr("metric-attr", "metric value")
testCases := []struct {
name string
promoteResourceAttributes []string
expectedLabels []prompb.Label
}{
{
name: "Successful conversion without resource attribute promotion",
promoteResourceAttributes: nil,
expectedLabels: []prompb.Label{
{
Name: "__name__",
Value: "test_metric",
},
{
Name: "instance",
Value: "service ID",
},
{
Name: "job",
Value: "service name",
},
{
Name: "metric_attr",
Value: "metric value",
},
},
},
{
name: "Successful conversion with resource attribute promotion",
promoteResourceAttributes: []string{"non-existent-attr", "existent-attr"},
expectedLabels: []prompb.Label{
{
Name: "__name__",
Value: "test_metric",
},
{
Name: "instance",
Value: "service ID",
},
{
Name: "job",
Value: "service name",
},
{
Name: "metric_attr",
Value: "metric value",
},
{
Name: "existent_attr",
Value: "resource value",
},
},
},
{
name: "Successful conversion with resource attribute promotion, conflicting resource attributes are ignored",
promoteResourceAttributes: []string{"non-existent-attr", "existent-attr", "metric-attr", "job", "instance"},
expectedLabels: []prompb.Label{
{
Name: "__name__",
Value: "test_metric",
},
{
Name: "instance",
Value: "service ID",
},
{
Name: "job",
Value: "service name",
},
{
Name: "existent_attr",
Value: "resource value",
},
{
Name: "metric_attr",
Value: "metric value",
},
},
},
{
name: "Successful conversion with resource attribute promotion, attributes are only promoted once",
promoteResourceAttributes: []string{"existent-attr", "existent-attr"},
expectedLabels: []prompb.Label{
{
Name: "__name__",
Value: "test_metric",
},
{
Name: "instance",
Value: "service ID",
},
{
Name: "job",
Value: "service name",
},
{
Name: "existent_attr",
Value: "resource value",
},
{
Name: "metric_attr",
Value: "metric value",
},
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
settings := Settings{
PromoteResourceAttributes: tc.promoteResourceAttributes,
}
lbls := createAttributes(resource, attrs, settings, nil, false)
assert.ElementsMatch(t, lbls, tc.expectedLabels)
})
}
}

2
storage/remote/otlptranslator/prometheusremotewrite/histograms.go
View file

@ -45,7 +45,7 @@ func (c *PrometheusConverter) addExponentialHistogramDataPoints(dataPoints pmetr
lbls := createAttributes(
resource,
pt.Attributes(),
settings.ExternalLabels,
settings,
nil,
true,
model.MetricNameLabel,

1
storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go
View file

@ -36,6 +36,7 @@ type Settings struct {
ExportCreatedMetric bool
AddMetricSuffixes bool
SendMetadata bool
PromoteResourceAttributes []string
}
// PrometheusConverter converts from OTel write format to Prometheus remote write format.

4
storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go
View file

@ -34,7 +34,7 @@ func (c *PrometheusConverter) addGaugeNumberDataPoints(dataPoints pmetric.Number
labels := createAttributes(
resource,
pt.Attributes(),
settings.ExternalLabels,
settings,
nil,
true,
model.MetricNameLabel,
@ -64,7 +64,7 @@ func (c *PrometheusConverter) addSumNumberDataPoints(dataPoints pmetric.NumberDa
lbls := createAttributes(
resource,
pt.Attributes(),
settings.ExternalLabels,
settings,
nil,
true,
model.MetricNameLabel,

123
storage/remote/queue_manager.go
View file

@ -391,7 +391,7 @@ func (m *queueManagerMetrics) unregister() {
// external timeseries database.
type WriteClient interface {
// Store stores the given samples in the remote storage.
Store(ctx context.Context, req []byte, retryAttempt int) error
Store(ctx context.Context, req []byte, retryAttempt int) (WriteResponseStats, error)
// Name uniquely identifies the remote storage.
Name() string
// Endpoint is the remote read or write endpoint for the storage client.
@ -597,14 +597,15 @@ func (t *QueueManager) sendMetadataWithBackoff(ctx context.Context, metadata []p
}
begin := time.Now()
err := t.storeClient.Store(ctx, req, try)
// Ignoring WriteResponseStats, because there is nothing for metadata, since it's
// embedded in v2 calls now, and we do v1 here.
_, err := t.storeClient.Store(ctx, req, try)
t.metrics.sentBatchDuration.Observe(time.Since(begin).Seconds())
if err != nil {
span.RecordError(err)
return err
}
return nil
}
@ -1108,9 +1109,9 @@ func (t *QueueManager) shouldReshard(desiredShards int) bool {
if desiredShards == t.numShards {
return false
}
// We shouldn't reshard if Prometheus hasn't been able to send to the
// remote endpoint successfully within some period of time.
minSendTimestamp := time.Now().Add(-2 * time.Duration(t.cfg.BatchSendDeadline)).Unix()
// We shouldn't reshard if Prometheus hasn't been able to send
// since the last time it checked if it should reshard.
minSendTimestamp := time.Now().Add(-1 * shardUpdateDuration).Unix()
lsts := t.lastSendTimestamp.Load()
if lsts < minSendTimestamp {
level.Warn(t.logger).Log("msg", "Skipping resharding, last successful send was beyond threshold", "lastSendTimestamp", lsts, "minSendTimestamp", minSendTimestamp)
@ -1661,8 +1662,8 @@ func populateTimeSeries(batch []timeSeries, pendingData []prompb.TimeSeries, sen
func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, sampleCount, exemplarCount, histogramCount int, pBuf *proto.Buffer, buf *[]byte, enc Compression) error {
begin := time.Now()
err := s.sendSamplesWithBackoff(ctx, samples, sampleCount, exemplarCount, histogramCount, 0, pBuf, buf, enc)
s.updateMetrics(ctx, err, sampleCount, exemplarCount, histogramCount, 0, time.Since(begin))
rs, err := s.sendSamplesWithBackoff(ctx, samples, sampleCount, exemplarCount, histogramCount, 0, pBuf, buf, enc)
s.updateMetrics(ctx, err, sampleCount, exemplarCount, histogramCount, 0, rs, time.Since(begin))
return err
}
@ -1670,17 +1671,29 @@ func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, s
// See https://github.com/prometheus/prometheus/issues/14409
func (s *shards) sendV2Samples(ctx context.Context, samples []writev2.TimeSeries, labels []string, sampleCount, exemplarCount, histogramCount, metadataCount int, pBuf, buf *[]byte, enc Compression) error {
begin := time.Now()
err := s.sendV2SamplesWithBackoff(ctx, samples, labels, sampleCount, exemplarCount, histogramCount, metadataCount, pBuf, buf, enc)
s.updateMetrics(ctx, err, sampleCount, exemplarCount, histogramCount, metadataCount, time.Since(begin))
rs, err := s.sendV2SamplesWithBackoff(ctx, samples, labels, sampleCount, exemplarCount, histogramCount, metadataCount, pBuf, buf, enc)
s.updateMetrics(ctx, err, sampleCount, exemplarCount, histogramCount, metadataCount, rs, time.Since(begin))
return err
}
func (s *shards) updateMetrics(_ context.Context, err error, sampleCount, exemplarCount, histogramCount, metadataCount int, duration time.Duration) {
func (s *shards) updateMetrics(_ context.Context, err error, sampleCount, exemplarCount, histogramCount, metadataCount int, rs WriteResponseStats, duration time.Duration) {
// Partial errors may happen -- account for that.
sampleDiff := sampleCount - rs.Samples
if sampleDiff > 0 {
s.qm.metrics.failedSamplesTotal.Add(float64(sampleDiff))
}
histogramDiff := histogramCount - rs.Histograms
if histogramDiff > 0 {
s.qm.metrics.failedHistogramsTotal.Add(float64(histogramDiff))
}
exemplarDiff := exemplarCount - rs.Exemplars
if exemplarDiff > 0 {
s.qm.metrics.failedExemplarsTotal.Add(float64(exemplarDiff))
}
if err != nil {
level.Error(s.qm.logger).Log("msg", "non-recoverable error", "count", sampleCount, "exemplarCount", exemplarCount, "histogramCount", histogramCount, "err", err)
s.qm.metrics.failedSamplesTotal.Add(float64(sampleCount))
s.qm.metrics.failedExemplarsTotal.Add(float64(exemplarCount))
s.qm.metrics.failedHistogramsTotal.Add(float64(histogramCount))
level.Error(s.qm.logger).Log("msg", "non-recoverable error", "failedSampleCount", sampleDiff, "failedHistogramCount", histogramDiff, "failedExemplarCount", exemplarDiff, "err", err)
} else if sampleDiff+exemplarDiff+histogramDiff > 0 {
level.Error(s.qm.logger).Log("msg", "we got 2xx status code from the Receiver yet statistics indicate some dat was not written; investigation needed", "failedSampleCount", sampleDiff, "failedHistogramCount", histogramDiff, "failedExemplarCount", exemplarDiff)
}
// These counters are used to calculate the dynamic sharding, and as such
@ -1688,6 +1701,7 @@ func (s *shards) updateMetrics(_ context.Context, err error, sampleCount, exempl
s.qm.dataOut.incr(int64(sampleCount + exemplarCount + histogramCount + metadataCount))
s.qm.dataOutDuration.incr(int64(duration))
s.qm.lastSendTimestamp.Store(time.Now().Unix())
// Pending samples/exemplars/histograms also should be subtracted, as an error means
// they will not be retried.
s.qm.metrics.pendingSamples.Sub(float64(sampleCount))
@ -1699,19 +1713,29 @@ func (s *shards) updateMetrics(_ context.Context, err error, sampleCount, exempl
}
// sendSamples to the remote storage with backoff for recoverable errors.
func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.TimeSeries, sampleCount, exemplarCount, histogramCount, metadataCount int, pBuf *proto.Buffer, buf *[]byte, enc Compression) error {
func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.TimeSeries, sampleCount, exemplarCount, histogramCount, metadataCount int, pBuf *proto.Buffer, buf *[]byte, enc Compression) (WriteResponseStats, error) {
// Build the WriteRequest with no metadata.
req, highest, lowest, err := buildWriteRequest(s.qm.logger, samples, nil, pBuf, buf, nil, enc)
s.qm.buildRequestLimitTimestamp.Store(lowest)
if err != nil {
// Failing to build the write request is non-recoverable, since it will
// only error if marshaling the proto to bytes fails.
return err
return WriteResponseStats{}, err
}
reqSize := len(req)
*buf = req
// Since we retry writes via attemptStore and sendWriteRequestWithBackoff we need
// to track the total amount of accepted data across the various attempts.
accumulatedStats := WriteResponseStats{}
var accumulatedStatsMu sync.Mutex
addStats := func(rs WriteResponseStats) {
accumulatedStatsMu.Lock()
accumulatedStats = accumulatedStats.Add(rs)
accumulatedStatsMu.Unlock()
}
// An anonymous function allows us to defer the completion of our per-try spans
// without causing a memory leak, and it has the nice effect of not propagating any
// parameters for sendSamplesWithBackoff/3.
@ -1759,17 +1783,21 @@ func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.Ti
s.qm.metrics.exemplarsTotal.Add(float64(exemplarCount))
s.qm.metrics.histogramsTotal.Add(float64(histogramCount))
s.qm.metrics.metadataTotal.Add(float64(metadataCount))
err := s.qm.client().Store(ctx, *buf, try)
// Technically for v1, we will likely have empty response stats, but for
// newer Receivers this might be not, so used it in a best effort.
rs, err := s.qm.client().Store(ctx, *buf, try)
s.qm.metrics.sentBatchDuration.Observe(time.Since(begin).Seconds())
// TODO(bwplotka): Revisit this once we have Receivers doing retriable partial error
// so far we don't have those, so it's ok to potentially skew statistics.
addStats(rs)
if err != nil {
if err == nil {
return nil
}
span.RecordError(err)
return err
}
return nil
}
onRetry := func() {
s.qm.metrics.retriedSamplesTotal.Add(float64(sampleCount))
s.qm.metrics.retriedExemplarsTotal.Add(float64(exemplarCount))
@ -1780,29 +1808,48 @@ func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.Ti
if errors.Is(err, context.Canceled) {
// When there is resharding, we cancel the context for this queue, which means the data is not sent.
// So we exit early to not update the metrics.
return err
return accumulatedStats, err
}
s.qm.metrics.sentBytesTotal.Add(float64(reqSize))
s.qm.metrics.highestSentTimestamp.Set(float64(highest / 1000))
return err
if err == nil && !accumulatedStats.Confirmed {
// No 2.0 response headers, and we sent v1 message, so likely it's 1.0 Receiver.
// Assume success, don't rely on headers.
return WriteResponseStats{
Samples: sampleCount,
Histograms: histogramCount,
Exemplars: exemplarCount,
}, nil
}
return accumulatedStats, err
}
// sendV2Samples to the remote storage with backoff for recoverable errors.
func (s *shards) sendV2SamplesWithBackoff(ctx context.Context, samples []writev2.TimeSeries, labels []string, sampleCount, exemplarCount, histogramCount, metadataCount int, pBuf, buf *[]byte, enc Compression) error {
func (s *shards) sendV2SamplesWithBackoff(ctx context.Context, samples []writev2.TimeSeries, labels []string, sampleCount, exemplarCount, histogramCount, metadataCount int, pBuf, buf *[]byte, enc Compression) (WriteResponseStats, error) {
// Build the WriteRequest with no metadata.
req, highest, lowest, err := buildV2WriteRequest(s.qm.logger, samples, labels, pBuf, buf, nil, enc)
s.qm.buildRequestLimitTimestamp.Store(lowest)
if err != nil {
// Failing to build the write request is non-recoverable, since it will
// only error if marshaling the proto to bytes fails.
return err
return WriteResponseStats{}, err
}
reqSize := len(req)
*buf = req
// Since we retry writes via attemptStore and sendWriteRequestWithBackoff we need
// to track the total amount of accepted data across the various attempts.
accumulatedStats := WriteResponseStats{}
var accumulatedStatsMu sync.Mutex
addStats := func(rs WriteResponseStats) {
accumulatedStatsMu.Lock()
accumulatedStats = accumulatedStats.Add(rs)
accumulatedStatsMu.Unlock()
}
// An anonymous function allows us to defer the completion of our per-try spans
// without causing a memory leak, and it has the nice effect of not propagating any
// parameters for sendSamplesWithBackoff/3.
@ -1850,16 +1897,29 @@ func (s *shards) sendV2SamplesWithBackoff(ctx context.Context, samples []writev2
s.qm.metrics.exemplarsTotal.Add(float64(exemplarCount))
s.qm.metrics.histogramsTotal.Add(float64(histogramCount))
s.qm.metrics.metadataTotal.Add(float64(metadataCount))
err := s.qm.client().Store(ctx, *buf, try)
rs, err := s.qm.client().Store(ctx, *buf, try)
s.qm.metrics.sentBatchDuration.Observe(time.Since(begin).Seconds())
// TODO(bwplotka): Revisit this once we have Receivers doing retriable partial error
// so far we don't have those, so it's ok to potentially skew statistics.
addStats(rs)
if err != nil {
if err == nil {
// Check the case mentioned in PRW 2.0
// https://prometheus.io/docs/specs/remote_write_spec_2_0/#required-written-response-headers.
if sampleCount+histogramCount+exemplarCount > 0 && rs.NoDataWritten() {
err = fmt.Errorf("sent v2 request with %v samples, %v histograms and %v exemplars; got 2xx, but PRW 2.0 response header statistics indicate %v samples, %v histograms and %v exemplars were accepted;"+
" assumining failure e.g. the target only supports PRW 1.0 prometheus.WriteRequest, but does not check the Content-Type header correctly",
sampleCount, histogramCount, exemplarCount,
rs.Samples, rs.Histograms, rs.Exemplars,
)
span.RecordError(err)
return err
}
return nil
}
span.RecordError(err)
return err
}
onRetry := func() {
s.qm.metrics.retriedSamplesTotal.Add(float64(sampleCount))
@ -1871,13 +1931,12 @@ func (s *shards) sendV2SamplesWithBackoff(ctx context.Context, samples []writev2
if errors.Is(err, context.Canceled) {
// When there is resharding, we cancel the context for this queue, which means the data is not sent.
// So we exit early to not update the metrics.
return err
return accumulatedStats, err
}
s.qm.metrics.sentBytesTotal.Add(float64(reqSize))
s.qm.metrics.highestSentTimestamp.Set(float64(highest / 1000))
return err
return accumulatedStats, err
}
func populateV2TimeSeries(symbolTable *writev2.SymbolsTable, batch []timeSeries, pendingData []writev2.TimeSeries, sendExemplars, sendNativeHistograms bool) (int, int, int, int) {

63
storage/remote/queue_manager_test.go
View file

@ -118,10 +118,10 @@ func TestBasicContentNegotiation(t *testing.T) {
expectFail: true,
},
{
name: "v2 talks to v1 that tries to unmarshal v2 payload with v1 proto",
name: "v2 talks to (broken) v1 that tries to unmarshal v2 payload with v1 proto",
senderProtoMsg: config.RemoteWriteProtoMsgV2, receiverProtoMsg: config.RemoteWriteProtoMsgV1,
injectErrs: []error{nil},
expectFail: true, // invalid request, no timeseries
expectFail: true, // We detect this thanks to https://github.com/prometheus/prometheus/issues/14359
},
// Opposite, v1 talking to v2 only server.
{
@ -130,12 +130,6 @@ func TestBasicContentNegotiation(t *testing.T) {
injectErrs: []error{errors.New("pretend unrecoverable err")},
expectFail: true,
},
{
name: "v1 talks to (broken) v2 that tries to unmarshal v1 payload with v2 proto",
senderProtoMsg: config.RemoteWriteProtoMsgV1, receiverProtoMsg: config.RemoteWriteProtoMsgV2,
injectErrs: []error{nil},
expectFail: true, // invalid request, no timeseries
},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
@ -182,7 +176,6 @@ func TestBasicContentNegotiation(t *testing.T) {
if !tc.expectFail {
// No error expected, so wait for data.
c.waitForExpectedData(t, 5*time.Second)
require.Equal(t, 1, c.writesReceived)
require.Equal(t, 0.0, client_testutil.ToFloat64(qm.metrics.failedSamplesTotal))
} else {
// Wait for failure to be recorded in metrics.
@ -190,11 +183,10 @@ func TestBasicContentNegotiation(t *testing.T) {
defer cancel()
require.NoError(t, runutil.Retry(500*time.Millisecond, ctx.Done(), func() error {
if client_testutil.ToFloat64(qm.metrics.failedSamplesTotal) != 1.0 {
return errors.New("expected one sample failed in qm metrics")
return fmt.Errorf("expected one sample failed in qm metrics; got %v", client_testutil.ToFloat64(qm.metrics.failedSamplesTotal))
}
return nil
}))
require.Equal(t, 0, c.writesReceived)
}
// samplesTotal means attempts.
@ -711,32 +703,35 @@ func TestShouldReshard(t *testing.T) {
startingShards int
samplesIn, samplesOut, lastSendTimestamp int64
expectedToReshard bool
sendDeadline model.Duration
}
cases := []testcase{
{
// Resharding shouldn't take place if the last successful send was > batch send deadline*2 seconds ago.
// resharding shouldn't take place if we haven't successfully sent
// since the last shardUpdateDuration, even if the send deadline is very low
startingShards: 10,
samplesIn: 1000,
samplesOut: 10,
lastSendTimestamp: time.Now().Unix() - int64(3*time.Duration(config.DefaultQueueConfig.BatchSendDeadline)/time.Second),
lastSendTimestamp: time.Now().Unix() - int64(shardUpdateDuration),
expectedToReshard: false,
sendDeadline: model.Duration(100 * time.Millisecond),
},
{
startingShards: 5,
startingShards: 10,
samplesIn: 1000,
samplesOut: 10,
lastSendTimestamp: time.Now().Unix(),
expectedToReshard: true,
sendDeadline: config.DefaultQueueConfig.BatchSendDeadline,
},
}
for _, c := range cases {
_, m := newTestClientAndQueueManager(t, defaultFlushDeadline, config.RemoteWriteProtoMsgV1)
_, m := newTestClientAndQueueManager(t, time.Duration(c.sendDeadline), config.RemoteWriteProtoMsgV1)
m.numShards = c.startingShards
m.dataIn.incr(c.samplesIn)
m.dataOut.incr(c.samplesOut)
m.lastSendTimestamp.Store(c.lastSendTimestamp)
m.Start()
desiredShards := m.calculateDesiredShards()
@ -764,10 +759,10 @@ func TestDisableReshardOnRetry(t *testing.T) {
metrics = newQueueManagerMetrics(nil, "", "")
client = &MockWriteClient{
StoreFunc: func(ctx context.Context, b []byte, i int) error {
StoreFunc: func(ctx context.Context, b []byte, i int) (WriteResponseStats, error) {
onStoreCalled()
return RecoverableError{
return WriteResponseStats{}, RecoverableError{
error: fmt.Errorf("fake error"),
retryAfter: model.Duration(retryAfter),
}
@ -1113,14 +1108,14 @@ func (c *TestWriteClient) SetReturnError(err error) {
c.returnError = err
}
func (c *TestWriteClient) Store(_ context.Context, req []byte, _ int) error {
func (c *TestWriteClient) Store(_ context.Context, req []byte, _ int) (WriteResponseStats, error) {
c.mtx.Lock()
defer c.mtx.Unlock()
if c.storeWait > 0 {
time.Sleep(c.storeWait)
}
if c.returnError != nil {
return c.returnError
return WriteResponseStats{}, c.returnError
}
// nil buffers are ok for snappy, ignore cast error.
if c.buf != nil {
@ -1130,14 +1125,14 @@ func (c *TestWriteClient) Store(_ context.Context, req []byte, _ int) error {
reqBuf, err := snappy.Decode(c.buf, req)
c.buf = reqBuf
if err != nil {
return err
return WriteResponseStats{}, err
}
// Check if we've been told to inject err for this call.
if len(c.injectedErrs) > 0 {
c.currErr++
if err = c.injectedErrs[c.currErr]; err != nil {
return err
return WriteResponseStats{}, err
}
}
@ -1156,13 +1151,10 @@ func (c *TestWriteClient) Store(_ context.Context, req []byte, _ int) error {
}
}
if err != nil {
return err
}
if len(reqProto.Timeseries) == 0 && len(reqProto.Metadata) == 0 {
return errors.New("invalid request, no timeseries")
return WriteResponseStats{}, err
}
rs := WriteResponseStats{}
b := labels.NewScratchBuilder(0)
for _, ts := range reqProto.Timeseries {
labels := ts.ToLabels(&b, nil)
@ -1170,10 +1162,12 @@ func (c *TestWriteClient) Store(_ context.Context, req []byte, _ int) error {
if len(ts.Samples) > 0 {
c.receivedSamples[tsID] = append(c.receivedSamples[tsID], ts.Samples...)
}
rs.Samples += len(ts.Samples)
if len(ts.Exemplars) > 0 {
c.receivedExemplars[tsID] = append(c.receivedExemplars[tsID], ts.Exemplars...)
}
rs.Exemplars += len(ts.Exemplars)
for _, h := range ts.Histograms {
if h.IsFloatHistogram() {
@ -1182,13 +1176,14 @@ func (c *TestWriteClient) Store(_ context.Context, req []byte, _ int) error {
c.receivedHistograms[tsID] = append(c.receivedHistograms[tsID], h)
}
}
rs.Histograms += len(ts.Histograms)
}
for _, m := range reqProto.Metadata {
c.receivedMetadata[m.MetricFamilyName] = append(c.receivedMetadata[m.MetricFamilyName], m)
}
c.writesReceived++
return nil
return rs, nil
}
func (c *TestWriteClient) Name() string {
@ -1256,10 +1251,10 @@ func NewTestBlockedWriteClient() *TestBlockingWriteClient {
return &TestBlockingWriteClient{}
}
func (c *TestBlockingWriteClient) Store(ctx context.Context, _ []byte, _ int) error {
func (c *TestBlockingWriteClient) Store(ctx context.Context, _ []byte, _ int) (WriteResponseStats, error) {
c.numCalls.Inc()
<-ctx.Done()
return nil
return WriteResponseStats{}, nil
}
func (c *TestBlockingWriteClient) NumCalls() uint64 {
@ -1278,19 +1273,19 @@ func (c *TestBlockingWriteClient) Endpoint() string {
type NopWriteClient struct{}
func NewNopWriteClient() *NopWriteClient { return &NopWriteClient{} }
func (c *NopWriteClient) Store(context.Context, []byte, int) error {
return nil
func (c *NopWriteClient) Store(context.Context, []byte, int) (WriteResponseStats, error) {
return WriteResponseStats{}, nil
}
func (c *NopWriteClient) Name() string { return "nopwriteclient" }
func (c *NopWriteClient) Endpoint() string { return "http://test-remote.com/1234" }
type MockWriteClient struct {
StoreFunc func(context.Context, []byte, int) error
StoreFunc func(context.Context, []byte, int) (WriteResponseStats, error)
NameFunc func() string
EndpointFunc func() string
}
func (c *MockWriteClient) Store(ctx context.Context, bb []byte, n int) error {
func (c *MockWriteClient) Store(ctx context.Context, bb []byte, n int) (WriteResponseStats, error) {
return c.StoreFunc(ctx, bb, n)
}
func (c *MockWriteClient) Name() string { return c.NameFunc() }

53
storage/remote/read_handler.go
View file

@ -202,16 +202,34 @@ func (h *readHandler) remoteReadStreamedXORChunks(ctx context.Context, w http.Re
return err
}
chunks := h.getChunkSeriesSet(ctx, query, filteredMatchers)
if err := chunks.Err(); err != nil {
querier, err := h.queryable.ChunkQuerier(query.StartTimestampMs, query.EndTimestampMs)
if err != nil {
return err
}
defer func() {
if err := querier.Close(); err != nil {
level.Warn(h.logger).Log("msg", "Error on chunk querier close", "err", err.Error())
}
}()
var hints *storage.SelectHints
if query.Hints != nil {
hints = &storage.SelectHints{
Start: query.Hints.StartMs,
End: query.Hints.EndMs,
Step: query.Hints.StepMs,
Func: query.Hints.Func,
Grouping: query.Hints.Grouping,
Range: query.Hints.RangeMs,
By: query.Hints.By,
}
}
ws, err := StreamChunkedReadResponses(
NewChunkedWriter(w, f),
int64(i),
// The streaming API has to provide the series sorted.
chunks,
querier.Select(ctx, true, hints, filteredMatchers...),
sortedExternalLabels,
h.remoteReadMaxBytesInFrame,
h.marshalPool,
@ -236,35 +254,6 @@ func (h *readHandler) remoteReadStreamedXORChunks(ctx context.Context, w http.Re
}
}
// getChunkSeriesSet executes a query to retrieve a ChunkSeriesSet,
// encapsulating the operation in its own function to ensure timely release of
// the querier resources.
func (h *readHandler) getChunkSeriesSet(ctx context.Context, query *prompb.Query, filteredMatchers []*labels.Matcher) storage.ChunkSeriesSet {
querier, err := h.queryable.ChunkQuerier(query.StartTimestampMs, query.EndTimestampMs)
if err != nil {
return storage.ErrChunkSeriesSet(err)
}
defer func() {
if err := querier.Close(); err != nil {
level.Warn(h.logger).Log("msg", "Error on chunk querier close", "err", err.Error())
}
}()
var hints *storage.SelectHints
if query.Hints != nil {
hints = &storage.SelectHints{
Start: query.Hints.StartMs,
End: query.Hints.EndMs,
Step: query.Hints.StepMs,
Func: query.Hints.Func,
Grouping: query.Hints.Grouping,
Range: query.Hints.RangeMs,
By: query.Hints.By,
}
}
return querier.Select(ctx, true, hints, filteredMatchers...)
}
// filterExtLabelsFromMatchers change equality matchers which match external labels
// to a matcher that looks for an empty label,
// as that label should not be present in the storage.

107
storage/remote/stats.go Normal file
View file

@ -0,0 +1,107 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"errors"
"net/http"
"strconv"
)
const (
rw20WrittenSamplesHeader = "X-Prometheus-Remote-Write-Samples-Written"
rw20WrittenHistogramsHeader = "X-Prometheus-Remote-Write-Histograms-Written"
rw20WrittenExemplarsHeader = "X-Prometheus-Remote-Write-Exemplars-Written"
)
// WriteResponseStats represents the response write statistics specified in https://github.com/prometheus/docs/pull/2486
type WriteResponseStats struct {
// Samples represents X-Prometheus-Remote-Write-Written-Samples
Samples int
// Histograms represents X-Prometheus-Remote-Write-Written-Histograms
Histograms int
// Exemplars represents X-Prometheus-Remote-Write-Written-Exemplars
Exemplars int
// Confirmed means we can trust those statistics from the point of view
// of the PRW 2.0 spec. When parsed from headers, it means we got at least one
// response header from the Receiver to confirm those numbers, meaning it must
// be a at least 2.0 Receiver. See ParseWriteResponseStats for details.
Confirmed bool
}
// NoDataWritten returns true if statistics indicate no data was written.
func (s WriteResponseStats) NoDataWritten() bool {
return (s.Samples + s.Histograms + s.Exemplars) == 0
}
// AllSamples returns both float and histogram sample numbers.
func (s WriteResponseStats) AllSamples() int {
return s.Samples + s.Histograms
}
// Add returns the sum of this WriteResponseStats plus the given WriteResponseStats.
func (s WriteResponseStats) Add(rs WriteResponseStats) WriteResponseStats {
s.Confirmed = rs.Confirmed
s.Samples += rs.Samples
s.Histograms += rs.Histograms
s.Exemplars += rs.Exemplars
return s
}
// SetHeaders sets response headers in a given response writer.
// Make sure to use it before http.ResponseWriter.WriteHeader and .Write.
func (s WriteResponseStats) SetHeaders(w http.ResponseWriter) {
h := w.Header()
h.Set(rw20WrittenSamplesHeader, strconv.Itoa(s.Samples))
h.Set(rw20WrittenHistogramsHeader, strconv.Itoa(s.Histograms))
h.Set(rw20WrittenExemplarsHeader, strconv.Itoa(s.Exemplars))
}
// ParseWriteResponseStats returns WriteResponseStats parsed from the response headers.
//
// As per 2.0 spec, missing header means 0. However, abrupt HTTP errors, 1.0 Receivers
// or buggy 2.0 Receivers might result in no response headers specified and that
// might NOT necessarily mean nothing was written. To represent that we set
// s.Confirmed = true only when see at least on response header.
//
// Error is returned when any of the header fails to parse as int64.
func ParseWriteResponseStats(r *http.Response) (s WriteResponseStats, err error) {
var (
errs []error
h = r.Header
)
if v := h.Get(rw20WrittenSamplesHeader); v != "" { // Empty means zero.
s.Confirmed = true
if s.Samples, err = strconv.Atoi(v); err != nil {
s.Samples = 0
errs = append(errs, err)
}
}
if v := h.Get(rw20WrittenHistogramsHeader); v != "" { // Empty means zero.
s.Confirmed = true
if s.Histograms, err = strconv.Atoi(v); err != nil {
s.Histograms = 0
errs = append(errs, err)
}
}
if v := h.Get(rw20WrittenExemplarsHeader); v != "" { // Empty means zero.
s.Confirmed = true
if s.Exemplars, err = strconv.Atoi(v); err != nil {
s.Exemplars = 0
errs = append(errs, err)
}
}
return s, errors.Join(errs...)
}

1
storage/remote/write.go
View file

@ -176,6 +176,7 @@ func (rws *WriteStorage) ApplyConfig(conf *config.Config) error {
HTTPClientConfig: rwConf.HTTPClientConfig,
SigV4Config: rwConf.SigV4Config,
AzureADConfig: rwConf.AzureADConfig,
GoogleIAMConfig: rwConf.GoogleIAMConfig,
Headers: rwConf.Headers,
RetryOnRateLimit: rwConf.QueueConfig.RetryOnRateLimit,
})

67
storage/remote/write_handler.go
View file

@ -19,7 +19,6 @@ import (
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
@ -201,7 +200,7 @@ func (h *writeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
respStats, errHTTPCode, err := h.writeV2(r.Context(), &req)
// Set required X-Prometheus-Remote-Write-Written-* response headers, in all cases.
respStats.SetResponseHeaders(w.Header())
respStats.SetHeaders(w)
if err != nil {
if errHTTPCode/5 == 100 { // 5xx
@ -318,24 +317,6 @@ func (h *writeHandler) appendV1Histograms(app storage.Appender, hh []prompb.Hist
return nil
}
const (
prw20WrittenSamplesHeader = "X-Prometheus-Remote-Write-Written-Samples"
rw20WrittenHistogramsHeader = "X-Prometheus-Remote-Write-Written-Histograms"
rw20WrittenExemplarsHeader = "X-Prometheus-Remote-Write-Written-Exemplars"
)
type responseStats struct {
samples int
histograms int
exemplars int
}
func (s responseStats) SetResponseHeaders(h http.Header) {
h.Set(prw20WrittenSamplesHeader, strconv.Itoa(s.samples))
h.Set(rw20WrittenHistogramsHeader, strconv.Itoa(s.histograms))
h.Set(rw20WrittenExemplarsHeader, strconv.Itoa(s.exemplars))
}
// writeV2 is similar to write, but it works with v2 proto message,
// allows partial 4xx writes and gathers statistics.
//
@ -345,14 +326,14 @@ func (s responseStats) SetResponseHeaders(h http.Header) {
//
// NOTE(bwplotka): TSDB storage is NOT idempotent, so we don't allow "partial retry-able" errors.
// Once we have 5xx type of error, we immediately stop and rollback all appends.
func (h *writeHandler) writeV2(ctx context.Context, req *writev2.Request) (_ responseStats, errHTTPCode int, _ error) {
func (h *writeHandler) writeV2(ctx context.Context, req *writev2.Request) (_ WriteResponseStats, errHTTPCode int, _ error) {
app := &timeLimitAppender{
Appender: h.appendable.Appender(ctx),
maxTime: timestamp.FromTime(time.Now().Add(maxAheadTime)),
}
rs := responseStats{}
samplesWithoutMetadata, errHTTPCode, err := h.appendV2(app, req, &rs)
s := WriteResponseStats{}
samplesWithoutMetadata, errHTTPCode, err := h.appendV2(app, req, &s)
if err != nil {
if errHTTPCode/5 == 100 {
// On 5xx, we always rollback, because we expect
@ -360,29 +341,29 @@ func (h *writeHandler) writeV2(ctx context.Context, req *writev2.Request) (_ res
if rerr := app.Rollback(); rerr != nil {
level.Error(h.logger).Log("msg", "writev2 rollback failed on retry-able error", "err", rerr)
}
return responseStats{}, errHTTPCode, err
return WriteResponseStats{}, errHTTPCode, err
}
// Non-retriable (e.g. bad request error case). Can be partially written.
commitErr := app.Commit()
if commitErr != nil {
// Bad requests does not matter as we have internal error (retryable).
return responseStats{}, http.StatusInternalServerError, commitErr
return WriteResponseStats{}, http.StatusInternalServerError, commitErr
}
// Bad request error happened, but rest of data (if any) was written.
h.samplesAppendedWithoutMetadata.Add(float64(samplesWithoutMetadata))
return rs, errHTTPCode, err
return s, errHTTPCode, err
}
// All good just commit.
if err := app.Commit(); err != nil {
return responseStats{}, http.StatusInternalServerError, err
return WriteResponseStats{}, http.StatusInternalServerError, err
}
h.samplesAppendedWithoutMetadata.Add(float64(samplesWithoutMetadata))
return rs, 0, nil
return s, 0, nil
}
func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs *responseStats) (samplesWithoutMetadata, errHTTPCode int, err error) {
func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs *WriteResponseStats) (samplesWithoutMetadata, errHTTPCode int, err error) {
var (
badRequestErrs []error
outOfOrderExemplarErrs, samplesWithInvalidLabels int
@ -400,14 +381,14 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs *
continue
}
allSamplesSoFar := rs.samples + rs.histograms
allSamplesSoFar := rs.AllSamples()
var ref storage.SeriesRef
// Samples.
for _, s := range ts.Samples {
ref, err = app.Append(ref, ls, s.GetTimestamp(), s.GetValue())
if err == nil {
rs.samples++
rs.Samples++
continue
}
// Handle append error.
@ -431,7 +412,7 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs *
ref, err = app.AppendHistogram(ref, ls, hp.Timestamp, hp.ToIntHistogram(), nil)
}
if err == nil {
rs.histograms++
rs.Histograms++
continue
}
// Handle append error.
@ -453,18 +434,19 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs *
e := ep.ToExemplar(&b, req.Symbols)
ref, err = app.AppendExemplar(ref, ls, e)
if err == nil {
rs.exemplars++
rs.Exemplars++
continue
}
// Handle append error.
// TODO(bwplotka): I left the logic as in v1, but we might want to make it consistent with samples and histograms.
// Since exemplar storage is still experimental, we don't fail in anyway, the request on ingestion errors.
if errors.Is(err, storage.ErrOutOfOrderExemplar) {
outOfOrderExemplarErrs++
level.Debug(h.logger).Log("msg", "Out of order exemplar", "err", err.Error(), "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e))
outOfOrderExemplarErrs++ // Maintain old metrics, but technically not needed, given we fail here.
level.Error(h.logger).Log("msg", "Out of order exemplar", "err", err.Error(), "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e))
badRequestErrs = append(badRequestErrs, fmt.Errorf("%w for series %v", err, ls.String()))
continue
}
level.Debug(h.logger).Log("msg", "Error while adding exemplar in AppendExemplar", "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e), "err", err)
// TODO(bwplotka): Add strict mode which would trigger rollback of everything if needed.
// For now we keep the previously released flow (just error not debug leve) of dropping them without rollback and 5xx.
level.Error(h.logger).Log("msg", "failed to ingest exemplar, emitting error log, but no error for PRW caller", "err", err.Error(), "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e))
}
m := ts.ToMetadata(req.Symbols)
@ -472,7 +454,7 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs *
level.Debug(h.logger).Log("msg", "error while updating metadata from remote write", "err", err)
// Metadata is attached to each series, so since Prometheus does not reject sample without metadata information,
// we don't report remote write error either. We increment metric instead.
samplesWithoutMetadata += (rs.samples + rs.histograms) - allSamplesSoFar
samplesWithoutMetadata += rs.AllSamples() - allSamplesSoFar
}
}
@ -490,7 +472,7 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs *
// NewOTLPWriteHandler creates a http.Handler that accepts OTLP write requests and
// writes them to the provided appendable.
func NewOTLPWriteHandler(logger log.Logger, appendable storage.Appendable) http.Handler {
func NewOTLPWriteHandler(logger log.Logger, appendable storage.Appendable, configFunc func() config.Config) http.Handler {
rwHandler := &writeHandler{
logger: logger,
appendable: appendable,
@ -499,12 +481,14 @@ func NewOTLPWriteHandler(logger log.Logger, appendable storage.Appendable) http.
return &otlpWriteHandler{
logger: logger,
rwHandler: rwHandler,
configFunc: configFunc,
}
}
type otlpWriteHandler struct {
logger log.Logger
rwHandler *writeHandler
configFunc func() config.Config
}
func (h *otlpWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -515,9 +499,12 @@ func (h *otlpWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
otlpCfg := h.configFunc().OTLPConfig
converter := otlptranslator.NewPrometheusConverter()
if err := converter.FromMetrics(req.Metrics(), otlptranslator.Settings{
AddMetricSuffixes: true,
PromoteResourceAttributes: otlpCfg.PromoteResourceAttributes,
}); err != nil {
level.Warn(h.logger).Log("msg", "Error translating OTLP metrics to Prometheus write request", "err", err)
}

16
storage/remote/write_handler_test.go
View file

@ -398,7 +398,7 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
{
desc: "Partial write; skipped exemplar; exemplar storage errs are noop",
input: writeV2RequestFixture.Timeseries,
appendExemplarErr: errors.New("some exemplar append error"),
appendExemplarErr: errors.New("some exemplar internal append error"),
expectedCode: http.StatusNoContent,
},
@ -449,9 +449,9 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
if tc.expectedCode == http.StatusInternalServerError {
// We don't expect writes for partial writes with retry-able code.
expectHeaderValue(t, 0, resp.Header.Get("X-Prometheus-Remote-Write-Written-Samples"))
expectHeaderValue(t, 0, resp.Header.Get("X-Prometheus-Remote-Write-Written-Histograms"))
expectHeaderValue(t, 0, resp.Header.Get("X-Prometheus-Remote-Write-Written-Exemplars"))
expectHeaderValue(t, 0, resp.Header.Get(rw20WrittenSamplesHeader))
expectHeaderValue(t, 0, resp.Header.Get(rw20WrittenHistogramsHeader))
expectHeaderValue(t, 0, resp.Header.Get(rw20WrittenExemplarsHeader))
require.Empty(t, len(appendable.samples))
require.Empty(t, len(appendable.histograms))
@ -462,12 +462,12 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
// Double check mandatory 2.0 stats.
// writeV2RequestFixture has 2 series with 1 sample, 2 histograms, 1 exemplar each.
expectHeaderValue(t, 2, resp.Header.Get("X-Prometheus-Remote-Write-Written-Samples"))
expectHeaderValue(t, 4, resp.Header.Get("X-Prometheus-Remote-Write-Written-Histograms"))
expectHeaderValue(t, 2, resp.Header.Get(rw20WrittenSamplesHeader))
expectHeaderValue(t, 4, resp.Header.Get(rw20WrittenHistogramsHeader))
if tc.appendExemplarErr != nil {
expectHeaderValue(t, 0, resp.Header.Get("X-Prometheus-Remote-Write-Written-Exemplars"))
expectHeaderValue(t, 0, resp.Header.Get(rw20WrittenExemplarsHeader))
} else {
expectHeaderValue(t, 2, resp.Header.Get("X-Prometheus-Remote-Write-Written-Exemplars"))
expectHeaderValue(t, 2, resp.Header.Get(rw20WrittenExemplarsHeader))
}
// Double check what was actually appended.

6
storage/remote/write_test.go
View file

@ -379,7 +379,11 @@ func TestOTLPWriteHandler(t *testing.T) {
req.Header.Set("Content-Type", "application/x-protobuf")
appendable := &mockAppendable{}
handler := NewOTLPWriteHandler(nil, appendable)
handler := NewOTLPWriteHandler(nil, appendable, func() config.Config {
return config.Config{
OTLPConfig: config.DefaultOTLPConfig,
}
})
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, req)

183
storage/series_test.go
View file

@ -126,6 +126,7 @@ func TestChunkSeriesSetToSeriesSet(t *testing.T) {
type histogramTest struct {
samples []chunks.Sample
expectedSamples []chunks.Sample
expectedCounterResetHeaders []chunkenc.CounterResetHeader
}
@ -141,6 +142,32 @@ func TestHistogramSeriesToChunks(t *testing.T) {
},
PositiveBuckets: []int64{2, 1}, // Abs: 2, 3
}
// h1 but with an extra empty bucket at offset -10.
// This can happen if h1 is from a recoded chunk, where a later histogram had a bucket at offset -10.
h1ExtraBuckets := &histogram.Histogram{
Count: 7,
ZeroCount: 2,
ZeroThreshold: 0.001,
Sum: 100,
Schema: 0,
PositiveSpans: []histogram.Span{
{Offset: -10, Length: 1},
{Offset: 9, Length: 2},
},
PositiveBuckets: []int64{0, 2, 1}, // Abs: 0, 2, 3
}
h1Recoded := &histogram.Histogram{
Count: 7,
ZeroCount: 2,
ZeroThreshold: 0.001,
Sum: 100,
Schema: 0,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{2, 1, -3, 0}, // Abs: 2, 3, 0, 0
}
// Appendable to h1.
h2 := &histogram.Histogram{
Count: 12,
@ -179,6 +206,32 @@ func TestHistogramSeriesToChunks(t *testing.T) {
},
PositiveBuckets: []float64{3, 1},
}
// fh1 but with an extra empty bucket at offset -10.
// This can happen if fh1 is from a recoded chunk, where a later histogram had a bucket at offset -10.
fh1ExtraBuckets := &histogram.FloatHistogram{
Count: 6,
ZeroCount: 2,
ZeroThreshold: 0.001,
Sum: 100,
Schema: 0,
PositiveSpans: []histogram.Span{
{Offset: -10, Length: 1},
{Offset: 9, Length: 2},
},
PositiveBuckets: []float64{0, 3, 1},
}
fh1Recoded := &histogram.FloatHistogram{
Count: 6,
ZeroCount: 2,
ZeroThreshold: 0.001,
Sum: 100,
Schema: 0,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []float64{3, 1, 0, 0},
}
// Appendable to fh1.
fh2 := &histogram.FloatHistogram{
Count: 17,
@ -219,6 +272,20 @@ func TestHistogramSeriesToChunks(t *testing.T) {
},
PositiveBuckets: []int64{2, 1}, // Abs: 2, 3
}
// gh1 recoded to add extra empty buckets at end.
gh1Recoded := &histogram.Histogram{
CounterResetHint: histogram.GaugeType,
Count: 7,
ZeroCount: 2,
ZeroThreshold: 0.001,
Sum: 100,
Schema: 0,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{2, 1, -3, 0}, // Abs: 2, 3, 0, 0
}
gh2 := &histogram.Histogram{
CounterResetHint: histogram.GaugeType,
Count: 12,
@ -246,6 +313,20 @@ func TestHistogramSeriesToChunks(t *testing.T) {
},
PositiveBuckets: []float64{3, 1},
}
// gfh1 recoded to add an extra empty buckets at end.
gfh1Recoded := &histogram.FloatHistogram{
CounterResetHint: histogram.GaugeType,
Count: 6,
ZeroCount: 2,
ZeroThreshold: 0.001,
Sum: 100,
Schema: 0,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []float64{3, 1, 0, 0},
}
gfh2 := &histogram.FloatHistogram{
CounterResetHint: histogram.GaugeType,
Count: 17,
@ -272,6 +353,9 @@ func TestHistogramSeriesToChunks(t *testing.T) {
samples: []chunks.Sample{
hSample{t: 1, h: h1},
},
expectedSamples: []chunks.Sample{
hSample{t: 1, h: h1},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset},
},
"two histograms encoded to a single chunk": {
@ -279,6 +363,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
hSample{t: 1, h: h1},
hSample{t: 2, h: h2},
},
expectedSamples: []chunks.Sample{
hSample{t: 1, h: h1Recoded},
hSample{t: 2, h: h2},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset},
},
"two histograms encoded to two chunks": {
@ -286,6 +374,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
hSample{t: 1, h: h2},
hSample{t: 2, h: h1},
},
expectedSamples: []chunks.Sample{
hSample{t: 1, h: h2},
hSample{t: 2, h: h1},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.CounterReset},
},
"histogram and stale sample encoded to two chunks": {
@ -293,6 +385,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
hSample{t: 1, h: staleHistogram},
hSample{t: 2, h: h1},
},
expectedSamples: []chunks.Sample{
hSample{t: 1, h: staleHistogram},
hSample{t: 2, h: h1},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.UnknownCounterReset},
},
"histogram and reduction in bucket encoded to two chunks": {
@ -300,6 +396,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
hSample{t: 1, h: h1},
hSample{t: 2, h: h2down},
},
expectedSamples: []chunks.Sample{
hSample{t: 1, h: h1},
hSample{t: 2, h: h2down},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.CounterReset},
},
// Float histograms.
@ -307,6 +407,9 @@ func TestHistogramSeriesToChunks(t *testing.T) {
samples: []chunks.Sample{
fhSample{t: 1, fh: fh1},
},
expectedSamples: []chunks.Sample{
fhSample{t: 1, fh: fh1},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset},
},
"two float histograms encoded to a single chunk": {
@ -314,6 +417,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
fhSample{t: 1, fh: fh1},
fhSample{t: 2, fh: fh2},
},
expectedSamples: []chunks.Sample{
fhSample{t: 1, fh: fh1Recoded},
fhSample{t: 2, fh: fh2},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset},
},
"two float histograms encoded to two chunks": {
@ -321,6 +428,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
fhSample{t: 1, fh: fh2},
fhSample{t: 2, fh: fh1},
},
expectedSamples: []chunks.Sample{
fhSample{t: 1, fh: fh2},
fhSample{t: 2, fh: fh1},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.CounterReset},
},
"float histogram and stale sample encoded to two chunks": {
@ -328,6 +439,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
fhSample{t: 1, fh: staleFloatHistogram},
fhSample{t: 2, fh: fh1},
},
expectedSamples: []chunks.Sample{
fhSample{t: 1, fh: staleFloatHistogram},
fhSample{t: 2, fh: fh1},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.UnknownCounterReset},
},
"float histogram and reduction in bucket encoded to two chunks": {
@ -335,6 +450,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
fhSample{t: 1, fh: fh1},
fhSample{t: 2, fh: fh2down},
},
expectedSamples: []chunks.Sample{
fhSample{t: 1, fh: fh1},
fhSample{t: 2, fh: fh2down},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.CounterReset},
},
// Mixed.
@ -343,6 +462,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
hSample{t: 1, h: h1},
fhSample{t: 2, fh: fh2},
},
expectedSamples: []chunks.Sample{
hSample{t: 1, h: h1},
fhSample{t: 2, fh: fh2},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.UnknownCounterReset},
},
"float histogram and histogram encoded to two chunks": {
@ -350,6 +473,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
fhSample{t: 1, fh: fh1},
hSample{t: 2, h: h2},
},
expectedSamples: []chunks.Sample{
fhSample{t: 1, fh: fh1},
hSample{t: 2, h: h2},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.UnknownCounterReset},
},
"histogram and stale float histogram encoded to two chunks": {
@ -357,12 +484,19 @@ func TestHistogramSeriesToChunks(t *testing.T) {
hSample{t: 1, h: h1},
fhSample{t: 2, fh: staleFloatHistogram},
},
expectedSamples: []chunks.Sample{
hSample{t: 1, h: h1},
fhSample{t: 2, fh: staleFloatHistogram},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset, chunkenc.UnknownCounterReset},
},
"single gauge histogram encoded to one chunk": {
samples: []chunks.Sample{
hSample{t: 1, h: gh1},
},
expectedSamples: []chunks.Sample{
hSample{t: 1, h: gh1},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.GaugeType},
},
"two gauge histograms encoded to one chunk when counter increases": {
@ -370,6 +504,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
hSample{t: 1, h: gh1},
hSample{t: 2, h: gh2},
},
expectedSamples: []chunks.Sample{
hSample{t: 1, h: gh1Recoded},
hSample{t: 2, h: gh2},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.GaugeType},
},
"two gauge histograms encoded to one chunk when counter decreases": {
@ -377,12 +515,19 @@ func TestHistogramSeriesToChunks(t *testing.T) {
hSample{t: 1, h: gh2},
hSample{t: 2, h: gh1},
},
expectedSamples: []chunks.Sample{
hSample{t: 1, h: gh2},
hSample{t: 2, h: gh1Recoded},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.GaugeType},
},
"single gauge float histogram encoded to one chunk": {
samples: []chunks.Sample{
fhSample{t: 1, fh: gfh1},
},
expectedSamples: []chunks.Sample{
fhSample{t: 1, fh: gfh1},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.GaugeType},
},
"two float gauge histograms encoded to one chunk when counter increases": {
@ -390,6 +535,10 @@ func TestHistogramSeriesToChunks(t *testing.T) {
fhSample{t: 1, fh: gfh1},
fhSample{t: 2, fh: gfh2},
},
expectedSamples: []chunks.Sample{
fhSample{t: 1, fh: gfh1Recoded},
fhSample{t: 2, fh: gfh2},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.GaugeType},
},
"two float gauge histograms encoded to one chunk when counter decreases": {
@ -397,8 +546,34 @@ func TestHistogramSeriesToChunks(t *testing.T) {
fhSample{t: 1, fh: gfh2},
fhSample{t: 2, fh: gfh1},
},
expectedSamples: []chunks.Sample{
fhSample{t: 1, fh: gfh2},
fhSample{t: 2, fh: gfh1Recoded},
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.GaugeType},
},
"histogram with extra empty bucket followed by histogram encodes to one chunk": {
samples: []chunks.Sample{
hSample{t: 1, h: h1ExtraBuckets},
hSample{t: 2, h: h1},
},
expectedSamples: []chunks.Sample{
hSample{t: 1, h: h1ExtraBuckets},
hSample{t: 2, h: h1ExtraBuckets}, // Recoded to add the missing buckets.
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset},
},
"float histogram with extra empty bucket followed by float histogram encodes to one chunk": {
samples: []chunks.Sample{
fhSample{t: 1, fh: fh1ExtraBuckets},
fhSample{t: 2, fh: fh1},
},
expectedSamples: []chunks.Sample{
fhSample{t: 1, fh: fh1ExtraBuckets},
fhSample{t: 2, fh: fh1ExtraBuckets}, // Recoded to add the missing buckets.
},
expectedCounterResetHeaders: []chunkenc.CounterResetHeader{chunkenc.UnknownCounterReset},
},
}
for testName, test := range tests {
@ -431,9 +606,9 @@ func testHistogramsSeriesToChunks(t *testing.T, test histogramTest) {
// Decode all encoded samples and assert they are equal to the original ones.
encodedSamples := chunks.ChunkMetasToSamples(chks)
require.Equal(t, len(test.samples), len(encodedSamples))
require.Equal(t, len(test.expectedSamples), len(encodedSamples))
for i, s := range test.samples {
for i, s := range test.expectedSamples {
encodedSample := encodedSamples[i]
switch expectedSample := s.(type) {
case hSample:
@ -447,7 +622,7 @@ func testHistogramsSeriesToChunks(t *testing.T, test histogramTest) {
require.True(t, value.IsStaleNaN(h.Sum), fmt.Sprintf("at idx %d", i))
continue
}
require.Equal(t, *expectedSample.h, *h.Compact(0), fmt.Sprintf("at idx %d", i))
require.Equal(t, *expectedSample.h, *h, fmt.Sprintf("at idx %d", i))
case fhSample:
require.Equal(t, chunkenc.ValFloatHistogram, encodedSample.Type(), "expect float histogram", fmt.Sprintf("at idx %d", i))
fh := encodedSample.FH()
@ -459,7 +634,7 @@ func testHistogramsSeriesToChunks(t *testing.T, test histogramTest) {
require.True(t, value.IsStaleNaN(fh.Sum), fmt.Sprintf("at idx %d", i))
continue
}
require.Equal(t, *expectedSample.fh, *fh.Compact(0), fmt.Sprintf("at idx %d", i))
require.Equal(t, *expectedSample.fh, *fh, fmt.Sprintf("at idx %d", i))
default:
t.Error("internal error, unexpected type")
}

2
tracing/tracing.go
View file

@ -30,7 +30,7 @@ import (
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
tracesdk "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.25.0"
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/noop"
"google.golang.org/grpc/credentials"

304
tsdb/chunkenc/float_histogram.go
View file

@ -219,16 +219,25 @@ func (a *FloatHistogramAppender) Append(int64, float64) {
}
// appendable returns whether the chunk can be appended to, and if so whether
// any recoding needs to happen using the provided inserts (in case of any new
// buckets, positive or negative range, respectively). If the sample is a gauge
// histogram, AppendableGauge must be used instead.
// 1. Any recoding needs to happen to the chunk using the provided forward
// inserts (in case of any new buckets, positive or negative range,
// respectively).
// 2. Any recoding needs to happen for the histogram being appended, using the
// backward inserts (in case of any missing buckets, positive or negative
// range, respectively).
//
// If the sample is a gauge histogram, AppendableGauge must be used instead.
//
// The chunk is not appendable in the following cases:
//
// - The schema has changed.
// - The custom bounds have changed if the current schema is custom buckets.
// - The threshold for the zero bucket has changed.
// - Any buckets have disappeared.
// - There was a counter reset in the count of observations or in any bucket, including the zero bucket.
// - Any buckets have disappeared, unless the bucket count was 0, unused.
// Empty bucket can happen if the chunk was recoded and we're merging a non
// recoded histogram. In this case backward inserts will be provided.
// - There was a counter reset in the count of observations or in any bucket,
// including the zero bucket.
// - The last sample in the chunk was stale while the current sample is not stale.
//
// The method returns an additional boolean set to true if it is not appendable
@ -236,6 +245,7 @@ func (a *FloatHistogramAppender) Append(int64, float64) {
// append. If counterReset is true, okToAppend is always false.
func (a *FloatHistogramAppender) appendable(h *histogram.FloatHistogram) (
positiveInserts, negativeInserts []Insert,
backwardPositiveInserts, backwardNegativeInserts []Insert,
okToAppend, counterReset bool,
) {
if a.NumSamples() > 0 && a.GetCounterResetHeader() == GaugeType {
@ -279,27 +289,214 @@ func (a *FloatHistogramAppender) appendable(h *histogram.FloatHistogram) (
}
var ok bool
positiveInserts, ok = expandSpansForward(a.pSpans, h.PositiveSpans)
positiveInserts, backwardPositiveInserts, ok = expandFloatSpansAndBuckets(a.pSpans, h.PositiveSpans, a.pBuckets, h.PositiveBuckets)
if !ok {
counterReset = true
return
}
negativeInserts, ok = expandSpansForward(a.nSpans, h.NegativeSpans)
negativeInserts, backwardNegativeInserts, ok = expandFloatSpansAndBuckets(a.nSpans, h.NegativeSpans, a.nBuckets, h.NegativeBuckets)
if !ok {
counterReset = true
return
}
if counterResetInAnyFloatBucket(a.pBuckets, h.PositiveBuckets, a.pSpans, h.PositiveSpans) ||
counterResetInAnyFloatBucket(a.nBuckets, h.NegativeBuckets, a.nSpans, h.NegativeSpans) {
counterReset, positiveInserts, negativeInserts = true, nil, nil
return
}
okToAppend = true
return
}
// expandFloatSpansAndBuckets returns the inserts to expand the bucket spans 'a' so that
// they match the spans in 'b'. 'b' must cover the same or more buckets than
// 'a', otherwise the function will return false.
// The function also returns the inserts to expand 'b' to also cover all the
// buckets that are missing in 'b', but are present with 0 counter value in 'a'.
// The function also checks for counter resets between 'a' and 'b'.
//
// Example:
//
// Let's say the old buckets look like this:
//
// span syntax: [offset, length]
// spans : [ 0 , 2 ] [2,1] [ 3 , 2 ] [3,1] [1,1]
// bucket idx : [0] [1] 2 3 [4] 5 6 7 [8] [9] 10 11 12 [13] 14 [15]
// raw values 6 3 3 2 4 5 1
// deltas 6 -3 0 -1 2 1 -4
//
// But now we introduce a new bucket layout. (Carefully chosen example where we
// have a span appended, one unchanged[*], one prepended, and two merge - in
// that order.)
//
// [*] unchanged in terms of which bucket indices they represent. but to achieve
// that, their offset needs to change if "disrupted" by spans changing ahead of
// them
//
// \/ this one is "unchanged"
// spans : [ 0 , 3 ] [1,1] [ 1 , 4 ] [ 3 , 3 ]
// bucket idx : [0] [1] [2] 3 [4] 5 [6] [7] [8] [9] 10 11 12 [13] [14] [15]
// raw values 6 3 0 3 0 0 2 4 5 0 1
// deltas 6 -3 -3 3 -3 0 2 2 1 -5 1
// delta mods: / \ / \ / \
//
// Note for histograms with delta-encoded buckets: Whenever any new buckets are
// introduced, the subsequent "old" bucket needs to readjust its delta to the
// new base of 0. Thus, for the caller who wants to transform the set of
// original deltas to a new set of deltas to match a new span layout that adds
// buckets, we simply need to generate a list of inserts.
//
// Note: Within expandSpansForward we don't have to worry about the changes to the
// spans themselves, thanks to the iterators we get to work with the more useful
// bucket indices (which of course directly correspond to the buckets we have to
// adjust).
func expandFloatSpansAndBuckets(a, b []histogram.Span, aBuckets []xorValue, bBuckets []float64) (forward, backward []Insert, ok bool) {
ai := newBucketIterator(a)
bi := newBucketIterator(b)
var aInserts []Insert // To insert into buckets of a, to make up for missing buckets in b.
var bInserts []Insert // To insert into buckets of b, to make up for missing empty(!) buckets in a.
// When aInter.num or bInter.num becomes > 0, this becomes a valid insert that should
// be yielded when we finish a streak of new buckets.
var aInter Insert
var bInter Insert
aIdx, aOK := ai.Next()
bIdx, bOK := bi.Next()
// Bucket count. Initialize the absolute count and index into the
// positive/negative counts or deltas array. The bucket count is
// used to detect counter reset as well as unused buckets in a.
var (
aCount float64
bCount float64
aCountIdx int
bCountIdx int
)
if aOK {
aCount = aBuckets[aCountIdx].value
}
if bOK {
bCount = bBuckets[bCountIdx]
}
loop:
for {
switch {
case aOK && bOK:
switch {
case aIdx == bIdx: // Both have an identical bucket index.
// Bucket count. Check bucket for reset from a to b.
if aCount > bCount {
return nil, nil, false
}
// Finish WIP insert for a and reset.
if aInter.num > 0 {
aInserts = append(aInserts, aInter)
aInter.num = 0
}
// Finish WIP insert for b and reset.
if bInter.num > 0 {
bInserts = append(bInserts, bInter)
bInter.num = 0
}
aIdx, aOK = ai.Next()
bIdx, bOK = bi.Next()
aInter.pos++ // Advance potential insert position.
aCountIdx++ // Advance absolute bucket count index for a.
if aOK {
aCount = aBuckets[aCountIdx].value
}
bInter.pos++ // Advance potential insert position.
bCountIdx++ // Advance absolute bucket count index for b.
if bOK {
bCount = bBuckets[bCountIdx]
}
continue
case aIdx < bIdx: // b misses a bucket index that is in a.
// This is ok if the count in a is 0, in which case we make a note to
// fill in the bucket in b and advance a.
if aCount == 0 {
bInter.num++ // Mark that we need to insert a bucket in b.
// Advance a
if aInter.num > 0 {
aInserts = append(aInserts, aInter)
aInter.num = 0
}
aIdx, aOK = ai.Next()
aInter.pos++
aCountIdx++
if aOK {
aCount = aBuckets[aCountIdx].value
}
continue
}
// Otherwise we are missing a bucket that was in use in a, which is a reset.
return nil, nil, false
case aIdx > bIdx: // a misses a value that is in b. Forward b and recompare.
aInter.num++
// Advance b
if bInter.num > 0 {
bInserts = append(bInserts, bInter)
bInter.num = 0
}
bIdx, bOK = bi.Next()
bInter.pos++
bCountIdx++
if bOK {
bCount = bBuckets[bCountIdx]
}
}
case aOK && !bOK: // b misses a value that is in a.
// This is ok if the count in a is 0, in which case we make a note to
// fill in the bucket in b and advance a.
if aCount == 0 {
bInter.num++
// Advance a
if aInter.num > 0 {
aInserts = append(aInserts, aInter)
aInter.num = 0
}
aIdx, aOK = ai.Next()
aInter.pos++ // Advance potential insert position.
// Update absolute bucket counts for a.
aCountIdx++
if aOK {
aCount = aBuckets[aCountIdx].value
}
continue
}
// Otherwise we are missing a bucket that was in use in a, which is a reset.
return nil, nil, false
case !aOK && bOK: // a misses a value that is in b. Forward b and recompare.
aInter.num++
// Advance b
if bInter.num > 0 {
bInserts = append(bInserts, bInter)
bInter.num = 0
}
bIdx, bOK = bi.Next()
bInter.pos++ // Advance potential insert position.
// Update absolute bucket counts for b.
bCountIdx++
if bOK {
bCount = bBuckets[bCountIdx]
}
default: // Both iterators ran out. We're done.
if aInter.num > 0 {
aInserts = append(aInserts, aInter)
}
if bInter.num > 0 {
bInserts = append(bInserts, bInter)
}
break loop
}
}
return aInserts, bInserts, true
}
// appendableGauge returns whether the chunk can be appended to, and if so
// whether:
// 1. Any recoding needs to happen to the chunk using the provided inserts
@ -349,76 +546,6 @@ func (a *FloatHistogramAppender) appendableGauge(h *histogram.FloatHistogram) (
return
}
// counterResetInAnyFloatBucket returns true if there was a counter reset for any
// bucket. This should be called only when the bucket layout is the same or new
// buckets were added. It does not handle the case of buckets missing.
func counterResetInAnyFloatBucket(oldBuckets []xorValue, newBuckets []float64, oldSpans, newSpans []histogram.Span) bool {
if len(oldSpans) == 0 || len(oldBuckets) == 0 {
return false
}
var (
oldSpanSliceIdx, newSpanSliceIdx int = -1, -1 // Index for the span slices. Starts at -1 to indicate that the first non empty span is not yet found.
oldInsideSpanIdx, newInsideSpanIdx uint32 // Index inside a span.
oldIdx, newIdx int32 // Index inside a bucket slice.
oldBucketSliceIdx, newBucketSliceIdx int // Index inside bucket slice.
)
// Find first non empty spans.
oldSpanSliceIdx, oldIdx = nextNonEmptySpanSliceIdx(oldSpanSliceIdx, oldIdx, oldSpans)
newSpanSliceIdx, newIdx = nextNonEmptySpanSliceIdx(newSpanSliceIdx, newIdx, newSpans)
oldVal, newVal := oldBuckets[0].value, newBuckets[0]
// Since we assume that new spans won't have missing buckets, there will never be a case
// where the old index will not find a matching new index.
for {
if oldIdx == newIdx {
if newVal < oldVal {
return true
}
}
if oldIdx <= newIdx {
// Moving ahead old bucket and span by 1 index.
if oldInsideSpanIdx+1 >= oldSpans[oldSpanSliceIdx].Length {
// Current span is over.
oldSpanSliceIdx, oldIdx = nextNonEmptySpanSliceIdx(oldSpanSliceIdx, oldIdx, oldSpans)
oldInsideSpanIdx = 0
if oldSpanSliceIdx >= len(oldSpans) {
// All old spans are over.
break
}
} else {
oldInsideSpanIdx++
oldIdx++
}
oldBucketSliceIdx++
oldVal = oldBuckets[oldBucketSliceIdx].value
}
if oldIdx > newIdx {
// Moving ahead new bucket and span by 1 index.
if newInsideSpanIdx+1 >= newSpans[newSpanSliceIdx].Length {
// Current span is over.
newSpanSliceIdx, newIdx = nextNonEmptySpanSliceIdx(newSpanSliceIdx, newIdx, newSpans)
newInsideSpanIdx = 0
if newSpanSliceIdx >= len(newSpans) {
// All new spans are over.
// This should not happen, old spans above should catch this first.
panic("new spans over before old spans in counterReset")
}
} else {
newInsideSpanIdx++
newIdx++
}
newBucketSliceIdx++
newVal = newBuckets[newBucketSliceIdx]
}
}
return false
}
// appendFloatHistogram appends a float histogram to the chunk. The caller must ensure that
// the histogram is properly structured, e.g. the number of buckets used
// corresponds to the number conveyed by the span structures. First call
@ -614,7 +741,7 @@ func (a *FloatHistogramAppender) AppendFloatHistogram(prev *FloatHistogramAppend
a.setCounterResetHeader(CounterReset)
case prev != nil:
// This is a new chunk, but continued from a previous one. We need to calculate the reset header unless already set.
_, _, _, counterReset := prev.appendable(h)
_, _, _, _, _, counterReset := prev.appendable(h)
if counterReset {
a.setCounterResetHeader(CounterReset)
} else {
@ -626,7 +753,7 @@ func (a *FloatHistogramAppender) AppendFloatHistogram(prev *FloatHistogramAppend
// Adding counter-like histogram.
if h.CounterResetHint != histogram.GaugeType {
pForwardInserts, nForwardInserts, okToAppend, counterReset := a.appendable(h)
pForwardInserts, nForwardInserts, pBackwardInserts, nBackwardInserts, okToAppend, counterReset := a.appendable(h)
if !okToAppend || counterReset {
if appendOnly {
if counterReset {
@ -657,6 +784,13 @@ func (a *FloatHistogramAppender) AppendFloatHistogram(prev *FloatHistogramAppend
app.(*FloatHistogramAppender).appendFloatHistogram(t, h)
return chk, true, app, nil
}
if len(pBackwardInserts) > 0 || len(nBackwardInserts) > 0 {
// The histogram needs to be expanded to have the extra empty buckets
// of the chunk.
h.PositiveSpans = a.pSpans
h.NegativeSpans = a.nSpans
a.recodeHistogram(h, pBackwardInserts, nBackwardInserts)
}
a.appendFloatHistogram(t, h)
return nil, false, a, nil
}

106
tsdb/chunkenc/float_histogram_test.go
View file

@ -245,9 +245,11 @@ func TestFloatHistogramChunkBucketChanges(t *testing.T) {
h2.NegativeBuckets = []int64{2, -1} // 2 1 (total 3)
// This is how span changes will be handled.
hApp, _ := app.(*FloatHistogramAppender)
posInterjections, negInterjections, ok, cr := hApp.appendable(h2.ToFloat(nil))
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2.ToFloat(nil))
require.NotEmpty(t, posInterjections)
require.NotEmpty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.True(t, ok) // Only new buckets came in.
require.False(t, cr)
c, app = hApp.recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans)
@ -333,7 +335,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Schema++
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
@ -343,7 +345,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.ZeroThreshold += 0.1
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
@ -363,9 +365,11 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
h2.Sum = 30
h2.PositiveBuckets = []float64{7, 5, 1, 3, 1, 0, 2, 5, 5, 0, 1}
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.NotEmpty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.True(t, ok) // Only new buckets came in.
require.False(t, cr)
@ -385,24 +389,56 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
h2.Sum = 21
h2.PositiveBuckets = []float64{6, 3, 2, 4, 5, 1}
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.False(t, ok) // Need to cut a new chunk.
require.True(t, cr)
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
}
{ // New histogram that has buckets missing but the buckets missing were empty.
emptyBucketH := eh.Copy()
emptyBucketH.PositiveBuckets = []float64{6, 0, 3, 2, 4, 0, 1}
c, hApp, ts, h1 := setup(emptyBucketH)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 1},
{Offset: 3, Length: 1},
{Offset: 3, Length: 2},
{Offset: 5, Length: 1},
}
h2.PositiveBuckets = []float64{7, 4, 3, 5, 2}
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.NotEmpty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.True(t, ok)
require.False(t, cr)
assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
// Check that h2 was recoded.
require.Equal(t, []float64{7, 0, 4, 3, 5, 0, 2}, h2.PositiveBuckets)
require.Equal(t, emptyBucketH.PositiveSpans, h2.PositiveSpans)
}
{ // New histogram that has a counter reset while buckets are same.
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Sum = 23
h2.PositiveBuckets = []float64{6, 2, 3, 2, 4, 5, 1}
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.False(t, ok) // Need to cut a new chunk.
require.True(t, cr)
@ -421,9 +457,11 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
h2.Sum = 29
h2.PositiveBuckets = []float64{7, 5, 1, 3, 1, 0, 2, 5, 5, 0, 0}
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.False(t, ok) // Need to cut a new chunk.
require.True(t, cr)
@ -448,9 +486,11 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
h2.Sum = 26
h2.PositiveBuckets = []float64{1, 2, 5, 3, 3, 2, 4, 5, 1}
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.False(t, ok) // Need to cut a new chunk.
require.True(t, cr)
@ -524,10 +564,44 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader())
}
{
// Start a new chunk with a histogram that has an empty bucket.
// Add a histogram that has the same bucket missing.
// This should be appendable and can happen if we are merging from chunks
// where the first sample came from a recoded chunk that added the
// empty bucket.
h1 := eh.Copy()
// Add a bucket that is empty -10 offsets from the first bucket.
h1.PositiveSpans = make([]histogram.Span, len(eh.PositiveSpans)+1)
h1.PositiveSpans[0] = histogram.Span{Offset: eh.PositiveSpans[0].Offset - 10, Length: 1}
h1.PositiveSpans[1] = histogram.Span{Offset: eh.PositiveSpans[0].Offset + 9, Length: eh.PositiveSpans[0].Length}
for i, v := range eh.PositiveSpans[1:] {
h1.PositiveSpans[i+2] = v
}
h1.PositiveBuckets = make([]float64, len(eh.PositiveBuckets)+1)
h1.PositiveBuckets[0] = 0
for i, v := range eh.PositiveBuckets {
h1.PositiveBuckets[i+1] = v
}
c, hApp, ts, _ := setup(h1)
h2 := eh.Copy()
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.NotEmpty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.True(t, ok)
require.False(t, cr)
assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
}
{ // Custom buckets, no change.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.True(t, ok)
assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
@ -538,7 +612,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
h2 := h1.Copy()
h2.Count++
h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 2}
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.True(t, ok)
assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
@ -549,7 +623,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
h2 := h1.Copy()
h2.Count--
h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 0}
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
@ -559,7 +633,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
@ -581,9 +655,11 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2.PositiveBuckets = []float64{7, 5, 1, 3, 1, 0, 2, 5, 5, 0, 1} // (total 30)
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.NotEmpty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.True(t, ok) // Only new buckets came in.
require.False(t, cr)
@ -839,9 +915,11 @@ func TestFloatHistogramChunkAppendableWithEmptySpan(t *testing.T) {
require.Equal(t, 1, c.NumSamples())
hApp, _ := app.(*FloatHistogramAppender)
pI, nI, okToAppend, counterReset := hApp.appendable(tc.h2)
pI, nI, bpI, bnI, okToAppend, counterReset := hApp.appendable(tc.h2)
require.Empty(t, pI)
require.Empty(t, nI)
require.Empty(t, bpI)
require.Empty(t, bnI)
require.True(t, okToAppend)
require.False(t, counterReset)
})

305
tsdb/chunkenc/histogram.go
View file

@ -237,16 +237,23 @@ func (a *HistogramAppender) Append(int64, float64) {
}
// appendable returns whether the chunk can be appended to, and if so whether
// any recoding needs to happen using the provided inserts (in case of any new
// buckets, positive or negative range, respectively). If the sample is a gauge
// histogram, AppendableGauge must be used instead.
// 1. Any recoding needs to happen to the chunk using the provided forward
// inserts (in case of any new buckets, positive or negative range,
// respectively).
// 2. Any recoding needs to happen for the histogram being appended, using the
// backward inserts (in case of any missing buckets, positive or negative
// range, respectively).
//
// If the sample is a gauge histogram, AppendableGauge must be used instead.
//
// The chunk is not appendable in the following cases:
//
// - The schema has changed.
// - The custom bounds have changed if the current schema is custom buckets.
// - The threshold for the zero bucket has changed.
// - Any buckets have disappeared.
// - Any buckets have disappeared, unless the bucket count was 0, unused.
// Empty bucket can happen if the chunk was recoded and we're merging a non
// recoded histogram. In this case backward inserts will be provided.
// - There was a counter reset in the count of observations or in any bucket,
// including the zero bucket.
// - The last sample in the chunk was stale while the current sample is not stale.
@ -256,6 +263,7 @@ func (a *HistogramAppender) Append(int64, float64) {
// append. If counterReset is true, okToAppend is always false.
func (a *HistogramAppender) appendable(h *histogram.Histogram) (
positiveInserts, negativeInserts []Insert,
backwardPositiveInserts, backwardNegativeInserts []Insert,
okToAppend, counterReset bool,
) {
if a.NumSamples() > 0 && a.GetCounterResetHeader() == GaugeType {
@ -299,31 +307,219 @@ func (a *HistogramAppender) appendable(h *histogram.Histogram) (
}
var ok bool
positiveInserts, ok = expandSpansForward(a.pSpans, h.PositiveSpans)
positiveInserts, backwardPositiveInserts, ok = expandIntSpansAndBuckets(a.pSpans, h.PositiveSpans, a.pBuckets, h.PositiveBuckets)
if !ok {
counterReset = true
return
}
negativeInserts, ok = expandSpansForward(a.nSpans, h.NegativeSpans)
negativeInserts, backwardNegativeInserts, ok = expandIntSpansAndBuckets(a.nSpans, h.NegativeSpans, a.nBuckets, h.NegativeBuckets)
if !ok {
counterReset = true
return
}
if counterResetInAnyBucket(a.pBuckets, h.PositiveBuckets, a.pSpans, h.PositiveSpans) ||
counterResetInAnyBucket(a.nBuckets, h.NegativeBuckets, a.nSpans, h.NegativeSpans) {
counterReset, positiveInserts, negativeInserts = true, nil, nil
return
}
okToAppend = true
return
}
// expandIntSpansAndBuckets returns the inserts to expand the bucket spans 'a' so that
// they match the spans in 'b'. 'b' must cover the same or more buckets than
// 'a', otherwise the function will return false.
// The function also returns the inserts to expand 'b' to also cover all the
// buckets that are missing in 'b', but are present with 0 counter value in 'a'.
// The function also checks for counter resets between 'a' and 'b'.
//
// Example:
//
// Let's say the old buckets look like this:
//
// span syntax: [offset, length]
// spans : [ 0 , 2 ] [2,1] [ 3 , 2 ] [3,1] [1,1]
// bucket idx : [0] [1] 2 3 [4] 5 6 7 [8] [9] 10 11 12 [13] 14 [15]
// raw values 6 3 3 2 4 5 1
// deltas 6 -3 0 -1 2 1 -4
//
// But now we introduce a new bucket layout. (Carefully chosen example where we
// have a span appended, one unchanged[*], one prepended, and two merge - in
// that order.)
//
// [*] unchanged in terms of which bucket indices they represent. but to achieve
// that, their offset needs to change if "disrupted" by spans changing ahead of
// them
//
// \/ this one is "unchanged"
// spans : [ 0 , 3 ] [1,1] [ 1 , 4 ] [ 3 , 3 ]
// bucket idx : [0] [1] [2] 3 [4] 5 [6] [7] [8] [9] 10 11 12 [13] [14] [15]
// raw values 6 3 0 3 0 0 2 4 5 0 1
// deltas 6 -3 -3 3 -3 0 2 2 1 -5 1
// delta mods: / \ / \ / \
//
// Note for histograms with delta-encoded buckets: Whenever any new buckets are
// introduced, the subsequent "old" bucket needs to readjust its delta to the
// new base of 0. Thus, for the caller who wants to transform the set of
// original deltas to a new set of deltas to match a new span layout that adds
// buckets, we simply need to generate a list of inserts.
//
// Note: Within expandSpansForward we don't have to worry about the changes to the
// spans themselves, thanks to the iterators we get to work with the more useful
// bucket indices (which of course directly correspond to the buckets we have to
// adjust).
func expandIntSpansAndBuckets(a, b []histogram.Span, aBuckets, bBuckets []int64) (forward, backward []Insert, ok bool) {
ai := newBucketIterator(a)
bi := newBucketIterator(b)
var aInserts []Insert // To insert into buckets of a, to make up for missing buckets in b.
var bInserts []Insert // To insert into buckets of b, to make up for missing empty(!) buckets in a.
// When aInter.num or bInter.num becomes > 0, this becomes a valid insert that should
// be yielded when we finish a streak of new buckets.
var aInter Insert
var bInter Insert
aIdx, aOK := ai.Next()
bIdx, bOK := bi.Next()
// Bucket count. Initialize the absolute count and index into the
// positive/negative counts or deltas array. The bucket count is
// used to detect counter reset as well as unused buckets in a.
var (
aCount int64
bCount int64
aCountIdx int
bCountIdx int
)
if aOK {
aCount = aBuckets[aCountIdx]
}
if bOK {
bCount = bBuckets[bCountIdx]
}
loop:
for {
switch {
case aOK && bOK:
switch {
case aIdx == bIdx: // Both have an identical bucket index.
// Bucket count. Check bucket for reset from a to b.
if aCount > bCount {
return nil, nil, false
}
// Finish WIP insert for a and reset.
if aInter.num > 0 {
aInserts = append(aInserts, aInter)
aInter.num = 0
}
// Finish WIP insert for b and reset.
if bInter.num > 0 {
bInserts = append(bInserts, bInter)
bInter.num = 0
}
aIdx, aOK = ai.Next()
bIdx, bOK = bi.Next()
aInter.pos++ // Advance potential insert position.
aCountIdx++ // Advance absolute bucket count index for a.
if aOK {
aCount += aBuckets[aCountIdx]
}
bInter.pos++ // Advance potential insert position.
bCountIdx++ // Advance absolute bucket count index for b.
if bOK {
bCount += bBuckets[bCountIdx]
}
continue
case aIdx < bIdx: // b misses a bucket index that is in a.
// This is ok if the count in a is 0, in which case we make a note to
// fill in the bucket in b and advance a.
if aCount == 0 {
bInter.num++ // Mark that we need to insert a bucket in b.
// Advance a
if aInter.num > 0 {
aInserts = append(aInserts, aInter)
aInter.num = 0
}
aIdx, aOK = ai.Next()
aInter.pos++
aCountIdx++
if aOK {
aCount += aBuckets[aCountIdx]
}
continue
}
// Otherwise we are missing a bucket that was in use in a, which is a reset.
return nil, nil, false
case aIdx > bIdx: // a misses a value that is in b. Forward b and recompare.
aInter.num++
// Advance b
if bInter.num > 0 {
bInserts = append(bInserts, bInter)
bInter.num = 0
}
bIdx, bOK = bi.Next()
bInter.pos++
bCountIdx++
if bOK {
bCount += bBuckets[bCountIdx]
}
}
case aOK && !bOK: // b misses a value that is in a.
// This is ok if the count in a is 0, in which case we make a note to
// fill in the bucket in b and advance a.
if aCount == 0 {
bInter.num++
// Advance a
if aInter.num > 0 {
aInserts = append(aInserts, aInter)
aInter.num = 0
}
aIdx, aOK = ai.Next()
aInter.pos++ // Advance potential insert position.
// Update absolute bucket counts for a.
aCountIdx++
if aOK {
aCount += aBuckets[aCountIdx]
}
continue
}
// Otherwise we are missing a bucket that was in use in a, which is a reset.
return nil, nil, false
case !aOK && bOK: // a misses a value that is in b. Forward b and recompare.
aInter.num++
// Advance b
if bInter.num > 0 {
bInserts = append(bInserts, bInter)
bInter.num = 0
}
bIdx, bOK = bi.Next()
bInter.pos++ // Advance potential insert position.
// Update absolute bucket counts for b.
bCountIdx++
if bOK {
bCount += bBuckets[bCountIdx]
}
default: // Both iterators ran out. We're done.
if aInter.num > 0 {
aInserts = append(aInserts, aInter)
}
if bInter.num > 0 {
bInserts = append(bInserts, bInter)
}
break loop
}
}
return aInserts, bInserts, true
}
// appendableGauge returns whether the chunk can be appended to, and if so
// whether:
// 1. Any recoding needs to happen to the chunk using the provided inserts
// (in case of any new buckets, positive or negative range, respectively).
// 1. Any recoding needs to happen to the chunk using the provided forward
// inserts (in case of any new buckets, positive or negative range,
// respectively).
// 2. Any recoding needs to happen for the histogram being appended, using the
// backward inserts (in case of any missing buckets, positive or negative
// range, respectively).
@ -369,76 +565,6 @@ func (a *HistogramAppender) appendableGauge(h *histogram.Histogram) (
return
}
// counterResetInAnyBucket returns true if there was a counter reset for any
// bucket. This should be called only when the bucket layout is the same or new
// buckets were added. It does not handle the case of buckets missing.
func counterResetInAnyBucket(oldBuckets, newBuckets []int64, oldSpans, newSpans []histogram.Span) bool {
if len(oldSpans) == 0 || len(oldBuckets) == 0 {
return false
}
var (
oldSpanSliceIdx, newSpanSliceIdx int = -1, -1 // Index for the span slices. Starts at -1 to indicate that the first non empty span is not yet found.
oldInsideSpanIdx, newInsideSpanIdx uint32 // Index inside a span.
oldIdx, newIdx int32 // Index inside a bucket slice.
oldBucketSliceIdx, newBucketSliceIdx int // Index inside bucket slice.
)
// Find first non empty spans.
oldSpanSliceIdx, oldIdx = nextNonEmptySpanSliceIdx(oldSpanSliceIdx, oldIdx, oldSpans)
newSpanSliceIdx, newIdx = nextNonEmptySpanSliceIdx(newSpanSliceIdx, newIdx, newSpans)
oldVal, newVal := oldBuckets[0], newBuckets[0]
// Since we assume that new spans won't have missing buckets, there will never be a case
// where the old index will not find a matching new index.
for {
if oldIdx == newIdx {
if newVal < oldVal {
return true
}
}
if oldIdx <= newIdx {
// Moving ahead old bucket and span by 1 index.
if oldInsideSpanIdx+1 >= oldSpans[oldSpanSliceIdx].Length {
// Current span is over.
oldSpanSliceIdx, oldIdx = nextNonEmptySpanSliceIdx(oldSpanSliceIdx, oldIdx, oldSpans)
oldInsideSpanIdx = 0
if oldSpanSliceIdx >= len(oldSpans) {
// All old spans are over.
break
}
} else {
oldInsideSpanIdx++
oldIdx++
}
oldBucketSliceIdx++
oldVal += oldBuckets[oldBucketSliceIdx]
}
if oldIdx > newIdx {
// Moving ahead new bucket and span by 1 index.
if newInsideSpanIdx+1 >= newSpans[newSpanSliceIdx].Length {
// Current span is over.
newSpanSliceIdx, newIdx = nextNonEmptySpanSliceIdx(newSpanSliceIdx, newIdx, newSpans)
newInsideSpanIdx = 0
if newSpanSliceIdx >= len(newSpans) {
// All new spans are over.
// This should not happen, old spans above should catch this first.
panic("new spans over before old spans in counterReset")
}
} else {
newInsideSpanIdx++
newIdx++
}
newBucketSliceIdx++
newVal += newBuckets[newBucketSliceIdx]
}
}
return false
}
// appendHistogram appends a histogram to the chunk. The caller must ensure that
// the histogram is properly structured, e.g. the number of buckets used
// corresponds to the number conveyed by the span structures. First call
@ -649,7 +775,7 @@ func (a *HistogramAppender) AppendHistogram(prev *HistogramAppender, t int64, h
a.setCounterResetHeader(CounterReset)
case prev != nil:
// This is a new chunk, but continued from a previous one. We need to calculate the reset header unless already set.
_, _, _, counterReset := prev.appendable(h)
_, _, _, _, _, counterReset := prev.appendable(h)
if counterReset {
a.setCounterResetHeader(CounterReset)
} else {
@ -661,7 +787,7 @@ func (a *HistogramAppender) AppendHistogram(prev *HistogramAppender, t int64, h
// Adding counter-like histogram.
if h.CounterResetHint != histogram.GaugeType {
pForwardInserts, nForwardInserts, okToAppend, counterReset := a.appendable(h)
pForwardInserts, nForwardInserts, pBackwardInserts, nBackwardInserts, okToAppend, counterReset := a.appendable(h)
if !okToAppend || counterReset {
if appendOnly {
if counterReset {
@ -692,6 +818,13 @@ func (a *HistogramAppender) AppendHistogram(prev *HistogramAppender, t int64, h
app.(*HistogramAppender).appendHistogram(t, h)
return chk, true, app, nil
}
if len(pBackwardInserts) > 0 || len(nBackwardInserts) > 0 {
// The histogram needs to be expanded to have the extra empty buckets
// of the chunk.
h.PositiveSpans = a.pSpans
h.NegativeSpans = a.nSpans
a.recodeHistogram(h, pBackwardInserts, nBackwardInserts)
}
a.appendHistogram(t, h)
return nil, false, a, nil
}

15
tsdb/chunkenc/histogram_meta.go
View file

@ -280,6 +280,9 @@ type Insert struct {
num int
}
// Deprecated: expandSpansForward, use expandIntSpansAndBuckets or
// expandFloatSpansAndBuckets instead.
// expandSpansForward is left here for reference.
// expandSpansForward returns the inserts to expand the bucket spans 'a' so that
// they match the spans in 'b'. 'b' must cover the same or more buckets than
// 'a', otherwise the function will return false.
@ -574,15 +577,3 @@ func counterResetHint(crh CounterResetHeader, numRead uint16) histogram.CounterR
return histogram.UnknownCounterReset
}
}
// Handle pathological case of empty span when advancing span idx.
// Call it with idx==-1 to find the first non empty span.
func nextNonEmptySpanSliceIdx(idx int, bucketIdx int32, spans []histogram.Span) (newIdx int, newBucketIdx int32) {
for idx++; idx < len(spans); idx++ {
if spans[idx].Length > 0 {
return idx, bucketIdx + spans[idx].Offset + 1
}
bucketIdx += spans[idx].Offset
}
return idx, 0
}

154
tsdb/chunkenc/histogram_test.go
View file

@ -256,9 +256,11 @@ func TestHistogramChunkBucketChanges(t *testing.T) {
h2.NegativeBuckets = []int64{2, -1} // 2 1 (total 3)
// This is how span changes will be handled.
hApp, _ := app.(*HistogramAppender)
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.NotEmpty(t, posInterjections)
require.NotEmpty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.True(t, ok) // Only new buckets came in.
require.False(t, cr)
c, app = hApp.recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans)
@ -347,7 +349,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Schema++
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
@ -357,7 +359,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.ZeroThreshold += 0.1
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
@ -380,9 +382,11 @@ func TestHistogramChunkAppendable(t *testing.T) {
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30)
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.NotEmpty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.True(t, ok) // Only new buckets came in.
require.False(t, cr)
@ -401,24 +405,57 @@ func TestHistogramChunkAppendable(t *testing.T) {
h2.Sum = 21
h2.PositiveBuckets = []int64{6, -3, -1, 2, 1, -4} // counts: 6, 3, 2, 4, 5, 1 (total 21)
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.False(t, ok) // Need to cut a new chunk.
require.True(t, cr)
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
}
{ // New histogram that has buckets missing but the buckets missing were empty.
emptyBucketH := eh.Copy()
emptyBucketH.PositiveBuckets = []int64{6, -6, 1, 1, -2, 1, 1} // counts: 6, 0, 1, 2, 0, 1, 2 (total 12)
c, hApp, ts, h1 := setup(emptyBucketH)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{ // Missing buckets at offset 1 and 9.
{Offset: 0, Length: 1},
{Offset: 3, Length: 1},
{Offset: 3, Length: 1},
{Offset: 4, Length: 1},
{Offset: 1, Length: 1},
}
h2.PositiveBuckets = []int64{7, -5, 1, 0, 1} // counts: 7, 2, 3, 3, 4 (total 18)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.NotEmpty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.True(t, ok)
require.False(t, cr)
assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
// Check that h2 was recoded.
require.Equal(t, []int64{7, -7, 2, 1, -3, 3, 1}, h2.PositiveBuckets) // counts: 7, 0, 2, 3 , 0, 3, 4 (total 18)
require.Equal(t, emptyBucketH.PositiveSpans, h2.PositiveSpans)
}
{ // New histogram that has a counter reset while buckets are same.
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Sum = 23
h2.PositiveBuckets = []int64{6, -4, 1, -1, 2, 1, -4} // counts: 6, 2, 3, 2, 4, 5, 1 (total 23)
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.False(t, ok) // Need to cut a new chunk.
require.True(t, cr)
@ -440,9 +477,11 @@ func TestHistogramChunkAppendable(t *testing.T) {
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 0} // 7 5 1 3 1 0 2 5 5 0 0 (total 29)
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.False(t, ok) // Need to cut a new chunk.
require.True(t, cr)
@ -470,9 +509,11 @@ func TestHistogramChunkAppendable(t *testing.T) {
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2.PositiveBuckets = []int64{1, 1, 3, -2, 0, -1, 2, 1, -4} // counts: 1, 2, 5, 3, 3, 2, 4, 5, 1 (total 26)
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.False(t, ok) // Need to cut a new chunk.
require.True(t, cr)
@ -549,10 +590,44 @@ func TestHistogramChunkAppendable(t *testing.T) {
require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader())
}
{
// Start a new chunk with a histogram that has an empty bucket.
// Add a histogram that has the same bucket missing.
// This should be appendable and can happen if we are merging from chunks
// where the first sample came from a recoded chunk that added the
// empty bucket.
h1 := eh.Copy()
// Add a bucket that is empty -10 offsets from the first bucket.
h1.PositiveSpans = make([]histogram.Span, len(eh.PositiveSpans)+1)
h1.PositiveSpans[0] = histogram.Span{Offset: eh.PositiveSpans[0].Offset - 10, Length: 1}
h1.PositiveSpans[1] = histogram.Span{Offset: eh.PositiveSpans[0].Offset + 9, Length: eh.PositiveSpans[0].Length}
for i, v := range eh.PositiveSpans[1:] {
h1.PositiveSpans[i+2] = v
}
h1.PositiveBuckets = make([]int64, len(eh.PositiveBuckets)+1)
h1.PositiveBuckets[0] = 0
for i, v := range eh.PositiveBuckets {
h1.PositiveBuckets[i+1] = v
}
c, hApp, ts, _ := setup(h1)
h2 := eh.Copy()
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.Empty(t, posInterjections)
require.Empty(t, negInterjections)
require.NotEmpty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.True(t, ok)
require.False(t, cr)
assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
}
{ // Custom buckets, no change.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.True(t, ok)
assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
@ -563,7 +638,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
h2 := h1.Copy()
h2.Count++
h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -3}
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.True(t, ok)
assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
@ -574,7 +649,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
h2 := h1.Copy()
h2.Count--
h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -5}
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
@ -584,7 +659,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
_, _, ok, _ := hApp.appendable(h2)
_, _, _, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
@ -606,9 +681,11 @@ func TestHistogramChunkAppendable(t *testing.T) {
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30)
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
posInterjections, negInterjections, backwardPositiveInserts, backwardNegativeInserts, ok, cr := hApp.appendable(h2)
require.NotEmpty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, backwardPositiveInserts)
require.Empty(t, backwardNegativeInserts)
require.True(t, ok) // Only new buckets came in.
require.False(t, cr)
@ -875,9 +952,11 @@ func TestHistogramChunkAppendableWithEmptySpan(t *testing.T) {
require.Equal(t, 1, c.NumSamples())
hApp, _ := app.(*HistogramAppender)
pI, nI, okToAppend, counterReset := hApp.appendable(tc.h2)
pI, nI, bpI, bnI, okToAppend, counterReset := hApp.appendable(tc.h2)
require.Empty(t, pI)
require.Empty(t, nI)
require.Empty(t, bpI)
require.Empty(t, bnI)
require.True(t, okToAppend)
require.False(t, counterReset)
})
@ -1368,3 +1447,50 @@ func TestHistogramAppendOnlyErrors(t *testing.T) {
require.EqualError(t, err, "histogram counter reset")
})
}
func BenchmarkAppendable(b *testing.B) {
// Create a histogram with a bunch of spans and buckets.
const (
numSpans = 1000
spanLength = 10
)
h := &histogram.Histogram{
Schema: 0,
Count: 100,
Sum: 1000,
ZeroThreshold: 0.001,
ZeroCount: 5,
}
for i := 0; i < numSpans; i++ {
h.PositiveSpans = append(h.PositiveSpans, histogram.Span{Offset: 5, Length: spanLength})
h.NegativeSpans = append(h.NegativeSpans, histogram.Span{Offset: 5, Length: spanLength})
for j := 0; j < spanLength; j++ {
h.PositiveBuckets = append(h.PositiveBuckets, int64(j))
h.NegativeBuckets = append(h.NegativeBuckets, int64(j))
}
}
c := Chunk(NewHistogramChunk())
// Create fresh appender and add the first histogram.
app, err := c.Appender()
if err != nil {
b.Fatal(err)
}
_, _, _, err = app.AppendHistogram(nil, 1, h, true)
if err != nil {
b.Fatal(err)
}
hApp := app.(*HistogramAppender)
isAppendable := true
for i := 0; i < b.N; i++ {
_, _, _, _, ok, _ := hApp.appendable(h)
isAppendable = isAppendable && ok
}
if !isAppendable {
b.Fail()
}
}

5
tsdb/db_test.go
View file

@ -63,7 +63,10 @@ func TestMain(m *testing.M) {
flag.Parse()
defaultIsolationDisabled = !isolationEnabled
goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("github.com/prometheus/prometheus/tsdb.(*SegmentWAL).cut.func1"), goleak.IgnoreTopFunction("github.com/prometheus/prometheus/tsdb.(*SegmentWAL).cut.func2"))
goleak.VerifyTestMain(m,
goleak.IgnoreTopFunction("github.com/prometheus/prometheus/tsdb.(*SegmentWAL).cut.func1"),
goleak.IgnoreTopFunction("github.com/prometheus/prometheus/tsdb.(*SegmentWAL).cut.func2"),
goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"))
}
func openTestDB(t testing.TB, opts *Options, rngs []int64) (db *DB) {

43
tsdb/head.go
View file

@ -178,6 +178,7 @@ type HeadOptions struct {
WALReplayConcurrency int
// EnableSharding enables ShardedPostings() support in the Head.
// EnableSharding is temporarily disabled during Init().
EnableSharding bool
}
@ -609,7 +610,7 @@ const cardinalityCacheExpirationTime = time.Duration(30) * time.Second
// Init loads data from the write ahead log and prepares the head for writes.
// It should be called before using an appender so that it
// limits the ingested samples to the head min valid time.
func (h *Head) Init(minValidTime int64) error {
func (h *Head) Init(minValidTime int64) (err error) {
h.minValidTime.Store(minValidTime)
defer func() {
h.postings.EnsureOrder(h.opts.WALReplayConcurrency)
@ -623,6 +624,24 @@ func (h *Head) Init(minValidTime int64) error {
}
}()
// If sharding is enabled, disable it while initializing, and calculate the shards later.
// We're going to use that field for other purposes during WAL replay,
// so we don't want to waste time on calculating the shard that we're going to lose anyway.
if h.opts.EnableSharding {
h.opts.EnableSharding = false
defer func() {
h.opts.EnableSharding = true
if err == nil {
// No locking is needed here as nobody should be writing while we're in Init.
for _, stripe := range h.series.series {
for _, s := range stripe {
s.shardHashOrMemoryMappedMaxTime = labels.StableHash(s.lset)
}
}
}
}()
}
level.Info(h.logger).Log("msg", "Replaying on-disk memory mappable chunks if any")
start := time.Now()
@ -683,7 +702,6 @@ func (h *Head) Init(minValidTime int64) error {
mmappedChunks map[chunks.HeadSeriesRef][]*mmappedChunk
oooMmappedChunks map[chunks.HeadSeriesRef][]*mmappedChunk
lastMmapRef chunks.ChunkDiskMapperRef
err error
mmapChunkReplayDuration time.Duration
)
@ -2068,9 +2086,11 @@ type memSeries struct {
ref chunks.HeadSeriesRef
meta *metadata.Metadata
// Series labels hash to use for sharding purposes. The value is always 0 when sharding has not
// been explicitly enabled in TSDB.
shardHash uint64
// Series labels hash to use for sharding purposes.
// The value is always 0 when sharding has not been explicitly enabled in TSDB.
// While the WAL replay the value stored here is the max time of any mmapped chunk,
// and the shard hash is re-calculated after WAL replay is complete.
shardHashOrMemoryMappedMaxTime uint64
// Everything after here should only be accessed with the lock held.
sync.Mutex
@ -2095,10 +2115,9 @@ type memSeries struct {
ooo *memSeriesOOOFields
mmMaxTime int64 // Max time of any mmapped chunk, only used during WAL replay.
nextAt int64 // Timestamp at which to cut the next chunk.
histogramChunkHasComputedEndTime bool // True if nextAt has been predicted for the current histograms chunk; false otherwise.
pendingCommit bool // Whether there are samples waiting to be committed to this series.
// We keep the last value here (in addition to appending it to the chunk) so we can check for duplicates.
lastValue float64
@ -2114,8 +2133,6 @@ type memSeries struct {
// txs is nil if isolation is disabled.
txs *txRing
pendingCommit bool // Whether there are samples waiting to be committed to this series.
}
// memSeriesOOOFields contains the fields required by memSeries
@ -2131,7 +2148,7 @@ func newMemSeries(lset labels.Labels, id chunks.HeadSeriesRef, shardHash uint64,
lset: lset,
ref: id,
nextAt: math.MinInt64,
shardHash: shardHash,
shardHashOrMemoryMappedMaxTime: shardHash,
}
if !isolationDisabled {
s.txs = newTxRing(0)
@ -2219,6 +2236,12 @@ func (s *memSeries) truncateChunksBefore(mint int64, minOOOMmapRef chunks.ChunkD
return removedInOrder + removedOOO
}
// shardHash returns the shard hash of the series, only available after WAL replay.
func (s *memSeries) shardHash() uint64 { return s.shardHashOrMemoryMappedMaxTime }
// mmMaxTime returns the max time of any mmapped chunk in the series, only available during WAL replay.
func (s *memSeries) mmMaxTime() int64 { return int64(s.shardHashOrMemoryMappedMaxTime) }
// cleanupAppendIDsBelow cleans up older appendIDs. Has to be called after
// acquiring lock.
func (s *memSeries) cleanupAppendIDsBelow(bound uint64) {

2
tsdb/head_read.go
View file

@ -170,7 +170,7 @@ func (h *headIndexReader) ShardedPostings(p index.Postings, shardIndex, shardCou
}
// Check if the series belong to the shard.
if s.shardHash%shardCount != shardIndex {
if s.shardHash()%shardCount != shardIndex {
continue
}

38
tsdb/head_test.go
View file

@ -23,6 +23,7 @@ import (
"path"
"path/filepath"
"reflect"
"runtime/pprof"
"sort"
"strconv"
"strings"
@ -89,6 +90,43 @@ func newTestHeadWithOptions(t testing.TB, compressWAL wlog.CompressionType, opts
return h, wal
}
// BenchmarkLoadRealWLs will be skipped unless the BENCHMARK_LOAD_REAL_WLS_DIR environment variable is set.
// BENCHMARK_LOAD_REAL_WLS_DIR should be the folder where `wal` and `chunks_head` are located.
// Optionally, BENCHMARK_LOAD_REAL_WLS_PROFILE can be set to a file path to write a CPU profile.
func BenchmarkLoadRealWLs(b *testing.B) {
dir := os.Getenv("BENCHMARK_LOAD_REAL_WLS_DIR")
if dir == "" {
b.Skipped()
}
profileFile := os.Getenv("BENCHMARK_LOAD_REAL_WLS_PROFILE")
if profileFile != "" {
b.Logf("Will profile in %s", profileFile)
f, err := os.Create(profileFile)
require.NoError(b, err)
b.Cleanup(func() { f.Close() })
require.NoError(b, pprof.StartCPUProfile(f))
b.Cleanup(pprof.StopCPUProfile)
}
wal, err := wlog.New(nil, nil, filepath.Join(dir, "wal"), wlog.CompressionNone)
require.NoError(b, err)
b.Cleanup(func() { wal.Close() })
wbl, err := wlog.New(nil, nil, filepath.Join(dir, "wbl"), wlog.CompressionNone)
require.NoError(b, err)
b.Cleanup(func() { wbl.Close() })
// Load the WAL.
for i := 0; i < b.N; i++ {
opts := DefaultHeadOptions()
opts.ChunkDirRoot = dir
h, err := NewHead(nil, nil, wal, wbl, opts, nil)
require.NoError(b, err)
h.Init(0)
}
}
func BenchmarkCreateSeries(b *testing.B) {
series := genSeries(b.N, 10, 0, 0)
h, _ := newTestHead(b, 10000, wlog.CompressionNone, false)

13
tsdb/head_wal.go
View file

@ -435,6 +435,8 @@ Outer:
return nil
}
func minInt64() int64 { return math.MinInt64 }
// resetSeriesWithMMappedChunks is only used during the WAL replay.
func (h *Head) resetSeriesWithMMappedChunks(mSeries *memSeries, mmc, oooMmc []*mmappedChunk, walSeriesRef chunks.HeadSeriesRef) (overlapped bool) {
if mSeries.ref != walSeriesRef {
@ -481,10 +483,11 @@ func (h *Head) resetSeriesWithMMappedChunks(mSeries *memSeries, mmc, oooMmc []*m
}
// Cache the last mmapped chunk time, so we can skip calling append() for samples it will reject.
if len(mmc) == 0 {
mSeries.mmMaxTime = math.MinInt64
mSeries.shardHashOrMemoryMappedMaxTime = uint64(minInt64())
} else {
mSeries.mmMaxTime = mmc[len(mmc)-1].maxTime
h.updateMinMaxTime(mmc[0].minTime, mSeries.mmMaxTime)
mmMaxTime := mmc[len(mmc)-1].maxTime
mSeries.shardHashOrMemoryMappedMaxTime = uint64(mmMaxTime)
h.updateMinMaxTime(mmc[0].minTime, mmMaxTime)
}
if len(oooMmc) != 0 {
// Mint and maxt can be in any chunk, they are not sorted.
@ -585,7 +588,7 @@ func (wp *walSubsetProcessor) processWALSamples(h *Head, mmappedChunks, oooMmapp
unknownRefs++
continue
}
if s.T <= ms.mmMaxTime {
if s.T <= ms.mmMaxTime() {
continue
}
if _, chunkCreated := ms.append(s.T, s.V, 0, appendChunkOpts); chunkCreated {
@ -614,7 +617,7 @@ func (wp *walSubsetProcessor) processWALSamples(h *Head, mmappedChunks, oooMmapp
unknownHistogramRefs++
continue
}
if s.t <= ms.mmMaxTime {
if s.t <= ms.mmMaxTime() {
continue
}
var chunkCreated bool

10
tsdb/tsdbutil/histogram.go
View file

@ -30,12 +30,10 @@ func GenerateTestHistograms(n int) (r []*histogram.Histogram) {
return r
}
func GenerateTestHistogramsWithUnknownResetHint(n int) []*histogram.Histogram {
hs := GenerateTestHistograms(n)
for i := range hs {
hs[i].CounterResetHint = histogram.UnknownCounterReset
}
return hs
func GenerateTestHistogramWithHint(n int, hint histogram.CounterResetHint) *histogram.Histogram {
h := GenerateTestHistogram(n)
h.CounterResetHint = hint
return h
}
// GenerateTestHistogram but it is up to the user to set any known counter reset hint.

81
util/junitxml/junitxml.go Normal file
View file

@ -0,0 +1,81 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package junitxml
import (
"encoding/xml"
"io"
)
type JUnitXML struct {
XMLName xml.Name `xml:"testsuites"`
Suites []*TestSuite `xml:"testsuite"`
}
type TestSuite struct {
Name string `xml:"name,attr"`
TestCount int `xml:"tests,attr"`
FailureCount int `xml:"failures,attr"`
ErrorCount int `xml:"errors,attr"`
SkippedCount int `xml:"skipped,attr"`
Timestamp string `xml:"timestamp,attr"`
Cases []*TestCase `xml:"testcase"`
}
type TestCase struct {
Name string `xml:"name,attr"`
Failures []string `xml:"failure,omitempty"`
Error string `xml:"error,omitempty"`
}
func (j *JUnitXML) WriteXML(h io.Writer) error {
return xml.NewEncoder(h).Encode(j)
}
func (j *JUnitXML) Suite(name string) *TestSuite {
ts := &TestSuite{Name: name}
j.Suites = append(j.Suites, ts)
return ts
}
func (ts *TestSuite) Fail(f string) {
ts.FailureCount++
curt := ts.lastCase()
curt.Failures = append(curt.Failures, f)
}
func (ts *TestSuite) lastCase() *TestCase {
if len(ts.Cases) == 0 {
ts.Case("unknown")
}
return ts.Cases[len(ts.Cases)-1]
}
func (ts *TestSuite) Case(name string) *TestSuite {
j := &TestCase{
Name: name,
}
ts.Cases = append(ts.Cases, j)
ts.TestCount++
return ts
}
func (ts *TestSuite) Settime(name string) {
ts.Timestamp = name
}
func (ts *TestSuite) Abort(e error) {
ts.ErrorCount++
curt := ts.lastCase()
curt.Error = e.Error()
}

66
util/junitxml/junitxml_test.go Normal file
View file

@ -0,0 +1,66 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package junitxml
import (
"bytes"
"encoding/xml"
"errors"
"testing"
)
func TestJunitOutput(t *testing.T) {
var buf bytes.Buffer
var test JUnitXML
x := FakeTestSuites()
if err := x.WriteXML(&buf); err != nil {
t.Fatalf("Failed to encode XML: %v", err)
}
output := buf.Bytes()
err := xml.Unmarshal(output, &test)
if err != nil {
t.Errorf("Unmarshal failed with error: %v", err)
}
var total int
var cases int
total = len(test.Suites)
if total != 3 {
t.Errorf("JUnit output had %d testsuite elements; expected 3\n", total)
}
for _, i := range test.Suites {
cases += len(i.Cases)
}
if cases != 7 {
t.Errorf("JUnit output had %d testcase; expected 7\n", cases)
}
}
func FakeTestSuites() *JUnitXML {
ju := &JUnitXML{}
good := ju.Suite("all good")
good.Case("alpha")
good.Case("beta")
good.Case("gamma")
mixed := ju.Suite("mixed")
mixed.Case("good")
bad := mixed.Case("bad")
bad.Fail("once")
bad.Fail("twice")
mixed.Case("ugly").Abort(errors.New("buggy"))
ju.Suite("fast").Fail("fail early")
return ju
}

2
web/api/v1/api.go
View file

@ -295,7 +295,7 @@ func NewAPI(
a.remoteWriteHandler = remote.NewWriteHandler(logger, registerer, ap, acceptRemoteWriteProtoMsgs)
}
if otlpEnabled {
a.otlpWriteHandler = remote.NewOTLPWriteHandler(logger, ap)
a.otlpWriteHandler = remote.NewOTLPWriteHandler(logger, ap, configFunc)
}
return a

1
web/api/v1/api_test.go
View file

@ -359,6 +359,7 @@ var samplePrometheusCfg = config.Config{
ScrapeConfigs: []*config.ScrapeConfig{},
RemoteWriteConfigs: []*config.RemoteWriteConfig{},
RemoteReadConfigs: []*config.RemoteReadConfig{},
OTLPConfig: config.OTLPConfig{},
}
var sampleFlagMap = map[string]string{

2
web/ui/module/codemirror-promql/src/complete/hybrid.test.ts
View file

@ -775,7 +775,7 @@ describe('computeStartCompletePosition test', () => {
it(value.title, () => {
const state = createEditorState(value.expr);
const node = syntaxTree(state).resolve(value.pos, -1);
const result = computeStartCompletePosition(node, value.pos);
const result = computeStartCompletePosition(state, node, value.pos);
expect(value.expectedStart).toEqual(result);
});
});

24
web/ui/module/codemirror-promql/src/complete/hybrid.ts
View file

@ -21,7 +21,6 @@ import {
BinaryExpr,
BoolModifier,
Div,
Duration,
Eql,
EqlRegex,
EqlSingle,
@ -40,7 +39,6 @@ import {
Mul,
Neq,
NeqRegex,
NumberLiteral,
OffsetExpr,
Or,
Pow,
@ -54,6 +52,8 @@ import {
UnquotedLabelMatcher,
QuotedLabelMatcher,
QuotedLabelName,
NumberDurationLiteralInDurationContext,
NumberDurationLiteral,
} from '@prometheus-io/lezer-promql';
import { Completion, CompletionContext, CompletionResult } from '@codemirror/autocomplete';
import { EditorState } from '@codemirror/state';
@ -179,7 +179,8 @@ function computeStartCompleteLabelPositionInLabelMatcherOrInGroupingLabel(node:
// It is an important step because the start position will be used by CMN to find the string and then to use it to filter the CompletionResult.
// A wrong `start` position will lead to have the completion not working.
// Note: this method is exported only for testing purpose.
export function computeStartCompletePosition(node: SyntaxNode, pos: number): number {
export function computeStartCompletePosition(state: EditorState, node: SyntaxNode, pos: number): number {
const currentText = state.doc.slice(node.from, pos).toString();
let start = node.from;
if (node.type.id === LabelMatchers || node.type.id === GroupingLabels) {
start = computeStartCompleteLabelPositionInLabelMatcherOrInGroupingLabel(node, pos);
@ -191,11 +192,16 @@ export function computeStartCompletePosition(node: SyntaxNode, pos: number): num
start++;
} else if (
node.type.id === OffsetExpr ||
(node.type.id === NumberLiteral && node.parent?.type.id === 0 && node.parent.parent?.type.id === SubqueryExpr) ||
// Since duration and number are equivalent, writing go[5] or go[5d] is syntactically accurate.
// Before we were able to guess when we had to autocomplete the duration later based on the error node,
// which is not possible anymore.
// So we have to analyze the string about the current node to see if the duration unit is already present or not.
(node.type.id === NumberDurationLiteralInDurationContext && !durationTerms.map((v) => v.label).includes(currentText[currentText.length - 1])) ||
(node.type.id === NumberDurationLiteral && node.parent?.type.id === 0 && node.parent.parent?.type.id === SubqueryExpr) ||
(node.type.id === 0 &&
(node.parent?.type.id === OffsetExpr ||
node.parent?.type.id === MatrixSelector ||
(node.parent?.type.id === SubqueryExpr && containsAtLeastOneChild(node.parent, Duration))))
(node.parent?.type.id === SubqueryExpr && containsAtLeastOneChild(node.parent, NumberDurationLiteralInDurationContext))))
) {
start = pos;
}
@ -230,7 +236,7 @@ export function analyzeCompletion(state: EditorState, node: SyntaxNode): Context
result.push({ kind: ContextKind.Duration });
break;
}
if (node.parent?.type.id === SubqueryExpr && containsAtLeastOneChild(node.parent, Duration)) {
if (node.parent?.type.id === SubqueryExpr && containsAtLeastOneChild(node.parent, NumberDurationLiteralInDurationContext)) {
// we are likely in the given situation:
// `rate(foo[5d:5])`
// so we should autocomplete a duration
@ -434,7 +440,7 @@ export function analyzeCompletion(state: EditorState, node: SyntaxNode): Context
result.push({ kind: ContextKind.MetricName, metricName: state.sliceDoc(node.from, node.to).slice(1, -1) });
}
break;
case NumberLiteral:
case NumberDurationLiteral:
if (node.parent?.type.id === 0 && node.parent.parent?.type.id === SubqueryExpr) {
// Here we are likely in this situation:
// `go[5d:4]`
@ -449,7 +455,7 @@ export function analyzeCompletion(state: EditorState, node: SyntaxNode): Context
result.push({ kind: ContextKind.Number });
}
break;
case Duration:
case NumberDurationLiteralInDurationContext:
case OffsetExpr:
result.push({ kind: ContextKind.Duration });
break;
@ -591,7 +597,7 @@ export class HybridComplete implements CompleteStrategy {
}
}
return asyncResult.then((result) => {
return arrayToCompletionResult(result, computeStartCompletePosition(tree, pos), pos, completeSnippet, span);
return arrayToCompletionResult(result, computeStartCompletePosition(state, tree, pos), pos, completeSnippet, span);
});
}

4
web/ui/module/codemirror-promql/src/parser/type.ts
View file

@ -17,7 +17,7 @@ import {
BinaryExpr,
FunctionCall,
MatrixSelector,
NumberLiteral,
NumberDurationLiteral,
OffsetExpr,
ParenExpr,
StepInvariantExpr,
@ -42,7 +42,7 @@ export function getType(node: SyntaxNode | null): ValueType {
return getType(node.firstChild);
case StringLiteral:
return ValueType.string;
case NumberLiteral:
case NumberDurationLiteral:
return ValueType.scalar;
case MatrixSelector:
return ValueType.matrix;

2
web/ui/module/lezer-promql/package.json
View file

@ -30,7 +30,7 @@
"test": "NODE_OPTIONS=--experimental-vm-modules jest"
},
"devDependencies": {
"@lezer/generator": "^1.7.0",
"@lezer/generator": "^1.7.1",
"@lezer/highlight": "^1.2.0",
"@lezer/lr": "^1.4.1"
},

4
web/ui/module/lezer-promql/src/highlight.js
View file

@ -17,8 +17,8 @@ export const promQLHighLight = styleTags({
LineComment: tags.comment,
LabelName: tags.labelName,
StringLiteral: tags.string,
NumberLiteral: tags.number,
Duration: tags.number,
NumberDurationLiteral: tags.number,
NumberDurationLiteralInDurationContext: tags.number,
Identifier: tags.variableName,
'Abs Absent AbsentOverTime Acos Acosh Asin Asinh Atan Atanh AvgOverTime Ceil Changes Clamp ClampMax ClampMin Cos Cosh CountOverTime DaysInMonth DayOfMonth DayOfWeek DayOfYear Deg Delta Deriv Exp Floor HistogramAvg HistogramCount HistogramFraction HistogramQuantile HistogramSum HoltWinters Hour Idelta Increase Irate LabelReplace LabelJoin LastOverTime Ln Log10 Log2 MaxOverTime MinOverTime Minute Month Pi PredictLinear PresentOverTime QuantileOverTime Rad Rate Resets Round Scalar Sgn Sin Sinh Sort SortDesc SortByLabel SortByLabelDesc Sqrt StddevOverTime StdvarOverTime SumOverTime Tan Tanh Time Timestamp Vector Year':
tags.function(tags.variableName),

20
web/ui/module/lezer-promql/src/promql.grammar
View file

@ -29,7 +29,7 @@ expr[@isGroup=Expr] {
BinaryExpr |
FunctionCall |
MatrixSelector |
NumberLiteral |
NumberDurationLiteral |
OffsetExpr |
ParenExpr |
StringLiteral |
@ -194,16 +194,16 @@ ParenExpr {
}
OffsetExpr {
expr Offset Sub? Duration
expr Offset NumberDurationLiteralInDurationContext
}
MatrixSelector {
// TODO: Can this not be more specific than "expr"?
expr "[" Duration "]"
expr "[" NumberDurationLiteralInDurationContext "]"
}
SubqueryExpr {
expr "[" Duration ":" ("" | Duration) "]"
expr "[" NumberDurationLiteralInDurationContext ":" ("" | NumberDurationLiteralInDurationContext) "]"
}
UnaryExpr {
@ -245,14 +245,18 @@ QuotedLabelName {
}
StepInvariantExpr {
expr At ( NumberLiteral | AtModifierPreprocessors "(" ")" )
expr At ( NumberDurationLiteral | AtModifierPreprocessors "(" ")" )
}
AtModifierPreprocessors {
Start | End
}
NumberLiteral {
NumberDurationLiteral {
("-"|"+")?~signed (number | inf | nan)
}
NumberDurationLiteralInDurationContext {
("-"|"+")?~signed (number | inf | nan)
}
@ -264,7 +268,7 @@ NumberLiteral {
number {
(std.digit+ (("_")? std.digit)* ("." std.digit+ (("_")? std.digit)*)? | "." std.digit+ (("_")? std.digit)*) (("e" | "E") ("+" | "-")? std.digit+ (("_")? std.digit)*)? |
"0x" (std.digit | $[a-fA-F])+
"0x" (std.digit | $[a-fA-F])+ | duration
}
StringLiteral { // TODO: This is for JS, make this work for PromQL.
'"' (![\\\n"] | "\\" _)* '"'? |
@ -272,7 +276,7 @@ NumberLiteral {
"`" ![`]* "`"
}
Duration {
duration {
// Each line below is just the same regex repeated over and over, but each time with one of the units made non-optional,
// to ensure that at least one <number>+<unit> pair is provided and an empty string is not recognized as a valid duration.
( ( std.digit+ "y" ) ( std.digit+ "w" )? ( std.digit+ "d" )? ( std.digit+ "h" )? ( std.digit+ "m" )? ( std.digit+ "s" )? ( std.digit+ "ms" )? ) |

68
web/ui/module/lezer-promql/test/expression.txt
View file

@ -4,7 +4,7 @@
==>
PromQL(NumberLiteral)
PromQL(NumberDurationLiteral)
# Double-quoted string literal
@ -46,7 +46,7 @@ PromQL(StringLiteral)
==>
PromQL(BinaryExpr(NumberLiteral, Add, NumberLiteral))
PromQL(BinaryExpr(NumberDurationLiteral, Add, NumberDurationLiteral))
# Complex expression
@ -73,7 +73,7 @@ PromQL(
VectorSelector(
Identifier
),
Duration
NumberDurationLiteralInDurationContext
)
)
)
@ -103,7 +103,7 @@ PromQL(
VectorSelector(
Identifier
),
Duration
NumberDurationLiteralInDurationContext
)
)
)
@ -240,21 +240,21 @@ PromQL(
)
)
# Duration units
# NumberDurationLiteralInDurationContext units
foo[1y2w3d4h5m6s7ms]
==>
PromQL(MatrixSelector(VectorSelector(Identifier),Duration))
PromQL(MatrixSelector(VectorSelector(Identifier),NumberDurationLiteralInDurationContext))
# Incorrectly ordered duration units
# Incorrectly ordered NumberDurationLiteralInDurationContext units
foo[1m2h]
==>
PromQL(SubqueryExpr(VectorSelector(Identifier),Duration,⚠,Duration))
PromQL(MatrixSelector(VectorSelector(Identifier),NumberDurationLiteralInDurationContext,⚠))
# Using a function name as a metric name
@ -311,7 +311,7 @@ PromQL(
),
Gtr,
BoolModifier(Bool),
NumberLiteral
NumberDurationLiteral
)
)
@ -357,7 +357,7 @@ PromQL(
VectorSelector(
Identifier
),
Duration
NumberDurationLiteralInDurationContext
)
)
)
@ -389,8 +389,8 @@ PromQL(
FunctionIdentifier(Clamp),
FunctionCallBody(
VectorSelector(Identifier),
NumberLiteral,
NumberLiteral
NumberDurationLiteral,
NumberDurationLiteral
)
)
)
@ -450,7 +450,7 @@ PromQL(
Identifier
),
At,
NumberLiteral
NumberDurationLiteral
)
)
@ -483,7 +483,7 @@ PromQL(
FunctionCallBody(
MatrixSelector(
VectorSelector(Identifier),
Duration
NumberDurationLiteralInDurationContext
)
)
),
@ -491,14 +491,14 @@ PromQL(
AggregateExpr(
AggregateOp(Topk),
FunctionCallBody(
NumberLiteral,
NumberDurationLiteral,
FunctionCall(
FunctionIdentifier(Rate),
FunctionCallBody(
StepInvariantExpr(
MatrixSelector(VectorSelector(Identifier), Duration),
MatrixSelector(VectorSelector(Identifier), NumberDurationLiteralInDurationContext),
At,
NumberLiteral
NumberDurationLiteral
)
)
)
@ -518,7 +518,7 @@ PromQL(
Identifier
),
At,
NumberLiteral
NumberDurationLiteral
)
)
@ -533,7 +533,7 @@ PromQL(
Identifier
),
At,
NumberLiteral
NumberDurationLiteral
)
)
@ -556,98 +556,98 @@ PromQL(VectorSelector(Identifier))
NaN
==>
PromQL(NumberLiteral)
PromQL(NumberDurationLiteral)
# Lower-cased NaN.
nan
==>
PromQL(NumberLiteral)
PromQL(NumberDurationLiteral)
# Inf.
Inf
==>
PromQL(NumberLiteral)
PromQL(NumberDurationLiteral)
# Negative Inf.
-Inf
==>
PromQL(NumberLiteral)
PromQL(NumberDurationLiteral)
# Positive Inf.
+Inf
==>
PromQL(NumberLiteral)
PromQL(NumberDurationLiteral)
# Lower-cased Inf.
inf
==>
PromQL(NumberLiteral)
PromQL(NumberDurationLiteral)
# Upper-cased Inf.
INF
==>
PromQL(NumberLiteral)
PromQL(NumberDurationLiteral)
# Negative number literal.
-42
==>
PromQL(NumberLiteral)
PromQL(NumberDurationLiteral)
# Explicitly positive number literal.
+42
==>
PromQL(NumberLiteral)
PromQL(NumberDurationLiteral)
# Trying to illegally use NaN as a metric name.
NaN{foo="bar"}
==>
PromQL(BinaryExpr(NumberLiteral,⚠,VectorSelector(LabelMatchers(UnquotedLabelMatcher(LabelName,MatchOp(EqlSingle),StringLiteral)))))
PromQL(BinaryExpr(NumberDurationLiteral,⚠,VectorSelector(LabelMatchers(UnquotedLabelMatcher(LabelName,MatchOp(EqlSingle),StringLiteral)))))
# Trying to illegally use Inf as a metric name.
Inf{foo="bar"}
==>
PromQL(BinaryExpr(NumberLiteral,⚠,VectorSelector(LabelMatchers(UnquotedLabelMatcher(LabelName,MatchOp(EqlSingle),StringLiteral)))))
PromQL(BinaryExpr(NumberDurationLiteral,⚠,VectorSelector(LabelMatchers(UnquotedLabelMatcher(LabelName,MatchOp(EqlSingle),StringLiteral)))))
# Negative offset
foo offset -5d
==>
PromQL(OffsetExpr(VectorSelector(Identifier), Offset, Sub, Duration))
PromQL(OffsetExpr(VectorSelector(Identifier), Offset, NumberDurationLiteralInDurationContext))
# Negative offset with space
foo offset - 5d
==>
PromQL(OffsetExpr(VectorSelector(Identifier), Offset, Sub, Duration))
PromQL(OffsetExpr(VectorSelector(Identifier), Offset, NumberDurationLiteralInDurationContext))
# Positive offset
foo offset 5d
==>
PromQL(OffsetExpr(VectorSelector(Identifier), Offset, Duration))
PromQL(OffsetExpr(VectorSelector(Identifier), Offset, NumberDurationLiteralInDurationContext))
# Parsing only metric names with alternative @top { "top": "MetricName" }
@ -661,7 +661,7 @@ MetricName(Identifier)
1 + foo atan2 bar
==>
PromQL(BinaryExpr(NumberLiteral,Add,BinaryExpr(VectorSelector(Identifier),Atan2,VectorSelector(Identifier))))
PromQL(BinaryExpr(NumberDurationLiteral,Add,BinaryExpr(VectorSelector(Identifier),Atan2,VectorSelector(Identifier))))
# Testing quoted metric name

8
web/ui/package-lock.json generated
View file

@ -72,7 +72,7 @@
"version": "0.53.1",
"license": "Apache-2.0",
"devDependencies": {
"@lezer/generator": "^1.7.0",
"@lezer/generator": "^1.7.1",
"@lezer/highlight": "^1.2.0",
"@lezer/lr": "^1.4.1"
},
@ -3371,9 +3371,9 @@
"integrity": "sha512-yemX0ZD2xS/73llMZIK6KplkjIjf2EvAHcinDi/TfJ9hS25G0388+ClHt6/3but0oOxinTcQHJLDXh6w1crzFQ=="
},
"node_modules/@lezer/generator": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-1.7.0.tgz",
"integrity": "sha512-IJ16tx3biLKlCXUzcK4v8S10AVa2BSM2rB12rtAL6f1hL2TS/HQQlGCoWRvanlL2J4mCYEEIv9uG7n4kVMkVDA==",
"version": "1.7.1",
"resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-1.7.1.tgz",
"integrity": "sha512-MgPJN9Si+ccxzXl3OAmCeZuUKw4XiPl4y664FX/hnnyG9CTqUPq65N3/VGPA2jD23D7QgMTtNqflta+cPN+5mQ==",
"dev": true,
"dependencies": {
"@lezer/common": "^1.1.0",