From 04ae4d85b444aa9fed7f1e5a0b746cb6103439fe Mon Sep 17 00:00:00 2001 From: kaffarell Date: Fri, 28 Oct 2022 20:05:15 +0200 Subject: [PATCH 001/107] docs: changed from 'go install' to 'go get' in contributing.md 'go install' is not really the correct command. It does not add the package to the go.mod file when a specific version is used. The correct command is 'go get'. See here: https://go.dev/ref/mod#go-install Signed-off-by: kaffarell --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 844f8e09f..16fc40712 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,10 +64,10 @@ To add or update a new dependency, use the `go get` command: ```bash # Pick the latest tagged release. -go install example.com/some/module/pkg@latest +go get example.com/some/module/pkg@latest # Pick a specific version. -go install example.com/some/module/pkg@vX.Y.Z +go get example.com/some/module/pkg@vX.Y.Z ``` Tidy up the `go.mod` and `go.sum` files: From 23f87b3278a743b59e15263bf2ff842d646a5424 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Tue, 15 Nov 2022 09:10:43 +0100 Subject: [PATCH 002/107] Makefile: Fix targets order Signed-off-by: Julien Pivotto --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c4a9a06cf..4c40efd36 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ plugins/plugins.go: plugins.yml plugins/generate.go plugins: plugins/plugins.go .PHONY: build -build: assets npm_licenses assets-compress common-build plugins +build: assets npm_licenses assets-compress plugins common-build .PHONY: bench_tsdb bench_tsdb: $(PROMU) From f81fae24144a8e3ab41cd44de7e3819c0c783888 Mon Sep 17 00:00:00 2001 From: Levi Harrison Date: Tue, 22 Nov 2022 10:09:14 -0500 Subject: [PATCH 003/107] Add common HTTP client to AWS SDs (#11611) * Common client in EC2 and Lightsail Signed-off-by: Levi Harrison * Azure -> AWS Signed-off-by: Levi Harrison Signed-off-by: Levi Harrison --- config/config_test.go | 14 +++--- discovery/aws/ec2.go | 13 ++++- discovery/aws/lightsail.go | 13 ++++- docs/configuration/configuration.md | 78 +++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 10 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index ddf65a068..9ee8fe1a0 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -586,6 +586,7 @@ var expectedConf = &Config{ Values: []string{"web", "db"}, }, }, + HTTPClientConfig: config.DefaultHTTPClientConfig, }, }, }, @@ -602,12 +603,13 @@ var expectedConf = &Config{ ServiceDiscoveryConfigs: discovery.Configs{ &aws.LightsailSDConfig{ - Region: "us-east-1", - AccessKey: "access", - SecretKey: "mysecret", - Profile: "profile", - RefreshInterval: model.Duration(60 * time.Second), - Port: 80, + Region: "us-east-1", + AccessKey: "access", + SecretKey: "mysecret", + Profile: "profile", + RefreshInterval: model.Duration(60 * time.Second), + Port: 80, + HTTPClientConfig: config.DefaultHTTPClientConfig, }, }, }, diff --git a/discovery/aws/ec2.go b/discovery/aws/ec2.go index 7519f58da..ca9921159 100644 --- a/discovery/aws/ec2.go +++ b/discovery/aws/ec2.go @@ -66,8 +66,9 @@ const ( // DefaultEC2SDConfig is the default EC2 SD configuration. var DefaultEC2SDConfig = EC2SDConfig{ - Port: 80, - RefreshInterval: model.Duration(60 * time.Second), + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + HTTPClientConfig: config.DefaultHTTPClientConfig, } func init() { @@ -91,6 +92,8 @@ type EC2SDConfig struct { RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` Port int `yaml:"port"` Filters []*EC2Filter `yaml:"filters"` + + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` } // Name returns the name of the EC2 Config. @@ -171,11 +174,17 @@ func (d *EC2Discovery) ec2Client(ctx context.Context) (*ec2.EC2, error) { creds = nil } + client, err := config.NewClientFromConfig(d.cfg.HTTPClientConfig, "ec2_sd") + if err != nil { + return nil, err + } + sess, err := session.NewSessionWithOptions(session.Options{ Config: aws.Config{ Endpoint: &d.cfg.Endpoint, Region: &d.cfg.Region, Credentials: creds, + HTTPClient: client, }, Profile: d.cfg.Profile, }) diff --git a/discovery/aws/lightsail.go b/discovery/aws/lightsail.go index 016d78a67..e671769ca 100644 --- a/discovery/aws/lightsail.go +++ b/discovery/aws/lightsail.go @@ -56,8 +56,9 @@ const ( // DefaultLightsailSDConfig is the default Lightsail SD configuration. var DefaultLightsailSDConfig = LightsailSDConfig{ - Port: 80, - RefreshInterval: model.Duration(60 * time.Second), + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + HTTPClientConfig: config.DefaultHTTPClientConfig, } func init() { @@ -74,6 +75,8 @@ type LightsailSDConfig struct { RoleARN string `yaml:"role_arn,omitempty"` RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` Port int `yaml:"port"` + + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` } // Name returns the name of the Lightsail Config. @@ -144,11 +147,17 @@ func (d *LightsailDiscovery) lightsailClient() (*lightsail.Lightsail, error) { creds = nil } + client, err := config.NewClientFromConfig(d.cfg.HTTPClientConfig, "lightsail_sd") + if err != nil { + return nil, err + } + sess, err := session.NewSessionWithOptions(session.Options{ Config: aws.Config{ Endpoint: &d.cfg.Endpoint, Region: &d.cfg.Region, Credentials: creds, + HTTPClient: client, }, Profile: d.cfg.Profile, }) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 7345aca4a..c6edcf233 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -1064,6 +1064,45 @@ See below for the configuration options for EC2 discovery: filters: [ - name: values: , [...] ] + +# Authentication information used to authenticate to the EC2 API. +# Note that `basic_auth`, `authorization` and `oauth2` options are +# mutually exclusive. +# `password` and `password_file` are mutually exclusive. + +# Optional HTTP basic authentication information, currently not supported by AWS. +basic_auth: + [ username: ] + [ password: ] + [ password_file: ] + +# Optional `Authorization` header configuration, currently not supported by AWS. +authorization: + # Sets the authentication type. + [ type: | default: Bearer ] + # Sets the credentials. It is mutually exclusive with + # `credentials_file`. + [ credentials: ] + # Sets the credentials to the credentials read from the configured file. + # It is mutuall exclusive with `credentials`. + [ credentials_file: ] + +# Optional OAuth 2.0 configuration, currently not supported by AWS. +oauth2: + [ ] + +# Optional proxy URL. +[ proxy_url: ] + +# Configure whether HTTP requests follow HTTP 3xx redirects. +[ follow_redirects: | default = true ] + +# Whether to enable HTTP2. +[ enable_http2: | default: true ] + +# TLS configuration. +tls_config: + [ ] ``` The [relabeling phase](#relabel_config) is the preferred and more powerful @@ -2066,6 +2105,45 @@ See below for the configuration options for Lightsail discovery: # The port to scrape metrics from. If using the public IP address, this must # instead be specified in the relabeling rule. [ port: | default = 80 ] + +# Authentication information used to authenticate to the Lightsail API. +# Note that `basic_auth`, `authorization` and `oauth2` options are +# mutually exclusive. +# `password` and `password_file` are mutually exclusive. + +# Optional HTTP basic authentication information, currently not supported by AWS. +basic_auth: + [ username: ] + [ password: ] + [ password_file: ] + +# Optional `Authorization` header configuration, currently not supported by AWS. +authorization: + # Sets the authentication type. + [ type: | default: Bearer ] + # Sets the credentials. It is mutually exclusive with + # `credentials_file`. + [ credentials: ] + # Sets the credentials to the credentials read from the configured file. + # It is mutuall exclusive with `credentials`. + [ credentials_file: ] + +# Optional OAuth 2.0 configuration, currently not supported by AWS. +oauth2: + [ ] + +# Optional proxy URL. +[ proxy_url: ] + +# Configure whether HTTP requests follow HTTP 3xx redirects. +[ follow_redirects: | default = true ] + +# Whether to enable HTTP2. +[ enable_http2: | default: true ] + +# TLS configuration. +tls_config: + [ ] ``` ### `` From d0e683e26d2ae14602d9274034ea502d0974f483 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 23 Nov 2022 17:31:18 +0530 Subject: [PATCH 004/107] Add TestCompactHeadWithDeletion to test compaction failure after deletion Signed-off-by: Ganesh Vernekar --- tsdb/db_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 83ffb5dbc..d4c2840c2 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -2960,6 +2960,24 @@ func TestCompactHead(t *testing.T) { require.NoError(t, seriesSet.Err()) } +// TestCompactHeadWithDeletion tests https://github.com/prometheus/prometheus/issues/11585. +func TestCompactHeadWithDeletion(t *testing.T) { + db, err := Open(t.TempDir(), log.NewNopLogger(), prometheus.NewRegistry(), nil, nil) + require.NoError(t, err) + + app := db.Appender(context.Background()) + _, err = app.Append(0, labels.FromStrings("a", "b"), 10, rand.Float64()) + require.NoError(t, err) + require.NoError(t, app.Commit()) + + err = db.Delete(0, 100, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) + require.NoError(t, err) + + // This recreates the bug. + require.NoError(t, db.CompactHead(NewRangeHead(db.Head(), 0, 100))) + require.NoError(t, db.Close()) +} + func deleteNonBlocks(dbDir string) error { dirs, err := os.ReadDir(dbDir) if err != nil { From ad79fb9f25eadef795362327fa3f7846d262f96b Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 23 Nov 2022 17:32:28 +0530 Subject: [PATCH 005/107] Do not error on empty chunk during iteration in populateWithDelChunkSeriesIterator Signed-off-by: Ganesh Vernekar --- tsdb/querier.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tsdb/querier.go b/tsdb/querier.go index 70b384e02..0f9b4d3b4 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -705,12 +705,7 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if valueType == chunkenc.ValNone { if err := p.currDelIter.Err(); err != nil { p.err = errors.Wrap(err, "iterate chunk while re-encoding") - return false } - - // Empty chunk, this should not happen, as we assume full - // deletions being filtered before this iterator. - p.err = errors.New("populateWithDelChunkSeriesIterator: unexpected empty chunk found while rewriting chunk") return false } From aa3f8d0975888fe02bc7b6fb8e6fb38b46a4eac1 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 23 Nov 2022 17:36:25 +0530 Subject: [PATCH 006/107] Cut v2.40.3 Signed-off-by: Ganesh Vernekar --- CHANGELOG.md | 4 ++++ VERSION | 2 +- web/ui/module/codemirror-promql/package.json | 4 ++-- web/ui/module/lezer-promql/package.json | 2 +- web/ui/package-lock.json | 14 +++++++------- web/ui/react-app/package.json | 4 ++-- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69eacf5a2..a5b8c7107 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 2.40.3 / 2022-11-23 + +* [BUGFIX] TSDB: Fix compaction after a deletion is called. #11623 + ## 2.40.2 / 2022-11-16 * [BUGFIX] UI: Fix black-on-black metric name color in dark mode. #11572 diff --git a/VERSION b/VERSION index 29508491c..e3bb0d0de 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.40.2 +2.40.3 diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index 39728d2e2..72c9c040f 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.2", + "version": "0.40.3", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.2", + "@prometheus-io/lezer-promql": "^0.40.3", "lru-cache": "^6.0.0" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index ec887468b..7d6731fa4 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.40.2", + "version": "0.40.3", "description": "lezer-based PromQL grammar", "main": "index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index 1f3221c02..145c2602b 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -28,10 +28,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.2", + "version": "0.40.3", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.2", + "@prometheus-io/lezer-promql": "^0.40.3", "lru-cache": "^6.0.0" }, "devDependencies": { @@ -61,7 +61,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.40.2", + "version": "0.40.3", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.1.1", @@ -17625,7 +17625,7 @@ }, "react-app": { "name": "@prometheus-io/app", - "version": "0.40.2", + "version": "0.40.3", "dependencies": { "@codemirror/autocomplete": "^6.2.0", "@codemirror/commands": "^6.1.0", @@ -17643,7 +17643,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.2", + "@prometheus-io/codemirror-promql": "^0.40.3", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11", @@ -19883,7 +19883,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.2", + "@prometheus-io/codemirror-promql": "^0.40.3", "@testing-library/react-hooks": "^7.0.2", "@types/enzyme": "^3.10.12", "@types/flot": "0.0.32", @@ -19935,7 +19935,7 @@ "@lezer/common": "^1.0.1", "@lezer/highlight": "^1.1.0", "@lezer/lr": "^1.2.3", - "@prometheus-io/lezer-promql": "^0.40.2", + "@prometheus-io/lezer-promql": "^0.40.3", "@types/lru-cache": "^5.1.1", "isomorphic-fetch": "^3.0.0", "lru-cache": "^6.0.0", diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index 5b4a48be5..c4bc1475d 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/app", - "version": "0.40.2", + "version": "0.40.3", "private": true, "dependencies": { "@codemirror/autocomplete": "^6.2.0", @@ -19,7 +19,7 @@ "@lezer/common": "^1.0.1", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.2", + "@prometheus-io/codemirror-promql": "^0.40.3", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11", From 0c05f95e929f1b62f500c9ed4a806a768acc34aa Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 26 Nov 2022 14:56:22 +0000 Subject: [PATCH 007/107] tsdb: use smaller allocation in blockBaseSeriesSet This reduces garbage, hence goes faster, when a short time range is required compared to the amount of chunks in the block. For example recording rules and alerts often look only at the last few minutes. Signed-off-by: Bryan Boreham --- tsdb/querier.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tsdb/querier.go b/tsdb/querier.go index 70b384e02..cf6b3c9dc 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -471,7 +471,14 @@ func (b *blockBaseSeriesSet) Next() bool { var trimFront, trimBack bool // Copy chunks as iterables are reusable. - chks := make([]chunks.Meta, 0, len(b.bufChks)) + // Count those in range to size allocation (roughly - ignoring tombstones). + nChks := 0 + for _, chk := range b.bufChks { + if !(chk.MaxTime < b.mint || chk.MinTime > b.maxt) { + nChks++ + } + } + chks := make([]chunks.Meta, 0, nChks) // Prefilter chunks and pick those which are not entirely deleted or totally outside of the requested range. for _, chk := range b.bufChks { From 1226922ff549edd27a2b98fa9b6222b4c05620ab Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 26 Nov 2022 15:22:59 +0000 Subject: [PATCH 008/107] tsdb: improve blockBaseSeriesSet scan Inverting the test for chunks deleted by tombstones makes all three rejections consistent, and also avoids the case where a chunk is excluded but still causes `trimFront` or `trimBack` to be set. Signed-off-by: Bryan Boreham --- tsdb/querier.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tsdb/querier.go b/tsdb/querier.go index 70b384e02..035bf449c 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -481,10 +481,10 @@ func (b *blockBaseSeriesSet) Next() bool { if chk.MinTime > b.maxt { continue } - - if !(tombstones.Interval{Mint: chk.MinTime, Maxt: chk.MaxTime}.IsSubrange(intervals)) { - chks = append(chks, chk) + if (tombstones.Interval{Mint: chk.MinTime, Maxt: chk.MaxTime}.IsSubrange(intervals)) { + continue } + chks = append(chks, chk) // If still not entirely deleted, check if trim is needed based on requested time range. if !b.disableTrimming { From af838ccf83ffdf6021c3c17dc3e44881101a6f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Mon, 28 Nov 2022 09:12:54 +0100 Subject: [PATCH 009/107] Include source block in error message when loading chunk fails. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- tsdb/block_test.go | 6 +++--- tsdb/compact.go | 2 +- tsdb/querier.go | 18 +++++++++++++----- tsdb/querier_test.go | 13 +++++++------ 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/tsdb/block_test.go b/tsdb/block_test.go index d90f974bc..799efbf85 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -142,7 +142,7 @@ func TestCorruptedChunk(t *testing.T) { // Truncate one byte after the segment header. require.NoError(t, f.Truncate(chunks.SegmentHeaderSize+1)) }, - iterErr: errors.New("cannot populate chunk 8: segment doesn't include enough bytes to read the chunk size data field - required:13, available:9"), + iterErr: errors.New("cannot populate chunk 8 from block 00000000000000000000000000: segment doesn't include enough bytes to read the chunk size data field - required:13, available:9"), }, { name: "chunk not enough bytes to read the data", @@ -151,7 +151,7 @@ func TestCorruptedChunk(t *testing.T) { require.NoError(t, err) require.NoError(t, f.Truncate(fi.Size()-1)) }, - iterErr: errors.New("cannot populate chunk 8: segment doesn't include enough bytes to read the chunk - required:26, available:25"), + iterErr: errors.New("cannot populate chunk 8 from block 00000000000000000000000000: segment doesn't include enough bytes to read the chunk - required:26, available:25"), }, { name: "checksum mismatch", @@ -169,7 +169,7 @@ func TestCorruptedChunk(t *testing.T) { require.NoError(t, err) require.Equal(t, n, 1) }, - iterErr: errors.New("cannot populate chunk 8: checksum mismatch expected:cfc0526c, actual:34815eae"), + iterErr: errors.New("cannot populate chunk 8 from block 00000000000000000000000000: checksum mismatch expected:cfc0526c, actual:34815eae"), }, } { t.Run(tc.name, func(t *testing.T) { diff --git a/tsdb/compact.go b/tsdb/compact.go index 08fd27a31..9fe50fda1 100644 --- a/tsdb/compact.go +++ b/tsdb/compact.go @@ -727,7 +727,7 @@ func (c *LeveledCompactor) populateBlock(blocks []BlockReader, meta *BlockMeta, } all = indexr.SortedPostings(all) // Blocks meta is half open: [min, max), so subtract 1 to ensure we don't hold samples with exact meta.MaxTime timestamp. - sets = append(sets, newBlockChunkSeriesSet(indexr, chunkr, tombsr, all, meta.MinTime, meta.MaxTime-1, false)) + sets = append(sets, newBlockChunkSeriesSet(b.Meta().ULID, indexr, chunkr, tombsr, all, meta.MinTime, meta.MaxTime-1, false)) syms := indexr.Symbols() if i == 0 { symbols = syms diff --git a/tsdb/querier.go b/tsdb/querier.go index 70b384e02..9c7fcf2c1 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -19,6 +19,7 @@ import ( "strings" "unicode/utf8" + "github.com/oklog/ulid" "github.com/pkg/errors" "golang.org/x/exp/slices" @@ -47,6 +48,7 @@ func init() { } type blockBaseQuerier struct { + blockID ulid.ULID index IndexReader chunks ChunkReader tombstones tombstones.Reader @@ -77,6 +79,7 @@ func newBlockBaseQuerier(b BlockReader, mint, maxt int64) (*blockBaseQuerier, er tombsr = tombstones.NewMemTombstones() } return &blockBaseQuerier{ + blockID: b.Meta().ULID, mint: mint, maxt: maxt, index: indexr, @@ -178,7 +181,7 @@ func (q *blockChunkQuerier) Select(sortSeries bool, hints *storage.SelectHints, if sortSeries { p = q.index.SortedPostings(p) } - return newBlockChunkSeriesSet(q.index, q.chunks, q.tombstones, p, mint, maxt, disableTrimming) + return newBlockChunkSeriesSet(q.blockID, q.index, q.chunks, q.tombstones, p, mint, maxt, disableTrimming) } func findSetMatches(pattern string) []string { @@ -427,6 +430,7 @@ func labelNamesWithMatchers(r IndexReader, matchers ...*labels.Matcher) ([]strin // Iterated series are trimmed with given min and max time as well as tombstones. // See newBlockSeriesSet and newBlockChunkSeriesSet to use it for either sample or chunk iterating. type blockBaseSeriesSet struct { + blockID ulid.ULID p index.Postings index IndexReader chunks ChunkReader @@ -512,7 +516,7 @@ func (b *blockBaseSeriesSet) Next() bool { copy(b.currLabels, b.bufLbls) b.currIterFn = func() *populateWithDelGenericSeriesIterator { - return newPopulateWithDelGenericSeriesIterator(b.chunks, chks, intervals) + return newPopulateWithDelGenericSeriesIterator(b.blockID, b.chunks, chks, intervals) } return true } @@ -539,7 +543,8 @@ func (b *blockBaseSeriesSet) Warnings() storage.Warnings { return nil } // means that the chunk iterator in currChkMeta is invalid and a chunk rewrite // is needed, for which currDelIter should be used. type populateWithDelGenericSeriesIterator struct { - chunks ChunkReader + blockID ulid.ULID + chunks ChunkReader // chks are expected to be sorted by minTime and should be related to // the same, single series. chks []chunks.Meta @@ -554,11 +559,13 @@ type populateWithDelGenericSeriesIterator struct { } func newPopulateWithDelGenericSeriesIterator( + blockID ulid.ULID, chunks ChunkReader, chks []chunks.Meta, intervals tombstones.Intervals, ) *populateWithDelGenericSeriesIterator { return &populateWithDelGenericSeriesIterator{ + blockID: blockID, chunks: chunks, chks: chks, i: -1, @@ -577,7 +584,7 @@ func (p *populateWithDelGenericSeriesIterator) next() bool { p.currChkMeta.Chunk, p.err = p.chunks.Chunk(p.currChkMeta) if p.err != nil { - p.err = errors.Wrapf(p.err, "cannot populate chunk %d", p.currChkMeta.Ref) + p.err = errors.Wrapf(p.err, "cannot populate chunk %d from block %s", p.currChkMeta.Ref, p.blockID.String()) return false } @@ -842,9 +849,10 @@ type blockChunkSeriesSet struct { blockBaseSeriesSet } -func newBlockChunkSeriesSet(i IndexReader, c ChunkReader, t tombstones.Reader, p index.Postings, mint, maxt int64, disableTrimming bool) storage.ChunkSeriesSet { +func newBlockChunkSeriesSet(id ulid.ULID, i IndexReader, c ChunkReader, t tombstones.Reader, p index.Postings, mint, maxt int64, disableTrimming bool) storage.ChunkSeriesSet { return &blockChunkSeriesSet{ blockBaseSeriesSet{ + blockID: id, index: i, chunks: c, tombstones: t, diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 199234bed..ffb24b17b 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/stretchr/testify/require" @@ -858,7 +859,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Run("sample", func(t *testing.T) { f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) - it := newPopulateWithDelGenericSeriesIterator(f, chkMetas, tc.intervals).toSeriesIterator() + it := newPopulateWithDelGenericSeriesIterator(ulid.ULID{}, f, chkMetas, tc.intervals).toSeriesIterator() var r []tsdbutil.Sample if tc.seek != 0 { @@ -878,7 +879,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }) t.Run("chunk", func(t *testing.T) { f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) - it := newPopulateWithDelGenericSeriesIterator(f, chkMetas, tc.intervals).toChunkSeriesIterator() + it := newPopulateWithDelGenericSeriesIterator(ulid.ULID{}, f, chkMetas, tc.intervals).toChunkSeriesIterator() if tc.seek != 0 { // Chunk iterator does not have Seek method. @@ -910,7 +911,7 @@ func TestPopulateWithDelSeriesIterator_DoubleSeek(t *testing.T) { []tsdbutil.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, ) - it := newPopulateWithDelGenericSeriesIterator(f, chkMetas, nil).toSeriesIterator() + it := newPopulateWithDelGenericSeriesIterator(ulid.ULID{}, f, chkMetas, nil).toSeriesIterator() require.Equal(t, chunkenc.ValFloat, it.Seek(1)) require.Equal(t, chunkenc.ValFloat, it.Seek(2)) require.Equal(t, chunkenc.ValFloat, it.Seek(2)) @@ -928,7 +929,7 @@ func TestPopulateWithDelSeriesIterator_SeekInCurrentChunk(t *testing.T) { []tsdbutil.Sample{}, ) - it := newPopulateWithDelGenericSeriesIterator(f, chkMetas, nil).toSeriesIterator() + it := newPopulateWithDelGenericSeriesIterator(ulid.ULID{}, f, chkMetas, nil).toSeriesIterator() require.Equal(t, chunkenc.ValFloat, it.Next()) ts, v := it.At() require.Equal(t, int64(1), ts) @@ -945,7 +946,7 @@ func TestPopulateWithDelSeriesIterator_SeekWithMinTime(t *testing.T) { []tsdbutil.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, ) - it := newPopulateWithDelGenericSeriesIterator(f, chkMetas, nil).toSeriesIterator() + it := newPopulateWithDelGenericSeriesIterator(ulid.ULID{}, f, chkMetas, nil).toSeriesIterator() require.Equal(t, chunkenc.ValNone, it.Seek(7)) require.Equal(t, chunkenc.ValFloat, it.Seek(3)) } @@ -958,7 +959,7 @@ func TestPopulateWithDelSeriesIterator_NextWithMinTime(t *testing.T) { ) it := newPopulateWithDelGenericSeriesIterator( - f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64}), + ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64}), ).toSeriesIterator() require.Equal(t, chunkenc.ValNone, it.Next()) } From 8d4140a06e44245b55bc01ccc14affc027162ad0 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 28 Nov 2022 16:16:55 +0000 Subject: [PATCH 010/107] labels: note that Hash may change For performance reasons we may use a different implementation of Hash() in future, so note this so callers can be warned. Signed-off-by: Bryan Boreham --- model/labels/labels.go | 1 + 1 file changed, 1 insertion(+) diff --git a/model/labels/labels.go b/model/labels/labels.go index 48237bdc0..d33fe0e29 100644 --- a/model/labels/labels.go +++ b/model/labels/labels.go @@ -134,6 +134,7 @@ func (ls Labels) MatchLabels(on bool, names ...string) Labels { } // Hash returns a hash value for the label set. +// Note: the result is not guaranteed to be consistent across different runs of Prometheus. func (ls Labels) Hash() uint64 { // Use xxhash.Sum64(b) for fast path as it's faster. b := make([]byte, 0, 1024) From 6bdecf377cea8e856509914f35234e948c4fcb80 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 28 Nov 2022 17:09:18 +0000 Subject: [PATCH 011/107] Switch from 'sanity' to more inclusive lanuage (#9376) * Switch from 'sanity' to more inclusive lanuage "Removing ableist language in code is important; it helps to create and maintain an environment that welcomes all developers of all backgrounds, while emphasizing that we as developers select the most articulate, precise, descriptive language we can rather than relying on metaphors. The phrase sanity check is ableist, and unnecessarily references mental health in our code bases. It denotes that people with mental illnesses are inferior, wrong, or incorrect, and the phrase sanity continues to be used by employers and other individuals to discriminate against these people." From https://gist.github.com/seanmhanson/fe370c2d8bd2b3228680e38899baf5cc Signed-off-by: Bryan Boreham --- promql/functions.go | 2 +- promql/parser/parse.go | 2 +- promql/test.go | 4 ++-- tsdb/agent/db.go | 2 +- tsdb/db.go | 2 +- tsdb/head_append.go | 2 +- web/ui/react-app/src/utils/utils.test.ts | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/promql/functions.go b/promql/functions.go index 8e9e6f965..d481cb735 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -302,7 +302,7 @@ func funcHoltWinters(vals []parser.Value, args parser.Expressions, enh *EvalNode // The trend factor argument. tf := vals[2].(Vector)[0].V - // Sanity check the input. + // Check that the input parameters are valid. if sf <= 0 || sf >= 1 { panic(fmt.Errorf("invalid smoothing factor. Expected: 0 < sf < 1, got: %f", sf)) } diff --git a/promql/parser/parse.go b/promql/parser/parse.go index 7a0e48464..6c37ce6fc 100644 --- a/promql/parser/parse.go +++ b/promql/parser/parse.go @@ -432,7 +432,7 @@ func (p *parser) expectType(node Node, want ValueType, context string) { } } -// checkAST checks the sanity of the provided AST. This includes type checking. +// checkAST checks the validity of the provided AST. This includes type checking. func (p *parser) checkAST(node Node) (typ ValueType) { // For expressions the type is determined by their Type function. // Lists do not have a type but are not invalid either. diff --git a/promql/test.go b/promql/test.go index 41526ccdc..ab2a159c4 100644 --- a/promql/test.go +++ b/promql/test.go @@ -491,8 +491,8 @@ func atModifierTestCases(exprStr string, evalTime time.Time) ([]atModifierTestCa }) if containsNonStepInvariant { - // Since there is a step invariant function, we cannot automatically - // generate step invariant test cases for it sanely. + // Expression contains a function whose result can vary with evaluation + // time, even though its arguments are step invariant: skip it. return nil, nil } diff --git a/tsdb/agent/db.go b/tsdb/agent/db.go index 9574c4b8a..0675d5a28 100644 --- a/tsdb/agent/db.go +++ b/tsdb/agent/db.go @@ -77,7 +77,7 @@ type Options struct { NoLockfile bool } -// DefaultOptions used for the WAL storage. They are sane for setups using +// DefaultOptions used for the WAL storage. They are reasonable for setups using // millisecond-precision timestamps. func DefaultOptions() *Options { return &Options{ diff --git a/tsdb/db.go b/tsdb/db.go index d18f4fee6..7a165e431 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -66,7 +66,7 @@ const ( // ErrNotReady is returned if the underlying storage is not ready yet. var ErrNotReady = errors.New("TSDB not ready") -// DefaultOptions used for the DB. They are sane for setups using +// DefaultOptions used for the DB. They are reasonable for setups using // millisecond precision timestamps. func DefaultOptions() *Options { return &Options{ diff --git a/tsdb/head_append.go b/tsdb/head_append.go index 8de71114f..d5003188d 100644 --- a/tsdb/head_append.go +++ b/tsdb/head_append.go @@ -440,7 +440,7 @@ func (s *memSeries) appendableHistogram(t int64, h *histogram.Histogram) error { } // AppendExemplar for headAppender assumes the series ref already exists, and so it doesn't -// use getOrCreate or make any of the lset sanity checks that Append does. +// use getOrCreate or make any of the lset validity checks that Append does. func (a *headAppender) AppendExemplar(ref storage.SeriesRef, lset labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { // Check if exemplar storage is enabled. if !a.head.opts.EnableExemplarStorage || a.head.opts.MaxExemplars.Load() <= 0 { diff --git a/web/ui/react-app/src/utils/utils.test.ts b/web/ui/react-app/src/utils/utils.test.ts index 99a72d7b8..4db3e28a6 100644 --- a/web/ui/react-app/src/utils/utils.test.ts +++ b/web/ui/react-app/src/utils/utils.test.ts @@ -190,7 +190,7 @@ describe('Utils', () => { it('renders never for pre-beginning-of-time strings', () => { expect(formatRelative('0001-01-01T00:00:00Z', now())).toEqual('Never'); }); - it('renders a humanized duration for sane durations', () => { + it('renders a humanized duration for durations', () => { expect(formatRelative('2019-11-04T09:15:29.578701-07:00', parseTime('2019-11-04T09:15:35.8701-07:00'))).toEqual( '6.292s ago' ); From 31a2db3ae9c0f4b486b6895973beabc1d1beac93 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Tue, 29 Nov 2022 10:44:09 +0100 Subject: [PATCH 012/107] Fix Basic Authentication Bypass (CVE-2022-46146) Signed-off-by: Julien Pivotto --- CHANGELOG.md | 4 ++++ VERSION | 2 +- go.mod | 2 +- go.sum | 4 ++-- web/ui/module/codemirror-promql/package.json | 4 ++-- web/ui/module/lezer-promql/package.json | 2 +- web/ui/package-lock.json | 14 +++++++------- web/ui/react-app/package.json | 4 ++-- 8 files changed, 20 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5b8c7107..a68ed6627 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 2.40.4 / 2022-11-29 + +* [SECURITY] Fix basic authentication bypass vulnerability (CVE-2022-46146). GHSA-4v48-4q5m-8vx4 + ## 2.40.3 / 2022-11-23 * [BUGFIX] TSDB: Fix compaction after a deletion is called. #11623 diff --git a/VERSION b/VERSION index e3bb0d0de..2d6c3fe67 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.40.3 +2.40.4 diff --git a/go.mod b/go.mod index 56aff971c..e1ee2751a 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/prometheus/common v0.37.0 github.com/prometheus/common/assets v0.2.0 github.com/prometheus/common/sigv4 v0.1.0 - github.com/prometheus/exporter-toolkit v0.8.1 + github.com/prometheus/exporter-toolkit v0.8.2 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 github.com/stretchr/testify v1.8.1 diff --git a/go.sum b/go.sum index bf56c6562..8cac275fb 100644 --- a/go.sum +++ b/go.sum @@ -704,8 +704,8 @@ github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwB github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= github.com/prometheus/exporter-toolkit v0.7.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= -github.com/prometheus/exporter-toolkit v0.8.1 h1:TpKt8z55q1zF30BYaZKqh+bODY0WtByHDOhDA2M9pEs= -github.com/prometheus/exporter-toolkit v0.8.1/go.mod h1:00shzmJL7KxcsabLWcONwpyNEuWhREOnFqZW7vadFS0= +github.com/prometheus/exporter-toolkit v0.8.2 h1:sbJAfBXQFkG6sUkbwBun8MNdzW9+wd5YfPYofbmj0YM= +github.com/prometheus/exporter-toolkit v0.8.2/go.mod h1:00shzmJL7KxcsabLWcONwpyNEuWhREOnFqZW7vadFS0= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index 72c9c040f..0b8743728 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.3", + "version": "0.40.4", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.3", + "@prometheus-io/lezer-promql": "^0.40.4", "lru-cache": "^6.0.0" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index 7d6731fa4..6a8beb98a 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.40.3", + "version": "0.40.4", "description": "lezer-based PromQL grammar", "main": "index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index 145c2602b..23afbdca7 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -28,10 +28,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.3", + "version": "0.40.4", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.3", + "@prometheus-io/lezer-promql": "^0.40.4", "lru-cache": "^6.0.0" }, "devDependencies": { @@ -61,7 +61,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.40.3", + "version": "0.40.4", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.1.1", @@ -17625,7 +17625,7 @@ }, "react-app": { "name": "@prometheus-io/app", - "version": "0.40.3", + "version": "0.40.4", "dependencies": { "@codemirror/autocomplete": "^6.2.0", "@codemirror/commands": "^6.1.0", @@ -17643,7 +17643,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.3", + "@prometheus-io/codemirror-promql": "^0.40.4", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11", @@ -19883,7 +19883,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.3", + "@prometheus-io/codemirror-promql": "^0.40.4", "@testing-library/react-hooks": "^7.0.2", "@types/enzyme": "^3.10.12", "@types/flot": "0.0.32", @@ -19935,7 +19935,7 @@ "@lezer/common": "^1.0.1", "@lezer/highlight": "^1.1.0", "@lezer/lr": "^1.2.3", - "@prometheus-io/lezer-promql": "^0.40.3", + "@prometheus-io/lezer-promql": "^0.40.4", "@types/lru-cache": "^5.1.1", "isomorphic-fetch": "^3.0.0", "lru-cache": "^6.0.0", diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index c4bc1475d..5bd05ca32 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/app", - "version": "0.40.3", + "version": "0.40.4", "private": true, "dependencies": { "@codemirror/autocomplete": "^6.2.0", @@ -19,7 +19,7 @@ "@lezer/common": "^1.0.1", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.3", + "@prometheus-io/codemirror-promql": "^0.40.4", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11", From b8b0d45d6982a1587acb669409982b6917620db0 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 30 Nov 2022 17:49:33 +0530 Subject: [PATCH 013/107] Fix reset of a histogram chunk iterator Signed-off-by: Ganesh Vernekar --- tsdb/chunkenc/histogram.go | 4 ++-- tsdb/chunkenc/histogram_test.go | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tsdb/chunkenc/histogram.go b/tsdb/chunkenc/histogram.go index 6e326888a..e57b96db9 100644 --- a/tsdb/chunkenc/histogram.go +++ b/tsdb/chunkenc/histogram.go @@ -624,9 +624,9 @@ func (it *histogramIterator) Err() error { } func (it *histogramIterator) Reset(b []byte) { - // The first 2 bytes contain chunk headers. + // The first 3 bytes contain chunk headers. // We skip that for actual samples. - it.br = newBReader(b[2:]) + it.br = newBReader(b[3:]) it.numTotal = binary.BigEndian.Uint16(b) it.numRead = 0 diff --git a/tsdb/chunkenc/histogram_test.go b/tsdb/chunkenc/histogram_test.go index 1a187667c..09c08ae3c 100644 --- a/tsdb/chunkenc/histogram_test.go +++ b/tsdb/chunkenc/histogram_test.go @@ -84,14 +84,14 @@ func TestHistogramChunkSameBuckets(t *testing.T) { require.Equal(t, exp, act) // 2. Expand second iterator while reusing first one. - // it2 := c.Iterator(it1) - // var res2 []pair - // for it2.Next() { - // ts, v := it2.At() - // res2 = append(res2, pair{t: ts, v: v}) - // } - // require.NoError(t, it2.Err()) - // require.Equal(t, exp, res2) + it2 := c.Iterator(it) + var res2 []res + for it2.Next() == ValHistogram { + ts, h := it2.AtHistogram() + res2 = append(res2, res{t: ts, h: h}) + } + require.NoError(t, it2.Err()) + require.Equal(t, exp, res2) // 3. Test iterator Seek. // mid := len(exp) / 2 From 44af4716c86138869aa621737139e6dacf0e2550 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Thu, 1 Dec 2022 18:16:11 +0530 Subject: [PATCH 014/107] Cut v2.40.5 (#11646) Signed-off-by: Ganesh Vernekar --- CHANGELOG.md | 4 ++++ VERSION | 2 +- web/ui/module/codemirror-promql/package.json | 4 ++-- web/ui/module/lezer-promql/package.json | 2 +- web/ui/package-lock.json | 14 +++++++------- web/ui/react-app/package.json | 4 ++-- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a68ed6627..4dba2e19d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 2.40.5 / 2022-12-01 + +* [BUGFIX] TSDB: Fix queries involving native histograms due to improper reset of iterators. #11643 + ## 2.40.4 / 2022-11-29 * [SECURITY] Fix basic authentication bypass vulnerability (CVE-2022-46146). GHSA-4v48-4q5m-8vx4 diff --git a/VERSION b/VERSION index 2d6c3fe67..0644c8fe6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.40.4 +2.40.5 diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index 0b8743728..7eab09449 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.4", + "version": "0.40.5", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.4", + "@prometheus-io/lezer-promql": "^0.40.5", "lru-cache": "^6.0.0" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index 6a8beb98a..0ffb9e5f7 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.40.4", + "version": "0.40.5", "description": "lezer-based PromQL grammar", "main": "index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index 23afbdca7..bc5b81c6e 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -28,10 +28,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.4", + "version": "0.40.5", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.4", + "@prometheus-io/lezer-promql": "^0.40.5", "lru-cache": "^6.0.0" }, "devDependencies": { @@ -61,7 +61,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.40.4", + "version": "0.40.5", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.1.1", @@ -17625,7 +17625,7 @@ }, "react-app": { "name": "@prometheus-io/app", - "version": "0.40.4", + "version": "0.40.5", "dependencies": { "@codemirror/autocomplete": "^6.2.0", "@codemirror/commands": "^6.1.0", @@ -17643,7 +17643,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.4", + "@prometheus-io/codemirror-promql": "^0.40.5", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11", @@ -19883,7 +19883,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.4", + "@prometheus-io/codemirror-promql": "^0.40.5", "@testing-library/react-hooks": "^7.0.2", "@types/enzyme": "^3.10.12", "@types/flot": "0.0.32", @@ -19935,7 +19935,7 @@ "@lezer/common": "^1.0.1", "@lezer/highlight": "^1.1.0", "@lezer/lr": "^1.2.3", - "@prometheus-io/lezer-promql": "^0.40.4", + "@prometheus-io/lezer-promql": "^0.40.5", "@types/lru-cache": "^5.1.1", "isomorphic-fetch": "^3.0.0", "lru-cache": "^6.0.0", diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index 5bd05ca32..f1b5edca6 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/app", - "version": "0.40.4", + "version": "0.40.5", "private": true, "dependencies": { "@codemirror/autocomplete": "^6.2.0", @@ -19,7 +19,7 @@ "@lezer/common": "^1.0.1", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.4", + "@prometheus-io/codemirror-promql": "^0.40.5", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11", From 9619d3fd3b05122d2fad9d80391b100f976d3764 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 4 Dec 2022 16:49:12 +0000 Subject: [PATCH 015/107] notifier: remove unused code None of the actions on `lb` have any effect because its result is not read. Signed-off-by: Bryan Boreham --- notifier/notifier.go | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/notifier/notifier.go b/notifier/notifier.go index 99886bd4b..fd89a029c 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -19,11 +19,9 @@ import ( "encoding/json" "fmt" "io" - "net" "net/http" "net/url" "path" - "strings" "sync" "time" @@ -727,47 +725,11 @@ func AlertmanagerFromGroup(tg *targetgroup.Group, cfg *config.AlertmanagerConfig continue } - lb := labels.NewBuilder(lset) - - // addPort checks whether we should add a default port to the address. - // If the address is not valid, we don't append a port either. - addPort := func(s string) bool { - // If we can split, a port exists and we don't have to add one. - if _, _, err := net.SplitHostPort(s); err == nil { - return false - } - // If adding a port makes it valid, the previous error - // was not due to an invalid address and we can append a port. - _, _, err := net.SplitHostPort(s + ":1234") - return err == nil - } addr := lset.Get(model.AddressLabel) - // If it's an address with no trailing port, infer it based on the used scheme. - if addPort(addr) { - // Addresses reaching this point are already wrapped in [] if necessary. - switch lset.Get(model.SchemeLabel) { - case "http", "": - addr = addr + ":80" - case "https": - addr = addr + ":443" - default: - return nil, nil, fmt.Errorf("invalid scheme: %q", cfg.Scheme) - } - lb.Set(model.AddressLabel, addr) - } - if err := config.CheckTargetAddress(model.LabelValue(addr)); err != nil { return nil, nil, err } - // Meta labels are deleted after relabelling. Other internal labels propagate to - // the target which decides whether they will be part of their label set. - for _, l := range lset { - if strings.HasPrefix(l.Name, model.MetaLabelPrefix) { - lb.Del(l.Name) - } - } - res = append(res, alertmanagerLabels{lset}) } return res, droppedAlertManagers, nil From 9853888f9be07783e29a7df4ffe321a5f74ef560 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 5 Dec 2022 17:15:02 +0000 Subject: [PATCH 016/107] tsdb tests: allocate more reasonable sample slice Typical parameters are one hour by 1 minute step, where the function would allocate a slice of 3.6 million samples instead of 60. Signed-off-by: Bryan Boreham --- tsdb/block_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsdb/block_test.go b/tsdb/block_test.go index 799efbf85..6cb00b348 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -678,7 +678,7 @@ func genSeriesFromSampleGenerator(totalSeries, labelCount int, mint, maxt, step for j := 1; len(lbls) < labelCount; j++ { lbls[defaultLabelName+strconv.Itoa(j)] = defaultLabelValue + strconv.Itoa(j) } - samples := make([]tsdbutil.Sample, 0, maxt-mint+1) + samples := make([]tsdbutil.Sample, 0, (maxt-mint)/step+1) for t := mint; t < maxt; t += step { samples = append(samples, generator(t)) } From 9979024a30b75eecdc49b4b7aa2af1c5f1b0c182 Mon Sep 17 00:00:00 2001 From: "Xiaochao Dong (@damnever)" Date: Thu, 8 Dec 2022 11:09:43 +0800 Subject: [PATCH 017/107] Report error if the series contains invalid metric names or labels during scrape Signed-off-by: Xiaochao Dong (@damnever) --- model/labels/labels.go | 14 ++++++++++ model/labels/labels_test.go | 56 +++++++++++++++++++++++++++++++++++++ scrape/scrape.go | 4 +++ scrape/scrape_test.go | 45 +++++++++++++++++++++++++++++ 4 files changed, 119 insertions(+) diff --git a/model/labels/labels.go b/model/labels/labels.go index 48237bdc0..7652b4e12 100644 --- a/model/labels/labels.go +++ b/model/labels/labels.go @@ -20,6 +20,7 @@ import ( "strconv" "github.com/cespare/xxhash/v2" + "github.com/prometheus/common/model" ) // Well-known label names used by Prometheus components. @@ -311,6 +312,19 @@ func (ls Labels) WithoutEmpty() Labels { return ls } +// IsValid checks if the metric name or label names are valid. +func (ls Labels) IsValid() bool { + for _, l := range ls { + if l.Name == model.MetricNameLabel && !model.IsValidMetricName(model.LabelValue(l.Value)) { + return false + } + if !model.LabelName(l.Name).IsValid() || !model.LabelValue(l.Value).IsValid() { + return false + } + } + return true +} + // Equal returns whether the two label sets are equal. func Equal(ls, o Labels) bool { if len(ls) != len(o) { diff --git a/model/labels/labels_test.go b/model/labels/labels_test.go index 57ccf1fef..0fd0edacc 100644 --- a/model/labels/labels_test.go +++ b/model/labels/labels_test.go @@ -216,6 +216,62 @@ func TestLabels_WithoutEmpty(t *testing.T) { } } +func TestLabels_IsValid(t *testing.T) { + for _, test := range []struct { + input Labels + expected bool + }{ + { + input: FromStrings( + "__name__", "test", + "hostname", "localhost", + "job", "check", + ), + expected: true, + }, + { + input: FromStrings( + "__name__", "test:ms", + "hostname_123", "localhost", + "_job", "check", + ), + expected: true, + }, + { + input: FromStrings("__name__", "test-ms"), + expected: false, + }, + { + input: FromStrings("__name__", "0zz"), + expected: false, + }, + { + input: FromStrings("abc:xyz", "invalid"), + expected: false, + }, + { + input: FromStrings("123abc", "invalid"), + expected: false, + }, + { + input: FromStrings("中文abc", "invalid"), + expected: false, + }, + { + input: FromStrings("invalid", "aa\xe2"), + expected: false, + }, + { + input: FromStrings("invalid", "\xF7\xBF\xBF\xBF"), + expected: false, + }, + } { + t.Run("", func(t *testing.T) { + require.Equal(t, test.expected, test.input.IsValid()) + }) + } +} + func TestLabels_Equal(t *testing.T) { labels := FromStrings( "aaa", "111", diff --git a/scrape/scrape.go b/scrape/scrape.go index 04375ab56..e9d172a3e 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -1609,6 +1609,10 @@ loop: err = errNameLabelMandatory break loop } + if !lset.IsValid() { + err = fmt.Errorf("invalid metric name or label names: %s", lset.String()) + break loop + } // If any label limits is exceeded the scrape should fail. if err = verifyLabelLimits(lset, sl.labelLimits); err != nil { diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index 27749dbc6..b22f7f095 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -1020,6 +1020,51 @@ func TestScrapeLoopSeriesAdded(t *testing.T) { require.Equal(t, 0, seriesAdded) } +func TestScrapeLoopFailWithInvalidLabelsAfterRelabel(t *testing.T) { + s := teststorage.New(t) + defer s.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + target := &Target{ + labels: labels.FromStrings("pod_label_invalid_012", "test"), + } + relabelConfig := []*relabel.Config{{ + Action: relabel.LabelMap, + Regex: relabel.MustNewRegexp("pod_label_invalid_(.+)"), + Separator: ";", + Replacement: "$1", + }} + sl := newScrapeLoop(ctx, + &testScraper{}, + nil, nil, + func(l labels.Labels) labels.Labels { + return mutateSampleLabels(l, target, true, relabelConfig) + }, + nopMutator, + s.Appender, + nil, + 0, + true, + 0, + nil, + 0, + 0, + false, + false, + nil, + false, + ) + + slApp := sl.appender(ctx) + total, added, seriesAdded, err := sl.append(slApp, []byte("test_metric 1\n"), "", time.Time{}) + require.ErrorContains(t, err, "invalid metric name or label names") + require.NoError(t, slApp.Rollback()) + require.Equal(t, 1, total) + require.Equal(t, 0, added) + require.Equal(t, 0, seriesAdded) +} + func makeTestMetrics(n int) []byte { // Construct a metrics string to parse sb := bytes.Buffer{} From 01382cadc3e4e8e48b2d870034c01db37ecdcaf9 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Thu, 8 Dec 2022 14:49:12 +0100 Subject: [PATCH 018/107] Update Prometheus/common - Check if TLS certificate and key file have been modified https://github.com/prometheus/common/issues/345 - Add the ability to specify the maximum acceptable TLS version https://github.com/prometheus/common/issues/414 Signed-off-by: Julien Pivotto --- docs/configuration/configuration.md | 5 +++++ go.mod | 5 ++--- go.sum | 10 ++++------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index c6edcf233..701c62546 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -389,6 +389,11 @@ A `tls_config` allows configuring TLS connections. # If unset, Prometheus will use Go default minimum version, which is TLS 1.2. # See MinVersion in https://pkg.go.dev/crypto/tls#Config. [ min_version: ] +# Maximum acceptable TLS version. Accepted values: TLS10 (TLS 1.0), TLS11 (TLS +# 1.1), TLS12 (TLS 1.2), TLS13 (TLS 1.3). +# If unset, Prometheus will use Go default maximum version, which is TLS 1.3. +# See MaxVersion in https://pkg.go.dev/crypto/tls#Config. +[ max_version: ] ``` ### `` diff --git a/go.mod b/go.mod index dc9a0c2e2..398b17b9b 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/prometheus/alertmanager v0.24.0 github.com/prometheus/client_golang v1.13.1 github.com/prometheus/client_model v0.3.0 - github.com/prometheus/common v0.37.0 + github.com/prometheus/common v0.38.0 github.com/prometheus/common/assets v0.2.0 github.com/prometheus/common/sigv4 v0.1.0 github.com/prometheus/exporter-toolkit v0.8.2 @@ -153,7 +153,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect @@ -178,7 +178,6 @@ require ( golang.org/x/term v0.1.0 // indirect golang.org/x/text v0.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.6 // indirect gotest.tools/v3 v3.0.3 // indirect diff --git a/go.sum b/go.sum index ecae3bd58..343f1c710 100644 --- a/go.sum +++ b/go.sum @@ -570,8 +570,8 @@ github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -697,8 +697,8 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.38.0 h1:VTQitp6mXTdUoCmDMugDVOJ1opi6ADftKfp/yeqTR/E= +github.com/prometheus/common v0.38.0/go.mod h1:MBXfmBQZrK5XpbCkjofnXs96LD2QQ7fEq4C0xjC/yec= github.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM= github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI= github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= @@ -989,7 +989,6 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.1.0 h1:isLCZuhj4v+tYv7eskaN4v/TM+A1begWWgyVJDdl1+Y= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1296,7 +1295,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= From 5198eac6f425ce094115638cfaace0560782c66b Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Fri, 9 Dec 2022 11:04:13 +0100 Subject: [PATCH 019/107] Release 2.40.6 with security fixes Signed-off-by: Julien Pivotto --- CHANGELOG.md | 5 +++++ VERSION | 2 +- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- web/ui/module/codemirror-promql/package.json | 4 ++-- web/ui/module/lezer-promql/package.json | 2 +- web/ui/package-lock.json | 14 +++++++------- web/ui/react-app/package.json | 4 ++-- 8 files changed, 33 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dba2e19d..071022709 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 2.40.6 / 2022-12-09 + +* [SECURITY] Security upgrade from go and upstream dependencies that include + security fixes to the net/http and os packages. #11691 + ## 2.40.5 / 2022-12-01 * [BUGFIX] TSDB: Fix queries involving native histograms due to improper reset of iterators. #11643 diff --git a/VERSION b/VERSION index 0644c8fe6..ad572faeb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.40.5 +2.40.6 diff --git a/go.mod b/go.mod index e1ee2751a..5fccf86fe 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/prometheus/alertmanager v0.24.0 github.com/prometheus/client_golang v1.13.1 github.com/prometheus/client_model v0.3.0 - github.com/prometheus/common v0.37.0 + github.com/prometheus/common v0.37.1 github.com/prometheus/common/assets v0.2.0 github.com/prometheus/common/sigv4 v0.1.0 github.com/prometheus/exporter-toolkit v0.8.2 @@ -61,10 +61,10 @@ require ( go.uber.org/atomic v1.10.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/goleak v1.2.0 - golang.org/x/net v0.1.0 + golang.org/x/net v0.4.0 golang.org/x/oauth2 v0.1.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.1.0 + golang.org/x/sys v0.3.0 golang.org/x/time v0.1.0 golang.org/x/tools v0.2.0 google.golang.org/api v0.102.0 @@ -175,8 +175,8 @@ require ( golang.org/x/crypto v0.1.0 // indirect golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 golang.org/x/mod v0.6.0 // indirect - golang.org/x/term v0.1.0 // indirect - golang.org/x/text v0.4.0 // indirect + golang.org/x/term v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 8cac275fb..8fefa97cf 100644 --- a/go.sum +++ b/go.sum @@ -697,8 +697,8 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.37.1 h1:pYY6b5sGXqEB0WwcRGAoVGKbxVthy9qF17R4gbHZVe0= +github.com/prometheus/common v0.37.1/go.mod h1:jEuMeTn4pKGSAxwr7rXtOD70GeY0ERpt0d9FkKf9sK4= github.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM= github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI= github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= @@ -977,8 +977,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220907135653-1e95f45603a7/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1084,13 +1084,13 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908150016-7ac13a9a928d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1100,8 +1100,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index 7eab09449..1b86f596f 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.5", + "version": "0.40.6", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.5", + "@prometheus-io/lezer-promql": "^0.40.6", "lru-cache": "^6.0.0" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index 0ffb9e5f7..b429732a7 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.40.5", + "version": "0.40.6", "description": "lezer-based PromQL grammar", "main": "index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index bc5b81c6e..3b3eedd5c 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -28,10 +28,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.5", + "version": "0.40.6", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.5", + "@prometheus-io/lezer-promql": "^0.40.6", "lru-cache": "^6.0.0" }, "devDependencies": { @@ -61,7 +61,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.40.5", + "version": "0.40.6", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.1.1", @@ -17625,7 +17625,7 @@ }, "react-app": { "name": "@prometheus-io/app", - "version": "0.40.5", + "version": "0.40.6", "dependencies": { "@codemirror/autocomplete": "^6.2.0", "@codemirror/commands": "^6.1.0", @@ -17643,7 +17643,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.5", + "@prometheus-io/codemirror-promql": "^0.40.6", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11", @@ -19883,7 +19883,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.5", + "@prometheus-io/codemirror-promql": "^0.40.6", "@testing-library/react-hooks": "^7.0.2", "@types/enzyme": "^3.10.12", "@types/flot": "0.0.32", @@ -19935,7 +19935,7 @@ "@lezer/common": "^1.0.1", "@lezer/highlight": "^1.1.0", "@lezer/lr": "^1.2.3", - "@prometheus-io/lezer-promql": "^0.40.5", + "@prometheus-io/lezer-promql": "^0.40.6", "@types/lru-cache": "^5.1.1", "isomorphic-fetch": "^3.0.0", "lru-cache": "^6.0.0", diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index f1b5edca6..0f2437a09 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/app", - "version": "0.40.5", + "version": "0.40.6", "private": true, "dependencies": { "@codemirror/autocomplete": "^6.2.0", @@ -19,7 +19,7 @@ "@lezer/common": "^1.0.1", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.5", + "@prometheus-io/codemirror-promql": "^0.40.6", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11", From 5f366e9b62b81a8c289d3d4fc821a30ff839b47c Mon Sep 17 00:00:00 2001 From: beorn7 Date: Sun, 11 Dec 2022 23:39:53 +0100 Subject: [PATCH 020/107] histograms: Improve tests and fix exposed bugs This adds negative buckets and access of float histograms to TestHistogramChunkSameBuckets and TestHistogramChunkBucketChanges. It also exercises a specific pattern of reusing an iterator (one where no access has happened). This exposes two bugs (where entries for positive buckets where used where the corresponding entries for negative buckets should have been used). One was fixed in #11627 (not merged), which triggered the work in this commit. This commit fixes both issues, so #11627 can be closed. It also simplifies the code in the histogramIterator.Next method that aims to recycle existing slice capacity. Furthermore, this is on top of the release-2.40 branch because we should probably cut a bugfix release for this. Signed-off-by: beorn7 --- tsdb/chunkenc/histogram.go | 39 +++------- tsdb/chunkenc/histogram_test.go | 130 +++++++++++++++++++++----------- 2 files changed, 95 insertions(+), 74 deletions(-) diff --git a/tsdb/chunkenc/histogram.go b/tsdb/chunkenc/histogram.go index e57b96db9..5aad382b2 100644 --- a/tsdb/chunkenc/histogram.go +++ b/tsdb/chunkenc/histogram.go @@ -176,7 +176,7 @@ func newHistogramIterator(b []byte) *histogramIterator { } func (c *HistogramChunk) iterator(it Iterator) *histogramIterator { - // This commet is copied from XORChunk.iterator: + // This comment is copied from XORChunk.iterator: // Should iterators guarantee to act on a copy of the data so it doesn't lock append? // When using striped locks to guard access to chunks, probably yes. // Could only copy data if the chunk is not completed yet. @@ -651,7 +651,7 @@ func (it *histogramIterator) Reset(b []byte) { } it.pBucketsDelta = it.pBucketsDelta[:0] - it.pBucketsDelta = it.pBucketsDelta[:0] + it.nBucketsDelta = it.nBucketsDelta[:0] it.sum = 0 it.leading = 0 @@ -677,36 +677,17 @@ func (it *histogramIterator) Next() ValueType { it.zThreshold = zeroThreshold it.pSpans, it.nSpans = posSpans, negSpans numPBuckets, numNBuckets := countSpans(posSpans), countSpans(negSpans) - // Allocate bucket slices as needed, recycling existing slices - // in case this iterator was reset and already has slices of a - // sufficient capacity. + // The code below recycles existing slices in case this iterator + // was reset and already has slices of a sufficient capacity. if numPBuckets > 0 { - if cap(it.pBuckets) < numPBuckets { - it.pBuckets = make([]int64, numPBuckets) - // If cap(it.pBuckets) isn't sufficient, neither is the cap of the others. - it.pBucketsDelta = make([]int64, numPBuckets) - it.pFloatBuckets = make([]float64, numPBuckets) - } else { - for i := 0; i < numPBuckets; i++ { - it.pBuckets = append(it.pBuckets, 0) - it.pBucketsDelta = append(it.pBucketsDelta, 0) - it.pFloatBuckets = append(it.pFloatBuckets, 0) - } - } + it.pBuckets = append(it.pBuckets, make([]int64, numPBuckets)...) + it.pBucketsDelta = append(it.pBucketsDelta, make([]int64, numPBuckets)...) + it.pFloatBuckets = append(it.pFloatBuckets, make([]float64, numPBuckets)...) } if numNBuckets > 0 { - if cap(it.nBuckets) < numNBuckets { - it.nBuckets = make([]int64, numNBuckets) - // If cap(it.nBuckets) isn't sufficient, neither is the cap of the others. - it.nBucketsDelta = make([]int64, numNBuckets) - it.nFloatBuckets = make([]float64, numNBuckets) - } else { - for i := 0; i < numNBuckets; i++ { - it.nBuckets = append(it.nBuckets, 0) - it.nBucketsDelta = append(it.nBucketsDelta, 0) - it.pFloatBuckets = append(it.pFloatBuckets, 0) - } - } + it.nBuckets = append(it.nBuckets, make([]int64, numNBuckets)...) + it.nBucketsDelta = append(it.nBucketsDelta, make([]int64, numNBuckets)...) + it.nFloatBuckets = append(it.nFloatBuckets, make([]float64, numNBuckets)...) } // Now read the actual data. diff --git a/tsdb/chunkenc/histogram_test.go b/tsdb/chunkenc/histogram_test.go index 09c08ae3c..59483f86f 100644 --- a/tsdb/chunkenc/histogram_test.go +++ b/tsdb/chunkenc/histogram_test.go @@ -21,9 +21,15 @@ import ( "github.com/prometheus/prometheus/model/histogram" ) +type result struct { + t int64 + h *histogram.Histogram + fh *histogram.FloatHistogram +} + func TestHistogramChunkSameBuckets(t *testing.T) { c := NewHistogramChunk() - var exp []res + var exp []result // Create fresh appender and add the first histogram. app, err := c.Appender() @@ -32,7 +38,7 @@ func TestHistogramChunkSameBuckets(t *testing.T) { ts := int64(1234567890) h := &histogram.Histogram{ - Count: 5, + Count: 15, ZeroCount: 2, Sum: 18.4, ZeroThreshold: 1e-100, @@ -42,20 +48,26 @@ func TestHistogramChunkSameBuckets(t *testing.T) { {Offset: 1, Length: 2}, }, PositiveBuckets: []int64{1, 1, -1, 0}, // counts: 1, 2, 1, 1 (total 5) + NegativeSpans: []histogram.Span{ + {Offset: 1, Length: 1}, + {Offset: 2, Length: 3}, + }, + NegativeBuckets: []int64{2, 1, -1, -1}, // counts: 2, 3, 2, 1 (total 8) } app.AppendHistogram(ts, h) - exp = append(exp, res{t: ts, h: h}) + exp = append(exp, result{t: ts, h: h, fh: h.ToFloat()}) require.Equal(t, 1, c.NumSamples()) // Add an updated histogram. ts += 16 h = h.Copy() - h.Count += 9 + h.Count = 32 h.ZeroCount++ h.Sum = 24.4 h.PositiveBuckets = []int64{5, -2, 1, -2} // counts: 5, 3, 4, 2 (total 14) + h.NegativeBuckets = []int64{4, -1, 1, -1} // counts: 4, 3, 4, 4 (total 15) app.AppendHistogram(ts, h) - exp = append(exp, res{t: ts, h: h}) + exp = append(exp, result{t: ts, h: h, fh: h.ToFloat()}) require.Equal(t, 2, c.NumSamples()) // Add update with new appender. @@ -64,59 +76,77 @@ func TestHistogramChunkSameBuckets(t *testing.T) { ts += 14 h = h.Copy() - h.Count += 13 + h.Count = 54 h.ZeroCount += 2 h.Sum = 24.4 h.PositiveBuckets = []int64{6, 1, -3, 6} // counts: 6, 7, 4, 10 (total 27) + h.NegativeBuckets = []int64{5, 1, -2, 3} // counts: 5, 6, 4, 7 (total 22) app.AppendHistogram(ts, h) - exp = append(exp, res{t: ts, h: h}) + exp = append(exp, result{t: ts, h: h, fh: h.ToFloat()}) require.Equal(t, 3, c.NumSamples()) // 1. Expand iterator in simple case. - it := c.iterator(nil) + it := c.Iterator(nil) require.NoError(t, it.Err()) - var act []res + var act []result for it.Next() == ValHistogram { ts, h := it.AtHistogram() - act = append(act, res{t: ts, h: h}) + fts, fh := it.AtFloatHistogram() + require.Equal(t, ts, fts) + act = append(act, result{t: ts, h: h, fh: fh}) } require.NoError(t, it.Err()) require.Equal(t, exp, act) // 2. Expand second iterator while reusing first one. it2 := c.Iterator(it) - var res2 []res + var act2 []result for it2.Next() == ValHistogram { ts, h := it2.AtHistogram() - res2 = append(res2, res{t: ts, h: h}) + fts, fh := it2.AtFloatHistogram() + require.Equal(t, ts, fts) + act2 = append(act2, result{t: ts, h: h, fh: fh}) } require.NoError(t, it2.Err()) - require.Equal(t, exp, res2) + require.Equal(t, exp, act2) - // 3. Test iterator Seek. - // mid := len(exp) / 2 + // 3. Now recycle an iterator that was never used to access anything. + itX := c.Iterator(nil) + for itX.Next() == ValHistogram { + // Just iterate through without accessing anything. + } + it3 := c.iterator(itX) + var act3 []result + for it3.Next() == ValHistogram { + ts, h := it3.AtHistogram() + fts, fh := it3.AtFloatHistogram() + require.Equal(t, ts, fts) + act3 = append(act3, result{t: ts, h: h, fh: fh}) + } + require.NoError(t, it3.Err()) + require.Equal(t, exp, act3) - // it3 := c.Iterator(nil) - // var res3 []pair - // require.Equal(t, true, it3.Seek(exp[mid].t)) + // 4. Test iterator Seek. + mid := len(exp) / 2 + it4 := c.Iterator(nil) + var act4 []result + require.Equal(t, ValHistogram, it4.Seek(exp[mid].t)) // Below ones should not matter. - // require.Equal(t, true, it3.Seek(exp[mid].t)) - // require.Equal(t, true, it3.Seek(exp[mid].t)) - // ts, v = it3.At() - // res3 = append(res3, pair{t: ts, v: v}) - - // for it3.Next() { - // ts, v := it3.At() - // res3 = append(res3, pair{t: ts, v: v}) - // } - // require.NoError(t, it3.Err()) - // require.Equal(t, exp[mid:], res3) - // require.Equal(t, false, it3.Seek(exp[len(exp)-1].t+1)) -} - -type res struct { - t int64 - h *histogram.Histogram + require.Equal(t, ValHistogram, it4.Seek(exp[mid].t)) + require.Equal(t, ValHistogram, it4.Seek(exp[mid].t)) + ts, h = it4.AtHistogram() + fts, fh := it4.AtFloatHistogram() + require.Equal(t, ts, fts) + act4 = append(act4, result{t: ts, h: h, fh: fh}) + for it4.Next() == ValHistogram { + ts, h := it4.AtHistogram() + fts, fh := it4.AtFloatHistogram() + require.Equal(t, ts, fts) + act4 = append(act4, result{t: ts, h: h, fh: fh}) + } + require.NoError(t, it4.Err()) + require.Equal(t, exp[mid:], act4) + require.Equal(t, ValNone, it4.Seek(exp[len(exp)-1].t+1)) } // Mimics the scenario described for compareSpans(). @@ -130,7 +160,7 @@ func TestHistogramChunkBucketChanges(t *testing.T) { ts1 := int64(1234567890) h1 := &histogram.Histogram{ - Count: 5, + Count: 27, ZeroCount: 2, Sum: 18.4, ZeroThreshold: 1e-125, @@ -143,6 +173,8 @@ func TestHistogramChunkBucketChanges(t *testing.T) { {Offset: 1, Length: 1}, }, PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // counts: 6, 3, 3, 2, 4, 5, 1 (total 24) + NegativeSpans: []histogram.Span{{Offset: 1, Length: 1}}, + NegativeBuckets: []int64{1}, } app.AppendHistogram(ts1, h1) @@ -157,19 +189,23 @@ func TestHistogramChunkBucketChanges(t *testing.T) { {Offset: 1, Length: 4}, {Offset: 3, Length: 3}, } - h2.Count += 9 + h2.NegativeSpans = []histogram.Span{{Offset: 0, Length: 2}} + h2.Count = 35 h2.ZeroCount++ h2.Sum = 30 // Existing histogram should get values converted from the above to: // 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between) // so the new histogram should have new counts >= these per-bucket counts, e.g.: h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30) - + // Existing histogram should get values converted from the above to: + // 0 1 (previous values with some new empty buckets in between) + // so the new histogram should have new counts >= these per-bucket counts, e.g.: + h2.NegativeBuckets = []int64{2, -1} // 2 1 (total 3) // This is how span changes will be handled. hApp, _ := app.(*HistogramAppender) posInterjections, negInterjections, ok, cr := hApp.Appendable(h2) require.Greater(t, len(posInterjections), 0) - require.Equal(t, 0, len(negInterjections)) + require.Greater(t, len(negInterjections), 0) require.True(t, ok) // Only new buckets came in. require.False(t, cr) c, app = hApp.Recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans) @@ -182,21 +218,25 @@ func TestHistogramChunkBucketChanges(t *testing.T) { // metadata as well as the expanded buckets. h1.PositiveSpans = h2.PositiveSpans h1.PositiveBuckets = []int64{6, -3, -3, 3, -3, 0, 2, 2, 1, -5, 1} - exp := []res{ - {t: ts1, h: h1}, - {t: ts2, h: h2}, + h1.NegativeSpans = h2.NegativeSpans + h1.NegativeBuckets = []int64{0, 1} + exp := []result{ + {t: ts1, h: h1, fh: h1.ToFloat()}, + {t: ts2, h: h2, fh: h2.ToFloat()}, } it := c.Iterator(nil) - var act []res + var act []result for it.Next() == ValHistogram { ts, h := it.AtHistogram() - act = append(act, res{t: ts, h: h}) + fts, fh := it.AtFloatHistogram() + require.Equal(t, ts, fts) + act = append(act, result{t: ts, h: h, fh: fh}) } require.NoError(t, it.Err()) require.Equal(t, exp, act) } -func TestHistoChunkAppendable(t *testing.T) { +func TestHistogramChunkAppendable(t *testing.T) { c := Chunk(NewHistogramChunk()) // Create fresh appender and add the first histogram. From a971bdd5f51f61ef801eabf5f808c4e34f3ddf32 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Fri, 9 Dec 2022 19:02:05 +0100 Subject: [PATCH 021/107] Do not build with netgo on Windows Fix #11480 Signed-off-by: Julien Pivotto --- .promu.yml | 8 +++++++- Makefile.common | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.promu.yml b/.promu.yml index 50c0be257..233295f85 100644 --- a/.promu.yml +++ b/.promu.yml @@ -10,7 +10,13 @@ build: path: ./cmd/prometheus - name: promtool path: ./cmd/promtool - flags: -a -tags netgo,builtinassets + tags: + all: + - netgo + - builtinassets + windows: + - builtinassets + flags: -a ldflags: | -X github.com/prometheus/common/version.Version={{.Version}} -X github.com/prometheus/common/version.Revision={{.Revision}} diff --git a/Makefile.common b/Makefile.common index 7642c4485..e358db69c 100644 --- a/Makefile.common +++ b/Makefile.common @@ -55,7 +55,7 @@ ifneq ($(shell which gotestsum),) endif endif -PROMU_VERSION ?= 0.13.0 +PROMU_VERSION ?= 0.14.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz SKIP_GOLANGCI_LINT := From 924ba90c3f753d1299f465bc39625cbdac56e28f Mon Sep 17 00:00:00 2001 From: John Carlo Roberto <10111643+irizwaririz@users.noreply.github.com> Date: Mon, 12 Dec 2022 23:08:45 +0800 Subject: [PATCH 022/107] Add link to best practices in "Defining Recording Rules" page (#11696) * docs: Add link to best practices in "Defining Recording Rules" page Signed-off-by: John Carlo Roberto <10111643+Irizwaririz@users.noreply.github.com> * docs: Improve wording Signed-off-by: John Carlo Roberto <10111643+Irizwaririz@users.noreply.github.com> Signed-off-by: John Carlo Roberto <10111643+Irizwaririz@users.noreply.github.com> --- docs/configuration/recording_rules.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuration/recording_rules.md b/docs/configuration/recording_rules.md index 8ac6c15fc..60ba67d1f 100644 --- a/docs/configuration/recording_rules.md +++ b/docs/configuration/recording_rules.md @@ -132,6 +132,9 @@ annotations: [ : ] ``` +See also the +[best practices for naming metrics created by recording rules](https://prometheus.io/docs/practices/rules/#recording-rules). + # Limiting alerts and series A limit for alerts produced by alerting rules and series produced recording rules From f113d58a57a889ac36c298f3099a91f5e777f848 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Fri, 9 Dec 2022 19:02:05 +0100 Subject: [PATCH 023/107] Do not build with netgo on Windows Fix #11480 Signed-off-by: Julien Pivotto --- .promu.yml | 8 +++++++- Makefile.common | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.promu.yml b/.promu.yml index 50c0be257..233295f85 100644 --- a/.promu.yml +++ b/.promu.yml @@ -10,7 +10,13 @@ build: path: ./cmd/prometheus - name: promtool path: ./cmd/promtool - flags: -a -tags netgo,builtinassets + tags: + all: + - netgo + - builtinassets + windows: + - builtinassets + flags: -a ldflags: | -X github.com/prometheus/common/version.Version={{.Version}} -X github.com/prometheus/common/version.Revision={{.Revision}} diff --git a/Makefile.common b/Makefile.common index 7642c4485..e358db69c 100644 --- a/Makefile.common +++ b/Makefile.common @@ -55,7 +55,7 @@ ifneq ($(shell which gotestsum),) endif endif -PROMU_VERSION ?= 0.13.0 +PROMU_VERSION ?= 0.14.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz SKIP_GOLANGCI_LINT := From 300d6e43901e0fbca1b618ea64b5978217e54b10 Mon Sep 17 00:00:00 2001 From: Danny Staple Date: Tue, 13 Dec 2022 11:11:13 +0000 Subject: [PATCH 024/107] Add an explanation to the expanding notation After some team discussion, we found this to be a useful was to explain the samples. Signed-off-by: Danny Staple --- docs/configuration/unit_testing_rules.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index d99255f01..1f69dd32d 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -77,13 +77,15 @@ series: # This uses expanding notation. # Expanding notation: # 'a+bxc' becomes 'a a+b a+(2*b) a+(3*b) … a+(c*b)' +# Read this as series starts at a, then c further samples incrementing by b. # 'a-bxc' becomes 'a a-b a-(2*b) a-(3*b) … a-(c*b)' +# Read this as series starts at a, then c further samples decrementing by b (or incrementing by negative b). # There are special values to indicate missing and stale samples: # '_' represents a missing sample from scrape # 'stale' indicates a stale sample # Examples: -# 1. '-2+4x3' becomes '-2 2 6 10' -# 2. ' 1-2x4' becomes '1 -1 -3 -5 -7' +# 1. '-2+4x3' becomes '-2 2 6 10' - series starts at -2, then 3 further samples incrementing by 4. +# 2. ' 1-2x4' becomes '1 -1 -3 -5 -7' - series starts at 1, then 4 further samples decrementing by 2. # 3. ' 1 _x3 stale' becomes '1 _ _ _ stale' values: ``` From b614fdd8a7f453f9d81342f2f515681ec227707a Mon Sep 17 00:00:00 2001 From: Danny Staple Date: Tue, 13 Dec 2022 15:52:40 +0000 Subject: [PATCH 025/107] Update unit_testing_rules.md Update the shorthand, and note the different behaviour between missing samples and numbers. Signed-off-by: Danny Staple --- docs/configuration/unit_testing_rules.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index 1f69dd32d..44a61fb30 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -86,7 +86,8 @@ series: # Examples: # 1. '-2+4x3' becomes '-2 2 6 10' - series starts at -2, then 3 further samples incrementing by 4. # 2. ' 1-2x4' becomes '1 -1 -3 -5 -7' - series starts at 1, then 4 further samples decrementing by 2. -# 3. ' 1 _x3 stale' becomes '1 _ _ _ stale' +# 3. ' 1x4' become '1 1 1 1' - shorthand for '1+0x4', series starts at 1, then 3 further samples incrementing by 0. +# 4. ' 1 _x3 stale' becomes '1 _ _ _ stale' - the missing sample cannot increment, so only 3 samples are produced by the '_x3' expression. values: ``` From 8c5da5e0781a170dab088ed0c5603bd45a6a5fc8 Mon Sep 17 00:00:00 2001 From: Julien Levesy Date: Tue, 29 Nov 2022 15:24:07 +0100 Subject: [PATCH 026/107] feat(config): allow no remote write configs Signed-off-by: Julien Levesy --- config/config.go | 4 --- config/config_test.go | 27 +++++++++++++++++++ config/testdata/agent_mode.good.yml | 2 ++ .../agent_mode.with_alert_manager.yml | 6 +++++ .../agent_mode.with_alert_relabels.yml | 5 ++++ .../testdata/agent_mode.with_remote_reads.yml | 5 ++++ .../testdata/agent_mode.with_rule_files.yml | 3 +++ .../agent_mode.without_remote_writes.yml | 2 ++ 8 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 config/testdata/agent_mode.good.yml create mode 100644 config/testdata/agent_mode.with_alert_manager.yml create mode 100644 config/testdata/agent_mode.with_alert_relabels.yml create mode 100644 config/testdata/agent_mode.with_remote_reads.yml create mode 100644 config/testdata/agent_mode.with_rule_files.yml create mode 100644 config/testdata/agent_mode.without_remote_writes.yml diff --git a/config/config.go b/config/config.go index 8e8460d4c..8f32f2d53 100644 --- a/config/config.go +++ b/config/config.go @@ -112,10 +112,6 @@ func LoadFile(filename string, agentMode, expandExternalLabels bool, logger log. } if agentMode { - if len(cfg.RemoteWriteConfigs) == 0 { - return nil, errors.New("at least one remote_write target must be specified in agent mode") - } - if len(cfg.AlertingConfig.AlertmanagerConfigs) > 0 || len(cfg.AlertingConfig.AlertRelabelConfigs) > 0 { return nil, errors.New("field alerting is not allowed in agent mode") } diff --git a/config/config_test.go b/config/config_test.go index 9ee8fe1a0..26f257ea2 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1745,6 +1745,33 @@ func TestExpandExternalLabels(t *testing.T) { require.Equal(t, labels.FromStrings("bar", "foo", "baz", "fooTestValuebar", "foo", "TestValue", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels) } +func TestAgentMode(t *testing.T) { + _, err := LoadFile("testdata/agent_mode.with_alert_manager.yml", true, false, log.NewNopLogger()) + require.ErrorContains(t, err, "field alerting is not allowed in agent mode") + + _, err = LoadFile("testdata/agent_mode.with_alert_relabels.yml", true, false, log.NewNopLogger()) + require.ErrorContains(t, err, "field alerting is not allowed in agent mode") + + _, err = LoadFile("testdata/agent_mode.with_rule_files.yml", true, false, log.NewNopLogger()) + require.ErrorContains(t, err, "field rule_files is not allowed in agent mode") + + _, err = LoadFile("testdata/agent_mode.with_remote_reads.yml", true, false, log.NewNopLogger()) + require.ErrorContains(t, err, "field remote_read is not allowed in agent mode") + + c, err := LoadFile("testdata/agent_mode.without_remote_writes.yml", true, false, log.NewNopLogger()) + require.NoError(t, err) + require.Len(t, c.RemoteWriteConfigs, 0) + + c, err = LoadFile("testdata/agent_mode.good.yml", true, false, log.NewNopLogger()) + require.NoError(t, err) + require.Len(t, c.RemoteWriteConfigs, 1) + require.Equal( + t, + "http://remote1/push", + c.RemoteWriteConfigs[0].URL.String(), + ) +} + func TestEmptyGlobalBlock(t *testing.T) { c, err := Load("global:\n", false, log.NewNopLogger()) require.NoError(t, err) diff --git a/config/testdata/agent_mode.good.yml b/config/testdata/agent_mode.good.yml new file mode 100644 index 000000000..a16612095 --- /dev/null +++ b/config/testdata/agent_mode.good.yml @@ -0,0 +1,2 @@ +remote_write: + - url: http://remote1/push diff --git a/config/testdata/agent_mode.with_alert_manager.yml b/config/testdata/agent_mode.with_alert_manager.yml new file mode 100644 index 000000000..9a3929957 --- /dev/null +++ b/config/testdata/agent_mode.with_alert_manager.yml @@ -0,0 +1,6 @@ +alerting: + alertmanagers: + - scheme: https + static_configs: + - targets: + - "1.2.3.4:9093" diff --git a/config/testdata/agent_mode.with_alert_relabels.yml b/config/testdata/agent_mode.with_alert_relabels.yml new file mode 100644 index 000000000..67e70fc7f --- /dev/null +++ b/config/testdata/agent_mode.with_alert_relabels.yml @@ -0,0 +1,5 @@ +alerting: + alert_relabel_configs: + - action: uppercase + source_labels: [instance] + target_label: instance diff --git a/config/testdata/agent_mode.with_remote_reads.yml b/config/testdata/agent_mode.with_remote_reads.yml new file mode 100644 index 000000000..416676793 --- /dev/null +++ b/config/testdata/agent_mode.with_remote_reads.yml @@ -0,0 +1,5 @@ +remote_read: + - url: http://remote1/read + read_recent: true + name: default + enable_http2: false diff --git a/config/testdata/agent_mode.with_rule_files.yml b/config/testdata/agent_mode.with_rule_files.yml new file mode 100644 index 000000000..3aaa9ad47 --- /dev/null +++ b/config/testdata/agent_mode.with_rule_files.yml @@ -0,0 +1,3 @@ +rule_files: + - "first.rules" + - "my/*.rules" diff --git a/config/testdata/agent_mode.without_remote_writes.yml b/config/testdata/agent_mode.without_remote_writes.yml new file mode 100644 index 000000000..1285bef67 --- /dev/null +++ b/config/testdata/agent_mode.without_remote_writes.yml @@ -0,0 +1,2 @@ +global: + scrape_interval: 15s From 106d06a032b040428964f16182239b7dfd3f11f2 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Tue, 13 Dec 2022 13:06:55 +0100 Subject: [PATCH 027/107] Release 2.40.7 We have 2 bugfixes, one which is important for Windows users and another one on native histograms. I think it is worth cutting another bugfix release before 2.41 comes out. Signed-off-by: Julien Pivotto --- CHANGELOG.md | 5 +++++ VERSION | 2 +- web/ui/module/codemirror-promql/package.json | 4 ++-- web/ui/module/lezer-promql/package.json | 2 +- web/ui/package-lock.json | 14 +++++++------- web/ui/react-app/package.json | 4 ++-- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 071022709..e0f08c51c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 2.40.7 / 2022-12-14 + +* [BUGFIX] Use Windows native DNS resolver. #11704 +* [BUGFIX] TSDB: Fix queries involving negative buckets of native histograms. #11699 + ## 2.40.6 / 2022-12-09 * [SECURITY] Security upgrade from go and upstream dependencies that include diff --git a/VERSION b/VERSION index ad572faeb..5c4c670b5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.40.6 +2.40.7 diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index 1b86f596f..b12e50ac6 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.6", + "version": "0.40.7", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.6", + "@prometheus-io/lezer-promql": "^0.40.7", "lru-cache": "^6.0.0" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index b429732a7..e29f2c8d8 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.40.6", + "version": "0.40.7", "description": "lezer-based PromQL grammar", "main": "index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index 3b3eedd5c..692ee3477 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -28,10 +28,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.6", + "version": "0.40.7", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.6", + "@prometheus-io/lezer-promql": "^0.40.7", "lru-cache": "^6.0.0" }, "devDependencies": { @@ -61,7 +61,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.40.6", + "version": "0.40.7", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.1.1", @@ -17625,7 +17625,7 @@ }, "react-app": { "name": "@prometheus-io/app", - "version": "0.40.6", + "version": "0.40.7", "dependencies": { "@codemirror/autocomplete": "^6.2.0", "@codemirror/commands": "^6.1.0", @@ -17643,7 +17643,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.6", + "@prometheus-io/codemirror-promql": "^0.40.7", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11", @@ -19883,7 +19883,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.6", + "@prometheus-io/codemirror-promql": "^0.40.7", "@testing-library/react-hooks": "^7.0.2", "@types/enzyme": "^3.10.12", "@types/flot": "0.0.32", @@ -19935,7 +19935,7 @@ "@lezer/common": "^1.0.1", "@lezer/highlight": "^1.1.0", "@lezer/lr": "^1.2.3", - "@prometheus-io/lezer-promql": "^0.40.6", + "@prometheus-io/lezer-promql": "^0.40.7", "@types/lru-cache": "^5.1.1", "isomorphic-fetch": "^3.0.0", "lru-cache": "^6.0.0", diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index 0f2437a09..5a17ac416 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/app", - "version": "0.40.6", + "version": "0.40.7", "private": true, "dependencies": { "@codemirror/autocomplete": "^6.2.0", @@ -19,7 +19,7 @@ "@lezer/common": "^1.0.1", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.6", + "@prometheus-io/codemirror-promql": "^0.40.7", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11", From 3677d61a4b2baca1b9eeb496fdd7f6512e36591b Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Wed, 14 Dec 2022 10:43:53 +0100 Subject: [PATCH 028/107] Update kubernetes dependencies A new API is available for AddEventHandlers, to get errors but also be able to cancel handlers. Doing the easy thing for the release, which is just to log errors. We could see how to improve this in the future to handle the errors properly and cancel the handlers. Signed-off-by: Julien Pivotto --- discovery/kubernetes/endpoints.go | 15 +++++-- discovery/kubernetes/endpointslice.go | 15 +++++-- discovery/kubernetes/ingress.go | 5 ++- discovery/kubernetes/node.go | 5 ++- discovery/kubernetes/pod.go | 10 ++++- discovery/kubernetes/service.go | 5 ++- go.mod | 34 +++++++-------- go.sum | 60 ++++++++++++++------------- 8 files changed, 92 insertions(+), 57 deletions(-) diff --git a/discovery/kubernetes/endpoints.go b/discovery/kubernetes/endpoints.go index 1f39c23e7..039daf4fa 100644 --- a/discovery/kubernetes/endpoints.go +++ b/discovery/kubernetes/endpoints.go @@ -72,7 +72,7 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca queue: workqueue.NewNamed("endpoints"), } - e.endpointsInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err := e.endpointsInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { epAddCount.Inc() e.enqueue(o) @@ -86,6 +86,9 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca e.enqueue(o) }, }) + if err != nil { + level.Error(l).Log("msg", "Error adding endpoints event handler.", "err", err) + } serviceUpdate := func(o interface{}) { svc, err := convertToService(o) @@ -106,7 +109,7 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca level.Error(e.logger).Log("msg", "retrieving endpoints failed", "err", err) } } - e.serviceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err = e.serviceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ // TODO(fabxc): potentially remove add and delete event handlers. Those should // be triggered via the endpoint handlers already. AddFunc: func(o interface{}) { @@ -122,8 +125,11 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca serviceUpdate(o) }, }) + if err != nil { + level.Error(l).Log("msg", "Error adding services event handler.", "err", err) + } if e.withNodeMetadata { - e.nodeInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err = e.nodeInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { node := o.(*apiv1.Node) e.enqueueNode(node.Name) @@ -137,6 +143,9 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca e.enqueueNode(node.Name) }, }) + if err != nil { + level.Error(l).Log("msg", "Error adding nodes event handler.", "err", err) + } } return e diff --git a/discovery/kubernetes/endpointslice.go b/discovery/kubernetes/endpointslice.go index 594759f45..0d9c5a25b 100644 --- a/discovery/kubernetes/endpointslice.go +++ b/discovery/kubernetes/endpointslice.go @@ -73,7 +73,7 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod queue: workqueue.NewNamed("endpointSlice"), } - e.endpointSliceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err := e.endpointSliceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { epslAddCount.Inc() e.enqueue(o) @@ -87,6 +87,9 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod e.enqueue(o) }, }) + if err != nil { + level.Error(l).Log("msg", "Error adding endpoint slices event handler.", "err", err) + } serviceUpdate := func(o interface{}) { svc, err := convertToService(o) @@ -109,7 +112,7 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod } } } - e.serviceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err = e.serviceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { svcAddCount.Inc() serviceUpdate(o) @@ -123,9 +126,12 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod serviceUpdate(o) }, }) + if err != nil { + level.Error(l).Log("msg", "Error adding services event handler.", "err", err) + } if e.withNodeMetadata { - e.nodeInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err = e.nodeInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { node := o.(*apiv1.Node) e.enqueueNode(node.Name) @@ -139,6 +145,9 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod e.enqueueNode(node.Name) }, }) + if err != nil { + level.Error(l).Log("msg", "Error adding nodes event handler.", "err", err) + } } return e diff --git a/discovery/kubernetes/ingress.go b/discovery/kubernetes/ingress.go index de6d2a0b4..8c9249f54 100644 --- a/discovery/kubernetes/ingress.go +++ b/discovery/kubernetes/ingress.go @@ -48,7 +48,7 @@ type Ingress struct { // NewIngress returns a new ingress discovery. func NewIngress(l log.Logger, inf cache.SharedInformer) *Ingress { s := &Ingress{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("ingress")} - s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { ingressAddCount.Inc() s.enqueue(o) @@ -62,6 +62,9 @@ func NewIngress(l log.Logger, inf cache.SharedInformer) *Ingress { s.enqueue(o) }, }) + if err != nil { + level.Error(l).Log("msg", "Error adding ingresses event handler.", "err", err) + } return s } diff --git a/discovery/kubernetes/node.go b/discovery/kubernetes/node.go index ebb6bbced..93adf7825 100644 --- a/discovery/kubernetes/node.go +++ b/discovery/kubernetes/node.go @@ -55,7 +55,7 @@ func NewNode(l log.Logger, inf cache.SharedInformer) *Node { l = log.NewNopLogger() } n := &Node{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("node")} - n.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err := n.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { nodeAddCount.Inc() n.enqueue(o) @@ -69,6 +69,9 @@ func NewNode(l log.Logger, inf cache.SharedInformer) *Node { n.enqueue(o) }, }) + if err != nil { + level.Error(l).Log("msg", "Error adding nodes event handler.", "err", err) + } return n } diff --git a/discovery/kubernetes/pod.go b/discovery/kubernetes/pod.go index e19a33c1a..2e3687a06 100644 --- a/discovery/kubernetes/pod.go +++ b/discovery/kubernetes/pod.go @@ -65,7 +65,7 @@ func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInfo logger: l, queue: workqueue.NewNamed("pod"), } - p.podInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err := p.podInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { podAddCount.Inc() p.enqueue(o) @@ -79,9 +79,12 @@ func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInfo p.enqueue(o) }, }) + if err != nil { + level.Error(l).Log("msg", "Error adding pods event handler.", "err", err) + } if p.withNodeMetadata { - p.nodeInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err = p.nodeInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { node := o.(*apiv1.Node) p.enqueuePodsForNode(node.Name) @@ -95,6 +98,9 @@ func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInfo p.enqueuePodsForNode(node.Name) }, }) + if err != nil { + level.Error(l).Log("msg", "Error adding pods event handler.", "err", err) + } } return p diff --git a/discovery/kubernetes/service.go b/discovery/kubernetes/service.go index 44ebcbc19..a19f06e7d 100644 --- a/discovery/kubernetes/service.go +++ b/discovery/kubernetes/service.go @@ -51,7 +51,7 @@ func NewService(l log.Logger, inf cache.SharedInformer) *Service { l = log.NewNopLogger() } s := &Service{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("service")} - s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { svcAddCount.Inc() s.enqueue(o) @@ -65,6 +65,9 @@ func NewService(l log.Logger, inf cache.SharedInformer) *Service { s.enqueue(o) }, }) + if err != nil { + level.Error(l).Log("msg", "Error adding services event handler.", "err", err) + } return s } diff --git a/go.mod b/go.mod index 398b17b9b..4ec1aba54 100644 --- a/go.mod +++ b/go.mod @@ -61,10 +61,10 @@ require ( go.uber.org/atomic v1.10.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/goleak v1.2.0 - golang.org/x/net v0.1.0 + golang.org/x/net v0.4.0 golang.org/x/oauth2 v0.1.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.1.0 + golang.org/x/sys v0.3.0 golang.org/x/time v0.1.0 golang.org/x/tools v0.2.0 google.golang.org/api v0.102.0 @@ -74,11 +74,11 @@ require ( gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.25.3 - k8s.io/apimachinery v0.25.3 - k8s.io/client-go v0.25.3 + k8s.io/api v0.26.0 + k8s.io/apimachinery v0.26.0 + k8s.io/client-go v0.26.0 k8s.io/klog v1.0.0 - k8s.io/klog/v2 v2.80.0 + k8s.io/klog/v2 v2.80.1 ) require ( @@ -95,8 +95,6 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Microsoft/go-winio v0.5.1 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/armon/go-metrics v0.3.10 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect @@ -107,18 +105,18 @@ require ( github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.8.0 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fatih/color v1.13.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-kit/kit v0.10.0 // indirect + github.com/go-kit/kit v0.12.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.21.2 // indirect github.com/go-openapi/errors v0.20.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/loads v0.21.1 // indirect github.com/go-openapi/spec v0.20.4 // indirect github.com/go-openapi/swag v0.21.1 // indirect @@ -139,8 +137,8 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.1 // indirect github.com/hashicorp/cronexpr v1.1.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-hclog v0.14.1 // indirect - github.com/hashicorp/go-immutable-radix v1.3.0 // indirect + github.com/hashicorp/go-hclog v0.16.2 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.1 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect @@ -175,15 +173,15 @@ require ( golang.org/x/crypto v0.1.0 // indirect golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 golang.org/x/mod v0.6.0 // indirect - golang.org/x/term v0.1.0 // indirect - golang.org/x/text v0.4.0 // indirect + golang.org/x/term v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.6 // indirect gotest.tools/v3 v3.0.3 // indirect - k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect - k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 // indirect + k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 343f1c710..d1e83fb19 100644 --- a/go.sum +++ b/go.sum @@ -72,9 +72,7 @@ github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -180,8 +178,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= -github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= -github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -217,8 +215,9 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4= +github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= @@ -242,8 +241,9 @@ github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpX github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/loads v0.21.1 h1:Wb3nVZpdEzDTcly8S4HMkey6fjARRzb7iEaySimlDW0= github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= github.com/go-openapi/runtime v0.23.1/go.mod h1:AKurw9fNre+h3ELZfk6ILsfvPN+bvvlaU/M9q/r9hpk= @@ -268,7 +268,6 @@ github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 h1:JVrqSeQfdhY github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= @@ -433,11 +432,13 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v0.14.1 h1:nQcJDQwIAGnmoUWp8ubocEX40cCml/17YkF6csQLReU= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.16.2 h1:K4ev2ib4LdQETX5cSZBG0DVLk1jwGqSPXBjdah3veNs= +github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.0 h1:8exGP7ego3OmkfksihtSouGMZ+hQrhxx+FVELeXpVPE= github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -629,9 +630,9 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= +github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= +github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -753,7 +754,6 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= @@ -976,8 +976,9 @@ golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1080,13 +1081,15 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1096,8 +1099,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1335,21 +1339,21 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ= -k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI= -k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc= -k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= -k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0= -k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= +k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= +k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= +k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= +k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= +k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 h1:tBEbstoM+K0FiBV5KGAKQ0kuvf54v/hwpldiJt69w1s= +k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From c396c3e32f0b51cadf626edca2c9ee6bb4a7074b Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Wed, 14 Dec 2022 11:30:36 +0100 Subject: [PATCH 029/107] Update go dependencies before 2.41 Signed-off-by: Julien Pivotto --- docs/configuration/configuration.md | 76 +++++++++ go.mod | 74 ++++----- go.sum | 229 +++++++++++----------------- 3 files changed, 199 insertions(+), 180 deletions(-) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 701c62546..7b78e14b6 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -208,6 +208,10 @@ tls_config: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] + # List of Azure service discovery configurations. azure_sd_configs: @@ -427,6 +431,9 @@ tls_config: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] ``` ### `` @@ -505,6 +512,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -607,6 +617,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -685,6 +698,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -734,6 +750,9 @@ host: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # TLS configuration. tls_config: @@ -900,6 +919,9 @@ host: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # TLS configuration. tls_config: @@ -1098,6 +1120,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -1364,6 +1389,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -1572,6 +1600,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -1657,6 +1688,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -1731,6 +1765,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -1944,6 +1981,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -2021,6 +2061,9 @@ server: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # TLS configuration. tls_config: @@ -2139,6 +2182,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -2211,6 +2257,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -2310,6 +2359,9 @@ tls_config: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] ``` By default every app listed in Marathon will be scraped by Prometheus. If not all @@ -2400,6 +2452,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -2577,6 +2632,9 @@ tls_config: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -2693,6 +2751,9 @@ tags_filter: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # TLS configuration. tls_config: @@ -2760,6 +2821,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -2833,6 +2897,9 @@ oauth2: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -3026,6 +3093,9 @@ tls_config: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -3235,6 +3305,9 @@ tls_config: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] @@ -3339,6 +3412,9 @@ tls_config: # Optional proxy URL. [ proxy_url: ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_headers: + [ : [, ...] ] ] # Configure whether HTTP requests follow HTTP 3xx redirects. [ follow_redirects: | default = true ] diff --git a/go.mod b/go.mod index 4ec1aba54..8cd0093a2 100644 --- a/go.mod +++ b/go.mod @@ -7,15 +7,15 @@ require ( github.com/Azure/go-autorest/autorest v0.11.28 github.com/Azure/go-autorest/autorest/adal v0.9.21 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 - github.com/aws/aws-sdk-go v1.44.131 - github.com/cespare/xxhash/v2 v2.1.2 + github.com/aws/aws-sdk-go v1.44.159 + github.com/cespare/xxhash/v2 v2.2.0 github.com/dennwc/varint v1.0.0 github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245 - github.com/digitalocean/godo v1.89.0 + github.com/digitalocean/godo v1.91.1 github.com/docker/docker v20.10.21+incompatible github.com/edsrzf/mmap-go v1.1.0 github.com/envoyproxy/go-control-plane v0.10.3 - github.com/envoyproxy/protoc-gen-validate v0.8.0 + github.com/envoyproxy/protoc-gen-validate v0.9.1 github.com/fsnotify/fsnotify v1.6.0 github.com/go-kit/log v0.2.1 github.com/go-logfmt/logfmt v0.5.1 @@ -23,13 +23,13 @@ require ( github.com/go-zookeeper/zk v1.0.3 github.com/gogo/protobuf v1.3.2 github.com/golang/snappy v0.0.4 - github.com/google/pprof v0.0.0-20221102093814-76f304f74e5e - github.com/gophercloud/gophercloud v1.0.0 - github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6 + github.com/google/pprof v0.0.0-20221212185716-aee1124e3a93 + github.com/gophercloud/gophercloud v1.1.1 + github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd github.com/grpc-ecosystem/grpc-gateway v1.16.0 - github.com/hashicorp/consul/api v1.15.3 - github.com/hashicorp/nomad/api v0.0.0-20221102143410-8a95f1239005 - github.com/hetznercloud/hcloud-go v1.35.3 + github.com/hashicorp/consul/api v1.18.0 + github.com/hashicorp/nomad/api v0.0.0-20221214074818-7dbbf6bc584d + github.com/hetznercloud/hcloud-go v1.38.0 github.com/ionos-cloud/sdk-go/v6 v6.1.3 github.com/json-iterator/go v1.1.12 github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b @@ -41,35 +41,35 @@ require ( github.com/ovh/go-ovh v1.3.0 github.com/pkg/errors v0.9.1 github.com/prometheus/alertmanager v0.24.0 - github.com/prometheus/client_golang v1.13.1 + github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 - github.com/prometheus/common v0.38.0 + github.com/prometheus/common v0.39.0 github.com/prometheus/common/assets v0.2.0 github.com/prometheus/common/sigv4 v0.1.0 github.com/prometheus/exporter-toolkit v0.8.2 - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 github.com/stretchr/testify v1.8.1 github.com/vultr/govultr/v2 v2.17.2 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4 - go.opentelemetry.io/otel v1.11.1 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.1 - go.opentelemetry.io/otel/sdk v1.11.1 - go.opentelemetry.io/otel/trace v1.11.1 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0 + go.opentelemetry.io/otel v1.11.2 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2 + go.opentelemetry.io/otel/sdk v1.11.2 + go.opentelemetry.io/otel/trace v1.11.2 go.uber.org/atomic v1.10.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/goleak v1.2.0 golang.org/x/net v0.4.0 - golang.org/x/oauth2 v0.1.0 + golang.org/x/oauth2 v0.3.0 golang.org/x/sync v0.1.0 golang.org/x/sys v0.3.0 - golang.org/x/time v0.1.0 - golang.org/x/tools v0.2.0 - google.golang.org/api v0.102.0 - google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c - google.golang.org/grpc v1.50.1 + golang.org/x/time v0.3.0 + golang.org/x/tools v0.4.0 + google.golang.org/api v0.104.0 + google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 + google.golang.org/grpc v1.51.0 google.golang.org/protobuf v1.28.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 @@ -82,12 +82,12 @@ require ( ) require ( - cloud.google.com/go/compute/metadata v0.2.1 // indirect + cloud.google.com/go/compute/metadata v0.2.2 // indirect github.com/coreos/go-systemd/v22 v22.4.0 // indirect ) require ( - cloud.google.com/go/compute v1.12.1 // indirect + cloud.google.com/go/compute v1.13.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect @@ -99,7 +99,7 @@ require ( github.com/armon/go-metrics v0.3.10 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect @@ -132,7 +132,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.6.0 // indirect + github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.1 // indirect github.com/hashicorp/cronexpr v1.1.1 // indirect @@ -142,7 +142,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.1 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/hashicorp/serf v0.9.7 // indirect + github.com/hashicorp/serf v0.10.1 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -151,7 +151,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect @@ -166,13 +166,13 @@ require ( github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect go.mongodb.org/mongo-driver v1.10.2 // indirect - go.opencensus.io v0.23.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 // indirect - go.opentelemetry.io/otel/metric v0.33.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect + go.opentelemetry.io/otel/metric v0.34.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect golang.org/x/crypto v0.1.0 // indirect - golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 - golang.org/x/mod v0.6.0 // indirect + golang.org/x/exp v0.0.0-20221212164502-fae10dda9338 + golang.org/x/mod v0.7.0 // indirect golang.org/x/term v0.3.0 // indirect golang.org/x/text v0.5.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index d1e83fb19..e33c4a716 100644 --- a/go.sum +++ b/go.sum @@ -3,7 +3,6 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -13,9 +12,6 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -23,13 +19,13 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= -cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= -cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute v1.13.0 h1:AYrLkB8NPdDRslNp4Jxmzrhdr03fUAIDbiGFjLWowoU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute/metadata v0.2.2 h1:aWKAjYaBaOSrpKl57+jnS/3fJRQnxL7TvR/u1VVbt6k= +cloud.google.com/go/compute/metadata v0.2.2/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/longrunning v0.1.1 h1:y50CXG4j0+qvEukslYFBCrzaXX0qpFbBzc3PchSu/LE= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -39,7 +35,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -103,8 +98,8 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.131 h1:kd61x79ax0vyiC/SZ9X1hKh8E0pt1BUOOcVBJEFhxkg= -github.com/aws/aws-sdk-go v1.44.131/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.159 h1:9odtuHAYQE9tQKyuX6ny1U1MHeH5/yzeCJi96g9H4DU= +github.com/aws/aws-sdk-go v1.44.159/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -114,14 +109,15 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= +github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -130,7 +126,6 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= @@ -158,8 +153,8 @@ github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgz github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245 h1:9cOfvEwjQxdwKuNDTQSaMKNRvwKwgZG+U4HrjeRKHso= github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/digitalocean/godo v1.89.0 h1:UL3Ii4qfk86m4qEKg2iSwop0puvgOCKvwzXvwArU05E= -github.com/digitalocean/godo v1.89.0/go.mod h1:NRpFznZFvhHjBoqZAaOD3khVzsJ3EibzKqFL4R60dmA= +github.com/digitalocean/godo v1.91.1 h1:1o30VOCu1aC6488qBd0SkQiBeAZ35RSTvLwCA1pQMhc= +github.com/digitalocean/godo v1.91.1/go.mod h1:NRpFznZFvhHjBoqZAaOD3khVzsJ3EibzKqFL4R60dmA= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -184,7 +179,6 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= @@ -192,8 +186,8 @@ github.com/envoyproxy/go-control-plane v0.10.3 h1:xdCVXxEe0Y3FQith+0cj2irwZudqGY github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= -github.com/envoyproxy/protoc-gen-validate v0.8.0 h1:eZxAlfY5c/HTcV7aN9EUL3Ej/zY/WDmawwClR16nfDA= -github.com/envoyproxy/protoc-gen-validate v0.8.0/go.mod h1:z+FSjkCuAJYqUS2daO/NBFgbCao8JDHcYcpnFfD00cI= +github.com/envoyproxy/protoc-gen-validate v0.9.1 h1:PS7VIOgmSVhWUEeZwTe7z7zouA22Cr590PzXKbZHOVY= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -371,7 +365,6 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -379,11 +372,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20221102093814-76f304f74e5e h1:F1LLQqQ8WoIbyoxLUY+JUZe1kuHdxThM6CPUATzE6Io= -github.com/google/pprof v0.0.0-20221102093814-76f304f74e5e/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/pprof v0.0.0-20221212185716-aee1124e3a93 h1:D5iJJZKAi0rU4e/5E58BkrnN+xeCDjAIqcm1GGxAGSI= +github.com/google/pprof v0.0.0-20221212185716-aee1124e3a93/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -394,11 +384,10 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbez github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.6.0 h1:SXk3ABtQYDT/OH8jAyvEOQ58mgawq5C4o/4/89qN2ZU= -github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gophercloud/gophercloud v1.0.0 h1:9nTGx0jizmHxDobe4mck89FyQHVyA3CaXLIUSGJjP9k= -github.com/gophercloud/gophercloud v1.0.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= +github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/gophercloud/gophercloud v1.1.1 h1:MuGyqbSxiuVBqkPZ3+Nhbytk1xZxhmfCB2Rg1cJWFWM= +github.com/gophercloud/gophercloud v1.1.1/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -406,8 +395,8 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6 h1:A3dhViTeFDSQcGOXuUi6ukCQSMyDtDISBp2z6OOo2YM= -github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= +github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww= +github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -417,11 +406,11 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4Zs github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.1 h1:/sDbPb60SusIXjiJGYLUoS/rAQurQmvGWmwn2bBPM9c= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.1/go.mod h1:G+WkljZi4mflcqVxYSgvt8MNctRQHjEH8ubKtt1Ka3w= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/api v1.15.3 h1:WYONYL2rxTXtlekAqblR2SCdJsizMDIj/uXb5wNy9zU= -github.com/hashicorp/consul/api v1.15.3/go.mod h1:/g/qgcoBcEXALCNZgRRisyTW0nY86++L0KbeAMXYCeY= +github.com/hashicorp/consul/api v1.18.0 h1:R7PPNzTCeN6VuQNDwwhZWJvzCtGSrNpJqfb22h3yH9g= +github.com/hashicorp/consul/api v1.18.0/go.mod h1:owRRGJ9M5xReDC5nfT8FTJrNAPbT4NM6p/k+d03q2v4= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.11.0 h1:HRzj8YSCln2yGgCumN5CL8lYlD3gBurnervJRJAZyC4= -github.com/hashicorp/consul/sdk v0.11.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw= +github.com/hashicorp/consul/sdk v0.13.0 h1:lce3nFlpv8humJL8rNrrGHYSKc3q+Kxfeg3Ii1m6ZWU= +github.com/hashicorp/consul/sdk v0.13.0/go.mod h1:0hs/l5fOVhJy/VdcoaNqUSi2AUs95eF5WKtv+EYIQqE= github.com/hashicorp/cronexpr v1.1.1 h1:NJZDd87hGXjoZBdvyCF9mX4DCq5Wy7+A/w+A7q0wn6c= github.com/hashicorp/cronexpr v1.1.1/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -432,20 +421,16 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.16.2 h1:K4ev2ib4LdQETX5cSZBG0DVLk1jwGqSPXBjdah3veNs= github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= -github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= @@ -461,6 +446,8 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -470,21 +457,20 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1vWZfM= github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/nomad/api v0.0.0-20221102143410-8a95f1239005 h1:jKwXhVS4F7qk0g8laz+Anz0g/6yaSJ3HqmSAuSNLUcA= -github.com/hashicorp/nomad/api v0.0.0-20221102143410-8a95f1239005/go.mod h1:vgJmrz4Bz9E1cR/uy70oP9udUJKFRkcEYHlHTp4nFwI= +github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= +github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= +github.com/hashicorp/nomad/api v0.0.0-20221214074818-7dbbf6bc584d h1:kEWrUx7mld3c6HRcO2KhfD1MYBkofuZfEfDwCRQ9aMU= +github.com/hashicorp/nomad/api v0.0.0-20221214074818-7dbbf6bc584d/go.mod h1:8FB4gnSJAfRGxfG+v0pZEPfqhZG7nZ87xDeUyw3gEMI= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/serf v0.9.7 h1:hkdgbqizGQHuU5IPqYM1JdSMV8nKfpuOnZYXssk9muY= -github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/hetznercloud/hcloud-go v1.35.3 h1:WCmFAhLRooih2QHAsbCbEdpIHnshQQmrPqsr3rHE1Ow= -github.com/hetznercloud/hcloud-go v1.35.3/go.mod h1:mepQwR6va27S3UQthaEPGS86jtzSY9xWL1e9dyxXpgA= +github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= +github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= +github.com/hetznercloud/hcloud-go v1.38.0 h1:K6Pd/mMdcLfBhvwG39qyAaacp4pCS3dKa8gChmLKxLg= +github.com/hetznercloud/hcloud-go v1.38.0/go.mod h1:mepQwR6va27S3UQthaEPGS86jtzSY9xWL1e9dyxXpgA= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -545,7 +531,6 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U github.com/linode/linodego v1.9.3 h1:+lxNZw4avRxhCqGjwfPgQ2PvMT+vOL0OMsTdzixR7hQ= github.com/linode/linodego v1.9.3/go.mod h1:h6AuFR/JpqwwM/vkj7s8KV3iGN8/jxn+zc437F8SZ8w= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= -github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -571,8 +556,8 @@ github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -664,7 +649,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -680,8 +664,8 @@ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c= -github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -698,8 +682,8 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.38.0 h1:VTQitp6mXTdUoCmDMugDVOJ1opi6ADftKfp/yeqTR/E= -github.com/prometheus/common v0.38.0/go.mod h1:MBXfmBQZrK5XpbCkjofnXs96LD2QQ7fEq4C0xjC/yec= +github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= +github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= github.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM= github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI= github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= @@ -728,11 +712,11 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 h1:0roa6gXKgyta64uqh52AQG3wzZXH21unn+ltzQSXML0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 h1:wsfMs0iv+MJiViM37qh5VEKISi3/ZUq2nNKNdqmumAs= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shoenig/test v0.4.3 h1:3+CjrpqCwtL08S0wZQilu9WWR/S2CdsLKhHjbJqPj/I= +github.com/shoenig/test v0.4.6 h1:S1pAVs5L1TSRen3N1YQNtBZIh9Z6d1PyQSUDUweMTqk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -756,7 +740,6 @@ github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -820,27 +803,26 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4 h1:aUEBEdCa6iamGzg6fuYxDA8ThxvOG240mAvWDU+XLio= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4/go.mod h1:l2MdsbKTocpPS5nQZscqTR9jd8u96VYZdcpF8Sye7mA= -go.opentelemetry.io/otel v1.11.1 h1:4WLLAmcfkmDk2ukNXJyq3/kiz/3UzCaYq6PskJsaou4= -go.opentelemetry.io/otel v1.11.1/go.mod h1:1nNhXBbWSD0nsL38H6btgnFN2k4i0sNLHNNMZMSbUGE= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 h1:X2GndnMCsUPh6CiY2a+frAbNsXaPLbB0soHRYhAZ5Ig= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1/go.mod h1:i8vjiSzbiUC7wOQplijSXMYUpNM93DtlS5CbUT+C6oQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1 h1:MEQNafcNCB0uQIti/oHgU7CZpUMYQ7qigBwMVKycHvc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1/go.mod h1:19O5I2U5iys38SsmT2uDJja/300woyzE1KPIQxEUBUc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1 h1:LYyG/f1W/jzAix16jbksJfMQFpOH/Ma6T639pVPMgfI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1/go.mod h1:QrRRQiY3kzAoYPNLP0W/Ikg0gR6V3LMc+ODSxr7yyvg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.1 h1:tFl63cpAAcD9TOU6U8kZU7KyXuSRYAZlbx1C61aaB74= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.1/go.mod h1:X620Jww3RajCJXw/unA+8IRTgxkdS7pi+ZwK9b7KUJk= -go.opentelemetry.io/otel/metric v0.33.0 h1:xQAyl7uGEYvrLAiV/09iTJlp1pZnQ9Wl793qbVvED1E= -go.opentelemetry.io/otel/metric v0.33.0/go.mod h1:QlTYc+EnYNq/M2mNk1qDDMRLpqCOj2f/r5c7Fd5FYaI= -go.opentelemetry.io/otel/sdk v1.11.1 h1:F7KmQgoHljhUuJyA+9BiU+EkJfyX5nVVF4wyzWZpKxs= -go.opentelemetry.io/otel/sdk v1.11.1/go.mod h1:/l3FE4SupHJ12TduVjUkZtlfFqDCQJlOlithYrdktys= -go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZXF8gdQ= -go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0 h1:yt2NKzK7Vyo6h0+X8BA4FpreZQTlVEIarnsBP/H5mzs= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0/go.mod h1:+ARmXlUlc51J7sZeCBkBJNdHGySrdOzgzxp6VWRWM1U= +go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0= +go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 h1:htgM8vZIF8oPSCxa341e3IZ4yr/sKxgu8KZYllByiVY= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2/go.mod h1:rqbht/LlhVBgn5+k3M5QK96K5Xb0DvXpMJ5SFQpY6uw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 h1:fqR1kli93643au1RKo0Uma3d2aPQKT+WBKfTSBaKbOc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2/go.mod h1:5Qn6qvgkMsLDX+sYK64rHb1FPhpn0UtxF+ouX1uhyJE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2 h1:ERwKPn9Aer7Gxsc0+ZlutlH1bEEAUXAUhqm3Y45ABbk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2/go.mod h1:jWZUM2MWhWCJ9J9xVbRx7tzK1mXKpAlze4CeulycwVY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2 h1:Us8tbCmuN16zAnK5TC69AtODLycKbwnskQzaB6DfFhc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2/go.mod h1:GZWSQQky8AgdJj50r1KJm8oiQiIPaAX7uZCFQX9GzC8= +go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8= +go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8= +go.opentelemetry.io/otel/sdk v1.11.2 h1:GF4JoaEx7iihdMFu30sOyRx52HDHOkl9xQ8SMqNXUiU= +go.opentelemetry.io/otel/sdk v1.11.2/go.mod h1:wZ1WxImwpq+lVRo4vsmSOxdd+xwoUJ6rqyLc3SyX9aU= +go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0= +go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= @@ -872,13 +854,11 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -891,8 +871,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= -golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20221212164502-fae10dda9338 h1:OvjRkcNHnf6/W5FZXSxODbxwD+X7fspczG7Jn/xQVD4= +golang.org/x/exp v0.0.0-20221212164502-fae10dda9338/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -905,7 +885,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= @@ -916,14 +895,12 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -959,10 +936,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= @@ -984,14 +958,10 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.1.0 h1:isLCZuhj4v+tYv7eskaN4v/TM+A1begWWgyVJDdl1+Y= -golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= +golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= +golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1053,20 +1023,14 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1080,6 +1044,7 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= @@ -1095,7 +1060,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -1106,8 +1070,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1159,26 +1123,18 @@ golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1194,11 +1150,8 @@ google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/ google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I= -google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= +google.golang.org/api v0.104.0 h1:KBfmLRqdZEbwQleFlSLnzpQJwhjpmNOk4cKQIBDZ9mg= +google.golang.org/api v0.104.0/go.mod h1:JCspTXJbBxa5ySXw4UgUqVer7DfVxbvc/CTUFqAED5U= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1237,18 +1190,11 @@ google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1m google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo= -google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 h1:jmIfw8+gSvXcZSgaFAGyInDXeWzUhvYH57G/5GKMn70= +google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1265,17 +1211,14 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= -google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 9e26adfd20bd5f7868ab871a34ebbdc852910166 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Wed, 14 Dec 2022 12:05:16 +0100 Subject: [PATCH 030/107] Add myself as release shepherd (#11693) Signed-off-by: Julien Pivotto Signed-off-by: Julien Pivotto --- RELEASE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 6d5c7f060..48c5caa00 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -45,7 +45,8 @@ Release cadence of first pre-releases being cut is 6 weeks. | v2.38 | 2022-08-10 | Julius Volz (GitHub: @juliusv) | | v2.39 | 2022-09-21 | Ganesh Vernekar (GitHub: @codesome) | | v2.40 | 2022-11-02 | Ganesh Vernekar (GitHub: @codesome) | -| v2.41 | 2022-12-14 | **searching for volunteer** | +| v2.41 | 2022-12-14 | Julien Pivotto (GitHub: @roidelapluie) | +| v2.42 | 2023-01-25 | **searching for volunteer** | If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice. From 87b9f1d24a6e89dd7d43dc57f97c2a1246d9d818 Mon Sep 17 00:00:00 2001 From: Danny Staple Date: Wed, 14 Dec 2022 12:20:28 +0000 Subject: [PATCH 031/107] Fix typo I introduced in unit testing rules. Signed-off-by: Danny Staple --- docs/configuration/unit_testing_rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index 44a61fb30..d992678cd 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -86,7 +86,7 @@ series: # Examples: # 1. '-2+4x3' becomes '-2 2 6 10' - series starts at -2, then 3 further samples incrementing by 4. # 2. ' 1-2x4' becomes '1 -1 -3 -5 -7' - series starts at 1, then 4 further samples decrementing by 2. -# 3. ' 1x4' become '1 1 1 1' - shorthand for '1+0x4', series starts at 1, then 3 further samples incrementing by 0. +# 3. ' 1x4' becomes '1 1 1 1' - shorthand for '1+0x4', series starts at 1, then 3 further samples incrementing by 0. # 4. ' 1 _x3 stale' becomes '1 _ _ _ stale' - the missing sample cannot increment, so only 3 samples are produced by the '_x3' expression. values: ``` From 7269a6e21a3718cd73bb975a75600086734bd16c Mon Sep 17 00:00:00 2001 From: Danny Staple Date: Wed, 14 Dec 2022 12:21:34 +0000 Subject: [PATCH 032/107] Fix the output example (based on empirical unit testing) Signed-off-by: Danny Staple --- docs/configuration/unit_testing_rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index d992678cd..6d0725f19 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -86,7 +86,7 @@ series: # Examples: # 1. '-2+4x3' becomes '-2 2 6 10' - series starts at -2, then 3 further samples incrementing by 4. # 2. ' 1-2x4' becomes '1 -1 -3 -5 -7' - series starts at 1, then 4 further samples decrementing by 2. -# 3. ' 1x4' becomes '1 1 1 1' - shorthand for '1+0x4', series starts at 1, then 3 further samples incrementing by 0. +# 3. ' 1x4' becomes '1 1 1 1 1' - shorthand for '1+0x4', series starts at 1, then 4 further samples incrementing by 0. # 4. ' 1 _x3 stale' becomes '1 _ _ _ stale' - the missing sample cannot increment, so only 3 samples are produced by the '_x3' expression. values: ``` From 29707b156934373cc891669059eb09006523b90e Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Wed, 14 Dec 2022 16:34:36 +0100 Subject: [PATCH 033/107] Makefile: Avoid the use of --transform for tar This makes it compatible with macos tar as well Signed-off-by: Julien Pivotto --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4c40efd36..dd51d5817 100644 --- a/Makefile +++ b/Makefile @@ -90,8 +90,10 @@ endif .PHONY: npm_licenses npm_licenses: ui-install @echo ">> bundling npm licenses" - rm -f $(REACT_APP_NPM_LICENSES_TARBALL) - find $(UI_NODE_MODULES_PATH) -iname "license*" | tar cfj $(REACT_APP_NPM_LICENSES_TARBALL) --transform 's/^/npm_licenses\//' --files-from=- + rm -f $(REACT_APP_NPM_LICENSES_TARBALL) npm_licenses + ln -s . npm_licenses + find npm_licenses/$(UI_NODE_MODULES_PATH) -iname "license*" | tar cfj $(REACT_APP_NPM_LICENSES_TARBALL) --files-from=- + rm -f npm_licenses .PHONY: tarball tarball: npm_licenses common-tarball From f8f4ac14a86fae5e3e8cf912aadfed314cf4ee65 Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Wed, 14 Dec 2022 13:08:52 -0800 Subject: [PATCH 034/107] Finishing evalSpanTimer always before return Signed-off-by: Alan Protasio --- promql/engine.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/promql/engine.go b/promql/engine.go index a2b438489..8dd76074c 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -653,12 +653,12 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval query.sampleStats.InitStepTracking(start, start, 1) val, warnings, err := evaluator.Eval(s.Expr) + evalSpanTimer.Finish() + if err != nil { return nil, warnings, err } - evalSpanTimer.Finish() - var mat Matrix switch result := val.(type) { @@ -704,10 +704,12 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval } query.sampleStats.InitStepTracking(evaluator.startTimestamp, evaluator.endTimestamp, evaluator.interval) val, warnings, err := evaluator.Eval(s.Expr) + + evalSpanTimer.Finish() + if err != nil { return nil, warnings, err } - evalSpanTimer.Finish() mat, ok := val.(Matrix) if !ok { From 8460807475ad73592c5bb956f84aef024130be69 Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Wed, 14 Dec 2022 13:24:02 -0800 Subject: [PATCH 035/107] fix blank lines Signed-off-by: Alan Protasio --- promql/engine.go | 1 + 1 file changed, 1 insertion(+) diff --git a/promql/engine.go b/promql/engine.go index 8dd76074c..b3ad14b3d 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -653,6 +653,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval query.sampleStats.InitStepTracking(start, start, 1) val, warnings, err := evaluator.Eval(s.Expr) + evalSpanTimer.Finish() if err != nil { From 1a2c645dfa095cd20af2af1a7d40dbd41214c653 Mon Sep 17 00:00:00 2001 From: Julius Volz Date: Thu, 15 Dec 2022 12:11:25 +0100 Subject: [PATCH 036/107] Correctly handle error unwrapping in rules and remote write receiver errors.Unwrap() actually dangerously returns nil if the error does not have an Unwrap() method, which is the case in at least one of these places where I noticed that no error was being logged at all when it should have. Signed-off-by: Julius Volz --- rules/manager.go | 9 +++++++++ storage/remote/write_handler.go | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/rules/manager.go b/rules/manager.go index ced61ea49..42f1b59ce 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -673,6 +673,9 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) { rule.SetLastError(err) sp.SetStatus(codes.Error, err.Error()) unwrappedErr := errors.Unwrap(err) + if unwrappedErr == nil { + unwrappedErr = err + } switch { case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample): numOutOfOrder++ @@ -700,6 +703,9 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) { // Series no longer exposed, mark it stale. _, err = app.Append(0, lset, timestamp.FromTime(ts), math.Float64frombits(value.StaleNaN)) unwrappedErr := errors.Unwrap(err) + if unwrappedErr == nil { + unwrappedErr = err + } switch { case unwrappedErr == nil: case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample), errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp): @@ -727,6 +733,9 @@ func (g *Group) cleanupStaleSeries(ctx context.Context, ts time.Time) { // Rule that produced series no longer configured, mark it stale. _, err := app.Append(0, s, timestamp.FromTime(ts), math.Float64frombits(value.StaleNaN)) unwrappedErr := errors.Unwrap(err) + if unwrappedErr == nil { + unwrappedErr = err + } switch { case unwrappedErr == nil: case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample), errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp): diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go index ad1f0f3ae..7a2a9951c 100644 --- a/storage/remote/write_handler.go +++ b/storage/remote/write_handler.go @@ -67,11 +67,14 @@ func (h *writeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // checkAppendExemplarError modifies the AppendExamplar's returned error based on the error cause. func (h *writeHandler) checkAppendExemplarError(err error, e exemplar.Exemplar, outOfOrderErrs *int) error { - unwrapedErr := errors.Unwrap(err) + unwrappedErr := errors.Unwrap(err) + if unwrappedErr == nil { + unwrappedErr = err + } switch { - case errors.Is(unwrapedErr, storage.ErrNotFound): + case errors.Is(unwrappedErr, storage.ErrNotFound): return storage.ErrNotFound - case errors.Is(unwrapedErr, storage.ErrOutOfOrderExemplar): + case errors.Is(unwrappedErr, storage.ErrOutOfOrderExemplar): *outOfOrderErrs++ level.Debug(h.logger).Log("msg", "Out of order exemplar", "exemplar", fmt.Sprintf("%+v", e)) return nil @@ -98,8 +101,11 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err for _, s := range ts.Samples { _, err = app.Append(0, labels, s.Timestamp, s.Value) if err != nil { - unwrapedErr := errors.Unwrap(err) - if errors.Is(unwrapedErr, storage.ErrOutOfOrderSample) || errors.Is(unwrapedErr, storage.ErrOutOfBounds) || errors.Is(unwrapedErr, storage.ErrDuplicateSampleForTimestamp) { + unwrappedErr := errors.Unwrap(err) + if unwrappedErr == nil { + unwrappedErr = err + } + if errors.Is(err, storage.ErrOutOfOrderSample) || errors.Is(unwrappedErr, storage.ErrOutOfBounds) || errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp) { level.Error(h.logger).Log("msg", "Out of order sample from remote write", "err", err.Error(), "series", labels.String(), "timestamp", s.Timestamp) } return err @@ -123,6 +129,9 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err _, err = app.AppendHistogram(0, labels, hp.Timestamp, hs) if err != nil { unwrappedErr := errors.Unwrap(err) + if unwrappedErr == nil { + unwrappedErr = err + } // Althogh AppendHistogram does not currently return ErrDuplicateSampleForTimestamp there is // a note indicating its inclusion in the future. if errors.Is(unwrappedErr, storage.ErrOutOfOrderSample) || errors.Is(unwrappedErr, storage.ErrOutOfBounds) || errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp) { From 8aae683d43183f6f0dec014810cf351ecc5c1359 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Wed, 14 Dec 2022 14:26:10 +0100 Subject: [PATCH 037/107] Update docker dependency Dependabots complains about this Signed-off-by: Julien Pivotto --- documentation/examples/remote_storage/go.mod | 1 + documentation/examples/remote_storage/go.sum | 3 ++- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/documentation/examples/remote_storage/go.mod b/documentation/examples/remote_storage/go.mod index 45e02136a..fea3a913b 100644 --- a/documentation/examples/remote_storage/go.mod +++ b/documentation/examples/remote_storage/go.mod @@ -21,6 +21,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dennwc/varint v1.0.0 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.2.3 // indirect diff --git a/documentation/examples/remote_storage/go.sum b/documentation/examples/remote_storage/go.sum index c1de813cc..505df7ceb 100644 --- a/documentation/examples/remote_storage/go.sum +++ b/documentation/examples/remote_storage/go.sum @@ -38,7 +38,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/digitalocean/godo v1.82.0 h1:lqAit46H1CqJGjh7LDbsamng/UMBME5rvmfH3Vb5Yy8= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.17+incompatible h1:JYCuMrWaVNophQTOrMMoSwudOVEfcegoZZrleKc1xwE= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= diff --git a/go.mod b/go.mod index 8cd0093a2..b3eaec7c6 100644 --- a/go.mod +++ b/go.mod @@ -102,7 +102,7 @@ require ( github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect diff --git a/go.sum b/go.sum index e33c4a716..6c5e19993 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8 github.com/digitalocean/godo v1.91.1 h1:1o30VOCu1aC6488qBd0SkQiBeAZ35RSTvLwCA1pQMhc= github.com/digitalocean/godo v1.91.1/go.mod h1:NRpFznZFvhHjBoqZAaOD3khVzsJ3EibzKqFL4R60dmA= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.21+incompatible h1:UTLdBmHk3bEY+w8qeO5KttOhy6OmXWsl/FEet9Uswog= github.com/docker/docker v20.10.21+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= From 75af6531e43b81f18126930b01f8c864384a3a6f Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Wed, 14 Dec 2022 14:59:31 +0100 Subject: [PATCH 038/107] Release v2.41.0-rc.0 Signed-off-by: Julien Pivotto --- CHANGELOG.md | 15 +++++++++++++++ VERSION | 2 +- web/ui/module/codemirror-promql/package.json | 4 ++-- web/ui/module/lezer-promql/package.json | 2 +- web/ui/package-lock.json | 14 +++++++------- web/ui/react-app/package.json | 4 ++-- 6 files changed, 28 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 843789766..8c816b776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 2.41.0-rc.0 / 2022-12-14 + +* [FEATURE] Relabeling: Add `keepequal` and `dropequal` relabel actions. #11564 +* [FEATURE] Add support for HTTP proxy headers. #11712 +* [ENHANCEMENT] Reload private certificates when changed on disk. #11685 +* [ENHANCEMENT] Add `max_version` to specify maximum TLS version in `tls_config`. #11685 +* [ENHANCEMENT] Add `goos` and `goarch` labels to `prometheus_build_info`. #11685 +* [ENHANCEMENT] SD: Add proxy support for EC2 and LightSail SDs #11611 +* [ENHANCEMENT] SD: Add new metric `prometheus_sd_file_watcher_errors_total`. #11066 +* [ENHANCEMENT] Remote Read: Use a pool to speed up marshalling. #11357 +* [ENHANCEMENT] TSDB: Improve handling of tombstoned chunks in iterators. #11632 +* [ENHANCEMENT] TSDB: Optimize postings offset table reading. #11535 +* [BUGFIX] Scrape: Validate the metric name, label names, and label values after relabeling. #11074 +* [BUGFIX] Remote Write receiver and rule manager: Fix error handling. #11727 + ## 2.40.7 / 2022-12-14 * [BUGFIX] Use Windows native DNS resolver. #11704 diff --git a/VERSION b/VERSION index 5c4c670b5..29631c945 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.40.7 +2.41.0-rc.0 diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index b70d827b3..507283f64 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.7", + "version": "0.41.0-rc.0", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.7", + "@prometheus-io/lezer-promql": "^0.41.0-rc.0", "lru-cache": "^6.0.0" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index f7416de19..93ae3c074 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.40.7", + "version": "0.41.0-rc.0", "description": "lezer-based PromQL grammar", "main": "index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index 9186d7f91..e326c690e 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -28,10 +28,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.40.7", + "version": "0.41.0-rc.0", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "^0.40.7", + "@prometheus-io/lezer-promql": "^0.41.0-rc.0", "lru-cache": "^6.0.0" }, "devDependencies": { @@ -61,7 +61,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.40.7", + "version": "0.41.0-rc.0", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.1.1", @@ -20674,7 +20674,7 @@ }, "react-app": { "name": "@prometheus-io/app", - "version": "0.40.7", + "version": "0.41.0-rc.0", "dependencies": { "@codemirror/autocomplete": "^6.2.0", "@codemirror/commands": "^6.1.2", @@ -20692,7 +20692,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.7", + "@prometheus-io/codemirror-promql": "^0.41.0-rc.0", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^7.0.1", @@ -23321,7 +23321,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.7", + "@prometheus-io/codemirror-promql": "^0.41.0-rc.0", "@testing-library/react-hooks": "^7.0.2", "@types/enzyme": "^3.10.12", "@types/flot": "0.0.32", @@ -23372,7 +23372,7 @@ "@lezer/common": "^1.0.1", "@lezer/highlight": "^1.1.2", "@lezer/lr": "^1.2.3", - "@prometheus-io/lezer-promql": "^0.40.7", + "@prometheus-io/lezer-promql": "^0.41.0-rc.0", "@types/lru-cache": "^5.1.1", "isomorphic-fetch": "^3.0.0", "lru-cache": "^6.0.0", diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index 0f14638fd..835336119 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/app", - "version": "0.40.7", + "version": "0.41.0-rc.0", "private": true, "dependencies": { "@codemirror/autocomplete": "^6.2.0", @@ -19,7 +19,7 @@ "@lezer/common": "^1.0.1", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.40.7", + "@prometheus-io/codemirror-promql": "^0.41.0-rc.0", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^7.0.1", From 6197ed63d86a591eba72ff27fa11871d4e73c102 Mon Sep 17 00:00:00 2001 From: Oleg Zaytsev Date: Thu, 15 Dec 2022 16:57:44 +0100 Subject: [PATCH 039/107] Remove comments from the remote read docs I think these are not intended to be here. Signed-off-by: Oleg Zaytsev --- docs/querying/remote_read_api.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/querying/remote_read_api.md b/docs/querying/remote_read_api.md index aae8e5d1b..30073c3e2 100644 --- a/docs/querying/remote_read_api.md +++ b/docs/querying/remote_read_api.md @@ -65,8 +65,6 @@ Note: Names of query parameters that may be repeated end with `[]`. This API provides data read functionality from Prometheus. This interface expects [snappy](https://github.com/google/snappy) compression. The API definition is located [here](https://github.com/prometheus/prometheus/blob/master/prompb/remote.proto). -/// Can you clarify what you mean by this? -/// https://github.com/prometheus/prometheus/pull/7266#discussion_r426456791 Can we talk a little bit how negotiation works of sampled vs streamed ? Request are made to the following endpoint. ``` @@ -74,12 +72,10 @@ Request are made to the following endpoint. ``` ### Samples -/// Does it return a message that includes a list, or does it return a list of raw samples? This returns a message that includes a list of raw samples. ### Streamed Chunks -/// This is a little much detail, the relevant point is they're the internal implementation of the chunks. These streamed chunks utilize an XOR algorithm inspired by the [Gorilla](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) compression to encode the chunks. However, it provides resolution to the millisecond instead of to the second. From f3f800ea6f8e601cba27bd265dfe3f4955d7a911 Mon Sep 17 00:00:00 2001 From: Danny Staple Date: Thu, 15 Dec 2022 16:22:40 +0000 Subject: [PATCH 040/107] Terminology amendment Signed-off-by: Danny Staple --- docs/configuration/unit_testing_rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index 6d0725f19..efd168b35 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -87,7 +87,7 @@ series: # 1. '-2+4x3' becomes '-2 2 6 10' - series starts at -2, then 3 further samples incrementing by 4. # 2. ' 1-2x4' becomes '1 -1 -3 -5 -7' - series starts at 1, then 4 further samples decrementing by 2. # 3. ' 1x4' becomes '1 1 1 1 1' - shorthand for '1+0x4', series starts at 1, then 4 further samples incrementing by 0. -# 4. ' 1 _x3 stale' becomes '1 _ _ _ stale' - the missing sample cannot increment, so only 3 samples are produced by the '_x3' expression. +# 4. ' 1 _x3 stale' becomes '1 _ _ _ stale' - the missing sample cannot increment, so 3 missing samples are produced by the '_x3' expression. values: ``` From 7ce09b4e3986b4ae07a3f48c7c337448e86fdff0 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 21 Sep 2022 11:56:44 +0100 Subject: [PATCH 041/107] storage: fix BenchmarkMergeSeriesSet The SeriesSets to be merged must be created each time round the loop, otherwise the benchmark is not doing any real work. Don't call ExpandSeries, because it spends most of its time allocating a memory buffer to hold the result, which we don't look at. Signed-off-by: Bryan Boreham Fix up merge test again --- storage/merge_test.go | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/storage/merge_test.go b/storage/merge_test.go index 726296dd5..a6576da13 100644 --- a/storage/merge_test.go +++ b/storage/merge_test.go @@ -868,9 +868,7 @@ func TestChainSampleIteratorSeek(t *testing.T) { } } -var result []tsdbutil.Sample - -func makeSeriesSet(numSeries, numSamples int) SeriesSet { +func makeSeries(numSeries, numSamples int) []Series { series := []Series{} for j := 0; j < numSeries; j++ { labels := labels.FromStrings("foo", fmt.Sprintf("bar%d", j)) @@ -880,30 +878,38 @@ func makeSeriesSet(numSeries, numSamples int) SeriesSet { } series = append(series, NewListSeries(labels, samples)) } - return NewMockSeriesSet(series...) + return series } -func makeMergeSeriesSet(numSeriesSets, numSeries, numSamples int) SeriesSet { - seriesSets := []genericSeriesSet{} - for i := 0; i < numSeriesSets; i++ { - seriesSets = append(seriesSets, &genericSeriesSetAdapter{makeSeriesSet(numSeries, numSamples)}) +func makeMergeSeriesSet(serieses [][]Series) SeriesSet { + seriesSets := make([]genericSeriesSet, len(serieses)) + for i, s := range serieses { + seriesSets[i] = &genericSeriesSetAdapter{NewMockSeriesSet(s...)} } return &seriesSetAdapter{newGenericMergeSeriesSet(seriesSets, (&seriesMergerAdapter{VerticalSeriesMergeFunc: ChainedSeriesMerge}).Merge)} } -func benchmarkDrain(seriesSet SeriesSet, b *testing.B) { +func benchmarkDrain(b *testing.B, makeSeriesSet func() SeriesSet) { var err error + var t int64 + var v float64 for n := 0; n < b.N; n++ { + seriesSet := makeSeriesSet() for seriesSet.Next() { - result, err = ExpandSamples(seriesSet.At().Iterator(), nil) - require.NoError(b, err) + iter := seriesSet.At().Iterator() + for iter.Next() == chunkenc.ValFloat { + t, v = iter.At() + } + err = iter.Err() } + require.NoError(b, err) + require.NotEqual(b, t, v) // To ensure the inner loop doesn't get optimised away. } } func BenchmarkNoMergeSeriesSet_100_100(b *testing.B) { - seriesSet := makeSeriesSet(100, 100) - benchmarkDrain(seriesSet, b) + series := makeSeries(100, 100) + benchmarkDrain(b, func() SeriesSet { return NewMockSeriesSet(series...) }) } func BenchmarkMergeSeriesSet(b *testing.B) { @@ -914,9 +920,12 @@ func BenchmarkMergeSeriesSet(b *testing.B) { {10, 100, 100}, {100, 100, 100}, } { - seriesSet := makeMergeSeriesSet(bm.numSeriesSets, bm.numSeries, bm.numSamples) + serieses := [][]Series{} + for i := 0; i < bm.numSeriesSets; i++ { + serieses = append(serieses, makeSeries(bm.numSeries, bm.numSamples)) + } b.Run(fmt.Sprintf("%d_%d_%d", bm.numSeriesSets, bm.numSeries, bm.numSamples), func(b *testing.B) { - benchmarkDrain(seriesSet, b) + benchmarkDrain(b, func() SeriesSet { return makeMergeSeriesSet(serieses) }) }) } } From 3c7de690598159b535ab1a4adf73aca44969a378 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 20 Sep 2022 18:16:45 +0100 Subject: [PATCH 042/107] storage: allow re-use of iterators Patterned after `Chunk.Iterator()`: pass the old iterator in so it can be re-used to avoid allocating a new object. (This commit does not do any re-use; it is just changing all the method signatures so re-use is possible in later commits.) Signed-off-by: Bryan Boreham --- cmd/promtool/backfill_test.go | 2 +- cmd/promtool/rules_test.go | 2 +- cmd/promtool/tsdb.go | 2 +- promql/engine.go | 16 ++++++++--- promql/test_test.go | 2 +- promql/value.go | 2 +- rules/manager.go | 2 +- rules/manager_test.go | 3 ++- scrape/scrape_test.go | 4 +-- storage/fanout_test.go | 6 +++-- storage/interface.go | 13 +++++---- storage/merge.go | 14 +++++----- storage/merge_test.go | 19 ++++++------- storage/remote/codec.go | 9 ++++--- storage/remote/codec_test.go | 2 +- storage/series.go | 26 +++++++++--------- tsdb/block_test.go | 8 +++--- tsdb/compact.go | 7 ++--- tsdb/db_test.go | 50 ++++++++++++++++++++++------------- tsdb/example_test.go | 2 +- tsdb/head_test.go | 24 ++++++++--------- tsdb/querier.go | 4 +-- tsdb/querier_test.go | 15 ++++++----- tsdb/tsdbblockutil.go | 3 ++- web/federate.go | 4 ++- 25 files changed, 140 insertions(+), 101 deletions(-) diff --git a/cmd/promtool/backfill_test.go b/cmd/promtool/backfill_test.go index 398f96766..2c551abeb 100644 --- a/cmd/promtool/backfill_test.go +++ b/cmd/promtool/backfill_test.go @@ -49,7 +49,7 @@ func queryAllSeries(t testing.TB, q storage.Querier, expectedMinTime, expectedMa samples := []backfillSample{} for ss.Next() { series := ss.At() - it := series.Iterator() + it := series.Iterator(nil) require.NoError(t, it.Err()) for it.Next() == chunkenc.ValFloat { ts, v := it.At() diff --git a/cmd/promtool/rules_test.go b/cmd/promtool/rules_test.go index 0e60a20fb..caa930616 100644 --- a/cmd/promtool/rules_test.go +++ b/cmd/promtool/rules_test.go @@ -139,7 +139,7 @@ func TestBackfillRuleIntegration(t *testing.T) { } else { require.Equal(t, 3, len(series.Labels())) } - it := series.Iterator() + it := series.Iterator(nil) for it.Next() == chunkenc.ValFloat { samplesCount++ ts, v := it.At() diff --git a/cmd/promtool/tsdb.go b/cmd/promtool/tsdb.go index 6934fad49..91b97f5c5 100644 --- a/cmd/promtool/tsdb.go +++ b/cmd/promtool/tsdb.go @@ -644,7 +644,7 @@ func dumpSamples(path string, mint, maxt int64) (err error) { for ss.Next() { series := ss.At() lbs := series.Labels() - it := series.Iterator() + it := series.Iterator(nil) for it.Next() == chunkenc.ValFloat { ts, val := it.At() fmt.Printf("%s %g %d\n", lbs, val, ts) diff --git a/promql/engine.go b/promql/engine.go index b3ad14b3d..0225f78d2 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -1393,10 +1393,12 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { enh := &EvalNodeHelper{Out: make(Vector, 0, 1)} // Process all the calls for one time series at a time. it := storage.NewBuffer(selRange) + var chkIter chunkenc.Iterator for i, s := range selVS.Series { ev.currentSamples -= len(points) points = points[:0] - it.Reset(s.Iterator()) + chkIter = s.Iterator(chkIter) + it.Reset(chkIter) metric := selVS.Series[i].Labels() // The last_over_time function acts like offset; thus, it // should keep the metric name. For all the other range @@ -1578,8 +1580,10 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { } mat := make(Matrix, 0, len(e.Series)) it := storage.NewMemoizedEmptyIterator(durationMilliseconds(ev.lookbackDelta)) + var chkIter chunkenc.Iterator for i, s := range e.Series { - it.Reset(s.Iterator()) + chkIter = s.Iterator(chkIter) + it.Reset(chkIter) ss := Series{ Metric: e.Series[i].Labels(), Points: getPointSlice(numSteps), @@ -1723,8 +1727,10 @@ func (ev *evaluator) vectorSelector(node *parser.VectorSelector, ts int64) (Vect } vec := make(Vector, 0, len(node.Series)) it := storage.NewMemoizedEmptyIterator(durationMilliseconds(ev.lookbackDelta)) + var chkIter chunkenc.Iterator for i, s := range node.Series { - it.Reset(s.Iterator()) + chkIter = s.Iterator(chkIter) + it.Reset(chkIter) t, v, h, ok := ev.vectorSelectorSingle(it, node, ts) if ok { @@ -1812,12 +1818,14 @@ func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) (Matrix, storag ev.error(errWithWarnings{fmt.Errorf("expanding series: %w", err), ws}) } + var chkIter chunkenc.Iterator series := vs.Series for i, s := range series { if err := contextDone(ev.ctx, "expression evaluation"); err != nil { ev.error(err) } - it.Reset(s.Iterator()) + chkIter = s.Iterator(chkIter) + it.Reset(chkIter) ss := Series{ Metric: series[i].Labels(), } diff --git a/promql/test_test.go b/promql/test_test.go index 5c16e57a2..c5cb41ed9 100644 --- a/promql/test_test.go +++ b/promql/test_test.go @@ -143,7 +143,7 @@ func TestLazyLoader_WithSamplesTill(t *testing.T) { got := Series{ Metric: storageSeries.Labels(), } - it := storageSeries.Iterator() + it := storageSeries.Iterator(nil) for it.Next() == chunkenc.ValFloat { t, v := it.At() got.Points = append(got.Points, Point{T: t, V: v}) diff --git a/promql/value.go b/promql/value.go index 507a5e6f1..78342859e 100644 --- a/promql/value.go +++ b/promql/value.go @@ -363,7 +363,7 @@ func (ss *StorageSeries) Labels() labels.Labels { } // Iterator returns a new iterator of the data of the series. -func (ss *StorageSeries) Iterator() chunkenc.Iterator { +func (ss *StorageSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { return newStorageSeriesIterator(ss.series) } diff --git a/rules/manager.go b/rules/manager.go index 42f1b59ce..4b9c8150a 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -807,7 +807,7 @@ func (g *Group) RestoreForState(ts time.Time) { // Series found for the 'for' state. var t int64 var v float64 - it := s.Iterator() + it := s.Iterator(nil) for it.Next() == chunkenc.ValFloat { t, v = it.At() } diff --git a/rules/manager_test.go b/rules/manager_test.go index 984bb81b9..5c580caf7 100644 --- a/rules/manager_test.go +++ b/rules/manager_test.go @@ -592,12 +592,13 @@ func TestStaleness(t *testing.T) { // Convert a SeriesSet into a form usable with require.Equal. func readSeriesSet(ss storage.SeriesSet) (map[string][]promql.Point, error) { result := map[string][]promql.Point{} + var it chunkenc.Iterator for ss.Next() { series := ss.At() points := []promql.Point{} - it := series.Iterator() + it := series.Iterator(it) for it.Next() == chunkenc.ValFloat { t, v := it.At() points = append(points, promql.Point{T: t, V: v}) diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index b22f7f095..bb851bd9e 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -2959,7 +2959,7 @@ func TestScrapeReportSingleAppender(t *testing.T) { c := 0 for series.Next() { - i := series.At().Iterator() + i := series.At().Iterator(nil) for i.Next() != chunkenc.ValNone { c++ } @@ -3032,7 +3032,7 @@ func TestScrapeReportLimit(t *testing.T) { var found bool for series.Next() { - i := series.At().Iterator() + i := series.At().Iterator(nil) for i.Next() == chunkenc.ValFloat { _, v := i.At() require.Equal(t, 1.0, v) diff --git a/storage/fanout_test.go b/storage/fanout_test.go index ee6623397..4996e8f64 100644 --- a/storage/fanout_test.go +++ b/storage/fanout_test.go @@ -86,11 +86,12 @@ func TestFanout_SelectSorted(t *testing.T) { result := make(map[int64]float64) var labelsResult labels.Labels + var iterator chunkenc.Iterator for seriesSet.Next() { series := seriesSet.At() seriesLabels := series.Labels() labelsResult = seriesLabels - iterator := series.Iterator() + iterator := series.Iterator(iterator) for iterator.Next() == chunkenc.ValFloat { timestamp, value := iterator.At() result[timestamp] = value @@ -112,11 +113,12 @@ func TestFanout_SelectSorted(t *testing.T) { result := make(map[int64]float64) var labelsResult labels.Labels + var iterator chunkenc.Iterator for seriesSet.Next() { series := seriesSet.At() seriesLabels := series.Labels() labelsResult = seriesLabels - iterator := series.Iterator() + iterator := series.Iterator(iterator) for iterator.Next() == chunkenc.ValFloat { timestamp, value := iterator.At() result[timestamp] = value diff --git a/storage/interface.go b/storage/interface.go index 22d3b4186..5f0be9db9 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -382,7 +382,7 @@ func (s mockSeries) Labels() labels.Labels { return labels.FromStrings(s.labelSet...) } -func (s mockSeries) Iterator() chunkenc.Iterator { +func (s mockSeries) Iterator(chunkenc.Iterator) chunkenc.Iterator { return chunkenc.MockSeriesIterator(s.timestamps, s.values) } @@ -421,14 +421,17 @@ type Labels interface { } type SampleIterable interface { - // Iterator returns a new, independent iterator of the data of the series. - Iterator() chunkenc.Iterator + // Iterator returns an iterator of the data of the series. + // The iterator passed as argument is for re-use. + // Depending on implementation, the iterator can + // be re-used or a new iterator can be allocated. + Iterator(chunkenc.Iterator) chunkenc.Iterator } type ChunkIterable interface { - // Iterator returns a new, independent iterator that iterates over potentially overlapping + // Iterator returns an iterator that iterates over potentially overlapping // chunks of the series, sorted by min time. - Iterator() chunks.Iterator + Iterator(chunks.Iterator) chunks.Iterator } type Warnings []error diff --git a/storage/merge.go b/storage/merge.go index 258e4e312..336d82c6f 100644 --- a/storage/merge.go +++ b/storage/merge.go @@ -425,10 +425,10 @@ func ChainedSeriesMerge(series ...Series) Series { } return &SeriesEntry{ Lset: series[0].Labels(), - SampleIteratorFn: func() chunkenc.Iterator { + SampleIteratorFn: func(chunkenc.Iterator) chunkenc.Iterator { iterators := make([]chunkenc.Iterator, 0, len(series)) for _, s := range series { - iterators = append(iterators, s.Iterator()) + iterators = append(iterators, s.Iterator(nil)) } return NewChainSampleIterator(iterators) }, @@ -607,10 +607,10 @@ func NewCompactingChunkSeriesMerger(mergeFunc VerticalSeriesMergeFunc) VerticalC } return &ChunkSeriesEntry{ Lset: series[0].Labels(), - ChunkIteratorFn: func() chunks.Iterator { + ChunkIteratorFn: func(chunks.Iterator) chunks.Iterator { iterators := make([]chunks.Iterator, 0, len(series)) for _, s := range series { - iterators = append(iterators, s.Iterator()) + iterators = append(iterators, s.Iterator(nil)) } return &compactChunkIterator{ mergeFunc: mergeFunc, @@ -693,7 +693,7 @@ func (c *compactChunkIterator) Next() bool { } // Add last as it's not yet included in overlap. We operate on same series, so labels does not matter here. - iter = NewSeriesToChunkEncoder(c.mergeFunc(append(overlapping, newChunkToSeriesDecoder(nil, c.curr))...)).Iterator() + iter = NewSeriesToChunkEncoder(c.mergeFunc(append(overlapping, newChunkToSeriesDecoder(nil, c.curr))...)).Iterator(nil) if !iter.Next() { if c.err = iter.Err(); c.err != nil { return false @@ -751,10 +751,10 @@ func NewConcatenatingChunkSeriesMerger() VerticalChunkSeriesMergeFunc { } return &ChunkSeriesEntry{ Lset: series[0].Labels(), - ChunkIteratorFn: func() chunks.Iterator { + ChunkIteratorFn: func(chunks.Iterator) chunks.Iterator { iterators := make([]chunks.Iterator, 0, len(series)) for _, s := range series { - iterators = append(iterators, s.Iterator()) + iterators = append(iterators, s.Iterator(nil)) } return &concatenatingChunkIterator{ iterators: iterators, diff --git a/storage/merge_test.go b/storage/merge_test.go index a6576da13..407fc4ea5 100644 --- a/storage/merge_test.go +++ b/storage/merge_test.go @@ -202,8 +202,8 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { expectedSeries := tc.expected.At() require.Equal(t, expectedSeries.Labels(), actualSeries.Labels()) - expSmpl, expErr := ExpandSamples(expectedSeries.Iterator(), nil) - actSmpl, actErr := ExpandSamples(actualSeries.Iterator(), nil) + expSmpl, expErr := ExpandSamples(expectedSeries.Iterator(nil), nil) + actSmpl, actErr := ExpandSamples(actualSeries.Iterator(nil), nil) require.Equal(t, expErr, actErr) require.Equal(t, expSmpl, actSmpl) } @@ -370,8 +370,8 @@ func TestMergeChunkQuerierWithNoVerticalChunkSeriesMerger(t *testing.T) { expectedSeries := tc.expected.At() require.Equal(t, expectedSeries.Labels(), actualSeries.Labels()) - expChks, expErr := ExpandChunks(expectedSeries.Iterator()) - actChks, actErr := ExpandChunks(actualSeries.Iterator()) + expChks, expErr := ExpandChunks(expectedSeries.Iterator(nil)) + actChks, actErr := ExpandChunks(actualSeries.Iterator(nil)) require.Equal(t, expErr, actErr) require.Equal(t, expChks, actChks) @@ -533,8 +533,8 @@ func TestCompactingChunkSeriesMerger(t *testing.T) { t.Run(tc.name, func(t *testing.T) { merged := m(tc.input...) require.Equal(t, tc.expected.Labels(), merged.Labels()) - actChks, actErr := ExpandChunks(merged.Iterator()) - expChks, expErr := ExpandChunks(tc.expected.Iterator()) + actChks, actErr := ExpandChunks(merged.Iterator(nil)) + expChks, expErr := ExpandChunks(tc.expected.Iterator(nil)) require.Equal(t, expErr, actErr) require.Equal(t, expChks, actChks) @@ -667,8 +667,8 @@ func TestConcatenatingChunkSeriesMerger(t *testing.T) { t.Run(tc.name, func(t *testing.T) { merged := m(tc.input...) require.Equal(t, tc.expected.Labels(), merged.Labels()) - actChks, actErr := ExpandChunks(merged.Iterator()) - expChks, expErr := ExpandChunks(tc.expected.Iterator()) + actChks, actErr := ExpandChunks(merged.Iterator(nil)) + expChks, expErr := ExpandChunks(tc.expected.Iterator(nil)) require.Equal(t, expErr, actErr) require.Equal(t, expChks, actChks) @@ -893,10 +893,11 @@ func benchmarkDrain(b *testing.B, makeSeriesSet func() SeriesSet) { var err error var t int64 var v float64 + var iter chunkenc.Iterator for n := 0; n < b.N; n++ { seriesSet := makeSeriesSet() for seriesSet.Next() { - iter := seriesSet.At().Iterator() + iter = seriesSet.At().Iterator(iter) for iter.Next() == chunkenc.ValFloat { t, v = iter.At() } diff --git a/storage/remote/codec.go b/storage/remote/codec.go index 48c2d8615..9b7516b87 100644 --- a/storage/remote/codec.go +++ b/storage/remote/codec.go @@ -33,6 +33,7 @@ import ( "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/chunks" ) // decodeReadLimit is the maximum size of a read request body in bytes. @@ -115,9 +116,10 @@ func ToQuery(from, to int64, matchers []*labels.Matcher, hints *storage.SelectHi func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, storage.Warnings, error) { numSamples := 0 resp := &prompb.QueryResult{} + var iter chunkenc.Iterator for ss.Next() { series := ss.At() - iter := series.Iterator() + iter = series.Iterator(iter) samples := []prompb.Sample{} for iter.Next() == chunkenc.ValFloat { @@ -199,11 +201,12 @@ func StreamChunkedReadResponses( var ( chks []prompb.Chunk lbls []prompb.Label + iter chunks.Iterator ) for ss.Next() { series := ss.At() - iter := series.Iterator() + iter = series.Iterator(iter) lbls = MergeLabels(labelsToLabelsProto(series.Labels(), lbls), sortedExternalLabels) frameBytesLeft := maxBytesInFrame @@ -346,7 +349,7 @@ func (c *concreteSeries) Labels() labels.Labels { return labels.New(c.labels...) } -func (c *concreteSeries) Iterator() chunkenc.Iterator { +func (c *concreteSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { return newConcreteSeriersIterator(c) } diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go index c806097c6..596eb0861 100644 --- a/storage/remote/codec_test.go +++ b/storage/remote/codec_test.go @@ -215,7 +215,7 @@ func TestConcreteSeriesIterator(t *testing.T) { {Value: 4, Timestamp: 4}, }, } - it := series.Iterator() + it := series.Iterator(nil) // Seek to the first sample with ts=1. require.Equal(t, chunkenc.ValFloat, it.Seek(1)) diff --git a/storage/series.go b/storage/series.go index 3259dd4d0..87b1256f6 100644 --- a/storage/series.go +++ b/storage/series.go @@ -27,25 +27,25 @@ import ( type SeriesEntry struct { Lset labels.Labels - SampleIteratorFn func() chunkenc.Iterator + SampleIteratorFn func(chunkenc.Iterator) chunkenc.Iterator } -func (s *SeriesEntry) Labels() labels.Labels { return s.Lset } -func (s *SeriesEntry) Iterator() chunkenc.Iterator { return s.SampleIteratorFn() } +func (s *SeriesEntry) Labels() labels.Labels { return s.Lset } +func (s *SeriesEntry) Iterator(it chunkenc.Iterator) chunkenc.Iterator { return s.SampleIteratorFn(it) } type ChunkSeriesEntry struct { Lset labels.Labels - ChunkIteratorFn func() chunks.Iterator + ChunkIteratorFn func(chunks.Iterator) chunks.Iterator } -func (s *ChunkSeriesEntry) Labels() labels.Labels { return s.Lset } -func (s *ChunkSeriesEntry) Iterator() chunks.Iterator { return s.ChunkIteratorFn() } +func (s *ChunkSeriesEntry) Labels() labels.Labels { return s.Lset } +func (s *ChunkSeriesEntry) Iterator(it chunks.Iterator) chunks.Iterator { return s.ChunkIteratorFn(it) } // NewListSeries returns series entry with iterator that allows to iterate over provided samples. func NewListSeries(lset labels.Labels, s []tsdbutil.Sample) *SeriesEntry { return &SeriesEntry{ Lset: lset, - SampleIteratorFn: func() chunkenc.Iterator { + SampleIteratorFn: func(it chunkenc.Iterator) chunkenc.Iterator { return NewListSeriesIterator(samples(s)) }, } @@ -56,7 +56,7 @@ func NewListSeries(lset labels.Labels, s []tsdbutil.Sample) *SeriesEntry { func NewListChunkSeriesFromSamples(lset labels.Labels, samples ...[]tsdbutil.Sample) *ChunkSeriesEntry { return &ChunkSeriesEntry{ Lset: lset, - ChunkIteratorFn: func() chunks.Iterator { + ChunkIteratorFn: func(it chunks.Iterator) chunks.Iterator { chks := make([]chunks.Meta, 0, len(samples)) for _, s := range samples { chks = append(chks, tsdbutil.ChunkFromSamples(s)) @@ -178,7 +178,7 @@ func (c *chunkSetToSeriesSet) Next() bool { return false } - iter := c.ChunkSeriesSet.At().Iterator() + iter := c.ChunkSeriesSet.At().Iterator(nil) c.sameSeriesChunks = c.sameSeriesChunks[:0] for iter.Next() { @@ -210,9 +210,9 @@ func (c *chunkSetToSeriesSet) Err() error { func newChunkToSeriesDecoder(labels labels.Labels, chk chunks.Meta) Series { return &SeriesEntry{ Lset: labels, - SampleIteratorFn: func() chunkenc.Iterator { + SampleIteratorFn: func(it chunkenc.Iterator) chunkenc.Iterator { // TODO(bwplotka): Can we provide any chunkenc buffer? - return chk.Chunk.Iterator(nil) + return chk.Chunk.Iterator(it) }, } } @@ -252,7 +252,7 @@ func NewSeriesToChunkEncoder(series Series) ChunkSeries { return &seriesToChunkEncoder{series} } -func (s *seriesToChunkEncoder) Iterator() chunks.Iterator { +func (s *seriesToChunkEncoder) Iterator(it chunks.Iterator) chunks.Iterator { var ( chk chunkenc.Chunk app chunkenc.Appender @@ -263,7 +263,7 @@ func (s *seriesToChunkEncoder) Iterator() chunks.Iterator { chks := []chunks.Meta{} i := 0 - seriesIter := s.Series.Iterator() + seriesIter := s.Series.Iterator(nil) lastType := chunkenc.ValNone for typ := seriesIter.Next(); typ != chunkenc.ValNone; typ = seriesIter.Next() { if typ != lastType || i >= seriesToChunkEncoderSplit { diff --git a/tsdb/block_test.go b/tsdb/block_test.go index 6cb00b348..c3a6ff576 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -203,7 +203,7 @@ func TestCorruptedChunk(t *testing.T) { // Check chunk errors during iter time. require.True(t, set.Next()) - it := set.At().Iterator() + it := set.At().Iterator(nil) require.Equal(t, chunkenc.ValNone, it.Next()) require.Equal(t, tc.iterErr.Error(), it.Err().Error()) }) @@ -505,11 +505,12 @@ func createHead(tb testing.TB, w *wlog.WL, series []storage.Series, chunkDir str head, err := NewHead(nil, nil, w, nil, opts, nil) require.NoError(tb, err) + var it chunkenc.Iterator ctx := context.Background() app := head.Appender(ctx) for _, s := range series { ref := storage.SeriesRef(0) - it := s.Iterator() + it = s.Iterator(it) lset := s.Labels() typ := it.Next() lastTyp := typ @@ -550,11 +551,12 @@ func createHeadWithOOOSamples(tb testing.TB, w *wlog.WL, series []storage.Series oooSampleLabels := make([]labels.Labels, 0, len(series)) oooSamples := make([]tsdbutil.SampleSlice, 0, len(series)) + var it chunkenc.Iterator totalSamples := 0 app := head.Appender(context.Background()) for _, s := range series { ref := storage.SeriesRef(0) - it := s.Iterator() + it = s.Iterator(it) lset := s.Labels() os := tsdbutil.SampleSlice{} count := 0 diff --git a/tsdb/compact.go b/tsdb/compact.go index 9fe50fda1..f216ad46a 100644 --- a/tsdb/compact.go +++ b/tsdb/compact.go @@ -746,8 +746,9 @@ func (c *LeveledCompactor) populateBlock(blocks []BlockReader, meta *BlockMeta, } var ( - ref = storage.SeriesRef(0) - chks []chunks.Meta + ref = storage.SeriesRef(0) + chks []chunks.Meta + chksIter chunks.Iterator ) set := sets[0] @@ -765,7 +766,7 @@ func (c *LeveledCompactor) populateBlock(blocks []BlockReader, meta *BlockMeta, default: } s := set.At() - chksIter := s.Iterator() + chksIter = s.Iterator(chksIter) chks = chks[:0] for chksIter.Next() { // We are not iterating in streaming way over chunk as diff --git a/tsdb/db_test.go b/tsdb/db_test.go index d4c2840c2..cea4b6e36 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -93,12 +93,13 @@ func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[str require.NoError(t, q.Close()) }() + var it chunkenc.Iterator result := map[string][]tsdbutil.Sample{} for ss.Next() { series := ss.At() samples := []tsdbutil.Sample{} - it := series.Iterator() + it = series.Iterator(it) for typ := it.Next(); typ != chunkenc.ValNone; typ = it.Next() { switch typ { case chunkenc.ValFloat: @@ -133,12 +134,13 @@ func queryChunks(t testing.TB, q storage.ChunkQuerier, matchers ...*labels.Match require.NoError(t, q.Close()) }() + var it chunks.Iterator result := map[string][]chunks.Meta{} for ss.Next() { series := ss.At() chks := []chunks.Meta{} - it := series.Iterator() + it = series.Iterator(it) for it.Next() { chks = append(chks, it.At()) } @@ -454,8 +456,8 @@ Outer: require.Equal(t, sexp.Labels(), sres.Labels()) - smplExp, errExp := storage.ExpandSamples(sexp.Iterator(), nil) - smplRes, errRes := storage.ExpandSamples(sres.Iterator(), nil) + smplExp, errExp := storage.ExpandSamples(sexp.Iterator(nil), nil) + smplRes, errRes := storage.ExpandSamples(sres.Iterator(nil), nil) require.Equal(t, errExp, errRes) require.Equal(t, smplExp, smplRes) @@ -628,9 +630,10 @@ func TestDB_Snapshot(t *testing.T) { // sum values seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + var series chunkenc.Iterator sum := 0.0 for seriesSet.Next() { - series := seriesSet.At().Iterator() + series = seriesSet.At().Iterator(series) for series.Next() == chunkenc.ValFloat { _, v := series.At() sum += v @@ -676,9 +679,10 @@ func TestDB_Snapshot_ChunksOutsideOfCompactedRange(t *testing.T) { // Sum values. seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + var series chunkenc.Iterator sum := 0.0 for seriesSet.Next() { - series := seriesSet.At().Iterator() + series = seriesSet.At().Iterator(series) for series.Next() == chunkenc.ValFloat { _, v := series.At() sum += v @@ -770,8 +774,8 @@ Outer: require.Equal(t, sexp.Labels(), sres.Labels()) - smplExp, errExp := storage.ExpandSamples(sexp.Iterator(), nil) - smplRes, errRes := storage.ExpandSamples(sres.Iterator(), nil) + smplExp, errExp := storage.ExpandSamples(sexp.Iterator(nil), nil) + smplRes, errRes := storage.ExpandSamples(sres.Iterator(nil), nil) require.Equal(t, errExp, errRes) require.Equal(t, smplExp, smplRes) @@ -921,7 +925,7 @@ func TestDB_e2e(t *testing.T) { for ss.Next() { x := ss.At() - smpls, err := storage.ExpandSamples(x.Iterator(), newSample) + smpls, err := storage.ExpandSamples(x.Iterator(nil), newSample) require.NoError(t, err) if len(smpls) > 0 { @@ -1108,12 +1112,13 @@ func testWALReplayRaceOnSamplesLoggedBeforeSeries(t *testing.T, numSamplesBefore set := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "series_id", ".+")) actualSeries := 0 + var chunksIt chunks.Iterator for set.Next() { actualSeries++ actualChunks := 0 - chunksIt := set.At().Iterator() + chunksIt = set.At().Iterator(chunksIt) for chunksIt.Next() { actualChunks++ } @@ -1205,8 +1210,8 @@ func TestTombstoneClean(t *testing.T) { require.Equal(t, sexp.Labels(), sres.Labels()) - smplExp, errExp := storage.ExpandSamples(sexp.Iterator(), nil) - smplRes, errRes := storage.ExpandSamples(sres.Iterator(), nil) + smplExp, errExp := storage.ExpandSamples(sexp.Iterator(nil), nil) + smplRes, errRes := storage.ExpandSamples(sres.Iterator(nil), nil) require.Equal(t, errExp, errRes) require.Equal(t, smplExp, smplRes) @@ -1479,11 +1484,12 @@ func TestSizeRetention(t *testing.T) { // Add some data to the WAL. headApp := db.Head().Appender(context.Background()) var aSeries labels.Labels + var it chunkenc.Iterator for _, m := range headBlocks { series := genSeries(100, 10, m.MinTime, m.MaxTime+1) for _, s := range series { aSeries = s.Labels() - it := s.Iterator() + it = s.Iterator(it) for it.Next() == chunkenc.ValFloat { tim, v := it.At() _, err := headApp.Append(0, s.Labels(), tim, v) @@ -1691,10 +1697,11 @@ func TestNotMatcherSelectsLabelsUnsetSeries(t *testing.T) { func expandSeriesSet(ss storage.SeriesSet) ([]labels.Labels, map[string][]sample, storage.Warnings, error) { resultLabels := []labels.Labels{} resultSamples := map[string][]sample{} + var it chunkenc.Iterator for ss.Next() { series := ss.At() samples := []sample{} - it := series.Iterator() + it = series.Iterator(it) for it.Next() == chunkenc.ValFloat { t, v := it.At() samples = append(samples, sample{t: t, v: v}) @@ -2500,10 +2507,11 @@ func TestDBReadOnly_FlushWAL(t *testing.T) { // Sum the values. seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, defaultLabelName, "flush")) + var series chunkenc.Iterator sum := 0.0 for seriesSet.Next() { - series := seriesSet.At().Iterator() + series = seriesSet.At().Iterator(series) for series.Next() == chunkenc.ValFloat { _, v := series.At() sum += v @@ -2946,10 +2954,11 @@ func TestCompactHead(t *testing.T) { defer func() { require.NoError(t, querier.Close()) }() seriesSet := querier.Select(false, nil, &labels.Matcher{Type: labels.MatchEqual, Name: "a", Value: "b"}) + var series chunkenc.Iterator var actSamples []sample for seriesSet.Next() { - series := seriesSet.At().Iterator() + series = seriesSet.At().Iterator(series) for series.Next() == chunkenc.ValFloat { time, val := series.At() actSamples = append(actSamples, sample{int64(time), val, nil, nil}) @@ -3347,7 +3356,7 @@ func testQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChunks(t actualSeries++ // Get the iterator and call Next() so that we're sure the chunk is loaded. - it := seriesSet.At().Iterator() + it := seriesSet.At().Iterator(nil) it.Next() it.At() @@ -3477,11 +3486,13 @@ func testChunkQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChun seriesSet := querier.Select(true, hints, labels.MustNewMatcher(labels.MatchRegexp, labels.MetricName, ".+")) // Iterate all series and get their chunks. + var it chunks.Iterator var chunks []chunkenc.Chunk actualSeries := 0 for seriesSet.Next() { actualSeries++ - for it := seriesSet.At().Iterator(); it.Next(); { + it = seriesSet.At().Iterator(it) + for it.Next() { chunks = append(chunks, it.At().Chunk) } } @@ -6025,13 +6036,14 @@ func TestQueryHistogramFromBlocksWithCompaction(t *testing.T) { ctx := context.Background() + var it chunkenc.Iterator exp := make(map[string][]tsdbutil.Sample) for _, series := range blockSeries { createBlock(t, db.Dir(), series) for _, s := range series { key := s.Labels().String() - it := s.Iterator() + it = s.Iterator(it) slice := exp[key] for typ := it.Next(); typ != chunkenc.ValNone; typ = it.Next() { switch typ { diff --git a/tsdb/example_test.go b/tsdb/example_test.go index c33bf6dc0..da0e37923 100644 --- a/tsdb/example_test.go +++ b/tsdb/example_test.go @@ -67,7 +67,7 @@ func Example() { series := ss.At() fmt.Println("series:", series.Labels().String()) - it := series.Iterator() + it := series.Iterator(nil) for it.Next() == chunkenc.ValFloat { _, v := it.At() // We ignore the timestamp here, only to have a predictable output we can test against (below) fmt.Println("sample", v) diff --git a/tsdb/head_test.go b/tsdb/head_test.go index 9b8eb0278..59824ae08 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -924,8 +924,8 @@ func TestHeadDeleteSimple(t *testing.T) { require.Equal(t, expSeries.Labels(), actSeries.Labels()) - smplExp, errExp := storage.ExpandSamples(expSeries.Iterator(), nil) - smplRes, errRes := storage.ExpandSamples(actSeries.Iterator(), nil) + smplExp, errExp := storage.ExpandSamples(expSeries.Iterator(nil), nil) + smplRes, errRes := storage.ExpandSamples(actSeries.Iterator(nil), nil) require.Equal(t, errExp, errRes) require.Equal(t, smplExp, smplRes) @@ -959,7 +959,7 @@ func TestDeleteUntilCurMax(t *testing.T) { res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) require.True(t, res.Next(), "series is not present") s := res.At() - it := s.Iterator() + it := s.Iterator(nil) require.Equal(t, chunkenc.ValNone, it.Next(), "expected no samples") for res.Next() { } @@ -976,7 +976,7 @@ func TestDeleteUntilCurMax(t *testing.T) { res = q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) require.True(t, res.Next(), "series don't exist") exps := res.At() - it = exps.Iterator() + it = exps.Iterator(nil) resSamples, err := storage.ExpandSamples(it, newSample) require.NoError(t, err) require.Equal(t, []tsdbutil.Sample{sample{11, 1, nil, nil}}, resSamples) @@ -1163,7 +1163,7 @@ func TestDelete_e2e(t *testing.T) { eok, rok := expSs.Next(), ss.Next() // Skip a series if iterator is empty. if rok { - for ss.At().Iterator().Next() == chunkenc.ValNone { + for ss.At().Iterator(nil).Next() == chunkenc.ValNone { rok = ss.Next() if !rok { break @@ -1177,8 +1177,8 @@ func TestDelete_e2e(t *testing.T) { sexp := expSs.At() sres := ss.At() require.Equal(t, sexp.Labels(), sres.Labels()) - smplExp, errExp := storage.ExpandSamples(sexp.Iterator(), nil) - smplRes, errRes := storage.ExpandSamples(sres.Iterator(), nil) + smplExp, errExp := storage.ExpandSamples(sexp.Iterator(nil), nil) + smplRes, errRes := storage.ExpandSamples(sres.Iterator(nil), nil) require.Equal(t, errExp, errRes) require.Equal(t, smplExp, smplRes) } @@ -2635,7 +2635,7 @@ func TestChunkNotFoundHeadGCRace(t *testing.T) { <-time.After(3 * time.Second) // Now consume after compaction when it's gone. - it := s.Iterator() + it := s.Iterator(nil) for it.Next() == chunkenc.ValFloat { _, _ = it.At() } @@ -2643,7 +2643,7 @@ func TestChunkNotFoundHeadGCRace(t *testing.T) { require.NoError(t, it.Err()) for ss.Next() { s = ss.At() - it := s.Iterator() + it = s.Iterator(it) for it.Next() == chunkenc.ValFloat { _, _ = it.At() } @@ -2841,7 +2841,7 @@ func TestAppendHistogram(t *testing.T) { s := ss.At() require.False(t, ss.Next()) - it := s.Iterator() + it := s.Iterator(nil) actHistograms := make([]timedHistogram, 0, len(expHistograms)) for it.Next() == chunkenc.ValHistogram { t, h := it.AtHistogram() @@ -3304,7 +3304,7 @@ func TestHistogramStaleSample(t *testing.T) { s := ss.At() require.False(t, ss.Next()) - it := s.Iterator() + it := s.Iterator(nil) actHistograms := make([]timedHistogram, 0, len(expHistograms)) for it.Next() == chunkenc.ValHistogram { t, h := it.AtHistogram() @@ -3581,7 +3581,7 @@ func TestAppendingDifferentEncodingToSameSeries(t *testing.T) { ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) require.True(t, ss.Next()) s := ss.At() - it := s.Iterator() + it := s.Iterator(nil) expIdx := 0 loop: for { diff --git a/tsdb/querier.go b/tsdb/querier.go index cc765903c..3ae1c4f1e 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -838,7 +838,7 @@ func (b *blockSeriesSet) At() storage.Series { currIterFn := b.currIterFn return &storage.SeriesEntry{ Lset: b.currLabels, - SampleIteratorFn: func() chunkenc.Iterator { + SampleIteratorFn: func(chunkenc.Iterator) chunkenc.Iterator { return currIterFn().toSeriesIterator() }, } @@ -872,7 +872,7 @@ func (b *blockChunkSeriesSet) At() storage.ChunkSeries { currIterFn := b.currIterFn return &storage.ChunkSeriesEntry{ Lset: b.currLabels, - ChunkIteratorFn: func() chunks.Iterator { + ChunkIteratorFn: func(chunks.Iterator) chunks.Iterator { return currIterFn().toChunkSeriesIterator() }, } diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index ffb24b17b..20e4c2f8f 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -194,8 +194,8 @@ func testBlockQuerier(t *testing.T, c blockQuerierTestCase, ir IndexReader, cr C sres := res.At() require.Equal(t, sexp.Labels(), sres.Labels()) - smplExp, errExp := storage.ExpandSamples(sexp.Iterator(), nil) - smplRes, errRes := storage.ExpandSamples(sres.Iterator(), nil) + smplExp, errExp := storage.ExpandSamples(sexp.Iterator(nil), nil) + smplRes, errRes := storage.ExpandSamples(sres.Iterator(nil), nil) require.Equal(t, errExp, errRes) require.Equal(t, smplExp, smplRes) @@ -230,9 +230,9 @@ func testBlockQuerier(t *testing.T, c blockQuerierTestCase, ir IndexReader, cr C require.Equal(t, sexpChks.Labels(), sres.Labels()) - chksExp, errExp := storage.ExpandChunks(sexpChks.Iterator()) + chksExp, errExp := storage.ExpandChunks(sexpChks.Iterator(nil)) rmChunkRefs(chksExp) - chksRes, errRes := storage.ExpandChunks(sres.Iterator()) + chksRes, errRes := storage.ExpandChunks(sres.Iterator(nil)) rmChunkRefs(chksRes) require.Equal(t, errExp, errRes) require.Equal(t, chksExp, chksRes) @@ -1433,9 +1433,10 @@ func BenchmarkQuerySeek(b *testing.B) { b.ResetTimer() b.ReportAllocs() + var it chunkenc.Iterator ss := sq.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*")) for ss.Next() { - it := ss.At().Iterator() + it = ss.At().Iterator(it) for t := mint; t <= maxt; t++ { it.Seek(t) } @@ -2042,11 +2043,13 @@ func benchQuery(b *testing.B, expExpansions int, q storage.Querier, selectors la for i := 0; i < b.N; i++ { ss := q.Select(false, nil, selectors...) var actualExpansions int + var it chunkenc.Iterator for ss.Next() { s := ss.At() s.Labels() - it := s.Iterator() + it = s.Iterator(it) for it.Next() != chunkenc.ValNone { + _, _ = it.At() } actualExpansions++ } diff --git a/tsdb/tsdbblockutil.go b/tsdb/tsdbblockutil.go index 777db5e90..8117f431c 100644 --- a/tsdb/tsdbblockutil.go +++ b/tsdb/tsdbblockutil.go @@ -49,10 +49,11 @@ func CreateBlock(series []storage.Series, dir string, chunkRange int64, logger l const commitAfter = 10000 ctx := context.Background() app := w.Appender(ctx) + var it chunkenc.Iterator for _, s := range series { ref := storage.SeriesRef(0) - it := s.Iterator() + it = s.Iterator(it) lset := s.Labels() typ := it.Next() lastTyp := typ diff --git a/web/federate.go b/web/federate.go index 5ba68fa28..baa3b5866 100644 --- a/web/federate.go +++ b/web/federate.go @@ -102,12 +102,14 @@ func (h *Handler) federation(w http.ResponseWriter, req *http.Request) { set := storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge) it := storage.NewBuffer(int64(h.lookbackDelta / 1e6)) + var chkIter chunkenc.Iterator for set.Next() { s := set.At() // TODO(fabxc): allow fast path for most recent sample either // in the storage itself or caching layer in Prometheus. - it.Reset(s.Iterator()) + chkIter = s.Iterator(chkIter) + it.Reset(chkIter) var t int64 var v float64 From f0866c0774a926a57868b8e38493131993c7f977 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 20 Sep 2022 19:27:44 +0100 Subject: [PATCH 043/107] tsdb: optimise block series iterators Re-use previous memory if it is already of the correct type. Also turn two levels of function closure into a single object that holds the required data. Signed-off-by: Bryan Boreham --- tsdb/querier.go | 123 +++++++++++++++++++++++++++---------------- tsdb/querier_test.go | 27 ++++++---- 2 files changed, 94 insertions(+), 56 deletions(-) diff --git a/tsdb/querier.go b/tsdb/querier.go index 3ae1c4f1e..642e089aa 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -426,6 +426,16 @@ func labelNamesWithMatchers(r IndexReader, matchers ...*labels.Matcher) ([]strin return r.LabelNamesFor(postings...) } +// These are the things fetched when we move from one series to another. +type seriesData struct { + chks []chunks.Meta + intervals tombstones.Intervals + labels labels.Labels +} + +// Labels implements part of storage.Series and storage.ChunkSeries. +func (s *seriesData) Labels() labels.Labels { return s.labels } + // blockBaseSeriesSet allows to iterate over all series in the single block. // Iterated series are trimmed with given min and max time as well as tombstones. // See newBlockSeriesSet and newBlockChunkSeriesSet to use it for either sample or chunk iterating. @@ -438,8 +448,7 @@ type blockBaseSeriesSet struct { mint, maxt int64 disableTrimming bool - currIterFn func() *populateWithDelGenericSeriesIterator - currLabels labels.Labels + curr seriesData bufChks []chunks.Meta bufLbls labels.Labels @@ -519,12 +528,11 @@ func (b *blockBaseSeriesSet) Next() bool { intervals = intervals.Add(tombstones.Interval{Mint: b.maxt + 1, Maxt: math.MaxInt64}) } - b.currLabels = make(labels.Labels, len(b.bufLbls)) - copy(b.currLabels, b.bufLbls) + b.curr.labels = make(labels.Labels, len(b.bufLbls)) + copy(b.curr.labels, b.bufLbls) + b.curr.chks = chks + b.curr.intervals = intervals - b.currIterFn = func() *populateWithDelGenericSeriesIterator { - return newPopulateWithDelGenericSeriesIterator(b.blockID, b.chunks, chks, intervals) - } return true } return false @@ -556,29 +564,26 @@ type populateWithDelGenericSeriesIterator struct { // the same, single series. chks []chunks.Meta - i int + i int // Index into chks; -1 if not started yet. err error - bufIter *DeletedIterator + bufIter DeletedIterator // Retained for memory re-use. currDelIter may point here. intervals tombstones.Intervals currDelIter chunkenc.Iterator currChkMeta chunks.Meta } -func newPopulateWithDelGenericSeriesIterator( - blockID ulid.ULID, - chunks ChunkReader, - chks []chunks.Meta, - intervals tombstones.Intervals, -) *populateWithDelGenericSeriesIterator { - return &populateWithDelGenericSeriesIterator{ - blockID: blockID, - chunks: chunks, - chks: chks, - i: -1, - bufIter: &DeletedIterator{}, - intervals: intervals, - } +func (p *populateWithDelGenericSeriesIterator) reset(blockID ulid.ULID, cr ChunkReader, chks []chunks.Meta, intervals tombstones.Intervals) { + p.blockID = blockID + p.chunks = cr + p.chks = chks + p.i = -1 + p.err = nil + p.bufIter.Iter = nil + p.bufIter.Intervals = p.bufIter.Intervals[:0] + p.intervals = intervals + p.currDelIter = nil + p.currChkMeta = chunks.Meta{} } func (p *populateWithDelGenericSeriesIterator) next() bool { @@ -618,28 +623,55 @@ func (p *populateWithDelGenericSeriesIterator) next() bool { // We don't want the full chunk, or it's potentially still opened, take // just a part of it. - p.bufIter.Iter = p.currChkMeta.Chunk.Iterator(nil) - p.currDelIter = p.bufIter + p.bufIter.Iter = p.currChkMeta.Chunk.Iterator(p.bufIter.Iter) + p.currDelIter = &p.bufIter return true } func (p *populateWithDelGenericSeriesIterator) Err() error { return p.err } -func (p *populateWithDelGenericSeriesIterator) toSeriesIterator() chunkenc.Iterator { - return &populateWithDelSeriesIterator{populateWithDelGenericSeriesIterator: p} +type blockSeriesEntry struct { + chunks ChunkReader + blockID ulid.ULID + seriesData } -func (p *populateWithDelGenericSeriesIterator) toChunkSeriesIterator() chunks.Iterator { - return &populateWithDelChunkSeriesIterator{populateWithDelGenericSeriesIterator: p} +func (s *blockSeriesEntry) Iterator(it chunkenc.Iterator) chunkenc.Iterator { + pi, ok := it.(*populateWithDelSeriesIterator) + if !ok { + pi = &populateWithDelSeriesIterator{} + } + pi.reset(s.blockID, s.chunks, s.chks, s.intervals) + return pi +} + +type chunkSeriesEntry struct { + chunks ChunkReader + blockID ulid.ULID + seriesData +} + +func (s *chunkSeriesEntry) Iterator(it chunks.Iterator) chunks.Iterator { + pi, ok := it.(*populateWithDelChunkSeriesIterator) + if !ok { + pi = &populateWithDelChunkSeriesIterator{} + } + pi.reset(s.blockID, s.chunks, s.chks, s.intervals) + return pi } // populateWithDelSeriesIterator allows to iterate over samples for the single series. type populateWithDelSeriesIterator struct { - *populateWithDelGenericSeriesIterator + populateWithDelGenericSeriesIterator curr chunkenc.Iterator } +func (p *populateWithDelSeriesIterator) reset(blockID ulid.ULID, cr ChunkReader, chks []chunks.Meta, intervals tombstones.Intervals) { + p.populateWithDelGenericSeriesIterator.reset(blockID, cr, chks, intervals) + p.curr = nil +} + func (p *populateWithDelSeriesIterator) Next() chunkenc.ValueType { if p.curr != nil { if valueType := p.curr.Next(); valueType != chunkenc.ValNone { @@ -701,11 +733,16 @@ func (p *populateWithDelSeriesIterator) Err() error { } type populateWithDelChunkSeriesIterator struct { - *populateWithDelGenericSeriesIterator + populateWithDelGenericSeriesIterator curr chunks.Meta } +func (p *populateWithDelChunkSeriesIterator) reset(blockID ulid.ULID, cr ChunkReader, chks []chunks.Meta, intervals tombstones.Intervals) { + p.populateWithDelGenericSeriesIterator.reset(blockID, cr, chks, intervals) + p.curr = chunks.Meta{} +} + func (p *populateWithDelChunkSeriesIterator) Next() bool { if !p.next() { return false @@ -834,13 +871,11 @@ func newBlockSeriesSet(i IndexReader, c ChunkReader, t tombstones.Reader, p inde } func (b *blockSeriesSet) At() storage.Series { - // At can be looped over before iterating, so save the current value locally. - currIterFn := b.currIterFn - return &storage.SeriesEntry{ - Lset: b.currLabels, - SampleIteratorFn: func(chunkenc.Iterator) chunkenc.Iterator { - return currIterFn().toSeriesIterator() - }, + // At can be looped over before iterating, so save the current values locally. + return &blockSeriesEntry{ + chunks: b.chunks, + blockID: b.blockID, + seriesData: b.curr, } } @@ -868,13 +903,11 @@ func newBlockChunkSeriesSet(id ulid.ULID, i IndexReader, c ChunkReader, t tombst } func (b *blockChunkSeriesSet) At() storage.ChunkSeries { - // At can be looped over before iterating, so save the current value locally. - currIterFn := b.currIterFn - return &storage.ChunkSeriesEntry{ - Lset: b.currLabels, - ChunkIteratorFn: func(chunks.Iterator) chunks.Iterator { - return currIterFn().toChunkSeriesIterator() - }, + // At can be looped over before iterating, so save the current values locally. + return &chunkSeriesEntry{ + chunks: b.chunks, + blockID: b.blockID, + seriesData: b.curr, } } diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 20e4c2f8f..3b44cef51 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -859,7 +859,8 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Run("sample", func(t *testing.T) { f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) - it := newPopulateWithDelGenericSeriesIterator(ulid.ULID{}, f, chkMetas, tc.intervals).toSeriesIterator() + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tc.intervals) var r []tsdbutil.Sample if tc.seek != 0 { @@ -879,7 +880,8 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }) t.Run("chunk", func(t *testing.T) { f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) - it := newPopulateWithDelGenericSeriesIterator(ulid.ULID{}, f, chkMetas, tc.intervals).toChunkSeriesIterator() + it := &populateWithDelChunkSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tc.intervals) if tc.seek != 0 { // Chunk iterator does not have Seek method. @@ -911,7 +913,8 @@ func TestPopulateWithDelSeriesIterator_DoubleSeek(t *testing.T) { []tsdbutil.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, ) - it := newPopulateWithDelGenericSeriesIterator(ulid.ULID{}, f, chkMetas, nil).toSeriesIterator() + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) require.Equal(t, chunkenc.ValFloat, it.Seek(1)) require.Equal(t, chunkenc.ValFloat, it.Seek(2)) require.Equal(t, chunkenc.ValFloat, it.Seek(2)) @@ -929,7 +932,8 @@ func TestPopulateWithDelSeriesIterator_SeekInCurrentChunk(t *testing.T) { []tsdbutil.Sample{}, ) - it := newPopulateWithDelGenericSeriesIterator(ulid.ULID{}, f, chkMetas, nil).toSeriesIterator() + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) require.Equal(t, chunkenc.ValFloat, it.Next()) ts, v := it.At() require.Equal(t, int64(1), ts) @@ -946,7 +950,8 @@ func TestPopulateWithDelSeriesIterator_SeekWithMinTime(t *testing.T) { []tsdbutil.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, ) - it := newPopulateWithDelGenericSeriesIterator(ulid.ULID{}, f, chkMetas, nil).toSeriesIterator() + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) require.Equal(t, chunkenc.ValNone, it.Seek(7)) require.Equal(t, chunkenc.ValFloat, it.Seek(3)) } @@ -958,9 +963,8 @@ func TestPopulateWithDelSeriesIterator_NextWithMinTime(t *testing.T) { []tsdbutil.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, ) - it := newPopulateWithDelGenericSeriesIterator( - ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64}), - ).toSeriesIterator() + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) require.Equal(t, chunkenc.ValNone, it.Next()) } @@ -2225,11 +2229,12 @@ func TestBlockBaseSeriesSet(t *testing.T) { i := 0 for bcs.Next() { - chks := bcs.currIterFn().chks + si := populateWithDelGenericSeriesIterator{} + si.reset(bcs.blockID, bcs.chunks, bcs.curr.chks, bcs.curr.intervals) idx := tc.expIdxs[i] - require.Equal(t, tc.series[idx].lset, bcs.currLabels) - require.Equal(t, tc.series[idx].chunks, chks) + require.Equal(t, tc.series[idx].lset, bcs.curr.labels) + require.Equal(t, tc.series[idx].chunks, si.chks) i++ } From 463f5cafdd243183c12ddb787d5bd8ef5fb42f5d Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 20 Sep 2022 19:31:28 +0100 Subject: [PATCH 044/107] storage: re-use iterators to save garbage Re-use previous memory if it is already of the correct type. In `NewListSeries` we hoist the conversion to an interface value out so it only allocates once. Signed-off-by: Bryan Boreham --- promql/value.go | 9 ++++++++ storage/merge.go | 51 ++++++++++++++++++++++++++++++----------- storage/merge_test.go | 4 ++-- storage/remote/codec.go | 9 ++++++++ storage/series.go | 49 +++++++++++++++++++++++++++++++++------ tsdb/head_read.go | 6 +---- 6 files changed, 100 insertions(+), 28 deletions(-) diff --git a/promql/value.go b/promql/value.go index 78342859e..4db976e97 100644 --- a/promql/value.go +++ b/promql/value.go @@ -364,6 +364,10 @@ func (ss *StorageSeries) Labels() labels.Labels { // Iterator returns a new iterator of the data of the series. func (ss *StorageSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { + if ssi, ok := it.(*storageSeriesIterator); ok { + ssi.reset(ss.series) + return ssi + } return newStorageSeriesIterator(ss.series) } @@ -379,6 +383,11 @@ func newStorageSeriesIterator(series Series) *storageSeriesIterator { } } +func (ssi *storageSeriesIterator) reset(series Series) { + ssi.points = series.Points + ssi.curr = -1 +} + func (ssi *storageSeriesIterator) Seek(t int64) chunkenc.ValueType { i := ssi.curr if i < 0 { diff --git a/storage/merge.go b/storage/merge.go index 336d82c6f..78a0125db 100644 --- a/storage/merge.go +++ b/storage/merge.go @@ -425,12 +425,8 @@ func ChainedSeriesMerge(series ...Series) Series { } return &SeriesEntry{ Lset: series[0].Labels(), - SampleIteratorFn: func(chunkenc.Iterator) chunkenc.Iterator { - iterators := make([]chunkenc.Iterator, 0, len(series)) - for _, s := range series { - iterators = append(iterators, s.Iterator(nil)) - } - return NewChainSampleIterator(iterators) + SampleIteratorFn: func(it chunkenc.Iterator) chunkenc.Iterator { + return ChainSampleIteratorFromSeries(it, series) }, } } @@ -446,15 +442,42 @@ type chainSampleIterator struct { lastT int64 } -// NewChainSampleIterator returns a single iterator that iterates over the samples from the given iterators in a sorted -// fashion. If samples overlap, one sample from overlapped ones is kept (randomly) and all others with the same -// timestamp are dropped. -func NewChainSampleIterator(iterators []chunkenc.Iterator) chunkenc.Iterator { - return &chainSampleIterator{ - iterators: iterators, - h: nil, - lastT: math.MinInt64, +// Return a chainSampleIterator initialized for length entries, re-using the memory from it if possible. +func getChainSampleIterator(it chunkenc.Iterator, length int) *chainSampleIterator { + csi, ok := it.(*chainSampleIterator) + if !ok { + csi = &chainSampleIterator{} } + if cap(csi.iterators) < length { + csi.iterators = make([]chunkenc.Iterator, length) + } else { + csi.iterators = csi.iterators[:length] + } + csi.h = nil + csi.lastT = math.MinInt64 + return csi +} + +func ChainSampleIteratorFromSeries(it chunkenc.Iterator, series []Series) chunkenc.Iterator { + csi := getChainSampleIterator(it, len(series)) + for i, s := range series { + csi.iterators[i] = s.Iterator(csi.iterators[i]) + } + return csi +} + +func ChainSampleIteratorFromMetas(it chunkenc.Iterator, chunks []chunks.Meta) chunkenc.Iterator { + csi := getChainSampleIterator(it, len(chunks)) + for i, c := range chunks { + csi.iterators[i] = c.Chunk.Iterator(csi.iterators[i]) + } + return csi +} + +func ChainSampleIteratorFromIterators(it chunkenc.Iterator, iterators []chunkenc.Iterator) chunkenc.Iterator { + csi := getChainSampleIterator(it, 0) + csi.iterators = iterators + return csi } func (c *chainSampleIterator) Seek(t int64) chunkenc.ValueType { diff --git a/storage/merge_test.go b/storage/merge_test.go index 407fc4ea5..ad68684c0 100644 --- a/storage/merge_test.go +++ b/storage/merge_test.go @@ -809,7 +809,7 @@ func TestChainSampleIterator(t *testing.T) { expected: []tsdbutil.Sample{sample{0, 0, nil, nil}, sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, }, } { - merged := NewChainSampleIterator(tc.input) + merged := ChainSampleIteratorFromIterators(nil, tc.input) actual, err := ExpandSamples(merged, nil) require.NoError(t, err) require.Equal(t, tc.expected, actual) @@ -855,7 +855,7 @@ func TestChainSampleIteratorSeek(t *testing.T) { expected: []tsdbutil.Sample{sample{0, 0, nil, nil}, sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, }, } { - merged := NewChainSampleIterator(tc.input) + merged := ChainSampleIteratorFromIterators(nil, tc.input) actual := []tsdbutil.Sample{} if merged.Seek(tc.seek) == chunkenc.ValFloat { t, v := merged.At() diff --git a/storage/remote/codec.go b/storage/remote/codec.go index 9b7516b87..a74ad2b7b 100644 --- a/storage/remote/codec.go +++ b/storage/remote/codec.go @@ -350,6 +350,10 @@ func (c *concreteSeries) Labels() labels.Labels { } func (c *concreteSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { + if csi, ok := it.(*concreteSeriesIterator); ok { + csi.reset(c) + return csi + } return newConcreteSeriersIterator(c) } @@ -366,6 +370,11 @@ func newConcreteSeriersIterator(series *concreteSeries) chunkenc.Iterator { } } +func (c *concreteSeriesIterator) reset(series *concreteSeries) { + c.cur = -1 + c.series = series +} + // Seek implements storage.SeriesIterator. func (c *concreteSeriesIterator) Seek(t int64) chunkenc.ValueType { if c.cur == -1 { diff --git a/storage/series.go b/storage/series.go index 87b1256f6..339beb2c9 100644 --- a/storage/series.go +++ b/storage/series.go @@ -43,10 +43,15 @@ func (s *ChunkSeriesEntry) Iterator(it chunks.Iterator) chunks.Iterator { return // NewListSeries returns series entry with iterator that allows to iterate over provided samples. func NewListSeries(lset labels.Labels, s []tsdbutil.Sample) *SeriesEntry { + samplesS := Samples(samples(s)) return &SeriesEntry{ Lset: lset, SampleIteratorFn: func(it chunkenc.Iterator) chunkenc.Iterator { - return NewListSeriesIterator(samples(s)) + if lsi, ok := it.(*listSeriesIterator); ok { + lsi.Reset(samplesS) + return lsi + } + return NewListSeriesIterator(samplesS) }, } } @@ -57,10 +62,20 @@ func NewListChunkSeriesFromSamples(lset labels.Labels, samples ...[]tsdbutil.Sam return &ChunkSeriesEntry{ Lset: lset, ChunkIteratorFn: func(it chunks.Iterator) chunks.Iterator { - chks := make([]chunks.Meta, 0, len(samples)) + lcsi, existing := it.(*listChunkSeriesIterator) + var chks []chunks.Meta + if existing { + chks = lcsi.chks[:0] + } else { + chks = make([]chunks.Meta, 0, len(samples)) + } for _, s := range samples { chks = append(chks, tsdbutil.ChunkFromSamples(s)) } + if existing { + lcsi.Reset(chks...) + return lcsi + } return NewListChunkSeriesIterator(chks...) }, } @@ -87,6 +102,11 @@ func NewListSeriesIterator(samples Samples) chunkenc.Iterator { return &listSeriesIterator{samples: samples, idx: -1} } +func (it *listSeriesIterator) Reset(samples Samples) { + it.samples = samples + it.idx = -1 +} + func (it *listSeriesIterator) At() (int64, float64) { s := it.samples.Get(it.idx) return s.T(), s.V() @@ -150,6 +170,11 @@ func NewListChunkSeriesIterator(chks ...chunks.Meta) chunks.Iterator { return &listChunkSeriesIterator{chks: chks, idx: -1} } +func (it *listChunkSeriesIterator) Reset(chks ...chunks.Meta) { + it.chks = chks + it.idx = -1 +} + func (it *listChunkSeriesIterator) At() chunks.Meta { return it.chks[it.idx] } @@ -164,6 +189,7 @@ func (it *listChunkSeriesIterator) Err() error { return nil } type chunkSetToSeriesSet struct { ChunkSeriesSet + iter chunks.Iterator chkIterErr error sameSeriesChunks []Series } @@ -178,18 +204,18 @@ func (c *chunkSetToSeriesSet) Next() bool { return false } - iter := c.ChunkSeriesSet.At().Iterator(nil) + c.iter = c.ChunkSeriesSet.At().Iterator(c.iter) c.sameSeriesChunks = c.sameSeriesChunks[:0] - for iter.Next() { + for c.iter.Next() { c.sameSeriesChunks = append( c.sameSeriesChunks, - newChunkToSeriesDecoder(c.ChunkSeriesSet.At().Labels(), iter.At()), + newChunkToSeriesDecoder(c.ChunkSeriesSet.At().Labels(), c.iter.At()), ) } - if iter.Err() != nil { - c.chkIterErr = iter.Err() + if c.iter.Err() != nil { + c.chkIterErr = c.iter.Err() return false } return true @@ -262,6 +288,11 @@ func (s *seriesToChunkEncoder) Iterator(it chunks.Iterator) chunks.Iterator { maxt := int64(math.MinInt64) chks := []chunks.Meta{} + lcsi, existing := it.(*listChunkSeriesIterator) + if existing { + chks = lcsi.chks[:0] + } + i := 0 seriesIter := s.Series.Iterator(nil) lastType := chunkenc.ValNone @@ -323,6 +354,10 @@ func (s *seriesToChunkEncoder) Iterator(it chunks.Iterator) chunks.Iterator { }) } + if existing { + lcsi.Reset(chks...) + return lcsi + } return NewListChunkSeriesIterator(chks...) } diff --git a/tsdb/head_read.go b/tsdb/head_read.go index 6a273a0fd..985a15792 100644 --- a/tsdb/head_read.go +++ b/tsdb/head_read.go @@ -503,11 +503,7 @@ func (o mergedOOOChunks) Appender() (chunkenc.Appender, error) { } func (o mergedOOOChunks) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator { - iterators := make([]chunkenc.Iterator, 0, len(o.chunks)) - for _, c := range o.chunks { - iterators = append(iterators, c.Chunk.Iterator(nil)) - } - return storage.NewChainSampleIterator(iterators) + return storage.ChainSampleIteratorFromMetas(iterator, o.chunks) } func (o mergedOOOChunks) NumSamples() int { From 085325069558be37b86318162f9dcaafcc83d227 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 15 Dec 2022 18:29:44 +0000 Subject: [PATCH 045/107] Review feedback Signed-off-by: Bryan Boreham --- storage/interface.go | 2 +- storage/series.go | 2 +- tsdb/querier.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/interface.go b/storage/interface.go index 5f0be9db9..3e8dd1086 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -422,7 +422,7 @@ type Labels interface { type SampleIterable interface { // Iterator returns an iterator of the data of the series. - // The iterator passed as argument is for re-use. + // The iterator passed as argument is for re-use, if not nil. // Depending on implementation, the iterator can // be re-used or a new iterator can be allocated. Iterator(chunkenc.Iterator) chunkenc.Iterator diff --git a/storage/series.go b/storage/series.go index 339beb2c9..377c060f7 100644 --- a/storage/series.go +++ b/storage/series.go @@ -287,7 +287,7 @@ func (s *seriesToChunkEncoder) Iterator(it chunks.Iterator) chunks.Iterator { mint := int64(math.MaxInt64) maxt := int64(math.MinInt64) - chks := []chunks.Meta{} + var chks []chunks.Meta lcsi, existing := it.(*listChunkSeriesIterator) if existing { chks = lcsi.chks[:0] diff --git a/tsdb/querier.go b/tsdb/querier.go index 642e089aa..89e3d5719 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -426,7 +426,7 @@ func labelNamesWithMatchers(r IndexReader, matchers ...*labels.Matcher) ([]strin return r.LabelNamesFor(postings...) } -// These are the things fetched when we move from one series to another. +// seriesData, used inside other iterators, are updated when we move from one series to another. type seriesData struct { chks []chunks.Meta intervals tombstones.Intervals From 89bf6e1df9e474488d4042bfdd3b4dcdfcb08889 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 29 Jun 2022 17:10:14 +0100 Subject: [PATCH 046/107] tsdb: Tidy up some test code Use simpler utility function to create Labels objects, making fewer assumptions about the data structure. Signed-off-by: Bryan Boreham --- tsdb/ooo_head_read_test.go | 16 ++++------------ tsdb/querier_test.go | 12 ++++++------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/tsdb/ooo_head_read_test.go b/tsdb/ooo_head_read_test.go index 8dca1ea59..b489c6e57 100644 --- a/tsdb/ooo_head_read_test.go +++ b/tsdb/ooo_head_read_test.go @@ -379,23 +379,15 @@ func TestOOOHeadChunkReader_LabelValues(t *testing.T) { app := head.Appender(context.Background()) // Add in-order samples - _, err := app.Append(0, labels.Labels{ - {Name: "foo", Value: "bar1"}, - }, 100, 1) + _, err := app.Append(0, labels.FromStrings("foo", "bar1"), 100, 1) require.NoError(t, err) - _, err = app.Append(0, labels.Labels{ - {Name: "foo", Value: "bar2"}, - }, 100, 2) + _, err = app.Append(0, labels.FromStrings("foo", "bar2"), 100, 2) require.NoError(t, err) // Add ooo samples for those series - _, err = app.Append(0, labels.Labels{ - {Name: "foo", Value: "bar1"}, - }, 90, 1) + _, err = app.Append(0, labels.FromStrings("foo", "bar1"), 90, 1) require.NoError(t, err) - _, err = app.Append(0, labels.Labels{ - {Name: "foo", Value: "bar2"}, - }, 90, 2) + _, err = app.Append(0, labels.FromStrings("foo", "bar2"), 90, 2) require.NoError(t, err) require.NoError(t, app.Commit()) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 3b44cef51..e6a75da2f 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -2160,7 +2160,7 @@ func TestBlockBaseSeriesSet(t *testing.T) { { series: []refdSeries{ { - lset: labels.New([]labels.Label{{Name: "a", Value: "a"}}...), + lset: labels.FromStrings("a", "a"), chunks: []chunks.Meta{ {Ref: 29}, {Ref: 45}, @@ -2173,19 +2173,19 @@ func TestBlockBaseSeriesSet(t *testing.T) { ref: 12, }, { - lset: labels.New([]labels.Label{{Name: "a", Value: "a"}, {Name: "b", Value: "b"}}...), + lset: labels.FromStrings("a", "a", "b", "b"), chunks: []chunks.Meta{ {Ref: 82}, {Ref: 23}, {Ref: 234}, {Ref: 65}, {Ref: 26}, }, ref: 10, }, { - lset: labels.New([]labels.Label{{Name: "b", Value: "c"}}...), + lset: labels.FromStrings("b", "c"), chunks: []chunks.Meta{{Ref: 8282}}, ref: 1, }, { - lset: labels.New([]labels.Label{{Name: "b", Value: "b"}}...), + lset: labels.FromStrings("b", "b"), chunks: []chunks.Meta{ {Ref: 829}, {Ref: 239}, {Ref: 2349}, {Ref: 659}, {Ref: 269}, }, @@ -2198,14 +2198,14 @@ func TestBlockBaseSeriesSet(t *testing.T) { { series: []refdSeries{ { - lset: labels.New([]labels.Label{{Name: "a", Value: "a"}, {Name: "b", Value: "b"}}...), + lset: labels.FromStrings("a", "a", "b", "b"), chunks: []chunks.Meta{ {Ref: 82}, {Ref: 23}, {Ref: 234}, {Ref: 65}, {Ref: 26}, }, ref: 10, }, { - lset: labels.New([]labels.Label{{Name: "b", Value: "c"}}...), + lset: labels.FromStrings("b", "c"), chunks: []chunks.Meta{{Ref: 8282}}, ref: 3, }, From 1695a7ee2f4a76e50eab6fabfa27f30050dfeb8e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 10 Jul 2022 15:29:04 +0100 Subject: [PATCH 047/107] promql: refactor BenchmarkRangeQuery so we can re-use test cases Signed-off-by: Bryan Boreham --- promql/bench_test.go | 56 +++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/promql/bench_test.go b/promql/bench_test.go index c3de6ca47..6fb20d1ab 100644 --- a/promql/bench_test.go +++ b/promql/bench_test.go @@ -27,17 +27,7 @@ import ( "github.com/prometheus/prometheus/util/teststorage" ) -func BenchmarkRangeQuery(b *testing.B) { - stor := teststorage.New(b) - defer stor.Close() - opts := EngineOpts{ - Logger: nil, - Reg: nil, - MaxSamples: 50000000, - Timeout: 100 * time.Second, - } - engine := NewEngine(opts) - +func setupRangeQueryTestData(stor *teststorage.TestStorage, engine *Engine, interval, numIntervals int) error { metrics := []labels.Labels{} metrics = append(metrics, labels.FromStrings("__name__", "a_one")) metrics = append(metrics, labels.FromStrings("__name__", "b_one")) @@ -65,25 +55,26 @@ func BenchmarkRangeQuery(b *testing.B) { } refs := make([]storage.SeriesRef, len(metrics)) - // A day of data plus 10k steps. - numIntervals := 8640 + 10000 - for s := 0; s < numIntervals; s++ { a := stor.Appender(context.Background()) - ts := int64(s * 10000) // 10s interval. + ts := int64(s * interval) for i, metric := range metrics { ref, _ := a.Append(refs[i], metric, ts, float64(s)+float64(i)/float64(len(metrics))) refs[i] = ref } if err := a.Commit(); err != nil { - b.Fatal(err) + return err } } + return nil +} - type benchCase struct { - expr string - steps int - } +type benchCase struct { + expr string + steps int +} + +func rangeQueryCases() []benchCase { cases := []benchCase{ // Plain retrieval. { @@ -210,7 +201,30 @@ func BenchmarkRangeQuery(b *testing.B) { tmp = append(tmp, benchCase{expr: c.expr, steps: 1000}) } } - cases = tmp + return tmp +} + +func BenchmarkRangeQuery(b *testing.B) { + stor := teststorage.New(b) + defer stor.Close() + opts := EngineOpts{ + Logger: nil, + Reg: nil, + MaxSamples: 50000000, + Timeout: 100 * time.Second, + } + engine := NewEngine(opts) + + const interval = 10000 // 10s interval. + // A day of data plus 10k steps. + numIntervals := 8640 + 10000 + + err := setupRangeQueryTestData(stor, engine, interval, numIntervals) + if err != nil { + b.Fatal(err) + } + cases := rangeQueryCases() + for _, c := range cases { name := fmt.Sprintf("expr=%s,steps=%d", c.expr, c.steps) b.Run(name, func(b *testing.B) { From 52adf55631c12154f3c757c9059a883382176945 Mon Sep 17 00:00:00 2001 From: David Fridman <119006078+davidifr@users.noreply.github.com> Date: Fri, 16 Dec 2022 20:14:35 +0200 Subject: [PATCH 048/107] Add VM size label to azure service discovery (#11575) (#11650) * Add VM size label to azure service discovery (#11575) Signed-off-by: davidifr * Add VM size label to azure service discovery (#11575) Signed-off-by: davidifr * Add VM size label to azure service discovery (#11575) Signed-off-by: davidifr Signed-off-by: davidifr --- discovery/azure/azure.go | 27 +++++++++++++++++++++------ discovery/azure/azure_test.go | 20 ++++++++++++++++++++ docs/configuration/configuration.md | 1 + 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/discovery/azure/azure.go b/discovery/azure/azure.go index 44576c79b..098fbb4c5 100644 --- a/discovery/azure/azure.go +++ b/discovery/azure/azure.go @@ -55,6 +55,7 @@ const ( azureLabelMachinePublicIP = azureLabel + "machine_public_ip" azureLabelMachineTag = azureLabel + "machine_tag_" azureLabelMachineScaleSet = azureLabel + "machine_scale_set" + azureLabelMachineSize = azureLabel + "machine_size" authMethodOAuth = "OAuth" authMethodManagedIdentity = "ManagedIdentity" @@ -261,6 +262,7 @@ type virtualMachine struct { ScaleSet string Tags map[string]*string NetworkInterfaces []string + Size string } // Create a new azureResource object from an ID string. @@ -343,6 +345,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { azureLabelMachineOSType: model.LabelValue(vm.OsType), azureLabelMachineLocation: model.LabelValue(vm.Location), azureLabelMachineResourceGroup: model.LabelValue(r.ResourceGroup), + azureLabelMachineSize: model.LabelValue(vm.Size), } if vm.ScaleSet != "" { @@ -514,6 +517,7 @@ func mapFromVM(vm compute.VirtualMachine) virtualMachine { tags := map[string]*string{} networkInterfaces := []string{} var computerName string + var size string if vm.Tags != nil { tags = vm.Tags @@ -525,10 +529,13 @@ func mapFromVM(vm compute.VirtualMachine) virtualMachine { } } - if vm.VirtualMachineProperties != nil && - vm.VirtualMachineProperties.OsProfile != nil && - vm.VirtualMachineProperties.OsProfile.ComputerName != nil { - computerName = *(vm.VirtualMachineProperties.OsProfile.ComputerName) + if vm.VirtualMachineProperties != nil { + if vm.VirtualMachineProperties.OsProfile != nil && vm.VirtualMachineProperties.OsProfile.ComputerName != nil { + computerName = *(vm.VirtualMachineProperties.OsProfile.ComputerName) + } + if vm.VirtualMachineProperties.HardwareProfile != nil { + size = string(vm.VirtualMachineProperties.HardwareProfile.VMSize) + } } return virtualMachine{ @@ -541,6 +548,7 @@ func mapFromVM(vm compute.VirtualMachine) virtualMachine { ScaleSet: "", Tags: tags, NetworkInterfaces: networkInterfaces, + Size: size, } } @@ -549,6 +557,7 @@ func mapFromVMScaleSetVM(vm compute.VirtualMachineScaleSetVM, scaleSetName strin tags := map[string]*string{} networkInterfaces := []string{} var computerName string + var size string if vm.Tags != nil { tags = vm.Tags @@ -560,8 +569,13 @@ func mapFromVMScaleSetVM(vm compute.VirtualMachineScaleSetVM, scaleSetName strin } } - if vm.VirtualMachineScaleSetVMProperties != nil && vm.VirtualMachineScaleSetVMProperties.OsProfile != nil { - computerName = *(vm.VirtualMachineScaleSetVMProperties.OsProfile.ComputerName) + if vm.VirtualMachineScaleSetVMProperties != nil { + if vm.VirtualMachineScaleSetVMProperties.OsProfile != nil && vm.VirtualMachineScaleSetVMProperties.OsProfile.ComputerName != nil { + computerName = *(vm.VirtualMachineScaleSetVMProperties.OsProfile.ComputerName) + } + if vm.VirtualMachineScaleSetVMProperties.HardwareProfile != nil { + size = string(vm.VirtualMachineScaleSetVMProperties.HardwareProfile.VMSize) + } } return virtualMachine{ @@ -574,6 +588,7 @@ func mapFromVMScaleSetVM(vm compute.VirtualMachineScaleSetVM, scaleSetName strin ScaleSet: scaleSetName, Tags: tags, NetworkInterfaces: networkInterfaces, + Size: size, } } diff --git a/discovery/azure/azure_test.go b/discovery/azure/azure_test.go index 744d182de..179b97ba6 100644 --- a/discovery/azure/azure_test.go +++ b/discovery/azure/azure_test.go @@ -28,6 +28,7 @@ func TestMain(m *testing.M) { func TestMapFromVMWithEmptyTags(t *testing.T) { id := "test" name := "name" + size := "size" vmType := "type" location := "westeurope" computerName := "computer_name" @@ -44,6 +45,9 @@ func TestMapFromVMWithEmptyTags(t *testing.T) { }, }, NetworkProfile: &networkProfile, + HardwareProfile: &compute.HardwareProfile{ + VMSize: compute.VirtualMachineSizeTypes(size), + }, } testVM := compute.VirtualMachine{ @@ -64,6 +68,7 @@ func TestMapFromVMWithEmptyTags(t *testing.T) { OsType: "Linux", Tags: map[string]*string{}, NetworkInterfaces: []string{}, + Size: size, } actualVM := mapFromVM(testVM) @@ -74,6 +79,7 @@ func TestMapFromVMWithEmptyTags(t *testing.T) { func TestMapFromVMWithTags(t *testing.T) { id := "test" name := "name" + size := "size" vmType := "type" location := "westeurope" computerName := "computer_name" @@ -93,6 +99,9 @@ func TestMapFromVMWithTags(t *testing.T) { }, }, NetworkProfile: &networkProfile, + HardwareProfile: &compute.HardwareProfile{ + VMSize: compute.VirtualMachineSizeTypes(size), + }, } testVM := compute.VirtualMachine{ @@ -113,6 +122,7 @@ func TestMapFromVMWithTags(t *testing.T) { OsType: "Linux", Tags: tags, NetworkInterfaces: []string{}, + Size: size, } actualVM := mapFromVM(testVM) @@ -123,6 +133,7 @@ func TestMapFromVMWithTags(t *testing.T) { func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) { id := "test" name := "name" + size := "size" vmType := "type" location := "westeurope" computerName := "computer_name" @@ -139,6 +150,9 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) { }, }, NetworkProfile: &networkProfile, + HardwareProfile: &compute.HardwareProfile{ + VMSize: compute.VirtualMachineSizeTypes(size), + }, } testVM := compute.VirtualMachineScaleSetVM{ @@ -161,6 +175,7 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) { Tags: map[string]*string{}, NetworkInterfaces: []string{}, ScaleSet: scaleSet, + Size: size, } actualVM := mapFromVMScaleSetVM(testVM, scaleSet) @@ -171,6 +186,7 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) { func TestMapFromVMScaleSetVMWithTags(t *testing.T) { id := "test" name := "name" + size := "size" vmType := "type" location := "westeurope" computerName := "computer_name" @@ -190,6 +206,9 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) { }, }, NetworkProfile: &networkProfile, + HardwareProfile: &compute.HardwareProfile{ + VMSize: compute.VirtualMachineSizeTypes(size), + }, } testVM := compute.VirtualMachineScaleSetVM{ @@ -212,6 +231,7 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) { Tags: tags, NetworkInterfaces: []string{}, ScaleSet: scaleSet, + Size: size, } actualVM := mapFromVMScaleSetVM(testVM, scaleSet) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 7b78e14b6..058a33da2 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -452,6 +452,7 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_azure_machine_resource_group`: the machine's resource group * `__meta_azure_machine_tag_`: each tag value of the machine * `__meta_azure_machine_scale_set`: the name of the scale set which the vm is part of (this value is only set if you are using a [scale set](https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/)) +* `__meta_azure_machine_size`: the machine size * `__meta_azure_subscription_id`: the subscription ID * `__meta_azure_tenant_id`: the tenant ID From 16738b00e358ff7fe17c89ecc7186929313c2bbc Mon Sep 17 00:00:00 2001 From: sniper91 Date: Tue, 29 Nov 2022 19:04:13 +0800 Subject: [PATCH 049/107] Do no re-use result slice in chunkSetToSeriesSet This is required to preserve the interface property of SeriesSet that says "At returns full series. Returned series should be iterable even after Next is called." Signed-off-by: sniper91 --- storage/series.go | 2 +- storage/series_test.go | 60 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/storage/series.go b/storage/series.go index 377c060f7..81f7b7baf 100644 --- a/storage/series.go +++ b/storage/series.go @@ -205,7 +205,7 @@ func (c *chunkSetToSeriesSet) Next() bool { } c.iter = c.ChunkSeriesSet.At().Iterator(c.iter) - c.sameSeriesChunks = c.sameSeriesChunks[:0] + c.sameSeriesChunks = nil for c.iter.Next() { c.sameSeriesChunks = append( diff --git a/storage/series_test.go b/storage/series_test.go index b200a3fc4..8a2675e35 100644 --- a/storage/series_test.go +++ b/storage/series_test.go @@ -18,7 +18,9 @@ import ( "github.com/stretchr/testify/require" + "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/tsdbutil" ) func TestListSeriesIterator(t *testing.T) { @@ -65,3 +67,61 @@ func TestListSeriesIterator(t *testing.T) { // And we don't go back. (This exposes issue #10027.) require.Equal(t, chunkenc.ValNone, it.Seek(2)) } + +// TestSeriesSetToChunkSet test the property of SeriesSet that says +// returned series should be iterable even after Next is called. +func TestChunkSeriesSetToSeriesSet(t *testing.T) { + series := []struct { + lbs labels.Labels + samples []tsdbutil.Sample + }{ + { + lbs: labels.Labels{ + {Name: "__name__", Value: "up"}, + {Name: "instance", Value: "localhost:8080"}, + }, + samples: []tsdbutil.Sample{ + sample{t: 1, v: 1}, + sample{t: 2, v: 2}, + sample{t: 3, v: 3}, + sample{t: 4, v: 4}, + }, + }, { + lbs: labels.Labels{ + {Name: "__name__", Value: "up"}, + {Name: "instance", Value: "localhost:8081"}, + }, + samples: []tsdbutil.Sample{ + sample{t: 1, v: 2}, + sample{t: 2, v: 3}, + sample{t: 3, v: 4}, + sample{t: 4, v: 5}, + sample{t: 5, v: 6}, + sample{t: 6, v: 7}, + }, + }, + } + var chunkSeries []ChunkSeries + for _, s := range series { + chunkSeries = append(chunkSeries, NewListChunkSeriesFromSamples(s.lbs, s.samples)) + } + css := NewMockChunkSeriesSet(chunkSeries...) + + ss := NewSeriesSetFromChunkSeriesSet(css) + var ssSlice []Series + for ss.Next() { + ssSlice = append(ssSlice, ss.At()) + } + require.Len(t, ssSlice, 2) + var iter chunkenc.Iterator + for i, s := range ssSlice { + require.EqualValues(t, series[i].lbs, s.Labels()) + iter = s.Iterator(iter) + j := 0 + for iter.Next() == chunkenc.ValFloat { + ts, v := iter.At() + require.EqualValues(t, series[i].samples[j], sample{t: ts, v: v}) + j++ + } + } +} From a19b369f9e518e6b89b59a72c82ba3bb687b2a7a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 13 Dec 2022 18:14:58 +0000 Subject: [PATCH 050/107] labels: avoid lint warning on New() This code is a bit cleaner. Signed-off-by: Bryan Boreham --- model/labels/labels.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/model/labels/labels.go b/model/labels/labels.go index aafba218a..103e8e5c3 100644 --- a/model/labels/labels.go +++ b/model/labels/labels.go @@ -357,9 +357,7 @@ func EmptyLabels() Labels { // The caller has to guarantee that all label names are unique. func New(ls ...Label) Labels { set := make(Labels, 0, len(ls)) - for _, l := range ls { - set = append(set, l) - } + set = append(set, ls...) sort.Sort(set) return set From ea7345a09c3f566ebd4ee30cafc8b77ff98f8f5b Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 13 Dec 2022 19:02:25 +0000 Subject: [PATCH 051/107] labels: improve comment on Builder.Set Signed-off-by: Bryan Boreham --- model/labels/labels.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/labels/labels.go b/model/labels/labels.go index 103e8e5c3..08c353c8a 100644 --- a/model/labels/labels.go +++ b/model/labels/labels.go @@ -468,7 +468,7 @@ Outer: return b } -// Set the name/value pair as a label. +// Set the name/value pair as a label. A value of "" means delete that label. func (b *Builder) Set(n, v string) *Builder { if v == "" { // Empty labels are the same as missing labels. From 2b8b8d9ac774bf5cacd650125aad29e6ecb69939 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 26 May 2022 14:59:13 +0000 Subject: [PATCH 052/107] labels: new methods to work without access to internals Without changing the definition of `labels.Labels`, add methods which enable code using it to work without knowledge of the internals. Signed-off-by: Bryan Boreham --- model/labels/labels.go | 81 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/model/labels/labels.go b/model/labels/labels.go index 08c353c8a..453c3f60d 100644 --- a/model/labels/labels.go +++ b/model/labels/labels.go @@ -412,6 +412,49 @@ func Compare(a, b Labels) int { return len(a) - len(b) } +// Copy labels from b on top of whatever was in ls previously, reusing memory or expanding if needed. +func (ls *Labels) CopyFrom(b Labels) { + (*ls) = append((*ls)[:0], b...) +} + +// IsEmpty returns true if ls represents an empty set of labels. +func (ls Labels) IsEmpty() bool { + return len(ls) == 0 +} + +// Range calls f on each label. +func (ls Labels) Range(f func(l Label)) { + for _, l := range ls { + f(l) + } +} + +// Validate calls f on each label. If f returns a non-nil error, then it returns that error cancelling the iteration. +func (ls Labels) Validate(f func(l Label) error) error { + for _, l := range ls { + if err := f(l); err != nil { + return err + } + } + return nil +} + +// InternStrings calls intern on every string value inside ls, replacing them with what it returns. +func (ls *Labels) InternStrings(intern func(string) string) { + for i, l := range *ls { + (*ls)[i].Name = intern(l.Name) + (*ls)[i].Value = intern(l.Value) + } +} + +// ReleaseStrings calls release on every string value inside ls. +func (ls Labels) ReleaseStrings(release func(string)) { + for _, l := range ls { + release(l.Name) + release(l.Value) + } +} + // Builder allows modifying Labels. type Builder struct { base Labels @@ -523,3 +566,41 @@ Outer: } return res } + +// ScratchBuilder allows efficient construction of a Labels from scratch. +type ScratchBuilder struct { + add Labels +} + +// NewScratchBuilder creates a ScratchBuilder initialized for Labels with n entries. +func NewScratchBuilder(n int) ScratchBuilder { + return ScratchBuilder{add: make([]Label, 0, n)} +} + +func (b *ScratchBuilder) Reset() { + b.add = b.add[:0] +} + +// Add a name/value pair. +// Note if you Add the same name twice you will get a duplicate label, which is invalid. +func (b *ScratchBuilder) Add(name, value string) { + b.add = append(b.add, Label{Name: name, Value: value}) +} + +// Sort the labels added so far by name. +func (b *ScratchBuilder) Sort() { + sort.Sort(b.add) +} + +// Return the name/value pairs added so far as a Labels object. +// Note: if you want them sorted, call Sort() first. +func (b *ScratchBuilder) Labels() Labels { + // Copy the slice, so the next use of ScratchBuilder doesn't overwrite. + return append([]Label{}, b.add...) +} + +// Write the newly-built Labels out to ls, reusing its buffer if long enough. +// Callers must ensure that there are no other references to ls. +func (b *ScratchBuilder) Overwrite(ls *Labels) { + (*ls) = append((*ls)[:0], b.add...) +} From 617bee60f19bfae850ffc6386e4be7637a0cb631 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 26 Jun 2022 20:11:53 +0100 Subject: [PATCH 053/107] labels: use ScratchBuilder in ReadLabels Instead of relying on being able to append to it like a slice. Signed-off-by: Bryan Boreham --- model/labels/test_utils.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/model/labels/test_utils.go b/model/labels/test_utils.go index a683588d1..05b816882 100644 --- a/model/labels/test_utils.go +++ b/model/labels/test_utils.go @@ -17,7 +17,6 @@ import ( "bufio" "fmt" "os" - "sort" "strings" ) @@ -51,13 +50,14 @@ func ReadLabels(fn string, n int) ([]Labels, error) { defer f.Close() scanner := bufio.NewScanner(f) + b := ScratchBuilder{} var mets []Labels hashes := map[uint64]struct{}{} i := 0 for scanner.Scan() && i < n { - m := make(Labels, 0, 10) + b.Reset() r := strings.NewReplacer("\"", "", "{", "", "}", "") s := r.Replace(scanner.Text()) @@ -65,10 +65,11 @@ func ReadLabels(fn string, n int) ([]Labels, error) { labelChunks := strings.Split(s, ",") for _, labelChunk := range labelChunks { split := strings.Split(labelChunk, ":") - m = append(m, Label{Name: split[0], Value: split[1]}) + b.Add(split[0], split[1]) } // Order of the k/v labels matters, don't assume we'll always receive them already sorted. - sort.Sort(m) + b.Sort() + m := b.Labels() h := m.Hash() if _, ok := hashes[h]; ok { From cbf432d2ac788b3bc83fd66630e3ea4cf0406644 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 26 May 2022 14:58:06 +0000 Subject: [PATCH 054/107] Update package labels tests for new labels.Labels type Re-did the FromStrings test to avoid assumptions about how it works. Signed-off-by: Bryan Boreham --- model/labels/labels_test.go | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/model/labels/labels_test.go b/model/labels/labels_test.go index 0fd0edacc..96b6ff9bc 100644 --- a/model/labels/labels_test.go +++ b/model/labels/labels_test.go @@ -36,10 +36,6 @@ func TestLabels_String(t *testing.T) { lables: Labels{}, expected: "{}", }, - { - lables: nil, - expected: "{}", - }, } for _, c := range cases { str := c.lables.String() @@ -316,18 +312,18 @@ func TestLabels_Equal(t *testing.T) { func TestLabels_FromStrings(t *testing.T) { labels := FromStrings("aaa", "111", "bbb", "222") - expected := Labels{ - { - Name: "aaa", - Value: "111", - }, - { - Name: "bbb", - Value: "222", - }, - } - - require.Equal(t, expected, labels, "unexpected labelset") + x := 0 + labels.Range(func(l Label) { + switch x { + case 0: + require.Equal(t, Label{Name: "aaa", Value: "111"}, l, "unexpected value") + case 1: + require.Equal(t, Label{Name: "bbb", Value: "222"}, l, "unexpected value") + default: + t.Fatalf("unexpected labelset value %d: %v", x, l) + } + x++ + }) require.Panics(t, func() { FromStrings("aaa", "111", "bbb") }) //nolint:staticcheck // Ignore SA5012, error is intentional test. } @@ -539,7 +535,6 @@ func TestBuilder(t *testing.T) { want: FromStrings("aaa", "111", "ccc", "333"), }, { - base: nil, set: []Label{{"aaa", "111"}, {"bbb", "222"}, {"ccc", "333"}}, del: []string{"bbb"}, want: FromStrings("aaa", "111", "ccc", "333"), @@ -604,8 +599,7 @@ func TestBuilder(t *testing.T) { func TestLabels_Hash(t *testing.T) { lbls := FromStrings("foo", "bar", "baz", "qux") require.Equal(t, lbls.Hash(), lbls.Hash()) - require.NotEqual(t, lbls.Hash(), Labels{lbls[1], lbls[0]}.Hash(), "unordered labels match.") - require.NotEqual(t, lbls.Hash(), Labels{lbls[0]}.Hash(), "different labels match.") + require.NotEqual(t, lbls.Hash(), FromStrings("foo", "bar").Hash(), "different labels match.") } var benchmarkLabelsResult uint64 @@ -623,7 +617,7 @@ func BenchmarkLabels_Hash(b *testing.B) { // Label ~20B name, 50B value. b.Set(fmt.Sprintf("abcdefghijabcdefghijabcdefghij%d", i), fmt.Sprintf("abcdefghijabcdefghijabcdefghijabcdefghijabcdefghij%d", i)) } - return b.Labels(nil) + return b.Labels(EmptyLabels()) }(), }, { @@ -634,7 +628,7 @@ func BenchmarkLabels_Hash(b *testing.B) { // Label ~50B name, 50B value. b.Set(fmt.Sprintf("abcdefghijabcdefghijabcdefghijabcdefghijabcdefghij%d", i), fmt.Sprintf("abcdefghijabcdefghijabcdefghijabcdefghijabcdefghij%d", i)) } - return b.Labels(nil) + return b.Labels(EmptyLabels()) }(), }, { From b10fd9aea3b169d60f9815d5baba6da7393c44ca Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 10 Jul 2022 15:22:49 +0100 Subject: [PATCH 055/107] model/labels: add a basic test for ScratchBuilder Signed-off-by: Bryan Boreham --- model/labels/labels_test.go | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/model/labels/labels_test.go b/model/labels/labels_test.go index 96b6ff9bc..3a1732d22 100644 --- a/model/labels/labels_test.go +++ b/model/labels/labels_test.go @@ -596,6 +596,46 @@ func TestBuilder(t *testing.T) { } } +func TestScratchBuilder(t *testing.T) { + for i, tcase := range []struct { + add []Label + want Labels + }{ + { + add: []Label{}, + want: EmptyLabels(), + }, + { + add: []Label{{"aaa", "111"}}, + want: FromStrings("aaa", "111"), + }, + { + add: []Label{{"aaa", "111"}, {"bbb", "222"}, {"ccc", "333"}}, + want: FromStrings("aaa", "111", "bbb", "222", "ccc", "333"), + }, + { + add: []Label{{"bbb", "222"}, {"aaa", "111"}, {"ccc", "333"}}, + want: FromStrings("aaa", "111", "bbb", "222", "ccc", "333"), + }, + { + add: []Label{{"ddd", "444"}}, + want: FromStrings("ddd", "444"), + }, + } { + overwriteTarget := EmptyLabels() + t.Run(fmt.Sprint(i), func(t *testing.T) { + b := ScratchBuilder{} + for _, lbl := range tcase.add { + b.Add(lbl.Name, lbl.Value) + } + b.Sort() + require.Equal(t, tcase.want, b.Labels()) + b.Overwrite(&overwriteTarget) + require.Equal(t, tcase.want, overwriteTarget) + }) + } +} + func TestLabels_Hash(t *testing.T) { lbls := FromStrings("foo", "bar", "baz", "qux") require.Equal(t, lbls.Hash(), lbls.Hash()) From 8ad7b64c0f1ba15f6a4b2f3275707548b26e5e5f Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 9 Mar 2022 22:13:03 +0000 Subject: [PATCH 056/107] Update package model/relabel for new labels.Labels type Signed-off-by: Bryan Boreham --- model/relabel/relabel.go | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/model/relabel/relabel.go b/model/relabel/relabel.go index c731f6e0d..0cc6eeeb7 100644 --- a/model/relabel/relabel.go +++ b/model/relabel/relabel.go @@ -203,20 +203,20 @@ func (re Regexp) String() string { // Process returns a relabeled copy of the given label set. The relabel configurations // are applied in order of input. -// If a label set is dropped, nil is returned. +// If a label set is dropped, EmptyLabels and false is returned. // May return the input labelSet modified. -func Process(lbls labels.Labels, cfgs ...*Config) labels.Labels { - lb := labels.NewBuilder(nil) +func Process(lbls labels.Labels, cfgs ...*Config) (ret labels.Labels, keep bool) { + lb := labels.NewBuilder(labels.EmptyLabels()) for _, cfg := range cfgs { - lbls = relabel(lbls, cfg, lb) - if lbls == nil { - return nil + lbls, keep = relabel(lbls, cfg, lb) + if !keep { + return labels.EmptyLabels(), false } } - return lbls + return lbls, true } -func relabel(lset labels.Labels, cfg *Config, lb *labels.Builder) labels.Labels { +func relabel(lset labels.Labels, cfg *Config, lb *labels.Builder) (ret labels.Labels, keep bool) { var va [16]string values := va[:0] if len(cfg.SourceLabels) > cap(values) { @@ -232,19 +232,19 @@ func relabel(lset labels.Labels, cfg *Config, lb *labels.Builder) labels.Labels switch cfg.Action { case Drop: if cfg.Regex.MatchString(val) { - return nil + return labels.EmptyLabels(), false } case Keep: if !cfg.Regex.MatchString(val) { - return nil + return labels.EmptyLabels(), false } case DropEqual: if lset.Get(cfg.TargetLabel) == val { - return nil + return labels.EmptyLabels(), false } case KeepEqual: if lset.Get(cfg.TargetLabel) != val { - return nil + return labels.EmptyLabels(), false } case Replace: indexes := cfg.Regex.FindStringSubmatchIndex(val) @@ -271,29 +271,29 @@ func relabel(lset labels.Labels, cfg *Config, lb *labels.Builder) labels.Labels mod := sum64(md5.Sum([]byte(val))) % cfg.Modulus lb.Set(cfg.TargetLabel, fmt.Sprintf("%d", mod)) case LabelMap: - for _, l := range lset { + lset.Range(func(l labels.Label) { if cfg.Regex.MatchString(l.Name) { res := cfg.Regex.ReplaceAllString(l.Name, cfg.Replacement) lb.Set(res, l.Value) } - } + }) case LabelDrop: - for _, l := range lset { + lset.Range(func(l labels.Label) { if cfg.Regex.MatchString(l.Name) { lb.Del(l.Name) } - } + }) case LabelKeep: - for _, l := range lset { + lset.Range(func(l labels.Label) { if !cfg.Regex.MatchString(l.Name) { lb.Del(l.Name) } - } + }) default: panic(fmt.Errorf("relabel: unknown relabel action type %q", cfg.Action)) } - return lb.Labels(lset) + return lb.Labels(lset), true } // sum64 sums the md5 hash to an uint64. From fe9fe0e1e5d7bc4efd796fadd594d3d6380d6a66 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 9 Mar 2022 22:13:18 +0000 Subject: [PATCH 057/107] Update package model/relabel tests for new labels.Labels type Signed-off-by: Bryan Boreham --- model/relabel/relabel_test.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/model/relabel/relabel_test.go b/model/relabel/relabel_test.go index 0b0dfd511..d277d778d 100644 --- a/model/relabel/relabel_test.go +++ b/model/relabel/relabel_test.go @@ -28,6 +28,7 @@ func TestRelabel(t *testing.T) { input labels.Labels relabel []*Config output labels.Labels + drop bool }{ { input: labels.FromMap(map[string]string{ @@ -101,7 +102,7 @@ func TestRelabel(t *testing.T) { Action: Replace, }, }, - output: nil, + drop: true, }, { input: labels.FromMap(map[string]string{ @@ -115,7 +116,7 @@ func TestRelabel(t *testing.T) { Action: Drop, }, }, - output: nil, + drop: true, }, { input: labels.FromMap(map[string]string{ @@ -177,7 +178,7 @@ func TestRelabel(t *testing.T) { Action: Keep, }, }, - output: nil, + drop: true, }, { input: labels.FromMap(map[string]string{ @@ -483,7 +484,7 @@ func TestRelabel(t *testing.T) { TargetLabel: "__port1", }, }, - output: nil, + drop: true, }, { input: labels.FromMap(map[string]string{ @@ -517,7 +518,7 @@ func TestRelabel(t *testing.T) { TargetLabel: "__port2", }, }, - output: nil, + drop: true, }, } @@ -538,8 +539,11 @@ func TestRelabel(t *testing.T) { } } - res := Process(test.input, test.relabel...) - require.Equal(t, test.output, res) + res, keep := Process(test.input, test.relabel...) + require.Equal(t, !test.drop, keep) + if keep { + require.Equal(t, test.output, res) + } } } @@ -721,7 +725,7 @@ func BenchmarkRelabel(b *testing.B) { for _, tt := range tests { b.Run(tt.name, func(b *testing.B) { for i := 0; i < b.N; i++ { - _ = Process(tt.lbls, tt.cfgs...) + _, _ = Process(tt.lbls, tt.cfgs...) } }) } From 1f04899ae33720830abcce683e72f9446627f253 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 9 Mar 2022 22:13:50 +0000 Subject: [PATCH 058/107] Update package model/textparse for new labels.Labels type Parse metrics using labels.ScratchBuilder, so we reduce assumptions about internals of Labels. Signed-off-by: Bryan Boreham --- model/textparse/openmetricsparse.go | 28 +++++++++++++--------------- model/textparse/promparse.go | 21 +++++++++------------ model/textparse/protobufparse.go | 27 ++++++++++++--------------- 3 files changed, 34 insertions(+), 42 deletions(-) diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go index 932a3d96d..3fc80b5d6 100644 --- a/model/textparse/openmetricsparse.go +++ b/model/textparse/openmetricsparse.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "math" - "sort" "strings" "unicode/utf8" @@ -82,6 +81,7 @@ func (l *openMetricsLexer) Error(es string) { // This is based on the working draft https://docs.google.com/document/u/1/d/1KwV0mAXwwbvvifBvDKH_LU1YjyXE_wxCkHNoCGq1GX0/edit type OpenMetricsParser struct { l *openMetricsLexer + builder labels.ScratchBuilder series []byte text []byte mtype MetricType @@ -158,14 +158,11 @@ func (p *OpenMetricsParser) Comment() []byte { // Metric writes the labels of the current sample into the passed labels. // It returns the string from which the metric was parsed. func (p *OpenMetricsParser) Metric(l *labels.Labels) string { - // Allocate the full immutable string immediately, so we just - // have to create references on it below. + // Copy the buffer to a string: this is only necessary for the return value. s := string(p.series) - *l = append(*l, labels.Label{ - Name: labels.MetricName, - Value: s[:p.offsets[0]-p.start], - }) + p.builder.Reset() + p.builder.Add(labels.MetricName, s[:p.offsets[0]-p.start]) for i := 1; i < len(p.offsets); i += 4 { a := p.offsets[i] - p.start @@ -173,16 +170,16 @@ func (p *OpenMetricsParser) Metric(l *labels.Labels) string { c := p.offsets[i+2] - p.start d := p.offsets[i+3] - p.start + value := s[c:d] // Replacer causes allocations. Replace only when necessary. if strings.IndexByte(s[c:d], byte('\\')) >= 0 { - *l = append(*l, labels.Label{Name: s[a:b], Value: lvalReplacer.Replace(s[c:d])}) - continue + value = lvalReplacer.Replace(value) } - *l = append(*l, labels.Label{Name: s[a:b], Value: s[c:d]}) + p.builder.Add(s[a:b], value) } - // Sort labels. - sort.Sort(*l) + p.builder.Sort() + *l = p.builder.Labels() return s } @@ -204,17 +201,18 @@ func (p *OpenMetricsParser) Exemplar(e *exemplar.Exemplar) bool { e.Ts = p.exemplarTs } + p.builder.Reset() for i := 0; i < len(p.eOffsets); i += 4 { a := p.eOffsets[i] - p.start b := p.eOffsets[i+1] - p.start c := p.eOffsets[i+2] - p.start d := p.eOffsets[i+3] - p.start - e.Labels = append(e.Labels, labels.Label{Name: s[a:b], Value: s[c:d]}) + p.builder.Add(s[a:b], s[c:d]) } - // Sort the labels. - sort.Sort(e.Labels) + p.builder.Sort() + e.Labels = p.builder.Labels() return true } diff --git a/model/textparse/promparse.go b/model/textparse/promparse.go index a3bb8bb9b..d503ff9a7 100644 --- a/model/textparse/promparse.go +++ b/model/textparse/promparse.go @@ -21,7 +21,6 @@ import ( "fmt" "io" "math" - "sort" "strconv" "strings" "unicode/utf8" @@ -144,6 +143,7 @@ func (l *promlexer) Error(es string) { // Prometheus text exposition format. type PromParser struct { l *promlexer + builder labels.ScratchBuilder series []byte text []byte mtype MetricType @@ -212,14 +212,11 @@ func (p *PromParser) Comment() []byte { // Metric writes the labels of the current sample into the passed labels. // It returns the string from which the metric was parsed. func (p *PromParser) Metric(l *labels.Labels) string { - // Allocate the full immutable string immediately, so we just - // have to create references on it below. + // Copy the buffer to a string: this is only necessary for the return value. s := string(p.series) - *l = append(*l, labels.Label{ - Name: labels.MetricName, - Value: s[:p.offsets[0]-p.start], - }) + p.builder.Reset() + p.builder.Add(labels.MetricName, s[:p.offsets[0]-p.start]) for i := 1; i < len(p.offsets); i += 4 { a := p.offsets[i] - p.start @@ -227,16 +224,16 @@ func (p *PromParser) Metric(l *labels.Labels) string { c := p.offsets[i+2] - p.start d := p.offsets[i+3] - p.start + value := s[c:d] // Replacer causes allocations. Replace only when necessary. if strings.IndexByte(s[c:d], byte('\\')) >= 0 { - *l = append(*l, labels.Label{Name: s[a:b], Value: lvalReplacer.Replace(s[c:d])}) - continue + value = lvalReplacer.Replace(value) } - *l = append(*l, labels.Label{Name: s[a:b], Value: s[c:d]}) + p.builder.Add(s[a:b], value) } - // Sort labels to maintain the sorted labels invariant. - sort.Sort(*l) + p.builder.Sort() + *l = p.builder.Labels() return s } diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go index a9c940879..37c6f0ebb 100644 --- a/model/textparse/protobufparse.go +++ b/model/textparse/protobufparse.go @@ -19,7 +19,6 @@ import ( "fmt" "io" "math" - "sort" "strings" "unicode/utf8" @@ -59,6 +58,8 @@ type ProtobufParser struct { // that we have to decode the next MetricFamily. state Entry + builder labels.ScratchBuilder // held here to reduce allocations when building Labels + mf *dto.MetricFamily // The following are just shenanigans to satisfy the Parser interface. @@ -245,23 +246,19 @@ func (p *ProtobufParser) Comment() []byte { // Metric writes the labels of the current sample into the passed labels. // It returns the string from which the metric was parsed. func (p *ProtobufParser) Metric(l *labels.Labels) string { - *l = append(*l, labels.Label{ - Name: labels.MetricName, - Value: p.getMagicName(), - }) + p.builder.Reset() + p.builder.Add(labels.MetricName, p.getMagicName()) for _, lp := range p.mf.GetMetric()[p.metricPos].GetLabel() { - *l = append(*l, labels.Label{ - Name: lp.GetName(), - Value: lp.GetValue(), - }) + p.builder.Add(lp.GetName(), lp.GetValue()) } if needed, name, value := p.getMagicLabel(); needed { - *l = append(*l, labels.Label{Name: name, Value: value}) + p.builder.Add(name, value) } // Sort labels to maintain the sorted labels invariant. - sort.Sort(*l) + p.builder.Sort() + *l = p.builder.Labels() return p.metricBytes.String() } @@ -305,12 +302,12 @@ func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool { ex.HasTs = true ex.Ts = ts.GetSeconds()*1000 + int64(ts.GetNanos()/1_000_000) } + p.builder.Reset() for _, lp := range exProto.GetLabel() { - ex.Labels = append(ex.Labels, labels.Label{ - Name: lp.GetName(), - Value: lp.GetValue(), - }) + p.builder.Add(lp.GetName(), lp.GetValue()) } + p.builder.Sort() + ex.Labels = p.builder.Labels() return true } From 8d350d9e0c84d90d3e639521da47e89eec57a4ab Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 9 Mar 2022 22:14:19 +0000 Subject: [PATCH 059/107] Update package model/textparse tests for new labels.Labels type We don't want to touch the result labels now we create them differently. Signed-off-by: Bryan Boreham --- model/textparse/openmetricsparse_test.go | 1 - model/textparse/promparse_test.go | 6 ++---- model/textparse/protobufparse_test.go | 2 -- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/model/textparse/openmetricsparse_test.go b/model/textparse/openmetricsparse_test.go index 9453db5d5..68b7fea8a 100644 --- a/model/textparse/openmetricsparse_test.go +++ b/model/textparse/openmetricsparse_test.go @@ -246,7 +246,6 @@ foo_total 17.0 1520879607.789 # {xx="yy"} 5` require.Equal(t, true, found) require.Equal(t, *exp[i].e, e) } - res = res[:0] case EntryType: m, typ := p.Type() diff --git a/model/textparse/promparse_test.go b/model/textparse/promparse_test.go index 6a1216da1..dadad3449 100644 --- a/model/textparse/promparse_test.go +++ b/model/textparse/promparse_test.go @@ -192,7 +192,6 @@ testmetric{label="\"bar\""} 1` require.Equal(t, exp[i].t, ts) require.Equal(t, exp[i].v, v) require.Equal(t, exp[i].lset, res) - res = res[:0] case EntryType: m, typ := p.Type() @@ -414,7 +413,7 @@ func BenchmarkParse(b *testing.B) { case EntrySeries: m, _, _ := p.Series() - res := make(labels.Labels, 0, 5) + var res labels.Labels p.Metric(&res) total += len(m) @@ -426,7 +425,7 @@ func BenchmarkParse(b *testing.B) { }) b.Run(parserName+"/decode-metric-reuse/"+fn, func(b *testing.B) { total := 0 - res := make(labels.Labels, 0, 5) + var res labels.Labels b.SetBytes(int64(len(buf) / promtestdataSampleCount)) b.ReportAllocs() @@ -451,7 +450,6 @@ func BenchmarkParse(b *testing.B) { total += len(m) i++ - res = res[:0] } } } diff --git a/model/textparse/protobufparse_test.go b/model/textparse/protobufparse_test.go index 33826ad8e..b8b868172 100644 --- a/model/textparse/protobufparse_test.go +++ b/model/textparse/protobufparse_test.go @@ -630,7 +630,6 @@ metric: < require.Equal(t, true, found) require.Equal(t, exp[i].e[0], e) } - res = res[:0] case EntryHistogram: m, ts, shs, fhs := p.Histogram() @@ -642,7 +641,6 @@ metric: < require.Equal(t, exp[i].t, int64(0)) } require.Equal(t, exp[i].lset, res) - res = res[:0] require.Equal(t, exp[i].m, string(m)) if shs != nil { require.Equal(t, exp[i].shs, shs) From 623d306f91103ea8e12960f2c3ba5c799b7c1a39 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 9 Mar 2022 22:26:05 +0000 Subject: [PATCH 060/107] Update package config for new labels.Labels type Signed-off-by: Bryan Boreham --- config/config.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/config/config.go b/config/config.go index 8e8460d4c..8eb88eb51 100644 --- a/config/config.go +++ b/config/config.go @@ -80,7 +80,8 @@ func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, erro return cfg, nil } - for i, v := range cfg.GlobalConfig.ExternalLabels { + b := labels.ScratchBuilder{} + cfg.GlobalConfig.ExternalLabels.Range(func(v labels.Label) { newV := os.Expand(v.Value, func(s string) string { if s == "$" { return "$" @@ -93,10 +94,10 @@ func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, erro }) if newV != v.Value { level.Debug(logger).Log("msg", "External label replaced", "label", v.Name, "input", v.Value, "output", newV) - v.Value = newV - cfg.GlobalConfig.ExternalLabels[i] = v } - } + b.Add(v.Name, newV) + }) + cfg.GlobalConfig.ExternalLabels = b.Labels() return cfg, nil } @@ -361,13 +362,16 @@ func (c *GlobalConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } - for _, l := range gc.ExternalLabels { + if err := gc.ExternalLabels.Validate(func(l labels.Label) error { if !model.LabelName(l.Name).IsValid() { return fmt.Errorf("%q is not a valid label name", l.Name) } if !model.LabelValue(l.Value).IsValid() { return fmt.Errorf("%q is not a valid label value", l.Value) } + return nil + }); err != nil { + return err } // First set the correct scrape interval, then check that the timeout @@ -394,7 +398,7 @@ func (c *GlobalConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { // isZero returns true iff the global config is the zero value. func (c *GlobalConfig) isZero() bool { - return c.ExternalLabels == nil && + return c.ExternalLabels.IsEmpty() && c.ScrapeInterval == 0 && c.ScrapeTimeout == 0 && c.EvaluationInterval == 0 && From 927a14b0e9f575b46163114c202d1470962d262c Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 9 Mar 2022 22:18:47 +0000 Subject: [PATCH 061/107] Update package tsdb/index for new labels.Labels type Incomplete - needs further changes to `Decoder.Series()`. Signed-off-by: Bryan Boreham --- tsdb/index/index.go | 11 +++++++---- tsdb/index/postings.go | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tsdb/index/index.go b/tsdb/index/index.go index 9d7897860..ba78b5712 100644 --- a/tsdb/index/index.go +++ b/tsdb/index/index.go @@ -423,7 +423,7 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ... return errors.Errorf("out-of-order series added with label set %q", lset) } - if ref < w.lastRef && len(w.lastSeries) != 0 { + if ref < w.lastRef && !w.lastSeries.IsEmpty() { return errors.Errorf("series with reference greater than %d already added", ref) } // We add padding to 16 bytes to increase the addressable space we get through 4 byte @@ -437,9 +437,9 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ... } w.buf2.Reset() - w.buf2.PutUvarint(len(lset)) + w.buf2.PutUvarint(lset.Len()) - for _, l := range lset { + if err := lset.Validate(func(l labels.Label) error { var err error cacheEntry, ok := w.symbolCache[l.Name] nameIndex := cacheEntry.index @@ -465,6 +465,9 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ... } } w.buf2.PutUvarint32(valueIndex) + return nil + }); err != nil { + return err } w.buf2.PutUvarint(len(chunks)) @@ -496,7 +499,7 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ... return errors.Wrap(err, "write series data") } - w.lastSeries = append(w.lastSeries[:0], lset...) + w.lastSeries.CopyFrom(lset) w.lastRef = ref return nil diff --git a/tsdb/index/postings.go b/tsdb/index/postings.go index 22e85ab36..64574f85c 100644 --- a/tsdb/index/postings.go +++ b/tsdb/index/postings.go @@ -353,9 +353,9 @@ func (p *MemPostings) Iter(f func(labels.Label, Postings) error) error { func (p *MemPostings) Add(id storage.SeriesRef, lset labels.Labels) { p.mtx.Lock() - for _, l := range lset { + lset.Range(func(l labels.Label) { p.addFor(id, l) - } + }) p.addFor(id, allPostingsKey) p.mtx.Unlock() From d3d96ec887b312157504489022991a7ff673b478 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 28 Jun 2022 16:03:26 +0100 Subject: [PATCH 062/107] tsdb/index: use ScratchBuilder to create Labels This necessitates a change to the `tsdb.IndexReader` interface: `index.Reader` is used from multiple goroutines concurrently, so we can't have state in it. We do retain a `ScratchBuilder` in `blockBaseSeriesSet` which is iterator-like. Signed-off-by: Bryan Boreham --- cmd/promtool/tsdb.go | 6 ++++-- tsdb/block.go | 9 +++++---- tsdb/db_test.go | 4 +++- tsdb/head_read.go | 2 +- tsdb/head_test.go | 14 ++++++++------ tsdb/index/index.go | 12 +++++++----- tsdb/index/index_test.go | 13 ++++++++----- tsdb/querier.go | 3 ++- tsdb/querier_test.go | 7 ++++--- tsdb/repair_test.go | 5 +++-- 10 files changed, 45 insertions(+), 30 deletions(-) diff --git a/cmd/promtool/tsdb.go b/cmd/promtool/tsdb.go index 91b97f5c5..1c6ae6f6b 100644 --- a/cmd/promtool/tsdb.go +++ b/cmd/promtool/tsdb.go @@ -472,8 +472,9 @@ func analyzeBlock(path, blockID string, limit int, runExtended bool) error { } lbls := labels.Labels{} chks := []chunks.Meta{} + builder := labels.ScratchBuilder{} for p.Next() { - if err = ir.Series(p.At(), &lbls, &chks); err != nil { + if err = ir.Series(p.At(), &builder, &lbls, &chks); err != nil { return err } // Amount of the block time range not covered by this series. @@ -589,10 +590,11 @@ func analyzeCompaction(block tsdb.BlockReader, indexr tsdb.IndexReader) (err err nBuckets := 10 histogram := make([]int, nBuckets) totalChunks := 0 + var builder labels.ScratchBuilder for postingsr.Next() { lbsl := labels.Labels{} var chks []chunks.Meta - if err := indexr.Series(postingsr.At(), &lbsl, &chks); err != nil { + if err := indexr.Series(postingsr.At(), &builder, &lbsl, &chks); err != nil { return err } diff --git a/tsdb/block.go b/tsdb/block.go index b06f0940a..33413bb21 100644 --- a/tsdb/block.go +++ b/tsdb/block.go @@ -82,7 +82,7 @@ type IndexReader interface { // Series populates the given labels and chunk metas for the series identified // by the reference. // Returns storage.ErrNotFound if the ref does not resolve to a known series. - Series(ref storage.SeriesRef, lset *labels.Labels, chks *[]chunks.Meta) error + Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, lset *labels.Labels, chks *[]chunks.Meta) error // LabelNames returns all the unique label names present in the index in sorted order. LabelNames(matchers ...*labels.Matcher) ([]string, error) @@ -499,8 +499,8 @@ func (r blockIndexReader) SortedPostings(p index.Postings) index.Postings { return r.ir.SortedPostings(p) } -func (r blockIndexReader) Series(ref storage.SeriesRef, lset *labels.Labels, chks *[]chunks.Meta) error { - if err := r.ir.Series(ref, lset, chks); err != nil { +func (r blockIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, lset *labels.Labels, chks *[]chunks.Meta) error { + if err := r.ir.Series(ref, builder, lset, chks); err != nil { return errors.Wrapf(err, "block: %s", r.b.Meta().ULID) } return nil @@ -563,10 +563,11 @@ func (pb *Block) Delete(mint, maxt int64, ms ...*labels.Matcher) error { var lset labels.Labels var chks []chunks.Meta + var builder labels.ScratchBuilder Outer: for p.Next() { - err := ir.Series(p.At(), &lset, &chks) + err := ir.Series(p.At(), &builder, &lset, &chks) if err != nil { return err } diff --git a/tsdb/db_test.go b/tsdb/db_test.go index cea4b6e36..7eb3e272e 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -1830,6 +1830,8 @@ func TestChunkAtBlockBoundary(t *testing.T) { err = db.Compact() require.NoError(t, err) + var builder labels.ScratchBuilder + for _, block := range db.Blocks() { r, err := block.Index() require.NoError(t, err) @@ -1849,7 +1851,7 @@ func TestChunkAtBlockBoundary(t *testing.T) { chunkCount := 0 for p.Next() { - err = r.Series(p.At(), &lset, &chks) + err = r.Series(p.At(), &builder, &lset, &chks) require.NoError(t, err) for _, c := range chks { require.True(t, meta.MinTime <= c.MinTime && c.MaxTime <= meta.MaxTime, diff --git a/tsdb/head_read.go b/tsdb/head_read.go index 985a15792..c6017e7b8 100644 --- a/tsdb/head_read.go +++ b/tsdb/head_read.go @@ -148,7 +148,7 @@ func (h *headIndexReader) SortedPostings(p index.Postings) index.Postings { } // Series returns the series for the given reference. -func (h *headIndexReader) Series(ref storage.SeriesRef, lbls *labels.Labels, chks *[]chunks.Meta) error { +func (h *headIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, lbls *labels.Labels, chks *[]chunks.Meta) error { s := h.head.series.getByID(chunks.HeadSeriesRef(ref)) if s == nil { diff --git a/tsdb/head_test.go b/tsdb/head_test.go index 59824ae08..f10dc9f64 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -1446,10 +1446,11 @@ func TestGCChunkAccess(t *testing.T) { idx := h.indexRange(0, 1500) var ( - lset labels.Labels - chunks []chunks.Meta + lset labels.Labels + chunks []chunks.Meta + builder labels.ScratchBuilder ) - require.NoError(t, idx.Series(1, &lset, &chunks)) + require.NoError(t, idx.Series(1, &builder, &lset, &chunks)) require.Equal(t, labels.FromStrings("a", "1"), lset) require.Equal(t, 2, len(chunks)) @@ -1499,10 +1500,11 @@ func TestGCSeriesAccess(t *testing.T) { idx := h.indexRange(0, 2000) var ( - lset labels.Labels - chunks []chunks.Meta + lset labels.Labels + chunks []chunks.Meta + builder labels.ScratchBuilder ) - require.NoError(t, idx.Series(1, &lset, &chunks)) + require.NoError(t, idx.Series(1, &builder, &lset, &chunks)) require.Equal(t, labels.FromStrings("a", "1"), lset) require.Equal(t, 2, len(chunks)) diff --git a/tsdb/index/index.go b/tsdb/index/index.go index ba78b5712..9cf2425ec 100644 --- a/tsdb/index/index.go +++ b/tsdb/index/index.go @@ -1597,7 +1597,7 @@ func (r *Reader) LabelValueFor(id storage.SeriesRef, label string) (string, erro } // Series reads the series with the given ID and writes its labels and chunks into lbls and chks. -func (r *Reader) Series(id storage.SeriesRef, lbls *labels.Labels, chks *[]chunks.Meta) error { +func (r *Reader) Series(id storage.SeriesRef, builder *labels.ScratchBuilder, lbls *labels.Labels, chks *[]chunks.Meta) error { offset := id // In version 2 series IDs are no longer exact references but series are 16-byte padded // and the ID is the multiple of 16 of the actual position. @@ -1608,7 +1608,7 @@ func (r *Reader) Series(id storage.SeriesRef, lbls *labels.Labels, chks *[]chunk if d.Err() != nil { return d.Err() } - return errors.Wrap(r.dec.Series(d.Get(), lbls, chks), "read series") + return errors.Wrap(r.dec.Series(d.Get(), builder, lbls, chks), "read series") } func (r *Reader) Postings(name string, values ...string) (Postings, error) { @@ -1836,8 +1836,9 @@ func (dec *Decoder) LabelValueFor(b []byte, label string) (string, error) { } // Series decodes a series entry from the given byte slice into lset and chks. -func (dec *Decoder) Series(b []byte, lbls *labels.Labels, chks *[]chunks.Meta) error { - *lbls = (*lbls)[:0] +// Previous contents of lbls can be overwritten - make sure you copy before retaining. +func (dec *Decoder) Series(b []byte, builder *labels.ScratchBuilder, lbls *labels.Labels, chks *[]chunks.Meta) error { + builder.Reset() *chks = (*chks)[:0] d := encoding.Decbuf{B: b} @@ -1861,8 +1862,9 @@ func (dec *Decoder) Series(b []byte, lbls *labels.Labels, chks *[]chunks.Meta) e return errors.Wrap(err, "lookup label value") } - *lbls = append(*lbls, labels.Label{Name: ln, Value: lv}) + builder.Add(ln, lv) } + builder.Overwrite(lbls) // Read the chunks meta data. k = d.Uvarint() diff --git a/tsdb/index/index_test.go b/tsdb/index/index_test.go index 6346f2ee8..ff22fb1c0 100644 --- a/tsdb/index/index_test.go +++ b/tsdb/index/index_test.go @@ -124,7 +124,7 @@ func (m mockIndex) SortedPostings(p Postings) Postings { return NewListPostings(ep) } -func (m mockIndex) Series(ref storage.SeriesRef, lset *labels.Labels, chks *[]chunks.Meta) error { +func (m mockIndex) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, lset *labels.Labels, chks *[]chunks.Meta) error { s, ok := m.series[ref] if !ok { return errors.New("not found") @@ -199,9 +199,10 @@ func TestIndexRW_Postings(t *testing.T) { var l labels.Labels var c []chunks.Meta + var builder labels.ScratchBuilder for i := 0; p.Next(); i++ { - err := ir.Series(p.At(), &l, &c) + err := ir.Series(p.At(), &builder, &l, &c) require.NoError(t, err) require.Equal(t, 0, len(c)) @@ -311,6 +312,7 @@ func TestPostingsMany(t *testing.T) { {in: []string{"126a", "126b", "127", "127a", "127b", "128", "128a", "128b", "129", "129a", "129b"}}, } + var builder labels.ScratchBuilder for _, c := range cases { it, err := ir.Postings("i", c.in...) require.NoError(t, err) @@ -319,7 +321,7 @@ func TestPostingsMany(t *testing.T) { var lbls labels.Labels var metas []chunks.Meta for it.Next() { - require.NoError(t, ir.Series(it.At(), &lbls, &metas)) + require.NoError(t, ir.Series(it.At(), &builder, &lbls, &metas)) got = append(got, lbls.Get("i")) } require.NoError(t, it.Err()) @@ -421,16 +423,17 @@ func TestPersistence_index_e2e(t *testing.T) { var lset, explset labels.Labels var chks, expchks []chunks.Meta + var builder labels.ScratchBuilder for gotp.Next() { require.True(t, expp.Next()) ref := gotp.At() - err := ir.Series(ref, &lset, &chks) + err := ir.Series(ref, &builder, &lset, &chks) require.NoError(t, err) - err = mi.Series(expp.At(), &explset, &expchks) + err = mi.Series(expp.At(), &builder, &explset, &expchks) require.NoError(t, err) require.Equal(t, explset, lset) require.Equal(t, expchks, chks) diff --git a/tsdb/querier.go b/tsdb/querier.go index 89e3d5719..4082279ae 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -452,12 +452,13 @@ type blockBaseSeriesSet struct { bufChks []chunks.Meta bufLbls labels.Labels + builder labels.ScratchBuilder err error } func (b *blockBaseSeriesSet) Next() bool { for b.p.Next() { - if err := b.index.Series(b.p.At(), &b.bufLbls, &b.bufChks); err != nil { + if err := b.index.Series(b.p.At(), &b.builder, &b.bufLbls, &b.bufChks); err != nil { // Postings may be stale. Skip if no underlying series exists. if errors.Cause(err) == storage.ErrNotFound { continue diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index e6a75da2f..9933b7158 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -1270,7 +1270,7 @@ func (m mockIndex) SortedPostings(p index.Postings) index.Postings { return index.NewListPostings(ep) } -func (m mockIndex) Series(ref storage.SeriesRef, lset *labels.Labels, chks *[]chunks.Meta) error { +func (m mockIndex) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, lset *labels.Labels, chks *[]chunks.Meta) error { s, ok := m.series[ref] if !ok { return storage.ErrNotFound @@ -1884,9 +1884,10 @@ func TestPostingsForMatchers(t *testing.T) { p, err := PostingsForMatchers(ir, c.matchers...) require.NoError(t, err) + var builder labels.ScratchBuilder for p.Next() { lbls := labels.Labels{} - require.NoError(t, ir.Series(p.At(), &lbls, &[]chunks.Meta{})) + require.NoError(t, ir.Series(p.At(), &builder, &lbls, &[]chunks.Meta{})) if _, ok := exp[lbls.String()]; !ok { t.Errorf("Evaluating %v, unexpected result %s", c.matchers, lbls.String()) } else { @@ -2097,7 +2098,7 @@ func (m mockMatcherIndex) SortedPostings(p index.Postings) index.Postings { return index.EmptyPostings() } -func (m mockMatcherIndex) Series(ref storage.SeriesRef, lset *labels.Labels, chks *[]chunks.Meta) error { +func (m mockMatcherIndex) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, lset *labels.Labels, chks *[]chunks.Meta) error { return nil } diff --git a/tsdb/repair_test.go b/tsdb/repair_test.go index 6d95fc0a6..d39411560 100644 --- a/tsdb/repair_test.go +++ b/tsdb/repair_test.go @@ -80,11 +80,12 @@ func TestRepairBadIndexVersion(t *testing.T) { require.NoError(t, err) p, err := r.Postings("b", "1") require.NoError(t, err) + var builder labels.ScratchBuilder for p.Next() { t.Logf("next ID %d", p.At()) var lset labels.Labels - require.Error(t, r.Series(p.At(), &lset, nil)) + require.Error(t, r.Series(p.At(), &builder, &lset, nil)) } require.NoError(t, p.Err()) require.NoError(t, r.Close()) @@ -106,7 +107,7 @@ func TestRepairBadIndexVersion(t *testing.T) { var lset labels.Labels var chks []chunks.Meta - require.NoError(t, r.Series(p.At(), &lset, &chks)) + require.NoError(t, r.Series(p.At(), &builder, &lset, &chks)) res = append(res, lset) } From a5bdff414b8d7b1925c5634c688f415262880de3 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 9 Mar 2022 22:18:37 +0000 Subject: [PATCH 063/107] Update package tsdb/index tests for new labels.Labels type Note in one cases we needed an extra copy of labels in case they change. Signed-off-by: Bryan Boreham --- tsdb/index/index_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tsdb/index/index_test.go b/tsdb/index/index_test.go index ff22fb1c0..e6b2e890c 100644 --- a/tsdb/index/index_test.go +++ b/tsdb/index/index_test.go @@ -68,14 +68,14 @@ func (m mockIndex) AddSeries(ref storage.SeriesRef, l labels.Labels, chunks ...c if _, ok := m.series[ref]; ok { return errors.Errorf("series with reference %d already added", ref) } - for _, lbl := range l { + l.Range(func(lbl labels.Label) { m.symbols[lbl.Name] = struct{}{} m.symbols[lbl.Value] = struct{}{} if _, ok := m.postings[lbl]; !ok { m.postings[lbl] = []storage.SeriesRef{} } m.postings[lbl] = append(m.postings[lbl], ref) - } + }) m.postings[allPostingsKey] = append(m.postings[allPostingsKey], ref) s := series{l: l} @@ -129,7 +129,7 @@ func (m mockIndex) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, if !ok { return errors.New("not found") } - *lset = append((*lset)[:0], s.l...) + lset.CopyFrom(s.l) *chks = append((*chks)[:0], s.chunks...) return nil @@ -322,7 +322,7 @@ func TestPostingsMany(t *testing.T) { var metas []chunks.Meta for it.Next() { require.NoError(t, ir.Series(it.At(), &builder, &lbls, &metas)) - got = append(got, lbls.Get("i")) + got = append(got, lbls.Copy().Get("i")) } require.NoError(t, it.Err()) exp := []string{} @@ -346,10 +346,10 @@ func TestPersistence_index_e2e(t *testing.T) { symbols := map[string]struct{}{} for _, lset := range lbls { - for _, l := range lset { + lset.Range(func(l labels.Label) { symbols[l.Name] = struct{}{} symbols[l.Value] = struct{}{} - } + }) } var input indexWriterSeriesSlice @@ -397,14 +397,14 @@ func TestPersistence_index_e2e(t *testing.T) { require.NoError(t, err) require.NoError(t, mi.AddSeries(storage.SeriesRef(i), s.labels, s.chunks...)) - for _, l := range s.labels { + s.labels.Range(func(l labels.Label) { valset, ok := values[l.Name] if !ok { valset = map[string]struct{}{} values[l.Name] = valset } valset[l.Value] = struct{}{} - } + }) postings.Add(storage.SeriesRef(i), s.labels) } From 14ad2e780be7e2f3ff49737a424ddc359f4adb1b Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 20 Feb 2022 20:42:11 +0000 Subject: [PATCH 064/107] Update package tsdb/agent for new labels.Labels type Signed-off-by: Bryan Boreham --- tsdb/agent/db.go | 10 ++++++--- tsdb/agent/db_test.go | 52 +++++++++++++++++++++---------------------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/tsdb/agent/db.go b/tsdb/agent/db.go index 0675d5a28..7e959573b 100644 --- a/tsdb/agent/db.go +++ b/tsdb/agent/db.go @@ -713,7 +713,7 @@ func (a *appender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v flo // Ensure no empty or duplicate labels have gotten through. This mirrors the // equivalent validation code in the TSDB's headAppender. l = l.WithoutEmpty() - if len(l) == 0 { + if l.IsEmpty() { return 0, errors.Wrap(tsdb.ErrInvalidSample, "empty labelset") } @@ -786,13 +786,17 @@ func (a *appender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exem // Exemplar label length does not include chars involved in text rendering such as quotes // equals sign, or commas. See definition of const ExemplarMaxLabelLength. labelSetLen := 0 - for _, l := range e.Labels { + err := e.Labels.Validate(func(l labels.Label) error { labelSetLen += utf8.RuneCountInString(l.Name) labelSetLen += utf8.RuneCountInString(l.Value) if labelSetLen > exemplar.ExemplarMaxLabelSetLength { - return 0, storage.ErrExemplarLabelLength + return storage.ErrExemplarLabelLength } + return nil + }) + if err != nil { + return 0, err } // Check for duplicate vs last stored exemplar for this series, and discard those. diff --git a/tsdb/agent/db_test.go b/tsdb/agent/db_test.go index c32e901e6..5933944de 100644 --- a/tsdb/agent/db_test.go +++ b/tsdb/agent/db_test.go @@ -49,28 +49,28 @@ func TestDB_InvalidSeries(t *testing.T) { _, err := app.Append(0, labels.Labels{}, 0, 0) require.ErrorIs(t, err, tsdb.ErrInvalidSample, "should reject empty labels") - _, err = app.Append(0, labels.Labels{{Name: "a", Value: "1"}, {Name: "a", Value: "2"}}, 0, 0) + _, err = app.Append(0, labels.FromStrings("a", "1", "a", "2"), 0, 0) require.ErrorIs(t, err, tsdb.ErrInvalidSample, "should reject duplicate labels") }) t.Run("Exemplars", func(t *testing.T) { - sRef, err := app.Append(0, labels.Labels{{Name: "a", Value: "1"}}, 0, 0) + sRef, err := app.Append(0, labels.FromStrings("a", "1"), 0, 0) require.NoError(t, err, "should not reject valid series") - _, err = app.AppendExemplar(0, nil, exemplar.Exemplar{}) + _, err = app.AppendExemplar(0, labels.EmptyLabels(), exemplar.Exemplar{}) require.EqualError(t, err, "unknown series ref when trying to add exemplar: 0") - e := exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}, {Name: "a", Value: "2"}}} - _, err = app.AppendExemplar(sRef, nil, e) + e := exemplar.Exemplar{Labels: labels.FromStrings("a", "1", "a", "2")} + _, err = app.AppendExemplar(sRef, labels.EmptyLabels(), e) require.ErrorIs(t, err, tsdb.ErrInvalidExemplar, "should reject duplicate labels") - e = exemplar.Exemplar{Labels: labels.Labels{{Name: "a_somewhat_long_trace_id", Value: "nYJSNtFrFTY37VR7mHzEE/LIDt7cdAQcuOzFajgmLDAdBSRHYPDzrxhMA4zz7el8naI/AoXFv9/e/G0vcETcIoNUi3OieeLfaIRQci2oa"}}} - _, err = app.AppendExemplar(sRef, nil, e) + e = exemplar.Exemplar{Labels: labels.FromStrings("a_somewhat_long_trace_id", "nYJSNtFrFTY37VR7mHzEE/LIDt7cdAQcuOzFajgmLDAdBSRHYPDzrxhMA4zz7el8naI/AoXFv9/e/G0vcETcIoNUi3OieeLfaIRQci2oa")} + _, err = app.AppendExemplar(sRef, labels.EmptyLabels(), e) require.ErrorIs(t, err, storage.ErrExemplarLabelLength, "should reject too long label length") // Inverse check - e = exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}}, Value: 20, Ts: 10, HasTs: true} - _, err = app.AppendExemplar(sRef, nil, e) + e = exemplar.Exemplar{Labels: labels.FromStrings("a", "1"), Value: 20, Ts: 10, HasTs: true} + _, err = app.AppendExemplar(sRef, labels.EmptyLabels(), e) require.NoError(t, err, "should not reject valid exemplars") }) } @@ -426,9 +426,7 @@ func Test_ExistingWAL_NextRef(t *testing.T) { // Append series app := db.Appender(context.Background()) for i := 0; i < seriesCount; i++ { - lset := labels.Labels{ - {Name: model.MetricNameLabel, Value: fmt.Sprintf("series_%d", i)}, - } + lset := labels.FromStrings(model.MetricNameLabel, fmt.Sprintf("series_%d", i)) _, err := app.Append(0, lset, 0, 100) require.NoError(t, err) } @@ -470,11 +468,11 @@ func startTime() (int64, error) { } // Create series for tests. -func labelsForTest(lName string, seriesCount int) []labels.Labels { - var series []labels.Labels +func labelsForTest(lName string, seriesCount int) [][]labels.Label { + var series [][]labels.Label for i := 0; i < seriesCount; i++ { - lset := labels.Labels{ + lset := []labels.Label{ {Name: "a", Value: lName}, {Name: "instance", Value: "localhost" + strconv.Itoa(i)}, {Name: "job", Value: "prometheus"}, @@ -507,28 +505,28 @@ func TestStorage_DuplicateExemplarsIgnored(t *testing.T) { app := s.Appender(context.Background()) defer s.Close() - sRef, err := app.Append(0, labels.Labels{{Name: "a", Value: "1"}}, 0, 0) + sRef, err := app.Append(0, labels.FromStrings("a", "1"), 0, 0) require.NoError(t, err, "should not reject valid series") // Write a few exemplars to our appender and call Commit(). // If the Labels, Value or Timestamp are different than the last exemplar, // then a new one should be appended; Otherwise, it should be skipped. - e := exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}}, Value: 20, Ts: 10, HasTs: true} - _, _ = app.AppendExemplar(sRef, nil, e) - _, _ = app.AppendExemplar(sRef, nil, e) + e := exemplar.Exemplar{Labels: labels.FromStrings("a", "1"), Value: 20, Ts: 10, HasTs: true} + _, _ = app.AppendExemplar(sRef, labels.EmptyLabels(), e) + _, _ = app.AppendExemplar(sRef, labels.EmptyLabels(), e) - e.Labels = labels.Labels{{Name: "b", Value: "2"}} - _, _ = app.AppendExemplar(sRef, nil, e) - _, _ = app.AppendExemplar(sRef, nil, e) - _, _ = app.AppendExemplar(sRef, nil, e) + e.Labels = labels.FromStrings("b", "2") + _, _ = app.AppendExemplar(sRef, labels.EmptyLabels(), e) + _, _ = app.AppendExemplar(sRef, labels.EmptyLabels(), e) + _, _ = app.AppendExemplar(sRef, labels.EmptyLabels(), e) e.Value = 42 - _, _ = app.AppendExemplar(sRef, nil, e) - _, _ = app.AppendExemplar(sRef, nil, e) + _, _ = app.AppendExemplar(sRef, labels.EmptyLabels(), e) + _, _ = app.AppendExemplar(sRef, labels.EmptyLabels(), e) e.Ts = 25 - _, _ = app.AppendExemplar(sRef, nil, e) - _, _ = app.AppendExemplar(sRef, nil, e) + _, _ = app.AppendExemplar(sRef, labels.EmptyLabels(), e) + _, _ = app.AppendExemplar(sRef, labels.EmptyLabels(), e) require.NoError(t, app.Commit()) From f0ec81baddf34cfb2c466816afed6764b52a8b67 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 20 Feb 2022 20:44:25 +0000 Subject: [PATCH 065/107] Update package tsdb/test for new labels.Labels type Signed-off-by: Bryan Boreham --- tsdb/test/labels_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tsdb/test/labels_test.go b/tsdb/test/labels_test.go index 354dbf836..f1ec34f3a 100644 --- a/tsdb/test/labels_test.go +++ b/tsdb/test/labels_test.go @@ -58,9 +58,7 @@ func BenchmarkLabelsClone(b *testing.B) { l := labels.FromMap(m) for i := 0; i < b.N; i++ { - res := make(labels.Labels, len(l)) - copy(res, l) - l = res + l = l.Copy() } } @@ -106,13 +104,13 @@ func BenchmarkLabelSetAccess(b *testing.B) { var v string - for _, l := range ls { + ls.Range(func(l labels.Label) { b.Run(l.Name, func(b *testing.B) { for i := 0; i < b.N; i++ { v = ls.Get(l.Name) } }) - } + }) _ = v } From 543c318ec2054e0be680ad28671b6d30efa314e2 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 9 Mar 2022 22:17:40 +0000 Subject: [PATCH 066/107] Update package tsdb for new labels.Labels type Signed-off-by: Bryan Boreham --- tsdb/db.go | 2 +- tsdb/exemplar.go | 5 ++++- tsdb/head_append.go | 8 ++++---- tsdb/head_read.go | 6 +++--- tsdb/ooo_head_read.go | 6 +++--- tsdb/querier.go | 5 +---- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tsdb/db.go b/tsdb/db.go index 7a165e431..616213b03 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -1002,7 +1002,7 @@ func (a dbAppender) GetRef(lset labels.Labels, hash uint64) (storage.SeriesRef, if g, ok := a.Appender.(storage.GetRef); ok { return g.GetRef(lset, hash) } - return 0, nil + return 0, labels.EmptyLabels() } func (a dbAppender) Commit() error { diff --git a/tsdb/exemplar.go b/tsdb/exemplar.go index 3718d9591..5ba3567e4 100644 --- a/tsdb/exemplar.go +++ b/tsdb/exemplar.go @@ -226,13 +226,16 @@ func (ce *CircularExemplarStorage) validateExemplar(key []byte, e exemplar.Exemp // Exemplar label length does not include chars involved in text rendering such as quotes // equals sign, or commas. See definition of const ExemplarMaxLabelLength. labelSetLen := 0 - for _, l := range e.Labels { + if err := e.Labels.Validate(func(l labels.Label) error { labelSetLen += utf8.RuneCountInString(l.Name) labelSetLen += utf8.RuneCountInString(l.Value) if labelSetLen > exemplar.ExemplarMaxLabelSetLength { return storage.ErrExemplarLabelLength } + return nil + }); err != nil { + return err } idx, ok := ce.index[string(key)] diff --git a/tsdb/head_append.go b/tsdb/head_append.go index d5003188d..822863276 100644 --- a/tsdb/head_append.go +++ b/tsdb/head_append.go @@ -102,7 +102,7 @@ func (a *initAppender) GetRef(lset labels.Labels, hash uint64) (storage.SeriesRe if g, ok := a.app.(storage.GetRef); ok { return g.GetRef(lset, hash) } - return 0, nil + return 0, labels.EmptyLabels() } func (a *initAppender) Commit() error { @@ -312,7 +312,7 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64 if s == nil { // Ensure no empty labels have gotten through. lset = lset.WithoutEmpty() - if len(lset) == 0 { + if lset.IsEmpty() { return 0, errors.Wrap(ErrInvalidSample, "empty labelset") } @@ -494,7 +494,7 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels if s == nil { // Ensure no empty labels have gotten through. lset = lset.WithoutEmpty() - if len(lset) == 0 { + if lset.IsEmpty() { return 0, errors.Wrap(ErrInvalidSample, "empty labelset") } @@ -650,7 +650,7 @@ var _ storage.GetRef = &headAppender{} func (a *headAppender) GetRef(lset labels.Labels, hash uint64) (storage.SeriesRef, labels.Labels) { s := a.head.series.getByHash(hash, lset) if s == nil { - return 0, nil + return 0, labels.EmptyLabels() } // returned labels must be suitable to pass to Append() return storage.SeriesRef(s.ref), s.lset diff --git a/tsdb/head_read.go b/tsdb/head_read.go index c6017e7b8..0a6aef2bd 100644 --- a/tsdb/head_read.go +++ b/tsdb/head_read.go @@ -155,7 +155,7 @@ func (h *headIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchB h.head.metrics.seriesNotFound.Inc() return storage.ErrNotFound } - *lbls = append((*lbls)[:0], s.lset...) + lbls.CopyFrom(s.lset) s.Lock() defer s.Unlock() @@ -222,9 +222,9 @@ func (h *headIndexReader) LabelNamesFor(ids ...storage.SeriesRef) ([]string, err if memSeries == nil { return nil, storage.ErrNotFound } - for _, lbl := range memSeries.lset { + memSeries.lset.Range(func(lbl labels.Label) { namesMap[lbl.Name] = struct{}{} - } + }) } names := make([]string, 0, len(namesMap)) for name := range namesMap { diff --git a/tsdb/ooo_head_read.go b/tsdb/ooo_head_read.go index f63607dc9..d0b09963d 100644 --- a/tsdb/ooo_head_read.go +++ b/tsdb/ooo_head_read.go @@ -47,7 +47,7 @@ func NewOOOHeadIndexReader(head *Head, mint, maxt int64) *OOOHeadIndexReader { return &OOOHeadIndexReader{hr} } -func (oh *OOOHeadIndexReader) Series(ref storage.SeriesRef, lbls *labels.Labels, chks *[]chunks.Meta) error { +func (oh *OOOHeadIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, lbls *labels.Labels, chks *[]chunks.Meta) error { return oh.series(ref, lbls, chks, 0) } @@ -61,7 +61,7 @@ func (oh *OOOHeadIndexReader) series(ref storage.SeriesRef, lbls *labels.Labels, oh.head.metrics.seriesNotFound.Inc() return storage.ErrNotFound } - *lbls = append((*lbls)[:0], s.lset...) + lbls.CopyFrom(s.lset) if chks == nil { return nil @@ -400,7 +400,7 @@ func (ir *OOOCompactionHeadIndexReader) SortedPostings(p index.Postings) index.P return p } -func (ir *OOOCompactionHeadIndexReader) Series(ref storage.SeriesRef, lset *labels.Labels, chks *[]chunks.Meta) error { +func (ir *OOOCompactionHeadIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, lset *labels.Labels, chks *[]chunks.Meta) error { return ir.ch.oooIR.series(ref, lset, chks, ir.ch.lastMmapRef) } diff --git a/tsdb/querier.go b/tsdb/querier.go index 4082279ae..34917dad0 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -529,8 +529,7 @@ func (b *blockBaseSeriesSet) Next() bool { intervals = intervals.Add(tombstones.Interval{Mint: b.maxt + 1, Maxt: math.MaxInt64}) } - b.curr.labels = make(labels.Labels, len(b.bufLbls)) - copy(b.curr.labels, b.bufLbls) + b.curr.labels = b.bufLbls.Copy() b.curr.chks = chks b.curr.intervals = intervals @@ -866,7 +865,6 @@ func newBlockSeriesSet(i IndexReader, c ChunkReader, t tombstones.Reader, p inde mint: mint, maxt: maxt, disableTrimming: disableTrimming, - bufLbls: make(labels.Labels, 0, 10), }, } } @@ -898,7 +896,6 @@ func newBlockChunkSeriesSet(id ulid.ULID, i IndexReader, c ChunkReader, t tombst mint: mint, maxt: maxt, disableTrimming: disableTrimming, - bufLbls: make(labels.Labels, 0, 10), }, } } From ce2cfad0cba11a37d1313b6b72675f622e4ce64a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 28 Jun 2022 16:02:08 +0100 Subject: [PATCH 067/107] Update package tsdb/record for new labels.Labels type Implement decoding via labels.ScratchBuilder, which we retain and re-use to reduce memory allocations. Signed-off-by: Bryan Boreham --- tsdb/record/record.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tsdb/record/record.go b/tsdb/record/record.go index 86cc93890..a1f05425f 100644 --- a/tsdb/record/record.go +++ b/tsdb/record/record.go @@ -17,7 +17,6 @@ package record import ( "math" - "sort" "github.com/pkg/errors" @@ -182,7 +181,9 @@ type RefMmapMarker struct { // Decoder decodes series, sample, metadata and tombstone records. // The zero value is ready to use. -type Decoder struct{} +type Decoder struct { + builder labels.ScratchBuilder +} // Type returns the type of the record. // Returns RecordUnknown if no valid record type is found. @@ -267,14 +268,15 @@ func (d *Decoder) Metadata(rec []byte, metadata []RefMetadata) ([]RefMetadata, e // DecodeLabels decodes one set of labels from buf. func (d *Decoder) DecodeLabels(dec *encoding.Decbuf) labels.Labels { - lset := make(labels.Labels, dec.Uvarint()) - - for i := range lset { - lset[i].Name = dec.UvarintStr() - lset[i].Value = dec.UvarintStr() + // TODO: reconsider if this function could be pushed down into labels.Labels to be more efficient. + d.builder.Reset() + nLabels := dec.Uvarint() + for i := 0; i < nLabels; i++ { + lName := dec.UvarintStr() + lValue := dec.UvarintStr() + d.builder.Add(lName, lValue) } - sort.Sort(lset) - return lset + return d.builder.Labels() } // Samples appends samples in rec to the given slice. @@ -525,12 +527,13 @@ func (e *Encoder) Metadata(metadata []RefMetadata, b []byte) []byte { // EncodeLabels encodes the contents of labels into buf. func EncodeLabels(buf *encoding.Encbuf, lbls labels.Labels) { - buf.PutUvarint(len(lbls)) + // TODO: reconsider if this function could be pushed down into labels.Labels to be more efficient. + buf.PutUvarint(lbls.Len()) - for _, l := range lbls { + lbls.Range(func(l labels.Label) { buf.PutUvarintStr(l.Name) buf.PutUvarintStr(l.Value) - } + }) } // Samples appends the encoded samples to b and returns the resulting slice. From 4b6a4d142510444730e3589a04f9d220e49ac03c Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 9 Mar 2022 22:17:29 +0000 Subject: [PATCH 068/107] Update package tsdb tests for new labels.Labels type Signed-off-by: Bryan Boreham --- tsdb/block_test.go | 42 +++++++++++++++++++------------------- tsdb/compact_test.go | 19 ++++++++--------- tsdb/db_test.go | 10 ++++----- tsdb/head_test.go | 24 ++++++++++++++-------- tsdb/ooo_head_read_test.go | 11 ++++++---- tsdb/querier_test.go | 27 +++++++++++++----------- 6 files changed, 72 insertions(+), 61 deletions(-) diff --git a/tsdb/block_test.go b/tsdb/block_test.go index c3a6ff576..380fa68bb 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -215,10 +215,10 @@ func TestLabelValuesWithMatchers(t *testing.T) { var seriesEntries []storage.Series for i := 0; i < 100; i++ { - seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{ - {Name: "tens", Value: fmt.Sprintf("value%d", i/10)}, - {Name: "unique", Value: fmt.Sprintf("value%d", i)}, - }, []tsdbutil.Sample{sample{100, 0, nil, nil}})) + seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings( + "tens", fmt.Sprintf("value%d", i/10), + "unique", fmt.Sprintf("value%d", i), + ), []tsdbutil.Sample{sample{100, 0, nil, nil}})) } blockDir := createBlock(t, tmpdir, seriesEntries) @@ -372,11 +372,11 @@ func BenchmarkLabelValuesWithMatchers(b *testing.B) { for i := 0; i < metricCount; i++ { // Note these series are not created in sort order: 'value2' sorts after 'value10'. // This makes a big difference to the benchmark timing. - seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{ - {Name: "a_unique", Value: fmt.Sprintf("value%d", i)}, - {Name: "b_tens", Value: fmt.Sprintf("value%d", i/(metricCount/10))}, - {Name: "c_ninety", Value: fmt.Sprintf("value%d", i/(metricCount/10)/9)}, // "0" for the first 90%, then "1" - }, []tsdbutil.Sample{sample{100, 0, nil, nil}})) + seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings( + "a_unique", fmt.Sprintf("value%d", i), + "b_tens", fmt.Sprintf("value%d", i/(metricCount/10)), + "c_ninety", fmt.Sprintf("value%d", i/(metricCount/10)/9), // "0" for the first 90%, then "1" + ), []tsdbutil.Sample{sample{100, 0, nil, nil}})) } blockDir := createBlock(b, tmpdir, seriesEntries) @@ -410,23 +410,23 @@ func TestLabelNamesWithMatchers(t *testing.T) { var seriesEntries []storage.Series for i := 0; i < 100; i++ { - seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{ - {Name: "unique", Value: fmt.Sprintf("value%d", i)}, - }, []tsdbutil.Sample{sample{100, 0, nil, nil}})) + seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings( + "unique", fmt.Sprintf("value%d", i), + ), []tsdbutil.Sample{sample{100, 0, nil, nil}})) if i%10 == 0 { - seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{ - {Name: "tens", Value: fmt.Sprintf("value%d", i/10)}, - {Name: "unique", Value: fmt.Sprintf("value%d", i)}, - }, []tsdbutil.Sample{sample{100, 0, nil, nil}})) + seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings( + "tens", fmt.Sprintf("value%d", i/10), + "unique", fmt.Sprintf("value%d", i), + ), []tsdbutil.Sample{sample{100, 0, nil, nil}})) } if i%20 == 0 { - seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{ - {Name: "tens", Value: fmt.Sprintf("value%d", i/10)}, - {Name: "twenties", Value: fmt.Sprintf("value%d", i/20)}, - {Name: "unique", Value: fmt.Sprintf("value%d", i)}, - }, []tsdbutil.Sample{sample{100, 0, nil, nil}})) + seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings( + "tens", fmt.Sprintf("value%d", i/10), + "twenties", fmt.Sprintf("value%d", i/20), + "unique", fmt.Sprintf("value%d", i), + ), []tsdbutil.Sample{sample{100, 0, nil, nil}})) } } diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go index a77d070e8..e7fe998ac 100644 --- a/tsdb/compact_test.go +++ b/tsdb/compact_test.go @@ -1478,11 +1478,11 @@ func TestSparseHistogramSpaceSavings(t *testing.T) { for sid, schema := range allSchemas { for i := 0; i < c.numSeriesPerSchema; i++ { - lbls := labels.Labels{ - {Name: "__name__", Value: fmt.Sprintf("rpc_durations_%d_histogram_seconds", i)}, - {Name: "instance", Value: "localhost:8080"}, - {Name: "job", Value: fmt.Sprintf("sparse_histogram_schema_%s", schemaDescription[sid])}, - } + lbls := labels.FromStrings( + "__name__", fmt.Sprintf("rpc_durations_%d_histogram_seconds", i), + "instance", "localhost:8080", + "job", fmt.Sprintf("sparse_histogram_schema_%s", schemaDescription[sid]), + ) allSparseSeries = append(allSparseSeries, struct { baseLabels labels.Labels hists []*histogram.Histogram @@ -1546,21 +1546,20 @@ func TestSparseHistogramSpaceSavings(t *testing.T) { for it.Next() { numOldSeriesPerHistogram++ b := it.At() - lbls := append(ah.baseLabels, labels.Label{Name: "le", Value: fmt.Sprintf("%.16f", b.Upper)}) + lbls := labels.NewBuilder(ah.baseLabels).Set("le", fmt.Sprintf("%.16f", b.Upper)).Labels(labels.EmptyLabels()) refs[itIdx], err = oldApp.Append(refs[itIdx], lbls, ts, float64(b.Count)) require.NoError(t, err) itIdx++ } + baseName := ah.baseLabels.Get(labels.MetricName) // _count metric. - countLbls := ah.baseLabels.Copy() - countLbls[0].Value = countLbls[0].Value + "_count" + countLbls := labels.NewBuilder(ah.baseLabels).Set(labels.MetricName, baseName+"_count").Labels(labels.EmptyLabels()) _, err = oldApp.Append(0, countLbls, ts, float64(h.Count)) require.NoError(t, err) numOldSeriesPerHistogram++ // _sum metric. - sumLbls := ah.baseLabels.Copy() - sumLbls[0].Value = sumLbls[0].Value + "_sum" + sumLbls := labels.NewBuilder(ah.baseLabels).Set(labels.MetricName, baseName+"_sum").Labels(labels.EmptyLabels()) _, err = oldApp.Append(0, sumLbls, ts, h.Sum) require.NoError(t, err) numOldSeriesPerHistogram++ diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 7eb3e272e..1d95cfe00 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -478,9 +478,9 @@ func TestAmendDatapointCausesError(t *testing.T) { require.NoError(t, app.Commit()) app = db.Appender(ctx) - _, err = app.Append(0, labels.Labels{{Name: "a", Value: "b"}}, 0, 0) + _, err = app.Append(0, labels.FromStrings("a", "b"), 0, 0) require.NoError(t, err) - _, err = app.Append(0, labels.Labels{{Name: "a", Value: "b"}}, 0, 1) + _, err = app.Append(0, labels.FromStrings("a", "b"), 0, 1) require.Equal(t, storage.ErrDuplicateSampleForTimestamp, err) require.NoError(t, app.Rollback()) @@ -498,15 +498,15 @@ func TestAmendDatapointCausesError(t *testing.T) { } app = db.Appender(ctx) - _, err = app.AppendHistogram(0, labels.Labels{{Name: "a", Value: "c"}}, 0, h.Copy()) + _, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy()) require.NoError(t, err) require.NoError(t, app.Commit()) app = db.Appender(ctx) - _, err = app.AppendHistogram(0, labels.Labels{{Name: "a", Value: "c"}}, 0, h.Copy()) + _, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy()) require.NoError(t, err) h.Schema = 2 - _, err = app.AppendHistogram(0, labels.Labels{{Name: "a", Value: "c"}}, 0, h.Copy()) + _, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy()) require.Equal(t, storage.ErrDuplicateSampleForTimestamp, err) require.NoError(t, app.Rollback()) } diff --git a/tsdb/head_test.go b/tsdb/head_test.go index f10dc9f64..cc0133b9d 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -388,7 +388,12 @@ func TestHead_HighConcurrencyReadAndWrite(t *testing.T) { querySeriesRef = (querySeriesRef + 1) % seriesCnt lbls := labelSets[querySeriesRef] - samples, err := queryHead(ts-qryRange, ts, lbls[0]) + // lbls has a single entry; extract it so we can run a query. + var lbl labels.Label + lbls.Range(func(l labels.Label) { + lbl = l + }) + samples, err := queryHead(ts-qryRange, ts, lbl) if err != nil { return false, err } @@ -1133,8 +1138,9 @@ func TestDelete_e2e(t *testing.T) { require.NoError(t, hb.Delete(r.Mint, r.Maxt, del.ms...)) } matched := labels.Slice{} - for _, ls := range lbls { + for _, l := range lbls { s := labels.Selector(del.ms) + ls := labels.New(l...) if s.Matches(ls) { matched = append(matched, ls) } @@ -2808,7 +2814,7 @@ func TestWaitForPendingReadersInTimeRange(t *testing.T) { } func TestAppendHistogram(t *testing.T) { - l := labels.Labels{{Name: "a", Value: "b"}} + l := labels.FromStrings("a", "b") for _, numHistograms := range []int{1, 10, 150, 200, 250, 300} { t.Run(fmt.Sprintf("%d", numHistograms), func(t *testing.T) { head, _ := newTestHead(t, 1000, false, false) @@ -2863,7 +2869,7 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { require.NoError(t, head.Init(0)) // Series with only histograms. - s1 := labels.Labels{{Name: "a", Value: "b1"}} + s1 := labels.FromStrings("a", "b1") k1 := s1.String() numHistograms := 450 exp := map[string][]tsdbutil.Sample{} @@ -2895,7 +2901,7 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { require.Greater(t, expHeadChunkSamples, 0) // Series with mix of histograms and float. - s2 := labels.Labels{{Name: "a", Value: "b2"}} + s2 := labels.FromStrings("a", "b2") k2 := s2.String() app = head.Appender(context.Background()) ts := 0 @@ -3256,7 +3262,7 @@ func TestHistogramMetrics(t *testing.T) { for x := 0; x < 5; x++ { expHSeries++ - l := labels.Labels{{Name: "a", Value: fmt.Sprintf("b%d", x)}} + l := labels.FromStrings("a", fmt.Sprintf("b%d", x)) for i, h := range GenerateTestHistograms(10) { app := head.Appender(context.Background()) _, err := app.AppendHistogram(0, l, int64(i), h) @@ -3279,7 +3285,7 @@ func TestHistogramMetrics(t *testing.T) { } func TestHistogramStaleSample(t *testing.T) { - l := labels.Labels{{Name: "a", Value: "b"}} + l := labels.FromStrings("a", "b") numHistograms := 20 head, _ := newTestHead(t, 100000, false, false) t.Cleanup(func() { @@ -3374,7 +3380,7 @@ func TestHistogramStaleSample(t *testing.T) { } func TestHistogramCounterResetHeader(t *testing.T) { - l := labels.Labels{{Name: "a", Value: "b"}} + l := labels.FromStrings("a", "b") head, _ := newTestHead(t, 1000, false, false) t.Cleanup(func() { require.NoError(t, head.Close()) @@ -3486,7 +3492,7 @@ func TestAppendingDifferentEncodingToSameSeries(t *testing.T) { db.DisableCompactions() hists := GenerateTestHistograms(10) - lbls := labels.Labels{{Name: "a", Value: "b"}} + lbls := labels.FromStrings("a", "b") type result struct { t int64 diff --git a/tsdb/ooo_head_read_test.go b/tsdb/ooo_head_read_test.go index b489c6e57..c48e95e46 100644 --- a/tsdb/ooo_head_read_test.go +++ b/tsdb/ooo_head_read_test.go @@ -358,12 +358,13 @@ func TestOOOHeadIndexReader_Series(t *testing.T) { var chks []chunks.Meta var respLset labels.Labels - err := ir.Series(storage.SeriesRef(s1ID), &respLset, &chks) + var b labels.ScratchBuilder + err := ir.Series(storage.SeriesRef(s1ID), &b, &respLset, &chks) require.NoError(t, err) require.Equal(t, s1Lset, respLset) require.Equal(t, expChunks, chks) - err = ir.Series(storage.SeriesRef(s1ID+1), &respLset, &chks) + err = ir.Series(storage.SeriesRef(s1ID+1), &b, &respLset, &chks) require.Equal(t, storage.ErrNotFound, err) }) } @@ -841,7 +842,8 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) { ir := NewOOOHeadIndexReader(db.head, tc.queryMinT, tc.queryMaxT) var chks []chunks.Meta var respLset labels.Labels - err := ir.Series(s1Ref, &respLset, &chks) + var b labels.ScratchBuilder + err := ir.Series(s1Ref, &b, &respLset, &chks) require.NoError(t, err) require.Equal(t, len(tc.expChunksSamples), len(chks)) @@ -1004,7 +1006,8 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding( ir := NewOOOHeadIndexReader(db.head, tc.queryMinT, tc.queryMaxT) var chks []chunks.Meta var respLset labels.Labels - err := ir.Series(s1Ref, &respLset, &chks) + var b labels.ScratchBuilder + err := ir.Series(s1Ref, &b, &respLset, &chks) require.NoError(t, err) require.Equal(t, len(tc.expChunksSamples), len(chks)) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 9933b7158..eb0c847fd 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -142,14 +142,14 @@ func createIdxChkReaders(t *testing.T, tc []seriesSamples) (IndexReader, ChunkRe postings.Add(storage.SeriesRef(i), ls) - for _, l := range ls { + ls.Range(func(l labels.Label) { vs, present := lblIdx[l.Name] if !present { vs = map[string]struct{}{} lblIdx[l.Name] = vs } vs[l.Value] = struct{}{} - } + }) } require.NoError(t, postings.Iter(func(l labels.Label, p index.Postings) error { @@ -1168,10 +1168,10 @@ func (m *mockIndex) AddSeries(ref storage.SeriesRef, l labels.Labels, chunks ... if _, ok := m.series[ref]; ok { return errors.Errorf("series with reference %d already added", ref) } - for _, lbl := range l { + l.Range(func(lbl labels.Label) { m.symbols[lbl.Name] = struct{}{} m.symbols[lbl.Value] = struct{}{} - } + }) s := series{l: l} // Actual chunk data is not stored in the index. @@ -1238,9 +1238,9 @@ func (m mockIndex) LabelValueFor(id storage.SeriesRef, label string) (string, er func (m mockIndex) LabelNamesFor(ids ...storage.SeriesRef) ([]string, error) { namesMap := make(map[string]bool) for _, id := range ids { - for _, lbl := range m.series[id].l { + m.series[id].l.Range(func(lbl labels.Label) { namesMap[lbl.Name] = true - } + }) } names := make([]string, 0, len(namesMap)) for name := range namesMap { @@ -1275,7 +1275,7 @@ func (m mockIndex) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, if !ok { return storage.ErrNotFound } - *lset = append((*lset)[:0], s.l...) + lset.CopyFrom(s.l) *chks = append((*chks)[:0], s.chunks...) return nil @@ -1297,9 +1297,9 @@ func (m mockIndex) LabelNames(matchers ...*labels.Matcher) ([]string, error) { } } if matches { - for _, lbl := range series.l { + series.l.Range(func(lbl labels.Label) { names[lbl.Name] = struct{}{} - } + }) } } } @@ -1974,7 +1974,7 @@ func BenchmarkQueries(b *testing.B) { // Add some common labels to make the matchers select these series. { - var commonLbls labels.Labels + var commonLbls []labels.Label for _, selector := range selectors { switch selector.Type { case labels.MatchEqual: @@ -1985,8 +1985,11 @@ func BenchmarkQueries(b *testing.B) { } for i := range commonLbls { s := series[i].(*storage.SeriesEntry) - allLabels := append(commonLbls, s.Labels()...) - newS := storage.NewListSeries(allLabels, nil) + allLabels := commonLbls + s.Labels().Range(func(l labels.Label) { + allLabels = append(allLabels, l) + }) + newS := storage.NewListSeries(labels.New(allLabels...), nil) newS.SampleIteratorFn = s.SampleIteratorFn series[i] = newS From d6b97f631a74f73afbde6ac6796cd9a07430a437 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 9 Mar 2022 22:21:57 +0000 Subject: [PATCH 069/107] Update package storage for new labels.Labels type Signed-off-by: Bryan Boreham --- storage/merge.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/merge.go b/storage/merge.go index 78a0125db..ba45b12bd 100644 --- a/storage/merge.go +++ b/storage/merge.go @@ -699,7 +699,7 @@ func (c *compactChunkIterator) Next() bool { // 1:1 duplicates, skip it. } else { // We operate on same series, so labels does not matter here. - overlapping = append(overlapping, newChunkToSeriesDecoder(nil, next)) + overlapping = append(overlapping, newChunkToSeriesDecoder(labels.EmptyLabels(), next)) if next.MaxTime > oMaxTime { oMaxTime = next.MaxTime } @@ -716,7 +716,7 @@ func (c *compactChunkIterator) Next() bool { } // Add last as it's not yet included in overlap. We operate on same series, so labels does not matter here. - iter = NewSeriesToChunkEncoder(c.mergeFunc(append(overlapping, newChunkToSeriesDecoder(nil, c.curr))...)).Iterator(nil) + iter = NewSeriesToChunkEncoder(c.mergeFunc(append(overlapping, newChunkToSeriesDecoder(labels.EmptyLabels(), c.curr))...)).Iterator(nil) if !iter.Next() { if c.err = iter.Err(); c.err != nil { return false From 56fefcd8120b6ce9df925bd299fb23660fdda4c0 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 9 Mar 2022 22:14:58 +0000 Subject: [PATCH 070/107] Update package promql for new labels.Labels type We use `labels.Builder` to parse metrics, to avoid depending on the internal implementation. This is not efficient, but the feature is only used in tests. It wasn't efficient previously either - calling `Sort()` after adding each label. `createLabelsForAbsentFunction` also uses a Builder now, and gets an extra `map` to replace the previous `Has()` usage. Signed-off-by: Bryan Boreham Fix up promql to compile with changes to Labels --- promql/engine.go | 12 ++++++------ promql/functions.go | 27 ++++++++++++++------------- promql/parser/generated_parser.y | 10 +++++----- promql/parser/generated_parser.y.go | 29 +++++++++++++++-------------- promql/test.go | 4 ++-- 5 files changed, 42 insertions(+), 40 deletions(-) diff --git a/promql/engine.go b/promql/engine.go index 0225f78d2..0455779a4 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -1567,7 +1567,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { case *parser.NumberLiteral: return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { - return append(enh.Out, Sample{Point: Point{V: e.Val}}), nil + return append(enh.Out, Sample{Point: Point{V: e.Val}, Metric: labels.EmptyLabels()}), nil }) case *parser.StringLiteral: @@ -2190,7 +2190,7 @@ func resultMetric(lhs, rhs labels.Labels, op parser.ItemType, matching *parser.V } } - ret := enh.lb.Labels(nil) + ret := enh.lb.Labels(labels.EmptyLabels()) enh.resultMetric[str] = ret return ret } @@ -2230,7 +2230,7 @@ func (ev *evaluator) VectorscalarBinop(op parser.ItemType, lhs Vector, rhs Scala } func dropMetricName(l labels.Labels) labels.Labels { - return labels.NewBuilder(l).Del(labels.MetricName).Labels(nil) + return labels.NewBuilder(l).Del(labels.MetricName).Labels(labels.EmptyLabels()) } // scalarBinop evaluates a binary operation between two Scalars. @@ -2357,7 +2357,7 @@ func (ev *evaluator) aggregation(op parser.ItemType, grouping []string, without } } - lb := labels.NewBuilder(nil) + lb := labels.NewBuilder(labels.EmptyLabels()) var buf []byte for si, s := range vec { metric := s.Metric @@ -2365,7 +2365,7 @@ func (ev *evaluator) aggregation(op parser.ItemType, grouping []string, without if op == parser.COUNT_VALUES { lb.Reset(metric) lb.Set(valueLabel, strconv.FormatFloat(s.V, 'f', -1, 64)) - metric = lb.Labels(nil) + metric = lb.Labels(labels.EmptyLabels()) // We've changed the metric so we have to recompute the grouping key. recomputeGroupingKey = true @@ -2389,7 +2389,7 @@ func (ev *evaluator) aggregation(op parser.ItemType, grouping []string, without } else { lb.Keep(grouping...) } - m := lb.Labels(nil) + m := lb.Labels(labels.EmptyLabels()) newAgg := &groupedAggregation{ labels: m, value: s.V, diff --git a/promql/functions.go b/promql/functions.go index d481cb735..c5922002b 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -957,7 +957,7 @@ func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *Ev if !ok { sample.Metric = labels.NewBuilder(sample.Metric). Del(excludedLabels...). - Labels(nil) + Labels(labels.EmptyLabels()) mb = &metricWithBuckets{sample.Metric, nil} enh.signatureToMetricWithBuckets[string(enh.lblBuf)] = mb @@ -1077,7 +1077,7 @@ func funcLabelReplace(vals []parser.Value, args parser.Expressions, enh *EvalNod if len(res) > 0 { lb.Set(dst, string(res)) } - outMetric = lb.Labels(nil) + outMetric = lb.Labels(labels.EmptyLabels()) enh.Dmn[h] = outMetric } } @@ -1145,7 +1145,7 @@ func funcLabelJoin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHe lb.Set(dst, strval) } - outMetric = lb.Labels(nil) + outMetric = lb.Labels(labels.EmptyLabels()) enh.Dmn[h] = outMetric } @@ -1383,7 +1383,7 @@ func (s *vectorByReverseValueHeap) Pop() interface{} { // createLabelsForAbsentFunction returns the labels that are uniquely and exactly matched // in a given expression. It is used in the absent functions. func createLabelsForAbsentFunction(expr parser.Expr) labels.Labels { - m := labels.Labels{} + b := labels.NewBuilder(labels.EmptyLabels()) var lm []*labels.Matcher switch n := expr.(type) { @@ -1392,25 +1392,26 @@ func createLabelsForAbsentFunction(expr parser.Expr) labels.Labels { case *parser.MatrixSelector: lm = n.VectorSelector.(*parser.VectorSelector).LabelMatchers default: - return m + return labels.EmptyLabels() } - empty := []string{} + // The 'has' map implements backwards-compatibility for historic behaviour: + // e.g. in `absent(x{job="a",job="b",foo="bar"})` then `job` is removed from the output. + // Note this gives arguably wrong behaviour for `absent(x{job="a",job="a",foo="bar"})`. + has := make(map[string]bool, len(lm)) for _, ma := range lm { if ma.Name == labels.MetricName { continue } - if ma.Type == labels.MatchEqual && !m.Has(ma.Name) { - m = labels.NewBuilder(m).Set(ma.Name, ma.Value).Labels(nil) + if ma.Type == labels.MatchEqual && !has[ma.Name] { + b.Set(ma.Name, ma.Value) + has[ma.Name] = true } else { - empty = append(empty, ma.Name) + b.Del(ma.Name) } } - for _, v := range empty { - m = labels.NewBuilder(m).Del(v).Labels(nil) - } - return m + return b.Labels(labels.EmptyLabels()) } func stringFromArg(e parser.Expr) string { diff --git a/promql/parser/generated_parser.y b/promql/parser/generated_parser.y index 433f45259..461e854ac 100644 --- a/promql/parser/generated_parser.y +++ b/promql/parser/generated_parser.y @@ -16,13 +16,13 @@ package parser import ( "math" - "sort" "strconv" "time" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/value" ) + %} %union { @@ -32,6 +32,7 @@ import ( matcher *labels.Matcher label labels.Label labels labels.Labels + lblList []labels.Label strings []string series []SequenceValue uint uint64 @@ -138,10 +139,9 @@ START_METRIC_SELECTOR // Type definitions for grammar rules. %type label_match_list %type label_matcher - %type aggregate_op grouping_label match_op maybe_label metric_identifier unary_op at_modifier_preprocessors - -%type label_set label_set_list metric +%type label_set metric +%type label_set_list %type