2015-08-20 04:03:56 -07:00
// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2013-01-07 14:24:26 -08:00
package config
import (
2022-05-23 00:54:32 -07:00
"errors"
2013-01-07 14:24:26 -08:00
"fmt"
2024-09-09 18:41:53 -07:00
"log/slog"
2024-10-18 08:12:31 -07:00
"mime"
2015-06-22 13:35:19 -07:00
"net/url"
2021-03-25 14:28:58 -07:00
"os"
2015-08-05 09:04:34 -07:00
"path/filepath"
2023-10-10 03:16:55 -07:00
"sort"
2024-04-01 08:34:35 -07:00
"strconv"
2015-04-20 03:24:25 -07:00
"strings"
2013-01-07 14:24:26 -08:00
"time"
2014-12-10 08:46:56 -08:00
2021-05-15 19:19:22 -07:00
"github.com/alecthomas/units"
2022-02-12 15:58:27 -08:00
"github.com/grafana/regexp"
2020-08-20 05:48:26 -07:00
"github.com/prometheus/common/config"
2015-08-20 08:18:46 -07:00
"github.com/prometheus/common/model"
2024-11-06 02:46:17 -08:00
"github.com/prometheus/sigv4"
2022-08-31 06:50:38 -07:00
"gopkg.in/yaml.v2"
2019-03-25 16:01:12 -07:00
2020-08-20 05:48:26 -07:00
"github.com/prometheus/prometheus/discovery"
2021-11-08 06:23:17 -08:00
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel"
2023-06-01 14:20:10 -07:00
"github.com/prometheus/prometheus/storage/remote/azuread"
2024-07-30 08:25:19 -07:00
"github.com/prometheus/prometheus/storage/remote/googleiam"
2013-01-07 14:24:26 -08:00
)
2015-05-13 02:28:04 -07:00
var (
2021-02-18 04:12:21 -08:00
patRulePath = regexp . MustCompile ( ` ^[^*]*(\*[^/]*)?$ ` )
reservedHeaders = map [ string ] struct { } {
2021-02-04 13:18:13 -08:00
// NOTE: authorization is checked specially,
// see RemoteWriteConfig.UnmarshalYAML.
// "authorization": {},
"host" : { } ,
"content-encoding" : { } ,
2021-02-18 04:12:21 -08:00
"content-length" : { } ,
2021-02-04 13:18:13 -08:00
"content-type" : { } ,
"user-agent" : { } ,
"connection" : { } ,
"keep-alive" : { } ,
"proxy-authenticate" : { } ,
"proxy-authorization" : { } ,
"www-authenticate" : { } ,
2021-02-18 04:12:21 -08:00
"accept-encoding" : { } ,
"x-prometheus-remote-write-version" : { } ,
"x-prometheus-remote-read-version" : { } ,
2021-03-08 11:20:09 -08:00
// Added by SigV4.
"x-amz-date" : { } ,
"x-amz-security-token" : { } ,
"x-amz-content-sha256" : { } ,
2021-02-04 13:18:13 -08:00
}
2015-05-13 02:28:04 -07:00
)
2013-01-07 14:24:26 -08:00
2024-07-18 11:08:21 -07:00
const (
LegacyValidationConfig = "legacy"
UTF8ValidationConfig = "utf8"
)
2015-05-07 01:55:03 -07:00
// Load parses the YAML input s into a Config.
2024-10-17 01:25:05 -07:00
func Load ( s string , logger * slog . Logger ) ( * Config , error ) {
2015-06-23 10:40:44 -07:00
cfg := & Config { }
2015-07-17 07:12:33 -07:00
// If the entire config body is empty the UnmarshalYAML method is
// never called. We thus have to set the DefaultConfig at the entry
// point as well.
* cfg = DefaultConfig
2018-04-04 01:07:39 -07:00
err := yaml . UnmarshalStrict ( [ ] byte ( s ) , cfg )
2015-05-07 01:55:03 -07:00
if err != nil {
return nil , err
}
2021-03-25 14:28:58 -07:00
2023-04-30 04:52:34 -07:00
b := labels . NewScratchBuilder ( 0 )
2022-03-09 14:26:05 -08:00
cfg . GlobalConfig . ExternalLabels . Range ( func ( v labels . Label ) {
2021-03-25 14:28:58 -07:00
newV := os . Expand ( v . Value , func ( s string ) string {
2022-01-17 07:43:55 -08:00
if s == "$" {
return "$"
}
2021-03-25 14:28:58 -07:00
if v := os . Getenv ( s ) ; v != "" {
return v
}
2024-09-09 18:41:53 -07:00
logger . Warn ( "Empty environment variable" , "name" , s )
2021-03-25 14:28:58 -07:00
return ""
} )
if newV != v . Value {
2024-09-09 18:41:53 -07:00
logger . Debug ( "External label replaced" , "label" , v . Name , "input" , v . Value , "output" , newV )
2021-03-25 14:28:58 -07:00
}
2023-04-30 04:52:34 -07:00
// Note newV can be blank. https://github.com/prometheus/prometheus/issues/11024
2022-03-09 14:26:05 -08:00
b . Add ( v . Name , newV )
} )
2024-10-17 01:25:05 -07:00
if ! b . Labels ( ) . IsEmpty ( ) {
cfg . GlobalConfig . ExternalLabels = b . Labels ( )
}
2024-11-08 05:02:48 -08:00
switch cfg . OTLPConfig . TranslationStrategy {
case UnderscoreEscapingWithSuffixes :
case "" :
case NoUTF8EscapingWithSuffixes :
if cfg . GlobalConfig . MetricNameValidationScheme == LegacyValidationConfig {
return nil , errors . New ( "OTLP translation strategy NoUTF8EscapingWithSuffixes is not allowed when UTF8 is disabled" )
}
default :
return nil , fmt . Errorf ( "unsupported OTLP translation strategy %q" , cfg . OTLPConfig . TranslationStrategy )
}
2015-05-07 01:55:03 -07:00
return cfg , nil
2013-01-07 14:24:26 -08:00
}
2015-08-05 09:30:37 -07:00
// LoadFile parses the given YAML file into a Config.
2024-10-17 01:25:05 -07:00
func LoadFile ( filename string , agentMode bool , logger * slog . Logger ) ( * Config , error ) {
2022-04-27 02:24:36 -07:00
content , err := os . ReadFile ( filename )
2015-05-07 01:55:03 -07:00
if err != nil {
return nil , err
}
2024-10-17 01:25:05 -07:00
cfg , err := Load ( string ( content ) , logger )
2015-08-05 09:04:34 -07:00
if err != nil {
2022-05-23 00:54:32 -07:00
return nil , fmt . Errorf ( "parsing YAML file %s: %w" , filename , err )
2015-08-05 09:04:34 -07:00
}
2021-10-29 16:41:40 -07:00
if agentMode {
if len ( cfg . AlertingConfig . AlertmanagerConfigs ) > 0 || len ( cfg . AlertingConfig . AlertRelabelConfigs ) > 0 {
return nil , errors . New ( "field alerting is not allowed in agent mode" )
}
if len ( cfg . RuleFiles ) > 0 {
return nil , errors . New ( "field rule_files is not allowed in agent mode" )
}
if len ( cfg . RemoteReadConfigs ) > 0 {
return nil , errors . New ( "field remote_read is not allowed in agent mode" )
}
}
2020-08-20 05:48:26 -07:00
cfg . SetDirectory ( filepath . Dir ( filename ) )
2015-08-05 09:04:34 -07:00
return cfg , nil
2013-01-07 14:24:26 -08:00
}
2015-05-07 01:55:03 -07:00
// The defaults applied before parsing the respective config sections.
var (
2015-08-24 06:07:27 -07:00
// DefaultConfig is the default top-level configuration.
2015-06-04 08:03:12 -07:00
DefaultConfig = Config {
2015-06-07 08:40:22 -07:00
GlobalConfig : DefaultGlobalConfig ,
2013-04-30 11:20:14 -07:00
}
2015-05-07 01:55:03 -07:00
2015-08-24 06:07:27 -07:00
// DefaultGlobalConfig is the default global configuration.
2015-06-04 08:03:12 -07:00
DefaultGlobalConfig = GlobalConfig {
2016-01-29 06:23:11 -08:00
ScrapeInterval : model . Duration ( 1 * time . Minute ) ,
ScrapeTimeout : model . Duration ( 10 * time . Second ) ,
EvaluationInterval : model . Duration ( 1 * time . Minute ) ,
2024-05-30 03:49:50 -07:00
RuleQueryOffset : model . Duration ( 0 * time . Minute ) ,
2023-10-10 03:16:55 -07:00
// When native histogram feature flag is enabled, ScrapeProtocols default
// changes to DefaultNativeHistogramScrapeProtocols.
ScrapeProtocols : DefaultScrapeProtocols ,
2013-01-07 14:24:26 -08:00
}
2024-04-01 08:34:35 -07:00
DefaultRuntimeConfig = RuntimeConfig {
// Go runtime tuning.
2024-06-10 00:34:55 -07:00
GoGC : 75 ,
2024-04-01 08:34:35 -07:00
}
2015-08-24 06:07:27 -07:00
// DefaultScrapeConfig is the default scrape configuration.
2015-06-04 08:03:12 -07:00
DefaultScrapeConfig = ScrapeConfig {
2023-10-10 03:16:55 -07:00
// ScrapeTimeout, ScrapeInterval and ScrapeProtocols default to the configured globals.
2024-10-18 00:32:15 -07:00
AlwaysScrapeClassicHistograms : false ,
MetricsPath : "/metrics" ,
Scheme : "http" ,
HonorLabels : false ,
HonorTimestamps : true ,
HTTPClientConfig : config . DefaultHTTPClientConfig ,
EnableCompression : true ,
2015-04-20 03:24:25 -07:00
}
2015-05-07 01:55:03 -07:00
2016-11-25 02:04:33 -08:00
// DefaultAlertmanagerConfig is the default alertmanager configuration.
DefaultAlertmanagerConfig = AlertmanagerConfig {
2021-02-26 13:48:06 -08:00
Scheme : "http" ,
Timeout : model . Duration ( 10 * time . Second ) ,
2021-03-19 10:19:12 -07:00
APIVersion : AlertmanagerAPIVersionV2 ,
2021-02-26 13:48:06 -08:00
HTTPClientConfig : config . DefaultHTTPClientConfig ,
2016-11-23 03:41:19 -08:00
}
2024-10-24 06:07:54 -07:00
DefaultRemoteWriteHTTPClientConfig = config . HTTPClientConfig {
FollowRedirects : true ,
EnableHTTP2 : false ,
}
2016-09-19 13:47:51 -07:00
// DefaultRemoteWriteConfig is the default remote write configuration.
DefaultRemoteWriteConfig = RemoteWriteConfig {
2021-02-26 13:48:06 -08:00
RemoteTimeout : model . Duration ( 30 * time . Second ) ,
[PRW 2.0] Merging `remote-write-2.0` feature branch to main (PRW 2.0 support + metadata in WAL) (#14395)
* Remote Write 1.1: e2e benchmarks (#13102)
* Remote Write e2e benchmarks
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Prometheus ports automatically assigned
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make dashboard editable + more modular to different job label values
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Dashboard improvements
* memory stats
* diffs look at counter increases
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* run script: absolute path for config templates
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* grafana dashboard improvements
* show actual values of metrics
* add memory stats and diff
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* dashboard changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
* replace snappy encoding library
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto types
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add decode function for new write request proto
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add lookup table struct that is used to build the symbol table in new
write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Implement code paths for new proto format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example server to include handler for new format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add new test client
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* tests and new -> original proto mapping util
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto support on receiver end
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Fix test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* no-brainer copypaste but more performance write support
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove some comented code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix mocks and fixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add basic reduce remote write handler benchmark
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor out common code between write methods
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix: queue manager to include float histograms in new requests
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add sender-side tests and fix failing ones
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor queue manager code to remove some duplication
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Improve sender benchmarks and some allocations
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use github.com/golang/snappy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup: remove hardcoded fake url for testing
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add 1.1 version handling code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Remove config, update proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* gofmt
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix NewWriteClient and change new flags wording
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fields rewording in handler
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remote write handler to checks version header
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix typo in log
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add minmized remote write proto format
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functions for translating between new proto formats symbol table and
actual prometheus labels
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functionality for new minimized remote write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor things
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Make LabelSymbols a fixed32
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove unused proto type
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update tests
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build for stringlabels tag
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use two uint32 to encode (offset,leng)
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* manually optimize varint marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use unsafe []byte->string cast to reuse buffer
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix writeRequestMinimizedFixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all code from previous interning approach
the 'minimized' version is now the only v1.1 version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* minimally-tested exemplar support for rw 1.1
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor new version flag to make it easier to pick a specific format
instead of having multiple flags, plus add new formats for testing
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use exp slices for backwards compat. to go 1.20 plus add copyright
header to test file
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix label ranging
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add bytes slice (instead of slice of 32bit vars) format for testing
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* test additional len and lenbytes formats
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove mistaken package lock changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove formats we've decided not to use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove more format types we probably won't use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* More cleanup
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use require instead of assert in custom marshal test
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup; remove some unused functions
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, mostly linting fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove package-lock.json change again
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, address review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix test panic
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor lint issue + use labels Range function since it looks like
the tests fail to do `range labels.Labels` on CI
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* new interning format based on []string indeces
Co-authored-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all new rw formats but the []string one
also adapt tests to the new format
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup rwSymbolTable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add some TODOs for later
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* don't reserve field 3 for new proto and add TODO
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix custom marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* additional merge fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix server example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* revert package-lock.json changes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example prometheus version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* define separate proto types for remote write 2.0
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* rename new proto types and move to separate pkg
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update prometheus version for example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make Metadata not nullable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove old MinSample proto message
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* change enum names to fit buf build recommend enum naming and lint rules
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* remote: Added test for classic histogram grouping when sending rw; Fixed queue manager test delay. (#13421)
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Remote write v2: metadata support in every write request (#13394)
* Approach bundling metadata along with samples and exemplars
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Add first test; rebase with main
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Alternative approach: bundle metadata in TimeSeries protobuf
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* update go mod to match main branch
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix after rebase
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* we're not going to modify the 1.X format anymore
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Modify AppendMetadata based on the fact that we be putting metadata into
timeseries
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Rename enums for remote write versions to something that makes more
sense + remove the added `sendMetadata` flag.
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename flag that enables writing of metadata records to the WAL
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* additional clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix usage of require.Len
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* some clean up from review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more review fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
* remote write 2.0: sync with `main` branch (#13510)
* consoles: exclude iowait and steal from CPU Utilisation
'iowait' and 'steal' indicate specific idle/wait states, which shouldn't
be counted into CPU Utilisation. Also see
https://github.com/prometheus-operator/kube-prometheus/pull/796 and
https://github.com/kubernetes-monitoring/kubernetes-mixin/pull/667.
Per the iostat man page:
%idle
Show the percentage of time that the CPU or CPUs were idle and the
system did not have an outstanding disk I/O request.
%iowait
Show the percentage of time that the CPU or CPUs were idle during
which the system had an outstanding disk I/O request.
%steal
Show the percentage of time spent in involuntary wait by the
virtual CPU or CPUs while the hypervisor was servicing another
virtual processor.
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
* tsdb: shrink txRing with smaller integers
4 billion active transactions ought to be enough for anyone.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: create isolation transaction slice on demand
When Prometheus restarts it creates every series read in from the WAL,
but many of those series will be finished, and never receive any more
samples. By defering allocation of the txRing slice to when it is first
needed, we save 32 bytes per stale series.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* add cluster variable to Overview dashboard
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* promql: simplify Native Histogram arithmetics
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
* Cut 2.49.0-rc.0 (#13270)
* Cut 2.49.0-rc.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed the duplicate.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Add unit protobuf parser
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Go on adding protobuf parsing for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* ui: create a reproduction for https://github.com/prometheus/prometheus/issues/13292
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Get conditional right
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Get VM Scale Set NIC (#13283)
Calling `*armnetwork.InterfacesClient.Get()` doesn't work for Scale Set
VM NIC, because these use a different Resource ID format.
Use `*armnetwork.InterfacesClient.GetVirtualMachineScaleSetNetworkInterface()`
instead. This needs both the scale set name and the instance ID, so
add an `InstanceID` field to the `virtualMachine` struct. `InstanceID`
is empty for a VM that isn't a ScaleSetVM.
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
* Cut v2.49.0-rc.1
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Delete debugging lines, amend error message for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Correct order in error message
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Consider storage.ErrTooOldSample as non-retryable
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
* scrape_test.go: Increase scrape interval in TestScrapeLoopCache to reduce potential flakiness
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Avoid creating string for suffix, consider counters without _total suffix
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* build(deps): bump github.com/prometheus/client_golang
Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.17.0 to 1.18.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/client_golang/compare/v1.17.0...v1.18.0)
---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump actions/setup-node from 3.8.1 to 4.0.1
Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3.8.1 to 4.0.1.
- [Release notes](https://github.com/actions/setup-node/releases)
- [Commits](https://github.com/actions/setup-node/compare/5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d...b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8)
---
updated-dependencies:
- dependency-name: actions/setup-node
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
* scripts: sort file list in embed directive
Otherwise the resulting string depends on find, which afaict depends on
the underlying filesystem. A stable file list make it easier to detect
UI changes in downstreams that need to track UI assets.
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
* Fix DataTableProps['data'] for resultType string
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* Fix handling of scalar and string in isHeatmapData
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* build(deps): bump github.com/influxdata/influxdb
Bumps [github.com/influxdata/influxdb](https://github.com/influxdata/influxdb) from 1.11.2 to 1.11.4.
- [Release notes](https://github.com/influxdata/influxdb/releases)
- [Commits](https://github.com/influxdata/influxdb/compare/v1.11.2...v1.11.4)
---
updated-dependencies:
- dependency-name: github.com/influxdata/influxdb
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump github.com/prometheus/prometheus
Bumps [github.com/prometheus/prometheus](https://github.com/prometheus/prometheus) from 0.48.0 to 0.48.1.
- [Release notes](https://github.com/prometheus/prometheus/releases)
- [Changelog](https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/prometheus/compare/v0.48.0...v0.48.1)
---
updated-dependencies:
- dependency-name: github.com/prometheus/prometheus
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* Bump client_golang to v1.18.0 (#13373)
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Drop old inmemory samples (#13002)
* Drop old inmemory samples
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Avoid copying timeseries when the feature is disabled
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Run gofmt
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Clarify docs
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Add more logging info
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Remove loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* optimize function and add tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Simplify filter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Update help info from metrics
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use metrics to keep track of drop elements during buildWriteRequest
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var in tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* pass time.Now as parameter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Change buildwriterequest during retries
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Revert "Remove loggers"
This reverts commit 54f91dfcae20488944162335ab4ad8be459df1ab.
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use log level debug for loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Fix linter
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove noisy debug-level logs; add 'reason' label to drop metrics
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove accidentally committed files
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Propagate logger to buildWriteRequest to log dropped data
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix docs comment
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Make drop reason more specific
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove unnecessary pass of logger
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Use snake_case for reason label
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix dropped samples metric
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
---------
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
* fix(discovery): allow requireUpdate util to timeout in discovery/file/file_test.go.
The loop ran indefinitely if the condition isn't met.
Before, each iteration created a new timer channel which was always outpaced by
the other timer channel with smaller duration.
minor detail: There was a memory leak: resources of the ~10 previous timers were
constantly kept. With the fix, we may keep the resources of one timer around for defaultWait
but this isn't worth the changes to make it right.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Merge pull request #13371 from kevinmingtarja/fix-isHeatmapData
ui: fix handling of scalar and string in isHeatmapData
* tsdb/{index,compact}: allow using custom postings encoding format (#13242)
* tsdb/{index,compact}: allow using custom postings encoding format
We would like to experiment with a different postings encoding format in
Thanos so in this change I am proposing adding another argument to
`NewWriter` which would allow users to change the format if needed.
Also, wire the leveled compactor so that it would be possible to change
the format there too.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb/compact: use a struct for leveled compactor options
As discussed on Slack, let's use a struct for the options in leveled
compactor.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: make changes after Bryan's review
- Make changes less intrusive
- Turn the postings encoder type into a function
- Add NewWriterWithEncoder()
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
---------
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0-rc.2
Signed-off-by: bwplotka <bwplotka@gmail.com>
* build(deps): bump actions/setup-go from 3.5.0 to 5.0.0 in /scripts (#13362)
Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3.5.0 to 5.0.0.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/6edd4406fa81c3da01a34fa6f6343087c207a568...0c52d547c9bc32b1aa3301fd7a9cb496313a4491)
---
updated-dependencies:
- dependency-name: actions/setup-go
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump github/codeql-action from 2.22.8 to 3.22.12 (#13358)
Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.22.8 to 3.22.12.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/407ffafae6a767df3e0230c3df91b6443ae8df75...012739e5082ff0c22ca6d6ab32e07c36df03c4a4)
---
updated-dependencies:
- dependency-name: github/codeql-action
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* put @nexucis has a release shepherd (#13383)
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
* Add analyze histograms command to promtool (#12331)
Add `query analyze` command to promtool
This command analyzes the buckets of classic and native histograms,
based on data queried from the Prometheus query API, i.e. it
doesn't require direct access to the TSDB files.
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* included instance in all necessary descriptions
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* tsdb/compact: fix passing merge func
Fixing a very small logical problem I've introduced :(.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: add enable overlapping compaction
This functionality is needed in downstream projects because they have a
separate component that does compaction.
Upstreaming
https://github.com/grafana/mimir-prometheus/blob/7c8e9a2a76fc729e9078889782928b2fdfe240e9/tsdb/compact.go#L323-L325.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* promtool: allow setting multiple matchers to "promtool tsdb dump" command. (#13296)
Conditions are ANDed inside the same matcher but matchers are ORed
Including unit tests for "promtool tsdb dump".
Refactor some matchers scraping utils.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Fixed changelog
Signed-off-by: bwplotka <bwplotka@gmail.com>
* tsdb/main: wire "EnableOverlappingCompaction" to tsdb.Options (#13398)
This added the https://github.com/prometheus/prometheus/pull/13393
"EnableOverlappingCompaction" parameter to the compactor code but not to
the tsdb.Options. I forgot about that. Add it to `tsdb.Options` too and
set it to `true` in Prometheus.
Copy/paste the description from
https://github.com/prometheus/prometheus/pull/13393#issuecomment-1891787986
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Issue #13268: fix quality value in accept header
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
* Cut 2.49.1 with scrape q= bugfix.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Cut 2.49.1 web package.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Restore more efficient version of NewPossibleNonCounterInfo annotation (#13022)
Restore more efficient version of NewPossibleNonCounterInfo annotation
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Fix regressions introduced by #13242
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* fix slice copy in 1.20 (#13389)
The slices package is added to the standard library in Go 1.21;
we need to import from the exp area to maintain compatibility with Go 1.20.
Signed-off-by: tyltr <tylitianrui@126.com>
* Docs: Query Basics: link to rate (#10538)
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
* chore(kubernetes): check preconditions earlier and avoid unnecessary checks or iterations
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Examples: link to `rate` for new users (#10535)
* Examples: link to `rate` for new users
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
* promql: use natural sort in sort_by_label and sort_by_label_desc (#13411)
These functions are intended for humans, as robots can already sort the results
however they please. Humans like things sorted "naturally":
* https://blog.codinghorror.com/sorting-for-humans-natural-sort-order/
A similar thing has been done to Grafana, which is also used by humans:
* https://github.com/grafana/grafana/pull/78024
* https://github.com/grafana/grafana/pull/78494
Signed-off-by: Ivan Babrou <github@ivan.computer>
* TestLabelValuesWithMatchers: Add test case
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* remove obsolete build tag
Signed-off-by: tyltr <tylitianrui@126.com>
* Upgrade some golang dependencies for resty 2.11
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
* Native Histograms: support `native_histogram_min_bucket_factor` in scrape_config (#13222)
Native Histograms: support native_histogram_min_bucket_factor in scrape_config
---------
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram (#13392)
Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Minor fixes to otlp vendor update script
Signed-off-by: Goutham <gouthamve@gmail.com>
* build(deps): bump github.com/hetznercloud/hcloud-go/v2
Bumps [github.com/hetznercloud/hcloud-go/v2](https://github.com/hetznercloud/hcloud-go) from 2.4.0 to 2.6.0.
- [Release notes](https://github.com/hetznercloud/hcloud-go/releases)
- [Changelog](https://github.com/hetznercloud/hcloud-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hetznercloud/hcloud-go/compare/v2.4.0...v2.6.0)
---
updated-dependencies:
- dependency-name: github.com/hetznercloud/hcloud-go/v2
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* Enhanced visibility for `promtool test rules` with JSON colored formatting (#13342)
* Added diff flag for unit test to improvise readability & debugging
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Removed blank spaces
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed linting error
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Added cli flags to documentation
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Revert unrrelated linting fixes
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed review suggestions
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Cleanup
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
---------
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* storage: skip merging when no remote storage configured
Prometheus is hard-coded to use a fanout storage between TSDB and
a remote storage which by default is empty.
This change detects the empty storage and skips merging between
result sets, which would make `Select()` sort results.
Bottom line: we skip a sort unless there really is some remote storage
configured.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Remove csmarchbanks from remote write owners (#13432)
I have not had the time to keep up with remote write and have no plans
to work on it in the near future so I am withdrawing my maintainership
of that part of the codebase. I continue to focus on client_python.
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
* add more context cancellation check at evaluation time
Signed-off-by: Ben Ye <benye@amazon.com>
* Optimize label values with matchers by taking shortcuts (#13426)
Don't calculate postings beforehand: we may not need them. If all
matchers are for the requested label, we can just filter its values.
Also, if there are no values at all, no need to run any kind of
logic.
Also add more labelValuesWithMatchers benchmarks
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
* Add automatic memory limit handling
Enable automatic detection of memory limits and configure GOMEMLIMIT to
match.
* Also includes a flag to allow controlling the reserved ratio.
Signed-off-by: SuperQ <superq@gmail.com>
* Update OSSF badge link (#13433)
Provide a more user friendly interface
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
* SD Managers taking over responsibility for registration of debug metrics (#13375)
SD Managers take over responsibility for SD metrics registration
---------
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Optimize histogram iterators (#13340)
Optimize histogram iterators
Histogram iterators allocate new objects in the AtHistogram and
AtFloatHistogram methods, which makes calculating rates over long
ranges expensive.
In #13215 we allowed an existing object to be reused
when converting an integer histogram to a float histogram. This commit follows
the same idea and allows injecting an existing object in the AtHistogram and
AtFloatHistogram methods. When the injected value is nil, iterators allocate
new histograms, otherwise they populate and return the injected object.
The commit also adds a CopyTo method to Histogram and FloatHistogram which
is used in the BufferedIterator to overwrite items in the ring instead of making
new copies.
Note that a specialized HPoint pool is needed for all of this to work
(`matrixSelectorHPool`).
---------
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* doc: Mark `mad_over_time` as experimental (#13440)
We forgot to do that in
https://github.com/prometheus/prometheus/pull/13059
Signed-off-by: beorn7 <beorn@grafana.com>
* Change metric label for Puppetdb from 'http' to 'puppetdb'
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
* mirror metrics.proto change & generate code
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
* TestHeadLabelValuesWithMatchers: Add test case (#13414)
Add test case to TestHeadLabelValuesWithMatchers, while fixing a couple
of typos in other test cases. Also enclosing some implicit sub-tests in a
`t.Run` call to make them explicitly sub-tests.
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* update all go dependencies (#13438)
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
* build(deps): bump the k8s-io group with 2 updates (#13454)
Bumps the k8s-io group with 2 updates: [k8s.io/api](https://github.com/kubernetes/api) and [k8s.io/client-go](https://github.com/kubernetes/client-go).
Updates `k8s.io/api` from 0.28.4 to 0.29.1
- [Commits](https://github.com/kubernetes/api/compare/v0.28.4...v0.29.1)
Updates `k8s.io/client-go` from 0.28.4 to 0.29.1
- [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/kubernetes/client-go/compare/v0.28.4...v0.29.1)
---
updated-dependencies:
- dependency-name: k8s.io/api
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
- dependency-name: k8s.io/client-go
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump the go-opentelemetry-io group with 1 update (#13453)
Bumps the go-opentelemetry-io group with 1 update: [go.opentelemetry.io/collector/semconv](https://github.com/open-telemetry/opentelemetry-collector).
Updates `go.opentelemetry.io/collector/semconv` from 0.92.0 to 0.93.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-collector/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-collector/blob/main/CHANGELOG-API.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-collector/compare/v0.92.0...v0.93.0)
---
updated-dependencies:
- dependency-name: go.opentelemetry.io/collector/semconv
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: go-opentelemetry-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump actions/upload-artifact from 3.1.3 to 4.0.0 (#13355)
Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.3 to 4.0.0.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/a8a3f3ad30e3422c9c7b888a15615d19a852ae32...c7d193f32edcb7bfad88892161225aeda64e9392)
---
updated-dependencies:
- dependency-name: actions/upload-artifact
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump bufbuild/buf-push-action (#13357)
Bumps [bufbuild/buf-push-action](https://github.com/bufbuild/buf-push-action) from 342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1 to a654ff18effe4641ebea4a4ce242c49800728459.
- [Release notes](https://github.com/bufbuild/buf-push-action/releases)
- [Commits](https://github.com/bufbuild/buf-push-action/compare/342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1...a654ff18effe4641ebea4a4ce242c49800728459)
---
updated-dependencies:
- dependency-name: bufbuild/buf-push-action
dependency-type: direct:production
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* Labels: Add DropMetricName function, used in PromQL (#13446)
This function is called very frequently when executing PromQL functions,
and we can do it much more efficiently inside Labels.
In the common case that `__name__` comes first in the labels, we simply
re-point to start at the next label, which is nearly free.
`DropMetricName` is now so cheap I removed the cache - benchmarks show
everything still goes faster.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: simplify internal series delete function (#13261)
Lifting an optimisation from Agent code, `seriesHashmap.del` can use
the unique series reference, doesn't need to check Labels.
Also streamline the logic for deleting from `unique` and `conflicts` maps,
and add some comments to help the next person.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* otlptranslator/update-copy.sh: Fix sed command lines
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Rollback k8s.io requirements (#13462)
Rollback k8s.io Go modules to v0.28.6 to avoid forcing upgrade of Go to
1.21. This allows us to keep compatibility with the currently supported
upstream Go releases.
Signed-off-by: SuperQ <superq@gmail.com>
* Make update-copy.sh work for both OSX and GNU sed
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Name @beorn7 and @krajorama as maintainers for native histograms
I have been the de-facto maintainer for native histograms from the
beginning. So let's put this into MAINTAINERS.md.
In addition, I hereby proposose George Krajcsovits AKA Krajo as a
co-maintainer. He has contributed a lot of native histogram code, but
more importantly, he has contributed substantially to reviewing other
contributors' native histogram code, up to a point where I was merely
rubberstamping the PRs he had already reviewed. I'm confident that he
is ready to to be granted commit rights as outlined in the
"Maintainers" section of the governance:
https://prometheus.io/governance/#maintainers
According to the same section of the governance, I will announce the
proposed change on the developers mailing list and will give some time
for lazy consensus before merging this PR.
Signed-off-by: beorn7 <beorn@grafana.com>
* ui/fix: correct url handling for stacked graphs (#13460)
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
* tsdb: use cheaper Mutex on series
Mutex is 8 bytes; RWMutex is 24 bytes and much more complicated. Since
`RLock` is only used in two places, `UpdateMetadata` and `Delete`,
neither of which are hotspots, we should use the cheaper one.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Fix last_over_time for native histograms
The last_over_time retains a histogram sample without making a copy.
This sample is now coming from the buffered iterator used for windowing functions,
and can be reused for reading subsequent samples as the iterator progresses.
I would propose copying the sample in the last_over_time function, similar to
how it is done for rate, sum_over_time and others.
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
* Implementation
NOTE:
Rebased from main after refactor in #13014
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Add feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactor concurrency control
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Optimising dependencies/dependents funcs to not produce new slices each request
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Rename flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring for performance, and to allow controller to be overridden
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Block until all rules, both sync & async, have completed evaluating
Updated & added tests
Review feedback nits
Return empty map if not indeterminate
Use highWatermark to track inflight requests counter
Appease the linter
Clarify feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Fix typo in CLI flag description
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Fixed auto-generated doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improve doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Simplify the design to update concurrency controller once the rule evaluation has done
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Add more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Added more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improved RuleConcurrencyController interface doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Introduced sequentialRuleEvalController
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Remove superfluous nil check in Group.metrics
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* api: Serialize discovered and target labels into JSON directly (#13469)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* api: Serialize discovered labels into JSON directly in dropped targets (#13484)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* Add ShardedPostings() support to TSDB (#10421)
This PR is a reference implementation of the proposal described in #10420.
In addition to what described in #10420, in this PR I've introduced labels.StableHash(). The idea is to offer an hashing function which doesn't change over time, and that's used by query sharding in order to get a stable behaviour over time. The implementation of labels.StableHash() is the hashing function used by Prometheus before stringlabels, and what's used by Grafana Mimir for query sharding (because built before stringlabels was a thing).
Follow up work
As mentioned in #10420, if this PR is accepted I'm also open to upload another foundamental piece used by Grafana Mimir query sharding to accelerate the query execution: an optional, configurable and fast in-memory cache for the series hashes.
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* storage/remote: document why two benchmarks are skipped
One was silently doing nothing; one was doing something but the work
didn't go up linearly with iteration count.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Pod status changes not discovered by Kube Endpoints SD (#13337)
* fix(discovery/kubernetes/endpoints): react to changes on Pods because some modifications can occur on them without triggering an update on the related Endpoints (The Pod phase changing from Pending to Running e.g.).
---------
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
* Small improvements, add const, remove copypasta (#8106)
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
* Proposal to improve FPointSlice and HPointSlice allocation. (#13448)
* Reusing points slice from previous series when the slice is under utilized
* Adding comments on the bench test
Signed-off-by: Alan Protasio <alanprot@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
---------
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
* remote write 2.0 - follow up improvements (#13478)
* move remote write proto version config from a remote storage config to a
per remote write configuration option
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename scrape config for metadata, fix 2.0 header var name/value (was
1.1), and more clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* address review comments, mostly lint fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* another lint fix
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint imports
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics. (#13502)
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Apply suggestions from code review
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
* prw2.0: Added support for "custom" layouts for native histogram proto (#13558)
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# prompb/write/v2/types.pb.go
* Update prompb/write/v2/types.proto
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Addressed comments, fixed test.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* first draft of content negotiation
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix race in test
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix another test race
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Almost done with lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix todos around 405 HEAD handling
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Changes based on review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Update storage/remote/client.go
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Latest updates to review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* latest tweaks
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* remote write 2.0 - content negotiation remediation (#13921)
* Consolidate renegotiation error into one, fix tests
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* fix metric name and actuall increment counter
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
---------
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fixes after main sync.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW 2.0] Moved rw2 proto to the full path (both package name and placement) (#13973)
undefined
* [PRW2.0] Remove benchmark scripts (#13949)
See rationales on https://docs.google.com/document/d/1Bpf7mYjrHUhPHkie0qlnZFxzgqf_L32kM8ZOknSdJrU/edit
Signed-off-by: bwplotka <bwplotka@gmail.com>
* rw20: Update prw commentary after Callum spec review (#14136)
* rw20: Update prw commentary after Callum spec review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update types.proto
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation. (#14330)
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* update custom marshalling
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed confusing comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (chain1) New Remote Write 2.0 Config options for 2.0-rc.1 spec. (#14335)
NOTE: For simple review this change does not touch remote/ packages, only main and configs.
Spec: https://prometheus.io/docs/specs/remote_write_spec_2_0
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (part 2) Removed automatic negotiation, updates for the latest spec semantics in remote pkg (#14329)
* [PRW-2.0] (part2) Moved to latest basic negotiation & spec semantics.
Spec: https://github.com/prometheus/docs/pull/2462
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# config/config.go
# docs/configuration/configuration.md
# storage/remote/queue_manager_test.go
# storage/remote/write.go
# web/api/v1/api.go
* Addressed comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* storage/remote tests: refactor: extract function newTestQueueManager
To reduce repetition.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use newTestQueueManager for test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* [PRW 2.0] (part3) moved type specific conversions to prompb and writev2 codecs.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added test for rwProtoMsgFlagParser; fixed TODO comment.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Renamed DecodeV2WriteRequestStr to DecodeWriteV2Request (with tests).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed comments on remote_storage example, updated it for 2.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed `--enable-feature=metadata-wal-records` docs and error when using PRW 2.0 without it.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments on custom*.go
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added TODO to genproto.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments in remote pkg.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added metadata validation to write handler test; fixed ToMetadata.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed rest of Callum comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed writev2.FromMetadataType (was wrongly using prompb).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* fix a few import whitespaces
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* add a default case with an error to the example RW receiver
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more minor import whitespace chagnes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Apply suggestions from code review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update storage/remote/queue_manager_test.go
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
Co-authored-by: Nicolás Pazos <32206519+npazosmendez@users.noreply.github.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: alexgreenbank <alex.greenbank@grafana.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
2024-07-04 14:29:20 -07:00
ProtobufMessage : RemoteWriteProtoMsgV1 ,
2021-02-26 13:48:06 -08:00
QueueConfig : DefaultQueueConfig ,
MetadataConfig : DefaultMetadataConfig ,
2024-10-24 06:07:54 -07:00
HTTPClientConfig : DefaultRemoteWriteHTTPClientConfig ,
2017-07-25 05:47:34 -07:00
}
2017-08-01 03:00:40 -07:00
// DefaultQueueConfig is the default remote queue configuration.
DefaultQueueConfig = QueueConfig {
2023-04-02 03:00:14 -07:00
// With a maximum of 50 shards, assuming an average of 100ms remote write
// time and 2000 samples per batch, we will be able to push 1M samples/s.
MaxShards : 50 ,
2018-12-04 09:32:14 -08:00
MinShards : 1 ,
2023-03-31 01:47:06 -07:00
MaxSamplesPerSend : 2000 ,
2017-07-25 05:47:34 -07:00
2023-04-02 03:00:14 -07:00
// Each shard will have a max of 10,000 samples pending in its channel, plus the pending
// samples that have been enqueued. Theoretically we should only ever have about 12,000 samples
// per shard pending. At 50 shards that's 600k.
Capacity : 10000 ,
2018-08-24 07:55:21 -07:00
BatchSendDeadline : model . Duration ( 5 * time . Second ) ,
2017-07-25 05:47:34 -07:00
2019-06-10 12:43:08 -07:00
// Backoff times for retrying a batch of samples on recoverable errors.
2018-08-24 07:55:21 -07:00
MinBackoff : model . Duration ( 30 * time . Millisecond ) ,
2021-11-09 14:08:24 -08:00
MaxBackoff : model . Duration ( 5 * time . Second ) ,
2016-09-19 13:47:51 -07:00
}
2017-03-10 03:53:27 -08:00
2020-11-19 07:23:03 -08:00
// DefaultMetadataConfig is the default metadata configuration for a remote write endpoint.
DefaultMetadataConfig = MetadataConfig {
2021-06-24 15:39:50 -07:00
Send : true ,
SendInterval : model . Duration ( 1 * time . Minute ) ,
2023-03-31 01:47:06 -07:00
MaxSamplesPerSend : 2000 ,
2020-11-19 07:23:03 -08:00
}
2017-03-10 03:53:27 -08:00
// DefaultRemoteReadConfig is the default remote read configuration.
DefaultRemoteReadConfig = RemoteReadConfig {
2022-02-16 13:12:47 -08:00
RemoteTimeout : model . Duration ( 1 * time . Minute ) ,
2024-08-27 23:23:54 -07:00
ChunkedReadLimit : DefaultChunkedReadLimit ,
2022-02-16 13:12:47 -08:00
HTTPClientConfig : config . DefaultHTTPClientConfig ,
FilterExternalLabels : true ,
2017-03-10 03:53:27 -08:00
}
2021-07-19 21:52:57 -07:00
// DefaultStorageConfig is the default TSDB/Exemplar storage configuration.
DefaultStorageConfig = StorageConfig {
ExemplarsConfig : & DefaultExemplarsConfig ,
}
DefaultExemplarsConfig = ExemplarsConfig {
MaxExemplars : 100000 ,
}
2024-06-03 09:02:26 -07:00
// DefaultOTLPConfig is the default OTLP configuration.
2024-11-08 05:02:48 -08:00
DefaultOTLPConfig = OTLPConfig {
TranslationStrategy : UnderscoreEscapingWithSuffixes ,
}
2015-05-07 01:55:03 -07:00
)
// Config is the top-level configuration for Prometheus's config files.
type Config struct {
2023-02-24 02:47:12 -08:00
GlobalConfig GlobalConfig ` yaml:"global" `
2024-04-01 08:34:35 -07:00
Runtime RuntimeConfig ` yaml:"runtime,omitempty" `
2023-02-24 02:47:12 -08:00
AlertingConfig AlertingConfig ` yaml:"alerting,omitempty" `
RuleFiles [ ] string ` yaml:"rule_files,omitempty" `
ScrapeConfigFiles [ ] string ` yaml:"scrape_config_files,omitempty" `
ScrapeConfigs [ ] * ScrapeConfig ` yaml:"scrape_configs,omitempty" `
StorageConfig StorageConfig ` yaml:"storage,omitempty" `
TracingConfig TracingConfig ` yaml:"tracing,omitempty" `
2015-05-07 01:55:03 -07:00
2017-02-13 12:43:20 -08:00
RemoteWriteConfigs [ ] * RemoteWriteConfig ` yaml:"remote_write,omitempty" `
2017-03-10 03:53:27 -08:00
RemoteReadConfigs [ ] * RemoteReadConfig ` yaml:"remote_read,omitempty" `
2024-06-03 09:02:26 -07:00
OTLPConfig OTLPConfig ` yaml:"otlp,omitempty" `
2015-05-07 01:55:03 -07:00
}
2020-08-20 05:48:26 -07:00
// SetDirectory joins any relative file paths with dir.
func ( c * Config ) SetDirectory ( dir string ) {
c . GlobalConfig . SetDirectory ( dir )
c . AlertingConfig . SetDirectory ( dir )
2022-01-29 14:56:44 -08:00
c . TracingConfig . SetDirectory ( dir )
2020-08-20 05:48:26 -07:00
for i , file := range c . RuleFiles {
c . RuleFiles [ i ] = config . JoinDir ( dir , file )
2016-11-24 06:17:50 -08:00
}
2023-02-24 02:47:12 -08:00
for i , file := range c . ScrapeConfigFiles {
c . ScrapeConfigFiles [ i ] = config . JoinDir ( dir , file )
}
2020-08-20 05:48:26 -07:00
for _ , c := range c . ScrapeConfigs {
c . SetDirectory ( dir )
2015-08-05 09:04:34 -07:00
}
2020-08-20 05:48:26 -07:00
for _ , c := range c . RemoteWriteConfigs {
c . SetDirectory ( dir )
2019-03-12 03:24:15 -07:00
}
2020-08-20 05:48:26 -07:00
for _ , c := range c . RemoteReadConfigs {
c . SetDirectory ( dir )
2019-03-12 03:24:15 -07:00
}
2015-08-05 09:04:34 -07:00
}
2015-05-07 07:47:18 -07:00
func ( c Config ) String ( ) string {
2017-05-29 04:46:23 -07:00
b , err := yaml . Marshal ( c )
if err != nil {
return fmt . Sprintf ( "<error creating config string: %s>" , err )
2013-01-07 14:24:26 -08:00
}
2017-05-29 04:46:23 -07:00
return string ( b )
2015-05-07 01:55:03 -07:00
}
2013-04-30 11:20:14 -07:00
2023-10-10 03:16:55 -07:00
// GetScrapeConfigs returns the scrape configurations.
2023-02-24 02:47:12 -08:00
func ( c * Config ) GetScrapeConfigs ( ) ( [ ] * ScrapeConfig , error ) {
scfgs := make ( [ ] * ScrapeConfig , len ( c . ScrapeConfigs ) )
jobNames := map [ string ] string { }
for i , scfg := range c . ScrapeConfigs {
[PRW 2.0] Merging `remote-write-2.0` feature branch to main (PRW 2.0 support + metadata in WAL) (#14395)
* Remote Write 1.1: e2e benchmarks (#13102)
* Remote Write e2e benchmarks
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Prometheus ports automatically assigned
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make dashboard editable + more modular to different job label values
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Dashboard improvements
* memory stats
* diffs look at counter increases
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* run script: absolute path for config templates
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* grafana dashboard improvements
* show actual values of metrics
* add memory stats and diff
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* dashboard changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
* replace snappy encoding library
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto types
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add decode function for new write request proto
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add lookup table struct that is used to build the symbol table in new
write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Implement code paths for new proto format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example server to include handler for new format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add new test client
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* tests and new -> original proto mapping util
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto support on receiver end
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Fix test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* no-brainer copypaste but more performance write support
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove some comented code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix mocks and fixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add basic reduce remote write handler benchmark
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor out common code between write methods
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix: queue manager to include float histograms in new requests
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add sender-side tests and fix failing ones
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor queue manager code to remove some duplication
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Improve sender benchmarks and some allocations
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use github.com/golang/snappy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup: remove hardcoded fake url for testing
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add 1.1 version handling code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Remove config, update proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* gofmt
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix NewWriteClient and change new flags wording
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fields rewording in handler
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remote write handler to checks version header
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix typo in log
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add minmized remote write proto format
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functions for translating between new proto formats symbol table and
actual prometheus labels
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functionality for new minimized remote write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor things
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Make LabelSymbols a fixed32
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove unused proto type
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update tests
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build for stringlabels tag
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use two uint32 to encode (offset,leng)
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* manually optimize varint marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use unsafe []byte->string cast to reuse buffer
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix writeRequestMinimizedFixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all code from previous interning approach
the 'minimized' version is now the only v1.1 version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* minimally-tested exemplar support for rw 1.1
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor new version flag to make it easier to pick a specific format
instead of having multiple flags, plus add new formats for testing
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use exp slices for backwards compat. to go 1.20 plus add copyright
header to test file
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix label ranging
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add bytes slice (instead of slice of 32bit vars) format for testing
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* test additional len and lenbytes formats
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove mistaken package lock changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove formats we've decided not to use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove more format types we probably won't use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* More cleanup
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use require instead of assert in custom marshal test
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup; remove some unused functions
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, mostly linting fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove package-lock.json change again
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, address review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix test panic
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor lint issue + use labels Range function since it looks like
the tests fail to do `range labels.Labels` on CI
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* new interning format based on []string indeces
Co-authored-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all new rw formats but the []string one
also adapt tests to the new format
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup rwSymbolTable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add some TODOs for later
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* don't reserve field 3 for new proto and add TODO
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix custom marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* additional merge fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix server example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* revert package-lock.json changes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example prometheus version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* define separate proto types for remote write 2.0
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* rename new proto types and move to separate pkg
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update prometheus version for example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make Metadata not nullable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove old MinSample proto message
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* change enum names to fit buf build recommend enum naming and lint rules
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* remote: Added test for classic histogram grouping when sending rw; Fixed queue manager test delay. (#13421)
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Remote write v2: metadata support in every write request (#13394)
* Approach bundling metadata along with samples and exemplars
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Add first test; rebase with main
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Alternative approach: bundle metadata in TimeSeries protobuf
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* update go mod to match main branch
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix after rebase
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* we're not going to modify the 1.X format anymore
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Modify AppendMetadata based on the fact that we be putting metadata into
timeseries
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Rename enums for remote write versions to something that makes more
sense + remove the added `sendMetadata` flag.
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename flag that enables writing of metadata records to the WAL
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* additional clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix usage of require.Len
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* some clean up from review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more review fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
* remote write 2.0: sync with `main` branch (#13510)
* consoles: exclude iowait and steal from CPU Utilisation
'iowait' and 'steal' indicate specific idle/wait states, which shouldn't
be counted into CPU Utilisation. Also see
https://github.com/prometheus-operator/kube-prometheus/pull/796 and
https://github.com/kubernetes-monitoring/kubernetes-mixin/pull/667.
Per the iostat man page:
%idle
Show the percentage of time that the CPU or CPUs were idle and the
system did not have an outstanding disk I/O request.
%iowait
Show the percentage of time that the CPU or CPUs were idle during
which the system had an outstanding disk I/O request.
%steal
Show the percentage of time spent in involuntary wait by the
virtual CPU or CPUs while the hypervisor was servicing another
virtual processor.
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
* tsdb: shrink txRing with smaller integers
4 billion active transactions ought to be enough for anyone.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: create isolation transaction slice on demand
When Prometheus restarts it creates every series read in from the WAL,
but many of those series will be finished, and never receive any more
samples. By defering allocation of the txRing slice to when it is first
needed, we save 32 bytes per stale series.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* add cluster variable to Overview dashboard
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* promql: simplify Native Histogram arithmetics
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
* Cut 2.49.0-rc.0 (#13270)
* Cut 2.49.0-rc.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed the duplicate.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Add unit protobuf parser
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Go on adding protobuf parsing for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* ui: create a reproduction for https://github.com/prometheus/prometheus/issues/13292
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Get conditional right
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Get VM Scale Set NIC (#13283)
Calling `*armnetwork.InterfacesClient.Get()` doesn't work for Scale Set
VM NIC, because these use a different Resource ID format.
Use `*armnetwork.InterfacesClient.GetVirtualMachineScaleSetNetworkInterface()`
instead. This needs both the scale set name and the instance ID, so
add an `InstanceID` field to the `virtualMachine` struct. `InstanceID`
is empty for a VM that isn't a ScaleSetVM.
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
* Cut v2.49.0-rc.1
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Delete debugging lines, amend error message for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Correct order in error message
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Consider storage.ErrTooOldSample as non-retryable
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
* scrape_test.go: Increase scrape interval in TestScrapeLoopCache to reduce potential flakiness
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Avoid creating string for suffix, consider counters without _total suffix
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* build(deps): bump github.com/prometheus/client_golang
Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.17.0 to 1.18.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/client_golang/compare/v1.17.0...v1.18.0)
---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump actions/setup-node from 3.8.1 to 4.0.1
Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3.8.1 to 4.0.1.
- [Release notes](https://github.com/actions/setup-node/releases)
- [Commits](https://github.com/actions/setup-node/compare/5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d...b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8)
---
updated-dependencies:
- dependency-name: actions/setup-node
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
* scripts: sort file list in embed directive
Otherwise the resulting string depends on find, which afaict depends on
the underlying filesystem. A stable file list make it easier to detect
UI changes in downstreams that need to track UI assets.
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
* Fix DataTableProps['data'] for resultType string
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* Fix handling of scalar and string in isHeatmapData
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* build(deps): bump github.com/influxdata/influxdb
Bumps [github.com/influxdata/influxdb](https://github.com/influxdata/influxdb) from 1.11.2 to 1.11.4.
- [Release notes](https://github.com/influxdata/influxdb/releases)
- [Commits](https://github.com/influxdata/influxdb/compare/v1.11.2...v1.11.4)
---
updated-dependencies:
- dependency-name: github.com/influxdata/influxdb
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump github.com/prometheus/prometheus
Bumps [github.com/prometheus/prometheus](https://github.com/prometheus/prometheus) from 0.48.0 to 0.48.1.
- [Release notes](https://github.com/prometheus/prometheus/releases)
- [Changelog](https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/prometheus/compare/v0.48.0...v0.48.1)
---
updated-dependencies:
- dependency-name: github.com/prometheus/prometheus
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* Bump client_golang to v1.18.0 (#13373)
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Drop old inmemory samples (#13002)
* Drop old inmemory samples
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Avoid copying timeseries when the feature is disabled
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Run gofmt
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Clarify docs
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Add more logging info
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Remove loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* optimize function and add tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Simplify filter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Update help info from metrics
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use metrics to keep track of drop elements during buildWriteRequest
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var in tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* pass time.Now as parameter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Change buildwriterequest during retries
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Revert "Remove loggers"
This reverts commit 54f91dfcae20488944162335ab4ad8be459df1ab.
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use log level debug for loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Fix linter
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove noisy debug-level logs; add 'reason' label to drop metrics
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove accidentally committed files
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Propagate logger to buildWriteRequest to log dropped data
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix docs comment
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Make drop reason more specific
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove unnecessary pass of logger
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Use snake_case for reason label
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix dropped samples metric
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
---------
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
* fix(discovery): allow requireUpdate util to timeout in discovery/file/file_test.go.
The loop ran indefinitely if the condition isn't met.
Before, each iteration created a new timer channel which was always outpaced by
the other timer channel with smaller duration.
minor detail: There was a memory leak: resources of the ~10 previous timers were
constantly kept. With the fix, we may keep the resources of one timer around for defaultWait
but this isn't worth the changes to make it right.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Merge pull request #13371 from kevinmingtarja/fix-isHeatmapData
ui: fix handling of scalar and string in isHeatmapData
* tsdb/{index,compact}: allow using custom postings encoding format (#13242)
* tsdb/{index,compact}: allow using custom postings encoding format
We would like to experiment with a different postings encoding format in
Thanos so in this change I am proposing adding another argument to
`NewWriter` which would allow users to change the format if needed.
Also, wire the leveled compactor so that it would be possible to change
the format there too.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb/compact: use a struct for leveled compactor options
As discussed on Slack, let's use a struct for the options in leveled
compactor.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: make changes after Bryan's review
- Make changes less intrusive
- Turn the postings encoder type into a function
- Add NewWriterWithEncoder()
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
---------
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0-rc.2
Signed-off-by: bwplotka <bwplotka@gmail.com>
* build(deps): bump actions/setup-go from 3.5.0 to 5.0.0 in /scripts (#13362)
Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3.5.0 to 5.0.0.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/6edd4406fa81c3da01a34fa6f6343087c207a568...0c52d547c9bc32b1aa3301fd7a9cb496313a4491)
---
updated-dependencies:
- dependency-name: actions/setup-go
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump github/codeql-action from 2.22.8 to 3.22.12 (#13358)
Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.22.8 to 3.22.12.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/407ffafae6a767df3e0230c3df91b6443ae8df75...012739e5082ff0c22ca6d6ab32e07c36df03c4a4)
---
updated-dependencies:
- dependency-name: github/codeql-action
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* put @nexucis has a release shepherd (#13383)
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
* Add analyze histograms command to promtool (#12331)
Add `query analyze` command to promtool
This command analyzes the buckets of classic and native histograms,
based on data queried from the Prometheus query API, i.e. it
doesn't require direct access to the TSDB files.
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* included instance in all necessary descriptions
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* tsdb/compact: fix passing merge func
Fixing a very small logical problem I've introduced :(.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: add enable overlapping compaction
This functionality is needed in downstream projects because they have a
separate component that does compaction.
Upstreaming
https://github.com/grafana/mimir-prometheus/blob/7c8e9a2a76fc729e9078889782928b2fdfe240e9/tsdb/compact.go#L323-L325.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* promtool: allow setting multiple matchers to "promtool tsdb dump" command. (#13296)
Conditions are ANDed inside the same matcher but matchers are ORed
Including unit tests for "promtool tsdb dump".
Refactor some matchers scraping utils.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Fixed changelog
Signed-off-by: bwplotka <bwplotka@gmail.com>
* tsdb/main: wire "EnableOverlappingCompaction" to tsdb.Options (#13398)
This added the https://github.com/prometheus/prometheus/pull/13393
"EnableOverlappingCompaction" parameter to the compactor code but not to
the tsdb.Options. I forgot about that. Add it to `tsdb.Options` too and
set it to `true` in Prometheus.
Copy/paste the description from
https://github.com/prometheus/prometheus/pull/13393#issuecomment-1891787986
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Issue #13268: fix quality value in accept header
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
* Cut 2.49.1 with scrape q= bugfix.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Cut 2.49.1 web package.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Restore more efficient version of NewPossibleNonCounterInfo annotation (#13022)
Restore more efficient version of NewPossibleNonCounterInfo annotation
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Fix regressions introduced by #13242
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* fix slice copy in 1.20 (#13389)
The slices package is added to the standard library in Go 1.21;
we need to import from the exp area to maintain compatibility with Go 1.20.
Signed-off-by: tyltr <tylitianrui@126.com>
* Docs: Query Basics: link to rate (#10538)
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
* chore(kubernetes): check preconditions earlier and avoid unnecessary checks or iterations
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Examples: link to `rate` for new users (#10535)
* Examples: link to `rate` for new users
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
* promql: use natural sort in sort_by_label and sort_by_label_desc (#13411)
These functions are intended for humans, as robots can already sort the results
however they please. Humans like things sorted "naturally":
* https://blog.codinghorror.com/sorting-for-humans-natural-sort-order/
A similar thing has been done to Grafana, which is also used by humans:
* https://github.com/grafana/grafana/pull/78024
* https://github.com/grafana/grafana/pull/78494
Signed-off-by: Ivan Babrou <github@ivan.computer>
* TestLabelValuesWithMatchers: Add test case
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* remove obsolete build tag
Signed-off-by: tyltr <tylitianrui@126.com>
* Upgrade some golang dependencies for resty 2.11
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
* Native Histograms: support `native_histogram_min_bucket_factor` in scrape_config (#13222)
Native Histograms: support native_histogram_min_bucket_factor in scrape_config
---------
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram (#13392)
Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Minor fixes to otlp vendor update script
Signed-off-by: Goutham <gouthamve@gmail.com>
* build(deps): bump github.com/hetznercloud/hcloud-go/v2
Bumps [github.com/hetznercloud/hcloud-go/v2](https://github.com/hetznercloud/hcloud-go) from 2.4.0 to 2.6.0.
- [Release notes](https://github.com/hetznercloud/hcloud-go/releases)
- [Changelog](https://github.com/hetznercloud/hcloud-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hetznercloud/hcloud-go/compare/v2.4.0...v2.6.0)
---
updated-dependencies:
- dependency-name: github.com/hetznercloud/hcloud-go/v2
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* Enhanced visibility for `promtool test rules` with JSON colored formatting (#13342)
* Added diff flag for unit test to improvise readability & debugging
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Removed blank spaces
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed linting error
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Added cli flags to documentation
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Revert unrrelated linting fixes
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed review suggestions
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Cleanup
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
---------
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* storage: skip merging when no remote storage configured
Prometheus is hard-coded to use a fanout storage between TSDB and
a remote storage which by default is empty.
This change detects the empty storage and skips merging between
result sets, which would make `Select()` sort results.
Bottom line: we skip a sort unless there really is some remote storage
configured.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Remove csmarchbanks from remote write owners (#13432)
I have not had the time to keep up with remote write and have no plans
to work on it in the near future so I am withdrawing my maintainership
of that part of the codebase. I continue to focus on client_python.
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
* add more context cancellation check at evaluation time
Signed-off-by: Ben Ye <benye@amazon.com>
* Optimize label values with matchers by taking shortcuts (#13426)
Don't calculate postings beforehand: we may not need them. If all
matchers are for the requested label, we can just filter its values.
Also, if there are no values at all, no need to run any kind of
logic.
Also add more labelValuesWithMatchers benchmarks
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
* Add automatic memory limit handling
Enable automatic detection of memory limits and configure GOMEMLIMIT to
match.
* Also includes a flag to allow controlling the reserved ratio.
Signed-off-by: SuperQ <superq@gmail.com>
* Update OSSF badge link (#13433)
Provide a more user friendly interface
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
* SD Managers taking over responsibility for registration of debug metrics (#13375)
SD Managers take over responsibility for SD metrics registration
---------
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Optimize histogram iterators (#13340)
Optimize histogram iterators
Histogram iterators allocate new objects in the AtHistogram and
AtFloatHistogram methods, which makes calculating rates over long
ranges expensive.
In #13215 we allowed an existing object to be reused
when converting an integer histogram to a float histogram. This commit follows
the same idea and allows injecting an existing object in the AtHistogram and
AtFloatHistogram methods. When the injected value is nil, iterators allocate
new histograms, otherwise they populate and return the injected object.
The commit also adds a CopyTo method to Histogram and FloatHistogram which
is used in the BufferedIterator to overwrite items in the ring instead of making
new copies.
Note that a specialized HPoint pool is needed for all of this to work
(`matrixSelectorHPool`).
---------
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* doc: Mark `mad_over_time` as experimental (#13440)
We forgot to do that in
https://github.com/prometheus/prometheus/pull/13059
Signed-off-by: beorn7 <beorn@grafana.com>
* Change metric label for Puppetdb from 'http' to 'puppetdb'
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
* mirror metrics.proto change & generate code
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
* TestHeadLabelValuesWithMatchers: Add test case (#13414)
Add test case to TestHeadLabelValuesWithMatchers, while fixing a couple
of typos in other test cases. Also enclosing some implicit sub-tests in a
`t.Run` call to make them explicitly sub-tests.
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* update all go dependencies (#13438)
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
* build(deps): bump the k8s-io group with 2 updates (#13454)
Bumps the k8s-io group with 2 updates: [k8s.io/api](https://github.com/kubernetes/api) and [k8s.io/client-go](https://github.com/kubernetes/client-go).
Updates `k8s.io/api` from 0.28.4 to 0.29.1
- [Commits](https://github.com/kubernetes/api/compare/v0.28.4...v0.29.1)
Updates `k8s.io/client-go` from 0.28.4 to 0.29.1
- [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/kubernetes/client-go/compare/v0.28.4...v0.29.1)
---
updated-dependencies:
- dependency-name: k8s.io/api
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
- dependency-name: k8s.io/client-go
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump the go-opentelemetry-io group with 1 update (#13453)
Bumps the go-opentelemetry-io group with 1 update: [go.opentelemetry.io/collector/semconv](https://github.com/open-telemetry/opentelemetry-collector).
Updates `go.opentelemetry.io/collector/semconv` from 0.92.0 to 0.93.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-collector/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-collector/blob/main/CHANGELOG-API.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-collector/compare/v0.92.0...v0.93.0)
---
updated-dependencies:
- dependency-name: go.opentelemetry.io/collector/semconv
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: go-opentelemetry-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump actions/upload-artifact from 3.1.3 to 4.0.0 (#13355)
Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.3 to 4.0.0.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/a8a3f3ad30e3422c9c7b888a15615d19a852ae32...c7d193f32edcb7bfad88892161225aeda64e9392)
---
updated-dependencies:
- dependency-name: actions/upload-artifact
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump bufbuild/buf-push-action (#13357)
Bumps [bufbuild/buf-push-action](https://github.com/bufbuild/buf-push-action) from 342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1 to a654ff18effe4641ebea4a4ce242c49800728459.
- [Release notes](https://github.com/bufbuild/buf-push-action/releases)
- [Commits](https://github.com/bufbuild/buf-push-action/compare/342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1...a654ff18effe4641ebea4a4ce242c49800728459)
---
updated-dependencies:
- dependency-name: bufbuild/buf-push-action
dependency-type: direct:production
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* Labels: Add DropMetricName function, used in PromQL (#13446)
This function is called very frequently when executing PromQL functions,
and we can do it much more efficiently inside Labels.
In the common case that `__name__` comes first in the labels, we simply
re-point to start at the next label, which is nearly free.
`DropMetricName` is now so cheap I removed the cache - benchmarks show
everything still goes faster.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: simplify internal series delete function (#13261)
Lifting an optimisation from Agent code, `seriesHashmap.del` can use
the unique series reference, doesn't need to check Labels.
Also streamline the logic for deleting from `unique` and `conflicts` maps,
and add some comments to help the next person.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* otlptranslator/update-copy.sh: Fix sed command lines
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Rollback k8s.io requirements (#13462)
Rollback k8s.io Go modules to v0.28.6 to avoid forcing upgrade of Go to
1.21. This allows us to keep compatibility with the currently supported
upstream Go releases.
Signed-off-by: SuperQ <superq@gmail.com>
* Make update-copy.sh work for both OSX and GNU sed
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Name @beorn7 and @krajorama as maintainers for native histograms
I have been the de-facto maintainer for native histograms from the
beginning. So let's put this into MAINTAINERS.md.
In addition, I hereby proposose George Krajcsovits AKA Krajo as a
co-maintainer. He has contributed a lot of native histogram code, but
more importantly, he has contributed substantially to reviewing other
contributors' native histogram code, up to a point where I was merely
rubberstamping the PRs he had already reviewed. I'm confident that he
is ready to to be granted commit rights as outlined in the
"Maintainers" section of the governance:
https://prometheus.io/governance/#maintainers
According to the same section of the governance, I will announce the
proposed change on the developers mailing list and will give some time
for lazy consensus before merging this PR.
Signed-off-by: beorn7 <beorn@grafana.com>
* ui/fix: correct url handling for stacked graphs (#13460)
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
* tsdb: use cheaper Mutex on series
Mutex is 8 bytes; RWMutex is 24 bytes and much more complicated. Since
`RLock` is only used in two places, `UpdateMetadata` and `Delete`,
neither of which are hotspots, we should use the cheaper one.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Fix last_over_time for native histograms
The last_over_time retains a histogram sample without making a copy.
This sample is now coming from the buffered iterator used for windowing functions,
and can be reused for reading subsequent samples as the iterator progresses.
I would propose copying the sample in the last_over_time function, similar to
how it is done for rate, sum_over_time and others.
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
* Implementation
NOTE:
Rebased from main after refactor in #13014
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Add feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactor concurrency control
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Optimising dependencies/dependents funcs to not produce new slices each request
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Rename flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring for performance, and to allow controller to be overridden
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Block until all rules, both sync & async, have completed evaluating
Updated & added tests
Review feedback nits
Return empty map if not indeterminate
Use highWatermark to track inflight requests counter
Appease the linter
Clarify feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Fix typo in CLI flag description
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Fixed auto-generated doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improve doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Simplify the design to update concurrency controller once the rule evaluation has done
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Add more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Added more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improved RuleConcurrencyController interface doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Introduced sequentialRuleEvalController
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Remove superfluous nil check in Group.metrics
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* api: Serialize discovered and target labels into JSON directly (#13469)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* api: Serialize discovered labels into JSON directly in dropped targets (#13484)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* Add ShardedPostings() support to TSDB (#10421)
This PR is a reference implementation of the proposal described in #10420.
In addition to what described in #10420, in this PR I've introduced labels.StableHash(). The idea is to offer an hashing function which doesn't change over time, and that's used by query sharding in order to get a stable behaviour over time. The implementation of labels.StableHash() is the hashing function used by Prometheus before stringlabels, and what's used by Grafana Mimir for query sharding (because built before stringlabels was a thing).
Follow up work
As mentioned in #10420, if this PR is accepted I'm also open to upload another foundamental piece used by Grafana Mimir query sharding to accelerate the query execution: an optional, configurable and fast in-memory cache for the series hashes.
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* storage/remote: document why two benchmarks are skipped
One was silently doing nothing; one was doing something but the work
didn't go up linearly with iteration count.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Pod status changes not discovered by Kube Endpoints SD (#13337)
* fix(discovery/kubernetes/endpoints): react to changes on Pods because some modifications can occur on them without triggering an update on the related Endpoints (The Pod phase changing from Pending to Running e.g.).
---------
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
* Small improvements, add const, remove copypasta (#8106)
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
* Proposal to improve FPointSlice and HPointSlice allocation. (#13448)
* Reusing points slice from previous series when the slice is under utilized
* Adding comments on the bench test
Signed-off-by: Alan Protasio <alanprot@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
---------
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
* remote write 2.0 - follow up improvements (#13478)
* move remote write proto version config from a remote storage config to a
per remote write configuration option
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename scrape config for metadata, fix 2.0 header var name/value (was
1.1), and more clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* address review comments, mostly lint fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* another lint fix
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint imports
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics. (#13502)
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Apply suggestions from code review
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
* prw2.0: Added support for "custom" layouts for native histogram proto (#13558)
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# prompb/write/v2/types.pb.go
* Update prompb/write/v2/types.proto
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Addressed comments, fixed test.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* first draft of content negotiation
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix race in test
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix another test race
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Almost done with lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix todos around 405 HEAD handling
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Changes based on review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Update storage/remote/client.go
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Latest updates to review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* latest tweaks
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* remote write 2.0 - content negotiation remediation (#13921)
* Consolidate renegotiation error into one, fix tests
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* fix metric name and actuall increment counter
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
---------
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fixes after main sync.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW 2.0] Moved rw2 proto to the full path (both package name and placement) (#13973)
undefined
* [PRW2.0] Remove benchmark scripts (#13949)
See rationales on https://docs.google.com/document/d/1Bpf7mYjrHUhPHkie0qlnZFxzgqf_L32kM8ZOknSdJrU/edit
Signed-off-by: bwplotka <bwplotka@gmail.com>
* rw20: Update prw commentary after Callum spec review (#14136)
* rw20: Update prw commentary after Callum spec review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update types.proto
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation. (#14330)
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* update custom marshalling
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed confusing comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (chain1) New Remote Write 2.0 Config options for 2.0-rc.1 spec. (#14335)
NOTE: For simple review this change does not touch remote/ packages, only main and configs.
Spec: https://prometheus.io/docs/specs/remote_write_spec_2_0
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (part 2) Removed automatic negotiation, updates for the latest spec semantics in remote pkg (#14329)
* [PRW-2.0] (part2) Moved to latest basic negotiation & spec semantics.
Spec: https://github.com/prometheus/docs/pull/2462
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# config/config.go
# docs/configuration/configuration.md
# storage/remote/queue_manager_test.go
# storage/remote/write.go
# web/api/v1/api.go
* Addressed comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* storage/remote tests: refactor: extract function newTestQueueManager
To reduce repetition.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use newTestQueueManager for test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* [PRW 2.0] (part3) moved type specific conversions to prompb and writev2 codecs.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added test for rwProtoMsgFlagParser; fixed TODO comment.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Renamed DecodeV2WriteRequestStr to DecodeWriteV2Request (with tests).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed comments on remote_storage example, updated it for 2.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed `--enable-feature=metadata-wal-records` docs and error when using PRW 2.0 without it.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments on custom*.go
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added TODO to genproto.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments in remote pkg.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added metadata validation to write handler test; fixed ToMetadata.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed rest of Callum comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed writev2.FromMetadataType (was wrongly using prompb).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* fix a few import whitespaces
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* add a default case with an error to the example RW receiver
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more minor import whitespace chagnes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Apply suggestions from code review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update storage/remote/queue_manager_test.go
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
Co-authored-by: Nicolás Pazos <32206519+npazosmendez@users.noreply.github.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: alexgreenbank <alex.greenbank@grafana.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
2024-07-04 14:29:20 -07:00
// We do these checks for library users that would not call validate in
2023-02-24 02:47:12 -08:00
// Unmarshal.
2023-05-30 01:22:23 -07:00
if err := scfg . Validate ( c . GlobalConfig ) ; err != nil {
2023-02-24 02:47:12 -08:00
return nil , err
}
if _ , ok := jobNames [ scfg . JobName ] ; ok {
return nil , fmt . Errorf ( "found multiple scrape configs with job name %q" , scfg . JobName )
}
jobNames [ scfg . JobName ] = "main config file"
scfgs [ i ] = scfg
}
for _ , pat := range c . ScrapeConfigFiles {
fs , err := filepath . Glob ( pat )
if err != nil {
// The only error can be a bad pattern.
return nil , fmt . Errorf ( "error retrieving scrape config files for %q: %w" , pat , err )
}
for _ , filename := range fs {
cfg := ScrapeConfigs { }
content , err := os . ReadFile ( filename )
if err != nil {
return nil , fileErr ( filename , err )
}
err = yaml . UnmarshalStrict ( content , & cfg )
if err != nil {
return nil , fileErr ( filename , err )
}
for _ , scfg := range cfg . ScrapeConfigs {
2023-05-30 01:22:23 -07:00
if err := scfg . Validate ( c . GlobalConfig ) ; err != nil {
2023-02-24 02:47:12 -08:00
return nil , fileErr ( filename , err )
}
if f , ok := jobNames [ scfg . JobName ] ; ok {
return nil , fileErr ( filename , fmt . Errorf ( "found multiple scrape configs with job name %q, first found in %s" , scfg . JobName , f ) )
}
jobNames [ scfg . JobName ] = fmt . Sprintf ( "%q" , filePath ( filename ) )
scfg . SetDirectory ( filepath . Dir ( filename ) )
scfgs = append ( scfgs , scfg )
}
}
}
return scfgs , nil
}
2015-05-27 18:12:42 -07:00
// UnmarshalYAML implements the yaml.Unmarshaler interface.
2015-05-07 01:55:03 -07:00
func ( c * Config ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
2015-06-04 08:03:12 -07:00
* c = DefaultConfig
// We want to set c to the defaults and then overwrite it with the input.
// To make unmarshal fill the plain data struct rather than calling UnmarshalYAML
// again, we have to hide it using a type indirection.
type plain Config
if err := unmarshal ( ( * plain ) ( c ) ) ; err != nil {
2015-05-07 01:55:03 -07:00
return err
}
2018-04-04 01:07:39 -07:00
2015-07-17 10:58:34 -07:00
// If a global block was open but empty the default global config is overwritten.
// We have to restore it here.
if c . GlobalConfig . isZero ( ) {
c . GlobalConfig = DefaultGlobalConfig
}
2024-04-01 08:34:35 -07:00
// If a runtime block was open but empty the default runtime config is overwritten.
// We have to restore it here.
if c . Runtime . isZero ( ) {
c . Runtime = DefaultRuntimeConfig
// Use the GOGC env var value if the runtime section is empty.
c . Runtime . GoGC = getGoGCEnv ( )
}
2015-05-27 22:36:21 -07:00
for _ , rf := range c . RuleFiles {
if ! patRulePath . MatchString ( rf ) {
2022-05-23 00:54:32 -07:00
return fmt . Errorf ( "invalid rule file path %q" , rf )
2015-05-27 22:36:21 -07:00
}
}
2023-02-24 02:47:12 -08:00
for _ , sf := range c . ScrapeConfigFiles {
if ! patRulePath . MatchString ( sf ) {
return fmt . Errorf ( "invalid scrape config file path %q" , sf )
}
}
2015-05-07 01:55:03 -07:00
// Do global overrides and validate unique names.
2015-04-25 03:59:05 -07:00
jobNames := map [ string ] struct { } { }
2015-05-07 01:55:03 -07:00
for _ , scfg := range c . ScrapeConfigs {
2023-05-30 01:22:23 -07:00
if err := scfg . Validate ( c . GlobalConfig ) ; err != nil {
2023-02-24 02:47:12 -08:00
return err
2016-02-15 02:08:49 -08:00
}
2013-01-07 14:24:26 -08:00
2015-05-07 01:55:03 -07:00
if _ , ok := jobNames [ scfg . JobName ] ; ok {
2022-05-23 00:54:32 -07:00
return fmt . Errorf ( "found multiple scrape configs with job name %q" , scfg . JobName )
2013-02-22 12:07:35 -08:00
}
2015-05-07 01:55:03 -07:00
jobNames [ scfg . JobName ] = struct { } { }
2013-02-22 12:07:35 -08:00
}
2019-12-12 12:47:23 -08:00
rwNames := map [ string ] struct { } { }
2018-12-03 02:09:02 -08:00
for _ , rwcfg := range c . RemoteWriteConfigs {
if rwcfg == nil {
2019-03-25 16:01:12 -07:00
return errors . New ( "empty or null remote write config section" )
2018-12-03 02:09:02 -08:00
}
2019-12-12 12:47:23 -08:00
// Skip empty names, we fill their name with their config hash in remote write code.
if _ , ok := rwNames [ rwcfg . Name ] ; ok && rwcfg . Name != "" {
2022-05-23 00:54:32 -07:00
return fmt . Errorf ( "found multiple remote write configs with job name %q" , rwcfg . Name )
2019-12-12 12:47:23 -08:00
}
rwNames [ rwcfg . Name ] = struct { } { }
2018-12-03 02:09:02 -08:00
}
2019-12-12 12:47:23 -08:00
rrNames := map [ string ] struct { } { }
2018-12-03 02:09:02 -08:00
for _ , rrcfg := range c . RemoteReadConfigs {
if rrcfg == nil {
2019-03-25 16:01:12 -07:00
return errors . New ( "empty or null remote read config section" )
2018-12-03 02:09:02 -08:00
}
2019-12-12 12:47:23 -08:00
// Skip empty names, we fill their name with their config hash in remote read code.
if _ , ok := rrNames [ rrcfg . Name ] ; ok && rrcfg . Name != "" {
2022-05-23 00:54:32 -07:00
return fmt . Errorf ( "found multiple remote read configs with job name %q" , rrcfg . Name )
2019-12-12 12:47:23 -08:00
}
rrNames [ rrcfg . Name ] = struct { } { }
2018-12-03 02:09:02 -08:00
}
2016-06-12 07:18:05 -07:00
return nil
2013-02-22 12:07:35 -08:00
}
2015-05-07 07:47:18 -07:00
// GlobalConfig configures values that are used across other configuration
2015-05-07 01:55:03 -07:00
// objects.
type GlobalConfig struct {
// How frequently to scrape targets by default.
2016-01-29 06:23:11 -08:00
ScrapeInterval model . Duration ` yaml:"scrape_interval,omitempty" `
2015-05-07 01:55:03 -07:00
// The default timeout when scraping targets.
2016-01-29 06:23:11 -08:00
ScrapeTimeout model . Duration ` yaml:"scrape_timeout,omitempty" `
2023-10-10 03:16:55 -07:00
// The protocols to negotiate during a scrape. It tells clients what
// protocol are accepted by Prometheus and with what weight (most wanted is first).
// Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1,
// OpenMetricsText1.0.0, PrometheusText0.0.4.
ScrapeProtocols [ ] ScrapeProtocol ` yaml:"scrape_protocols,omitempty" `
2015-05-07 01:55:03 -07:00
// How frequently to evaluate rules by default.
2016-01-29 06:23:11 -08:00
EvaluationInterval model . Duration ` yaml:"evaluation_interval,omitempty" `
2024-05-30 03:49:50 -07:00
// Offset the rule evaluation timestamp of this particular group by the specified duration into the past to ensure the underlying metrics have been received.
2024-06-05 08:35:52 -07:00
RuleQueryOffset model . Duration ` yaml:"rule_query_offset,omitempty" `
2020-01-08 05:28:43 -08:00
// File to which PromQL queries are logged.
QueryLogFile string ` yaml:"query_log_file,omitempty" `
2024-08-26 02:41:56 -07:00
// File to which scrape failures are logged.
ScrapeFailureLogFile string ` yaml:"scrape_failure_log_file,omitempty" `
2015-05-07 01:55:03 -07:00
// The labels to add to any timeseries that this Prometheus instance scrapes.
2019-03-08 08:29:25 -08:00
ExternalLabels labels . Labels ` yaml:"external_labels,omitempty" `
2023-05-30 01:22:23 -07:00
// An uncompressed response body larger than this many bytes will cause the
// scrape to fail. 0 means no limit.
BodySizeLimit units . Base2Bytes ` yaml:"body_size_limit,omitempty" `
// More than this many samples post metric-relabeling will cause the scrape to
// fail. 0 means no limit.
SampleLimit uint ` yaml:"sample_limit,omitempty" `
// More than this many targets after the target relabeling will cause the
// scrapes to fail. 0 means no limit.
TargetLimit uint ` yaml:"target_limit,omitempty" `
// More than this many labels post metric-relabeling will cause the scrape to
// fail. 0 means no limit.
LabelLimit uint ` yaml:"label_limit,omitempty" `
// More than this label name length post metric-relabeling will cause the
// scrape to fail. 0 means no limit.
LabelNameLengthLimit uint ` yaml:"label_name_length_limit,omitempty" `
// More than this label value length post metric-relabeling will cause the
// scrape to fail. 0 means no limit.
LabelValueLengthLimit uint ` yaml:"label_value_length_limit,omitempty" `
2023-08-14 07:39:25 -07:00
// Keep no more than this many dropped targets per job.
// 0 means no limit.
KeepDroppedTargets uint ` yaml:"keep_dropped_targets,omitempty" `
2024-07-18 11:08:21 -07:00
// Allow UTF8 Metric and Label Names.
MetricNameValidationScheme string ` yaml:"metric_name_validation_scheme,omitempty" `
2013-01-07 14:24:26 -08:00
}
2023-10-10 03:16:55 -07:00
// ScrapeProtocol represents supported protocol for scraping metrics.
type ScrapeProtocol string
// Validate returns error if given scrape protocol is not supported.
func ( s ScrapeProtocol ) Validate ( ) error {
if _ , ok := ScrapeProtocolsHeaders [ s ] ; ! ok {
return fmt . Errorf ( "unknown scrape protocol %v, supported: %v" ,
s , func ( ) ( ret [ ] string ) {
for k := range ScrapeProtocolsHeaders {
ret = append ( ret , string ( k ) )
}
sort . Strings ( ret )
return ret
} ( ) )
}
return nil
}
2024-10-18 08:12:31 -07:00
// HeaderMediaType returns the MIME mediaType for a particular ScrapeProtocol.
func ( s ScrapeProtocol ) HeaderMediaType ( ) string {
if _ , ok := ScrapeProtocolsHeaders [ s ] ; ! ok {
return ""
}
mediaType , _ , err := mime . ParseMediaType ( ScrapeProtocolsHeaders [ s ] )
if err != nil {
return ""
}
return mediaType
}
2023-10-10 03:16:55 -07:00
var (
PrometheusProto ScrapeProtocol = "PrometheusProto"
PrometheusText0_0_4 ScrapeProtocol = "PrometheusText0.0.4"
2024-10-18 08:12:31 -07:00
PrometheusText1_0_0 ScrapeProtocol = "PrometheusText1.0.0"
2023-10-10 03:16:55 -07:00
OpenMetricsText0_0_1 ScrapeProtocol = "OpenMetricsText0.0.1"
OpenMetricsText1_0_0 ScrapeProtocol = "OpenMetricsText1.0.0"
2024-07-18 11:08:21 -07:00
UTF8NamesHeader string = model . EscapingKey + "=" + model . AllowUTF8
2023-10-10 03:16:55 -07:00
ScrapeProtocolsHeaders = map [ ScrapeProtocol ] string {
PrometheusProto : "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited" ,
PrometheusText0_0_4 : "text/plain;version=0.0.4" ,
2024-10-18 08:12:31 -07:00
PrometheusText1_0_0 : "text/plain;version=1.0.0;escaping=allow-utf-8" ,
2023-10-10 03:16:55 -07:00
OpenMetricsText0_0_1 : "application/openmetrics-text;version=0.0.1" ,
OpenMetricsText1_0_0 : "application/openmetrics-text;version=1.0.0" ,
}
2023-12-11 00:43:42 -08:00
// DefaultScrapeProtocols is the set of scrape protocols that will be proposed
// to scrape target, ordered by priority.
2023-10-10 03:16:55 -07:00
DefaultScrapeProtocols = [ ] ScrapeProtocol {
OpenMetricsText1_0_0 ,
OpenMetricsText0_0_1 ,
2024-10-18 08:12:31 -07:00
PrometheusText1_0_0 ,
2023-10-10 03:16:55 -07:00
PrometheusText0_0_4 ,
}
2023-12-11 00:43:42 -08:00
// DefaultProtoFirstScrapeProtocols is like DefaultScrapeProtocols, but it
// favors protobuf Prometheus exposition format.
// Used by default for certain feature-flags like
// "native-histograms" and "created-timestamp-zero-ingestion".
DefaultProtoFirstScrapeProtocols = [ ] ScrapeProtocol {
2023-10-10 03:16:55 -07:00
PrometheusProto ,
OpenMetricsText1_0_0 ,
OpenMetricsText0_0_1 ,
2024-10-18 08:12:31 -07:00
PrometheusText1_0_0 ,
2023-10-10 03:16:55 -07:00
PrometheusText0_0_4 ,
}
)
// validateAcceptScrapeProtocols return errors if we see problems with accept scrape protocols option.
func validateAcceptScrapeProtocols ( sps [ ] ScrapeProtocol ) error {
if len ( sps ) == 0 {
return errors . New ( "scrape_protocols cannot be empty" )
}
dups := map [ string ] struct { } { }
for _ , sp := range sps {
if _ , ok := dups [ strings . ToLower ( string ( sp ) ) ] ; ok {
return fmt . Errorf ( "duplicated protocol in scrape_protocols, got %v" , sps )
}
if err := sp . Validate ( ) ; err != nil {
return fmt . Errorf ( "scrape_protocols: %w" , err )
}
dups [ strings . ToLower ( string ( sp ) ) ] = struct { } { }
}
return nil
}
2020-08-20 05:48:26 -07:00
// SetDirectory joins any relative file paths with dir.
func ( c * GlobalConfig ) SetDirectory ( dir string ) {
c . QueryLogFile = config . JoinDir ( dir , c . QueryLogFile )
2024-08-26 02:41:56 -07:00
c . ScrapeFailureLogFile = config . JoinDir ( dir , c . ScrapeFailureLogFile )
2020-08-20 05:48:26 -07:00
}
2015-05-27 18:12:42 -07:00
// UnmarshalYAML implements the yaml.Unmarshaler interface.
2015-06-04 08:03:12 -07:00
func ( c * GlobalConfig ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
2016-02-15 05:08:25 -08:00
// Create a clean global config as the previous one was already populated
// by the default due to the YAML parser behavior for empty blocks.
gc := & GlobalConfig { }
type plain GlobalConfig
if err := unmarshal ( ( * plain ) ( gc ) ) ; err != nil {
2015-05-07 01:55:03 -07:00
return err
}
2018-04-04 01:07:39 -07:00
2022-03-09 14:26:05 -08:00
if err := gc . ExternalLabels . Validate ( func ( l labels . Label ) error {
2019-03-08 08:29:25 -08:00
if ! model . LabelName ( l . Name ) . IsValid ( ) {
2022-05-23 00:54:32 -07:00
return fmt . Errorf ( "%q is not a valid label name" , l . Name )
2019-03-08 08:29:25 -08:00
}
2019-03-14 14:16:29 -07:00
if ! model . LabelValue ( l . Value ) . IsValid ( ) {
2022-05-23 00:54:32 -07:00
return fmt . Errorf ( "%q is not a valid label value" , l . Value )
2019-03-14 14:16:29 -07:00
}
2022-03-09 14:26:05 -08:00
return nil
} ) ; err != nil {
return err
2019-03-08 08:29:25 -08:00
}
2016-02-15 02:08:49 -08:00
// First set the correct scrape interval, then check that the timeout
// (inferred or explicit) is not greater than that.
2016-02-15 05:08:25 -08:00
if gc . ScrapeInterval == 0 {
gc . ScrapeInterval = DefaultGlobalConfig . ScrapeInterval
2016-02-15 02:08:49 -08:00
}
2016-02-15 05:08:25 -08:00
if gc . ScrapeTimeout > gc . ScrapeInterval {
2019-03-25 16:01:12 -07:00
return errors . New ( "global scrape timeout greater than scrape interval" )
2016-02-15 02:08:49 -08:00
}
2016-02-15 05:08:25 -08:00
if gc . ScrapeTimeout == 0 {
if DefaultGlobalConfig . ScrapeTimeout > gc . ScrapeInterval {
gc . ScrapeTimeout = gc . ScrapeInterval
2016-02-15 02:08:49 -08:00
} else {
2016-02-15 05:08:25 -08:00
gc . ScrapeTimeout = DefaultGlobalConfig . ScrapeTimeout
2016-02-15 02:08:49 -08:00
}
}
2016-02-15 05:08:25 -08:00
if gc . EvaluationInterval == 0 {
gc . EvaluationInterval = DefaultGlobalConfig . EvaluationInterval
2016-02-15 02:08:49 -08:00
}
2023-10-10 03:16:55 -07:00
if gc . ScrapeProtocols == nil {
gc . ScrapeProtocols = DefaultGlobalConfig . ScrapeProtocols
}
if err := validateAcceptScrapeProtocols ( gc . ScrapeProtocols ) ; err != nil {
return fmt . Errorf ( "%w for global config" , err )
}
2016-02-15 05:08:25 -08:00
* c = * gc
2016-06-12 07:18:05 -07:00
return nil
2013-01-07 14:24:26 -08:00
}
2013-08-16 09:17:48 -07:00
2015-07-17 10:58:34 -07:00
// isZero returns true iff the global config is the zero value.
func ( c * GlobalConfig ) isZero ( ) bool {
2022-03-09 14:26:05 -08:00
return c . ExternalLabels . IsEmpty ( ) &&
2015-07-17 10:58:34 -07:00
c . ScrapeInterval == 0 &&
c . ScrapeTimeout == 0 &&
2020-01-08 05:28:43 -08:00
c . EvaluationInterval == 0 &&
2024-05-30 03:49:50 -07:00
c . RuleQueryOffset == 0 &&
2023-10-10 03:16:55 -07:00
c . QueryLogFile == "" &&
2024-08-26 02:41:56 -07:00
c . ScrapeFailureLogFile == "" &&
2023-10-10 03:16:55 -07:00
c . ScrapeProtocols == nil
2015-07-17 10:58:34 -07:00
}
2024-04-01 08:34:35 -07:00
// RuntimeConfig configures the values for the process behavior.
type RuntimeConfig struct {
// The Go garbage collection target percentage.
GoGC int ` yaml:"gogc,omitempty" `
}
// isZero returns true iff the global config is the zero value.
func ( c * RuntimeConfig ) isZero ( ) bool {
return c . GoGC == 0
}
2023-02-24 02:47:12 -08:00
type ScrapeConfigs struct {
ScrapeConfigs [ ] * ScrapeConfig ` yaml:"scrape_configs,omitempty" `
}
2016-11-23 03:41:19 -08:00
// ScrapeConfig configures a scraping unit for Prometheus.
type ScrapeConfig struct {
// The job name to which the job label is set by default.
JobName string ` yaml:"job_name" `
// Indicator whether the scraped metrics should remain unmodified.
HonorLabels bool ` yaml:"honor_labels,omitempty" `
2019-03-15 03:04:15 -07:00
// Indicator whether the scraped timestamps should be respected.
HonorTimestamps bool ` yaml:"honor_timestamps" `
2023-10-31 13:58:42 -07:00
// Indicator whether to track the staleness of the scraped timestamps.
TrackTimestampsStaleness bool ` yaml:"track_timestamps_staleness" `
2016-11-23 03:41:19 -08:00
// A set of query parameters with which the target is scraped.
Params url . Values ` yaml:"params,omitempty" `
// How frequently to scrape the targets of this scrape config.
ScrapeInterval model . Duration ` yaml:"scrape_interval,omitempty" `
// The timeout for scraping targets of this config.
ScrapeTimeout model . Duration ` yaml:"scrape_timeout,omitempty" `
2023-10-10 03:16:55 -07:00
// The protocols to negotiate during a scrape. It tells clients what
// protocol are accepted by Prometheus and with what preference (most wanted is first).
// Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1,
2024-10-18 08:12:31 -07:00
// OpenMetricsText1.0.0, PrometheusText1.0.0, PrometheusText0.0.4.
2023-10-10 03:16:55 -07:00
ScrapeProtocols [ ] ScrapeProtocol ` yaml:"scrape_protocols,omitempty" `
2024-10-18 08:12:31 -07:00
// The fallback protocol to use if the Content-Type provided by the target
// is not provided, blank, or not one of the expected values.
// Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1,
// OpenMetricsText1.0.0, PrometheusText1.0.0, PrometheusText0.0.4.
ScrapeFallbackProtocol ScrapeProtocol ` yaml:"fallback_scrape_protocol,omitempty" `
2024-10-18 00:32:15 -07:00
// Whether to scrape a classic histogram, even if it is also exposed as a native histogram.
AlwaysScrapeClassicHistograms bool ` yaml:"always_scrape_classic_histograms,omitempty" `
2024-10-14 02:13:58 -07:00
// Whether to convert all scraped classic histograms into a native histogram with custom buckets.
2024-10-21 04:22:58 -07:00
ConvertClassicHistogramsToNHCB bool ` yaml:"convert_classic_histograms_to_nhcb,omitempty" `
2024-08-26 02:41:56 -07:00
// File to which scrape failures are logged.
ScrapeFailureLogFile string ` yaml:"scrape_failure_log_file,omitempty" `
2016-11-23 03:41:19 -08:00
// The HTTP resource path on which to fetch metrics from targets.
MetricsPath string ` yaml:"metrics_path,omitempty" `
// The URL scheme with which to fetch metrics from targets.
Scheme string ` yaml:"scheme,omitempty" `
2023-11-20 04:02:53 -08:00
// Indicator whether to request compressed response from the target.
EnableCompression bool ` yaml:"enable_compression" `
2021-05-15 19:19:22 -07:00
// An uncompressed response body larger than this many bytes will cause the
// scrape to fail. 0 means no limit.
BodySizeLimit units . Base2Bytes ` yaml:"body_size_limit,omitempty" `
2021-05-06 01:56:21 -07:00
// More than this many samples post metric-relabeling will cause the scrape to
2023-05-30 01:22:23 -07:00
// fail. 0 means no limit.
2016-12-16 07:08:50 -08:00
SampleLimit uint ` yaml:"sample_limit,omitempty" `
2020-07-30 05:20:24 -07:00
// More than this many targets after the target relabeling will cause the
2023-05-30 01:22:23 -07:00
// scrapes to fail. 0 means no limit.
2020-07-30 05:20:24 -07:00
TargetLimit uint ` yaml:"target_limit,omitempty" `
2021-05-06 01:56:21 -07:00
// More than this many labels post metric-relabeling will cause the scrape to
2023-05-30 01:22:23 -07:00
// fail. 0 means no limit.
2021-05-06 01:56:21 -07:00
LabelLimit uint ` yaml:"label_limit,omitempty" `
// More than this label name length post metric-relabeling will cause the
2023-05-30 01:22:23 -07:00
// scrape to fail. 0 means no limit.
2021-05-06 01:56:21 -07:00
LabelNameLengthLimit uint ` yaml:"label_name_length_limit,omitempty" `
// More than this label value length post metric-relabeling will cause the
2023-05-30 01:22:23 -07:00
// scrape to fail. 0 means no limit.
2021-05-06 01:56:21 -07:00
LabelValueLengthLimit uint ` yaml:"label_value_length_limit,omitempty" `
2024-01-17 07:58:54 -08:00
// If there are more than this many buckets in a native histogram,
// buckets will be merged to stay within the limit.
2023-05-04 11:29:50 -07:00
NativeHistogramBucketLimit uint ` yaml:"native_histogram_bucket_limit,omitempty" `
2024-01-17 07:58:54 -08:00
// If the growth factor of one bucket to the next is smaller than this,
// buckets will be merged to increase the factor sufficiently.
NativeHistogramMinBucketFactor float64 ` yaml:"native_histogram_min_bucket_factor,omitempty" `
2023-08-14 07:39:25 -07:00
// Keep no more than this many dropped targets per job.
// 0 means no limit.
KeepDroppedTargets uint ` yaml:"keep_dropped_targets,omitempty" `
2024-07-18 11:08:21 -07:00
// Allow UTF8 Metric and Label Names.
MetricNameValidationScheme string ` yaml:"metric_name_validation_scheme,omitempty" `
2016-11-23 03:41:19 -08:00
// We cannot do proper Go type embedding below as the parser will then parse
// values arbitrarily into the overflow maps of further-down types.
2020-08-20 05:48:26 -07:00
ServiceDiscoveryConfigs discovery . Configs ` yaml:"-" `
HTTPClientConfig config . HTTPClientConfig ` yaml:",inline" `
2016-11-23 03:41:19 -08:00
2015-06-12 14:16:13 -07:00
// List of target relabel configurations.
2018-12-18 03:26:36 -08:00
RelabelConfigs [ ] * relabel . Config ` yaml:"relabel_configs,omitempty" `
2015-06-12 14:16:13 -07:00
// List of metric relabel configurations.
2018-12-18 03:26:36 -08:00
MetricRelabelConfigs [ ] * relabel . Config ` yaml:"metric_relabel_configs,omitempty" `
2013-08-16 09:17:48 -07:00
}
2015-04-20 03:24:25 -07:00
2020-08-20 05:48:26 -07:00
// SetDirectory joins any relative file paths with dir.
func ( c * ScrapeConfig ) SetDirectory ( dir string ) {
c . ServiceDiscoveryConfigs . SetDirectory ( dir )
c . HTTPClientConfig . SetDirectory ( dir )
2024-08-26 02:41:56 -07:00
c . ScrapeFailureLogFile = config . JoinDir ( dir , c . ScrapeFailureLogFile )
2020-08-20 05:48:26 -07:00
}
2015-06-04 08:03:12 -07:00
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func ( c * ScrapeConfig ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
* c = DefaultScrapeConfig
2020-08-20 05:48:26 -07:00
if err := discovery . UnmarshalYAMLWithInlineConfigs ( c , unmarshal ) ; err != nil {
2015-06-04 08:03:12 -07:00
return err
}
2016-09-14 23:00:14 -07:00
if len ( c . JobName ) == 0 {
2019-03-25 16:01:12 -07:00
return errors . New ( "job_name is empty" )
2015-06-04 08:03:12 -07:00
}
2016-11-24 06:17:50 -08:00
2016-11-23 03:41:19 -08:00
// The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer.
// We cannot make it a pointer as the parser panics for inlined pointer structs.
// Thus we just do its validation here.
2018-04-04 01:07:39 -07:00
if err := c . HTTPClientConfig . Validate ( ) ; err != nil {
2016-11-24 06:17:50 -08:00
return err
2015-07-22 08:48:22 -07:00
}
2016-11-23 03:41:19 -08:00
2015-11-07 06:25:51 -08:00
// Check for users putting URLs in target groups.
if len ( c . RelabelConfigs ) == 0 {
2020-08-20 05:48:26 -07:00
if err := checkStaticTargets ( c . ServiceDiscoveryConfigs ) ; err != nil {
return err
2015-11-07 06:25:51 -08:00
}
}
2018-06-13 08:34:59 -07:00
2018-12-03 02:09:02 -08:00
for _ , rlcfg := range c . RelabelConfigs {
if rlcfg == nil {
2019-03-25 16:01:12 -07:00
return errors . New ( "empty or null target relabeling rule in scrape config" )
2018-12-03 02:09:02 -08:00
}
}
for _ , rlcfg := range c . MetricRelabelConfigs {
if rlcfg == nil {
2019-03-25 16:01:12 -07:00
return errors . New ( "empty or null metric relabeling rule in scrape config" )
2018-12-03 02:09:02 -08:00
}
}
2016-06-12 07:18:05 -07:00
return nil
2015-06-04 08:03:12 -07:00
}
2023-10-10 03:16:55 -07:00
// Validate validates scrape config, but also fills relevant default values from global config if needed.
2023-05-30 01:22:23 -07:00
func ( c * ScrapeConfig ) Validate ( globalConfig GlobalConfig ) error {
2023-02-24 02:47:12 -08:00
if c == nil {
return errors . New ( "empty or null scrape config section" )
}
// First set the correct scrape interval, then check that the timeout
// (inferred or explicit) is not greater than that.
if c . ScrapeInterval == 0 {
2023-05-30 01:22:23 -07:00
c . ScrapeInterval = globalConfig . ScrapeInterval
2023-02-24 02:47:12 -08:00
}
if c . ScrapeTimeout > c . ScrapeInterval {
return fmt . Errorf ( "scrape timeout greater than scrape interval for scrape config with job name %q" , c . JobName )
}
if c . ScrapeTimeout == 0 {
2023-05-30 01:22:23 -07:00
if globalConfig . ScrapeTimeout > c . ScrapeInterval {
2023-02-24 02:47:12 -08:00
c . ScrapeTimeout = c . ScrapeInterval
} else {
2023-05-30 01:22:23 -07:00
c . ScrapeTimeout = globalConfig . ScrapeTimeout
2023-02-24 02:47:12 -08:00
}
}
2023-05-30 01:22:23 -07:00
if c . BodySizeLimit == 0 {
c . BodySizeLimit = globalConfig . BodySizeLimit
}
if c . SampleLimit == 0 {
c . SampleLimit = globalConfig . SampleLimit
}
if c . TargetLimit == 0 {
c . TargetLimit = globalConfig . TargetLimit
}
if c . LabelLimit == 0 {
c . LabelLimit = globalConfig . LabelLimit
}
if c . LabelNameLengthLimit == 0 {
c . LabelNameLengthLimit = globalConfig . LabelNameLengthLimit
}
if c . LabelValueLengthLimit == 0 {
c . LabelValueLengthLimit = globalConfig . LabelValueLengthLimit
}
2023-08-14 07:39:25 -07:00
if c . KeepDroppedTargets == 0 {
c . KeepDroppedTargets = globalConfig . KeepDroppedTargets
}
2024-08-26 02:41:56 -07:00
if c . ScrapeFailureLogFile == "" {
c . ScrapeFailureLogFile = globalConfig . ScrapeFailureLogFile
}
2023-05-30 01:22:23 -07:00
2023-10-10 03:16:55 -07:00
if c . ScrapeProtocols == nil {
c . ScrapeProtocols = globalConfig . ScrapeProtocols
}
if err := validateAcceptScrapeProtocols ( c . ScrapeProtocols ) ; err != nil {
return fmt . Errorf ( "%w for scrape config with job name %q" , err , c . JobName )
}
2024-10-18 08:12:31 -07:00
if c . ScrapeFallbackProtocol != "" {
if err := c . ScrapeFallbackProtocol . Validate ( ) ; err != nil {
return fmt . Errorf ( "invalid fallback_scrape_protocol for scrape config with job name %q: %w" , c . JobName , err )
}
}
2024-07-18 11:08:21 -07:00
switch globalConfig . MetricNameValidationScheme {
2024-08-21 07:38:27 -07:00
case LegacyValidationConfig :
case "" , UTF8ValidationConfig :
2024-07-18 11:08:21 -07:00
if model . NameValidationScheme != model . UTF8Validation {
2024-08-21 07:38:27 -07:00
panic ( "utf8 name validation requested but model.NameValidationScheme is not set to UTF8" )
2024-07-18 11:08:21 -07:00
}
default :
return fmt . Errorf ( "unknown name validation method specified, must be either 'legacy' or 'utf8', got %s" , globalConfig . MetricNameValidationScheme )
}
2024-08-23 11:13:41 -07:00
if c . MetricNameValidationScheme == "" {
c . MetricNameValidationScheme = globalConfig . MetricNameValidationScheme
}
2024-07-18 11:08:21 -07:00
2023-02-24 02:47:12 -08:00
return nil
}
2020-08-20 05:48:26 -07:00
// MarshalYAML implements the yaml.Marshaler interface.
func ( c * ScrapeConfig ) MarshalYAML ( ) ( interface { } , error ) {
return discovery . MarshalYAMLWithInlineConfigs ( c )
}
2021-07-19 21:52:57 -07:00
// StorageConfig configures runtime reloadable configuration options.
type StorageConfig struct {
2022-09-20 10:05:50 -07:00
TSDBConfig * TSDBConfig ` yaml:"tsdb,omitempty" `
2021-07-19 21:52:57 -07:00
ExemplarsConfig * ExemplarsConfig ` yaml:"exemplars,omitempty" `
}
2022-09-20 10:05:50 -07:00
// TSDBConfig configures runtime reloadable configuration options.
type TSDBConfig struct {
// OutOfOrderTimeWindow sets how long back in time an out-of-order sample can be inserted
// into the TSDB. This flag is typically set while unmarshaling the configuration file and translating
// OutOfOrderTimeWindowFlag's duration. The unit of this flag is expected to be the same as any
// other timestamp in the TSDB.
OutOfOrderTimeWindow int64
// OutOfOrderTimeWindowFlag holds the parsed duration from the config file.
// During unmarshall, this is converted into milliseconds and stored in OutOfOrderTimeWindow.
// This should not be used directly and must be converted into OutOfOrderTimeWindow.
OutOfOrderTimeWindowFlag model . Duration ` yaml:"out_of_order_time_window,omitempty" `
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func ( t * TSDBConfig ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
* t = TSDBConfig { }
type plain TSDBConfig
if err := unmarshal ( ( * plain ) ( t ) ) ; err != nil {
return err
}
t . OutOfOrderTimeWindow = time . Duration ( t . OutOfOrderTimeWindowFlag ) . Milliseconds ( )
return nil
}
2022-01-25 02:08:04 -08:00
type TracingClientType string
const (
TracingClientHTTP TracingClientType = "http"
TracingClientGRPC TracingClientType = "grpc"
2022-02-22 08:07:30 -08:00
GzipCompression = "gzip"
2022-01-25 02:08:04 -08:00
)
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func ( t * TracingClientType ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
* t = TracingClientType ( "" )
type plain TracingClientType
if err := unmarshal ( ( * plain ) ( t ) ) ; err != nil {
return err
}
if * t != TracingClientHTTP && * t != TracingClientGRPC {
return fmt . Errorf ( "expected tracing client type to be to be %s or %s, but got %s" ,
TracingClientHTTP , TracingClientGRPC , * t ,
)
}
return nil
}
// TracingConfig configures the tracing options.
type TracingConfig struct {
ClientType TracingClientType ` yaml:"client_type,omitempty" `
Endpoint string ` yaml:"endpoint,omitempty" `
SamplingFraction float64 ` yaml:"sampling_fraction,omitempty" `
2022-01-29 14:56:44 -08:00
Insecure bool ` yaml:"insecure,omitempty" `
2022-01-25 02:08:04 -08:00
TLSConfig config . TLSConfig ` yaml:"tls_config,omitempty" `
2022-02-22 08:07:30 -08:00
Headers map [ string ] string ` yaml:"headers,omitempty" `
Compression string ` yaml:"compression,omitempty" `
Timeout model . Duration ` yaml:"timeout,omitempty" `
2022-01-25 02:08:04 -08:00
}
2022-01-29 14:56:44 -08:00
// SetDirectory joins any relative file paths with dir.
func ( t * TracingConfig ) SetDirectory ( dir string ) {
t . TLSConfig . SetDirectory ( dir )
}
2022-01-25 02:08:04 -08:00
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func ( t * TracingConfig ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
2022-01-29 14:56:44 -08:00
* t = TracingConfig {
ClientType : TracingClientGRPC ,
}
2022-01-25 02:08:04 -08:00
type plain TracingConfig
if err := unmarshal ( ( * plain ) ( t ) ) ; err != nil {
return err
}
2022-02-22 12:44:36 -08:00
if err := validateHeadersForTracing ( t . Headers ) ; err != nil {
2022-02-22 08:07:30 -08:00
return err
}
2022-01-25 02:08:04 -08:00
if t . Endpoint == "" {
return errors . New ( "tracing endpoint must be set" )
}
2022-02-22 08:07:30 -08:00
if t . Compression != "" && t . Compression != GzipCompression {
return fmt . Errorf ( "invalid compression type %s provided, valid options: %s" ,
t . Compression , GzipCompression )
}
2022-01-25 02:08:04 -08:00
return nil
}
2021-07-19 21:52:57 -07:00
// ExemplarsConfig configures runtime reloadable configuration options.
type ExemplarsConfig struct {
// MaxExemplars sets the size, in # of exemplars stored, of the single circular buffer used to store exemplars in memory.
// Use a value of 0 or less than 0 to disable the storage without having to restart Prometheus.
MaxExemplars int64 ` yaml:"max_exemplars,omitempty" `
}
2016-11-25 02:04:33 -08:00
// AlertingConfig configures alerting and alertmanager related configs.
2016-11-23 03:41:19 -08:00
type AlertingConfig struct {
2019-12-12 08:00:19 -08:00
AlertRelabelConfigs [ ] * relabel . Config ` yaml:"alert_relabel_configs,omitempty" `
AlertmanagerConfigs AlertmanagerConfigs ` yaml:"alertmanagers,omitempty" `
2016-11-23 03:41:19 -08:00
}
2020-08-20 05:48:26 -07:00
// SetDirectory joins any relative file paths with dir.
func ( c * AlertingConfig ) SetDirectory ( dir string ) {
for _ , c := range c . AlertmanagerConfigs {
c . SetDirectory ( dir )
}
}
2016-11-23 03:41:19 -08:00
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func ( c * AlertingConfig ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
// Create a clean global config as the previous one was already populated
// by the default due to the YAML parser behavior for empty blocks.
* c = AlertingConfig { }
type plain AlertingConfig
2018-12-03 02:09:02 -08:00
if err := unmarshal ( ( * plain ) ( c ) ) ; err != nil {
return err
}
for _ , rlcfg := range c . AlertRelabelConfigs {
if rlcfg == nil {
2019-03-25 16:01:12 -07:00
return errors . New ( "empty or null alert relabeling rule" )
2018-12-03 02:09:02 -08:00
}
}
return nil
2016-11-23 03:41:19 -08:00
}
2019-12-12 08:00:19 -08:00
// AlertmanagerConfigs is a slice of *AlertmanagerConfig.
type AlertmanagerConfigs [ ] * AlertmanagerConfig
// ToMap converts a slice of *AlertmanagerConfig to a map.
func ( a AlertmanagerConfigs ) ToMap ( ) map [ string ] * AlertmanagerConfig {
ret := make ( map [ string ] * AlertmanagerConfig )
for i := range a {
ret [ fmt . Sprintf ( "config-%d" , i ) ] = a [ i ]
}
return ret
}
2019-04-18 05:17:03 -07:00
// AlertmanagerAPIVersion represents a version of the
// github.com/prometheus/alertmanager/api, e.g. 'v1' or 'v2'.
2024-10-18 06:23:14 -07:00
// 'v1' is no longer supported.
2019-04-18 05:17:03 -07:00
type AlertmanagerAPIVersion string
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func ( v * AlertmanagerAPIVersion ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
* v = AlertmanagerAPIVersion ( "" )
type plain AlertmanagerAPIVersion
if err := unmarshal ( ( * plain ) ( v ) ) ; err != nil {
return err
}
for _ , supportedVersion := range SupportedAlertmanagerAPIVersions {
if * v == supportedVersion {
return nil
}
}
return fmt . Errorf ( "expected Alertmanager api version to be one of %v but got %v" , SupportedAlertmanagerAPIVersions , * v )
}
const (
// AlertmanagerAPIVersionV1 represents
// github.com/prometheus/alertmanager/api/v1.
AlertmanagerAPIVersionV1 AlertmanagerAPIVersion = "v1"
// AlertmanagerAPIVersionV2 represents
// github.com/prometheus/alertmanager/api/v2.
AlertmanagerAPIVersionV2 AlertmanagerAPIVersion = "v2"
)
var SupportedAlertmanagerAPIVersions = [ ] AlertmanagerAPIVersion {
2024-10-18 06:23:14 -07:00
AlertmanagerAPIVersionV2 ,
2019-04-18 05:17:03 -07:00
}
2017-03-18 14:32:08 -07:00
// AlertmanagerConfig configures how Alertmanagers can be discovered and communicated with.
2016-11-24 06:17:50 -08:00
type AlertmanagerConfig struct {
2016-11-23 03:42:33 -08:00
// We cannot do proper Go type embedding below as the parser will then parse
// values arbitrarily into the overflow maps of further-down types.
2020-08-20 05:48:26 -07:00
ServiceDiscoveryConfigs discovery . Configs ` yaml:"-" `
HTTPClientConfig config . HTTPClientConfig ` yaml:",inline" `
2023-08-31 18:43:48 -07:00
SigV4Config * sigv4 . SigV4Config ` yaml:"sigv4,omitempty" `
2016-11-23 03:42:33 -08:00
// The URL scheme to use when talking to Alertmanagers.
Scheme string ` yaml:"scheme,omitempty" `
// Path prefix to add in front of the push endpoint path.
PathPrefix string ` yaml:"path_prefix,omitempty" `
// The timeout used when sending alerts.
2018-08-24 07:55:21 -07:00
Timeout model . Duration ` yaml:"timeout,omitempty" `
2016-11-23 03:42:33 -08:00
2019-04-18 05:17:03 -07:00
// The api version of Alertmanager.
APIVersion AlertmanagerAPIVersion ` yaml:"api_version" `
2016-11-23 03:42:33 -08:00
// List of Alertmanager relabel configurations.
2018-12-18 03:26:36 -08:00
RelabelConfigs [ ] * relabel . Config ` yaml:"relabel_configs,omitempty" `
2023-07-11 14:44:23 -07:00
// Relabel alerts before sending to the specific alertmanager.
AlertRelabelConfigs [ ] * relabel . Config ` yaml:"alert_relabel_configs,omitempty" `
2016-11-23 03:42:33 -08:00
}
2020-08-20 05:48:26 -07:00
// SetDirectory joins any relative file paths with dir.
func ( c * AlertmanagerConfig ) SetDirectory ( dir string ) {
c . ServiceDiscoveryConfigs . SetDirectory ( dir )
c . HTTPClientConfig . SetDirectory ( dir )
}
2016-11-23 03:42:33 -08:00
// UnmarshalYAML implements the yaml.Unmarshaler interface.
2016-11-24 06:17:50 -08:00
func ( c * AlertmanagerConfig ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
2016-11-25 02:04:33 -08:00
* c = DefaultAlertmanagerConfig
2020-08-20 05:48:26 -07:00
if err := discovery . UnmarshalYAMLWithInlineConfigs ( c , unmarshal ) ; err != nil {
2016-11-23 03:42:33 -08:00
return err
}
2016-11-24 06:17:50 -08:00
2016-11-23 03:42:33 -08:00
// The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer.
// We cannot make it a pointer as the parser panics for inlined pointer structs.
// Thus we just do its validation here.
Refactor SD configuration to remove `config` dependency (#3629)
* refactor: move targetGroup struct and CheckOverflow() to their own package
* refactor: move auth and security related structs to a utility package, fix import error in utility package
* refactor: Azure SD, remove SD struct from config
* refactor: DNS SD, remove SD struct from config into dns package
* refactor: ec2 SD, move SD struct from config into the ec2 package
* refactor: file SD, move SD struct from config to file discovery package
* refactor: gce, move SD struct from config to gce discovery package
* refactor: move HTTPClientConfig and URL into util/config, fix import error in httputil
* refactor: consul, move SD struct from config into consul discovery package
* refactor: marathon, move SD struct from config into marathon discovery package
* refactor: triton, move SD struct from config to triton discovery package, fix test
* refactor: zookeeper, move SD structs from config to zookeeper discovery package
* refactor: openstack, remove SD struct from config, move into openstack discovery package
* refactor: kubernetes, move SD struct from config into kubernetes discovery package
* refactor: notifier, use targetgroup package instead of config
* refactor: tests for file, marathon, triton SD - use targetgroup package instead of config.TargetGroup
* refactor: retrieval, use targetgroup package instead of config.TargetGroup
* refactor: storage, use config util package
* refactor: discovery manager, use targetgroup package instead of config.TargetGroup
* refactor: use HTTPClient and TLS config from configUtil instead of config
* refactor: tests, use targetgroup package instead of config.TargetGroup
* refactor: fix tagetgroup.Group pointers that were removed by mistake
* refactor: openstack, kubernetes: drop prefixes
* refactor: remove import aliases forced due to vscode bug
* refactor: move main SD struct out of config into discovery/config
* refactor: rename configUtil to config_util
* refactor: rename yamlUtil to yaml_config
* refactor: kubernetes, remove prefixes
* refactor: move the TargetGroup package to discovery/
* refactor: fix order of imports
2017-12-29 12:01:34 -08:00
if err := c . HTTPClientConfig . Validate ( ) ; err != nil {
2016-11-24 06:17:50 -08:00
return err
2016-11-23 03:42:33 -08:00
}
2023-08-31 18:43:48 -07:00
httpClientConfigAuthEnabled := c . HTTPClientConfig . BasicAuth != nil ||
c . HTTPClientConfig . Authorization != nil || c . HTTPClientConfig . OAuth2 != nil
if httpClientConfigAuthEnabled && c . SigV4Config != nil {
2024-11-03 04:15:51 -08:00
return errors . New ( "at most one of basic_auth, authorization, oauth2, & sigv4 must be configured" )
2023-08-31 18:43:48 -07:00
}
2016-11-23 03:42:33 -08:00
// Check for users putting URLs in target groups.
if len ( c . RelabelConfigs ) == 0 {
2020-08-20 05:48:26 -07:00
if err := checkStaticTargets ( c . ServiceDiscoveryConfigs ) ; err != nil {
return err
2016-11-23 03:42:33 -08:00
}
}
2018-06-13 08:34:59 -07:00
2018-12-03 02:09:02 -08:00
for _ , rlcfg := range c . RelabelConfigs {
if rlcfg == nil {
2019-03-25 16:01:12 -07:00
return errors . New ( "empty or null Alertmanager target relabeling rule" )
2018-12-03 02:09:02 -08:00
}
}
2023-07-11 14:44:23 -07:00
for _ , rlcfg := range c . AlertRelabelConfigs {
if rlcfg == nil {
return errors . New ( "empty or null Alertmanager alert relabeling rule" )
}
}
2020-08-20 05:48:26 -07:00
return nil
}
// MarshalYAML implements the yaml.Marshaler interface.
func ( c * AlertmanagerConfig ) MarshalYAML ( ) ( interface { } , error ) {
return discovery . MarshalYAMLWithInlineConfigs ( c )
}
2018-06-13 08:34:59 -07:00
2020-08-20 05:48:26 -07:00
func checkStaticTargets ( configs discovery . Configs ) error {
for _ , cfg := range configs {
sc , ok := cfg . ( discovery . StaticConfig )
if ! ok {
continue
}
for _ , tg := range sc {
for _ , t := range tg . Targets {
if err := CheckTargetAddress ( t [ model . AddressLabel ] ) ; err != nil {
return err
}
}
}
}
2016-11-23 03:42:33 -08:00
return nil
}
2015-11-07 06:25:51 -08:00
// CheckTargetAddress checks if target address is valid.
func CheckTargetAddress ( address model . LabelValue ) error {
// For now check for a URL, we may want to expand this later.
if strings . Contains ( string ( address ) , "/" ) {
2022-05-23 00:54:32 -07:00
return fmt . Errorf ( "%q is not a valid hostname" , address )
2015-11-07 06:25:51 -08:00
}
return nil
}
[PRW 2.0] Merging `remote-write-2.0` feature branch to main (PRW 2.0 support + metadata in WAL) (#14395)
* Remote Write 1.1: e2e benchmarks (#13102)
* Remote Write e2e benchmarks
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Prometheus ports automatically assigned
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make dashboard editable + more modular to different job label values
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Dashboard improvements
* memory stats
* diffs look at counter increases
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* run script: absolute path for config templates
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* grafana dashboard improvements
* show actual values of metrics
* add memory stats and diff
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* dashboard changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
* replace snappy encoding library
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto types
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add decode function for new write request proto
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add lookup table struct that is used to build the symbol table in new
write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Implement code paths for new proto format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example server to include handler for new format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add new test client
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* tests and new -> original proto mapping util
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto support on receiver end
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Fix test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* no-brainer copypaste but more performance write support
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove some comented code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix mocks and fixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add basic reduce remote write handler benchmark
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor out common code between write methods
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix: queue manager to include float histograms in new requests
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add sender-side tests and fix failing ones
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor queue manager code to remove some duplication
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Improve sender benchmarks and some allocations
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use github.com/golang/snappy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup: remove hardcoded fake url for testing
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add 1.1 version handling code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Remove config, update proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* gofmt
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix NewWriteClient and change new flags wording
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fields rewording in handler
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remote write handler to checks version header
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix typo in log
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add minmized remote write proto format
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functions for translating between new proto formats symbol table and
actual prometheus labels
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functionality for new minimized remote write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor things
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Make LabelSymbols a fixed32
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove unused proto type
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update tests
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build for stringlabels tag
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use two uint32 to encode (offset,leng)
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* manually optimize varint marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use unsafe []byte->string cast to reuse buffer
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix writeRequestMinimizedFixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all code from previous interning approach
the 'minimized' version is now the only v1.1 version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* minimally-tested exemplar support for rw 1.1
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor new version flag to make it easier to pick a specific format
instead of having multiple flags, plus add new formats for testing
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use exp slices for backwards compat. to go 1.20 plus add copyright
header to test file
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix label ranging
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add bytes slice (instead of slice of 32bit vars) format for testing
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* test additional len and lenbytes formats
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove mistaken package lock changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove formats we've decided not to use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove more format types we probably won't use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* More cleanup
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use require instead of assert in custom marshal test
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup; remove some unused functions
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, mostly linting fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove package-lock.json change again
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, address review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix test panic
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor lint issue + use labels Range function since it looks like
the tests fail to do `range labels.Labels` on CI
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* new interning format based on []string indeces
Co-authored-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all new rw formats but the []string one
also adapt tests to the new format
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup rwSymbolTable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add some TODOs for later
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* don't reserve field 3 for new proto and add TODO
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix custom marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* additional merge fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix server example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* revert package-lock.json changes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example prometheus version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* define separate proto types for remote write 2.0
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* rename new proto types and move to separate pkg
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update prometheus version for example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make Metadata not nullable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove old MinSample proto message
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* change enum names to fit buf build recommend enum naming and lint rules
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* remote: Added test for classic histogram grouping when sending rw; Fixed queue manager test delay. (#13421)
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Remote write v2: metadata support in every write request (#13394)
* Approach bundling metadata along with samples and exemplars
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Add first test; rebase with main
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Alternative approach: bundle metadata in TimeSeries protobuf
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* update go mod to match main branch
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix after rebase
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* we're not going to modify the 1.X format anymore
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Modify AppendMetadata based on the fact that we be putting metadata into
timeseries
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Rename enums for remote write versions to something that makes more
sense + remove the added `sendMetadata` flag.
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename flag that enables writing of metadata records to the WAL
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* additional clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix usage of require.Len
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* some clean up from review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more review fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
* remote write 2.0: sync with `main` branch (#13510)
* consoles: exclude iowait and steal from CPU Utilisation
'iowait' and 'steal' indicate specific idle/wait states, which shouldn't
be counted into CPU Utilisation. Also see
https://github.com/prometheus-operator/kube-prometheus/pull/796 and
https://github.com/kubernetes-monitoring/kubernetes-mixin/pull/667.
Per the iostat man page:
%idle
Show the percentage of time that the CPU or CPUs were idle and the
system did not have an outstanding disk I/O request.
%iowait
Show the percentage of time that the CPU or CPUs were idle during
which the system had an outstanding disk I/O request.
%steal
Show the percentage of time spent in involuntary wait by the
virtual CPU or CPUs while the hypervisor was servicing another
virtual processor.
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
* tsdb: shrink txRing with smaller integers
4 billion active transactions ought to be enough for anyone.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: create isolation transaction slice on demand
When Prometheus restarts it creates every series read in from the WAL,
but many of those series will be finished, and never receive any more
samples. By defering allocation of the txRing slice to when it is first
needed, we save 32 bytes per stale series.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* add cluster variable to Overview dashboard
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* promql: simplify Native Histogram arithmetics
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
* Cut 2.49.0-rc.0 (#13270)
* Cut 2.49.0-rc.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed the duplicate.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Add unit protobuf parser
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Go on adding protobuf parsing for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* ui: create a reproduction for https://github.com/prometheus/prometheus/issues/13292
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Get conditional right
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Get VM Scale Set NIC (#13283)
Calling `*armnetwork.InterfacesClient.Get()` doesn't work for Scale Set
VM NIC, because these use a different Resource ID format.
Use `*armnetwork.InterfacesClient.GetVirtualMachineScaleSetNetworkInterface()`
instead. This needs both the scale set name and the instance ID, so
add an `InstanceID` field to the `virtualMachine` struct. `InstanceID`
is empty for a VM that isn't a ScaleSetVM.
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
* Cut v2.49.0-rc.1
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Delete debugging lines, amend error message for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Correct order in error message
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Consider storage.ErrTooOldSample as non-retryable
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
* scrape_test.go: Increase scrape interval in TestScrapeLoopCache to reduce potential flakiness
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Avoid creating string for suffix, consider counters without _total suffix
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* build(deps): bump github.com/prometheus/client_golang
Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.17.0 to 1.18.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/client_golang/compare/v1.17.0...v1.18.0)
---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump actions/setup-node from 3.8.1 to 4.0.1
Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3.8.1 to 4.0.1.
- [Release notes](https://github.com/actions/setup-node/releases)
- [Commits](https://github.com/actions/setup-node/compare/5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d...b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8)
---
updated-dependencies:
- dependency-name: actions/setup-node
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
* scripts: sort file list in embed directive
Otherwise the resulting string depends on find, which afaict depends on
the underlying filesystem. A stable file list make it easier to detect
UI changes in downstreams that need to track UI assets.
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
* Fix DataTableProps['data'] for resultType string
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* Fix handling of scalar and string in isHeatmapData
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* build(deps): bump github.com/influxdata/influxdb
Bumps [github.com/influxdata/influxdb](https://github.com/influxdata/influxdb) from 1.11.2 to 1.11.4.
- [Release notes](https://github.com/influxdata/influxdb/releases)
- [Commits](https://github.com/influxdata/influxdb/compare/v1.11.2...v1.11.4)
---
updated-dependencies:
- dependency-name: github.com/influxdata/influxdb
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump github.com/prometheus/prometheus
Bumps [github.com/prometheus/prometheus](https://github.com/prometheus/prometheus) from 0.48.0 to 0.48.1.
- [Release notes](https://github.com/prometheus/prometheus/releases)
- [Changelog](https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/prometheus/compare/v0.48.0...v0.48.1)
---
updated-dependencies:
- dependency-name: github.com/prometheus/prometheus
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* Bump client_golang to v1.18.0 (#13373)
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Drop old inmemory samples (#13002)
* Drop old inmemory samples
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Avoid copying timeseries when the feature is disabled
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Run gofmt
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Clarify docs
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Add more logging info
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Remove loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* optimize function and add tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Simplify filter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Update help info from metrics
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use metrics to keep track of drop elements during buildWriteRequest
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var in tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* pass time.Now as parameter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Change buildwriterequest during retries
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Revert "Remove loggers"
This reverts commit 54f91dfcae20488944162335ab4ad8be459df1ab.
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use log level debug for loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Fix linter
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove noisy debug-level logs; add 'reason' label to drop metrics
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove accidentally committed files
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Propagate logger to buildWriteRequest to log dropped data
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix docs comment
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Make drop reason more specific
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove unnecessary pass of logger
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Use snake_case for reason label
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix dropped samples metric
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
---------
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
* fix(discovery): allow requireUpdate util to timeout in discovery/file/file_test.go.
The loop ran indefinitely if the condition isn't met.
Before, each iteration created a new timer channel which was always outpaced by
the other timer channel with smaller duration.
minor detail: There was a memory leak: resources of the ~10 previous timers were
constantly kept. With the fix, we may keep the resources of one timer around for defaultWait
but this isn't worth the changes to make it right.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Merge pull request #13371 from kevinmingtarja/fix-isHeatmapData
ui: fix handling of scalar and string in isHeatmapData
* tsdb/{index,compact}: allow using custom postings encoding format (#13242)
* tsdb/{index,compact}: allow using custom postings encoding format
We would like to experiment with a different postings encoding format in
Thanos so in this change I am proposing adding another argument to
`NewWriter` which would allow users to change the format if needed.
Also, wire the leveled compactor so that it would be possible to change
the format there too.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb/compact: use a struct for leveled compactor options
As discussed on Slack, let's use a struct for the options in leveled
compactor.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: make changes after Bryan's review
- Make changes less intrusive
- Turn the postings encoder type into a function
- Add NewWriterWithEncoder()
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
---------
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0-rc.2
Signed-off-by: bwplotka <bwplotka@gmail.com>
* build(deps): bump actions/setup-go from 3.5.0 to 5.0.0 in /scripts (#13362)
Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3.5.0 to 5.0.0.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/6edd4406fa81c3da01a34fa6f6343087c207a568...0c52d547c9bc32b1aa3301fd7a9cb496313a4491)
---
updated-dependencies:
- dependency-name: actions/setup-go
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump github/codeql-action from 2.22.8 to 3.22.12 (#13358)
Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.22.8 to 3.22.12.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/407ffafae6a767df3e0230c3df91b6443ae8df75...012739e5082ff0c22ca6d6ab32e07c36df03c4a4)
---
updated-dependencies:
- dependency-name: github/codeql-action
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* put @nexucis has a release shepherd (#13383)
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
* Add analyze histograms command to promtool (#12331)
Add `query analyze` command to promtool
This command analyzes the buckets of classic and native histograms,
based on data queried from the Prometheus query API, i.e. it
doesn't require direct access to the TSDB files.
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* included instance in all necessary descriptions
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* tsdb/compact: fix passing merge func
Fixing a very small logical problem I've introduced :(.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: add enable overlapping compaction
This functionality is needed in downstream projects because they have a
separate component that does compaction.
Upstreaming
https://github.com/grafana/mimir-prometheus/blob/7c8e9a2a76fc729e9078889782928b2fdfe240e9/tsdb/compact.go#L323-L325.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* promtool: allow setting multiple matchers to "promtool tsdb dump" command. (#13296)
Conditions are ANDed inside the same matcher but matchers are ORed
Including unit tests for "promtool tsdb dump".
Refactor some matchers scraping utils.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Fixed changelog
Signed-off-by: bwplotka <bwplotka@gmail.com>
* tsdb/main: wire "EnableOverlappingCompaction" to tsdb.Options (#13398)
This added the https://github.com/prometheus/prometheus/pull/13393
"EnableOverlappingCompaction" parameter to the compactor code but not to
the tsdb.Options. I forgot about that. Add it to `tsdb.Options` too and
set it to `true` in Prometheus.
Copy/paste the description from
https://github.com/prometheus/prometheus/pull/13393#issuecomment-1891787986
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Issue #13268: fix quality value in accept header
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
* Cut 2.49.1 with scrape q= bugfix.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Cut 2.49.1 web package.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Restore more efficient version of NewPossibleNonCounterInfo annotation (#13022)
Restore more efficient version of NewPossibleNonCounterInfo annotation
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Fix regressions introduced by #13242
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* fix slice copy in 1.20 (#13389)
The slices package is added to the standard library in Go 1.21;
we need to import from the exp area to maintain compatibility with Go 1.20.
Signed-off-by: tyltr <tylitianrui@126.com>
* Docs: Query Basics: link to rate (#10538)
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
* chore(kubernetes): check preconditions earlier and avoid unnecessary checks or iterations
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Examples: link to `rate` for new users (#10535)
* Examples: link to `rate` for new users
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
* promql: use natural sort in sort_by_label and sort_by_label_desc (#13411)
These functions are intended for humans, as robots can already sort the results
however they please. Humans like things sorted "naturally":
* https://blog.codinghorror.com/sorting-for-humans-natural-sort-order/
A similar thing has been done to Grafana, which is also used by humans:
* https://github.com/grafana/grafana/pull/78024
* https://github.com/grafana/grafana/pull/78494
Signed-off-by: Ivan Babrou <github@ivan.computer>
* TestLabelValuesWithMatchers: Add test case
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* remove obsolete build tag
Signed-off-by: tyltr <tylitianrui@126.com>
* Upgrade some golang dependencies for resty 2.11
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
* Native Histograms: support `native_histogram_min_bucket_factor` in scrape_config (#13222)
Native Histograms: support native_histogram_min_bucket_factor in scrape_config
---------
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram (#13392)
Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Minor fixes to otlp vendor update script
Signed-off-by: Goutham <gouthamve@gmail.com>
* build(deps): bump github.com/hetznercloud/hcloud-go/v2
Bumps [github.com/hetznercloud/hcloud-go/v2](https://github.com/hetznercloud/hcloud-go) from 2.4.0 to 2.6.0.
- [Release notes](https://github.com/hetznercloud/hcloud-go/releases)
- [Changelog](https://github.com/hetznercloud/hcloud-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hetznercloud/hcloud-go/compare/v2.4.0...v2.6.0)
---
updated-dependencies:
- dependency-name: github.com/hetznercloud/hcloud-go/v2
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* Enhanced visibility for `promtool test rules` with JSON colored formatting (#13342)
* Added diff flag for unit test to improvise readability & debugging
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Removed blank spaces
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed linting error
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Added cli flags to documentation
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Revert unrrelated linting fixes
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed review suggestions
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Cleanup
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
---------
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* storage: skip merging when no remote storage configured
Prometheus is hard-coded to use a fanout storage between TSDB and
a remote storage which by default is empty.
This change detects the empty storage and skips merging between
result sets, which would make `Select()` sort results.
Bottom line: we skip a sort unless there really is some remote storage
configured.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Remove csmarchbanks from remote write owners (#13432)
I have not had the time to keep up with remote write and have no plans
to work on it in the near future so I am withdrawing my maintainership
of that part of the codebase. I continue to focus on client_python.
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
* add more context cancellation check at evaluation time
Signed-off-by: Ben Ye <benye@amazon.com>
* Optimize label values with matchers by taking shortcuts (#13426)
Don't calculate postings beforehand: we may not need them. If all
matchers are for the requested label, we can just filter its values.
Also, if there are no values at all, no need to run any kind of
logic.
Also add more labelValuesWithMatchers benchmarks
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
* Add automatic memory limit handling
Enable automatic detection of memory limits and configure GOMEMLIMIT to
match.
* Also includes a flag to allow controlling the reserved ratio.
Signed-off-by: SuperQ <superq@gmail.com>
* Update OSSF badge link (#13433)
Provide a more user friendly interface
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
* SD Managers taking over responsibility for registration of debug metrics (#13375)
SD Managers take over responsibility for SD metrics registration
---------
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Optimize histogram iterators (#13340)
Optimize histogram iterators
Histogram iterators allocate new objects in the AtHistogram and
AtFloatHistogram methods, which makes calculating rates over long
ranges expensive.
In #13215 we allowed an existing object to be reused
when converting an integer histogram to a float histogram. This commit follows
the same idea and allows injecting an existing object in the AtHistogram and
AtFloatHistogram methods. When the injected value is nil, iterators allocate
new histograms, otherwise they populate and return the injected object.
The commit also adds a CopyTo method to Histogram and FloatHistogram which
is used in the BufferedIterator to overwrite items in the ring instead of making
new copies.
Note that a specialized HPoint pool is needed for all of this to work
(`matrixSelectorHPool`).
---------
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* doc: Mark `mad_over_time` as experimental (#13440)
We forgot to do that in
https://github.com/prometheus/prometheus/pull/13059
Signed-off-by: beorn7 <beorn@grafana.com>
* Change metric label for Puppetdb from 'http' to 'puppetdb'
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
* mirror metrics.proto change & generate code
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
* TestHeadLabelValuesWithMatchers: Add test case (#13414)
Add test case to TestHeadLabelValuesWithMatchers, while fixing a couple
of typos in other test cases. Also enclosing some implicit sub-tests in a
`t.Run` call to make them explicitly sub-tests.
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* update all go dependencies (#13438)
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
* build(deps): bump the k8s-io group with 2 updates (#13454)
Bumps the k8s-io group with 2 updates: [k8s.io/api](https://github.com/kubernetes/api) and [k8s.io/client-go](https://github.com/kubernetes/client-go).
Updates `k8s.io/api` from 0.28.4 to 0.29.1
- [Commits](https://github.com/kubernetes/api/compare/v0.28.4...v0.29.1)
Updates `k8s.io/client-go` from 0.28.4 to 0.29.1
- [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/kubernetes/client-go/compare/v0.28.4...v0.29.1)
---
updated-dependencies:
- dependency-name: k8s.io/api
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
- dependency-name: k8s.io/client-go
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump the go-opentelemetry-io group with 1 update (#13453)
Bumps the go-opentelemetry-io group with 1 update: [go.opentelemetry.io/collector/semconv](https://github.com/open-telemetry/opentelemetry-collector).
Updates `go.opentelemetry.io/collector/semconv` from 0.92.0 to 0.93.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-collector/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-collector/blob/main/CHANGELOG-API.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-collector/compare/v0.92.0...v0.93.0)
---
updated-dependencies:
- dependency-name: go.opentelemetry.io/collector/semconv
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: go-opentelemetry-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump actions/upload-artifact from 3.1.3 to 4.0.0 (#13355)
Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.3 to 4.0.0.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/a8a3f3ad30e3422c9c7b888a15615d19a852ae32...c7d193f32edcb7bfad88892161225aeda64e9392)
---
updated-dependencies:
- dependency-name: actions/upload-artifact
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump bufbuild/buf-push-action (#13357)
Bumps [bufbuild/buf-push-action](https://github.com/bufbuild/buf-push-action) from 342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1 to a654ff18effe4641ebea4a4ce242c49800728459.
- [Release notes](https://github.com/bufbuild/buf-push-action/releases)
- [Commits](https://github.com/bufbuild/buf-push-action/compare/342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1...a654ff18effe4641ebea4a4ce242c49800728459)
---
updated-dependencies:
- dependency-name: bufbuild/buf-push-action
dependency-type: direct:production
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* Labels: Add DropMetricName function, used in PromQL (#13446)
This function is called very frequently when executing PromQL functions,
and we can do it much more efficiently inside Labels.
In the common case that `__name__` comes first in the labels, we simply
re-point to start at the next label, which is nearly free.
`DropMetricName` is now so cheap I removed the cache - benchmarks show
everything still goes faster.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: simplify internal series delete function (#13261)
Lifting an optimisation from Agent code, `seriesHashmap.del` can use
the unique series reference, doesn't need to check Labels.
Also streamline the logic for deleting from `unique` and `conflicts` maps,
and add some comments to help the next person.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* otlptranslator/update-copy.sh: Fix sed command lines
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Rollback k8s.io requirements (#13462)
Rollback k8s.io Go modules to v0.28.6 to avoid forcing upgrade of Go to
1.21. This allows us to keep compatibility with the currently supported
upstream Go releases.
Signed-off-by: SuperQ <superq@gmail.com>
* Make update-copy.sh work for both OSX and GNU sed
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Name @beorn7 and @krajorama as maintainers for native histograms
I have been the de-facto maintainer for native histograms from the
beginning. So let's put this into MAINTAINERS.md.
In addition, I hereby proposose George Krajcsovits AKA Krajo as a
co-maintainer. He has contributed a lot of native histogram code, but
more importantly, he has contributed substantially to reviewing other
contributors' native histogram code, up to a point where I was merely
rubberstamping the PRs he had already reviewed. I'm confident that he
is ready to to be granted commit rights as outlined in the
"Maintainers" section of the governance:
https://prometheus.io/governance/#maintainers
According to the same section of the governance, I will announce the
proposed change on the developers mailing list and will give some time
for lazy consensus before merging this PR.
Signed-off-by: beorn7 <beorn@grafana.com>
* ui/fix: correct url handling for stacked graphs (#13460)
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
* tsdb: use cheaper Mutex on series
Mutex is 8 bytes; RWMutex is 24 bytes and much more complicated. Since
`RLock` is only used in two places, `UpdateMetadata` and `Delete`,
neither of which are hotspots, we should use the cheaper one.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Fix last_over_time for native histograms
The last_over_time retains a histogram sample without making a copy.
This sample is now coming from the buffered iterator used for windowing functions,
and can be reused for reading subsequent samples as the iterator progresses.
I would propose copying the sample in the last_over_time function, similar to
how it is done for rate, sum_over_time and others.
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
* Implementation
NOTE:
Rebased from main after refactor in #13014
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Add feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactor concurrency control
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Optimising dependencies/dependents funcs to not produce new slices each request
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Rename flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring for performance, and to allow controller to be overridden
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Block until all rules, both sync & async, have completed evaluating
Updated & added tests
Review feedback nits
Return empty map if not indeterminate
Use highWatermark to track inflight requests counter
Appease the linter
Clarify feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Fix typo in CLI flag description
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Fixed auto-generated doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improve doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Simplify the design to update concurrency controller once the rule evaluation has done
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Add more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Added more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improved RuleConcurrencyController interface doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Introduced sequentialRuleEvalController
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Remove superfluous nil check in Group.metrics
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* api: Serialize discovered and target labels into JSON directly (#13469)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* api: Serialize discovered labels into JSON directly in dropped targets (#13484)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* Add ShardedPostings() support to TSDB (#10421)
This PR is a reference implementation of the proposal described in #10420.
In addition to what described in #10420, in this PR I've introduced labels.StableHash(). The idea is to offer an hashing function which doesn't change over time, and that's used by query sharding in order to get a stable behaviour over time. The implementation of labels.StableHash() is the hashing function used by Prometheus before stringlabels, and what's used by Grafana Mimir for query sharding (because built before stringlabels was a thing).
Follow up work
As mentioned in #10420, if this PR is accepted I'm also open to upload another foundamental piece used by Grafana Mimir query sharding to accelerate the query execution: an optional, configurable and fast in-memory cache for the series hashes.
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* storage/remote: document why two benchmarks are skipped
One was silently doing nothing; one was doing something but the work
didn't go up linearly with iteration count.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Pod status changes not discovered by Kube Endpoints SD (#13337)
* fix(discovery/kubernetes/endpoints): react to changes on Pods because some modifications can occur on them without triggering an update on the related Endpoints (The Pod phase changing from Pending to Running e.g.).
---------
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
* Small improvements, add const, remove copypasta (#8106)
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
* Proposal to improve FPointSlice and HPointSlice allocation. (#13448)
* Reusing points slice from previous series when the slice is under utilized
* Adding comments on the bench test
Signed-off-by: Alan Protasio <alanprot@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
---------
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
* remote write 2.0 - follow up improvements (#13478)
* move remote write proto version config from a remote storage config to a
per remote write configuration option
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename scrape config for metadata, fix 2.0 header var name/value (was
1.1), and more clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* address review comments, mostly lint fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* another lint fix
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint imports
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics. (#13502)
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Apply suggestions from code review
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
* prw2.0: Added support for "custom" layouts for native histogram proto (#13558)
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# prompb/write/v2/types.pb.go
* Update prompb/write/v2/types.proto
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Addressed comments, fixed test.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* first draft of content negotiation
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix race in test
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix another test race
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Almost done with lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix todos around 405 HEAD handling
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Changes based on review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Update storage/remote/client.go
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Latest updates to review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* latest tweaks
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* remote write 2.0 - content negotiation remediation (#13921)
* Consolidate renegotiation error into one, fix tests
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* fix metric name and actuall increment counter
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
---------
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fixes after main sync.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW 2.0] Moved rw2 proto to the full path (both package name and placement) (#13973)
undefined
* [PRW2.0] Remove benchmark scripts (#13949)
See rationales on https://docs.google.com/document/d/1Bpf7mYjrHUhPHkie0qlnZFxzgqf_L32kM8ZOknSdJrU/edit
Signed-off-by: bwplotka <bwplotka@gmail.com>
* rw20: Update prw commentary after Callum spec review (#14136)
* rw20: Update prw commentary after Callum spec review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update types.proto
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation. (#14330)
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* update custom marshalling
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed confusing comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (chain1) New Remote Write 2.0 Config options for 2.0-rc.1 spec. (#14335)
NOTE: For simple review this change does not touch remote/ packages, only main and configs.
Spec: https://prometheus.io/docs/specs/remote_write_spec_2_0
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (part 2) Removed automatic negotiation, updates for the latest spec semantics in remote pkg (#14329)
* [PRW-2.0] (part2) Moved to latest basic negotiation & spec semantics.
Spec: https://github.com/prometheus/docs/pull/2462
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# config/config.go
# docs/configuration/configuration.md
# storage/remote/queue_manager_test.go
# storage/remote/write.go
# web/api/v1/api.go
* Addressed comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* storage/remote tests: refactor: extract function newTestQueueManager
To reduce repetition.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use newTestQueueManager for test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* [PRW 2.0] (part3) moved type specific conversions to prompb and writev2 codecs.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added test for rwProtoMsgFlagParser; fixed TODO comment.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Renamed DecodeV2WriteRequestStr to DecodeWriteV2Request (with tests).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed comments on remote_storage example, updated it for 2.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed `--enable-feature=metadata-wal-records` docs and error when using PRW 2.0 without it.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments on custom*.go
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added TODO to genproto.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments in remote pkg.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added metadata validation to write handler test; fixed ToMetadata.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed rest of Callum comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed writev2.FromMetadataType (was wrongly using prompb).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* fix a few import whitespaces
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* add a default case with an error to the example RW receiver
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more minor import whitespace chagnes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Apply suggestions from code review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update storage/remote/queue_manager_test.go
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
Co-authored-by: Nicolás Pazos <32206519+npazosmendez@users.noreply.github.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: alexgreenbank <alex.greenbank@grafana.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
2024-07-04 14:29:20 -07:00
// RemoteWriteProtoMsg represents the known protobuf message for the remote write
// 1.0 and 2.0 specs.
type RemoteWriteProtoMsg string
// Validate returns error if the given reference for the protobuf message is not supported.
func ( s RemoteWriteProtoMsg ) Validate ( ) error {
switch s {
case RemoteWriteProtoMsgV1 , RemoteWriteProtoMsgV2 :
return nil
default :
return fmt . Errorf ( "unknown remote write protobuf message %v, supported: %v" , s , RemoteWriteProtoMsgs { RemoteWriteProtoMsgV1 , RemoteWriteProtoMsgV2 } . String ( ) )
}
}
type RemoteWriteProtoMsgs [ ] RemoteWriteProtoMsg
func ( m RemoteWriteProtoMsgs ) Strings ( ) [ ] string {
ret := make ( [ ] string , 0 , len ( m ) )
for _ , typ := range m {
ret = append ( ret , string ( typ ) )
}
return ret
}
func ( m RemoteWriteProtoMsgs ) String ( ) string {
return strings . Join ( m . Strings ( ) , ", " )
}
var (
2024-07-22 15:07:12 -07:00
// RemoteWriteProtoMsgV1 represents the `prometheus.WriteRequest` protobuf
// message introduced in the https://prometheus.io/docs/specs/remote_write_spec/,
// which will eventually be deprecated.
[PRW 2.0] Merging `remote-write-2.0` feature branch to main (PRW 2.0 support + metadata in WAL) (#14395)
* Remote Write 1.1: e2e benchmarks (#13102)
* Remote Write e2e benchmarks
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Prometheus ports automatically assigned
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make dashboard editable + more modular to different job label values
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Dashboard improvements
* memory stats
* diffs look at counter increases
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* run script: absolute path for config templates
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* grafana dashboard improvements
* show actual values of metrics
* add memory stats and diff
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* dashboard changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
* replace snappy encoding library
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto types
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add decode function for new write request proto
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add lookup table struct that is used to build the symbol table in new
write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Implement code paths for new proto format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example server to include handler for new format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add new test client
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* tests and new -> original proto mapping util
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto support on receiver end
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Fix test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* no-brainer copypaste but more performance write support
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove some comented code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix mocks and fixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add basic reduce remote write handler benchmark
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor out common code between write methods
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix: queue manager to include float histograms in new requests
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add sender-side tests and fix failing ones
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor queue manager code to remove some duplication
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Improve sender benchmarks and some allocations
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use github.com/golang/snappy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup: remove hardcoded fake url for testing
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add 1.1 version handling code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Remove config, update proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* gofmt
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix NewWriteClient and change new flags wording
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fields rewording in handler
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remote write handler to checks version header
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix typo in log
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add minmized remote write proto format
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functions for translating between new proto formats symbol table and
actual prometheus labels
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functionality for new minimized remote write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor things
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Make LabelSymbols a fixed32
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove unused proto type
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update tests
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build for stringlabels tag
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use two uint32 to encode (offset,leng)
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* manually optimize varint marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use unsafe []byte->string cast to reuse buffer
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix writeRequestMinimizedFixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all code from previous interning approach
the 'minimized' version is now the only v1.1 version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* minimally-tested exemplar support for rw 1.1
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor new version flag to make it easier to pick a specific format
instead of having multiple flags, plus add new formats for testing
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use exp slices for backwards compat. to go 1.20 plus add copyright
header to test file
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix label ranging
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add bytes slice (instead of slice of 32bit vars) format for testing
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* test additional len and lenbytes formats
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove mistaken package lock changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove formats we've decided not to use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove more format types we probably won't use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* More cleanup
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use require instead of assert in custom marshal test
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup; remove some unused functions
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, mostly linting fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove package-lock.json change again
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, address review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix test panic
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor lint issue + use labels Range function since it looks like
the tests fail to do `range labels.Labels` on CI
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* new interning format based on []string indeces
Co-authored-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all new rw formats but the []string one
also adapt tests to the new format
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup rwSymbolTable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add some TODOs for later
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* don't reserve field 3 for new proto and add TODO
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix custom marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* additional merge fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix server example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* revert package-lock.json changes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example prometheus version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* define separate proto types for remote write 2.0
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* rename new proto types and move to separate pkg
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update prometheus version for example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make Metadata not nullable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove old MinSample proto message
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* change enum names to fit buf build recommend enum naming and lint rules
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* remote: Added test for classic histogram grouping when sending rw; Fixed queue manager test delay. (#13421)
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Remote write v2: metadata support in every write request (#13394)
* Approach bundling metadata along with samples and exemplars
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Add first test; rebase with main
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Alternative approach: bundle metadata in TimeSeries protobuf
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* update go mod to match main branch
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix after rebase
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* we're not going to modify the 1.X format anymore
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Modify AppendMetadata based on the fact that we be putting metadata into
timeseries
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Rename enums for remote write versions to something that makes more
sense + remove the added `sendMetadata` flag.
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename flag that enables writing of metadata records to the WAL
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* additional clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix usage of require.Len
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* some clean up from review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more review fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
* remote write 2.0: sync with `main` branch (#13510)
* consoles: exclude iowait and steal from CPU Utilisation
'iowait' and 'steal' indicate specific idle/wait states, which shouldn't
be counted into CPU Utilisation. Also see
https://github.com/prometheus-operator/kube-prometheus/pull/796 and
https://github.com/kubernetes-monitoring/kubernetes-mixin/pull/667.
Per the iostat man page:
%idle
Show the percentage of time that the CPU or CPUs were idle and the
system did not have an outstanding disk I/O request.
%iowait
Show the percentage of time that the CPU or CPUs were idle during
which the system had an outstanding disk I/O request.
%steal
Show the percentage of time spent in involuntary wait by the
virtual CPU or CPUs while the hypervisor was servicing another
virtual processor.
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
* tsdb: shrink txRing with smaller integers
4 billion active transactions ought to be enough for anyone.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: create isolation transaction slice on demand
When Prometheus restarts it creates every series read in from the WAL,
but many of those series will be finished, and never receive any more
samples. By defering allocation of the txRing slice to when it is first
needed, we save 32 bytes per stale series.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* add cluster variable to Overview dashboard
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* promql: simplify Native Histogram arithmetics
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
* Cut 2.49.0-rc.0 (#13270)
* Cut 2.49.0-rc.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed the duplicate.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Add unit protobuf parser
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Go on adding protobuf parsing for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* ui: create a reproduction for https://github.com/prometheus/prometheus/issues/13292
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Get conditional right
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Get VM Scale Set NIC (#13283)
Calling `*armnetwork.InterfacesClient.Get()` doesn't work for Scale Set
VM NIC, because these use a different Resource ID format.
Use `*armnetwork.InterfacesClient.GetVirtualMachineScaleSetNetworkInterface()`
instead. This needs both the scale set name and the instance ID, so
add an `InstanceID` field to the `virtualMachine` struct. `InstanceID`
is empty for a VM that isn't a ScaleSetVM.
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
* Cut v2.49.0-rc.1
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Delete debugging lines, amend error message for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Correct order in error message
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Consider storage.ErrTooOldSample as non-retryable
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
* scrape_test.go: Increase scrape interval in TestScrapeLoopCache to reduce potential flakiness
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Avoid creating string for suffix, consider counters without _total suffix
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* build(deps): bump github.com/prometheus/client_golang
Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.17.0 to 1.18.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/client_golang/compare/v1.17.0...v1.18.0)
---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump actions/setup-node from 3.8.1 to 4.0.1
Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3.8.1 to 4.0.1.
- [Release notes](https://github.com/actions/setup-node/releases)
- [Commits](https://github.com/actions/setup-node/compare/5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d...b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8)
---
updated-dependencies:
- dependency-name: actions/setup-node
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
* scripts: sort file list in embed directive
Otherwise the resulting string depends on find, which afaict depends on
the underlying filesystem. A stable file list make it easier to detect
UI changes in downstreams that need to track UI assets.
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
* Fix DataTableProps['data'] for resultType string
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* Fix handling of scalar and string in isHeatmapData
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* build(deps): bump github.com/influxdata/influxdb
Bumps [github.com/influxdata/influxdb](https://github.com/influxdata/influxdb) from 1.11.2 to 1.11.4.
- [Release notes](https://github.com/influxdata/influxdb/releases)
- [Commits](https://github.com/influxdata/influxdb/compare/v1.11.2...v1.11.4)
---
updated-dependencies:
- dependency-name: github.com/influxdata/influxdb
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump github.com/prometheus/prometheus
Bumps [github.com/prometheus/prometheus](https://github.com/prometheus/prometheus) from 0.48.0 to 0.48.1.
- [Release notes](https://github.com/prometheus/prometheus/releases)
- [Changelog](https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/prometheus/compare/v0.48.0...v0.48.1)
---
updated-dependencies:
- dependency-name: github.com/prometheus/prometheus
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* Bump client_golang to v1.18.0 (#13373)
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Drop old inmemory samples (#13002)
* Drop old inmemory samples
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Avoid copying timeseries when the feature is disabled
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Run gofmt
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Clarify docs
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Add more logging info
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Remove loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* optimize function and add tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Simplify filter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Update help info from metrics
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use metrics to keep track of drop elements during buildWriteRequest
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var in tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* pass time.Now as parameter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Change buildwriterequest during retries
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Revert "Remove loggers"
This reverts commit 54f91dfcae20488944162335ab4ad8be459df1ab.
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use log level debug for loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Fix linter
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove noisy debug-level logs; add 'reason' label to drop metrics
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove accidentally committed files
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Propagate logger to buildWriteRequest to log dropped data
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix docs comment
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Make drop reason more specific
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove unnecessary pass of logger
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Use snake_case for reason label
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix dropped samples metric
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
---------
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
* fix(discovery): allow requireUpdate util to timeout in discovery/file/file_test.go.
The loop ran indefinitely if the condition isn't met.
Before, each iteration created a new timer channel which was always outpaced by
the other timer channel with smaller duration.
minor detail: There was a memory leak: resources of the ~10 previous timers were
constantly kept. With the fix, we may keep the resources of one timer around for defaultWait
but this isn't worth the changes to make it right.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Merge pull request #13371 from kevinmingtarja/fix-isHeatmapData
ui: fix handling of scalar and string in isHeatmapData
* tsdb/{index,compact}: allow using custom postings encoding format (#13242)
* tsdb/{index,compact}: allow using custom postings encoding format
We would like to experiment with a different postings encoding format in
Thanos so in this change I am proposing adding another argument to
`NewWriter` which would allow users to change the format if needed.
Also, wire the leveled compactor so that it would be possible to change
the format there too.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb/compact: use a struct for leveled compactor options
As discussed on Slack, let's use a struct for the options in leveled
compactor.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: make changes after Bryan's review
- Make changes less intrusive
- Turn the postings encoder type into a function
- Add NewWriterWithEncoder()
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
---------
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0-rc.2
Signed-off-by: bwplotka <bwplotka@gmail.com>
* build(deps): bump actions/setup-go from 3.5.0 to 5.0.0 in /scripts (#13362)
Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3.5.0 to 5.0.0.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/6edd4406fa81c3da01a34fa6f6343087c207a568...0c52d547c9bc32b1aa3301fd7a9cb496313a4491)
---
updated-dependencies:
- dependency-name: actions/setup-go
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump github/codeql-action from 2.22.8 to 3.22.12 (#13358)
Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.22.8 to 3.22.12.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/407ffafae6a767df3e0230c3df91b6443ae8df75...012739e5082ff0c22ca6d6ab32e07c36df03c4a4)
---
updated-dependencies:
- dependency-name: github/codeql-action
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* put @nexucis has a release shepherd (#13383)
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
* Add analyze histograms command to promtool (#12331)
Add `query analyze` command to promtool
This command analyzes the buckets of classic and native histograms,
based on data queried from the Prometheus query API, i.e. it
doesn't require direct access to the TSDB files.
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* included instance in all necessary descriptions
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* tsdb/compact: fix passing merge func
Fixing a very small logical problem I've introduced :(.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: add enable overlapping compaction
This functionality is needed in downstream projects because they have a
separate component that does compaction.
Upstreaming
https://github.com/grafana/mimir-prometheus/blob/7c8e9a2a76fc729e9078889782928b2fdfe240e9/tsdb/compact.go#L323-L325.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* promtool: allow setting multiple matchers to "promtool tsdb dump" command. (#13296)
Conditions are ANDed inside the same matcher but matchers are ORed
Including unit tests for "promtool tsdb dump".
Refactor some matchers scraping utils.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Fixed changelog
Signed-off-by: bwplotka <bwplotka@gmail.com>
* tsdb/main: wire "EnableOverlappingCompaction" to tsdb.Options (#13398)
This added the https://github.com/prometheus/prometheus/pull/13393
"EnableOverlappingCompaction" parameter to the compactor code but not to
the tsdb.Options. I forgot about that. Add it to `tsdb.Options` too and
set it to `true` in Prometheus.
Copy/paste the description from
https://github.com/prometheus/prometheus/pull/13393#issuecomment-1891787986
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Issue #13268: fix quality value in accept header
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
* Cut 2.49.1 with scrape q= bugfix.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Cut 2.49.1 web package.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Restore more efficient version of NewPossibleNonCounterInfo annotation (#13022)
Restore more efficient version of NewPossibleNonCounterInfo annotation
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Fix regressions introduced by #13242
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* fix slice copy in 1.20 (#13389)
The slices package is added to the standard library in Go 1.21;
we need to import from the exp area to maintain compatibility with Go 1.20.
Signed-off-by: tyltr <tylitianrui@126.com>
* Docs: Query Basics: link to rate (#10538)
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
* chore(kubernetes): check preconditions earlier and avoid unnecessary checks or iterations
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Examples: link to `rate` for new users (#10535)
* Examples: link to `rate` for new users
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
* promql: use natural sort in sort_by_label and sort_by_label_desc (#13411)
These functions are intended for humans, as robots can already sort the results
however they please. Humans like things sorted "naturally":
* https://blog.codinghorror.com/sorting-for-humans-natural-sort-order/
A similar thing has been done to Grafana, which is also used by humans:
* https://github.com/grafana/grafana/pull/78024
* https://github.com/grafana/grafana/pull/78494
Signed-off-by: Ivan Babrou <github@ivan.computer>
* TestLabelValuesWithMatchers: Add test case
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* remove obsolete build tag
Signed-off-by: tyltr <tylitianrui@126.com>
* Upgrade some golang dependencies for resty 2.11
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
* Native Histograms: support `native_histogram_min_bucket_factor` in scrape_config (#13222)
Native Histograms: support native_histogram_min_bucket_factor in scrape_config
---------
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram (#13392)
Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Minor fixes to otlp vendor update script
Signed-off-by: Goutham <gouthamve@gmail.com>
* build(deps): bump github.com/hetznercloud/hcloud-go/v2
Bumps [github.com/hetznercloud/hcloud-go/v2](https://github.com/hetznercloud/hcloud-go) from 2.4.0 to 2.6.0.
- [Release notes](https://github.com/hetznercloud/hcloud-go/releases)
- [Changelog](https://github.com/hetznercloud/hcloud-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hetznercloud/hcloud-go/compare/v2.4.0...v2.6.0)
---
updated-dependencies:
- dependency-name: github.com/hetznercloud/hcloud-go/v2
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* Enhanced visibility for `promtool test rules` with JSON colored formatting (#13342)
* Added diff flag for unit test to improvise readability & debugging
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Removed blank spaces
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed linting error
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Added cli flags to documentation
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Revert unrrelated linting fixes
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed review suggestions
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Cleanup
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
---------
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* storage: skip merging when no remote storage configured
Prometheus is hard-coded to use a fanout storage between TSDB and
a remote storage which by default is empty.
This change detects the empty storage and skips merging between
result sets, which would make `Select()` sort results.
Bottom line: we skip a sort unless there really is some remote storage
configured.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Remove csmarchbanks from remote write owners (#13432)
I have not had the time to keep up with remote write and have no plans
to work on it in the near future so I am withdrawing my maintainership
of that part of the codebase. I continue to focus on client_python.
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
* add more context cancellation check at evaluation time
Signed-off-by: Ben Ye <benye@amazon.com>
* Optimize label values with matchers by taking shortcuts (#13426)
Don't calculate postings beforehand: we may not need them. If all
matchers are for the requested label, we can just filter its values.
Also, if there are no values at all, no need to run any kind of
logic.
Also add more labelValuesWithMatchers benchmarks
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
* Add automatic memory limit handling
Enable automatic detection of memory limits and configure GOMEMLIMIT to
match.
* Also includes a flag to allow controlling the reserved ratio.
Signed-off-by: SuperQ <superq@gmail.com>
* Update OSSF badge link (#13433)
Provide a more user friendly interface
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
* SD Managers taking over responsibility for registration of debug metrics (#13375)
SD Managers take over responsibility for SD metrics registration
---------
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Optimize histogram iterators (#13340)
Optimize histogram iterators
Histogram iterators allocate new objects in the AtHistogram and
AtFloatHistogram methods, which makes calculating rates over long
ranges expensive.
In #13215 we allowed an existing object to be reused
when converting an integer histogram to a float histogram. This commit follows
the same idea and allows injecting an existing object in the AtHistogram and
AtFloatHistogram methods. When the injected value is nil, iterators allocate
new histograms, otherwise they populate and return the injected object.
The commit also adds a CopyTo method to Histogram and FloatHistogram which
is used in the BufferedIterator to overwrite items in the ring instead of making
new copies.
Note that a specialized HPoint pool is needed for all of this to work
(`matrixSelectorHPool`).
---------
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* doc: Mark `mad_over_time` as experimental (#13440)
We forgot to do that in
https://github.com/prometheus/prometheus/pull/13059
Signed-off-by: beorn7 <beorn@grafana.com>
* Change metric label for Puppetdb from 'http' to 'puppetdb'
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
* mirror metrics.proto change & generate code
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
* TestHeadLabelValuesWithMatchers: Add test case (#13414)
Add test case to TestHeadLabelValuesWithMatchers, while fixing a couple
of typos in other test cases. Also enclosing some implicit sub-tests in a
`t.Run` call to make them explicitly sub-tests.
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* update all go dependencies (#13438)
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
* build(deps): bump the k8s-io group with 2 updates (#13454)
Bumps the k8s-io group with 2 updates: [k8s.io/api](https://github.com/kubernetes/api) and [k8s.io/client-go](https://github.com/kubernetes/client-go).
Updates `k8s.io/api` from 0.28.4 to 0.29.1
- [Commits](https://github.com/kubernetes/api/compare/v0.28.4...v0.29.1)
Updates `k8s.io/client-go` from 0.28.4 to 0.29.1
- [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/kubernetes/client-go/compare/v0.28.4...v0.29.1)
---
updated-dependencies:
- dependency-name: k8s.io/api
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
- dependency-name: k8s.io/client-go
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump the go-opentelemetry-io group with 1 update (#13453)
Bumps the go-opentelemetry-io group with 1 update: [go.opentelemetry.io/collector/semconv](https://github.com/open-telemetry/opentelemetry-collector).
Updates `go.opentelemetry.io/collector/semconv` from 0.92.0 to 0.93.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-collector/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-collector/blob/main/CHANGELOG-API.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-collector/compare/v0.92.0...v0.93.0)
---
updated-dependencies:
- dependency-name: go.opentelemetry.io/collector/semconv
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: go-opentelemetry-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump actions/upload-artifact from 3.1.3 to 4.0.0 (#13355)
Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.3 to 4.0.0.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/a8a3f3ad30e3422c9c7b888a15615d19a852ae32...c7d193f32edcb7bfad88892161225aeda64e9392)
---
updated-dependencies:
- dependency-name: actions/upload-artifact
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump bufbuild/buf-push-action (#13357)
Bumps [bufbuild/buf-push-action](https://github.com/bufbuild/buf-push-action) from 342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1 to a654ff18effe4641ebea4a4ce242c49800728459.
- [Release notes](https://github.com/bufbuild/buf-push-action/releases)
- [Commits](https://github.com/bufbuild/buf-push-action/compare/342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1...a654ff18effe4641ebea4a4ce242c49800728459)
---
updated-dependencies:
- dependency-name: bufbuild/buf-push-action
dependency-type: direct:production
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* Labels: Add DropMetricName function, used in PromQL (#13446)
This function is called very frequently when executing PromQL functions,
and we can do it much more efficiently inside Labels.
In the common case that `__name__` comes first in the labels, we simply
re-point to start at the next label, which is nearly free.
`DropMetricName` is now so cheap I removed the cache - benchmarks show
everything still goes faster.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: simplify internal series delete function (#13261)
Lifting an optimisation from Agent code, `seriesHashmap.del` can use
the unique series reference, doesn't need to check Labels.
Also streamline the logic for deleting from `unique` and `conflicts` maps,
and add some comments to help the next person.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* otlptranslator/update-copy.sh: Fix sed command lines
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Rollback k8s.io requirements (#13462)
Rollback k8s.io Go modules to v0.28.6 to avoid forcing upgrade of Go to
1.21. This allows us to keep compatibility with the currently supported
upstream Go releases.
Signed-off-by: SuperQ <superq@gmail.com>
* Make update-copy.sh work for both OSX and GNU sed
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Name @beorn7 and @krajorama as maintainers for native histograms
I have been the de-facto maintainer for native histograms from the
beginning. So let's put this into MAINTAINERS.md.
In addition, I hereby proposose George Krajcsovits AKA Krajo as a
co-maintainer. He has contributed a lot of native histogram code, but
more importantly, he has contributed substantially to reviewing other
contributors' native histogram code, up to a point where I was merely
rubberstamping the PRs he had already reviewed. I'm confident that he
is ready to to be granted commit rights as outlined in the
"Maintainers" section of the governance:
https://prometheus.io/governance/#maintainers
According to the same section of the governance, I will announce the
proposed change on the developers mailing list and will give some time
for lazy consensus before merging this PR.
Signed-off-by: beorn7 <beorn@grafana.com>
* ui/fix: correct url handling for stacked graphs (#13460)
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
* tsdb: use cheaper Mutex on series
Mutex is 8 bytes; RWMutex is 24 bytes and much more complicated. Since
`RLock` is only used in two places, `UpdateMetadata` and `Delete`,
neither of which are hotspots, we should use the cheaper one.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Fix last_over_time for native histograms
The last_over_time retains a histogram sample without making a copy.
This sample is now coming from the buffered iterator used for windowing functions,
and can be reused for reading subsequent samples as the iterator progresses.
I would propose copying the sample in the last_over_time function, similar to
how it is done for rate, sum_over_time and others.
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
* Implementation
NOTE:
Rebased from main after refactor in #13014
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Add feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactor concurrency control
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Optimising dependencies/dependents funcs to not produce new slices each request
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Rename flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring for performance, and to allow controller to be overridden
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Block until all rules, both sync & async, have completed evaluating
Updated & added tests
Review feedback nits
Return empty map if not indeterminate
Use highWatermark to track inflight requests counter
Appease the linter
Clarify feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Fix typo in CLI flag description
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Fixed auto-generated doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improve doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Simplify the design to update concurrency controller once the rule evaluation has done
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Add more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Added more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improved RuleConcurrencyController interface doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Introduced sequentialRuleEvalController
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Remove superfluous nil check in Group.metrics
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* api: Serialize discovered and target labels into JSON directly (#13469)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* api: Serialize discovered labels into JSON directly in dropped targets (#13484)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* Add ShardedPostings() support to TSDB (#10421)
This PR is a reference implementation of the proposal described in #10420.
In addition to what described in #10420, in this PR I've introduced labels.StableHash(). The idea is to offer an hashing function which doesn't change over time, and that's used by query sharding in order to get a stable behaviour over time. The implementation of labels.StableHash() is the hashing function used by Prometheus before stringlabels, and what's used by Grafana Mimir for query sharding (because built before stringlabels was a thing).
Follow up work
As mentioned in #10420, if this PR is accepted I'm also open to upload another foundamental piece used by Grafana Mimir query sharding to accelerate the query execution: an optional, configurable and fast in-memory cache for the series hashes.
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* storage/remote: document why two benchmarks are skipped
One was silently doing nothing; one was doing something but the work
didn't go up linearly with iteration count.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Pod status changes not discovered by Kube Endpoints SD (#13337)
* fix(discovery/kubernetes/endpoints): react to changes on Pods because some modifications can occur on them without triggering an update on the related Endpoints (The Pod phase changing from Pending to Running e.g.).
---------
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
* Small improvements, add const, remove copypasta (#8106)
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
* Proposal to improve FPointSlice and HPointSlice allocation. (#13448)
* Reusing points slice from previous series when the slice is under utilized
* Adding comments on the bench test
Signed-off-by: Alan Protasio <alanprot@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
---------
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
* remote write 2.0 - follow up improvements (#13478)
* move remote write proto version config from a remote storage config to a
per remote write configuration option
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename scrape config for metadata, fix 2.0 header var name/value (was
1.1), and more clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* address review comments, mostly lint fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* another lint fix
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint imports
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics. (#13502)
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Apply suggestions from code review
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
* prw2.0: Added support for "custom" layouts for native histogram proto (#13558)
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# prompb/write/v2/types.pb.go
* Update prompb/write/v2/types.proto
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Addressed comments, fixed test.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* first draft of content negotiation
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix race in test
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix another test race
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Almost done with lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix todos around 405 HEAD handling
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Changes based on review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Update storage/remote/client.go
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Latest updates to review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* latest tweaks
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* remote write 2.0 - content negotiation remediation (#13921)
* Consolidate renegotiation error into one, fix tests
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* fix metric name and actuall increment counter
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
---------
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fixes after main sync.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW 2.0] Moved rw2 proto to the full path (both package name and placement) (#13973)
undefined
* [PRW2.0] Remove benchmark scripts (#13949)
See rationales on https://docs.google.com/document/d/1Bpf7mYjrHUhPHkie0qlnZFxzgqf_L32kM8ZOknSdJrU/edit
Signed-off-by: bwplotka <bwplotka@gmail.com>
* rw20: Update prw commentary after Callum spec review (#14136)
* rw20: Update prw commentary after Callum spec review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update types.proto
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation. (#14330)
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* update custom marshalling
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed confusing comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (chain1) New Remote Write 2.0 Config options for 2.0-rc.1 spec. (#14335)
NOTE: For simple review this change does not touch remote/ packages, only main and configs.
Spec: https://prometheus.io/docs/specs/remote_write_spec_2_0
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (part 2) Removed automatic negotiation, updates for the latest spec semantics in remote pkg (#14329)
* [PRW-2.0] (part2) Moved to latest basic negotiation & spec semantics.
Spec: https://github.com/prometheus/docs/pull/2462
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# config/config.go
# docs/configuration/configuration.md
# storage/remote/queue_manager_test.go
# storage/remote/write.go
# web/api/v1/api.go
* Addressed comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* storage/remote tests: refactor: extract function newTestQueueManager
To reduce repetition.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use newTestQueueManager for test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* [PRW 2.0] (part3) moved type specific conversions to prompb and writev2 codecs.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added test for rwProtoMsgFlagParser; fixed TODO comment.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Renamed DecodeV2WriteRequestStr to DecodeWriteV2Request (with tests).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed comments on remote_storage example, updated it for 2.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed `--enable-feature=metadata-wal-records` docs and error when using PRW 2.0 without it.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments on custom*.go
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added TODO to genproto.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments in remote pkg.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added metadata validation to write handler test; fixed ToMetadata.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed rest of Callum comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed writev2.FromMetadataType (was wrongly using prompb).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* fix a few import whitespaces
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* add a default case with an error to the example RW receiver
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more minor import whitespace chagnes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Apply suggestions from code review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update storage/remote/queue_manager_test.go
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
Co-authored-by: Nicolás Pazos <32206519+npazosmendez@users.noreply.github.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: alexgreenbank <alex.greenbank@grafana.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
2024-07-04 14:29:20 -07:00
//
// NOTE: This string is used for both HTTP header values and config value, so don't change
// this reference.
RemoteWriteProtoMsgV1 RemoteWriteProtoMsg = "prometheus.WriteRequest"
// RemoteWriteProtoMsgV2 represents the `io.prometheus.write.v2.Request` protobuf
// message introduced in https://prometheus.io/docs/specs/remote_write_spec_2_0/
//
// NOTE: This string is used for both HTTP header values and config value, so don't change
// this reference.
RemoteWriteProtoMsgV2 RemoteWriteProtoMsg = "io.prometheus.write.v2.Request"
)
2017-03-10 03:53:27 -08:00
// RemoteWriteConfig is the configuration for writing to remote storage.
2016-09-19 13:47:51 -07:00
type RemoteWriteConfig struct {
2022-07-14 06:13:12 -07:00
URL * config . URL ` yaml:"url" `
RemoteTimeout model . Duration ` yaml:"remote_timeout,omitempty" `
Headers map [ string ] string ` yaml:"headers,omitempty" `
WriteRelabelConfigs [ ] * relabel . Config ` yaml:"write_relabel_configs,omitempty" `
Name string ` yaml:"name,omitempty" `
SendExemplars bool ` yaml:"send_exemplars,omitempty" `
SendNativeHistograms bool ` yaml:"send_native_histograms,omitempty" `
2024-11-15 07:41:49 -08:00
RoundRobinDNS bool ` yaml:"round_robin_dns,omitempty" `
[PRW 2.0] Merging `remote-write-2.0` feature branch to main (PRW 2.0 support + metadata in WAL) (#14395)
* Remote Write 1.1: e2e benchmarks (#13102)
* Remote Write e2e benchmarks
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Prometheus ports automatically assigned
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make dashboard editable + more modular to different job label values
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Dashboard improvements
* memory stats
* diffs look at counter increases
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* run script: absolute path for config templates
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* grafana dashboard improvements
* show actual values of metrics
* add memory stats and diff
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* dashboard changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
* replace snappy encoding library
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto types
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add decode function for new write request proto
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add lookup table struct that is used to build the symbol table in new
write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Implement code paths for new proto format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example server to include handler for new format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add new test client
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* tests and new -> original proto mapping util
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto support on receiver end
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Fix test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* no-brainer copypaste but more performance write support
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove some comented code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix mocks and fixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add basic reduce remote write handler benchmark
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor out common code between write methods
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix: queue manager to include float histograms in new requests
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add sender-side tests and fix failing ones
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor queue manager code to remove some duplication
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Improve sender benchmarks and some allocations
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use github.com/golang/snappy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup: remove hardcoded fake url for testing
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add 1.1 version handling code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Remove config, update proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* gofmt
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix NewWriteClient and change new flags wording
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fields rewording in handler
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remote write handler to checks version header
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix typo in log
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add minmized remote write proto format
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functions for translating between new proto formats symbol table and
actual prometheus labels
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functionality for new minimized remote write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor things
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Make LabelSymbols a fixed32
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove unused proto type
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update tests
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build for stringlabels tag
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use two uint32 to encode (offset,leng)
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* manually optimize varint marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use unsafe []byte->string cast to reuse buffer
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix writeRequestMinimizedFixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all code from previous interning approach
the 'minimized' version is now the only v1.1 version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* minimally-tested exemplar support for rw 1.1
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor new version flag to make it easier to pick a specific format
instead of having multiple flags, plus add new formats for testing
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use exp slices for backwards compat. to go 1.20 plus add copyright
header to test file
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix label ranging
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add bytes slice (instead of slice of 32bit vars) format for testing
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* test additional len and lenbytes formats
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove mistaken package lock changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove formats we've decided not to use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove more format types we probably won't use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* More cleanup
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use require instead of assert in custom marshal test
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup; remove some unused functions
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, mostly linting fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove package-lock.json change again
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, address review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix test panic
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor lint issue + use labels Range function since it looks like
the tests fail to do `range labels.Labels` on CI
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* new interning format based on []string indeces
Co-authored-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all new rw formats but the []string one
also adapt tests to the new format
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup rwSymbolTable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add some TODOs for later
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* don't reserve field 3 for new proto and add TODO
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix custom marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* additional merge fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix server example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* revert package-lock.json changes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example prometheus version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* define separate proto types for remote write 2.0
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* rename new proto types and move to separate pkg
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update prometheus version for example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make Metadata not nullable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove old MinSample proto message
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* change enum names to fit buf build recommend enum naming and lint rules
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* remote: Added test for classic histogram grouping when sending rw; Fixed queue manager test delay. (#13421)
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Remote write v2: metadata support in every write request (#13394)
* Approach bundling metadata along with samples and exemplars
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Add first test; rebase with main
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Alternative approach: bundle metadata in TimeSeries protobuf
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* update go mod to match main branch
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix after rebase
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* we're not going to modify the 1.X format anymore
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Modify AppendMetadata based on the fact that we be putting metadata into
timeseries
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Rename enums for remote write versions to something that makes more
sense + remove the added `sendMetadata` flag.
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename flag that enables writing of metadata records to the WAL
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* additional clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix usage of require.Len
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* some clean up from review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more review fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
* remote write 2.0: sync with `main` branch (#13510)
* consoles: exclude iowait and steal from CPU Utilisation
'iowait' and 'steal' indicate specific idle/wait states, which shouldn't
be counted into CPU Utilisation. Also see
https://github.com/prometheus-operator/kube-prometheus/pull/796 and
https://github.com/kubernetes-monitoring/kubernetes-mixin/pull/667.
Per the iostat man page:
%idle
Show the percentage of time that the CPU or CPUs were idle and the
system did not have an outstanding disk I/O request.
%iowait
Show the percentage of time that the CPU or CPUs were idle during
which the system had an outstanding disk I/O request.
%steal
Show the percentage of time spent in involuntary wait by the
virtual CPU or CPUs while the hypervisor was servicing another
virtual processor.
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
* tsdb: shrink txRing with smaller integers
4 billion active transactions ought to be enough for anyone.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: create isolation transaction slice on demand
When Prometheus restarts it creates every series read in from the WAL,
but many of those series will be finished, and never receive any more
samples. By defering allocation of the txRing slice to when it is first
needed, we save 32 bytes per stale series.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* add cluster variable to Overview dashboard
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* promql: simplify Native Histogram arithmetics
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
* Cut 2.49.0-rc.0 (#13270)
* Cut 2.49.0-rc.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed the duplicate.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Add unit protobuf parser
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Go on adding protobuf parsing for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* ui: create a reproduction for https://github.com/prometheus/prometheus/issues/13292
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Get conditional right
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Get VM Scale Set NIC (#13283)
Calling `*armnetwork.InterfacesClient.Get()` doesn't work for Scale Set
VM NIC, because these use a different Resource ID format.
Use `*armnetwork.InterfacesClient.GetVirtualMachineScaleSetNetworkInterface()`
instead. This needs both the scale set name and the instance ID, so
add an `InstanceID` field to the `virtualMachine` struct. `InstanceID`
is empty for a VM that isn't a ScaleSetVM.
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
* Cut v2.49.0-rc.1
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Delete debugging lines, amend error message for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Correct order in error message
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Consider storage.ErrTooOldSample as non-retryable
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
* scrape_test.go: Increase scrape interval in TestScrapeLoopCache to reduce potential flakiness
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Avoid creating string for suffix, consider counters without _total suffix
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* build(deps): bump github.com/prometheus/client_golang
Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.17.0 to 1.18.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/client_golang/compare/v1.17.0...v1.18.0)
---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump actions/setup-node from 3.8.1 to 4.0.1
Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3.8.1 to 4.0.1.
- [Release notes](https://github.com/actions/setup-node/releases)
- [Commits](https://github.com/actions/setup-node/compare/5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d...b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8)
---
updated-dependencies:
- dependency-name: actions/setup-node
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
* scripts: sort file list in embed directive
Otherwise the resulting string depends on find, which afaict depends on
the underlying filesystem. A stable file list make it easier to detect
UI changes in downstreams that need to track UI assets.
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
* Fix DataTableProps['data'] for resultType string
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* Fix handling of scalar and string in isHeatmapData
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* build(deps): bump github.com/influxdata/influxdb
Bumps [github.com/influxdata/influxdb](https://github.com/influxdata/influxdb) from 1.11.2 to 1.11.4.
- [Release notes](https://github.com/influxdata/influxdb/releases)
- [Commits](https://github.com/influxdata/influxdb/compare/v1.11.2...v1.11.4)
---
updated-dependencies:
- dependency-name: github.com/influxdata/influxdb
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump github.com/prometheus/prometheus
Bumps [github.com/prometheus/prometheus](https://github.com/prometheus/prometheus) from 0.48.0 to 0.48.1.
- [Release notes](https://github.com/prometheus/prometheus/releases)
- [Changelog](https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/prometheus/compare/v0.48.0...v0.48.1)
---
updated-dependencies:
- dependency-name: github.com/prometheus/prometheus
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* Bump client_golang to v1.18.0 (#13373)
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Drop old inmemory samples (#13002)
* Drop old inmemory samples
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Avoid copying timeseries when the feature is disabled
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Run gofmt
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Clarify docs
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Add more logging info
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Remove loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* optimize function and add tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Simplify filter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Update help info from metrics
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use metrics to keep track of drop elements during buildWriteRequest
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var in tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* pass time.Now as parameter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Change buildwriterequest during retries
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Revert "Remove loggers"
This reverts commit 54f91dfcae20488944162335ab4ad8be459df1ab.
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use log level debug for loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Fix linter
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove noisy debug-level logs; add 'reason' label to drop metrics
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove accidentally committed files
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Propagate logger to buildWriteRequest to log dropped data
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix docs comment
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Make drop reason more specific
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove unnecessary pass of logger
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Use snake_case for reason label
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix dropped samples metric
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
---------
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
* fix(discovery): allow requireUpdate util to timeout in discovery/file/file_test.go.
The loop ran indefinitely if the condition isn't met.
Before, each iteration created a new timer channel which was always outpaced by
the other timer channel with smaller duration.
minor detail: There was a memory leak: resources of the ~10 previous timers were
constantly kept. With the fix, we may keep the resources of one timer around for defaultWait
but this isn't worth the changes to make it right.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Merge pull request #13371 from kevinmingtarja/fix-isHeatmapData
ui: fix handling of scalar and string in isHeatmapData
* tsdb/{index,compact}: allow using custom postings encoding format (#13242)
* tsdb/{index,compact}: allow using custom postings encoding format
We would like to experiment with a different postings encoding format in
Thanos so in this change I am proposing adding another argument to
`NewWriter` which would allow users to change the format if needed.
Also, wire the leveled compactor so that it would be possible to change
the format there too.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb/compact: use a struct for leveled compactor options
As discussed on Slack, let's use a struct for the options in leveled
compactor.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: make changes after Bryan's review
- Make changes less intrusive
- Turn the postings encoder type into a function
- Add NewWriterWithEncoder()
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
---------
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0-rc.2
Signed-off-by: bwplotka <bwplotka@gmail.com>
* build(deps): bump actions/setup-go from 3.5.0 to 5.0.0 in /scripts (#13362)
Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3.5.0 to 5.0.0.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/6edd4406fa81c3da01a34fa6f6343087c207a568...0c52d547c9bc32b1aa3301fd7a9cb496313a4491)
---
updated-dependencies:
- dependency-name: actions/setup-go
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump github/codeql-action from 2.22.8 to 3.22.12 (#13358)
Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.22.8 to 3.22.12.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/407ffafae6a767df3e0230c3df91b6443ae8df75...012739e5082ff0c22ca6d6ab32e07c36df03c4a4)
---
updated-dependencies:
- dependency-name: github/codeql-action
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* put @nexucis has a release shepherd (#13383)
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
* Add analyze histograms command to promtool (#12331)
Add `query analyze` command to promtool
This command analyzes the buckets of classic and native histograms,
based on data queried from the Prometheus query API, i.e. it
doesn't require direct access to the TSDB files.
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* included instance in all necessary descriptions
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* tsdb/compact: fix passing merge func
Fixing a very small logical problem I've introduced :(.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: add enable overlapping compaction
This functionality is needed in downstream projects because they have a
separate component that does compaction.
Upstreaming
https://github.com/grafana/mimir-prometheus/blob/7c8e9a2a76fc729e9078889782928b2fdfe240e9/tsdb/compact.go#L323-L325.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* promtool: allow setting multiple matchers to "promtool tsdb dump" command. (#13296)
Conditions are ANDed inside the same matcher but matchers are ORed
Including unit tests for "promtool tsdb dump".
Refactor some matchers scraping utils.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Fixed changelog
Signed-off-by: bwplotka <bwplotka@gmail.com>
* tsdb/main: wire "EnableOverlappingCompaction" to tsdb.Options (#13398)
This added the https://github.com/prometheus/prometheus/pull/13393
"EnableOverlappingCompaction" parameter to the compactor code but not to
the tsdb.Options. I forgot about that. Add it to `tsdb.Options` too and
set it to `true` in Prometheus.
Copy/paste the description from
https://github.com/prometheus/prometheus/pull/13393#issuecomment-1891787986
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Issue #13268: fix quality value in accept header
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
* Cut 2.49.1 with scrape q= bugfix.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Cut 2.49.1 web package.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Restore more efficient version of NewPossibleNonCounterInfo annotation (#13022)
Restore more efficient version of NewPossibleNonCounterInfo annotation
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Fix regressions introduced by #13242
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* fix slice copy in 1.20 (#13389)
The slices package is added to the standard library in Go 1.21;
we need to import from the exp area to maintain compatibility with Go 1.20.
Signed-off-by: tyltr <tylitianrui@126.com>
* Docs: Query Basics: link to rate (#10538)
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
* chore(kubernetes): check preconditions earlier and avoid unnecessary checks or iterations
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Examples: link to `rate` for new users (#10535)
* Examples: link to `rate` for new users
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
* promql: use natural sort in sort_by_label and sort_by_label_desc (#13411)
These functions are intended for humans, as robots can already sort the results
however they please. Humans like things sorted "naturally":
* https://blog.codinghorror.com/sorting-for-humans-natural-sort-order/
A similar thing has been done to Grafana, which is also used by humans:
* https://github.com/grafana/grafana/pull/78024
* https://github.com/grafana/grafana/pull/78494
Signed-off-by: Ivan Babrou <github@ivan.computer>
* TestLabelValuesWithMatchers: Add test case
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* remove obsolete build tag
Signed-off-by: tyltr <tylitianrui@126.com>
* Upgrade some golang dependencies for resty 2.11
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
* Native Histograms: support `native_histogram_min_bucket_factor` in scrape_config (#13222)
Native Histograms: support native_histogram_min_bucket_factor in scrape_config
---------
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram (#13392)
Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Minor fixes to otlp vendor update script
Signed-off-by: Goutham <gouthamve@gmail.com>
* build(deps): bump github.com/hetznercloud/hcloud-go/v2
Bumps [github.com/hetznercloud/hcloud-go/v2](https://github.com/hetznercloud/hcloud-go) from 2.4.0 to 2.6.0.
- [Release notes](https://github.com/hetznercloud/hcloud-go/releases)
- [Changelog](https://github.com/hetznercloud/hcloud-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hetznercloud/hcloud-go/compare/v2.4.0...v2.6.0)
---
updated-dependencies:
- dependency-name: github.com/hetznercloud/hcloud-go/v2
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* Enhanced visibility for `promtool test rules` with JSON colored formatting (#13342)
* Added diff flag for unit test to improvise readability & debugging
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Removed blank spaces
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed linting error
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Added cli flags to documentation
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Revert unrrelated linting fixes
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed review suggestions
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Cleanup
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
---------
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* storage: skip merging when no remote storage configured
Prometheus is hard-coded to use a fanout storage between TSDB and
a remote storage which by default is empty.
This change detects the empty storage and skips merging between
result sets, which would make `Select()` sort results.
Bottom line: we skip a sort unless there really is some remote storage
configured.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Remove csmarchbanks from remote write owners (#13432)
I have not had the time to keep up with remote write and have no plans
to work on it in the near future so I am withdrawing my maintainership
of that part of the codebase. I continue to focus on client_python.
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
* add more context cancellation check at evaluation time
Signed-off-by: Ben Ye <benye@amazon.com>
* Optimize label values with matchers by taking shortcuts (#13426)
Don't calculate postings beforehand: we may not need them. If all
matchers are for the requested label, we can just filter its values.
Also, if there are no values at all, no need to run any kind of
logic.
Also add more labelValuesWithMatchers benchmarks
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
* Add automatic memory limit handling
Enable automatic detection of memory limits and configure GOMEMLIMIT to
match.
* Also includes a flag to allow controlling the reserved ratio.
Signed-off-by: SuperQ <superq@gmail.com>
* Update OSSF badge link (#13433)
Provide a more user friendly interface
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
* SD Managers taking over responsibility for registration of debug metrics (#13375)
SD Managers take over responsibility for SD metrics registration
---------
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Optimize histogram iterators (#13340)
Optimize histogram iterators
Histogram iterators allocate new objects in the AtHistogram and
AtFloatHistogram methods, which makes calculating rates over long
ranges expensive.
In #13215 we allowed an existing object to be reused
when converting an integer histogram to a float histogram. This commit follows
the same idea and allows injecting an existing object in the AtHistogram and
AtFloatHistogram methods. When the injected value is nil, iterators allocate
new histograms, otherwise they populate and return the injected object.
The commit also adds a CopyTo method to Histogram and FloatHistogram which
is used in the BufferedIterator to overwrite items in the ring instead of making
new copies.
Note that a specialized HPoint pool is needed for all of this to work
(`matrixSelectorHPool`).
---------
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* doc: Mark `mad_over_time` as experimental (#13440)
We forgot to do that in
https://github.com/prometheus/prometheus/pull/13059
Signed-off-by: beorn7 <beorn@grafana.com>
* Change metric label for Puppetdb from 'http' to 'puppetdb'
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
* mirror metrics.proto change & generate code
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
* TestHeadLabelValuesWithMatchers: Add test case (#13414)
Add test case to TestHeadLabelValuesWithMatchers, while fixing a couple
of typos in other test cases. Also enclosing some implicit sub-tests in a
`t.Run` call to make them explicitly sub-tests.
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* update all go dependencies (#13438)
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
* build(deps): bump the k8s-io group with 2 updates (#13454)
Bumps the k8s-io group with 2 updates: [k8s.io/api](https://github.com/kubernetes/api) and [k8s.io/client-go](https://github.com/kubernetes/client-go).
Updates `k8s.io/api` from 0.28.4 to 0.29.1
- [Commits](https://github.com/kubernetes/api/compare/v0.28.4...v0.29.1)
Updates `k8s.io/client-go` from 0.28.4 to 0.29.1
- [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/kubernetes/client-go/compare/v0.28.4...v0.29.1)
---
updated-dependencies:
- dependency-name: k8s.io/api
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
- dependency-name: k8s.io/client-go
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump the go-opentelemetry-io group with 1 update (#13453)
Bumps the go-opentelemetry-io group with 1 update: [go.opentelemetry.io/collector/semconv](https://github.com/open-telemetry/opentelemetry-collector).
Updates `go.opentelemetry.io/collector/semconv` from 0.92.0 to 0.93.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-collector/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-collector/blob/main/CHANGELOG-API.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-collector/compare/v0.92.0...v0.93.0)
---
updated-dependencies:
- dependency-name: go.opentelemetry.io/collector/semconv
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: go-opentelemetry-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump actions/upload-artifact from 3.1.3 to 4.0.0 (#13355)
Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.3 to 4.0.0.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/a8a3f3ad30e3422c9c7b888a15615d19a852ae32...c7d193f32edcb7bfad88892161225aeda64e9392)
---
updated-dependencies:
- dependency-name: actions/upload-artifact
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump bufbuild/buf-push-action (#13357)
Bumps [bufbuild/buf-push-action](https://github.com/bufbuild/buf-push-action) from 342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1 to a654ff18effe4641ebea4a4ce242c49800728459.
- [Release notes](https://github.com/bufbuild/buf-push-action/releases)
- [Commits](https://github.com/bufbuild/buf-push-action/compare/342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1...a654ff18effe4641ebea4a4ce242c49800728459)
---
updated-dependencies:
- dependency-name: bufbuild/buf-push-action
dependency-type: direct:production
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* Labels: Add DropMetricName function, used in PromQL (#13446)
This function is called very frequently when executing PromQL functions,
and we can do it much more efficiently inside Labels.
In the common case that `__name__` comes first in the labels, we simply
re-point to start at the next label, which is nearly free.
`DropMetricName` is now so cheap I removed the cache - benchmarks show
everything still goes faster.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: simplify internal series delete function (#13261)
Lifting an optimisation from Agent code, `seriesHashmap.del` can use
the unique series reference, doesn't need to check Labels.
Also streamline the logic for deleting from `unique` and `conflicts` maps,
and add some comments to help the next person.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* otlptranslator/update-copy.sh: Fix sed command lines
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Rollback k8s.io requirements (#13462)
Rollback k8s.io Go modules to v0.28.6 to avoid forcing upgrade of Go to
1.21. This allows us to keep compatibility with the currently supported
upstream Go releases.
Signed-off-by: SuperQ <superq@gmail.com>
* Make update-copy.sh work for both OSX and GNU sed
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Name @beorn7 and @krajorama as maintainers for native histograms
I have been the de-facto maintainer for native histograms from the
beginning. So let's put this into MAINTAINERS.md.
In addition, I hereby proposose George Krajcsovits AKA Krajo as a
co-maintainer. He has contributed a lot of native histogram code, but
more importantly, he has contributed substantially to reviewing other
contributors' native histogram code, up to a point where I was merely
rubberstamping the PRs he had already reviewed. I'm confident that he
is ready to to be granted commit rights as outlined in the
"Maintainers" section of the governance:
https://prometheus.io/governance/#maintainers
According to the same section of the governance, I will announce the
proposed change on the developers mailing list and will give some time
for lazy consensus before merging this PR.
Signed-off-by: beorn7 <beorn@grafana.com>
* ui/fix: correct url handling for stacked graphs (#13460)
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
* tsdb: use cheaper Mutex on series
Mutex is 8 bytes; RWMutex is 24 bytes and much more complicated. Since
`RLock` is only used in two places, `UpdateMetadata` and `Delete`,
neither of which are hotspots, we should use the cheaper one.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Fix last_over_time for native histograms
The last_over_time retains a histogram sample without making a copy.
This sample is now coming from the buffered iterator used for windowing functions,
and can be reused for reading subsequent samples as the iterator progresses.
I would propose copying the sample in the last_over_time function, similar to
how it is done for rate, sum_over_time and others.
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
* Implementation
NOTE:
Rebased from main after refactor in #13014
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Add feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactor concurrency control
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Optimising dependencies/dependents funcs to not produce new slices each request
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Rename flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring for performance, and to allow controller to be overridden
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Block until all rules, both sync & async, have completed evaluating
Updated & added tests
Review feedback nits
Return empty map if not indeterminate
Use highWatermark to track inflight requests counter
Appease the linter
Clarify feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Fix typo in CLI flag description
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Fixed auto-generated doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improve doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Simplify the design to update concurrency controller once the rule evaluation has done
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Add more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Added more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improved RuleConcurrencyController interface doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Introduced sequentialRuleEvalController
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Remove superfluous nil check in Group.metrics
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* api: Serialize discovered and target labels into JSON directly (#13469)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* api: Serialize discovered labels into JSON directly in dropped targets (#13484)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* Add ShardedPostings() support to TSDB (#10421)
This PR is a reference implementation of the proposal described in #10420.
In addition to what described in #10420, in this PR I've introduced labels.StableHash(). The idea is to offer an hashing function which doesn't change over time, and that's used by query sharding in order to get a stable behaviour over time. The implementation of labels.StableHash() is the hashing function used by Prometheus before stringlabels, and what's used by Grafana Mimir for query sharding (because built before stringlabels was a thing).
Follow up work
As mentioned in #10420, if this PR is accepted I'm also open to upload another foundamental piece used by Grafana Mimir query sharding to accelerate the query execution: an optional, configurable and fast in-memory cache for the series hashes.
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* storage/remote: document why two benchmarks are skipped
One was silently doing nothing; one was doing something but the work
didn't go up linearly with iteration count.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Pod status changes not discovered by Kube Endpoints SD (#13337)
* fix(discovery/kubernetes/endpoints): react to changes on Pods because some modifications can occur on them without triggering an update on the related Endpoints (The Pod phase changing from Pending to Running e.g.).
---------
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
* Small improvements, add const, remove copypasta (#8106)
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
* Proposal to improve FPointSlice and HPointSlice allocation. (#13448)
* Reusing points slice from previous series when the slice is under utilized
* Adding comments on the bench test
Signed-off-by: Alan Protasio <alanprot@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
---------
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
* remote write 2.0 - follow up improvements (#13478)
* move remote write proto version config from a remote storage config to a
per remote write configuration option
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename scrape config for metadata, fix 2.0 header var name/value (was
1.1), and more clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* address review comments, mostly lint fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* another lint fix
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint imports
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics. (#13502)
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Apply suggestions from code review
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
* prw2.0: Added support for "custom" layouts for native histogram proto (#13558)
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# prompb/write/v2/types.pb.go
* Update prompb/write/v2/types.proto
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Addressed comments, fixed test.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* first draft of content negotiation
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix race in test
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix another test race
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Almost done with lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix todos around 405 HEAD handling
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Changes based on review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Update storage/remote/client.go
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Latest updates to review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* latest tweaks
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* remote write 2.0 - content negotiation remediation (#13921)
* Consolidate renegotiation error into one, fix tests
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* fix metric name and actuall increment counter
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
---------
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fixes after main sync.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW 2.0] Moved rw2 proto to the full path (both package name and placement) (#13973)
undefined
* [PRW2.0] Remove benchmark scripts (#13949)
See rationales on https://docs.google.com/document/d/1Bpf7mYjrHUhPHkie0qlnZFxzgqf_L32kM8ZOknSdJrU/edit
Signed-off-by: bwplotka <bwplotka@gmail.com>
* rw20: Update prw commentary after Callum spec review (#14136)
* rw20: Update prw commentary after Callum spec review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update types.proto
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation. (#14330)
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* update custom marshalling
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed confusing comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (chain1) New Remote Write 2.0 Config options for 2.0-rc.1 spec. (#14335)
NOTE: For simple review this change does not touch remote/ packages, only main and configs.
Spec: https://prometheus.io/docs/specs/remote_write_spec_2_0
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (part 2) Removed automatic negotiation, updates for the latest spec semantics in remote pkg (#14329)
* [PRW-2.0] (part2) Moved to latest basic negotiation & spec semantics.
Spec: https://github.com/prometheus/docs/pull/2462
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# config/config.go
# docs/configuration/configuration.md
# storage/remote/queue_manager_test.go
# storage/remote/write.go
# web/api/v1/api.go
* Addressed comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* storage/remote tests: refactor: extract function newTestQueueManager
To reduce repetition.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use newTestQueueManager for test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* [PRW 2.0] (part3) moved type specific conversions to prompb and writev2 codecs.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added test for rwProtoMsgFlagParser; fixed TODO comment.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Renamed DecodeV2WriteRequestStr to DecodeWriteV2Request (with tests).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed comments on remote_storage example, updated it for 2.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed `--enable-feature=metadata-wal-records` docs and error when using PRW 2.0 without it.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments on custom*.go
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added TODO to genproto.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments in remote pkg.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added metadata validation to write handler test; fixed ToMetadata.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed rest of Callum comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed writev2.FromMetadataType (was wrongly using prompb).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* fix a few import whitespaces
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* add a default case with an error to the example RW receiver
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more minor import whitespace chagnes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Apply suggestions from code review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update storage/remote/queue_manager_test.go
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
Co-authored-by: Nicolás Pazos <32206519+npazosmendez@users.noreply.github.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: alexgreenbank <alex.greenbank@grafana.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
2024-07-04 14:29:20 -07:00
// ProtobufMessage specifies the protobuf message to use against the remote
// receiver as specified in https://prometheus.io/docs/specs/remote_write_spec_2_0/
ProtobufMessage RemoteWriteProtoMsg ` yaml:"protobuf_message,omitempty" `
2016-09-19 13:47:51 -07:00
2017-03-20 05:37:50 -07:00
// We cannot do proper Go type embedding below as the parser will then parse
// values arbitrarily into the overflow maps of further-down types.
2020-08-20 05:48:26 -07:00
HTTPClientConfig config . HTTPClientConfig ` yaml:",inline" `
QueueConfig QueueConfig ` yaml:"queue_config,omitempty" `
2020-11-19 07:23:03 -08:00
MetadataConfig MetadataConfig ` yaml:"metadata_config,omitempty" `
2021-08-26 06:37:19 -07:00
SigV4Config * sigv4 . SigV4Config ` yaml:"sigv4,omitempty" `
2023-06-01 14:20:10 -07:00
AzureADConfig * azuread . AzureADConfig ` yaml:"azuread,omitempty" `
2024-07-30 08:25:19 -07:00
GoogleIAMConfig * googleiam . Config ` yaml:"google_iam,omitempty" `
2020-08-20 05:48:26 -07:00
}
// SetDirectory joins any relative file paths with dir.
func ( c * RemoteWriteConfig ) SetDirectory ( dir string ) {
c . HTTPClientConfig . SetDirectory ( dir )
2016-09-19 13:47:51 -07:00
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func ( c * RemoteWriteConfig ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
* c = DefaultRemoteWriteConfig
type plain RemoteWriteConfig
if err := unmarshal ( ( * plain ) ( c ) ) ; err != nil {
return err
}
2017-08-07 00:49:45 -07:00
if c . URL == nil {
2019-03-25 16:01:12 -07:00
return errors . New ( "url for remote_write is empty" )
2017-08-07 00:49:45 -07:00
}
2018-12-03 02:09:02 -08:00
for _ , rlcfg := range c . WriteRelabelConfigs {
if rlcfg == nil {
2019-03-25 16:01:12 -07:00
return errors . New ( "empty or null relabeling rule in remote write config" )
2018-12-03 02:09:02 -08:00
}
}
2021-02-18 04:12:21 -08:00
if err := validateHeaders ( c . Headers ) ; err != nil {
return err
2021-02-04 13:18:13 -08:00
}
2017-07-25 05:47:34 -07:00
[PRW 2.0] Merging `remote-write-2.0` feature branch to main (PRW 2.0 support + metadata in WAL) (#14395)
* Remote Write 1.1: e2e benchmarks (#13102)
* Remote Write e2e benchmarks
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Prometheus ports automatically assigned
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make dashboard editable + more modular to different job label values
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Dashboard improvements
* memory stats
* diffs look at counter increases
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* run script: absolute path for config templates
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* grafana dashboard improvements
* show actual values of metrics
* add memory stats and diff
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* dashboard changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
* replace snappy encoding library
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto types
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add decode function for new write request proto
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add lookup table struct that is used to build the symbol table in new
write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Implement code paths for new proto format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example server to include handler for new format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add new test client
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* tests and new -> original proto mapping util
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add new proto support on receiver end
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Fix test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* no-brainer copypaste but more performance write support
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove some comented code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix mocks and fixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add basic reduce remote write handler benchmark
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor out common code between write methods
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix: queue manager to include float histograms in new requests
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add sender-side tests and fix failing ones
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor queue manager code to remove some duplication
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Improve sender benchmarks and some allocations
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use github.com/golang/snappy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup: remove hardcoded fake url for testing
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add 1.1 version handling code
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Remove config, update proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* gofmt
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix NewWriteClient and change new flags wording
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fields rewording in handler
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remote write handler to checks version header
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix typo in log
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add minmized remote write proto format
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functions for translating between new proto formats symbol table and
actual prometheus labels
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add functionality for new minimized remote write request format
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor things
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Make LabelSymbols a fixed32
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove unused proto type
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update tests
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix build for stringlabels tag
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use two uint32 to encode (offset,leng)
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* manually optimize varint marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Use unsafe []byte->string cast to reuse buffer
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix writeRequestMinimizedFixture
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all code from previous interning approach
the 'minimized' version is now the only v1.1 version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* minimally-tested exemplar support for rw 1.1
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* refactor new version flag to make it easier to pick a specific format
instead of having multiple flags, plus add new formats for testing
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use exp slices for backwards compat. to go 1.20 plus add copyright
header to test file
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix label ranging
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Add bytes slice (instead of slice of 32bit vars) format for testing
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* test additional len and lenbytes formats
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove mistaken package lock changes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove formats we've decided not to use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove more format types we probably won't use
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* More cleanup
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use require instead of assert in custom marshal test
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup; remove some unused functions
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, mostly linting fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove package-lock.json change again
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* more cleanup, address review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix test panic
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix minor lint issue + use labels Range function since it looks like
the tests fail to do `range labels.Labels` on CI
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* new interning format based on []string indeces
Co-authored-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove all new rw formats but the []string one
also adapt tests to the new format
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* cleanup rwSymbolTable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* add some TODOs for later
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* don't reserve field 3 for new proto and add TODO
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix custom marshaling
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* additional merge fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint fixes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* fix server example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* revert package-lock.json changes
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update example prometheus version
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* define separate proto types for remote write 2.0
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* rename new proto types and move to separate pkg
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* update prometheus version for example
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make proto
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* make Metadata not nullable
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* remove old MinSample proto message
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* change enum names to fit buf build recommend enum naming and lint rules
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* remote: Added test for classic histogram grouping when sending rw; Fixed queue manager test delay. (#13421)
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Remote write v2: metadata support in every write request (#13394)
* Approach bundling metadata along with samples and exemplars
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Add first test; rebase with main
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* Alternative approach: bundle metadata in TimeSeries protobuf
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
* update go mod to match main branch
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix after rebase
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* we're not going to modify the 1.X format anymore
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Modify AppendMetadata based on the fact that we be putting metadata into
timeseries
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Rename enums for remote write versions to something that makes more
sense + remove the added `sendMetadata` flag.
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename flag that enables writing of metadata records to the WAL
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* additional clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* fix usage of require.Len
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* some clean up from review comments
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more review fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
* remote write 2.0: sync with `main` branch (#13510)
* consoles: exclude iowait and steal from CPU Utilisation
'iowait' and 'steal' indicate specific idle/wait states, which shouldn't
be counted into CPU Utilisation. Also see
https://github.com/prometheus-operator/kube-prometheus/pull/796 and
https://github.com/kubernetes-monitoring/kubernetes-mixin/pull/667.
Per the iostat man page:
%idle
Show the percentage of time that the CPU or CPUs were idle and the
system did not have an outstanding disk I/O request.
%iowait
Show the percentage of time that the CPU or CPUs were idle during
which the system had an outstanding disk I/O request.
%steal
Show the percentage of time spent in involuntary wait by the
virtual CPU or CPUs while the hypervisor was servicing another
virtual processor.
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
* tsdb: shrink txRing with smaller integers
4 billion active transactions ought to be enough for anyone.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: create isolation transaction slice on demand
When Prometheus restarts it creates every series read in from the WAL,
but many of those series will be finished, and never receive any more
samples. By defering allocation of the txRing slice to when it is first
needed, we save 32 bytes per stale series.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* add cluster variable to Overview dashboard
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* promql: simplify Native Histogram arithmetics
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
* Cut 2.49.0-rc.0 (#13270)
* Cut 2.49.0-rc.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed the duplicate.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Add unit protobuf parser
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Go on adding protobuf parsing for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* ui: create a reproduction for https://github.com/prometheus/prometheus/issues/13292
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Get conditional right
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Get VM Scale Set NIC (#13283)
Calling `*armnetwork.InterfacesClient.Get()` doesn't work for Scale Set
VM NIC, because these use a different Resource ID format.
Use `*armnetwork.InterfacesClient.GetVirtualMachineScaleSetNetworkInterface()`
instead. This needs both the scale set name and the instance ID, so
add an `InstanceID` field to the `virtualMachine` struct. `InstanceID`
is empty for a VM that isn't a ScaleSetVM.
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
* Cut v2.49.0-rc.1
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Delete debugging lines, amend error message for unit
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Correct order in error message
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* Consider storage.ErrTooOldSample as non-retryable
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
* scrape_test.go: Increase scrape interval in TestScrapeLoopCache to reduce potential flakiness
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Avoid creating string for suffix, consider counters without _total suffix
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
* build(deps): bump github.com/prometheus/client_golang
Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.17.0 to 1.18.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/client_golang/compare/v1.17.0...v1.18.0)
---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump actions/setup-node from 3.8.1 to 4.0.1
Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3.8.1 to 4.0.1.
- [Release notes](https://github.com/actions/setup-node/releases)
- [Commits](https://github.com/actions/setup-node/compare/5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d...b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8)
---
updated-dependencies:
- dependency-name: actions/setup-node
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
* scripts: sort file list in embed directive
Otherwise the resulting string depends on find, which afaict depends on
the underlying filesystem. A stable file list make it easier to detect
UI changes in downstreams that need to track UI assets.
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
* Fix DataTableProps['data'] for resultType string
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* Fix handling of scalar and string in isHeatmapData
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
* build(deps): bump github.com/influxdata/influxdb
Bumps [github.com/influxdata/influxdb](https://github.com/influxdata/influxdb) from 1.11.2 to 1.11.4.
- [Release notes](https://github.com/influxdata/influxdb/releases)
- [Commits](https://github.com/influxdata/influxdb/compare/v1.11.2...v1.11.4)
---
updated-dependencies:
- dependency-name: github.com/influxdata/influxdb
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* build(deps): bump github.com/prometheus/prometheus
Bumps [github.com/prometheus/prometheus](https://github.com/prometheus/prometheus) from 0.48.0 to 0.48.1.
- [Release notes](https://github.com/prometheus/prometheus/releases)
- [Changelog](https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/prometheus/compare/v0.48.0...v0.48.1)
---
updated-dependencies:
- dependency-name: github.com/prometheus/prometheus
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
* Bump client_golang to v1.18.0 (#13373)
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Drop old inmemory samples (#13002)
* Drop old inmemory samples
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Avoid copying timeseries when the feature is disabled
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Run gofmt
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Clarify docs
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Add more logging info
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Remove loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* optimize function and add tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Simplify filter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Update help info from metrics
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use metrics to keep track of drop elements during buildWriteRequest
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* rename var in tests
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* pass time.Now as parameter
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Change buildwriterequest during retries
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Revert "Remove loggers"
This reverts commit 54f91dfcae20488944162335ab4ad8be459df1ab.
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* use log level debug for loggers
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
* Fix linter
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove noisy debug-level logs; add 'reason' label to drop metrics
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove accidentally committed files
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Propagate logger to buildWriteRequest to log dropped data
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix docs comment
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Make drop reason more specific
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Remove unnecessary pass of logger
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Use snake_case for reason label
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
* Fix dropped samples metric
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
---------
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
* fix(discovery): allow requireUpdate util to timeout in discovery/file/file_test.go.
The loop ran indefinitely if the condition isn't met.
Before, each iteration created a new timer channel which was always outpaced by
the other timer channel with smaller duration.
minor detail: There was a memory leak: resources of the ~10 previous timers were
constantly kept. With the fix, we may keep the resources of one timer around for defaultWait
but this isn't worth the changes to make it right.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Merge pull request #13371 from kevinmingtarja/fix-isHeatmapData
ui: fix handling of scalar and string in isHeatmapData
* tsdb/{index,compact}: allow using custom postings encoding format (#13242)
* tsdb/{index,compact}: allow using custom postings encoding format
We would like to experiment with a different postings encoding format in
Thanos so in this change I am proposing adding another argument to
`NewWriter` which would allow users to change the format if needed.
Also, wire the leveled compactor so that it would be possible to change
the format there too.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb/compact: use a struct for leveled compactor options
As discussed on Slack, let's use a struct for the options in leveled
compactor.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: make changes after Bryan's review
- Make changes less intrusive
- Turn the postings encoder type into a function
- Add NewWriterWithEncoder()
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
---------
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0-rc.2
Signed-off-by: bwplotka <bwplotka@gmail.com>
* build(deps): bump actions/setup-go from 3.5.0 to 5.0.0 in /scripts (#13362)
Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3.5.0 to 5.0.0.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/6edd4406fa81c3da01a34fa6f6343087c207a568...0c52d547c9bc32b1aa3301fd7a9cb496313a4491)
---
updated-dependencies:
- dependency-name: actions/setup-go
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump github/codeql-action from 2.22.8 to 3.22.12 (#13358)
Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.22.8 to 3.22.12.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/407ffafae6a767df3e0230c3df91b6443ae8df75...012739e5082ff0c22ca6d6ab32e07c36df03c4a4)
---
updated-dependencies:
- dependency-name: github/codeql-action
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* put @nexucis has a release shepherd (#13383)
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
* Add analyze histograms command to promtool (#12331)
Add `query analyze` command to promtool
This command analyzes the buckets of classic and native histograms,
based on data queried from the Prometheus query API, i.e. it
doesn't require direct access to the TSDB files.
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* included instance in all necessary descriptions
Signed-off-by: Erik Sommer <ersotech@posteo.de>
* tsdb/compact: fix passing merge func
Fixing a very small logical problem I've introduced :(.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* tsdb: add enable overlapping compaction
This functionality is needed in downstream projects because they have a
separate component that does compaction.
Upstreaming
https://github.com/grafana/mimir-prometheus/blob/7c8e9a2a76fc729e9078889782928b2fdfe240e9/tsdb/compact.go#L323-L325.
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Cut 2.49.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* promtool: allow setting multiple matchers to "promtool tsdb dump" command. (#13296)
Conditions are ANDed inside the same matcher but matchers are ORed
Including unit tests for "promtool tsdb dump".
Refactor some matchers scraping utils.
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Fixed changelog
Signed-off-by: bwplotka <bwplotka@gmail.com>
* tsdb/main: wire "EnableOverlappingCompaction" to tsdb.Options (#13398)
This added the https://github.com/prometheus/prometheus/pull/13393
"EnableOverlappingCompaction" parameter to the compactor code but not to
the tsdb.Options. I forgot about that. Add it to `tsdb.Options` too and
set it to `true` in Prometheus.
Copy/paste the description from
https://github.com/prometheus/prometheus/pull/13393#issuecomment-1891787986
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
* Issue #13268: fix quality value in accept header
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
* Cut 2.49.1 with scrape q= bugfix.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Cut 2.49.1 web package.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Restore more efficient version of NewPossibleNonCounterInfo annotation (#13022)
Restore more efficient version of NewPossibleNonCounterInfo annotation
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Fix regressions introduced by #13242
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* fix slice copy in 1.20 (#13389)
The slices package is added to the standard library in Go 1.21;
we need to import from the exp area to maintain compatibility with Go 1.20.
Signed-off-by: tyltr <tylitianrui@126.com>
* Docs: Query Basics: link to rate (#10538)
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
* chore(kubernetes): check preconditions earlier and avoid unnecessary checks or iterations
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
* Examples: link to `rate` for new users (#10535)
* Examples: link to `rate` for new users
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
* promql: use natural sort in sort_by_label and sort_by_label_desc (#13411)
These functions are intended for humans, as robots can already sort the results
however they please. Humans like things sorted "naturally":
* https://blog.codinghorror.com/sorting-for-humans-natural-sort-order/
A similar thing has been done to Grafana, which is also used by humans:
* https://github.com/grafana/grafana/pull/78024
* https://github.com/grafana/grafana/pull/78494
Signed-off-by: Ivan Babrou <github@ivan.computer>
* TestLabelValuesWithMatchers: Add test case
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* remove obsolete build tag
Signed-off-by: tyltr <tylitianrui@126.com>
* Upgrade some golang dependencies for resty 2.11
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
* Native Histograms: support `native_histogram_min_bucket_factor` in scrape_config (#13222)
Native Histograms: support native_histogram_min_bucket_factor in scrape_config
---------
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram (#13392)
Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram
---------
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
* Minor fixes to otlp vendor update script
Signed-off-by: Goutham <gouthamve@gmail.com>
* build(deps): bump github.com/hetznercloud/hcloud-go/v2
Bumps [github.com/hetznercloud/hcloud-go/v2](https://github.com/hetznercloud/hcloud-go) from 2.4.0 to 2.6.0.
- [Release notes](https://github.com/hetznercloud/hcloud-go/releases)
- [Changelog](https://github.com/hetznercloud/hcloud-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hetznercloud/hcloud-go/compare/v2.4.0...v2.6.0)
---
updated-dependencies:
- dependency-name: github.com/hetznercloud/hcloud-go/v2
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* Enhanced visibility for `promtool test rules` with JSON colored formatting (#13342)
* Added diff flag for unit test to improvise readability & debugging
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Removed blank spaces
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed linting error
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Added cli flags to documentation
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Revert unrrelated linting fixes
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Fixed review suggestions
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Cleanup
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* Updated flag description
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
---------
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
* storage: skip merging when no remote storage configured
Prometheus is hard-coded to use a fanout storage between TSDB and
a remote storage which by default is empty.
This change detects the empty storage and skips merging between
result sets, which would make `Select()` sort results.
Bottom line: we skip a sort unless there really is some remote storage
configured.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Remove csmarchbanks from remote write owners (#13432)
I have not had the time to keep up with remote write and have no plans
to work on it in the near future so I am withdrawing my maintainership
of that part of the codebase. I continue to focus on client_python.
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
* add more context cancellation check at evaluation time
Signed-off-by: Ben Ye <benye@amazon.com>
* Optimize label values with matchers by taking shortcuts (#13426)
Don't calculate postings beforehand: we may not need them. If all
matchers are for the requested label, we can just filter its values.
Also, if there are no values at all, no need to run any kind of
logic.
Also add more labelValuesWithMatchers benchmarks
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
* Add automatic memory limit handling
Enable automatic detection of memory limits and configure GOMEMLIMIT to
match.
* Also includes a flag to allow controlling the reserved ratio.
Signed-off-by: SuperQ <superq@gmail.com>
* Update OSSF badge link (#13433)
Provide a more user friendly interface
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
* SD Managers taking over responsibility for registration of debug metrics (#13375)
SD Managers take over responsibility for SD metrics registration
---------
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
* Optimize histogram iterators (#13340)
Optimize histogram iterators
Histogram iterators allocate new objects in the AtHistogram and
AtFloatHistogram methods, which makes calculating rates over long
ranges expensive.
In #13215 we allowed an existing object to be reused
when converting an integer histogram to a float histogram. This commit follows
the same idea and allows injecting an existing object in the AtHistogram and
AtFloatHistogram methods. When the injected value is nil, iterators allocate
new histograms, otherwise they populate and return the injected object.
The commit also adds a CopyTo method to Histogram and FloatHistogram which
is used in the BufferedIterator to overwrite items in the ring instead of making
new copies.
Note that a specialized HPoint pool is needed for all of this to work
(`matrixSelectorHPool`).
---------
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* doc: Mark `mad_over_time` as experimental (#13440)
We forgot to do that in
https://github.com/prometheus/prometheus/pull/13059
Signed-off-by: beorn7 <beorn@grafana.com>
* Change metric label for Puppetdb from 'http' to 'puppetdb'
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
* mirror metrics.proto change & generate code
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
* TestHeadLabelValuesWithMatchers: Add test case (#13414)
Add test case to TestHeadLabelValuesWithMatchers, while fixing a couple
of typos in other test cases. Also enclosing some implicit sub-tests in a
`t.Run` call to make them explicitly sub-tests.
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* update all go dependencies (#13438)
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
* build(deps): bump the k8s-io group with 2 updates (#13454)
Bumps the k8s-io group with 2 updates: [k8s.io/api](https://github.com/kubernetes/api) and [k8s.io/client-go](https://github.com/kubernetes/client-go).
Updates `k8s.io/api` from 0.28.4 to 0.29.1
- [Commits](https://github.com/kubernetes/api/compare/v0.28.4...v0.29.1)
Updates `k8s.io/client-go` from 0.28.4 to 0.29.1
- [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/kubernetes/client-go/compare/v0.28.4...v0.29.1)
---
updated-dependencies:
- dependency-name: k8s.io/api
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
- dependency-name: k8s.io/client-go
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: k8s-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump the go-opentelemetry-io group with 1 update (#13453)
Bumps the go-opentelemetry-io group with 1 update: [go.opentelemetry.io/collector/semconv](https://github.com/open-telemetry/opentelemetry-collector).
Updates `go.opentelemetry.io/collector/semconv` from 0.92.0 to 0.93.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-collector/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-collector/blob/main/CHANGELOG-API.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-collector/compare/v0.92.0...v0.93.0)
---
updated-dependencies:
- dependency-name: go.opentelemetry.io/collector/semconv
dependency-type: direct:production
update-type: version-update:semver-minor
dependency-group: go-opentelemetry-io
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump actions/upload-artifact from 3.1.3 to 4.0.0 (#13355)
Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.3 to 4.0.0.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/a8a3f3ad30e3422c9c7b888a15615d19a852ae32...c7d193f32edcb7bfad88892161225aeda64e9392)
---
updated-dependencies:
- dependency-name: actions/upload-artifact
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* build(deps): bump bufbuild/buf-push-action (#13357)
Bumps [bufbuild/buf-push-action](https://github.com/bufbuild/buf-push-action) from 342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1 to a654ff18effe4641ebea4a4ce242c49800728459.
- [Release notes](https://github.com/bufbuild/buf-push-action/releases)
- [Commits](https://github.com/bufbuild/buf-push-action/compare/342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1...a654ff18effe4641ebea4a4ce242c49800728459)
---
updated-dependencies:
- dependency-name: bufbuild/buf-push-action
dependency-type: direct:production
...
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* Labels: Add DropMetricName function, used in PromQL (#13446)
This function is called very frequently when executing PromQL functions,
and we can do it much more efficiently inside Labels.
In the common case that `__name__` comes first in the labels, we simply
re-point to start at the next label, which is nearly free.
`DropMetricName` is now so cheap I removed the cache - benchmarks show
everything still goes faster.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* tsdb: simplify internal series delete function (#13261)
Lifting an optimisation from Agent code, `seriesHashmap.del` can use
the unique series reference, doesn't need to check Labels.
Also streamline the logic for deleting from `unique` and `conflicts` maps,
and add some comments to help the next person.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* otlptranslator/update-copy.sh: Fix sed command lines
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Rollback k8s.io requirements (#13462)
Rollback k8s.io Go modules to v0.28.6 to avoid forcing upgrade of Go to
1.21. This allows us to keep compatibility with the currently supported
upstream Go releases.
Signed-off-by: SuperQ <superq@gmail.com>
* Make update-copy.sh work for both OSX and GNU sed
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
* Name @beorn7 and @krajorama as maintainers for native histograms
I have been the de-facto maintainer for native histograms from the
beginning. So let's put this into MAINTAINERS.md.
In addition, I hereby proposose George Krajcsovits AKA Krajo as a
co-maintainer. He has contributed a lot of native histogram code, but
more importantly, he has contributed substantially to reviewing other
contributors' native histogram code, up to a point where I was merely
rubberstamping the PRs he had already reviewed. I'm confident that he
is ready to to be granted commit rights as outlined in the
"Maintainers" section of the governance:
https://prometheus.io/governance/#maintainers
According to the same section of the governance, I will announce the
proposed change on the developers mailing list and will give some time
for lazy consensus before merging this PR.
Signed-off-by: beorn7 <beorn@grafana.com>
* ui/fix: correct url handling for stacked graphs (#13460)
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
* tsdb: use cheaper Mutex on series
Mutex is 8 bytes; RWMutex is 24 bytes and much more complicated. Since
`RLock` is only used in two places, `UpdateMetadata` and `Delete`,
neither of which are hotspots, we should use the cheaper one.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Fix last_over_time for native histograms
The last_over_time retains a histogram sample without making a copy.
This sample is now coming from the buffered iterator used for windowing functions,
and can be reused for reading subsequent samples as the iterator progresses.
I would propose copying the sample in the last_over_time function, similar to
how it is done for rate, sum_over_time and others.
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
* Implementation
NOTE:
Rebased from main after refactor in #13014
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Add feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactor concurrency control
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Optimising dependencies/dependents funcs to not produce new slices each request
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Rename flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Refactoring for performance, and to allow controller to be overridden
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Block until all rules, both sync & async, have completed evaluating
Updated & added tests
Review feedback nits
Return empty map if not indeterminate
Use highWatermark to track inflight requests counter
Appease the linter
Clarify feature flag
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
* Fix typo in CLI flag description
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Fixed auto-generated doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improve doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Simplify the design to update concurrency controller once the rule evaluation has done
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Add more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Added more test cases to TestDependenciesEdgeCases
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Improved RuleConcurrencyController interface doc
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Introduced sequentialRuleEvalController
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* Remove superfluous nil check in Group.metrics
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* api: Serialize discovered and target labels into JSON directly (#13469)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* api: Serialize discovered labels into JSON directly in dropped targets (#13484)
Converted maps into labels.Labels to avoid a lot of copying of data which leads to very high memory consumption while opening the /service-discovery endpoint in the Prometheus UI
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
* Add ShardedPostings() support to TSDB (#10421)
This PR is a reference implementation of the proposal described in #10420.
In addition to what described in #10420, in this PR I've introduced labels.StableHash(). The idea is to offer an hashing function which doesn't change over time, and that's used by query sharding in order to get a stable behaviour over time. The implementation of labels.StableHash() is the hashing function used by Prometheus before stringlabels, and what's used by Grafana Mimir for query sharding (because built before stringlabels was a thing).
Follow up work
As mentioned in #10420, if this PR is accepted I'm also open to upload another foundamental piece used by Grafana Mimir query sharding to accelerate the query execution: an optional, configurable and fast in-memory cache for the series hashes.
Signed-off-by: Marco Pracucci <marco@pracucci.com>
* storage/remote: document why two benchmarks are skipped
One was silently doing nothing; one was doing something but the work
didn't go up linearly with iteration count.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
* Pod status changes not discovered by Kube Endpoints SD (#13337)
* fix(discovery/kubernetes/endpoints): react to changes on Pods because some modifications can occur on them without triggering an update on the related Endpoints (The Pod phase changing from Pending to Running e.g.).
---------
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
* Small improvements, add const, remove copypasta (#8106)
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
* Proposal to improve FPointSlice and HPointSlice allocation. (#13448)
* Reusing points slice from previous series when the slice is under utilized
* Adding comments on the bench test
Signed-off-by: Alan Protasio <alanprot@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
---------
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
* remote write 2.0 - follow up improvements (#13478)
* move remote write proto version config from a remote storage config to a
per remote write configuration option
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* rename scrape config for metadata, fix 2.0 header var name/value (was
1.1), and more clean up
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* address review comments, mostly lint fixes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* another lint fix
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* lint imports
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics. (#13502)
* Added commmentary to RW 2.0 protocol for easier adoption and explicit semantics.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Apply suggestions from code review
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nico Pazos <32206519+npazosmendez@users.noreply.github.com>
* prw2.0: Added support for "custom" layouts for native histogram proto (#13558)
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
* prw2.0: Added support for "custom" layouts for native histogram.
Result of the discussions:
* https://github.com/prometheus/prometheus/issues/13475#issuecomment-1931496924
* https://cloud-native.slack.com/archives/C02KR205UMU/p1707301006347199
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# prompb/write/v2/types.pb.go
* Update prompb/write/v2/types.proto
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Addressed comments, fixed test.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
* first draft of content negotiation
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix race in test
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix another test race
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Almost done with lint
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fix todos around 405 HEAD handling
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Changes based on review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Update storage/remote/client.go
Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Latest updates to review comments
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* latest tweaks
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* remote write 2.0 - content negotiation remediation (#13921)
* Consolidate renegotiation error into one, fix tests
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* fix metric name and actuall increment counter
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
---------
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
* Fixes after main sync.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW 2.0] Moved rw2 proto to the full path (both package name and placement) (#13973)
undefined
* [PRW2.0] Remove benchmark scripts (#13949)
See rationales on https://docs.google.com/document/d/1Bpf7mYjrHUhPHkie0qlnZFxzgqf_L32kM8ZOknSdJrU/edit
Signed-off-by: bwplotka <bwplotka@gmail.com>
* rw20: Update prw commentary after Callum spec review (#14136)
* rw20: Update prw commentary after Callum spec review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update types.proto
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation. (#14330)
* [PRW 2.0] Updated spec proto (2.0-rc.1); deterministic v1 interop; to be sympathetic with implementation.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* update custom marshalling
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Removed confusing comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (chain1) New Remote Write 2.0 Config options for 2.0-rc.1 spec. (#14335)
NOTE: For simple review this change does not touch remote/ packages, only main and configs.
Spec: https://prometheus.io/docs/specs/remote_write_spec_2_0
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
* [PRW-2.0] (part 2) Removed automatic negotiation, updates for the latest spec semantics in remote pkg (#14329)
* [PRW-2.0] (part2) Moved to latest basic negotiation & spec semantics.
Spec: https://github.com/prometheus/docs/pull/2462
Supersedes https://github.com/prometheus/prometheus/pull/13968
Signed-off-by: bwplotka <bwplotka@gmail.com>
# Conflicts:
# config/config.go
# docs/configuration/configuration.md
# storage/remote/queue_manager_test.go
# storage/remote/write.go
# web/api/v1/api.go
* Addressed comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
---------
Signed-off-by: bwplotka <bwplotka@gmail.com>
* lint
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* storage/remote tests: refactor: extract function newTestQueueManager
To reduce repetition.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* use newTestQueueManager for test
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* go mod tidy
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
* [PRW 2.0] (part3) moved type specific conversions to prompb and writev2 codecs.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added test for rwProtoMsgFlagParser; fixed TODO comment.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Renamed DecodeV2WriteRequestStr to DecodeWriteV2Request (with tests).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed comments on remote_storage example, updated it for 2.0
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed `--enable-feature=metadata-wal-records` docs and error when using PRW 2.0 without it.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments on custom*.go
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added TODO to genproto.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed Callum comments in remote pkg.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Added metadata validation to write handler test; fixed ToMetadata.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Addressed rest of Callum comments.
Signed-off-by: bwplotka <bwplotka@gmail.com>
* Fixed writev2.FromMetadataType (was wrongly using prompb).
Signed-off-by: bwplotka <bwplotka@gmail.com>
* fix a few import whitespaces
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* add a default case with an error to the example RW receiver
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* more minor import whitespace chagnes
Signed-off-by: Callum Styan <callumstyan@gmail.com>
* Apply suggestions from code review
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
* Update storage/remote/queue_manager_test.go
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
---------
Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalist0@gmail.com>
Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
Signed-off-by: Erik Sommer <ersotech@posteo.de>
Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Signed-off-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: tyltr <tylitianrui@126.com>
Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Signed-off-by: Ivan Babrou <github@ivan.computer>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Israel Blancas <iblancasa@gmail.com>
Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Signed-off-by: Goutham <gouthamve@gmail.com>
Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
Signed-off-by: SuperQ <superq@gmail.com>
Signed-off-by: Ben Kochie <superq@gmail.com>
Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: beorn7 <beorn@grafana.com>
Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
Signed-off-by: Yury Moladau <yurymolodov@gmail.com>
Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
Signed-off-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Signed-off-by: Mikhail Fesenko <proggga@gmail.com>
Signed-off-by: Jesus Vazquez <jesusvzpg@gmail.com>
Signed-off-by: Alan Protasio <alanprot@gmail.com>
Signed-off-by: Alex Greenbank <alex.greenbank@grafana.com>
Co-authored-by: Nicolás Pazos <32206519+npazosmendez@users.noreply.github.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Nicolás Pazos <npazosmendez@gmail.com>
Co-authored-by: alexgreenbank <alex.greenbank@grafana.com>
Co-authored-by: Marco Pracucci <marco@pracucci.com>
Co-authored-by: Paschalis Tsilias <paschalist0@gmail.com>
Co-authored-by: Julian Wiedmann <jwi@linux.ibm.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Erik Sommer <ersotech@posteo.de>
Co-authored-by: Linas Medziunas <linas.medziunas@gmail.com>
Co-authored-by: Arianna Vespri <arianna.vespri@yahoo.it>
Co-authored-by: machine424 <ayoubmrini424@gmail.com>
Co-authored-by: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Co-authored-by: Daniel Kerbel <nmdanny@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jan Fajerski <jfajersk@redhat.com>
Co-authored-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
Co-authored-by: Augustin Husson <husson.augustin@gmail.com>
Co-authored-by: Björn Rabenstein <beorn@grafana.com>
Co-authored-by: zenador <zenador@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>
Co-authored-by: Ben Kochie <superq@gmail.com>
Co-authored-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Co-authored-by: tyltr <tylitianrui@126.com>
Co-authored-by: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
Co-authored-by: Matthias Loibl <mail@matthiasloibl.com>
Co-authored-by: Ivan Babrou <github@ivan.computer>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Israel Blancas <iblancasa@gmail.com>
Co-authored-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Goutham <gouthamve@gmail.com>
Co-authored-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Co-authored-by: Chris Marchbanks <csmarchbanks@gmail.com>
Co-authored-by: Ben Ye <benye@amazon.com>
Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com>
Co-authored-by: Matthieu MOREL <matthieu.morel35@gmail.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>
Co-authored-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
Co-authored-by: Leegin <114397475+Leegin-darknight@users.noreply.github.com>
Co-authored-by: Guillermo Sanchez Gavier <gsanchez@newrelic.com>
Co-authored-by: Mikhail Fesenko <proggga@gmail.com>
Co-authored-by: Alan Protasio <alanprot@gmail.com>
2024-07-04 14:29:20 -07:00
if err := c . ProtobufMessage . Validate ( ) ; err != nil {
return fmt . Errorf ( "invalid protobuf_message value: %w" , err )
}
2017-07-25 05:47:34 -07:00
// The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer.
// We cannot make it a pointer as the parser panics for inlined pointer structs.
// Thus we just do its validation here.
2021-03-08 11:20:09 -08:00
if err := c . HTTPClientConfig . Validate ( ) ; err != nil {
return err
}
2024-07-30 08:25:19 -07:00
return validateAuthConfigs ( c )
}
2021-03-08 11:20:09 -08:00
2024-07-30 08:25:19 -07:00
// validateAuthConfigs validates that at most one of basic_auth, authorization, oauth2, sigv4, azuread or google_iam must be configured.
func validateAuthConfigs ( c * RemoteWriteConfig ) error {
var authConfigured [ ] string
if c . HTTPClientConfig . BasicAuth != nil {
authConfigured = append ( authConfigured , "basic_auth" )
2023-06-01 14:20:10 -07:00
}
2024-07-30 08:25:19 -07:00
if c . HTTPClientConfig . Authorization != nil {
authConfigured = append ( authConfigured , "authorization" )
}
if c . HTTPClientConfig . OAuth2 != nil {
authConfigured = append ( authConfigured , "oauth2" )
}
if c . SigV4Config != nil {
authConfigured = append ( authConfigured , "sigv4" )
}
if c . AzureADConfig != nil {
authConfigured = append ( authConfigured , "azuread" )
}
if c . GoogleIAMConfig != nil {
authConfigured = append ( authConfigured , "google_iam" )
}
if len ( authConfigured ) > 1 {
return fmt . Errorf ( "at most one of basic_auth, authorization, oauth2, sigv4, azuread or google_iam must be configured. Currently configured: %v" , authConfigured )
2021-03-08 11:20:09 -08:00
}
return nil
2016-09-19 13:47:51 -07:00
}
2017-03-10 03:53:27 -08:00
2022-02-22 12:44:36 -08:00
func validateHeadersForTracing ( headers map [ string ] string ) error {
for header := range headers {
if strings . ToLower ( header ) == "authorization" {
return errors . New ( "custom authorization header configuration is not yet supported" )
}
if _ , ok := reservedHeaders [ strings . ToLower ( header ) ] ; ok {
2022-05-23 00:54:32 -07:00
return fmt . Errorf ( "%s is a reserved header. It must not be changed" , header )
2022-02-22 12:44:36 -08:00
}
}
return nil
}
2021-02-18 04:12:21 -08:00
func validateHeaders ( headers map [ string ] string ) error {
for header := range headers {
if strings . ToLower ( header ) == "authorization" {
2024-07-30 08:25:19 -07:00
return errors . New ( "authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, azuread or google_iam parameter" )
2021-02-18 04:12:21 -08:00
}
if _ , ok := reservedHeaders [ strings . ToLower ( header ) ] ; ok {
2022-05-23 00:54:32 -07:00
return fmt . Errorf ( "%s is a reserved header. It must not be changed" , header )
2021-02-18 04:12:21 -08:00
}
}
return nil
}
2017-08-01 03:00:40 -07:00
// QueueConfig is the configuration for the queue used to write to remote
2017-07-25 05:47:34 -07:00
// storage.
2017-08-01 03:00:40 -07:00
type QueueConfig struct {
2019-08-13 02:10:21 -07:00
// Number of samples to buffer per shard before we block. Defaults to
// MaxSamplesPerSend.
2017-08-01 03:00:40 -07:00
Capacity int ` yaml:"capacity,omitempty" `
2017-07-25 05:47:34 -07:00
// Max number of shards, i.e. amount of concurrency.
2017-08-01 03:00:40 -07:00
MaxShards int ` yaml:"max_shards,omitempty" `
2017-07-25 05:47:34 -07:00
2018-12-04 09:32:14 -08:00
// Min number of shards, i.e. amount of concurrency.
MinShards int ` yaml:"min_shards,omitempty" `
2017-07-25 05:47:34 -07:00
// Maximum number of samples per send.
2017-08-01 03:00:40 -07:00
MaxSamplesPerSend int ` yaml:"max_samples_per_send,omitempty" `
2017-07-25 05:47:34 -07:00
// Maximum time sample will wait in buffer.
2018-08-24 07:55:21 -07:00
BatchSendDeadline model . Duration ` yaml:"batch_send_deadline,omitempty" `
2017-07-25 05:47:34 -07:00
// On recoverable errors, backoff exponentially.
2021-02-11 09:24:49 -08:00
MinBackoff model . Duration ` yaml:"min_backoff,omitempty" `
MaxBackoff model . Duration ` yaml:"max_backoff,omitempty" `
RetryOnRateLimit bool ` yaml:"retry_on_http_429,omitempty" `
2024-01-05 10:40:30 -08:00
// Samples older than the limit will be dropped.
SampleAgeLimit model . Duration ` yaml:"sample_age_limit,omitempty" `
2017-07-25 05:47:34 -07:00
}
2020-11-19 07:23:03 -08:00
// MetadataConfig is the configuration for sending metadata to remote
// storage.
type MetadataConfig struct {
// Send controls whether we send metric metadata to remote storage.
Send bool ` yaml:"send" `
// SendInterval controls how frequently we send metric metadata.
SendInterval model . Duration ` yaml:"send_interval" `
2021-06-24 15:39:50 -07:00
// Maximum number of samples per send.
MaxSamplesPerSend int ` yaml:"max_samples_per_send,omitempty" `
2020-11-19 07:23:03 -08:00
}
2024-08-27 23:23:54 -07:00
const (
// DefaultChunkedReadLimit is the default value for the maximum size of the protobuf frame client allows.
// 50MB is the default. This is equivalent to ~100k full XOR chunks and average labelset.
DefaultChunkedReadLimit = 5e+7
)
2017-03-10 03:53:27 -08:00
// RemoteReadConfig is the configuration for reading from remote storage.
type RemoteReadConfig struct {
2024-08-27 23:23:54 -07:00
URL * config . URL ` yaml:"url" `
RemoteTimeout model . Duration ` yaml:"remote_timeout,omitempty" `
ChunkedReadLimit uint64 ` yaml:"chunked_read_limit,omitempty" `
Headers map [ string ] string ` yaml:"headers,omitempty" `
ReadRecent bool ` yaml:"read_recent,omitempty" `
Name string ` yaml:"name,omitempty" `
2019-12-12 12:47:23 -08:00
2017-03-20 05:37:50 -07:00
// We cannot do proper Go type embedding below as the parser will then parse
// values arbitrarily into the overflow maps of further-down types.
2020-08-20 05:48:26 -07:00
HTTPClientConfig config . HTTPClientConfig ` yaml:",inline" `
2017-03-10 03:53:27 -08:00
2017-11-11 17:23:20 -08:00
// RequiredMatchers is an optional list of equality matchers which have to
// be present in a selector to query the remote read endpoint.
RequiredMatchers model . LabelSet ` yaml:"required_matchers,omitempty" `
2022-02-16 13:12:47 -08:00
// Whether to use the external labels as selectors for the remote read endpoint.
FilterExternalLabels bool ` yaml:"filter_external_labels,omitempty" `
2017-03-10 03:53:27 -08:00
}
2020-08-20 05:48:26 -07:00
// SetDirectory joins any relative file paths with dir.
func ( c * RemoteReadConfig ) SetDirectory ( dir string ) {
c . HTTPClientConfig . SetDirectory ( dir )
}
2017-03-10 03:53:27 -08:00
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func ( c * RemoteReadConfig ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
* c = DefaultRemoteReadConfig
type plain RemoteReadConfig
if err := unmarshal ( ( * plain ) ( c ) ) ; err != nil {
return err
}
2017-08-07 00:49:45 -07:00
if c . URL == nil {
2019-03-25 16:01:12 -07:00
return errors . New ( "url for remote_read is empty" )
2017-08-07 00:49:45 -07:00
}
2021-02-18 04:12:21 -08:00
if err := validateHeaders ( c . Headers ) ; err != nil {
return err
}
2017-07-25 05:47:34 -07:00
// The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer.
// We cannot make it a pointer as the parser panics for inlined pointer structs.
// Thus we just do its validation here.
2018-04-04 01:07:39 -07:00
return c . HTTPClientConfig . Validate ( )
2017-03-10 03:53:27 -08:00
}
2023-02-24 02:47:12 -08:00
func filePath ( filename string ) string {
absPath , err := filepath . Abs ( filename )
if err != nil {
return filename
}
return absPath
}
func fileErr ( filename string , err error ) error {
return fmt . Errorf ( "%q: %w" , filePath ( filename ) , err )
}
2024-04-01 08:34:35 -07:00
func getGoGCEnv ( ) int {
goGCEnv := os . Getenv ( "GOGC" )
// If the GOGC env var is set, use the same logic as upstream Go.
if goGCEnv != "" {
// Special case for GOGC=off.
if strings . ToLower ( goGCEnv ) == "off" {
return - 1
}
i , err := strconv . Atoi ( goGCEnv )
if err == nil {
return i
}
}
return DefaultRuntimeConfig . GoGC
}
2024-06-03 09:02:26 -07:00
2024-11-08 05:02:48 -08:00
type translationStrategyOption string
var (
// NoUTF8EscapingWithSuffixes will keep UTF-8 characters as they are, units and type suffixes will still be added.
NoUTF8EscapingWithSuffixes translationStrategyOption = "NoUTF8EscapingWithSuffixes"
// UnderscoreEscapingWithSuffixes is the default option for translating OTLP to Prometheus.
// This option will translate all UTF-8 characters to underscores, while adding units and type suffixes.
UnderscoreEscapingWithSuffixes translationStrategyOption = "UnderscoreEscapingWithSuffixes"
)
2024-06-03 09:02:26 -07:00
// OTLPConfig is the configuration for writing to the OTLP endpoint.
type OTLPConfig struct {
2024-11-08 05:02:48 -08:00
PromoteResourceAttributes [ ] string ` yaml:"promote_resource_attributes,omitempty" `
TranslationStrategy translationStrategyOption ` yaml:"translation_strategy,omitempty" `
2024-11-29 06:12:57 -08:00
ServiceNameInTargetInfo bool ` yaml:"service_name_in_target_info,omitempty" `
2024-06-03 09:02:26 -07:00
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func ( c * OTLPConfig ) UnmarshalYAML ( unmarshal func ( interface { } ) error ) error {
* c = DefaultOTLPConfig
type plain OTLPConfig
2024-07-16 05:32:24 -07:00
if err := unmarshal ( ( * plain ) ( c ) ) ; err != nil {
return err
}
seen := map [ string ] struct { } { }
2024-07-18 01:40:47 -07:00
var err error
for i , attr := range c . PromoteResourceAttributes {
attr = strings . TrimSpace ( attr )
if attr == "" {
2024-11-03 04:15:51 -08:00
err = errors . Join ( err , errors . New ( "empty promoted OTel resource attribute" ) )
2024-07-18 01:40:47 -07:00
continue
}
if _ , exists := seen [ attr ] ; exists {
err = errors . Join ( err , fmt . Errorf ( "duplicated promoted OTel resource attribute %q" , attr ) )
2024-07-16 05:32:24 -07:00
continue
}
2024-07-18 01:40:47 -07:00
seen [ attr ] = struct { } { }
c . PromoteResourceAttributes [ i ] = attr
2024-07-16 05:32:24 -07:00
}
2024-07-18 01:40:47 -07:00
return err
2024-06-03 09:02:26 -07:00
}