From 2ced3c78c011c681715e824803d6c9cce9c64db1 Mon Sep 17 00:00:00 2001 From: Augustin Husson Date: Sat, 24 Dec 2022 18:30:11 +0100 Subject: [PATCH 01/23] fix ui bump version on mac Signed-off-by: Augustin Husson --- scripts/ui_release.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/ui_release.sh b/scripts/ui_release.sh index 3693cc718f..9f9ad3edaf 100755 --- a/scripts/ui_release.sh +++ b/scripts/ui_release.sh @@ -75,12 +75,17 @@ function bumpVersion() { if [[ "${version}" == v* ]]; then version="${version:1}" fi - # increase the version on all packages - npm version "${version}" --workspaces # upgrade the @prometheus-io/* dependencies on all packages for workspace in ${workspaces}; do - sed -E -i "s|(\"@prometheus-io/.+\": )\".+\"|\1\"\^${version}\"|" "${workspace}"/package.json + # sed -i syntax is different on mac and linux + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -E -i "" "s|(\"@prometheus-io/.+\": )\".+\"|\1\"${version}\"|" "${workspace}"/package.json + else + sed -E -i "s|(\"@prometheus-io/.+\": )\".+\"|\1\"${version}\"|" "${workspace}"/package.json + fi done + # increase the version on all packages + npm version "${version}" --workspaces } if [[ "$1" == "--copy" ]]; then From f3c480e7bc2e53269c78720506c9e198f40e0170 Mon Sep 17 00:00:00 2001 From: Augustin Husson Date: Thu, 29 Dec 2022 11:23:46 +0100 Subject: [PATCH 02/23] fix indentation issue Signed-off-by: Augustin Husson --- scripts/ui_release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ui_release.sh b/scripts/ui_release.sh index 9f9ad3edaf..0f251690b7 100755 --- a/scripts/ui_release.sh +++ b/scripts/ui_release.sh @@ -84,8 +84,8 @@ function bumpVersion() { sed -E -i "s|(\"@prometheus-io/.+\": )\".+\"|\1\"${version}\"|" "${workspace}"/package.json fi done - # increase the version on all packages - npm version "${version}" --workspaces + # increase the version on all packages + npm version "${version}" --workspaces } if [[ "$1" == "--copy" ]]; then From 2d61d012ff7c96a12a8d1aba52d3eb19e8d2db46 Mon Sep 17 00:00:00 2001 From: "Xiaochao Dong (@damnever)" Date: Fri, 30 Dec 2022 19:11:29 +0800 Subject: [PATCH 03/23] Avoid copy during remote write if external labels is empty Signed-off-by: Xiaochao Dong (@damnever) --- storage/remote/queue_manager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go index f907615fc9..15634aa68d 100644 --- a/storage/remote/queue_manager.go +++ b/storage/remote/queue_manager.go @@ -847,6 +847,10 @@ func (t *QueueManager) releaseLabels(ls labels.Labels) { // processExternalLabels merges externalLabels into ls. If ls contains // a label in externalLabels, the value in ls wins. func processExternalLabels(ls labels.Labels, externalLabels []labels.Label) labels.Labels { + if len(externalLabels) == 0 { + return ls + } + b := labels.NewScratchBuilder(ls.Len() + len(externalLabels)) j := 0 ls.Range(func(l labels.Label) { From e3b5416d05704bc766256cf76d57cf9992bb714b Mon Sep 17 00:00:00 2001 From: Damien Grisonnet Date: Wed, 7 Dec 2022 14:32:03 +0100 Subject: [PATCH 04/23] model/textparse: improve error outputs Parsing errors in the Prometheus HTTP format parser are very hard to investigate since they only approximately indicate what is going wrong in the parser and don't provide any information about the incorrect input. As such it is very hard to tell what is wrong in the format exposed by the application. Signed-off-by: Damien Grisonnet --- model/textparse/openmetricsparse.go | 4 +++ model/textparse/promparse.go | 36 ++++++++++++++------------ model/textparse/promparse_test.go | 40 ++++++++++++++--------------- 3 files changed, 44 insertions(+), 36 deletions(-) diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go index 15a95a9592..2ab8c5503e 100644 --- a/model/textparse/openmetricsparse.go +++ b/model/textparse/openmetricsparse.go @@ -223,6 +223,10 @@ func (p *OpenMetricsParser) nextToken() token { return tok } +func parseError(exp string, got token) error { + return fmt.Errorf("%s, got %q", exp, got) +} + // Next advances the parser to the next sample. It returns false if no // more samples were read or an error occurred. func (p *OpenMetricsParser) Next() (Entry, error) { diff --git a/model/textparse/promparse.go b/model/textparse/promparse.go index b0c963392d..37ccde002d 100644 --- a/model/textparse/promparse.go +++ b/model/textparse/promparse.go @@ -254,8 +254,12 @@ func (p *PromParser) nextToken() token { } } -func parseError(exp string, got token) error { - return fmt.Errorf("%s, got %q", exp, got) +func (p *PromParser) parseError(exp string, got token) error { + e := p.l.i + 1 + if len(p.l.b) <= e { + e = len(p.l.b) + } + return fmt.Errorf("%s, got %q (%q) while parsing: %q", exp, p.l.b[p.l.start:e], got, p.l.b[p.start:e]) } // Next advances the parser to the next sample. It returns false if no @@ -278,7 +282,7 @@ func (p *PromParser) Next() (Entry, error) { case tMName: p.offsets = append(p.offsets, p.l.start, p.l.i) default: - return EntryInvalid, parseError("expected metric name after "+t.String(), t2) + return EntryInvalid, p.parseError("expected metric name after "+t.String(), t2) } switch t2 := p.nextToken(); t2 { case tText: @@ -308,11 +312,11 @@ func (p *PromParser) Next() (Entry, error) { } case tHelp: if !utf8.Valid(p.text) { - return EntryInvalid, fmt.Errorf("help text is not a valid utf8 string") + return EntryInvalid, fmt.Errorf("help text %q is not a valid utf8 string", p.text) } } if t := p.nextToken(); t != tLinebreak { - return EntryInvalid, parseError("linebreak expected after metadata", t) + return EntryInvalid, p.parseError("linebreak expected after metadata", t) } switch t { case tHelp: @@ -323,7 +327,7 @@ func (p *PromParser) Next() (Entry, error) { case tComment: p.text = p.l.buf() if t := p.nextToken(); t != tLinebreak { - return EntryInvalid, parseError("linebreak expected after comment", t) + return EntryInvalid, p.parseError("linebreak expected after comment", t) } return EntryComment, nil @@ -340,10 +344,10 @@ func (p *PromParser) Next() (Entry, error) { t2 = p.nextToken() } if t2 != tValue { - return EntryInvalid, parseError("expected value after metric", t2) + return EntryInvalid, p.parseError("expected value after metric", t2) } if p.val, err = parseFloat(yoloString(p.l.buf())); err != nil { - return EntryInvalid, err + return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i]) } // Ensure canonical NaN value. if math.IsNaN(p.val) { @@ -356,18 +360,18 @@ func (p *PromParser) Next() (Entry, error) { case tTimestamp: p.hasTS = true if p.ts, err = strconv.ParseInt(yoloString(p.l.buf()), 10, 64); err != nil { - return EntryInvalid, err + return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i]) } if t2 := p.nextToken(); t2 != tLinebreak { - return EntryInvalid, parseError("expected next entry after timestamp", t2) + return EntryInvalid, p.parseError("expected next entry after timestamp", t2) } default: - return EntryInvalid, parseError("expected timestamp or new record", t) + return EntryInvalid, p.parseError("expected timestamp or new record", t) } return EntrySeries, nil default: - err = fmt.Errorf("%q is not a valid start token", t) + err = p.parseError("expected a valid start token", t) } return EntryInvalid, err } @@ -380,18 +384,18 @@ func (p *PromParser) parseLVals() error { return nil case tLName: default: - return parseError("expected label name", t) + return p.parseError("expected label name", t) } p.offsets = append(p.offsets, p.l.start, p.l.i) if t := p.nextToken(); t != tEqual { - return parseError("expected equal", t) + return p.parseError("expected equal", t) } if t := p.nextToken(); t != tLValue { - return parseError("expected label value", t) + return p.parseError("expected label value", t) } if !utf8.Valid(p.l.buf()) { - return fmt.Errorf("invalid UTF-8 label value") + return fmt.Errorf("invalid UTF-8 label value: %q", p.l.buf()) } // The promlexer ensures the value string is quoted. Strip first diff --git a/model/textparse/promparse_test.go b/model/textparse/promparse_test.go index 6788b3c2e4..e0ecf62f5d 100644 --- a/model/textparse/promparse_test.go +++ b/model/textparse/promparse_test.go @@ -219,63 +219,63 @@ func TestPromParseErrors(t *testing.T) { }{ { input: "a", - err: "expected value after metric, got \"INVALID\"", + err: "expected value after metric, got \"\\n\" (\"INVALID\") while parsing: \"a\\n\"", }, { input: "a{b='c'} 1\n", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"'\" (\"INVALID\") while parsing: \"a{b='\"", }, { input: "a{b=\n", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"\\n\" (\"INVALID\") while parsing: \"a{b=\\n\"", }, { input: "a{\xff=\"foo\"} 1\n", - err: "expected label name, got \"INVALID\"", + err: "expected label name, got \"\\xff\" (\"INVALID\") while parsing: \"a{\\xff\"", }, { input: "a{b=\"\xff\"} 1\n", - err: "invalid UTF-8 label value", + err: "invalid UTF-8 label value: \"\\\"\\xff\\\"\"", }, { input: "a true\n", - err: "strconv.ParseFloat: parsing \"true\": invalid syntax", + err: "strconv.ParseFloat: parsing \"true\": invalid syntax while parsing: \"a true\"", }, { input: "something_weird{problem=\"", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"\\\"\\n\" (\"INVALID\") while parsing: \"something_weird{problem=\\\"\\n\"", }, { input: "empty_label_name{=\"\"} 0", - err: "expected label name, got \"EQUAL\"", + err: "expected label name, got \"=\\\"\" (\"EQUAL\") while parsing: \"empty_label_name{=\\\"\"", }, { input: "foo 1_2\n", - err: "unsupported character in float", + err: "unsupported character in float while parsing: \"foo 1_2\"", }, { input: "foo 0x1p-3\n", - err: "unsupported character in float", + err: "unsupported character in float while parsing: \"foo 0x1p-3\"", }, { input: "foo 0x1P-3\n", - err: "unsupported character in float", + err: "unsupported character in float while parsing: \"foo 0x1P-3\"", }, { input: "foo 0 1_2\n", - err: "expected next entry after timestamp, got \"INVALID\"", + err: "expected next entry after timestamp, got \"_\" (\"INVALID\") while parsing: \"foo 0 1_\"", }, { input: `{a="ok"} 1`, - err: `"INVALID" is not a valid start token`, + err: "expected a valid start token, got \"{\" (\"INVALID\") while parsing: \"{\"", }, { input: "# TYPE #\n#EOF\n", - err: "expected metric name after TYPE, got \"INVALID\"", + err: "expected metric name after TYPE, got \"#\" (\"INVALID\") while parsing: \"# TYPE #\"", }, { input: "# HELP #\n#EOF\n", - err: "expected metric name after HELP, got \"INVALID\"", + err: "expected metric name after HELP, got \"#\" (\"INVALID\") while parsing: \"# HELP #\"", }, } @@ -313,23 +313,23 @@ func TestPromNullByteHandling(t *testing.T) { }, { input: "a{b=\x00\"ssss\"} 1\n", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"\\x00\" (\"INVALID\") while parsing: \"a{b=\\x00\"", }, { input: "a{b=\"\x00", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"\\\"\\x00\\n\" (\"INVALID\") while parsing: \"a{b=\\\"\\x00\\n\"", }, { input: "a{b\x00=\"hiih\"} 1", - err: "expected equal, got \"INVALID\"", + err: "expected equal, got \"\\x00\" (\"INVALID\") while parsing: \"a{b\\x00\"", }, { input: "a\x00{b=\"ddd\"} 1", - err: "expected value after metric, got \"INVALID\"", + err: "expected value after metric, got \"\\x00\" (\"INVALID\") while parsing: \"a\\x00\"", }, { input: "a 0 1\x00", - err: "expected next entry after timestamp, got \"INVALID\"", + err: "expected next entry after timestamp, got \"\\x00\" (\"INVALID\") while parsing: \"a 0 1\\x00\"", }, } From a13249a98f483146122de5203e4f3c241e96c884 Mon Sep 17 00:00:00 2001 From: Jimmie Han Date: Tue, 21 Feb 2023 13:14:01 +0800 Subject: [PATCH 05/23] scrape: fix prometheus_target_scrape_pool_target_limit metric not set on creating scrape pool (#12001) Signed-off-by: Jimmie Han --- scrape/scrape.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrape/scrape.go b/scrape/scrape.go index 562b376caa..a92bd22cb2 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -328,7 +328,7 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, jitterSeed options.PassMetadataInContext, ) } - + targetScrapePoolTargetLimit.WithLabelValues(sp.config.JobName).Set(float64(sp.config.TargetLimit)) return sp, nil } From 0fee61571951db385c7a1cc30ecd0a073a975ae4 Mon Sep 17 00:00:00 2001 From: Damien Grisonnet Date: Fri, 3 Feb 2023 16:50:15 +0100 Subject: [PATCH 06/23] model/textparse: improve openmetrics error outputs Signed-off-by: Damien Grisonnet --- model/textparse/openmetricsparse.go | 59 ++++++----- model/textparse/openmetricsparse_test.go | 120 +++++++++++------------ model/textparse/promparse.go | 2 +- 3 files changed, 89 insertions(+), 92 deletions(-) diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go index 2ab8c5503e..0fe8010876 100644 --- a/model/textparse/openmetricsparse.go +++ b/model/textparse/openmetricsparse.go @@ -46,13 +46,6 @@ func (l *openMetricsLexer) buf() []byte { return l.b[l.start:l.i] } -func (l *openMetricsLexer) cur() byte { - if l.i < len(l.b) { - return l.b[l.i] - } - return byte(' ') -} - // next advances the openMetricsLexer to the next character. func (l *openMetricsLexer) next() byte { l.i++ @@ -223,8 +216,12 @@ func (p *OpenMetricsParser) nextToken() token { return tok } -func parseError(exp string, got token) error { - return fmt.Errorf("%s, got %q", exp, got) +func (p *OpenMetricsParser) parseError(exp string, got token) error { + e := p.l.i + 1 + if len(p.l.b) < e { + e = len(p.l.b) + } + return fmt.Errorf("%s, got %q (%q) while parsing: %q", exp, p.l.b[p.l.start:e], got, p.l.b[p.start:e]) } // Next advances the parser to the next sample. It returns false if no @@ -252,7 +249,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) { case tMName: p.offsets = append(p.offsets, p.l.start, p.l.i) default: - return EntryInvalid, parseError("expected metric name after "+t.String(), t2) + return EntryInvalid, p.parseError("expected metric name after "+t.String(), t2) } switch t2 := p.nextToken(); t2 { case tText: @@ -288,7 +285,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) { } case tHelp: if !utf8.Valid(p.text) { - return EntryInvalid, errors.New("help text is not a valid utf8 string") + return EntryInvalid, fmt.Errorf("help text %q is not a valid utf8 string", p.text) } } switch t { @@ -301,7 +298,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) { u := yoloString(p.text) if len(u) > 0 { if !strings.HasSuffix(m, u) || len(m) < len(u)+1 || p.l.b[p.offsets[1]-len(u)-1] != '_' { - return EntryInvalid, fmt.Errorf("unit not a suffix of metric %q", m) + return EntryInvalid, fmt.Errorf("unit %q not a suffix of metric %q", u, m) } } return EntryUnit, nil @@ -340,10 +337,10 @@ func (p *OpenMetricsParser) Next() (Entry, error) { var ts float64 // A float is enough to hold what we need for millisecond resolution. if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil { - return EntryInvalid, err + return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i]) } if math.IsNaN(ts) || math.IsInf(ts, 0) { - return EntryInvalid, errors.New("invalid timestamp") + return EntryInvalid, fmt.Errorf("invalid timestamp %f", ts) } p.ts = int64(ts * 1000) switch t3 := p.nextToken(); t3 { @@ -353,15 +350,15 @@ func (p *OpenMetricsParser) Next() (Entry, error) { return EntryInvalid, err } default: - return EntryInvalid, parseError("expected next entry after timestamp", t3) + return EntryInvalid, p.parseError("expected next entry after timestamp", t3) } default: - return EntryInvalid, parseError("expected timestamp or # symbol", t2) + return EntryInvalid, p.parseError("expected timestamp or # symbol", t2) } return EntrySeries, nil default: - err = fmt.Errorf("%q %q is not a valid start token", t, string(p.l.cur())) + err = p.parseError("expected a valid start token", t) } return EntryInvalid, err } @@ -399,19 +396,19 @@ func (p *OpenMetricsParser) parseComment() error { var ts float64 // A float is enough to hold what we need for millisecond resolution. if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil { - return err + return fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i]) } if math.IsNaN(ts) || math.IsInf(ts, 0) { - return errors.New("invalid exemplar timestamp") + return fmt.Errorf("invalid exemplar timestamp %f", ts) } p.exemplarTs = int64(ts * 1000) switch t3 := p.nextToken(); t3 { case tLinebreak: default: - return parseError("expected next entry after exemplar timestamp", t3) + return p.parseError("expected next entry after exemplar timestamp", t3) } default: - return parseError("expected timestamp or comment", t2) + return p.parseError("expected timestamp or comment", t2) } return nil } @@ -425,21 +422,21 @@ func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) { return offsets, nil case tComma: if first { - return nil, parseError("expected label name or left brace", t) + return nil, p.parseError("expected label name or left brace", t) } t = p.nextToken() if t != tLName { - return nil, parseError("expected label name", t) + return nil, p.parseError("expected label name", t) } case tLName: if !first { - return nil, parseError("expected comma", t) + return nil, p.parseError("expected comma", t) } default: if first { - return nil, parseError("expected label name or left brace", t) + return nil, p.parseError("expected label name or left brace", t) } - return nil, parseError("expected comma or left brace", t) + return nil, p.parseError("expected comma or left brace", t) } first = false @@ -448,13 +445,13 @@ func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) { offsets = append(offsets, p.l.start, p.l.i) if t := p.nextToken(); t != tEqual { - return nil, parseError("expected equal", t) + return nil, p.parseError("expected equal", t) } if t := p.nextToken(); t != tLValue { - return nil, parseError("expected label value", t) + return nil, p.parseError("expected label value", t) } if !utf8.Valid(p.l.buf()) { - return nil, errors.New("invalid UTF-8 label value") + return nil, fmt.Errorf("invalid UTF-8 label value: %q", p.l.buf()) } // The openMetricsLexer ensures the value string is quoted. Strip first @@ -465,11 +462,11 @@ func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) { func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error) { if t != tValue { - return 0, parseError(fmt.Sprintf("expected value after %v", after), t) + return 0, p.parseError(fmt.Sprintf("expected value after %v", after), t) } val, err := parseFloat(yoloString(p.l.buf()[1:])) if err != nil { - return 0, err + return 0, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i]) } // Ensure canonical NaN value. if math.IsNaN(p.exemplarVal) { diff --git a/model/textparse/openmetricsparse_test.go b/model/textparse/openmetricsparse_test.go index 68b7fea8a0..12fb03f015 100644 --- a/model/textparse/openmetricsparse_test.go +++ b/model/textparse/openmetricsparse_test.go @@ -293,11 +293,11 @@ func TestOpenMetricsParseErrors(t *testing.T) { }, { input: "\n", - err: "\"INVALID\" \"\\n\" is not a valid start token", + err: "expected a valid start token, got \"\\n\" (\"INVALID\") while parsing: \"\\n\"", }, { input: "metric", - err: "expected value after metric, got \"EOF\"", + err: "expected value after metric, got \"metric\" (\"EOF\") while parsing: \"metric\"", }, { input: "metric 1", @@ -313,19 +313,19 @@ func TestOpenMetricsParseErrors(t *testing.T) { }, { input: "a\n#EOF\n", - err: "expected value after metric, got \"INVALID\"", + err: "expected value after metric, got \"\\n\" (\"INVALID\") while parsing: \"a\\n\"", }, { input: "\n\n#EOF\n", - err: "\"INVALID\" \"\\n\" is not a valid start token", + err: "expected a valid start token, got \"\\n\" (\"INVALID\") while parsing: \"\\n\"", }, { input: " a 1\n#EOF\n", - err: "\"INVALID\" \" \" is not a valid start token", + err: "expected a valid start token, got \" \" (\"INVALID\") while parsing: \" \"", }, { input: "9\n#EOF\n", - err: "\"INVALID\" \"9\" is not a valid start token", + err: "expected a valid start token, got \"9\" (\"INVALID\") while parsing: \"9\"", }, { input: "# TYPE u untyped\n#EOF\n", @@ -337,11 +337,11 @@ func TestOpenMetricsParseErrors(t *testing.T) { }, { input: "# TYPE c counter\n#EOF\n", - err: "\"INVALID\" \" \" is not a valid start token", + err: "expected a valid start token, got \"# \" (\"INVALID\") while parsing: \"# \"", }, { input: "# TYPE \n#EOF\n", - err: "expected metric name after TYPE, got \"INVALID\"", + err: "expected metric name after TYPE, got \"\\n\" (\"INVALID\") while parsing: \"# TYPE \\n\"", }, { input: "# TYPE m\n#EOF\n", @@ -349,19 +349,19 @@ func TestOpenMetricsParseErrors(t *testing.T) { }, { input: "# UNIT metric suffix\n#EOF\n", - err: "unit not a suffix of metric \"metric\"", + err: "unit \"suffix\" not a suffix of metric \"metric\"", }, { input: "# UNIT metricsuffix suffix\n#EOF\n", - err: "unit not a suffix of metric \"metricsuffix\"", + err: "unit \"suffix\" not a suffix of metric \"metricsuffix\"", }, { input: "# UNIT m suffix\n#EOF\n", - err: "unit not a suffix of metric \"m\"", + err: "unit \"suffix\" not a suffix of metric \"m\"", }, { input: "# UNIT \n#EOF\n", - err: "expected metric name after UNIT, got \"INVALID\"", + err: "expected metric name after UNIT, got \"\\n\" (\"INVALID\") while parsing: \"# UNIT \\n\"", }, { input: "# UNIT m\n#EOF\n", @@ -369,7 +369,7 @@ func TestOpenMetricsParseErrors(t *testing.T) { }, { input: "# HELP \n#EOF\n", - err: "expected metric name after HELP, got \"INVALID\"", + err: "expected metric name after HELP, got \"\\n\" (\"INVALID\") while parsing: \"# HELP \\n\"", }, { input: "# HELP m\n#EOF\n", @@ -377,27 +377,27 @@ func TestOpenMetricsParseErrors(t *testing.T) { }, { input: "a\t1\n#EOF\n", - err: "expected value after metric, got \"INVALID\"", + err: "expected value after metric, got \"\\t\" (\"INVALID\") while parsing: \"a\\t\"", }, { input: "a 1\t2\n#EOF\n", - err: "strconv.ParseFloat: parsing \"1\\t2\": invalid syntax", + err: "strconv.ParseFloat: parsing \"1\\t2\": invalid syntax while parsing: \"a 1\\t2\"", }, { input: "a 1 2 \n#EOF\n", - err: "expected next entry after timestamp, got \"INVALID\"", + err: "expected next entry after timestamp, got \" \\n\" (\"INVALID\") while parsing: \"a 1 2 \\n\"", }, { input: "a 1 2 #\n#EOF\n", - err: "expected next entry after timestamp, got \"TIMESTAMP\"", + err: "expected next entry after timestamp, got \" #\\n\" (\"TIMESTAMP\") while parsing: \"a 1 2 #\\n\"", }, { input: "a 1 1z\n#EOF\n", - err: "strconv.ParseFloat: parsing \"1z\": invalid syntax", + err: "strconv.ParseFloat: parsing \"1z\": invalid syntax while parsing: \"a 1 1z\"", }, { input: " # EOF\n", - err: "\"INVALID\" \" \" is not a valid start token", + err: "expected a valid start token, got \" \" (\"INVALID\") while parsing: \" \"", }, { input: "# EOF\na 1", @@ -413,7 +413,7 @@ func TestOpenMetricsParseErrors(t *testing.T) { }, { input: "#\tTYPE c counter\n", - err: "\"INVALID\" \"\\t\" is not a valid start token", + err: "expected a valid start token, got \"#\\t\" (\"INVALID\") while parsing: \"#\\t\"", }, { input: "# TYPE c counter\n", @@ -421,79 +421,79 @@ func TestOpenMetricsParseErrors(t *testing.T) { }, { input: "a 1 1 1\n# EOF\n", - err: "expected next entry after timestamp, got \"TIMESTAMP\"", + err: "expected next entry after timestamp, got \" 1\\n\" (\"TIMESTAMP\") while parsing: \"a 1 1 1\\n\"", }, { input: "a{b='c'} 1\n# EOF\n", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"'\" (\"INVALID\") while parsing: \"a{b='\"", }, { input: "a{b=\"c\",} 1\n# EOF\n", - err: "expected label name, got \"BCLOSE\"", + err: "expected label name, got \"} \" (\"BCLOSE\") while parsing: \"a{b=\\\"c\\\",} \"", }, { input: "a{,b=\"c\"} 1\n# EOF\n", - err: "expected label name or left brace, got \"COMMA\"", + err: "expected label name or left brace, got \",b\" (\"COMMA\") while parsing: \"a{,b\"", }, { input: "a{b=\"c\"d=\"e\"} 1\n# EOF\n", - err: "expected comma, got \"LNAME\"", + err: "expected comma, got \"d=\" (\"LNAME\") while parsing: \"a{b=\\\"c\\\"d=\"", }, { input: "a{b=\"c\",,d=\"e\"} 1\n# EOF\n", - err: "expected label name, got \"COMMA\"", + err: "expected label name, got \",d\" (\"COMMA\") while parsing: \"a{b=\\\"c\\\",,d\"", }, { input: "a{b=\n# EOF\n", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"\\n\" (\"INVALID\") while parsing: \"a{b=\\n\"", }, { input: "a{\xff=\"foo\"} 1\n# EOF\n", - err: "expected label name or left brace, got \"INVALID\"", + err: "expected label name or left brace, got \"\\xff\" (\"INVALID\") while parsing: \"a{\\xff\"", }, { input: "a{b=\"\xff\"} 1\n# EOF\n", - err: "invalid UTF-8 label value", + err: "invalid UTF-8 label value: \"\\\"\\xff\\\"\"", }, { input: "a true\n", - err: "strconv.ParseFloat: parsing \"true\": invalid syntax", + err: "strconv.ParseFloat: parsing \"true\": invalid syntax while parsing: \"a true\"", }, { input: "something_weird{problem=\"\n# EOF\n", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"\\\"\\n\" (\"INVALID\") while parsing: \"something_weird{problem=\\\"\\n\"", }, { input: "empty_label_name{=\"\"} 0\n# EOF\n", - err: "expected label name or left brace, got \"EQUAL\"", + err: "expected label name or left brace, got \"=\\\"\" (\"EQUAL\") while parsing: \"empty_label_name{=\\\"\"", }, { input: "foo 1_2\n\n# EOF\n", - err: "unsupported character in float", + err: "unsupported character in float while parsing: \"foo 1_2\"", }, { input: "foo 0x1p-3\n\n# EOF\n", - err: "unsupported character in float", + err: "unsupported character in float while parsing: \"foo 0x1p-3\"", }, { input: "foo 0x1P-3\n\n# EOF\n", - err: "unsupported character in float", + err: "unsupported character in float while parsing: \"foo 0x1P-3\"", }, { input: "foo 0 1_2\n\n# EOF\n", - err: "unsupported character in float", + err: "unsupported character in float while parsing: \"foo 0 1_2\"", }, { input: "custom_metric_total 1 # {aa=bb}\n# EOF\n", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"b\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=b\"", }, { input: "custom_metric_total 1 # {aa=\"bb\"}\n# EOF\n", - err: "expected value after exemplar labels, got \"INVALID\"", + err: "expected value after exemplar labels, got \"\\n\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"}\\n\"", }, { input: `custom_metric_total 1 # {aa="bb"}`, - err: "expected value after exemplar labels, got \"EOF\"", + err: "expected value after exemplar labels, got \"}\" (\"EOF\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"}\"", }, { input: `custom_metric 1 # {aa="bb"}`, @@ -501,55 +501,55 @@ func TestOpenMetricsParseErrors(t *testing.T) { }, { input: `custom_metric_total 1 # {aa="bb",,cc="dd"} 1`, - err: "expected label name, got \"COMMA\"", + err: "expected label name, got \",c\" (\"COMMA\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\",,c\"", }, { input: `custom_metric_total 1 # {aa="bb"} 1_2`, - err: "unsupported character in float", + err: "unsupported character in float while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"} 1_2\"", }, { input: `custom_metric_total 1 # {aa="bb"} 0x1p-3`, - err: "unsupported character in float", + err: "unsupported character in float while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"} 0x1p-3\"", }, { input: `custom_metric_total 1 # {aa="bb"} true`, - err: "strconv.ParseFloat: parsing \"true\": invalid syntax", + err: "strconv.ParseFloat: parsing \"true\": invalid syntax while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"} true\"", }, { input: `custom_metric_total 1 # {aa="bb",cc=}`, - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"}\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\",cc=}\"", }, { input: `custom_metric_total 1 # {aa=\"\xff\"} 9.0`, - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"\\\\\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=\\\\\"", }, { input: `{b="c",} 1`, - err: `"INVALID" "{" is not a valid start token`, + err: "expected a valid start token, got \"{\" (\"INVALID\") while parsing: \"{\"", }, { input: `a 1 NaN`, - err: `invalid timestamp`, + err: `invalid timestamp NaN`, }, { input: `a 1 -Inf`, - err: `invalid timestamp`, + err: `invalid timestamp -Inf`, }, { input: `a 1 Inf`, - err: `invalid timestamp`, + err: `invalid timestamp +Inf`, }, { input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 NaN", - err: `invalid exemplar timestamp`, + err: `invalid exemplar timestamp NaN`, }, { input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 -Inf", - err: `invalid exemplar timestamp`, + err: `invalid exemplar timestamp -Inf`, }, { input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 Inf", - err: `invalid exemplar timestamp`, + err: `invalid exemplar timestamp +Inf`, }, } @@ -586,35 +586,35 @@ func TestOMNullByteHandling(t *testing.T) { }, { input: "a{b=\x00\"ssss\"} 1\n# EOF\n", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"\\x00\" (\"INVALID\") while parsing: \"a{b=\\x00\"", }, { input: "a{b=\"\x00", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"\\\"\\x00\" (\"INVALID\") while parsing: \"a{b=\\\"\\x00\"", }, { input: "a{b\x00=\"hiih\"} 1", - err: "expected equal, got \"INVALID\"", + err: "expected equal, got \"\\x00\" (\"INVALID\") while parsing: \"a{b\\x00\"", }, { input: "a\x00{b=\"ddd\"} 1", - err: "expected value after metric, got \"INVALID\"", + err: "expected value after metric, got \"\\x00\" (\"INVALID\") while parsing: \"a\\x00\"", }, { input: "#", - err: "\"INVALID\" \" \" is not a valid start token", + err: "expected a valid start token, got \"#\" (\"INVALID\") while parsing: \"#\"", }, { input: "# H", - err: "\"INVALID\" \" \" is not a valid start token", + err: "expected a valid start token, got \"# H\" (\"INVALID\") while parsing: \"# H\"", }, { input: "custom_metric_total 1 # {b=\x00\"ssss\"} 1\n", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"\\x00\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {b=\\x00\"", }, { input: "custom_metric_total 1 # {b=\"\x00ss\"} 1\n", - err: "expected label value, got \"INVALID\"", + err: "expected label value, got \"\\\"\\x00\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {b=\\\"\\x00\"", }, } diff --git a/model/textparse/promparse.go b/model/textparse/promparse.go index 37ccde002d..2c981f050e 100644 --- a/model/textparse/promparse.go +++ b/model/textparse/promparse.go @@ -256,7 +256,7 @@ func (p *PromParser) nextToken() token { func (p *PromParser) parseError(exp string, got token) error { e := p.l.i + 1 - if len(p.l.b) <= e { + if len(p.l.b) < e { e = len(p.l.b) } return fmt.Errorf("%s, got %q (%q) while parsing: %q", exp, p.l.b[p.l.start:e], got, p.l.b[p.start:e]) From f34b2cede3c61897dd0e9248dbc6b94b26b0689e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 20 Dec 2022 15:26:17 +0000 Subject: [PATCH 07/23] Remove microbenchmarks These benchmarks are all testing things related to what Prometheus does, so perhaps have some historical interest, but we should not retain them in the main repo. Signed-off-by: Bryan Boreham --- go.mod | 1 - go.sum | 2 - tsdb/test/conv_test.go | 58 ----------- tsdb/test/hash_test.go | 123 ---------------------- tsdb/test/labels_test.go | 214 --------------------------------------- 5 files changed, 398 deletions(-) delete mode 100644 tsdb/test/conv_test.go delete mode 100644 tsdb/test/hash_test.go delete mode 100644 tsdb/test/labels_test.go diff --git a/go.mod b/go.mod index ac94408e4b..8df8696b6d 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/aws/aws-sdk-go v1.44.187 github.com/cespare/xxhash/v2 v2.2.0 github.com/dennwc/varint v1.0.0 - github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245 github.com/digitalocean/godo v1.95.0 github.com/docker/docker v20.10.23+incompatible github.com/edsrzf/mmap-go v1.1.0 diff --git a/go.sum b/go.sum index 06b37a3099..28342ab228 100644 --- a/go.sum +++ b/go.sum @@ -148,8 +148,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245 h1:9cOfvEwjQxdwKuNDTQSaMKNRvwKwgZG+U4HrjeRKHso= -github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/digitalocean/godo v1.95.0 h1:S48/byPKui7RHZc1wYEPfRvkcEvToADNb5I3guu95xg= github.com/digitalocean/godo v1.95.0/go.mod h1:NRpFznZFvhHjBoqZAaOD3khVzsJ3EibzKqFL4R60dmA= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= diff --git a/tsdb/test/conv_test.go b/tsdb/test/conv_test.go deleted file mode 100644 index 0d34b99876..0000000000 --- a/tsdb/test/conv_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2017 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import "testing" - -func BenchmarkMapConversion(b *testing.B) { - type key string - type val string - - m := map[key]val{ - "job": "node", - "instance": "123.123.1.211:9090", - "path": "/api/v1/namespaces//deployments/", - "method": "GET", - "namespace": "system", - "status": "500", - } - - var sm map[string]string - - for i := 0; i < b.N; i++ { - sm = make(map[string]string, len(m)) - for k, v := range m { - sm[string(k)] = string(v) - } - } -} - -func BenchmarkListIter(b *testing.B) { - var list []uint32 - for i := 0; i < 1e4; i++ { - list = append(list, uint32(i)) - } - - b.ResetTimer() - - total := uint32(0) - - for j := 0; j < b.N; j++ { - sum := uint32(0) - for _, k := range list { - sum += k - } - total += sum - } -} diff --git a/tsdb/test/hash_test.go b/tsdb/test/hash_test.go deleted file mode 100644 index 1242f5db5d..0000000000 --- a/tsdb/test/hash_test.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2017 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "crypto/rand" - "fmt" - "hash/crc32" - "testing" - - "github.com/cespare/xxhash/v2" - sip13 "github.com/dgryski/go-sip13" -) - -type pair struct { - name, value string -} - -var testInput = []pair{ - {"job", "node"}, - {"instance", "123.123.1.211:9090"}, - {"path", "/api/v1/namespaces//deployments/"}, - {"method", "GET"}, - {"namespace", "system"}, - {"status", "500"}, -} - -func BenchmarkHash(b *testing.B) { - input := []byte{} - for _, v := range testInput { - input = append(input, v.name...) - input = append(input, '\xff') - input = append(input, v.value...) - input = append(input, '\xff') - } - - var total uint64 - - var k0 uint64 = 0x0706050403020100 - var k1 uint64 = 0x0f0e0d0c0b0a0908 - - for name, f := range map[string]func(b []byte) uint64{ - "xxhash": xxhash.Sum64, - "fnv64": fnv64a, - "sip13": func(b []byte) uint64 { return sip13.Sum64(k0, k1, b) }, - } { - b.Run(name, func(b *testing.B) { - b.SetBytes(int64(len(input))) - total = 0 - for i := 0; i < b.N; i++ { - total += f(input) - } - }) - } -} - -// hashAdd adds a string to a fnv64a hash value, returning the updated hash. -func fnv64a(b []byte) uint64 { - const ( - offset64 = 14695981039346656037 - prime64 = 1099511628211 - ) - - h := uint64(offset64) - for x := range b { - h ^= uint64(x) - h *= prime64 - } - return h -} - -func BenchmarkCRC32_diff(b *testing.B) { - data := [][]byte{} - - for i := 0; i < 1000; i++ { - b := make([]byte, 512) - rand.Read(b) - data = append(data, b) - } - - ctab := crc32.MakeTable(crc32.Castagnoli) - total := uint32(0) - - b.Run("direct", func(b *testing.B) { - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - total += crc32.Checksum(data[i%1000], ctab) - } - }) - b.Run("hash-reuse", func(b *testing.B) { - b.ReportAllocs() - h := crc32.New(ctab) - - for i := 0; i < b.N; i++ { - h.Reset() - h.Write(data[i%1000]) - total += h.Sum32() - } - }) - b.Run("hash-new", func(b *testing.B) { - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - h := crc32.New(ctab) - h.Write(data[i%1000]) - total += h.Sum32() - } - }) - - fmt.Println(total) -} diff --git a/tsdb/test/labels_test.go b/tsdb/test/labels_test.go deleted file mode 100644 index f1ec34f3a7..0000000000 --- a/tsdb/test/labels_test.go +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright 2017 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "bytes" - "crypto/rand" - "testing" - - "github.com/prometheus/prometheus/model/labels" -) - -func BenchmarkMapClone(b *testing.B) { - m := map[string]string{ - "job": "node", - "instance": "123.123.1.211:9090", - "path": "/api/v1/namespaces//deployments/", - "method": "GET", - "namespace": "system", - "status": "500", - "prometheus": "prometheus-core-1", - "datacenter": "eu-west-1", - "pod_name": "abcdef-99999-defee", - } - - for i := 0; i < b.N; i++ { - res := make(map[string]string, len(m)) - for k, v := range m { - res[k] = v - } - m = res - } -} - -func BenchmarkLabelsClone(b *testing.B) { - m := map[string]string{ - "job": "node", - "instance": "123.123.1.211:9090", - "path": "/api/v1/namespaces//deployments/", - "method": "GET", - "namespace": "system", - "status": "500", - "prometheus": "prometheus-core-1", - "datacenter": "eu-west-1", - "pod_name": "abcdef-99999-defee", - } - l := labels.FromMap(m) - - for i := 0; i < b.N; i++ { - l = l.Copy() - } -} - -func BenchmarkLabelMapAccess(b *testing.B) { - m := map[string]string{ - "job": "node", - "instance": "123.123.1.211:9090", - "path": "/api/v1/namespaces//deployments/", - "method": "GET", - "namespace": "system", - "status": "500", - "prometheus": "prometheus-core-1", - "datacenter": "eu-west-1", - "pod_name": "abcdef-99999-defee", - } - - var v string - - for k := range m { - b.Run(k, func(b *testing.B) { - for i := 0; i < b.N; i++ { - v = m[k] - } - }) - } - - _ = v -} - -func BenchmarkLabelSetAccess(b *testing.B) { - m := map[string]string{ - "job": "node", - "instance": "123.123.1.211:9090", - "path": "/api/v1/namespaces//deployments/", - "method": "GET", - "namespace": "system", - "status": "500", - "prometheus": "prometheus-core-1", - "datacenter": "eu-west-1", - "pod_name": "abcdef-99999-defee", - } - ls := labels.FromMap(m) - - var v string - - ls.Range(func(l labels.Label) { - b.Run(l.Name, func(b *testing.B) { - for i := 0; i < b.N; i++ { - v = ls.Get(l.Name) - } - }) - }) - - _ = v -} - -func BenchmarkStringBytesEquals(b *testing.B) { - randBytes := func(n int) ([]byte, []byte) { - buf1 := make([]byte, n) - if _, err := rand.Read(buf1); err != nil { - b.Fatal(err) - } - buf2 := make([]byte, n) - copy(buf1, buf2) - - return buf1, buf2 - } - - cases := []struct { - name string - f func() ([]byte, []byte) - }{ - { - name: "equal", - f: func() ([]byte, []byte) { - return randBytes(60) - }, - }, - { - name: "1-flip-end", - f: func() ([]byte, []byte) { - b1, b2 := randBytes(60) - b2[59] ^= b2[59] - return b1, b2 - }, - }, - { - name: "1-flip-middle", - f: func() ([]byte, []byte) { - b1, b2 := randBytes(60) - b2[29] ^= b2[29] - return b1, b2 - }, - }, - { - name: "1-flip-start", - f: func() ([]byte, []byte) { - b1, b2 := randBytes(60) - b2[0] ^= b2[0] - return b1, b2 - }, - }, - { - name: "different-length", - f: func() ([]byte, []byte) { - b1, b2 := randBytes(60) - return b1, b2[:59] - }, - }, - } - - for _, c := range cases { - b.Run(c.name+"-strings", func(b *testing.B) { - ab, bb := c.f() - as, bs := string(ab), string(bb) - b.SetBytes(int64(len(as))) - - var r bool - - for i := 0; i < b.N; i++ { - r = as == bs - } - _ = r - }) - - b.Run(c.name+"-bytes", func(b *testing.B) { - ab, bb := c.f() - b.SetBytes(int64(len(ab))) - - var r bool - - for i := 0; i < b.N; i++ { - r = bytes.Equal(ab, bb) - } - _ = r - }) - - b.Run(c.name+"-bytes-length-check", func(b *testing.B) { - ab, bb := c.f() - b.SetBytes(int64(len(ab))) - - var r bool - - for i := 0; i < b.N; i++ { - if len(ab) != len(bb) { - continue - } - r = bytes.Equal(ab, bb) - } - _ = r - }) - } -} From f03b8d0968f7a9ba403f5be6079c0c663a4784a1 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 10 Jan 2023 15:41:39 +0000 Subject: [PATCH 08/23] Add benchmark copying labels Taken from previous tsdb/test/BenchmarkLabelsClone. Signed-off-by: Bryan Boreham --- model/labels/labels_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/model/labels/labels_test.go b/model/labels/labels_test.go index 69beb2fc3e..1902f2f71d 100644 --- a/model/labels/labels_test.go +++ b/model/labels/labels_test.go @@ -696,6 +696,25 @@ func BenchmarkLabels_Hash(b *testing.B) { } } +func BenchmarkLabels_Copy(b *testing.B) { + m := map[string]string{ + "job": "node", + "instance": "123.123.1.211:9090", + "path": "/api/v1/namespaces//deployments/", + "method": "GET", + "namespace": "system", + "status": "500", + "prometheus": "prometheus-core-1", + "datacenter": "eu-west-1", + "pod_name": "abcdef-99999-defee", + } + l := FromMap(m) + + for i := 0; i < b.N; i++ { + l = l.Copy() + } +} + func TestMarshaling(t *testing.T) { lbls := FromStrings("aaa", "111", "bbb", "2222", "ccc", "33333") expectedJSON := "{\"aaa\":\"111\",\"bbb\":\"2222\",\"ccc\":\"33333\"}" From 4126985908ebb08a0e93021ef2476fd77731227e Mon Sep 17 00:00:00 2001 From: "Amr Hanafi (MAHDI))" Date: Fri, 24 Feb 2023 09:32:05 -0800 Subject: [PATCH 09/23] Update golang.org/x/net to v0.7.0 ato address GHSA-vvpx-j8f3-3w6h https://github.com/advisories/GHSA-vvpx-j8f3-3w6h Signed-off-by: Amr Hanafi (MAHDI)) --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 8df8696b6d..41bd49a9dd 100644 --- a/go.mod +++ b/go.mod @@ -60,10 +60,10 @@ require ( go.uber.org/atomic v1.10.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/goleak v1.2.0 - golang.org/x/net v0.5.0 + golang.org/x/net v0.7.0 golang.org/x/oauth2 v0.4.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.4.0 + golang.org/x/sys v0.5.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.5.0 google.golang.org/api v0.108.0 @@ -174,8 +174,8 @@ require ( golang.org/x/crypto v0.1.0 // indirect golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874 golang.org/x/mod v0.7.0 // indirect - golang.org/x/term v0.4.0 // indirect - golang.org/x/text v0.6.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.6 // indirect diff --git a/go.sum b/go.sum index 28342ab228..e8352cbf3f 100644 --- a/go.sum +++ b/go.sum @@ -917,8 +917,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1009,13 +1009,13 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1025,8 +1025,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 53bb046d21fd1101c9e91f3cc3a598117efac3a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:34:24 +0100 Subject: [PATCH 10/23] build(deps): bump github.com/aws/aws-sdk-go from 1.44.187 to 1.44.207 (#12015) Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.187 to 1.44.207. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.187...v1.44.207) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 41bd49a9dd..17c367bfe4 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Azure/go-autorest/autorest v0.11.28 github.com/Azure/go-autorest/autorest/adal v0.9.22 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 - github.com/aws/aws-sdk-go v1.44.187 + github.com/aws/aws-sdk-go v1.44.207 github.com/cespare/xxhash/v2 v2.2.0 github.com/dennwc/varint v1.0.0 github.com/digitalocean/godo v1.95.0 diff --git a/go.sum b/go.sum index e8352cbf3f..35baac090d 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,8 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.44.187 h1:D5CsRomPnlwDHJCanL2mtaLIcbhjiWxNh5j8zvaWdJA= -github.com/aws/aws-sdk-go v1.44.187/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.207 h1:7O0AMKxTm+/GUx6zw+3dqc+fD3tTzv8xaZPYo+ywRwE= +github.com/aws/aws-sdk-go v1.44.207/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= From 5ec1b4baaf01dae0b22567db53886c83b74e0e76 Mon Sep 17 00:00:00 2001 From: Renning Bruns Date: Tue, 28 Feb 2023 09:36:58 -0800 Subject: [PATCH 11/23] make hashmod a lot easier to read and a little faster (#11551) Previous code was effectively doing BigEndian.Uint64, so call that and save time. An md5.Sum result is always 16 bytes. The first 8 are not used in the result, just as before. Signed-off-by: Renning Bruns --- model/relabel/relabel.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/model/relabel/relabel.go b/model/relabel/relabel.go index 0cc6eeeb7e..eb05817c0c 100644 --- a/model/relabel/relabel.go +++ b/model/relabel/relabel.go @@ -15,6 +15,7 @@ package relabel import ( "crypto/md5" + "encoding/binary" "fmt" "strings" @@ -268,7 +269,9 @@ func relabel(lset labels.Labels, cfg *Config, lb *labels.Builder) (ret labels.La case Uppercase: lb.Set(cfg.TargetLabel, strings.ToUpper(val)) case HashMod: - mod := sum64(md5.Sum([]byte(val))) % cfg.Modulus + hash := md5.Sum([]byte(val)) + // Use only the last 8 bytes of the hash to give the same result as earlier versions of this code. + mod := binary.BigEndian.Uint64(hash[8:]) % cfg.Modulus lb.Set(cfg.TargetLabel, fmt.Sprintf("%d", mod)) case LabelMap: lset.Range(func(l labels.Label) { @@ -295,15 +298,3 @@ func relabel(lset labels.Labels, cfg *Config, lb *labels.Builder) (ret labels.La return lb.Labels(lset), true } - -// sum64 sums the md5 hash to an uint64. -func sum64(hash [md5.Size]byte) uint64 { - var s uint64 - - for i, b := range hash { - shift := uint64((md5.Size - i - 1) * 8) - - s |= uint64(b) << shift - } - return s -} From d33eb3ab17616a54b97d9f7791c791a79823f279 Mon Sep 17 00:00:00 2001 From: Rens Groothuijsen Date: Wed, 1 Mar 2023 13:21:02 +0100 Subject: [PATCH 12/23] Automatically remove incorrect snapshot with index that is ahead of WAL (#11859) Signed-off-by: Rens Groothuijsen Signed-off-by: Ganesh Vernekar Co-authored-by: Ganesh Vernekar --- tsdb/head.go | 49 +++++++++++++++++++++++++++++++---------- tsdb/head_test.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 11 deletions(-) diff --git a/tsdb/head.go b/tsdb/head.go index 833aa8c5f3..f539388f27 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -568,20 +568,47 @@ func (h *Head) Init(minValidTime int64) error { if h.opts.EnableMemorySnapshotOnShutdown { level.Info(h.logger).Log("msg", "Chunk snapshot is enabled, replaying from the snapshot") - var err error - snapIdx, snapOffset, refSeries, err = h.loadChunkSnapshot() - if err != nil { - snapIdx, snapOffset = -1, 0 - refSeries = make(map[chunks.HeadSeriesRef]*memSeries) + // If there are any WAL files, there should be at least one WAL file with an index that is current or newer + // than the snapshot index. If the WAL index is behind the snapshot index somehow, the snapshot is assumed + // to be outdated. + loadSnapshot := true + if h.wal != nil { + _, endAt, err := wlog.Segments(h.wal.Dir()) + if err != nil { + return errors.Wrap(err, "finding WAL segments") + } - h.metrics.snapshotReplayErrorTotal.Inc() - level.Error(h.logger).Log("msg", "Failed to load chunk snapshot", "err", err) - // We clear the partially loaded data to replay fresh from the WAL. - if err := h.resetInMemoryState(); err != nil { - return err + _, idx, _, err := LastChunkSnapshot(h.opts.ChunkDirRoot) + if err != nil && err != record.ErrNotFound { + level.Error(h.logger).Log("msg", "Could not find last snapshot", "err", err) + } + + if err == nil && endAt < idx { + loadSnapshot = false + level.Warn(h.logger).Log("msg", "Last WAL file is behind snapshot, removing snapshots") + if err := DeleteChunkSnapshots(h.opts.ChunkDirRoot, math.MaxInt, math.MaxInt); err != nil { + level.Error(h.logger).Log("msg", "Error while deleting snapshot directories", "err", err) + } + } + } + if loadSnapshot { + var err error + snapIdx, snapOffset, refSeries, err = h.loadChunkSnapshot() + if err == nil { + level.Info(h.logger).Log("msg", "Chunk snapshot loading time", "duration", time.Since(start).String()) + } + if err != nil { + snapIdx, snapOffset = -1, 0 + refSeries = make(map[chunks.HeadSeriesRef]*memSeries) + + h.metrics.snapshotReplayErrorTotal.Inc() + level.Error(h.logger).Log("msg", "Failed to load chunk snapshot", "err", err) + // We clear the partially loaded data to replay fresh from the WAL. + if err := h.resetInMemoryState(); err != nil { + return err + } } } - level.Info(h.logger).Log("msg", "Chunk snapshot loading time", "duration", time.Since(start).String()) } mmapChunkReplayStart := time.Now() diff --git a/tsdb/head_test.go b/tsdb/head_test.go index 4fe6c8f91b..b5afed64b3 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -4786,3 +4786,58 @@ func TestGaugeFloatHistogramWALAndChunkHeader(t *testing.T) { checkHeaders() } + +func TestSnapshotAheadOfWALError(t *testing.T) { + head, _ := newTestHead(t, 120*4, false, false) + head.opts.EnableMemorySnapshotOnShutdown = true + // Add a sample to fill WAL. + app := head.Appender(context.Background()) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), 10, 10) + require.NoError(t, err) + require.NoError(t, app.Commit()) + + // Increment snapshot index to create sufficiently large difference. + for i := 0; i < 2; i++ { + _, err = head.wal.NextSegment() + require.NoError(t, err) + } + require.NoError(t, head.Close()) // This will create a snapshot. + + _, idx, _, err := LastChunkSnapshot(head.opts.ChunkDirRoot) + require.NoError(t, err) + require.Equal(t, 2, idx) + + // Restart the WAL while keeping the old snapshot. The new head is created manually in this case in order + // to keep using the same snapshot directory instead of a random one. + require.NoError(t, os.RemoveAll(head.wal.Dir())) + head.opts.EnableMemorySnapshotOnShutdown = false + w, _ := wlog.NewSize(nil, nil, head.wal.Dir(), 32768, false) + head, err = NewHead(nil, nil, w, nil, head.opts, nil) + require.NoError(t, err) + // Add a sample to fill WAL. + app = head.Appender(context.Background()) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), 10, 10) + require.NoError(t, err) + require.NoError(t, app.Commit()) + lastSegment, _, _ := w.LastSegmentAndOffset() + require.Equal(t, 0, lastSegment) + require.NoError(t, head.Close()) + + // New WAL is saved, but old snapshot still exists. + _, idx, _, err = LastChunkSnapshot(head.opts.ChunkDirRoot) + require.NoError(t, err) + require.Equal(t, 2, idx) + + // Create new Head which should detect the incorrect index and delete the snapshot. + head.opts.EnableMemorySnapshotOnShutdown = true + w, _ = wlog.NewSize(nil, nil, head.wal.Dir(), 32768, false) + head, err = NewHead(nil, nil, w, nil, head.opts, nil) + require.NoError(t, err) + require.NoError(t, head.Init(math.MinInt64)) + + // Verify that snapshot directory does not exist anymore. + _, _, _, err = LastChunkSnapshot(head.opts.ChunkDirRoot) + require.Equal(t, record.ErrNotFound, err) + + require.NoError(t, head.Close()) +} From e1a6e6faa69985691b7ff9e86228394fd876c902 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 23:58:57 +0000 Subject: [PATCH 13/23] build(deps): bump go.uber.org/goleak from 1.2.0 to 1.2.1 Bumps [go.uber.org/goleak](https://github.com/uber-go/goleak) from 1.2.0 to 1.2.1. - [Release notes](https://github.com/uber-go/goleak/releases) - [Changelog](https://github.com/uber-go/goleak/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber-go/goleak/compare/v1.2.0...v1.2.1) --- updated-dependencies: - dependency-name: go.uber.org/goleak dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 17c367bfe4..1d6c3614f3 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( go.opentelemetry.io/otel/trace v1.11.2 go.uber.org/atomic v1.10.0 go.uber.org/automaxprocs v1.5.1 - go.uber.org/goleak v1.2.0 + go.uber.org/goleak v1.2.1 golang.org/x/net v0.7.0 golang.org/x/oauth2 v0.4.0 golang.org/x/sync v0.1.0 diff --git a/go.sum b/go.sum index 35baac090d..da45de3a9b 100644 --- a/go.sum +++ b/go.sum @@ -806,8 +806,8 @@ go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= @@ -855,7 +855,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= From 59cc7389fa8abffab119024049a5246caf341a5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:00:04 +0000 Subject: [PATCH 14/23] build(deps): bump github.com/stretchr/testify Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.1 to 1.8.2. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.1...v1.8.2) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- documentation/examples/remote_storage/go.mod | 2 +- documentation/examples/remote_storage/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/examples/remote_storage/go.mod b/documentation/examples/remote_storage/go.mod index 2d455916f1..51dd63b3ff 100644 --- a/documentation/examples/remote_storage/go.mod +++ b/documentation/examples/remote_storage/go.mod @@ -9,7 +9,7 @@ require ( github.com/influxdata/influxdb v1.11.0 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/common v0.37.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 gopkg.in/alecthomas/kingpin.v2 v2.2.6 ) diff --git a/documentation/examples/remote_storage/go.sum b/documentation/examples/remote_storage/go.sum index 3a1b54813b..dd4894f8ab 100644 --- a/documentation/examples/remote_storage/go.sum +++ b/documentation/examples/remote_storage/go.sum @@ -228,8 +228,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From c1c444504e4a58a7cc5861d5e9049649d4882a9f Mon Sep 17 00:00:00 2001 From: ansalamdaniel <98307038+ansalamdaniel@users.noreply.github.com> Date: Thu, 2 Mar 2023 15:25:56 +0530 Subject: [PATCH 15/23] Feat: metrics for head_chunks & wal folders (#12013) Signed-off-by: ansalamdaniel --- tsdb/head.go | 11 +++++++++++ tsdb/wlog/wlog.go | 17 +++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tsdb/head.go b/tsdb/head.go index f539388f27..cf2c15d79b 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -502,6 +502,17 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { }, func() float64 { return float64(h.iso.lastAppendID()) }), + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "prometheus_tsdb_head_chunks_storage_size_bytes", + Help: "Size of the chunks_head directory.", + }, func() float64 { + val, err := h.chunkDiskMapper.Size() + if err != nil { + level.Error(h.logger).Log("msg", "Failed to calculate size of \"chunks_head\" dir", + "err", err.Error()) + } + return float64(val) + }), ) } return m diff --git a/tsdb/wlog/wlog.go b/tsdb/wlog/wlog.go index 5ae308d4ea..df8bab53ff 100644 --- a/tsdb/wlog/wlog.go +++ b/tsdb/wlog/wlog.go @@ -199,9 +199,10 @@ type wlMetrics struct { truncateTotal prometheus.Counter currentSegment prometheus.Gauge writesFailed prometheus.Counter + walFileSize prometheus.GaugeFunc } -func newWLMetrics(r prometheus.Registerer) *wlMetrics { +func newWLMetrics(w *WL, r prometheus.Registerer) *wlMetrics { m := &wlMetrics{} m.fsyncDuration = prometheus.NewSummary(prometheus.SummaryOpts{ @@ -233,6 +234,17 @@ func newWLMetrics(r prometheus.Registerer) *wlMetrics { Name: "writes_failed_total", Help: "Total number of write log writes that failed.", }) + m.walFileSize = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "storage_size_bytes", + Help: "Size of the write log directory.", + }, func() float64 { + val, err := w.Size() + if err != nil { + level.Error(w.logger).Log("msg", "Failed to calculate size of \"wal\" dir", + "err", err.Error()) + } + return float64(val) + }) if r != nil { r.MustRegister( @@ -243,6 +255,7 @@ func newWLMetrics(r prometheus.Registerer) *wlMetrics { m.truncateTotal, m.currentSegment, m.writesFailed, + m.walFileSize, ) } @@ -279,7 +292,7 @@ func NewSize(logger log.Logger, reg prometheus.Registerer, dir string, segmentSi if filepath.Base(dir) == WblDirName { prefix = "prometheus_tsdb_out_of_order_wbl_" } - w.metrics = newWLMetrics(prometheus.WrapRegistererWithPrefix(prefix, reg)) + w.metrics = newWLMetrics(w, prometheus.WrapRegistererWithPrefix(prefix, reg)) _, last, err := Segments(w.Dir()) if err != nil { From 8e4350dd59e3f8575c7f5296db73721c7789a81c Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Mon, 27 Feb 2023 21:37:30 +0100 Subject: [PATCH 16/23] Directly include SVG logo in the page. Use HTML element to include Prometheus logo in the status page. Signed-off-by: Julien Pivotto --- web/ui/react-app/src/App.tsx | 78 ++++++++++--------- web/ui/react-app/src/Navbar.tsx | 7 +- .../src/contexts/AnimateLogoContext.tsx | 3 + web/ui/react-app/src/pages/status/Status.tsx | 43 +++++++++- web/ui/react-app/src/themes/_shared.scss | 20 ++++- 5 files changed, 108 insertions(+), 43 deletions(-) create mode 100644 web/ui/react-app/src/contexts/AnimateLogoContext.tsx diff --git a/web/ui/react-app/src/App.tsx b/web/ui/react-app/src/App.tsx index 30b07f2613..31cf72ab00 100755 --- a/web/ui/react-app/src/App.tsx +++ b/web/ui/react-app/src/App.tsx @@ -6,6 +6,7 @@ import { BrowserRouter as Router, Redirect, Route, Switch } from 'react-router-d import { PathPrefixContext } from './contexts/PathPrefixContext'; import { ThemeContext, themeName, themeSetting } from './contexts/ThemeContext'; import { ReadyContext } from './contexts/ReadyContext'; +import { AnimateLogoContext } from './contexts/AnimateLogoContext'; import { useLocalStorage } from './hooks/useLocalStorage'; import useMedia from './hooks/useMedia'; import { @@ -60,6 +61,7 @@ const App: FC = ({ consolesLink, agentMode, ready }) => { const [userTheme, setUserTheme] = useLocalStorage(themeLocalStorageKey, 'auto'); const browserHasThemes = useMedia('(prefers-color-scheme)'); const browserWantsDarkTheme = useMedia('(prefers-color-scheme: dark)'); + const [animateLogo, setAnimateLogo] = useLocalStorage('animateLogo', false); let theme: themeName; if (userTheme !== 'auto') { @@ -76,46 +78,48 @@ const App: FC = ({ consolesLink, agentMode, ready }) => { - - - - - {/* + + + + + + {/* NOTE: Any route added here needs to also be added to the list of React-handled router paths ("reactRouterPaths") in /web/web.go. */} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/web/ui/react-app/src/Navbar.tsx b/web/ui/react-app/src/Navbar.tsx index 10ffe5f0d9..a58f6914fe 100644 --- a/web/ui/react-app/src/Navbar.tsx +++ b/web/ui/react-app/src/Navbar.tsx @@ -13,21 +13,22 @@ import { DropdownItem, } from 'reactstrap'; import { ThemeToggle } from './Theme'; -import logo from './images/prometheus_logo_grey.svg'; +import { ReactComponent as PromLogo } from './images/prometheus_logo_grey.svg'; interface NavbarProps { consolesLink: string | null; agentMode: boolean; + animateLogo?: boolean | false; } -const Navigation: FC = ({ consolesLink, agentMode }) => { +const Navigation: FC = ({ consolesLink, agentMode, animateLogo }) => { const [isOpen, setIsOpen] = useState(false); const toggle = () => setIsOpen(!isOpen); return ( - Prometheus logo + Prometheus{agentMode && ' Agent'} diff --git a/web/ui/react-app/src/contexts/AnimateLogoContext.tsx b/web/ui/react-app/src/contexts/AnimateLogoContext.tsx new file mode 100644 index 0000000000..4834a60a88 --- /dev/null +++ b/web/ui/react-app/src/contexts/AnimateLogoContext.tsx @@ -0,0 +1,3 @@ +import { createContext } from 'react'; + +export const AnimateLogoContext = createContext(false); diff --git a/web/ui/react-app/src/pages/status/Status.tsx b/web/ui/react-app/src/pages/status/Status.tsx index 0d37db9619..374c672dd7 100644 --- a/web/ui/react-app/src/pages/status/Status.tsx +++ b/web/ui/react-app/src/pages/status/Status.tsx @@ -1,4 +1,4 @@ -import React, { Fragment, FC } from 'react'; +import React, { Fragment, FC, useState, useEffect } from 'react'; import { Table } from 'reactstrap'; import { withStatusIndicator } from '../../components/withStatusIndicator'; import { useFetch } from '../../hooks/useFetch'; @@ -97,7 +97,46 @@ const StatusResult: FC<{ fetchPath: string; title: string }> = ({ fetchPath, tit ); }; -const Status: FC<{ agentMode: boolean }> = ({ agentMode }) => { +interface StatusProps { + agentMode?: boolean | false; + setAnimateLogo?: (animateLogo: boolean) => void; +} + +const Status: FC = ({ agentMode, setAnimateLogo }) => { + /* _ + * /' \ + * | | + * \__/ */ + + const [inputText, setInputText] = useState(''); + + useEffect(() => { + const handleKeyPress = (event: KeyboardEvent) => { + const keyPressed = event.key.toUpperCase(); + setInputText((prevInputText) => { + const newInputText = prevInputText.slice(-3) + String.fromCharCode(((keyPressed.charCodeAt(0) - 64) % 26) + 65); + return newInputText; + }); + }; + + document.addEventListener('keypress', handleKeyPress); + + return () => { + document.removeEventListener('keypress', handleKeyPress); + }; + }, []); + + useEffect(() => { + if (setAnimateLogo && inputText != '') { + setAnimateLogo(inputText.toUpperCase() === 'QSPN'); + } + }, [inputText]); + + /* _ + * /' \ + * | | + * \__/ */ + const pathPrefix = usePathPrefix(); const path = `${pathPrefix}/${API_PATH}`; diff --git a/web/ui/react-app/src/themes/_shared.scss b/web/ui/react-app/src/themes/_shared.scss index 20658a8afd..8745a6d775 100644 --- a/web/ui/react-app/src/themes/_shared.scss +++ b/web/ui/react-app/src/themes/_shared.scss @@ -104,9 +104,14 @@ button.execute-btn { margin-bottom: 20px; } -.navbar-brand img { +.navbar-brand svg { padding-right: 1rem; height: 1.9rem; + width: 2.9rem; +} + +.navbar-brand svg.animate path { + animation: flamecolor 4s ease-in 1 forwards,flame 1s ease-in infinite; } .navbar-brand { @@ -357,3 +362,16 @@ input[type='checkbox']:checked + label { display: block; font-family: monospace; } + +@keyframes flamecolor { + 100% { + fill: #e95224; + } +} +@keyframes flame { + 0% { d: path("M 56.667,0.667 C 25.372,0.667 0,26.036 0,57.332 c 0,31.295 25.372,56.666 56.667,56.666 31.295,0 56.666,-25.371 56.666,-56.666 0,-31.296 -25.372,-56.665 -56.666,-56.665 z m 0,106.055 c -8.904,0 -16.123,-5.948 -16.123,-13.283 H 72.79 c 0,7.334 -7.219,13.283 -16.123,13.283 z M 83.297,89.04 H 30.034 V 79.382 H 83.298 V 89.04 Z M 83.106,74.411 H 30.186 C 30.01,74.208 29.83,74.008 29.66,73.802 24.208,67.182 22.924,63.726 21.677,60.204 c -0.021,-0.116 6.611,1.355 11.314,2.413 0,0 2.42,0.56 5.958,1.205 -3.397,-3.982 -5.414,-9.044 -5.414,-14.218 0,-11.359 8.712,-21.285 5.569,-29.308 3.059,0.249 6.331,6.456 6.552,16.161 3.252,-4.494 4.613,-12.701 4.613,-17.733 0,-5.21 3.433,-11.262 6.867,-11.469 -3.061,5.045 0.793,9.37 4.219,20.099 1.285,4.03 1.121,10.812 2.113,15.113 C 63.797,33.534 65.333,20.5 71,16 c -2.5,5.667 0.37,12.758 2.333,16.167 3.167,5.5 5.087,9.667 5.087,17.548 0,5.284 -1.951,10.259 -5.242,14.148 3.742,-0.702 6.326,-1.335 6.326,-1.335 l 12.152,-2.371 c 10e-4,-10e-4 -1.765,7.261 -8.55,14.254 z"); + } + 50% { d: path("M 56.667,0.667 C 25.372,0.667 0,26.036 0,57.332 c 0,31.295 25.372,56.666 56.667,56.666 31.295,0 56.666,-25.371 56.666,-56.666 0,-31.296 -25.372,-56.665 -56.666,-56.665 z m 0,106.055 c -8.904,0 -16.123,-5.948 -16.123,-13.283 H 72.79 c 0,7.334 -7.219,13.283 -16.123,13.283 z M 83.297,89.04 H 30.034 V 79.382 H 83.298 V 89.04 Z M 83.106,74.411 H 30.186 C 30.01,74.208 29.83,74.008 29.66,73.802 24.208,67.182 22.924,63.726 21.677,60.204 c -0.021,-0.116 6.611,1.355 11.314,2.413 0,0 2.42,0.56 5.958,1.205 -3.397,-3.982 -5.414,-9.044 -5.414,-14.218 0,-11.359 1.640181,-23.047128 7.294982,-29.291475 C 39.391377,29.509803 45.435,26.752 45.656,36.457 c 3.252,-4.494 7.100362,-8.366957 7.100362,-13.398957 0,-5.21 0.137393,-8.650513 -3.479689,-15.0672265 7.834063,1.6180944 8.448052,4.2381285 11.874052,14.9671285 1.285,4.03 1.325275,15.208055 2.317275,19.509055 0.329,-8.933 6.441001,-14.01461 5.163951,-21.391003 5.755224,5.771457 4.934508,10.495521 7.126537,14.288218 3.167,5.5 2.382625,7.496239 2.382625,15.377239 0,5.284 -1.672113,9.232546 -4.963113,13.121546 3.742,-0.702 6.326,-1.335 6.326,-1.335 l 12.152,-2.371 c 10e-4,-10e-4 -1.765,7.261 -8.55,14.254 z"); } + 100% { + d: path("M 56.667,0.667 C 25.372,0.667 0,26.036 0,57.332 c 0,31.295 25.372,56.666 56.667,56.666 31.295,0 56.666,-25.371 56.666,-56.666 0,-31.296 -25.372,-56.665 -56.666,-56.665 z m 0,106.055 c -8.904,0 -16.123,-5.948 -16.123,-13.283 H 72.79 c 0,7.334 -7.219,13.283 -16.123,13.283 z M 83.297,89.04 H 30.034 V 79.382 H 83.298 V 89.04 Z M 83.106,74.411 H 30.186 C 30.01,74.208 29.83,74.008 29.66,73.802 24.208,67.182 22.924,63.726 21.677,60.204 c -0.021,-0.116 6.611,1.355 11.314,2.413 0,0 2.42,0.56 5.958,1.205 -3.397,-3.982 -5.414,-9.044 -5.414,-14.218 0,-11.359 8.712,-21.285 5.569,-29.308 3.059,0.249 6.331,6.456 6.552,16.161 3.252,-4.494 4.613,-12.701 4.613,-17.733 0,-5.21 3.433,-11.262 6.867,-11.469 -3.061,5.045 0.793,9.37 4.219,20.099 1.285,4.03 1.121,10.812 2.113,15.113 C 63.797,33.534 65.333,20.5 71,16 c -2.5,5.667 0.37,12.758 2.333,16.167 3.167,5.5 5.087,9.667 5.087,17.548 0,5.284 -1.951,10.259 -5.242,14.148 3.742,-0.702 6.326,-1.335 6.326,-1.335 l 12.152,-2.371 c 10e-4,-10e-4 -1.765,7.261 -8.55,14.254 z"); } +} From 9225d118dc2c839a4858bf3c6f6a6f4041d93ea5 Mon Sep 17 00:00:00 2001 From: Jonatan Ivanov Date: Thu, 16 Feb 2023 19:25:54 -0800 Subject: [PATCH 17/23] Add Exemplars support for all time series This change removes restrictions to allow adding exemplars to all time series. It also contains some improvements in test values so that it is easier to track what is tested. The advantage of doing this is having a little less error-prone tests: "yy" is not really descriptive but "counter-test" can give people a better idea about what is tested so it is harder to make mistakes. Closes gh-11982 Signed-off-by: Jonatan Ivanov --- model/textparse/openmetricsparse.go | 18 ---------- model/textparse/openmetricsparse_test.go | 44 ++++++++++++++++++------ 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go index 0fe8010876..c17d40020a 100644 --- a/model/textparse/openmetricsparse.go +++ b/model/textparse/openmetricsparse.go @@ -17,7 +17,6 @@ package textparse import ( - "bytes" "errors" "fmt" "io" @@ -31,8 +30,6 @@ import ( "github.com/prometheus/prometheus/model/value" ) -var allowedSuffixes = [][]byte{[]byte("_total"), []byte("_bucket")} - type openMetricsLexer struct { b []byte i int @@ -364,12 +361,6 @@ func (p *OpenMetricsParser) Next() (Entry, error) { } func (p *OpenMetricsParser) parseComment() error { - // Validate the name of the metric. It must have _total or _bucket as - // suffix for exemplars to be supported. - if err := p.validateNameForExemplar(p.series[:p.offsets[0]-p.start]); err != nil { - return err - } - var err error // Parse the labels. p.eOffsets, err = p.parseLVals(p.eOffsets) @@ -474,12 +465,3 @@ func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error } return val, nil } - -func (p *OpenMetricsParser) validateNameForExemplar(name []byte) error { - for _, suffix := range allowedSuffixes { - if bytes.HasSuffix(name, suffix) { - return nil - } - } - return fmt.Errorf("metric name %v does not support exemplars", string(name)) -} diff --git a/model/textparse/openmetricsparse_test.go b/model/textparse/openmetricsparse_test.go index 12fb03f015..d65e4977ef 100644 --- a/model/textparse/openmetricsparse_test.go +++ b/model/textparse/openmetricsparse_test.go @@ -45,9 +45,14 @@ hh_bucket{le="+Inf"} 1 # TYPE gh gaugehistogram gh_bucket{le="+Inf"} 1 # TYPE hhh histogram -hhh_bucket{le="+Inf"} 1 # {aa="bb"} 4 +hhh_bucket{le="+Inf"} 1 # {id="histogram-bucket-test"} 4 +hhh_count 1 # {id="histogram-count-test"} 4 # TYPE ggh gaugehistogram -ggh_bucket{le="+Inf"} 1 # {cc="dd",xx="yy"} 4 123.123 +ggh_bucket{le="+Inf"} 1 # {id="gaugehistogram-bucket-test",xx="yy"} 4 123.123 +ggh_count 1 # {id="gaugehistogram-count-test",xx="yy"} 4 123.123 +# TYPE smr_seconds summary +smr_seconds_count 2.0 # {id="summary-count-test"} 1 123.321 +smr_seconds_sum 42.0 # {id="summary-sum-test"} 1 123.321 # TYPE ii info ii{foo="bar"} 1 # TYPE ss stateset @@ -59,7 +64,7 @@ _metric_starting_with_underscore 1 testmetric{_label_starting_with_underscore="foo"} 1 testmetric{label="\"bar\""} 1 # TYPE foo counter -foo_total 17.0 1520879607.789 # {xx="yy"} 5` +foo_total 17.0 1520879607.789 # {id="counter-test"} 5` input += "\n# HELP metric foo\x00bar" input += "\nnull_byte_metric{a=\"abc\x00\"} 1" @@ -152,7 +157,12 @@ foo_total 17.0 1520879607.789 # {xx="yy"} 5` m: `hhh_bucket{le="+Inf"}`, v: 1, lset: labels.FromStrings("__name__", "hhh_bucket", "le", "+Inf"), - e: &exemplar.Exemplar{Labels: labels.FromStrings("aa", "bb"), Value: 4}, + e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "histogram-bucket-test"), Value: 4}, + }, { + m: `hhh_count`, + v: 1, + lset: labels.FromStrings("__name__", "hhh_count"), + e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "histogram-count-test"), Value: 4}, }, { m: "ggh", typ: MetricTypeGaugeHistogram, @@ -160,7 +170,25 @@ foo_total 17.0 1520879607.789 # {xx="yy"} 5` m: `ggh_bucket{le="+Inf"}`, v: 1, lset: labels.FromStrings("__name__", "ggh_bucket", "le", "+Inf"), - e: &exemplar.Exemplar{Labels: labels.FromStrings("cc", "dd", "xx", "yy"), Value: 4, HasTs: true, Ts: 123123}, + e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "gaugehistogram-bucket-test", "xx", "yy"), Value: 4, HasTs: true, Ts: 123123}, + }, { + m: `ggh_count`, + v: 1, + lset: labels.FromStrings("__name__", "ggh_count"), + e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "gaugehistogram-count-test", "xx", "yy"), Value: 4, HasTs: true, Ts: 123123}, + }, { + m: "smr_seconds", + typ: MetricTypeSummary, + }, { + m: `smr_seconds_count`, + v: 2, + lset: labels.FromStrings("__name__", "smr_seconds_count"), + e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "summary-count-test"), Value: 1, HasTs: true, Ts: 123321}, + }, { + m: `smr_seconds_sum`, + v: 42, + lset: labels.FromStrings("__name__", "smr_seconds_sum"), + e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "summary-sum-test"), Value: 1, HasTs: true, Ts: 123321}, }, { m: "ii", typ: MetricTypeInfo, @@ -206,7 +234,7 @@ foo_total 17.0 1520879607.789 # {xx="yy"} 5` v: 17, lset: labels.FromStrings("__name__", "foo_total"), t: int64p(1520879607789), - e: &exemplar.Exemplar{Labels: labels.FromStrings("xx", "yy"), Value: 5}, + e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5}, }, { m: "metric", help: "foo\x00bar", @@ -495,10 +523,6 @@ func TestOpenMetricsParseErrors(t *testing.T) { input: `custom_metric_total 1 # {aa="bb"}`, err: "expected value after exemplar labels, got \"}\" (\"EOF\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"}\"", }, - { - input: `custom_metric 1 # {aa="bb"}`, - err: "metric name custom_metric does not support exemplars", - }, { input: `custom_metric_total 1 # {aa="bb",,cc="dd"} 1`, err: "expected label name, got \",c\" (\"COMMA\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\",,c\"", From adac7bcd1eb65e6c311663f06907dc50a8b828cf Mon Sep 17 00:00:00 2001 From: Dup4 Date: Sun, 5 Mar 2023 22:54:54 +0800 Subject: [PATCH 18/23] chore: fix ignore file_name Signed-off-by: Dup4 --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a63f499515..ed269dbf17 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,7 @@ benchmark.txt !/.promu.yml !/.golangci.yml /documentation/examples/remote_storage/remote_storage_adapter/remote_storage_adapter -/documentation/examples/remote_storage/example_write_adapter/example_writer_adapter +/documentation/examples/remote_storage/example_write_adapter/example_write_adapter npm_licenses.tar.bz2 /web/ui/static/react From ba1918b6d3ba00e952b77bd274e6e85751c33153 Mon Sep 17 00:00:00 2001 From: Harold Dost Date: Sat, 4 Mar 2023 16:51:39 +0100 Subject: [PATCH 19/23] promql: Add a Makefile target for goyacc * Add documentation for goyacc as relevant * Ignore the y.output since it's not needed for build process. Signed-off-by: Harold Dost --- .gitignore | 3 +++ CONTRIBUTING.md | 17 +++++++++++++++++ Makefile | 11 +++++++++++ 3 files changed, 31 insertions(+) diff --git a/.gitignore b/.gitignore index a63f499515..6ccfb146d5 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ npm_licenses.tar.bz2 /.build /**/node_modules + +# Ignore parser debug +y.output diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 16fc40712f..2fbed38809 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,3 +78,20 @@ GO111MODULE=on go mod tidy ``` You have to commit the changes to `go.mod` and `go.sum` before submitting the pull request. + +## Working with the PromQL parser + +The PromQL parser grammar is located in `promql/parser/generated_parser.y` and it can be built using `make parser`. +The parser is built using [goyacc](https://pkg.go.dev/golang.org/x/tools/cmd/goyacc) + +If doing some sort of debugging, then it is possible to add some verbose output. After generating the parser, then you +can modify the the `./promql/parser/generated_parser.y.go` manually. + +```golang +// As of writing this was somewhere around line 600. +var ( + yyDebug = 0 // This can be be a number 0 -> 5. + yyErrorVerbose = false // This can be set to true. +) + +``` diff --git a/Makefile b/Makefile index dd51d5817a..e345c1a886 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,17 @@ assets-tarball: assets @echo '>> packaging assets' scripts/package_assets.sh +# We only want to generate the parser when there's changes to the grammar. +.PHONY: parser +parser: + @echo ">> running goyacc to generate the .go file." +ifeq (, $(shell which goyacc)) + @echo "goyacc not installed so skipping" + @echo "To install: go install golang.org/x/tools/cmd/goyacc@v0.6.0" +else + goyacc -o promql/parser/generated_parser.y.go promql/parser/generated_parser.y +endif + .PHONY: test # If we only want to only test go code we have to change the test target # which is called by all. From 30297f0d9bd4a7c2179191c9896989161fb1aa5c Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 6 Mar 2023 16:29:21 +0000 Subject: [PATCH 20/23] stringlabels: size buffer for added labels This makes the buffer the correct size for the common case that labels have only been added. It will be too large for the case that labels are changed, but the current buffer resize logic in `appendLabelTo` doubles the buffer, so a small over-estimate is better. Signed-off-by: Bryan Boreham --- model/labels/labels_string.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/model/labels/labels_string.go b/model/labels/labels_string.go index e912882fe3..57d8236a5a 100644 --- a/model/labels/labels_string.go +++ b/model/labels/labels_string.go @@ -599,7 +599,8 @@ func (b *Builder) Labels(res Labels) Labels { sort.Strings(b.del) a, d := 0, 0 - buf := make([]byte, 0, len(b.base.data)) // TODO: see if we can re-use the buffer from res. + bufSize := len(b.base.data) + labelsSize(b.add) + buf := make([]byte, 0, bufSize) // TODO: see if we can re-use the buffer from res. for pos := 0; pos < len(b.base.data); { oldPos := pos var lName string From a07a0be024ade806b7a8fbcf58a241a307f1ab8b Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 6 Mar 2023 18:17:31 +0000 Subject: [PATCH 21/23] Add benchmark for labels.Builder Signed-off-by: Bryan Boreham --- model/labels/labels_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/model/labels/labels_test.go b/model/labels/labels_test.go index 1902f2f71d..4832be3375 100644 --- a/model/labels/labels_test.go +++ b/model/labels/labels_test.go @@ -696,6 +696,31 @@ func BenchmarkLabels_Hash(b *testing.B) { } } +func BenchmarkBuilder(b *testing.B) { + m := []Label{ + {"job", "node"}, + {"instance", "123.123.1.211:9090"}, + {"path", "/api/v1/namespaces//deployments/"}, + {"method", "GET"}, + {"namespace", "system"}, + {"status", "500"}, + {"prometheus", "prometheus-core-1"}, + {"datacenter", "eu-west-1"}, + {"pod_name", "abcdef-99999-defee"}, + } + + var l Labels + builder := NewBuilder(EmptyLabels()) + for i := 0; i < b.N; i++ { + builder.Reset(EmptyLabels()) + for _, l := range m { + builder.Set(l.Name, l.Value) + } + l = builder.Labels(EmptyLabels()) + } + require.Equal(b, 9, l.Len()) +} + func BenchmarkLabels_Copy(b *testing.B) { m := map[string]string{ "job": "node", From 38c6d3da9ffb1dbff19edb6ea161b5f30a9b5e8c Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 6 Mar 2023 18:22:49 +0000 Subject: [PATCH 22/23] labels: use slices.Sort for better performance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The difference is modest, but we've used `slices.Sort` in lots of other places so why not here. name old time/op new time/op delta Builder 1.04µs ± 3% 0.95µs ± 3% -8.27% (p=0.008 n=5+5) name old alloc/op new alloc/op delta Builder 312B ± 0% 288B ± 0% -7.69% (p=0.008 n=5+5) name old allocs/op new allocs/op delta Builder 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.008 n=5+5) Signed-off-by: Bryan Boreham --- model/labels/labels.go | 10 +++++----- model/labels/labels_string.go | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/model/labels/labels.go b/model/labels/labels.go index 18b8f8dc62..056bc63740 100644 --- a/model/labels/labels.go +++ b/model/labels/labels.go @@ -18,11 +18,11 @@ package labels import ( "bytes" "encoding/json" - "sort" "strconv" "github.com/cespare/xxhash/v2" "github.com/prometheus/common/model" + "golang.org/x/exp/slices" ) // Well-known label names used by Prometheus components. @@ -360,7 +360,7 @@ func EmptyLabels() Labels { func New(ls ...Label) Labels { set := make(Labels, 0, len(ls)) set = append(set, ls...) - sort.Sort(set) + slices.SortFunc(set, func(a, b Label) bool { return a.Name < b.Name }) return set } @@ -384,7 +384,7 @@ func FromStrings(ss ...string) Labels { res = append(res, Label{Name: ss[i], Value: ss[i+1]}) } - sort.Sort(res) + slices.SortFunc(res, func(a, b Label) bool { return a.Name < b.Name }) return res } @@ -564,7 +564,7 @@ Outer: } if len(b.add) > 0 { // Base is already in order, so we only need to sort if we add to it. res = append(res, b.add...) - sort.Sort(res) + slices.SortFunc(res, func(a, b Label) bool { return a.Name < b.Name }) } return res } @@ -591,7 +591,7 @@ func (b *ScratchBuilder) Add(name, value string) { // Sort the labels added so far by name. func (b *ScratchBuilder) Sort() { - sort.Sort(b.add) + slices.SortFunc(b.add, func(a, b Label) bool { return a.Name < b.Name }) } // Asssign is for when you already have a Labels which you want this ScratchBuilder to return. diff --git a/model/labels/labels_string.go b/model/labels/labels_string.go index e912882fe3..ccba604ee2 100644 --- a/model/labels/labels_string.go +++ b/model/labels/labels_string.go @@ -19,12 +19,12 @@ import ( "bytes" "encoding/json" "reflect" - "sort" "strconv" "unsafe" "github.com/cespare/xxhash/v2" "github.com/prometheus/common/model" + "golang.org/x/exp/slices" ) // Well-known label names used by Prometheus components. @@ -385,7 +385,7 @@ func yoloBytes(s string) (b []byte) { // New returns a sorted Labels from the given labels. // The caller has to guarantee that all label names are unique. func New(ls ...Label) Labels { - sort.Sort(labelSlice(ls)) + slices.SortFunc(ls, func(a, b Label) bool { return a.Name < b.Name }) size := labelsSize(ls) buf := make([]byte, size) marshalLabelsToSizedBuffer(ls, buf) @@ -411,7 +411,7 @@ func FromStrings(ss ...string) Labels { ls = append(ls, Label{Name: ss[i], Value: ss[i+1]}) } - sort.Sort(labelSlice(ls)) + slices.SortFunc(ls, func(a, b Label) bool { return a.Name < b.Name }) return New(ls...) } @@ -595,8 +595,8 @@ func (b *Builder) Labels(res Labels) Labels { return b.base } - sort.Sort(labelSlice(b.add)) - sort.Strings(b.del) + slices.SortFunc(b.add, func(a, b Label) bool { return a.Name < b.Name }) + slices.Sort(b.del) a, d := 0, 0 buf := make([]byte, 0, len(b.base.data)) // TODO: see if we can re-use the buffer from res. @@ -753,7 +753,7 @@ func (b *ScratchBuilder) Add(name, value string) { // Sort the labels added so far by name. func (b *ScratchBuilder) Sort() { - sort.Sort(labelSlice(b.add)) + slices.SortFunc(b.add, func(a, b Label) bool { return a.Name < b.Name }) } // Asssign is for when you already have a Labels which you want this ScratchBuilder to return. From c9b85afd93d03df31f1b320353521ee7ac9e51d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90urica=20Yuri=20Nikoli=C4=87?= Date: Tue, 7 Mar 2023 17:41:33 +0100 Subject: [PATCH 23/23] Making the number of CPUs used for WAL replay configurable (#12066) Adds `WALReplayConcurrency` as an option on tsdb `Options` and `HeadOptions`. If it is not set or set <=0, then `GOMAXPROCS` is used, which matches the previous behaviour. Signed-off-by: Yuri Nikolic --- tsdb/db.go | 7 +++++ tsdb/head.go | 13 +++++++++ tsdb/head_wal.go | 73 ++++++++++++++++++++++++------------------------ 3 files changed, 56 insertions(+), 37 deletions(-) diff --git a/tsdb/db.go b/tsdb/db.go index 616213b031..561867025b 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -125,6 +125,10 @@ type Options struct { // WALCompression will turn on Snappy compression for records on the WAL. WALCompression bool + // Maximum number of CPUs that can simultaneously processes WAL replay. + // If it is <=0, then GOMAXPROCS is used. + WALReplayConcurrency int + // StripeSize is the size in entries of the series hash map. Reducing the size will save memory but impact performance. StripeSize int @@ -782,6 +786,9 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs headOpts.EnableNativeHistograms.Store(opts.EnableNativeHistograms) headOpts.OutOfOrderTimeWindow.Store(opts.OutOfOrderTimeWindow) headOpts.OutOfOrderCapMax.Store(opts.OutOfOrderCapMax) + if opts.WALReplayConcurrency > 0 { + headOpts.WALReplayConcurrency = opts.WALReplayConcurrency + } if opts.IsolationDisabled { // We only override this flag if isolation is disabled at DB level. We use the default otherwise. headOpts.IsolationDisabled = opts.IsolationDisabled diff --git a/tsdb/head.go b/tsdb/head.go index cf2c15d79b..ef176d1c52 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -18,6 +18,7 @@ import ( "io" "math" "path/filepath" + "runtime" "sync" "time" @@ -58,6 +59,8 @@ var ( // defaultIsolationDisabled is true if isolation is disabled by default. defaultIsolationDisabled = false + + defaultWALReplayConcurrency = runtime.GOMAXPROCS(0) ) // Head handles reads and writes of time series data within a time window. @@ -155,6 +158,11 @@ type HeadOptions struct { EnableMemorySnapshotOnShutdown bool IsolationDisabled bool + + // Maximum number of CPUs that can simultaneously processes WAL replay. + // The default value is GOMAXPROCS. + // If it is set to a negative value or zero, the default value is used. + WALReplayConcurrency int } const ( @@ -172,6 +180,7 @@ func DefaultHeadOptions() *HeadOptions { StripeSize: DefaultStripeSize, SeriesCallback: &noopSeriesLifecycleCallback{}, IsolationDisabled: defaultIsolationDisabled, + WALReplayConcurrency: defaultWALReplayConcurrency, } ho.OutOfOrderCapMax.Store(DefaultOutOfOrderCapMax) return ho @@ -247,6 +256,10 @@ func NewHead(r prometheus.Registerer, l log.Logger, wal, wbl *wlog.WL, opts *Hea opts.ChunkPool = chunkenc.NewPool() } + if opts.WALReplayConcurrency <= 0 { + opts.WALReplayConcurrency = defaultWALReplayConcurrency + } + h.chunkDiskMapper, err = chunks.NewChunkDiskMapper( r, mmappedChunksDir(opts.ChunkDirRoot), diff --git a/tsdb/head_wal.go b/tsdb/head_wal.go index f2ef4c1e08..dd55f438d8 100644 --- a/tsdb/head_wal.go +++ b/tsdb/head_wal.go @@ -18,7 +18,6 @@ import ( "math" "os" "path/filepath" - "runtime" "strconv" "strings" "sync" @@ -65,13 +64,13 @@ func (h *Head) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks. // Start workers that each process samples for a partition of the series ID space. var ( wg sync.WaitGroup - n = runtime.GOMAXPROCS(0) - processors = make([]walSubsetProcessor, n) + concurrency = h.opts.WALReplayConcurrency + processors = make([]walSubsetProcessor, concurrency) exemplarsInput chan record.RefExemplar dec record.Decoder - shards = make([][]record.RefSample, n) - histogramShards = make([][]histogramRecord, n) + shards = make([][]record.RefSample, concurrency) + histogramShards = make([][]histogramRecord, concurrency) decoded = make(chan interface{}, 10) decodeErr, seriesCreationErr error @@ -116,7 +115,7 @@ func (h *Head) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks. // For CorruptionErr ensure to terminate all workers before exiting. _, ok := err.(*wlog.CorruptionErr) if ok || seriesCreationErr != nil { - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { processors[i].closeAndDrain() } close(exemplarsInput) @@ -124,8 +123,8 @@ func (h *Head) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks. } }() - wg.Add(n) - for i := 0; i < n; i++ { + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { processors[i].setup() go func(wp *walSubsetProcessor) { @@ -276,7 +275,7 @@ Outer: multiRef[walSeries.Ref] = mSeries.ref } - idx := uint64(mSeries.ref) % uint64(n) + idx := uint64(mSeries.ref) % uint64(concurrency) processors[idx].input <- walSubsetProcessorInputItem{walSeriesRef: walSeries.Ref, existingSeries: mSeries} } //nolint:staticcheck // Ignore SA6002 relax staticcheck verification. @@ -293,7 +292,7 @@ Outer: if len(samples) < m { m = len(samples) } - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { if shards[i] == nil { shards[i] = processors[i].reuseBuf() } @@ -305,10 +304,10 @@ Outer: if r, ok := multiRef[sam.Ref]; ok { sam.Ref = r } - mod := uint64(sam.Ref) % uint64(n) + mod := uint64(sam.Ref) % uint64(concurrency) shards[mod] = append(shards[mod], sam) } - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { if len(shards[i]) > 0 { processors[i].input <- walSubsetProcessorInputItem{samples: shards[i]} shards[i] = nil @@ -351,7 +350,7 @@ Outer: if len(samples) < m { m = len(samples) } - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { if histogramShards[i] == nil { histogramShards[i] = processors[i].reuseHistogramBuf() } @@ -363,10 +362,10 @@ Outer: if r, ok := multiRef[sam.Ref]; ok { sam.Ref = r } - mod := uint64(sam.Ref) % uint64(n) + mod := uint64(sam.Ref) % uint64(concurrency) histogramShards[mod] = append(histogramShards[mod], histogramRecord{ref: sam.Ref, t: sam.T, h: sam.H}) } - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { if len(histogramShards[i]) > 0 { processors[i].input <- walSubsetProcessorInputItem{histogramSamples: histogramShards[i]} histogramShards[i] = nil @@ -388,7 +387,7 @@ Outer: if len(samples) < m { m = len(samples) } - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { if histogramShards[i] == nil { histogramShards[i] = processors[i].reuseHistogramBuf() } @@ -400,10 +399,10 @@ Outer: if r, ok := multiRef[sam.Ref]; ok { sam.Ref = r } - mod := uint64(sam.Ref) % uint64(n) + mod := uint64(sam.Ref) % uint64(concurrency) histogramShards[mod] = append(histogramShards[mod], histogramRecord{ref: sam.Ref, t: sam.T, fh: sam.FH}) } - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { if len(histogramShards[i]) > 0 { processors[i].input <- walSubsetProcessorInputItem{histogramSamples: histogramShards[i]} histogramShards[i] = nil @@ -444,7 +443,7 @@ Outer: } // Signal termination to each worker and wait for it to close its output channel. - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { processors[i].closeAndDrain() } close(exemplarsInput) @@ -685,12 +684,12 @@ func (h *Head) loadWBL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks. lastSeq, lastOff := lastMmapRef.Unpack() // Start workers that each process samples for a partition of the series ID space. var ( - wg sync.WaitGroup - n = runtime.GOMAXPROCS(0) - processors = make([]wblSubsetProcessor, n) + wg sync.WaitGroup + concurrency = h.opts.WALReplayConcurrency + processors = make([]wblSubsetProcessor, concurrency) dec record.Decoder - shards = make([][]record.RefSample, n) + shards = make([][]record.RefSample, concurrency) decodedCh = make(chan interface{}, 10) decodeErr error @@ -712,15 +711,15 @@ func (h *Head) loadWBL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks. _, ok := err.(*wlog.CorruptionErr) if ok { err = &errLoadWbl{err: err} - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { processors[i].closeAndDrain() } wg.Wait() } }() - wg.Add(n) - for i := 0; i < n; i++ { + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { processors[i].setup() go func(wp *wblSubsetProcessor) { @@ -779,17 +778,17 @@ func (h *Head) loadWBL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks. if len(samples) < m { m = len(samples) } - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { shards[i] = processors[i].reuseBuf() } for _, sam := range samples[:m] { if r, ok := multiRef[sam.Ref]; ok { sam.Ref = r } - mod := uint64(sam.Ref) % uint64(n) + mod := uint64(sam.Ref) % uint64(concurrency) shards[mod] = append(shards[mod], sam) } - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { processors[i].input <- shards[i] } samples = samples[m:] @@ -816,7 +815,7 @@ func (h *Head) loadWBL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks. mmapMarkerUnknownRefs.Inc() continue } - idx := uint64(ms.ref) % uint64(n) + idx := uint64(ms.ref) % uint64(concurrency) // It is possible that some old sample is being processed in processWALSamples that // could cause race below. So we wait for the goroutine to empty input the buffer and finish // processing all old samples after emptying the buffer. @@ -845,7 +844,7 @@ func (h *Head) loadWBL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks. } // Signal termination to each worker and wait for it to close its output channel. - for i := 0; i < n; i++ { + for i := 0; i < concurrency; i++ { processors[i].closeAndDrain() } wg.Wait() @@ -1381,18 +1380,18 @@ func (h *Head) loadChunkSnapshot() (int, int, map[chunks.HeadSeriesRef]*memSerie var ( numSeries = 0 unknownRefs = int64(0) - n = runtime.GOMAXPROCS(0) + concurrency = h.opts.WALReplayConcurrency wg sync.WaitGroup - recordChan = make(chan chunkSnapshotRecord, 5*n) - shardedRefSeries = make([]map[chunks.HeadSeriesRef]*memSeries, n) - errChan = make(chan error, n) + recordChan = make(chan chunkSnapshotRecord, 5*concurrency) + shardedRefSeries = make([]map[chunks.HeadSeriesRef]*memSeries, concurrency) + errChan = make(chan error, concurrency) refSeries map[chunks.HeadSeriesRef]*memSeries exemplarBuf []record.RefExemplar dec record.Decoder ) - wg.Add(n) - for i := 0; i < n; i++ { + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { go func(idx int, rc <-chan chunkSnapshotRecord) { defer wg.Done() defer func() {