mirror of
https://github.com/prometheus/prometheus.git
synced 2024-09-19 23:37:31 -07:00
textparse: Implement CreatedTimestamp()
in openmetricsparse.go
(#14356)
* feat: initial implement of createedTimestamp() with tests Signed-off-by: Manik Rana <manikrana54@gmail.com> * feat: return ct after finding it Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: remove unneeded test Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: add comments Signed-off-by: Manik Rana <manikrana54@gmail.com> * feat: multiple changes - implement changes from pair programming session - use newParse.val() - advance parser p if ct is found Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: check if err from p.Next() Signed-off-by: Manik Rana <manikrana54@gmail.com> * feat: advance parser and parse histograms + summary Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: restore previous tests Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: retore failing tests Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: remove unneeded comments Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: return nil when mtype doesn't match Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: update go fmt version Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: cleanup Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: comments Signed-off-by: Manik Rana <manikrana54@gmail.com> * feat: document deepcopyparser Co-authored-by: Arthur Silva Sens <arthursens2005@gmail.com> Signed-off-by: Manik Rana <Manikrana54@gmail.com> Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: lint Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: cover edgecase of `gauge_created` in CreatedTimestamp() Signed-off-by: Manik Rana <manikrana54@gmail.com> * refac: readability updates Signed-off-by: Manik Rana <manikrana54@gmail.com> * refac: dedeuplicate labeldiff checks Signed-off-by: Manik Rana <manikrana54@gmail.com> * tests: add tests for new label functions Signed-off-by: Manik Rana <manikrana54@gmail.com> * feat: document CreatedTimestamp func Signed-off-by: Manik Rana <manikrana54@gmail.com> * refac: optimize `CreatedTimestamp()` - Use refactored CreatedTimestamp function with bug fixes - Remove unused code in labels.go - Improve code documentation Signed-off-by: Manik Rana <manikrana54@gmail.com> Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: add tests and lint fixes Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: remove mName Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: lint Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: comments Signed-off-by: Manik Rana <manikrana54@gmail.com> * tests: add tests for CT parse failures and deepCopy Signed-off-by: Manik Rana <manikrana54@gmail.com> * refac: edit expectCT struct Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: lint Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: add new label in deepCopy Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: use p.builder in deepCopy Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: add NewMetricsParserWithOpts Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: lint Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: comments Co-authored-by: Arthur Silva Sens <arthursens2005@gmail.com> Signed-off-by: Manik Rana <Manikrana54@gmail.com> * chore: comments Co-authored-by: Arthur Silva Sens <arthursens2005@gmail.com> Signed-off-by: Manik Rana <Manikrana54@gmail.com> * chore: rename var Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: add condition for OM fuzzing Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: build tags Signed-off-by: Manik Rana <manikrana54@gmail.com> * refac: default skipCT to false Signed-off-by: Manik Rana <manikrana54@gmail.com> * refac: rename skipCT to skipCTSeries Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: formatting Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: comments and readability updates Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: comments Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com> Signed-off-by: Manik Rana <Manikrana54@gmail.com> * refac: remove NewOpenMetricsParserWithOpts Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: lint Signed-off-by: Manik Rana <manikrana54@gmail.com> * refac: extract skipCTSeries logic from parseMetricSuffix Signed-off-by: Manik Rana <manikrana54@gmail.com> * refac: inline create a NewOpenMetricsParser Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: comments Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: comments Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com> Signed-off-by: Manik Rana <Manikrana54@gmail.com> * refac: improve error handling Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: return error instead of nil Signed-off-by: Manik Rana <manikrana54@gmail.com> * fix: remove skipCT check from tBraceOpen Signed-off-by: Manik Rana <manikrana54@gmail.com> * Pair programming with Manik, Arthur and Daniel. Signed-off-by: bwplotka <bwplotka@gmail.com> * chore: comments and use helper funcs Signed-off-by: Manik Rana <manikrana54@gmail.com> * chore: lint Signed-off-by: Manik Rana <manikrana54@gmail.com> --------- Signed-off-by: Manik Rana <manikrana54@gmail.com> Signed-off-by: Manik Rana <Manikrana54@gmail.com> Signed-off-by: bwplotka <bwplotka@gmail.com> Co-authored-by: Arthur Silva Sens <arthursens2005@gmail.com> Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
This commit is contained in:
parent
b7a58dcf3d
commit
02c465bf58
|
@ -94,16 +94,46 @@ type OpenMetricsParser struct {
|
||||||
exemplarVal float64
|
exemplarVal float64
|
||||||
exemplarTs int64
|
exemplarTs int64
|
||||||
hasExemplarTs bool
|
hasExemplarTs bool
|
||||||
|
|
||||||
|
skipCTSeries bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewOpenMetricsParser returns a new parser of the byte slice.
|
type openMetricsParserOptions struct {
|
||||||
func NewOpenMetricsParser(b []byte, st *labels.SymbolTable) Parser {
|
SkipCTSeries bool
|
||||||
return &OpenMetricsParser{
|
}
|
||||||
l: &openMetricsLexer{b: b},
|
|
||||||
builder: labels.NewScratchBuilderWithSymbolTable(st, 16),
|
type OpenMetricsOption func(*openMetricsParserOptions)
|
||||||
|
|
||||||
|
// WithOMParserCTSeriesSkipped turns off exposing _created lines
|
||||||
|
// as series, which makes those only used for parsing created timestamp
|
||||||
|
// for `CreatedTimestamp` method purposes.
|
||||||
|
//
|
||||||
|
// It's recommended to use this option to avoid using _created lines for other
|
||||||
|
// purposes than created timestamp, but leave false by default for the
|
||||||
|
// best-effort compatibility.
|
||||||
|
func WithOMParserCTSeriesSkipped() OpenMetricsOption {
|
||||||
|
return func(o *openMetricsParserOptions) {
|
||||||
|
o.SkipCTSeries = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewOpenMetricsParser returns a new parser for the byte slice with option to skip CT series parsing.
|
||||||
|
func NewOpenMetricsParser(b []byte, st *labels.SymbolTable, opts ...OpenMetricsOption) Parser {
|
||||||
|
options := &openMetricsParserOptions{}
|
||||||
|
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(options)
|
||||||
|
}
|
||||||
|
|
||||||
|
parser := &OpenMetricsParser{
|
||||||
|
l: &openMetricsLexer{b: b},
|
||||||
|
builder: labels.NewScratchBuilderWithSymbolTable(st, 16),
|
||||||
|
skipCTSeries: options.SkipCTSeries,
|
||||||
|
}
|
||||||
|
|
||||||
|
return parser
|
||||||
|
}
|
||||||
|
|
||||||
// Series returns the bytes of the series, the timestamp if set, and the value
|
// Series returns the bytes of the series, the timestamp if set, and the value
|
||||||
// of the current sample.
|
// of the current sample.
|
||||||
func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) {
|
func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) {
|
||||||
|
@ -219,10 +249,90 @@ func (p *OpenMetricsParser) Exemplar(e *exemplar.Exemplar) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreatedTimestamp returns nil as it's not implemented yet.
|
// CreatedTimestamp returns the created timestamp for a current Metric if exists or nil.
|
||||||
// TODO(bwplotka): https://github.com/prometheus/prometheus/issues/12980
|
// NOTE(Maniktherana): Might use additional CPU/mem resources due to deep copy of parser required for peeking given 1.0 OM specification on _created series.
|
||||||
func (p *OpenMetricsParser) CreatedTimestamp() *int64 {
|
func (p *OpenMetricsParser) CreatedTimestamp() *int64 {
|
||||||
return nil
|
if !TypeRequiresCT(p.mtype) {
|
||||||
|
// Not a CT supported metric type, fast path.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
currLset labels.Labels
|
||||||
|
buf []byte
|
||||||
|
peekWithoutNameLsetHash uint64
|
||||||
|
)
|
||||||
|
p.Metric(&currLset)
|
||||||
|
currFamilyLsetHash, buf := currLset.HashWithoutLabels(buf, labels.MetricName, "le", "quantile")
|
||||||
|
// Search for the _created line for the currFamilyLsetHash using ephemeral parser until
|
||||||
|
// we see EOF or new metric family. We have to do it as we don't know where (and if)
|
||||||
|
// that CT line is.
|
||||||
|
// TODO(bwplotka): Make sure OM 1.1/2.0 pass CT via metadata or exemplar-like to avoid this.
|
||||||
|
peek := deepCopy(p)
|
||||||
|
for {
|
||||||
|
eType, err := peek.Next()
|
||||||
|
if err != nil {
|
||||||
|
// This means peek will give error too later on, so def no CT line found.
|
||||||
|
// This might result in partial scrape with wrong/missing CT, but only
|
||||||
|
// spec improvement would help.
|
||||||
|
// TODO(bwplotka): Make sure OM 1.1/2.0 pass CT via metadata or exemplar-like to avoid this.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if eType != EntrySeries {
|
||||||
|
// Assume we hit different family, no CT line found.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var peekedLset labels.Labels
|
||||||
|
peek.Metric(&peekedLset)
|
||||||
|
peekedName := peekedLset.Get(model.MetricNameLabel)
|
||||||
|
if !strings.HasSuffix(peekedName, "_created") {
|
||||||
|
// Not a CT line, search more.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// We got a CT line here, but let's search if CT line is actually for our series, edge case.
|
||||||
|
peekWithoutNameLsetHash, _ = peekedLset.HashWithoutLabels(buf, labels.MetricName, "le", "quantile")
|
||||||
|
if peekWithoutNameLsetHash != currFamilyLsetHash {
|
||||||
|
// CT line for a different series, for our series no CT.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ct := int64(peek.val)
|
||||||
|
return &ct
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TypeRequiresCT returns true if the metric type requires a _created timestamp.
|
||||||
|
func TypeRequiresCT(t model.MetricType) bool {
|
||||||
|
switch t {
|
||||||
|
case model.MetricTypeCounter, model.MetricTypeSummary, model.MetricTypeHistogram:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// deepCopy creates a copy of a parser without re-using the slices' original memory addresses.
|
||||||
|
func deepCopy(p *OpenMetricsParser) OpenMetricsParser {
|
||||||
|
newB := make([]byte, len(p.l.b))
|
||||||
|
copy(newB, p.l.b)
|
||||||
|
|
||||||
|
newLexer := &openMetricsLexer{
|
||||||
|
b: newB,
|
||||||
|
i: p.l.i,
|
||||||
|
start: p.l.start,
|
||||||
|
err: p.l.err,
|
||||||
|
state: p.l.state,
|
||||||
|
}
|
||||||
|
|
||||||
|
newParser := OpenMetricsParser{
|
||||||
|
l: newLexer,
|
||||||
|
builder: p.builder,
|
||||||
|
mtype: p.mtype,
|
||||||
|
val: p.val,
|
||||||
|
skipCTSeries: false,
|
||||||
|
}
|
||||||
|
return newParser
|
||||||
}
|
}
|
||||||
|
|
||||||
// nextToken returns the next token from the openMetricsLexer.
|
// nextToken returns the next token from the openMetricsLexer.
|
||||||
|
@ -337,7 +447,13 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
p.series = p.l.b[p.start:p.l.i]
|
p.series = p.l.b[p.start:p.l.i]
|
||||||
return p.parseMetricSuffix(p.nextToken())
|
if err := p.parseSeriesEndOfLine(p.nextToken()); err != nil {
|
||||||
|
return EntryInvalid, err
|
||||||
|
}
|
||||||
|
if p.skipCTSeries && p.isCreatedSeries() {
|
||||||
|
return p.Next()
|
||||||
|
}
|
||||||
|
return EntrySeries, nil
|
||||||
case tMName:
|
case tMName:
|
||||||
p.offsets = append(p.offsets, p.start, p.l.i)
|
p.offsets = append(p.offsets, p.start, p.l.i)
|
||||||
p.series = p.l.b[p.start:p.l.i]
|
p.series = p.l.b[p.start:p.l.i]
|
||||||
|
@ -351,8 +467,14 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
|
||||||
p.series = p.l.b[p.start:p.l.i]
|
p.series = p.l.b[p.start:p.l.i]
|
||||||
t2 = p.nextToken()
|
t2 = p.nextToken()
|
||||||
}
|
}
|
||||||
return p.parseMetricSuffix(t2)
|
|
||||||
|
|
||||||
|
if err := p.parseSeriesEndOfLine(t2); err != nil {
|
||||||
|
return EntryInvalid, err
|
||||||
|
}
|
||||||
|
if p.skipCTSeries && p.isCreatedSeries() {
|
||||||
|
return p.Next()
|
||||||
|
}
|
||||||
|
return EntrySeries, nil
|
||||||
default:
|
default:
|
||||||
err = p.parseError("expected a valid start token", t)
|
err = p.parseError("expected a valid start token", t)
|
||||||
}
|
}
|
||||||
|
@ -467,51 +589,64 @@ func (p *OpenMetricsParser) parseLVals(offsets []int, isExemplar bool) ([]int, e
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseMetricSuffix parses the end of the line after the metric name and
|
// isCreatedSeries returns true if the current series is a _created series.
|
||||||
// labels. It starts parsing with the provided token.
|
func (p *OpenMetricsParser) isCreatedSeries() bool {
|
||||||
func (p *OpenMetricsParser) parseMetricSuffix(t token) (Entry, error) {
|
var newLbs labels.Labels
|
||||||
|
p.Metric(&newLbs)
|
||||||
|
name := newLbs.Get(model.MetricNameLabel)
|
||||||
|
if TypeRequiresCT(p.mtype) && strings.HasSuffix(name, "_created") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseSeriesEndOfLine parses the series end of the line (value, optional
|
||||||
|
// timestamp, commentary, etc.) after the metric name and labels.
|
||||||
|
// It starts parsing with the provided token.
|
||||||
|
func (p *OpenMetricsParser) parseSeriesEndOfLine(t token) error {
|
||||||
if p.offsets[0] == -1 {
|
if p.offsets[0] == -1 {
|
||||||
return EntryInvalid, fmt.Errorf("metric name not set while parsing: %q", p.l.b[p.start:p.l.i])
|
return fmt.Errorf("metric name not set while parsing: %q", p.l.b[p.start:p.l.i])
|
||||||
}
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
p.val, err = p.getFloatValue(t, "metric")
|
p.val, err = p.getFloatValue(t, "metric")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return EntryInvalid, err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
p.hasTS = false
|
p.hasTS = false
|
||||||
switch t2 := p.nextToken(); t2 {
|
switch t2 := p.nextToken(); t2 {
|
||||||
case tEOF:
|
case tEOF:
|
||||||
return EntryInvalid, errors.New("data does not end with # EOF")
|
return errors.New("data does not end with # EOF")
|
||||||
case tLinebreak:
|
case tLinebreak:
|
||||||
break
|
break
|
||||||
case tComment:
|
case tComment:
|
||||||
if err := p.parseComment(); err != nil {
|
if err := p.parseComment(); err != nil {
|
||||||
return EntryInvalid, err
|
return err
|
||||||
}
|
}
|
||||||
case tTimestamp:
|
case tTimestamp:
|
||||||
p.hasTS = true
|
p.hasTS = true
|
||||||
var ts float64
|
var ts float64
|
||||||
// A float is enough to hold what we need for millisecond resolution.
|
// A float is enough to hold what we need for millisecond resolution.
|
||||||
if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
|
if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
|
||||||
return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
|
return fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
|
||||||
}
|
}
|
||||||
if math.IsNaN(ts) || math.IsInf(ts, 0) {
|
if math.IsNaN(ts) || math.IsInf(ts, 0) {
|
||||||
return EntryInvalid, fmt.Errorf("invalid timestamp %f", ts)
|
return fmt.Errorf("invalid timestamp %f", ts)
|
||||||
}
|
}
|
||||||
p.ts = int64(ts * 1000)
|
p.ts = int64(ts * 1000)
|
||||||
switch t3 := p.nextToken(); t3 {
|
switch t3 := p.nextToken(); t3 {
|
||||||
case tLinebreak:
|
case tLinebreak:
|
||||||
case tComment:
|
case tComment:
|
||||||
if err := p.parseComment(); err != nil {
|
if err := p.parseComment(); err != nil {
|
||||||
return EntryInvalid, err
|
return err
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return EntryInvalid, p.parseError("expected next entry after timestamp", t3)
|
return p.parseError("expected next entry after timestamp", t3)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return EntrySeries, nil
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error) {
|
func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error) {
|
||||||
|
|
|
@ -14,6 +14,7 @@
|
||||||
package textparse
|
package textparse
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
@ -24,6 +25,8 @@ import (
|
||||||
"github.com/prometheus/prometheus/model/labels"
|
"github.com/prometheus/prometheus/model/labels"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func int64p(x int64) *int64 { return &x }
|
||||||
|
|
||||||
func TestOpenMetricsParse(t *testing.T) {
|
func TestOpenMetricsParse(t *testing.T) {
|
||||||
input := `# HELP go_gc_duration_seconds A summary of the GC invocation durations.
|
input := `# HELP go_gc_duration_seconds A summary of the GC invocation durations.
|
||||||
# TYPE go_gc_duration_seconds summary
|
# TYPE go_gc_duration_seconds summary
|
||||||
|
@ -63,15 +66,34 @@ ss{A="a"} 0
|
||||||
_metric_starting_with_underscore 1
|
_metric_starting_with_underscore 1
|
||||||
testmetric{_label_starting_with_underscore="foo"} 1
|
testmetric{_label_starting_with_underscore="foo"} 1
|
||||||
testmetric{label="\"bar\""} 1
|
testmetric{label="\"bar\""} 1
|
||||||
|
# HELP foo Counter with and without labels to certify CT is parsed for both cases
|
||||||
# TYPE foo counter
|
# TYPE foo counter
|
||||||
foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
|
foo_total 17.0 1520879607.789 # {id="counter-test"} 5
|
||||||
|
foo_created 1000
|
||||||
|
foo_total{a="b"} 17.0 1520879607.789 # {id="counter-test"} 5
|
||||||
|
foo_created{a="b"} 1000
|
||||||
|
# HELP bar Summary with CT at the end, making sure we find CT even if it's multiple lines a far
|
||||||
|
# TYPE bar summary
|
||||||
|
bar_count 17.0
|
||||||
|
bar_sum 324789.3
|
||||||
|
bar{quantile="0.95"} 123.7
|
||||||
|
bar{quantile="0.99"} 150.0
|
||||||
|
bar_created 1520430000
|
||||||
|
# HELP baz Histogram with the same objective as above's summary
|
||||||
|
# TYPE baz histogram
|
||||||
|
baz_bucket{le="0.0"} 0
|
||||||
|
baz_bucket{le="+Inf"} 17
|
||||||
|
baz_count 17
|
||||||
|
baz_sum 324789.3
|
||||||
|
baz_created 1520430000
|
||||||
|
# HELP fizz_created Gauge which shouldn't be parsed as CT
|
||||||
|
# TYPE fizz_created gauge
|
||||||
|
fizz_created 17.0`
|
||||||
|
|
||||||
input += "\n# HELP metric foo\x00bar"
|
input += "\n# HELP metric foo\x00bar"
|
||||||
input += "\nnull_byte_metric{a=\"abc\x00\"} 1"
|
input += "\nnull_byte_metric{a=\"abc\x00\"} 1"
|
||||||
input += "\n# EOF\n"
|
input += "\n# EOF\n"
|
||||||
|
|
||||||
int64p := func(x int64) *int64 { return &x }
|
|
||||||
|
|
||||||
exp := []expectedParse{
|
exp := []expectedParse{
|
||||||
{
|
{
|
||||||
m: "go_gc_duration_seconds",
|
m: "go_gc_duration_seconds",
|
||||||
|
@ -216,6 +238,9 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
|
||||||
m: "testmetric{label=\"\\\"bar\\\"\"}",
|
m: "testmetric{label=\"\\\"bar\\\"\"}",
|
||||||
v: 1,
|
v: 1,
|
||||||
lset: labels.FromStrings("__name__", "testmetric", "label", `"bar"`),
|
lset: labels.FromStrings("__name__", "testmetric", "label", `"bar"`),
|
||||||
|
}, {
|
||||||
|
m: "foo",
|
||||||
|
help: "Counter with and without labels to certify CT is parsed for both cases",
|
||||||
}, {
|
}, {
|
||||||
m: "foo",
|
m: "foo",
|
||||||
typ: model.MetricTypeCounter,
|
typ: model.MetricTypeCounter,
|
||||||
|
@ -225,6 +250,76 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
|
||||||
lset: labels.FromStrings("__name__", "foo_total"),
|
lset: labels.FromStrings("__name__", "foo_total"),
|
||||||
t: int64p(1520879607789),
|
t: int64p(1520879607789),
|
||||||
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5},
|
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5},
|
||||||
|
ct: int64p(1000),
|
||||||
|
}, {
|
||||||
|
m: `foo_total{a="b"}`,
|
||||||
|
v: 17.0,
|
||||||
|
lset: labels.FromStrings("__name__", "foo_total", "a", "b"),
|
||||||
|
t: int64p(1520879607789),
|
||||||
|
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5},
|
||||||
|
ct: int64p(1000),
|
||||||
|
}, {
|
||||||
|
m: "bar",
|
||||||
|
help: "Summary with CT at the end, making sure we find CT even if it's multiple lines a far",
|
||||||
|
}, {
|
||||||
|
m: "bar",
|
||||||
|
typ: model.MetricTypeSummary,
|
||||||
|
}, {
|
||||||
|
m: "bar_count",
|
||||||
|
v: 17.0,
|
||||||
|
lset: labels.FromStrings("__name__", "bar_count"),
|
||||||
|
ct: int64p(1520430000),
|
||||||
|
}, {
|
||||||
|
m: "bar_sum",
|
||||||
|
v: 324789.3,
|
||||||
|
lset: labels.FromStrings("__name__", "bar_sum"),
|
||||||
|
ct: int64p(1520430000),
|
||||||
|
}, {
|
||||||
|
m: `bar{quantile="0.95"}`,
|
||||||
|
v: 123.7,
|
||||||
|
lset: labels.FromStrings("__name__", "bar", "quantile", "0.95"),
|
||||||
|
ct: int64p(1520430000),
|
||||||
|
}, {
|
||||||
|
m: `bar{quantile="0.99"}`,
|
||||||
|
v: 150.0,
|
||||||
|
lset: labels.FromStrings("__name__", "bar", "quantile", "0.99"),
|
||||||
|
ct: int64p(1520430000),
|
||||||
|
}, {
|
||||||
|
m: "baz",
|
||||||
|
help: "Histogram with the same objective as above's summary",
|
||||||
|
}, {
|
||||||
|
m: "baz",
|
||||||
|
typ: model.MetricTypeHistogram,
|
||||||
|
}, {
|
||||||
|
m: `baz_bucket{le="0.0"}`,
|
||||||
|
v: 0,
|
||||||
|
lset: labels.FromStrings("__name__", "baz_bucket", "le", "0.0"),
|
||||||
|
ct: int64p(1520430000),
|
||||||
|
}, {
|
||||||
|
m: `baz_bucket{le="+Inf"}`,
|
||||||
|
v: 17,
|
||||||
|
lset: labels.FromStrings("__name__", "baz_bucket", "le", "+Inf"),
|
||||||
|
ct: int64p(1520430000),
|
||||||
|
}, {
|
||||||
|
m: `baz_count`,
|
||||||
|
v: 17,
|
||||||
|
lset: labels.FromStrings("__name__", "baz_count"),
|
||||||
|
ct: int64p(1520430000),
|
||||||
|
}, {
|
||||||
|
m: `baz_sum`,
|
||||||
|
v: 324789.3,
|
||||||
|
lset: labels.FromStrings("__name__", "baz_sum"),
|
||||||
|
ct: int64p(1520430000),
|
||||||
|
}, {
|
||||||
|
m: "fizz_created",
|
||||||
|
help: "Gauge which shouldn't be parsed as CT",
|
||||||
|
}, {
|
||||||
|
m: "fizz_created",
|
||||||
|
typ: model.MetricTypeGauge,
|
||||||
|
}, {
|
||||||
|
m: `fizz_created`,
|
||||||
|
v: 17,
|
||||||
|
lset: labels.FromStrings("__name__", "fizz_created"),
|
||||||
}, {
|
}, {
|
||||||
m: "metric",
|
m: "metric",
|
||||||
help: "foo\x00bar",
|
help: "foo\x00bar",
|
||||||
|
@ -235,8 +330,8 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable())
|
p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable(), WithOMParserCTSeriesSkipped())
|
||||||
checkParseResults(t, p, exp)
|
checkParseResultsWithCT(t, p, exp, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUTF8OpenMetricsParse(t *testing.T) {
|
func TestUTF8OpenMetricsParse(t *testing.T) {
|
||||||
|
@ -251,6 +346,7 @@ func TestUTF8OpenMetricsParse(t *testing.T) {
|
||||||
# UNIT "go.gc_duration_seconds" seconds
|
# UNIT "go.gc_duration_seconds" seconds
|
||||||
{"go.gc_duration_seconds",quantile="0"} 4.9351e-05
|
{"go.gc_duration_seconds",quantile="0"} 4.9351e-05
|
||||||
{"go.gc_duration_seconds",quantile="0.25"} 7.424100000000001e-05
|
{"go.gc_duration_seconds",quantile="0.25"} 7.424100000000001e-05
|
||||||
|
{"go.gc_duration_seconds_created"} 12313
|
||||||
{"go.gc_duration_seconds",quantile="0.5",a="b"} 8.3835e-05
|
{"go.gc_duration_seconds",quantile="0.5",a="b"} 8.3835e-05
|
||||||
{"http.status",q="0.9",a="b"} 8.3835e-05
|
{"http.status",q="0.9",a="b"} 8.3835e-05
|
||||||
{"http.status",q="0.9",a="b"} 8.3835e-05
|
{"http.status",q="0.9",a="b"} 8.3835e-05
|
||||||
|
@ -274,10 +370,12 @@ func TestUTF8OpenMetricsParse(t *testing.T) {
|
||||||
m: `{"go.gc_duration_seconds",quantile="0"}`,
|
m: `{"go.gc_duration_seconds",quantile="0"}`,
|
||||||
v: 4.9351e-05,
|
v: 4.9351e-05,
|
||||||
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0"),
|
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0"),
|
||||||
|
ct: int64p(12313),
|
||||||
}, {
|
}, {
|
||||||
m: `{"go.gc_duration_seconds",quantile="0.25"}`,
|
m: `{"go.gc_duration_seconds",quantile="0.25"}`,
|
||||||
v: 7.424100000000001e-05,
|
v: 7.424100000000001e-05,
|
||||||
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.25"),
|
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.25"),
|
||||||
|
ct: int64p(12313),
|
||||||
}, {
|
}, {
|
||||||
m: `{"go.gc_duration_seconds",quantile="0.5",a="b"}`,
|
m: `{"go.gc_duration_seconds",quantile="0.5",a="b"}`,
|
||||||
v: 8.3835e-05,
|
v: 8.3835e-05,
|
||||||
|
@ -306,8 +404,8 @@ choices}`, "strange©™\n'quoted' \"name\"", "6"),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable())
|
p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable(), WithOMParserCTSeriesSkipped())
|
||||||
checkParseResults(t, p, exp)
|
checkParseResultsWithCT(t, p, exp, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOpenMetricsParseErrors(t *testing.T) {
|
func TestOpenMetricsParseErrors(t *testing.T) {
|
||||||
|
@ -598,10 +696,6 @@ func TestOpenMetricsParseErrors(t *testing.T) {
|
||||||
input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 -Inf",
|
input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 -Inf",
|
||||||
err: `invalid exemplar timestamp -Inf`,
|
err: `invalid exemplar timestamp -Inf`,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 Inf",
|
|
||||||
err: `invalid exemplar timestamp +Inf`,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for i, c := range cases {
|
for i, c := range cases {
|
||||||
|
@ -684,3 +778,217 @@ func TestOMNullByteHandling(t *testing.T) {
|
||||||
require.Equal(t, c.err, err.Error(), "test %d", i)
|
require.Equal(t, c.err, err.Error(), "test %d", i)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// While not desirable, there are cases were CT fails to parse and
|
||||||
|
// these tests show them.
|
||||||
|
// TODO(maniktherana): Make sure OM 1.1/2.0 pass CT via metadata or exemplar-like to avoid this.
|
||||||
|
func TestCTParseFailures(t *testing.T) {
|
||||||
|
input := `# HELP something Histogram with _created between buckets and summary
|
||||||
|
# TYPE something histogram
|
||||||
|
something_count 17
|
||||||
|
something_sum 324789.3
|
||||||
|
something_created 1520430001
|
||||||
|
something_bucket{le="0.0"} 0
|
||||||
|
something_bucket{le="+Inf"} 17
|
||||||
|
# HELP thing Histogram with _created as first line
|
||||||
|
# TYPE thing histogram
|
||||||
|
thing_created 1520430002
|
||||||
|
thing_count 17
|
||||||
|
thing_sum 324789.3
|
||||||
|
thing_bucket{le="0.0"} 0
|
||||||
|
thing_bucket{le="+Inf"} 17
|
||||||
|
# HELP yum Summary with _created between sum and quantiles
|
||||||
|
# TYPE yum summary
|
||||||
|
yum_count 17.0
|
||||||
|
yum_sum 324789.3
|
||||||
|
yum_created 1520430003
|
||||||
|
yum{quantile="0.95"} 123.7
|
||||||
|
yum{quantile="0.99"} 150.0
|
||||||
|
# HELP foobar Summary with _created as the first line
|
||||||
|
# TYPE foobar summary
|
||||||
|
foobar_created 1520430004
|
||||||
|
foobar_count 17.0
|
||||||
|
foobar_sum 324789.3
|
||||||
|
foobar{quantile="0.95"} 123.7
|
||||||
|
foobar{quantile="0.99"} 150.0`
|
||||||
|
|
||||||
|
input += "\n# EOF\n"
|
||||||
|
|
||||||
|
int64p := func(x int64) *int64 { return &x }
|
||||||
|
|
||||||
|
type expectCT struct {
|
||||||
|
m string
|
||||||
|
ct *int64
|
||||||
|
typ model.MetricType
|
||||||
|
help string
|
||||||
|
isErr bool
|
||||||
|
}
|
||||||
|
|
||||||
|
exp := []expectCT{
|
||||||
|
{
|
||||||
|
m: "something",
|
||||||
|
help: "Histogram with _created between buckets and summary",
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: "something",
|
||||||
|
typ: model.MetricTypeHistogram,
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: `something_count`,
|
||||||
|
ct: int64p(1520430001),
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: `something_sum`,
|
||||||
|
ct: int64p(1520430001),
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: `something_bucket{le="0.0"}`,
|
||||||
|
ct: int64p(1520430001),
|
||||||
|
isErr: true,
|
||||||
|
}, {
|
||||||
|
m: `something_bucket{le="+Inf"}`,
|
||||||
|
ct: int64p(1520430001),
|
||||||
|
isErr: true,
|
||||||
|
}, {
|
||||||
|
m: "thing",
|
||||||
|
help: "Histogram with _created as first line",
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: "thing",
|
||||||
|
typ: model.MetricTypeHistogram,
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: `thing_count`,
|
||||||
|
ct: int64p(1520430002),
|
||||||
|
isErr: true,
|
||||||
|
}, {
|
||||||
|
m: `thing_sum`,
|
||||||
|
ct: int64p(1520430002),
|
||||||
|
isErr: true,
|
||||||
|
}, {
|
||||||
|
m: `thing_bucket{le="0.0"}`,
|
||||||
|
ct: int64p(1520430002),
|
||||||
|
isErr: true,
|
||||||
|
}, {
|
||||||
|
m: `thing_bucket{le="+Inf"}`,
|
||||||
|
ct: int64p(1520430002),
|
||||||
|
isErr: true,
|
||||||
|
}, {
|
||||||
|
m: "yum",
|
||||||
|
help: "Summary with _created between summary and quantiles",
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: "yum",
|
||||||
|
typ: model.MetricTypeSummary,
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: "yum_count",
|
||||||
|
ct: int64p(1520430003),
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: "yum_sum",
|
||||||
|
ct: int64p(1520430003),
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: `yum{quantile="0.95"}`,
|
||||||
|
ct: int64p(1520430003),
|
||||||
|
isErr: true,
|
||||||
|
}, {
|
||||||
|
m: `yum{quantile="0.99"}`,
|
||||||
|
ct: int64p(1520430003),
|
||||||
|
isErr: true,
|
||||||
|
}, {
|
||||||
|
m: "foobar",
|
||||||
|
help: "Summary with _created as the first line",
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: "foobar",
|
||||||
|
typ: model.MetricTypeSummary,
|
||||||
|
isErr: false,
|
||||||
|
}, {
|
||||||
|
m: "foobar_count",
|
||||||
|
ct: int64p(1520430004),
|
||||||
|
isErr: true,
|
||||||
|
}, {
|
||||||
|
m: "foobar_sum",
|
||||||
|
ct: int64p(1520430004),
|
||||||
|
isErr: true,
|
||||||
|
}, {
|
||||||
|
m: `foobar{quantile="0.95"}`,
|
||||||
|
ct: int64p(1520430004),
|
||||||
|
isErr: true,
|
||||||
|
}, {
|
||||||
|
m: `foobar{quantile="0.99"}`,
|
||||||
|
ct: int64p(1520430004),
|
||||||
|
isErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable(), WithOMParserCTSeriesSkipped())
|
||||||
|
i := 0
|
||||||
|
|
||||||
|
var res labels.Labels
|
||||||
|
for {
|
||||||
|
et, err := p.Next()
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
switch et {
|
||||||
|
case EntrySeries:
|
||||||
|
p.Metric(&res)
|
||||||
|
|
||||||
|
if ct := p.CreatedTimestamp(); exp[i].isErr {
|
||||||
|
require.Nil(t, ct)
|
||||||
|
} else {
|
||||||
|
require.Equal(t, *exp[i].ct, *ct)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeepCopy(t *testing.T) {
|
||||||
|
input := []byte(`# HELP go_goroutines A gauge goroutines.
|
||||||
|
# TYPE go_goroutines gauge
|
||||||
|
go_goroutines 33 123.123
|
||||||
|
# TYPE go_gc_duration_seconds summary
|
||||||
|
go_gc_duration_seconds
|
||||||
|
go_gc_duration_seconds_created`)
|
||||||
|
|
||||||
|
st := labels.NewSymbolTable()
|
||||||
|
parser := NewOpenMetricsParser(input, st, WithOMParserCTSeriesSkipped()).(*OpenMetricsParser)
|
||||||
|
|
||||||
|
// Modify the original parser state
|
||||||
|
_, err := parser.Next()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, "go_goroutines", string(parser.l.b[parser.offsets[0]:parser.offsets[1]]))
|
||||||
|
require.True(t, parser.skipCTSeries)
|
||||||
|
|
||||||
|
// Create a deep copy of the parser
|
||||||
|
copyParser := deepCopy(parser)
|
||||||
|
etype, err := copyParser.Next()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, EntryType, etype)
|
||||||
|
require.True(t, parser.skipCTSeries)
|
||||||
|
require.False(t, copyParser.skipCTSeries)
|
||||||
|
|
||||||
|
// Modify the original parser further
|
||||||
|
parser.Next()
|
||||||
|
parser.Next()
|
||||||
|
parser.Next()
|
||||||
|
require.Equal(t, "go_gc_duration_seconds", string(parser.l.b[parser.offsets[0]:parser.offsets[1]]))
|
||||||
|
require.Equal(t, "summary", string(parser.mtype))
|
||||||
|
require.False(t, copyParser.skipCTSeries)
|
||||||
|
require.True(t, parser.skipCTSeries)
|
||||||
|
|
||||||
|
// Ensure the copy remains unchanged
|
||||||
|
copyParser.Next()
|
||||||
|
copyParser.Next()
|
||||||
|
require.Equal(t, "go_gc_duration_seconds", string(copyParser.l.b[copyParser.offsets[0]:copyParser.offsets[1]]))
|
||||||
|
require.False(t, copyParser.skipCTSeries)
|
||||||
|
}
|
||||||
|
|
|
@ -18,6 +18,7 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/klauspost/compress/gzip"
|
"github.com/klauspost/compress/gzip"
|
||||||
|
@ -41,6 +42,7 @@ type expectedParse struct {
|
||||||
unit string
|
unit string
|
||||||
comment string
|
comment string
|
||||||
e *exemplar.Exemplar
|
e *exemplar.Exemplar
|
||||||
|
ct *int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPromParse(t *testing.T) {
|
func TestPromParse(t *testing.T) {
|
||||||
|
@ -188,6 +190,10 @@ testmetric{label="\"bar\""} 1`
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkParseResults(t *testing.T, p Parser, exp []expectedParse) {
|
func checkParseResults(t *testing.T, p Parser, exp []expectedParse) {
|
||||||
|
checkParseResultsWithCT(t, p, exp, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkParseResultsWithCT(t *testing.T, p Parser, exp []expectedParse, ctLinesRemoved bool) {
|
||||||
i := 0
|
i := 0
|
||||||
|
|
||||||
var res labels.Labels
|
var res labels.Labels
|
||||||
|
@ -205,6 +211,14 @@ func checkParseResults(t *testing.T, p Parser, exp []expectedParse) {
|
||||||
|
|
||||||
p.Metric(&res)
|
p.Metric(&res)
|
||||||
|
|
||||||
|
if ctLinesRemoved {
|
||||||
|
// Are CT series skipped?
|
||||||
|
_, typ := p.Type()
|
||||||
|
if TypeRequiresCT(typ) && strings.HasSuffix(res.Get(labels.MetricName), "_created") {
|
||||||
|
t.Fatalf("we exped created lines skipped")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
require.Equal(t, exp[i].m, string(m))
|
require.Equal(t, exp[i].m, string(m))
|
||||||
require.Equal(t, exp[i].t, ts)
|
require.Equal(t, exp[i].t, ts)
|
||||||
require.Equal(t, exp[i].v, v)
|
require.Equal(t, exp[i].v, v)
|
||||||
|
@ -218,6 +232,11 @@ func checkParseResults(t *testing.T, p Parser, exp []expectedParse) {
|
||||||
require.True(t, found)
|
require.True(t, found)
|
||||||
testutil.RequireEqual(t, *exp[i].e, e)
|
testutil.RequireEqual(t, *exp[i].e, e)
|
||||||
}
|
}
|
||||||
|
if ct := p.CreatedTimestamp(); ct != nil {
|
||||||
|
require.Equal(t, *exp[i].ct, *ct)
|
||||||
|
} else {
|
||||||
|
require.Nil(t, exp[i].ct)
|
||||||
|
}
|
||||||
|
|
||||||
case EntryType:
|
case EntryType:
|
||||||
m, typ := p.Type()
|
m, typ := p.Type()
|
||||||
|
@ -475,8 +494,10 @@ const (
|
||||||
|
|
||||||
func BenchmarkParse(b *testing.B) {
|
func BenchmarkParse(b *testing.B) {
|
||||||
for parserName, parser := range map[string]func([]byte, *labels.SymbolTable) Parser{
|
for parserName, parser := range map[string]func([]byte, *labels.SymbolTable) Parser{
|
||||||
"prometheus": NewPromParser,
|
"prometheus": NewPromParser,
|
||||||
"openmetrics": NewOpenMetricsParser,
|
"openmetrics": func(b []byte, st *labels.SymbolTable) Parser {
|
||||||
|
return NewOpenMetricsParser(b, st)
|
||||||
|
},
|
||||||
} {
|
} {
|
||||||
for _, fn := range []string{"promtestdata.txt", "promtestdata.nometa.txt"} {
|
for _, fn := range []string{"promtestdata.txt", "promtestdata.nometa.txt"} {
|
||||||
f, err := os.Open(fn)
|
f, err := os.Open(fn)
|
||||||
|
|
|
@ -68,6 +68,10 @@ func fuzzParseMetricWithContentType(in []byte, contentType string) int {
|
||||||
panic(warning)
|
panic(warning)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if contentType == "application/openmetrics-text" {
|
||||||
|
p = textparse.NewOpenMetricsParser(in, symbolTable)
|
||||||
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
for {
|
for {
|
||||||
_, err = p.Next()
|
_, err = p.Next()
|
||||||
|
|
Loading…
Reference in a new issue