From abb0502685ee7ee745c84fa07f246f158fe8b170 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 2 Sep 2024 13:38:05 +0100 Subject: [PATCH] [ENHANCEMENT] PromQL: Add detail to tracing spans For aggregates, operators, calls, show what operation is performed. Also add an event when series are expanded, typically time spent accessing TSDB. Signed-off-by: Bryan Boreham --- promql/engine.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/promql/engine.go b/promql/engine.go index 537eafb41..3094ca3b2 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -991,6 +991,8 @@ func checkAndExpandSeriesSet(ctx context.Context, expr parser.Expr) (annotations if e.Series != nil { return nil, nil } + span := trace.SpanFromContext(ctx) + span.AddEvent("expand start", trace.WithAttributes(attribute.String("selector", e.String()))) series, ws, err := expandSeriesSet(ctx, e.UnexpandedSeriesSet) if e.SkipHistogramBuckets { for i := range series { @@ -998,6 +1000,7 @@ func checkAndExpandSeriesSet(ctx context.Context, expr parser.Expr) (annotations } } e.Series = series + span.AddEvent("expand end", trace.WithAttributes(attribute.Int("num_series", len(series)))) return ws, err } return nil, nil @@ -1475,6 +1478,9 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value, // Create a new span to help investigate inner evaluation performances. ctx, span := otel.Tracer("").Start(ctx, stats.InnerEvalTime.SpanOperation()+" eval "+reflect.TypeOf(expr).String()) defer span.End() + if ss, ok := expr.(interface{ ShortString() string }); ok { + span.SetAttributes(attribute.String("operation", ss.ShortString())) + } switch e := expr.(type) { case *parser.AggregateExpr: @@ -1849,11 +1855,13 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value, } case *parser.NumberLiteral: + span.SetAttributes(attribute.Float64("value", e.Val)) return ev.rangeEval(ctx, nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return append(enh.Out, Sample{F: e.Val, Metric: labels.EmptyLabels()}), nil }) case *parser.StringLiteral: + span.SetAttributes(attribute.String("value", e.Val)) return String{V: e.Val, T: ev.startTimestamp}, nil case *parser.VectorSelector: