From 2e7c739d4475102e894b1ac538dc4e205573158c Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Fri, 25 Oct 2024 16:46:00 +0200 Subject: [PATCH 1/4] chore: add tcp events to remote store span Signed-off-by: Juraj Michalek --- storage/remote/client.go | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/storage/remote/client.go b/storage/remote/client.go index 62218cfba..54c8b34fc 100644 --- a/storage/remote/client.go +++ b/storage/remote/client.go @@ -18,8 +18,10 @@ import ( "context" "errors" "fmt" + "go.opentelemetry.io/otel/attribute" "io" "net/http" + "net/http/httptrace" "strconv" "strings" "time" @@ -279,7 +281,36 @@ func (c *Client) Store(ctx context.Context, req []byte, attempt int) (WriteRespo ctx, span := otel.Tracer("").Start(ctx, "Remote Store", trace.WithSpanKind(trace.SpanKindClient)) defer span.End() - httpResp, err := c.Client.Do(httpReq.WithContext(ctx)) + httpReqTrace := &httptrace.ClientTrace{ + GetConn: func(hostPort string) { + span.AddEvent("GetConn", trace.WithAttributes(attribute.String("host", hostPort))) + }, + GotConn: func(info httptrace.GotConnInfo) { + span.AddEvent("GotConn", trace.WithAttributes( + attribute.Bool("reused", info.Reused), + attribute.Bool("wasIdle", info.WasIdle), + //attribute.Duration("idleTime", info.IdleTime), + )) + }, + DNSStart: func(info httptrace.DNSStartInfo) { + span.AddEvent("DNSStart", trace.WithAttributes(attribute.String("host", info.Host))) + }, + DNSDone: func(info httptrace.DNSDoneInfo) { + span.AddEvent("DNSDone", trace.WithAttributes(attribute.Bool("coalesced", info.Coalesced))) + }, + ConnectStart: func(network, addr string) { + span.AddEvent("ConnectStart", trace.WithAttributes(attribute.String("network", network), attribute.String("addr", addr))) + }, + ConnectDone: func(network, addr string, err error) { + attrs := []attribute.KeyValue{attribute.String("network", network), attribute.String("addr", addr)} + if err != nil { + attrs = append(attrs, attribute.String("error", err.Error())) + } + span.AddEvent("ConnectDone", trace.WithAttributes(attrs...)) + }, + } + + httpResp, err := c.Client.Do(httpReq.WithContext(httptrace.WithClientTrace(ctx, httpReqTrace))) if err != nil { // Errors from Client.Do are from (for example) network errors, so are // recoverable. From 3c1ffbb2fdb646ff7382d6debfe6771b5590066c Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Fri, 25 Oct 2024 17:01:52 +0200 Subject: [PATCH 2/4] chore: added idleTimeMs and fixed linting issues Signed-off-by: Juraj Michalek --- storage/remote/client.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/storage/remote/client.go b/storage/remote/client.go index 54c8b34fc..ad898cd3f 100644 --- a/storage/remote/client.go +++ b/storage/remote/client.go @@ -18,7 +18,6 @@ import ( "context" "errors" "fmt" - "go.opentelemetry.io/otel/attribute" "io" "net/http" "net/http/httptrace" @@ -26,6 +25,8 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/attribute" + "github.com/gogo/protobuf/proto" "github.com/golang/snappy" "github.com/prometheus/client_golang/prometheus" @@ -289,7 +290,7 @@ func (c *Client) Store(ctx context.Context, req []byte, attempt int) (WriteRespo span.AddEvent("GotConn", trace.WithAttributes( attribute.Bool("reused", info.Reused), attribute.Bool("wasIdle", info.WasIdle), - //attribute.Duration("idleTime", info.IdleTime), + attribute.Float64("idleTimeMs", float64(info.IdleTime.Milliseconds())), )) }, DNSStart: func(info httptrace.DNSStartInfo) { From 7ecdb55b02345dc7c52f078840c75f9ec65b7c14 Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Mon, 28 Oct 2024 16:02:09 +0100 Subject: [PATCH 3/4] chore: use otelhttptrace instead Signed-off-by: Juraj Michalek --- go.mod | 9 +++++---- go.sum | 18 ++++++++++-------- storage/remote/client.go | 41 +++++++--------------------------------- 3 files changed, 22 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index 6d33d2ed2..3399ffb00 100644 --- a/go.mod +++ b/go.mod @@ -62,13 +62,14 @@ require ( github.com/vultr/govultr/v2 v2.17.2 go.opentelemetry.io/collector/pdata v1.16.0 go.opentelemetry.io/collector/semconv v0.110.0 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 - go.opentelemetry.io/otel v1.30.0 + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 + go.opentelemetry.io/otel v1.31.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 go.opentelemetry.io/otel/sdk v1.30.0 - go.opentelemetry.io/otel/trace v1.30.0 + go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/atomic v1.11.0 go.uber.org/automaxprocs v1.6.0 go.uber.org/goleak v1.3.0 @@ -184,7 +185,7 @@ require ( github.com/xhit/go-str2duration/v2 v2.1.0 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect diff --git a/go.sum b/go.sum index 3d415cf34..1dce748ba 100644 --- a/go.sum +++ b/go.sum @@ -591,22 +591,24 @@ go.opentelemetry.io/collector/pdata v1.16.0 h1:g02K8jlRnmQ7TQDuXpdgVL6vIxIVqr5Gb go.opentelemetry.io/collector/pdata v1.16.0/go.mod h1:YZZJIt2ehxosYf/Y1pbvexjNWsIGNNrzzlCTO9jC1F4= go.opentelemetry.io/collector/semconv v0.110.0 h1:KHQnOHe3gUz0zsxe8ph9kN5OTypCFD4V+06AiBTfeNk= go.opentelemetry.io/collector/semconv v0.110.0/go.mod h1:zCJ5njhWpejR+A40kiEoeFm1xq1uzyZwMnRNX6/D82A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 h1:4BZHA+B1wXEQoGNHxW8mURaLhcdGwvRnmhGbm+odRbc= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0 h1:m0yTiGDLUvVYaTFbAvCkVYIYcvwKt3G7OLoN77NUs/8= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0/go.mod h1:wBQbT4UekBfegL2nx0Xk1vBcnzyBPsIVm9hRG4fYcr4= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 h1:umZgi92IyxfXd/l4kaDhnKgY8rnN/cZcF1LKc6I8OQ8= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0/go.mod h1:4lVs6obhSVRb1EW5FhOuBTyiQhtRtAnnva9vD3yRfq8= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= diff --git a/storage/remote/client.go b/storage/remote/client.go index ad898cd3f..d37b1505c 100644 --- a/storage/remote/client.go +++ b/storage/remote/client.go @@ -25,8 +25,6 @@ import ( "strings" "time" - "go.opentelemetry.io/otel/attribute" - "github.com/gogo/protobuf/proto" "github.com/golang/snappy" "github.com/prometheus/client_golang/prometheus" @@ -34,6 +32,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/common/sigv4" "github.com/prometheus/common/version" + "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" @@ -216,8 +215,11 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) { if conf.WriteProtoMsg != "" { writeProtoMsg = conf.WriteProtoMsg } - - httpClient.Transport = otelhttp.NewTransport(t) + httpClient.Transport = otelhttp.NewTransport( + t, + otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { + return otelhttptrace.NewClientTrace(ctx) + })) return &Client{ remoteName: name, urlString: conf.URL.String(), @@ -282,36 +284,7 @@ func (c *Client) Store(ctx context.Context, req []byte, attempt int) (WriteRespo ctx, span := otel.Tracer("").Start(ctx, "Remote Store", trace.WithSpanKind(trace.SpanKindClient)) defer span.End() - httpReqTrace := &httptrace.ClientTrace{ - GetConn: func(hostPort string) { - span.AddEvent("GetConn", trace.WithAttributes(attribute.String("host", hostPort))) - }, - GotConn: func(info httptrace.GotConnInfo) { - span.AddEvent("GotConn", trace.WithAttributes( - attribute.Bool("reused", info.Reused), - attribute.Bool("wasIdle", info.WasIdle), - attribute.Float64("idleTimeMs", float64(info.IdleTime.Milliseconds())), - )) - }, - DNSStart: func(info httptrace.DNSStartInfo) { - span.AddEvent("DNSStart", trace.WithAttributes(attribute.String("host", info.Host))) - }, - DNSDone: func(info httptrace.DNSDoneInfo) { - span.AddEvent("DNSDone", trace.WithAttributes(attribute.Bool("coalesced", info.Coalesced))) - }, - ConnectStart: func(network, addr string) { - span.AddEvent("ConnectStart", trace.WithAttributes(attribute.String("network", network), attribute.String("addr", addr))) - }, - ConnectDone: func(network, addr string, err error) { - attrs := []attribute.KeyValue{attribute.String("network", network), attribute.String("addr", addr)} - if err != nil { - attrs = append(attrs, attribute.String("error", err.Error())) - } - span.AddEvent("ConnectDone", trace.WithAttributes(attrs...)) - }, - } - - httpResp, err := c.Client.Do(httpReq.WithContext(httptrace.WithClientTrace(ctx, httpReqTrace))) + httpResp, err := c.Client.Do(httpReq.WithContext(ctx)) if err != nil { // Errors from Client.Do are from (for example) network errors, so are // recoverable. From 76ff12b32a9ec946c4c2241ea5fea148a2f952c2 Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Wed, 30 Oct 2024 09:41:16 +0100 Subject: [PATCH 4/4] chore: only create span events Signed-off-by: Juraj Michalek --- storage/remote/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/remote/client.go b/storage/remote/client.go index d37b1505c..23775122e 100644 --- a/storage/remote/client.go +++ b/storage/remote/client.go @@ -218,7 +218,7 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) { httpClient.Transport = otelhttp.NewTransport( t, otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { - return otelhttptrace.NewClientTrace(ctx) + return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans()) })) return &Client{ remoteName: name,