From 080e6ed31a1abd7186d8521172a51b9f1cc78bb9 Mon Sep 17 00:00:00 2001 From: Krasi Georgiev Date: Fri, 23 Nov 2018 17:57:31 +0200 Subject: [PATCH] collect cpu and trace profiles with the promtool debug command (#4897) Signed-off-by: Krasi Georgiev --- cmd/promtool/debug.go | 120 +++++++++++------------------------------- cmd/promtool/main.go | 113 ++++++++++++++++++++++++--------------- 2 files changed, 102 insertions(+), 131 deletions(-) diff --git a/cmd/promtool/debug.go b/cmd/promtool/debug.go index 6704fbb573..280a0d44b2 100644 --- a/cmd/promtool/debug.go +++ b/cmd/promtool/debug.go @@ -14,112 +14,56 @@ package main import ( - "bytes" "fmt" + "io/ioutil" "net/http" - "os" - "github.com/google/pprof/profile" + "github.com/pkg/errors" ) type debugWriterConfig struct { serverURL string tarballName string - pathToFileName map[string]string - postProcess func(b []byte) ([]byte, error) + endPointGroups []endpointsGroup } -type debugWriter struct { - archiver - httpClient - requestToFile map[*http.Request]string - postProcess func(b []byte) ([]byte, error) -} - -func newDebugWriter(cfg debugWriterConfig) (*debugWriter, error) { - client, err := newPrometheusHTTPClient(cfg.serverURL) - if err != nil { - return nil, err - } +func debugWrite(cfg debugWriterConfig) error { archiver, err := newTarGzFileWriter(cfg.tarballName) if err != nil { - return nil, err + return errors.Wrap(err, "error creating a new archiver") } - reqs := make(map[*http.Request]string) - for path, filename := range cfg.pathToFileName { - req, err := http.NewRequest(http.MethodGet, client.urlJoin(path), nil) - if err != nil { - return nil, err - } - reqs[req] = filename - } - return &debugWriter{ - archiver, - client, - reqs, - cfg.postProcess, - }, nil -} -func (w *debugWriter) Write() int { - for req, filename := range w.requestToFile { - _, body, err := w.do(req) - if err != nil { - fmt.Fprintln(os.Stderr, "error executing HTTP request:", err) - return 1 + for _, endPointGroup := range cfg.endPointGroups { + for url, filename := range endPointGroup.urlToFilename { + url := cfg.serverURL + url + fmt.Println("collecting:", url) + res, err := http.Get(url) + if err != nil { + return errors.Wrap(err, "error executing HTTP request") + } + body, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + return errors.Wrap(err, "error reading the response body") + } + + if endPointGroup.postProcess != nil { + body, err = endPointGroup.postProcess(body) + if err != nil { + return errors.Wrap(err, "error post-processing HTTP response body") + } + } + if err := archiver.write(filename, body); err != nil { + return errors.Wrap(err, "error writing into the archive") + } } - buf, err := w.postProcess(body) - if err != nil { - fmt.Fprintln(os.Stderr, "error post-processing HTTP response body:", err) - return 1 - } - - if err := w.archiver.write(filename, buf); err != nil { - fmt.Fprintln(os.Stderr, "error writing into archive:", err) - return 1 - } } - if err := w.close(); err != nil { - fmt.Fprintln(os.Stderr, "error closing archiver:", err) - return 1 + if err := archiver.close(); err != nil { + return errors.Wrap(err, "error closing archive writer") } - fmt.Printf("Compiling debug information complete, all files written in %q.\n", w.filename()) - return 0 -} - -func validate(b []byte) (*profile.Profile, error) { - p, err := profile.Parse(bytes.NewReader(b)) - if err != nil { - return nil, err - } - return p, nil -} - -var pprofPostProcess = func(b []byte) ([]byte, error) { - p, err := validate(b) - if err != nil { - return nil, err - } - var buf bytes.Buffer - if err := p.WriteUncompressed(&buf); err != nil { - return nil, err - } - fmt.Println(p.String()) - return buf.Bytes(), nil -} - -var metricsPostProcess = func(b []byte) ([]byte, error) { - fmt.Println(string(b)) - return b, nil -} - -var allPostProcess = func(b []byte) ([]byte, error) { - _, err := validate(b) - if err != nil { - return metricsPostProcess(b) - } - return pprofPostProcess(b) + fmt.Printf("Compiling debug information complete, all files written in %q.\n", cfg.tarballName) + return nil } diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go index 6033219933..7a9192ea62 100644 --- a/cmd/promtool/main.go +++ b/cmd/promtool/main.go @@ -14,6 +14,7 @@ package main import ( + "bytes" "context" "encoding/json" "fmt" @@ -27,6 +28,8 @@ import ( "gopkg.in/alecthomas/kingpin.v2" + "github.com/google/pprof/profile" + "github.com/pkg/errors" "github.com/prometheus/client_golang/api" "github.com/prometheus/client_golang/api/prometheus/v1" config_util "github.com/prometheus/common/config" @@ -510,61 +513,85 @@ func parseTime(s string) (time.Time, error) { return time.Time{}, fmt.Errorf("cannot parse %q to a valid timestamp", s) } -func debugPprof(url string) int { - w, err := newDebugWriter(debugWriterConfig{ - serverURL: url, - tarballName: "debug.tar.gz", - pathToFileName: map[string]string{ - "/debug/pprof/block": "block.pb", - "/debug/pprof/goroutine": "goroutine.pb", - "/debug/pprof/heap": "heap.pb", - "/debug/pprof/mutex": "mutex.pb", - "/debug/pprof/threadcreate": "threadcreate.pb", +type endpointsGroup struct { + urlToFilename map[string]string + postProcess func(b []byte) ([]byte, error) +} + +var ( + pprofEndpoints = []endpointsGroup{ + { + urlToFilename: map[string]string{ + "/debug/pprof/profile?seconds=30": "cpu.pb", + "/debug/pprof/block": "block.pb", + "/debug/pprof/goroutine": "goroutine.pb", + "/debug/pprof/heap": "heap.pb", + "/debug/pprof/mutex": "mutex.pb", + "/debug/pprof/threadcreate": "threadcreate.pb", + }, + postProcess: func(b []byte) ([]byte, error) { + p, err := profile.Parse(bytes.NewReader(b)) + if err != nil { + return nil, err + } + var buf bytes.Buffer + if err := p.WriteUncompressed(&buf); err != nil { + return nil, errors.Wrap(err, "writing the profile to the buffer") + } + + return buf.Bytes(), nil + }, }, - postProcess: pprofPostProcess, - }) - if err != nil { - fmt.Fprintln(os.Stderr, "error creating debug writer:", err) + { + urlToFilename: map[string]string{ + "/debug/pprof/trace?seconds=30": "trace.pb", + }, + }, + } + metricsEndpoints = []endpointsGroup{ + { + urlToFilename: map[string]string{ + "/metrics": "metrics.txt", + }, + }, + } + allEndpoints = append(pprofEndpoints, metricsEndpoints...) +) + +func debugPprof(url string) int { + if err := debugWrite(debugWriterConfig{ + serverURL: url, + tarballName: "debug.tar.gz", + endPointGroups: pprofEndpoints, + }); err != nil { + fmt.Fprintln(os.Stderr, "error completing debug command:", err) return 1 } - return w.Write() + return 0 } func debugMetrics(url string) int { - w, err := newDebugWriter(debugWriterConfig{ - serverURL: url, - tarballName: "debug.tar.gz", - pathToFileName: map[string]string{ - "/metrics": "metrics.txt", - }, - postProcess: metricsPostProcess, - }) - if err != nil { - fmt.Fprintln(os.Stderr, "error creating debug writer:", err) + if err := debugWrite(debugWriterConfig{ + serverURL: url, + tarballName: "debug.tar.gz", + endPointGroups: metricsEndpoints, + }); err != nil { + fmt.Fprintln(os.Stderr, "error completing debug command:", err) return 1 } - return w.Write() + return 0 } func debugAll(url string) int { - w, err := newDebugWriter(debugWriterConfig{ - serverURL: url, - tarballName: "debug.tar.gz", - pathToFileName: map[string]string{ - "/debug/pprof/block": "block.pb", - "/debug/pprof/goroutine": "goroutine.pb", - "/debug/pprof/heap": "heap.pb", - "/debug/pprof/mutex": "mutex.pb", - "/debug/pprof/threadcreate": "threadcreate.pb", - "/metrics": "metrics.txt", - }, - postProcess: allPostProcess, - }) - if err != nil { - fmt.Fprintln(os.Stderr, "error creating debug writer:", err) + if err := debugWrite(debugWriterConfig{ + serverURL: url, + tarballName: "debug.tar.gz", + endPointGroups: allEndpoints, + }); err != nil { + fmt.Fprintln(os.Stderr, "error completing debug command:", err) return 1 } - return w.Write() + return 0 } type printer interface { @@ -583,7 +610,7 @@ func (p *promqlPrinter) printSeries(val []model.LabelSet) { fmt.Println(v) } } -func (j *promqlPrinter) printLabelValues(val model.LabelValues) { +func (p *promqlPrinter) printLabelValues(val model.LabelValues) { for _, v := range val { fmt.Println(v) }