From bb86c3f62ba95145013506856fcba65e8c93a75a Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 21 Mar 2018 17:08:37 +0100 Subject: [PATCH] Report internal runtime information on status page (#3921) Add information about tsdb, wal and config reload --- cmd/prometheus/main.go | 10 +++--- web/ui/bindata.go | 6 ++-- web/ui/templates/status.html | 22 +++++++++++++- web/web.go | 59 ++++++++++++++++++++++++++++++------ 4 files changed, 78 insertions(+), 19 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index f54927299..962cd9a57 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -61,14 +61,12 @@ import ( var ( configSuccess = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "prometheus", - Name: "config_last_reload_successful", - Help: "Whether the last configuration reload attempt was successful.", + Name: "prometheus_config_last_reload_successful", + Help: "Whether the last configuration reload attempt was successful.", }) configSuccessTime = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "prometheus", - Name: "config_last_reload_success_timestamp_seconds", - Help: "Timestamp of the last successful configuration reload.", + Name: "prometheus_config_last_reload_success_timestamp_seconds", + Help: "Timestamp of the last successful configuration reload.", }) ) diff --git a/web/ui/bindata.go b/web/ui/bindata.go index c5341915f..e753f4845 100644 --- a/web/ui/bindata.go +++ b/web/ui/bindata.go @@ -246,12 +246,12 @@ func webUiTemplatesServiceDiscoveryHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "web/ui/templates/service-discovery.html", size: 2547, mode: os.FileMode(420), modTime: time.Unix(1520602638, 0)} + info := bindataFileInfo{name: "web/ui/templates/service-discovery.html", size: 2547, mode: os.FileMode(420), modTime: time.Unix(1521019344, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _webUiTemplatesStatusHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x55\xc1\x8e\xda\x30\x10\xbd\xe7\x2b\xa6\x3e\xae\x1a\x2c\xed\xb1\x0a\x91\x76\xa1\xa2\x95\xda\xee\x8a\x2d\xdd\x5e\x4d\x3c\x10\xab\xc1\x46\x63\x87\x2d\xb2\xfc\xef\x55\x02\x09\x89\x04\xcb\x52\x0e\xed\x25\xf1\xf3\x3c\xbd\x37\x9a\x19\xdb\xde\x4b\x5c\x28\x8d\xc0\x72\x14\x92\x85\x90\xbc\x8b\x63\xd0\xea\x37\xc4\x71\xea\x3d\x6a\x19\x42\x14\x1d\x58\x99\xd1\x0e\xb5\x63\x21\x44\x00\x89\x54\x1b\xc8\x0a\x61\xed\xb0\x0e\x08\xa5\x91\xe2\x45\x51\x2a\xc9\xd2\x08\x00\x20\xc9\x6f\x41\xc9\x21\xa3\x52\x3b\xb5\x42\x96\x4e\x77\x0b\xf8\xac\x17\x86\x56\xc2\x29\xa3\x13\x9e\xdf\xee\xd9\x4e\xcc\x0b\x6c\x14\x77\xa0\xfe\xc6\x99\xd1\x12\xb5\x45\xb9\xc7\x73\x43\x12\xa9\x85\xd6\x91\x5a\xb7\x28\x37\x1b\xa4\x7d\x02\x95\xe8\xdc\xc8\x6d\x83\x2a\x4c\x07\x50\xc1\x3c\x9d\xad\xab\x9c\x12\xee\xf2\x7e\x44\xa6\xde\x0f\xee\x15\xb9\x7c\x30\xfb\x3e\x0a\x21\xe1\x4e\x76\x84\x78\x57\xe9\x88\xec\xb3\xa1\x5f\x4a\x2f\x61\xac\x08\x33\x67\x68\x7b\xc2\x61\xf4\x3c\xbe\x58\xfb\x5b\xb9\x9a\x23\x81\x59\xc0\xd2\x90\x29\x9d\xd2\x68\x4f\xc8\x4f\x1a\xc2\xc8\x94\xda\x5d\xec\x34\x79\xf8\x7a\xf7\xf3\x71\xfa\x30\x7a\x3a\xa5\xdf\x12\xfe\x42\x7b\x32\x3a\xa9\x3a\x79\xb5\xe2\x09\xef\xf4\x35\xe1\x75\xe7\xd3\xa8\x37\x74\xf3\x52\x15\x52\x1d\x06\x8d\xa5\xf7\xd5\xce\x7f\x35\x7b\x60\x33\xb3\xc6\x21\x23\xf3\xc2\xd2\x1f\x48\xb6\x4e\xea\x68\x45\xf6\xd1\xe6\x7f\x69\xb1\x7b\x4e\x53\xdc\xa8\x37\x58\x35\xb4\xab\xbc\xee\x49\xe8\x2c\x3f\xe3\xb4\x23\x5d\xe7\x53\x35\x77\x66\x91\xce\x59\x35\xbc\xeb\xdd\xc6\xc2\x9d\xba\x36\x7a\x6e\x15\xef\x2a\xb7\x89\x79\xdb\x6c\xb4\xbc\x2b\x8f\x8e\x28\x90\xdc\x4a\x68\xb1\x44\xb2\x2c\xbd\xeb\xc2\x7f\x7b\x66\xea\x7b\xe3\xa3\x96\x6b\xa3\xb4\xeb\x57\xa3\x5f\x51\xef\x49\xe8\x25\xc2\xa0\x97\x7c\xfd\x6e\x1d\x15\xf6\x9e\xdf\x40\x97\x0b\xb3\xe9\x17\x0b\xa2\x78\x11\x5b\x0b\xb9\xd8\x20\x3c\x65\x39\xae\xf0\x3d\x7c\x32\xd6\x81\xd0\x12\x1e\x45\xd5\x27\x74\x70\xc3\x3b\xc2\x6d\x57\x76\xfc\x10\x3e\x70\x9e\x08\xc8\x09\x17\x43\xd6\xdf\xf6\x7e\x50\x89\x85\xc0\xd2\x76\x99\x70\x51\x81\x4a\xfb\xf5\x99\x69\x9e\xe7\x33\x5d\x4d\xb8\x54\x9b\x34\x6a\xd8\x7f\x02\x00\x00\xff\xff\x6f\x51\x02\x01\xf2\x07\x00\x00") +var _webUiTemplatesStatusHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x56\x41\x6f\xe2\x3c\x10\xbd\xf3\x2b\xe6\xf3\xb1\xfa\x20\x52\x8f\x9f\x42\xa4\x96\x7e\x4b\x57\xea\xaa\x15\x94\xed\x5e\x4d\x3c\x21\x56\x83\x8d\xc6\x0e\xdd\xca\xf2\x7f\x5f\x39\x24\x29\x59\x91\xd2\xc2\x61\xf7\x02\x79\xf1\xcb\x7b\x2f\x33\xce\x24\xce\x09\xcc\xa4\x42\x60\x39\x72\xc1\xbc\x8f\xff\x19\x0e\x41\xc9\x9f\x30\x1c\x26\xce\xa1\x12\xde\x0f\x06\x6f\xac\x54\x2b\x8b\xca\x32\xef\x07\x00\xb1\x90\x5b\x48\x0b\x6e\xcc\xb8\x5a\xe0\x52\x21\x0d\xb3\xa2\x94\x82\x25\x03\x00\x80\x38\xbf\x04\x29\xc6\x8c\x4a\x65\xe5\x1a\x59\x32\xdb\x1d\xc0\x57\x95\x69\x5a\x73\x2b\xb5\x8a\xa3\xfc\xb2\x66\x5b\xbe\x2c\xb0\x51\xdc\x81\xea\x77\x98\x6a\x25\x50\x19\x14\x35\x5e\x6a\x12\x48\x2d\x34\x96\xe4\xa6\x45\xb9\xde\x22\xd5\x01\x82\xe8\x52\x8b\xd7\x06\x05\x4c\x6f\x20\xc0\x3c\x59\x6c\x42\xa6\x38\xb2\x79\x77\x45\x24\xce\x8d\xae\x25\xd9\x7c\xb4\x78\x9c\x78\x1f\x47\x56\xec\x09\x45\xfb\x4a\x07\x64\x9f\x34\x3d\x4b\xb5\x82\x1b\x49\x98\x5a\x4d\xaf\x3d\x0e\x93\xa7\x9b\xa3\xda\xce\xc9\x0c\x94\xb6\x30\x9a\x61\xa1\xb9\x98\x68\x95\xc9\xd5\xbc\x4c\x53\x34\xc6\xfb\xa6\x66\x82\xab\x15\x12\xab\x1b\xf7\x7b\xa0\xdd\x45\x25\x55\x65\x07\xaa\x84\x0e\x67\x92\x59\x8f\x51\x7d\x90\x95\x85\x73\x58\x18\xf4\xfe\x0b\x97\x05\x8a\xda\xf1\x93\x15\xba\xe3\xc6\x82\x69\x25\x21\xfd\x70\xc0\x51\xb8\x74\x97\xed\x51\xae\xf1\xa4\xfe\xdc\x22\x17\x90\xe6\xa5\x7a\x36\x7d\x9d\x09\x8b\x13\x5d\x2a\x7b\x9a\x78\xb5\xd5\x0d\x92\xc4\x3e\x87\x10\x7e\x5e\x11\x4e\xb3\x79\xba\xba\x83\x54\x13\x95\x9b\x50\xb2\xde\xfb\x68\x19\xa7\xb9\x4c\x35\xe9\xd2\x4a\xd5\x7b\x1b\x2d\xe1\x44\xfd\xfb\x6f\x57\x3f\x1e\x66\xf7\x93\x79\x9f\x7e\x4b\x38\x41\x7b\x3a\xe9\x55\x9d\xbe\xbb\x6b\xe2\x68\x6f\x76\xc4\x51\x35\x5d\x92\x41\x67\xb0\x2d\x4b\x59\x08\xf9\x36\xcc\x58\x72\x1d\xce\xfc\x55\xf3\x0d\x4c\xaa\x37\x38\x66\xa4\x5f\x58\xf2\x1d\xc9\x54\xa1\x0e\x56\xa4\x5e\x6d\xfe\x3f\x5b\xec\x8e\xd3\x0c\xb7\xf2\x03\x56\x0d\xed\x2c\xaf\x6b\xe2\x2a\xcd\x8f\x38\xed\x48\xe7\xf9\x84\xe6\x2e\x0c\xd2\x31\xab\x86\x77\xbe\xdb\x0d\xb7\x7d\xaf\xa6\x8e\x5b\xe0\x9d\xe5\x36\xd5\x1f\xdb\x1b\x2d\xef\xcc\x47\x87\x17\x48\x76\xcd\x15\x5f\x21\x19\x96\x5c\xed\xc3\x3f\xfb\xcc\x54\x73\xe3\x7f\x25\x36\x5a\x2a\xdb\xad\x46\xb7\xa2\xce\x51\x78\xe1\xc2\xa8\x13\xbe\xfa\x36\x3a\x28\xec\x5c\x74\x01\xfb\x5c\x58\xcc\xee\x0c\xf0\xe2\x85\xbf\x1a\xc8\xf9\x16\x61\x9e\xe6\xb8\xc6\x7f\xe1\x56\x1b\x0b\x5c\x09\x78\xe0\xa1\x4f\x68\xe1\x22\xda\x13\x6e\xbb\xb2\xe3\x7b\xff\x5f\x14\xc5\x1c\x72\xc2\x6c\xcc\xba\xa7\x9d\x1b\x05\x31\xef\x59\xd2\x1e\xc6\x11\x0f\x20\x68\xbf\xbf\x67\x9a\x4f\xc0\x23\x5d\x8d\x23\x21\xb7\xc9\xa0\x61\xff\x0a\x00\x00\xff\xff\xe5\xe4\xe6\xaa\x56\x0a\x00\x00") func webUiTemplatesStatusHtmlBytes() ([]byte, error) { return bindataRead( @@ -266,7 +266,7 @@ func webUiTemplatesStatusHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "web/ui/templates/status.html", size: 2034, mode: os.FileMode(420), modTime: time.Unix(1520481515, 0)} + info := bindataFileInfo{name: "web/ui/templates/status.html", size: 2646, mode: os.FileMode(420), modTime: time.Unix(1521019352, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/web/ui/templates/status.html b/web/ui/templates/status.html index dd4022002..18c4cb9aa 100644 --- a/web/ui/templates/status.html +++ b/web/ui/templates/status.html @@ -13,8 +13,28 @@ Working Directory {{.CWD}} + + Configuration reload + {{if .ReloadConfigSuccess}}Successful{{else}}Failed{{end}} + - Number of goroutines + Last successful configuration reload + {{.LastConfigTime.UTC}} + + + Head chunks + {{.ChunkCount}} + + + Head time series + {{.TimeSeriesCount}} + + + WAL corruptions + {{.CorruptionCount}} + + + Goroutines {{.GoroutineCount}} diff --git a/web/web.go b/web/web.go index 5bf004958..a2212ad69 100644 --- a/web/web.go +++ b/web/web.go @@ -21,6 +21,7 @@ import ( "io" "io/ioutil" stdlog "log" + "math" "net" "net/http" "net/http/pprof" @@ -48,6 +49,7 @@ import ( "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/client_model/go" "github.com/prometheus/common/model" "github.com/prometheus/common/route" "github.com/prometheus/tsdb" @@ -587,14 +589,19 @@ func (h *Handler) graph(w http.ResponseWriter, r *http.Request) { } func (h *Handler) status(w http.ResponseWriter, r *http.Request) { - h.executeTemplate(w, "status.html", struct { - Birth time.Time - CWD string - Version *PrometheusVersion - Alertmanagers []*url.URL - GoroutineCount int - GOMAXPROCS int - GOGC string + status := struct { + Birth time.Time + CWD string + Version *PrometheusVersion + Alertmanagers []*url.URL + GoroutineCount int + GOMAXPROCS int + GOGC string + CorruptionCount int64 + ChunkCount int64 + TimeSeriesCount int64 + LastConfigTime time.Time + ReloadConfigSuccess bool }{ Birth: h.birth, CWD: h.cwd, @@ -603,7 +610,41 @@ func (h *Handler) status(w http.ResponseWriter, r *http.Request) { GoroutineCount: runtime.NumGoroutine(), GOMAXPROCS: runtime.GOMAXPROCS(0), GOGC: os.Getenv("GOGC"), - }) + } + metrics, err := prometheus.DefaultGatherer.Gather() + if err != nil { + http.Error(w, fmt.Sprintf("error gathering runtime status: %s", err), http.StatusInternalServerError) + return + } + for _, mF := range metrics { + switch *mF.Name { + case "prometheus_tsdb_head_chunks": + status.ChunkCount = int64(toFloat64(mF)) + case "prometheus_tsdb_head_series": + status.TimeSeriesCount = int64(toFloat64(mF)) + case "prometheus_tsdb_wal_corruptions_total": + status.CorruptionCount = int64(toFloat64(mF)) + case "prometheus_config_last_reload_successful": + status.ReloadConfigSuccess = toFloat64(mF) != 0 + case "prometheus_config_last_reload_success_timestamp_seconds": + status.LastConfigTime = time.Unix(int64(toFloat64(mF)), 0) + } + } + h.executeTemplate(w, "status.html", status) +} + +func toFloat64(f *io_prometheus_client.MetricFamily) float64 { + m := *f.Metric[0] + if m.Gauge != nil { + return m.Gauge.GetValue() + } + if m.Counter != nil { + return m.Counter.GetValue() + } + if m.Untyped != nil { + return m.Untyped.GetValue() + } + return math.NaN() } func (h *Handler) flags(w http.ResponseWriter, r *http.Request) {