prometheus/web/api/v1/api.go

1539 lines
45 KiB
Go
Raw Normal View History

// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"context"
"fmt"
"math"
"math/rand"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
2017-10-26 03:44:49 -07:00
"sort"
"strconv"
"strings"
"time"
"unsafe"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
jsoniter "github.com/json-iterator/go"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
2015-09-24 08:07:11 -07:00
"github.com/prometheus/common/route"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/pkg/gate"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/textparse"
"github.com/prometheus/prometheus/pkg/timestamp"
2017-10-23 13:28:17 -07:00
"github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/promql/parser"
"github.com/prometheus/prometheus/rules"
"github.com/prometheus/prometheus/scrape"
"github.com/prometheus/prometheus/storage"
2017-10-23 13:28:17 -07:00
"github.com/prometheus/prometheus/storage/remote"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/tsdb/index"
"github.com/prometheus/prometheus/util/httputil"
"github.com/prometheus/prometheus/util/stats"
)
const (
namespace = "prometheus"
subsystem = "api"
)
type status string
const (
statusSuccess status = "success"
2017-12-02 05:52:43 -08:00
statusError status = "error"
)
type errorType string
const (
errorNone errorType = ""
errorTimeout errorType = "timeout"
errorCanceled errorType = "canceled"
errorExec errorType = "execution"
errorBadData errorType = "bad_data"
errorInternal errorType = "internal"
errorUnavailable errorType = "unavailable"
errorNotFound errorType = "not_found"
)
var (
LocalhostRepresentations = []string{"127.0.0.1", "localhost"}
remoteReadQueries = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "remote_read_queries",
Help: "The current number of remote read queries being executed or waiting.",
})
)
type apiError struct {
typ errorType
err error
}
func (e *apiError) Error() string {
return fmt.Sprintf("%s: %s", e.typ, e.err)
}
2016-12-02 04:31:43 -08:00
type targetRetriever interface {
TargetsActive() map[string][]*scrape.Target
TargetsDropped() map[string][]*scrape.Target
2016-12-02 04:31:43 -08:00
}
2017-01-13 01:20:11 -08:00
type alertmanagerRetriever interface {
Alertmanagers() []*url.URL
DroppedAlertmanagers() []*url.URL
2016-12-02 04:31:43 -08:00
}
type rulesRetriever interface {
RuleGroups() []*rules.Group
AlertingRules() []*rules.AlertingRule
}
2019-11-02 08:53:32 -07:00
// PrometheusVersion contains build information about Prometheus.
type PrometheusVersion struct {
Version string `json:"version"`
Revision string `json:"revision"`
Branch string `json:"branch"`
BuildUser string `json:"buildUser"`
BuildDate string `json:"buildDate"`
GoVersion string `json:"goVersion"`
}
// RuntimeInfo contains runtime information about Prometheus.
type RuntimeInfo struct {
StartTime time.Time `json:"startTime"`
CWD string `json:"CWD"`
ReloadConfigSuccess bool `json:"reloadConfigSuccess"`
LastConfigTime time.Time `json:"lastConfigTime"`
ChunkCount int64 `json:"chunkCount"`
TimeSeriesCount int64 `json:"timeSeriesCount"`
CorruptionCount int64 `json:"corruptionCount"`
GoroutineCount int `json:"goroutineCount"`
GOMAXPROCS int `json:"GOMAXPROCS"`
GOGC string `json:"GOGC"`
GODEBUG string `json:"GODEBUG"`
StorageRetention string `json:"storageRetention"`
}
type response struct {
Status status `json:"status"`
Data interface{} `json:"data,omitempty"`
ErrorType errorType `json:"errorType,omitempty"`
Error string `json:"error,omitempty"`
Warnings []string `json:"warnings,omitempty"`
}
type apiFuncResult struct {
data interface{}
err *apiError
warnings storage.Warnings
finalizer func()
}
type apiFunc func(r *http.Request) apiFuncResult
// TSDBAdmin defines the tsdb interfaces used by the v1 API for admin operations.
type TSDBAdmin interface {
CleanTombstones() error
Delete(mint, maxt int64, ms ...*labels.Matcher) error
Dir() string
Snapshot(dir string, withHead bool) error
Head() *tsdb.Head
}
// API can register a set of endpoints in a router and handle
// them using the provided storage and query engine.
type API struct {
Queryable storage.Queryable
QueryEngine *promql.Engine
2017-01-13 01:20:11 -08:00
targetRetriever targetRetriever
alertmanagerRetriever alertmanagerRetriever
rulesRetriever rulesRetriever
now func() time.Time
config func() config.Config
flagsMap map[string]string
ready func(http.HandlerFunc) http.HandlerFunc
globalURLOptions GlobalURLOptions
db func() TSDBAdmin
enableAdmin bool
logger log.Logger
remoteReadSampleLimit int
remoteReadMaxBytesInFrame int
remoteReadGate *gate.Gate
CORSOrigin *regexp.Regexp
2019-11-02 08:53:32 -07:00
buildInfo *PrometheusVersion
runtimeInfo func() (RuntimeInfo, error)
}
func init() {
jsoniter.RegisterTypeEncoderFunc("promql.Point", marshalPointJSON, marshalPointJSONIsEmpty)
prometheus.MustRegister(remoteReadQueries)
}
// NewAPI returns an initialized API type.
func NewAPI(
qe *promql.Engine,
q storage.Queryable,
tr targetRetriever,
ar alertmanagerRetriever,
configFunc func() config.Config,
flagsMap map[string]string,
globalURLOptions GlobalURLOptions,
readyFunc func(http.HandlerFunc) http.HandlerFunc,
db func() TSDBAdmin,
enableAdmin bool,
logger log.Logger,
rr rulesRetriever,
remoteReadSampleLimit int,
remoteReadConcurrencyLimit int,
remoteReadMaxBytesInFrame int,
CORSOrigin *regexp.Regexp,
2019-11-02 08:53:32 -07:00
runtimeInfo func() (RuntimeInfo, error),
buildInfo *PrometheusVersion,
) *API {
return &API{
2017-01-13 01:20:11 -08:00
QueryEngine: qe,
2017-07-06 05:38:40 -07:00
Queryable: q,
2017-01-13 01:20:11 -08:00
targetRetriever: tr,
alertmanagerRetriever: ar,
now: time.Now,
config: configFunc,
flagsMap: flagsMap,
ready: readyFunc,
globalURLOptions: globalURLOptions,
db: db,
enableAdmin: enableAdmin,
rulesRetriever: rr,
remoteReadSampleLimit: remoteReadSampleLimit,
remoteReadGate: gate.New(remoteReadConcurrencyLimit),
remoteReadMaxBytesInFrame: remoteReadMaxBytesInFrame,
logger: logger,
CORSOrigin: CORSOrigin,
2019-11-02 08:53:32 -07:00
runtimeInfo: runtimeInfo,
buildInfo: buildInfo,
}
}
// Register the API's endpoints in the given router.
func (api *API) Register(r *route.Router) {
wrap := func(f apiFunc) http.HandlerFunc {
hf := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
httputil.SetCORS(w, api.CORSOrigin, r)
result := f(r)
if result.err != nil {
api.respondError(w, result.err, result.data)
} else if result.data != nil {
api.respond(w, result.data, result.warnings)
} else {
w.WriteHeader(http.StatusNoContent)
}
if result.finalizer != nil {
result.finalizer()
Optimise PromQL (#3966) * Move range logic to 'eval' Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make aggregegate range aware Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * PromQL is statically typed, so don't eval to find the type. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Extend rangewrapper to multiple exprs Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Start making function evaluation ranged Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make instant queries a special case of range queries Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Eliminate evalString Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Evaluate range vector functions one series at a time Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make unary operators range aware Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make binops range aware Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Pass time to range-aware functions. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make simple _over_time functions range aware Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Reduce allocs when working with matrix selectors Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Add basic benchmark for range evaluation Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Reuse objects for function arguments Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Do dropmetricname and allocating output vector only once. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Add range-aware support for range vector functions with params Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Optimise holt_winters, cut cpu and allocs by ~25% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make rate&friends range aware Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make more functions range aware. Document calling convention. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make date functions range aware Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make simple math functions range aware Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Convert more functions to be range aware Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make more functions range aware Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Specialcase timestamp() with vector selector arg for range awareness Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Remove transition code for functions Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Remove the rest of the engine transition code Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Remove more obselete code Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Remove the last uses of the eval* functions Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Remove engine finalizers to prevent corruption The finalizers set by matrixSelector were being called just before the value they were retruning to the pool was then being provided to the caller. Thus a concurrent query could corrupt the data that the user has just been returned. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Add new benchmark suite for range functinos Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Migrate existing benchmarks to new system Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Expand promql benchmarks Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Simply test by removing unused range code Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * When testing instant queries, check range queries too. To protect against subsequent steps in a range query being affected by the previous steps, add a test that evaluates an instant query that we know works again as a range query with the tiimestamp we care about not being the first step. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Reuse ring for matrix iters. Put query results back in pool. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Reuse buffer when iterating over matrix selectors Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Unary minus should remove metric name Cut down benchmarks for faster runs. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Reduce repetition in benchmark test cases Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Work series by series when doing normal vectorSelectors Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Optimise benchmark setup, cuts time by 60% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Have rangeWrapper use an evalNodeHelper to cache across steps Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Use evalNodeHelper with functions Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Cache dropMetricName within a node evaluation. This saves both the calculations and allocs done by dropMetricName across steps. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Reuse input vectors in rangewrapper Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Reuse the point slices in the matrixes input/output by rangeWrapper Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make benchmark setup faster using AddFast Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Simplify benchmark code. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Add caching in VectorBinop Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Use xor to have one-level resultMetric hash key Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Add more benchmarks Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Call Query.Close in apiv1 This allows point slices allocated for the response data to be reused by later queries, saving allocations. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Optimise histogram_quantile It's now 5-10% faster with 97% less garbage generated for 1k steps Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make the input collection in rangeVector linear rather than quadratic Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Optimise label_replace, for 1k steps 15x fewer allocs and 3x faster Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Optimise label_join, 1.8x faster and 11x less memory for 1k steps Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Expand benchmarks, cleanup comments, simplify numSteps logic. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Address Fabian's comments Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Comments from Alin. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Address jrv's comments Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Remove dead code Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Address Simon's comments. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Rename populateIterators, pre-init some sizes Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Handle case where function has non-matrix args first Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Split rangeWrapper out to rangeEval function, improve comments Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Cleanup and make things more consistent Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Make EvalNodeHelper public Signed-off-by: Brian Brazil <brian.brazil@robustperception.io> * Fabian's comments. Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2018-06-04 06:47:45 -07:00
}
})
return api.ready(httputil.CompressionHandler{
2015-09-18 07:51:53 -07:00
Handler: hf,
}.ServeHTTP)
}
r.Options("/*path", wrap(api.options))
r.Get("/query", wrap(api.query))
r.Post("/query", wrap(api.query))
r.Get("/query_range", wrap(api.queryRange))
r.Post("/query_range", wrap(api.queryRange))
r.Get("/labels", wrap(api.labelNames))
r.Post("/labels", wrap(api.labelNames))
r.Get("/label/:name/values", wrap(api.labelValues))
r.Get("/series", wrap(api.series))
r.Post("/series", wrap(api.series))
r.Del("/series", wrap(api.dropSeries))
2016-12-02 04:31:43 -08:00
r.Get("/targets", wrap(api.targets))
r.Get("/targets/metadata", wrap(api.targetMetadata))
r.Get("/alertmanagers", wrap(api.alertmanagers))
r.Get("/metadata", wrap(api.metricMetadata))
r.Get("/status/config", wrap(api.serveConfig))
2019-11-02 08:53:32 -07:00
r.Get("/status/runtimeinfo", wrap(api.serveRuntimeInfo))
r.Get("/status/buildinfo", wrap(api.serveBuildInfo))
r.Get("/status/flags", wrap(api.serveFlags))
r.Get("/status/tsdb", wrap(api.serveTSDBStatus))
r.Post("/read", api.ready(http.HandlerFunc(api.remoteRead)))
r.Get("/alerts", wrap(api.alerts))
r.Get("/rules", wrap(api.rules))
// Admin APIs
r.Post("/admin/tsdb/delete_series", wrap(api.deleteSeries))
r.Post("/admin/tsdb/clean_tombstones", wrap(api.cleanTombstones))
r.Post("/admin/tsdb/snapshot", wrap(api.snapshot))
r.Put("/admin/tsdb/delete_series", wrap(api.deleteSeries))
r.Put("/admin/tsdb/clean_tombstones", wrap(api.cleanTombstones))
r.Put("/admin/tsdb/snapshot", wrap(api.snapshot))
}
type queryData struct {
ResultType parser.ValueType `json:"resultType"`
Result parser.Value `json:"result"`
Stats *stats.QueryStats `json:"stats,omitempty"`
}
func (api *API) options(r *http.Request) apiFuncResult {
return apiFuncResult{nil, nil, nil, nil}
}
func (api *API) query(r *http.Request) apiFuncResult {
ts, err := parseTimeParam(r, "time", api.now())
if err != nil {
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
ctx := r.Context()
2017-03-06 09:32:21 -08:00
if to := r.FormValue("timeout"); to != "" {
var cancel context.CancelFunc
timeout, err := parseDuration(to)
if err != nil {
err = errors.Wrapf(err, "invalid parameter 'timeout'")
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
2017-03-06 09:32:21 -08:00
}
ctx, cancel = context.WithTimeout(ctx, timeout)
defer cancel()
}
qry, err := api.QueryEngine.NewInstantQuery(api.Queryable, r.FormValue("query"), ts)
if err != nil {
err = errors.Wrapf(err, "invalid parameter 'query'")
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
ctx = httputil.ContextFromRequest(ctx, r)
2017-03-06 09:32:21 -08:00
res := qry.Exec(ctx)
if res.Err != nil {
return apiFuncResult{nil, returnAPIError(res.Err), res.Warnings, qry.Close}
}
// Optional stats field in response if parameter "stats" is not empty.
var qs *stats.QueryStats
if r.FormValue("stats") != "" {
qs = stats.NewQueryStats(qry.Stats())
}
return apiFuncResult{&queryData{
ResultType: res.Value.Type(),
Result: res.Value,
Stats: qs,
}, nil, res.Warnings, qry.Close}
}
func (api *API) queryRange(r *http.Request) apiFuncResult {
start, err := parseTime(r.FormValue("start"))
if err != nil {
err = errors.Wrapf(err, "invalid parameter 'start'")
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
end, err := parseTime(r.FormValue("end"))
if err != nil {
err = errors.Wrapf(err, "invalid parameter 'end'")
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
if end.Before(start) {
err := errors.New("end timestamp must not be before start time")
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
step, err := parseDuration(r.FormValue("step"))
if err != nil {
err = errors.Wrapf(err, "invalid parameter 'step'")
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
if step <= 0 {
err := errors.New("zero or negative query resolution step widths are not accepted. Try a positive integer")
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
// For safety, limit the number of returned points per timeseries.
// This is sufficient for 60s resolution for a week or 1h resolution for a year.
if end.Sub(start)/step > 11000 {
err := errors.New("exceeded maximum resolution of 11,000 points per timeseries. Try decreasing the query resolution (?step=XX)")
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
ctx := r.Context()
2017-03-06 09:32:21 -08:00
if to := r.FormValue("timeout"); to != "" {
var cancel context.CancelFunc
timeout, err := parseDuration(to)
if err != nil {
err = errors.Wrap(err, "invalid parameter 'timeout'")
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
2017-03-06 09:32:21 -08:00
}
ctx, cancel = context.WithTimeout(ctx, timeout)
defer cancel()
}
qry, err := api.QueryEngine.NewRangeQuery(api.Queryable, r.FormValue("query"), start, end, step)
if err != nil {
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
ctx = httputil.ContextFromRequest(ctx, r)
2017-03-06 09:32:21 -08:00
res := qry.Exec(ctx)
if res.Err != nil {
return apiFuncResult{nil, returnAPIError(res.Err), res.Warnings, qry.Close}
}
// Optional stats field in response if parameter "stats" is not empty.
var qs *stats.QueryStats
if r.FormValue("stats") != "" {
qs = stats.NewQueryStats(qry.Stats())
}
return apiFuncResult{&queryData{
ResultType: res.Value.Type(),
Result: res.Value,
Stats: qs,
}, nil, res.Warnings, qry.Close}
}
func returnAPIError(err error) *apiError {
if err == nil {
return nil
}
switch err.(type) {
case promql.ErrQueryCanceled:
return &apiError{errorCanceled, err}
case promql.ErrQueryTimeout:
return &apiError{errorTimeout, err}
case promql.ErrStorage:
return &apiError{errorInternal, err}
}
return &apiError{errorExec, err}
}
func (api *API) labelNames(r *http.Request) apiFuncResult {
q, err := api.Queryable.Querier(r.Context(), math.MinInt64, math.MaxInt64)
if err != nil {
return apiFuncResult{nil, &apiError{errorExec, err}, nil, nil}
}
defer q.Close()
names, warnings, err := q.LabelNames()
if err != nil {
return apiFuncResult{nil, &apiError{errorExec, err}, warnings, nil}
}
return apiFuncResult{names, nil, warnings, nil}
}
func (api *API) labelValues(r *http.Request) apiFuncResult {
ctx := r.Context()
name := route.Param(ctx, "name")
if !model.LabelNameRE.MatchString(name) {
return apiFuncResult{nil, &apiError{errorBadData, errors.Errorf("invalid label name: %q", name)}, nil, nil}
}
q, err := api.Queryable.Querier(ctx, math.MinInt64, math.MaxInt64)
if err != nil {
return apiFuncResult{nil, &apiError{errorExec, err}, nil, nil}
}
Load only some offsets into the symbol table into memory. Rather than keeping the entire symbol table in memory, keep every nth offset and walk from there to the entry we need. This ends up slightly slower, ~360ms per 1M series returned from PostingsForMatchers which is not much considering the rest of the CPU such a query would go on to use. Make LabelValues use the postings tables, rather than having to do symbol lookups. Use yoloString, as PostingsForMatchers doesn't need the strings to stick around and adjust the API call to keep the Querier open until it's all marshalled. Remove allocatedSymbols memory optimisation, we no longer keep all the symbol strings in heap memory. Remove LabelValuesFor and LabelIndices, they're dead code. Ensure we've still tests for label indices, and add missing test that we can work with old V1 Format index files. PostingForMatchers performance is slightly better, with a big drop in allocation counts due to using yoloString for LabelValues: benchmark old ns/op new ns/op delta BenchmarkPostingsForMatchers/Block/n="1"-4 36698 36681 -0.05% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 522786 560887 +7.29% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 511652 537680 +5.09% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 522102 564239 +8.07% BenchmarkPostingsForMatchers/Block/i=~".*"-4 113689911 111795919 -1.67% BenchmarkPostingsForMatchers/Block/i=~".+"-4 135825572 132871085 -2.18% BenchmarkPostingsForMatchers/Block/i=~""-4 40782628 38038181 -6.73% BenchmarkPostingsForMatchers/Block/i!=""-4 31267869 29194327 -6.63% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112733329 111568823 -1.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 112868153 111232029 -1.45% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 31338257 29349446 -6.35% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 32054482 29972436 -6.50% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 136504654 133968442 -1.86% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 27960350 27264997 -2.49% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 136765564 133860724 -2.12% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 163714583 159453668 -2.60% benchmark old allocs new allocs delta BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 17 15 -11.76% BenchmarkPostingsForMatchers/Block/i=~".*"-4 100012 12 -99.99% BenchmarkPostingsForMatchers/Block/i=~".+"-4 200040 100040 -49.99% BenchmarkPostingsForMatchers/Block/i=~""-4 200045 100045 -49.99% BenchmarkPostingsForMatchers/Block/i!=""-4 200041 100041 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 100017 17 -99.98% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 100023 23 -99.98% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 200046 100046 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 200050 100050 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 200049 100049 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 111150 11150 -89.97% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 200055 100055 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 311238 111238 -64.26% benchmark old bytes new bytes delta BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 552 1544 +179.71% BenchmarkPostingsForMatchers/Block/i=~".*"-4 1600482 1606125 +0.35% BenchmarkPostingsForMatchers/Block/i=~".+"-4 17259065 17264709 +0.03% BenchmarkPostingsForMatchers/Block/i=~""-4 17259150 17264780 +0.03% BenchmarkPostingsForMatchers/Block/i!=""-4 17259048 17264680 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1600610 1606242 +0.35% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1600813 1606434 +0.35% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17259176 17264808 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17259304 17264936 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17259333 17264965 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3142628 3148262 +0.18% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17259509 17265141 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20405680 20416944 +0.06% However overall Select performance is down and involves more allocs, due to having to do more than a simple map lookup to resolve a symbol and that all the strings returned are allocated: benchmark old ns/op new ns/op delta BenchmarkQuerierSelect/Block/1of1000000-4 506092636 862678244 +70.46% BenchmarkQuerierSelect/Block/10of1000000-4 505638968 860917636 +70.26% BenchmarkQuerierSelect/Block/100of1000000-4 505229450 882150048 +74.60% BenchmarkQuerierSelect/Block/1000of1000000-4 515905414 862241115 +67.13% BenchmarkQuerierSelect/Block/10000of1000000-4 516785354 874841110 +69.29% BenchmarkQuerierSelect/Block/100000of1000000-4 540742808 907030187 +67.74% BenchmarkQuerierSelect/Block/1000000of1000000-4 815224288 1181236903 +44.90% benchmark old allocs new allocs delta BenchmarkQuerierSelect/Block/1of1000000-4 4000020 6000020 +50.00% BenchmarkQuerierSelect/Block/10of1000000-4 4000038 6000038 +50.00% BenchmarkQuerierSelect/Block/100of1000000-4 4000218 6000218 +50.00% BenchmarkQuerierSelect/Block/1000of1000000-4 4002018 6002018 +49.97% BenchmarkQuerierSelect/Block/10000of1000000-4 4020018 6020018 +49.75% BenchmarkQuerierSelect/Block/100000of1000000-4 4200018 6200018 +47.62% BenchmarkQuerierSelect/Block/1000000of1000000-4 6000018 8000019 +33.33% benchmark old bytes new bytes delta BenchmarkQuerierSelect/Block/1of1000000-4 176001468 227201476 +29.09% BenchmarkQuerierSelect/Block/10of1000000-4 176002620 227202628 +29.09% BenchmarkQuerierSelect/Block/100of1000000-4 176014140 227214148 +29.09% BenchmarkQuerierSelect/Block/1000of1000000-4 176129340 227329348 +29.07% BenchmarkQuerierSelect/Block/10000of1000000-4 177281340 228481348 +28.88% BenchmarkQuerierSelect/Block/100000of1000000-4 188801340 240001348 +27.12% BenchmarkQuerierSelect/Block/1000000of1000000-4 304001340 355201616 +16.84% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2019-12-12 08:55:32 -08:00
closer := func() {
q.Close()
}
vals, warnings, err := q.LabelValues(name)
if err != nil {
Load only some offsets into the symbol table into memory. Rather than keeping the entire symbol table in memory, keep every nth offset and walk from there to the entry we need. This ends up slightly slower, ~360ms per 1M series returned from PostingsForMatchers which is not much considering the rest of the CPU such a query would go on to use. Make LabelValues use the postings tables, rather than having to do symbol lookups. Use yoloString, as PostingsForMatchers doesn't need the strings to stick around and adjust the API call to keep the Querier open until it's all marshalled. Remove allocatedSymbols memory optimisation, we no longer keep all the symbol strings in heap memory. Remove LabelValuesFor and LabelIndices, they're dead code. Ensure we've still tests for label indices, and add missing test that we can work with old V1 Format index files. PostingForMatchers performance is slightly better, with a big drop in allocation counts due to using yoloString for LabelValues: benchmark old ns/op new ns/op delta BenchmarkPostingsForMatchers/Block/n="1"-4 36698 36681 -0.05% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 522786 560887 +7.29% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 511652 537680 +5.09% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 522102 564239 +8.07% BenchmarkPostingsForMatchers/Block/i=~".*"-4 113689911 111795919 -1.67% BenchmarkPostingsForMatchers/Block/i=~".+"-4 135825572 132871085 -2.18% BenchmarkPostingsForMatchers/Block/i=~""-4 40782628 38038181 -6.73% BenchmarkPostingsForMatchers/Block/i!=""-4 31267869 29194327 -6.63% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112733329 111568823 -1.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 112868153 111232029 -1.45% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 31338257 29349446 -6.35% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 32054482 29972436 -6.50% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 136504654 133968442 -1.86% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 27960350 27264997 -2.49% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 136765564 133860724 -2.12% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 163714583 159453668 -2.60% benchmark old allocs new allocs delta BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 17 15 -11.76% BenchmarkPostingsForMatchers/Block/i=~".*"-4 100012 12 -99.99% BenchmarkPostingsForMatchers/Block/i=~".+"-4 200040 100040 -49.99% BenchmarkPostingsForMatchers/Block/i=~""-4 200045 100045 -49.99% BenchmarkPostingsForMatchers/Block/i!=""-4 200041 100041 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 100017 17 -99.98% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 100023 23 -99.98% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 200046 100046 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 200050 100050 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 200049 100049 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 111150 11150 -89.97% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 200055 100055 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 311238 111238 -64.26% benchmark old bytes new bytes delta BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 552 1544 +179.71% BenchmarkPostingsForMatchers/Block/i=~".*"-4 1600482 1606125 +0.35% BenchmarkPostingsForMatchers/Block/i=~".+"-4 17259065 17264709 +0.03% BenchmarkPostingsForMatchers/Block/i=~""-4 17259150 17264780 +0.03% BenchmarkPostingsForMatchers/Block/i!=""-4 17259048 17264680 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1600610 1606242 +0.35% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1600813 1606434 +0.35% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17259176 17264808 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17259304 17264936 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17259333 17264965 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3142628 3148262 +0.18% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17259509 17265141 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20405680 20416944 +0.06% However overall Select performance is down and involves more allocs, due to having to do more than a simple map lookup to resolve a symbol and that all the strings returned are allocated: benchmark old ns/op new ns/op delta BenchmarkQuerierSelect/Block/1of1000000-4 506092636 862678244 +70.46% BenchmarkQuerierSelect/Block/10of1000000-4 505638968 860917636 +70.26% BenchmarkQuerierSelect/Block/100of1000000-4 505229450 882150048 +74.60% BenchmarkQuerierSelect/Block/1000of1000000-4 515905414 862241115 +67.13% BenchmarkQuerierSelect/Block/10000of1000000-4 516785354 874841110 +69.29% BenchmarkQuerierSelect/Block/100000of1000000-4 540742808 907030187 +67.74% BenchmarkQuerierSelect/Block/1000000of1000000-4 815224288 1181236903 +44.90% benchmark old allocs new allocs delta BenchmarkQuerierSelect/Block/1of1000000-4 4000020 6000020 +50.00% BenchmarkQuerierSelect/Block/10of1000000-4 4000038 6000038 +50.00% BenchmarkQuerierSelect/Block/100of1000000-4 4000218 6000218 +50.00% BenchmarkQuerierSelect/Block/1000of1000000-4 4002018 6002018 +49.97% BenchmarkQuerierSelect/Block/10000of1000000-4 4020018 6020018 +49.75% BenchmarkQuerierSelect/Block/100000of1000000-4 4200018 6200018 +47.62% BenchmarkQuerierSelect/Block/1000000of1000000-4 6000018 8000019 +33.33% benchmark old bytes new bytes delta BenchmarkQuerierSelect/Block/1of1000000-4 176001468 227201476 +29.09% BenchmarkQuerierSelect/Block/10of1000000-4 176002620 227202628 +29.09% BenchmarkQuerierSelect/Block/100of1000000-4 176014140 227214148 +29.09% BenchmarkQuerierSelect/Block/1000of1000000-4 176129340 227329348 +29.07% BenchmarkQuerierSelect/Block/10000of1000000-4 177281340 228481348 +28.88% BenchmarkQuerierSelect/Block/100000of1000000-4 188801340 240001348 +27.12% BenchmarkQuerierSelect/Block/1000000of1000000-4 304001340 355201616 +16.84% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2019-12-12 08:55:32 -08:00
return apiFuncResult{nil, &apiError{errorExec, err}, warnings, closer}
}
Load only some offsets into the symbol table into memory. Rather than keeping the entire symbol table in memory, keep every nth offset and walk from there to the entry we need. This ends up slightly slower, ~360ms per 1M series returned from PostingsForMatchers which is not much considering the rest of the CPU such a query would go on to use. Make LabelValues use the postings tables, rather than having to do symbol lookups. Use yoloString, as PostingsForMatchers doesn't need the strings to stick around and adjust the API call to keep the Querier open until it's all marshalled. Remove allocatedSymbols memory optimisation, we no longer keep all the symbol strings in heap memory. Remove LabelValuesFor and LabelIndices, they're dead code. Ensure we've still tests for label indices, and add missing test that we can work with old V1 Format index files. PostingForMatchers performance is slightly better, with a big drop in allocation counts due to using yoloString for LabelValues: benchmark old ns/op new ns/op delta BenchmarkPostingsForMatchers/Block/n="1"-4 36698 36681 -0.05% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 522786 560887 +7.29% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 511652 537680 +5.09% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 522102 564239 +8.07% BenchmarkPostingsForMatchers/Block/i=~".*"-4 113689911 111795919 -1.67% BenchmarkPostingsForMatchers/Block/i=~".+"-4 135825572 132871085 -2.18% BenchmarkPostingsForMatchers/Block/i=~""-4 40782628 38038181 -6.73% BenchmarkPostingsForMatchers/Block/i!=""-4 31267869 29194327 -6.63% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112733329 111568823 -1.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 112868153 111232029 -1.45% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 31338257 29349446 -6.35% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 32054482 29972436 -6.50% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 136504654 133968442 -1.86% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 27960350 27264997 -2.49% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 136765564 133860724 -2.12% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 163714583 159453668 -2.60% benchmark old allocs new allocs delta BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 17 15 -11.76% BenchmarkPostingsForMatchers/Block/i=~".*"-4 100012 12 -99.99% BenchmarkPostingsForMatchers/Block/i=~".+"-4 200040 100040 -49.99% BenchmarkPostingsForMatchers/Block/i=~""-4 200045 100045 -49.99% BenchmarkPostingsForMatchers/Block/i!=""-4 200041 100041 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 100017 17 -99.98% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 100023 23 -99.98% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 200046 100046 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 200050 100050 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 200049 100049 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 111150 11150 -89.97% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 200055 100055 -49.99% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 311238 111238 -64.26% benchmark old bytes new bytes delta BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 552 1544 +179.71% BenchmarkPostingsForMatchers/Block/i=~".*"-4 1600482 1606125 +0.35% BenchmarkPostingsForMatchers/Block/i=~".+"-4 17259065 17264709 +0.03% BenchmarkPostingsForMatchers/Block/i=~""-4 17259150 17264780 +0.03% BenchmarkPostingsForMatchers/Block/i!=""-4 17259048 17264680 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1600610 1606242 +0.35% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1600813 1606434 +0.35% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17259176 17264808 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17259304 17264936 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17259333 17264965 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3142628 3148262 +0.18% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17259509 17265141 +0.03% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20405680 20416944 +0.06% However overall Select performance is down and involves more allocs, due to having to do more than a simple map lookup to resolve a symbol and that all the strings returned are allocated: benchmark old ns/op new ns/op delta BenchmarkQuerierSelect/Block/1of1000000-4 506092636 862678244 +70.46% BenchmarkQuerierSelect/Block/10of1000000-4 505638968 860917636 +70.26% BenchmarkQuerierSelect/Block/100of1000000-4 505229450 882150048 +74.60% BenchmarkQuerierSelect/Block/1000of1000000-4 515905414 862241115 +67.13% BenchmarkQuerierSelect/Block/10000of1000000-4 516785354 874841110 +69.29% BenchmarkQuerierSelect/Block/100000of1000000-4 540742808 907030187 +67.74% BenchmarkQuerierSelect/Block/1000000of1000000-4 815224288 1181236903 +44.90% benchmark old allocs new allocs delta BenchmarkQuerierSelect/Block/1of1000000-4 4000020 6000020 +50.00% BenchmarkQuerierSelect/Block/10of1000000-4 4000038 6000038 +50.00% BenchmarkQuerierSelect/Block/100of1000000-4 4000218 6000218 +50.00% BenchmarkQuerierSelect/Block/1000of1000000-4 4002018 6002018 +49.97% BenchmarkQuerierSelect/Block/10000of1000000-4 4020018 6020018 +49.75% BenchmarkQuerierSelect/Block/100000of1000000-4 4200018 6200018 +47.62% BenchmarkQuerierSelect/Block/1000000of1000000-4 6000018 8000019 +33.33% benchmark old bytes new bytes delta BenchmarkQuerierSelect/Block/1of1000000-4 176001468 227201476 +29.09% BenchmarkQuerierSelect/Block/10of1000000-4 176002620 227202628 +29.09% BenchmarkQuerierSelect/Block/100of1000000-4 176014140 227214148 +29.09% BenchmarkQuerierSelect/Block/1000of1000000-4 176129340 227329348 +29.07% BenchmarkQuerierSelect/Block/10000of1000000-4 177281340 228481348 +28.88% BenchmarkQuerierSelect/Block/100000of1000000-4 188801340 240001348 +27.12% BenchmarkQuerierSelect/Block/1000000of1000000-4 304001340 355201616 +16.84% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2019-12-12 08:55:32 -08:00
return apiFuncResult{vals, nil, warnings, closer}
}
var (
minTime = time.Unix(math.MinInt64/1000+62135596801, 0).UTC()
maxTime = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC()
minTimeFormatted = minTime.Format(time.RFC3339Nano)
maxTimeFormatted = maxTime.Format(time.RFC3339Nano)
)
func (api *API) series(r *http.Request) apiFuncResult {
if err := r.ParseForm(); err != nil {
return apiFuncResult{nil, &apiError{errorBadData, errors.Wrapf(err, "error parsing form values")}, nil, nil}
}
if len(r.Form["match[]"]) == 0 {
return apiFuncResult{nil, &apiError{errorBadData, errors.New("no match[] parameter provided")}, nil, nil}
}
start, err := parseTimeParam(r, "start", minTime)
if err != nil {
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
end, err := parseTimeParam(r, "end", maxTime)
if err != nil {
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
var matcherSets [][]*labels.Matcher
for _, s := range r.Form["match[]"] {
matchers, err := parser.ParseMetricSelector(s)
if err != nil {
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
matcherSets = append(matcherSets, matchers)
}
q, err := api.Queryable.Querier(r.Context(), timestamp.FromTime(start), timestamp.FromTime(end))
if err != nil {
return apiFuncResult{nil, &apiError{errorExec, err}, nil, nil}
}
defer q.Close()
var sets []storage.SeriesSet
var warnings storage.Warnings
for _, mset := range matcherSets {
s, wrn, err := q.Select(false, nil, mset...)
warnings = append(warnings, wrn...)
2017-11-23 04:50:06 -08:00
if err != nil {
return apiFuncResult{nil, &apiError{errorExec, err}, warnings, nil}
2017-11-23 04:50:06 -08:00
}
sets = append(sets, s)
}
storage: Added Chunks{Queryable/Querier/SeriesSet/Series/Iteratable. Added generic Merge{SeriesSet/Querier} implementation. (#7005) * storage: Added Chunks{Queryable/Querier/SeriesSet/Series/Iteratable. Added generic Merge{SeriesSet/Querier} implementation. ## Rationales: In many places (e.g. chunk Remote read, Thanos Receive fetching chunk from TSDB), we operate on encoded chunks not samples. This means that we unnecessary decode/encode, wasting CPU, time and memory. This PR adds chunk iterator interfaces and makes the merge code to be reused between both seriesSets I will make the use of it in following PR inside tsdb itself. For now fanout implements it and mergers. All merges now also allows passing series mergers. This opens doors for custom deduplications other than TSDB vertical ones (e.g. offline one we have in Thanos). ## Changes * Added Chunk versions of all iterating methods. It all starts in Querier/ChunkQuerier. The plan is that Storage will implement both chunked and samples. * Added Seek to chunks.Iterator interface for iterating over chunks. * NewMergeChunkQuerier was added; Both this and NewMergeQuerier are now using generigMergeQuerier to share the code. Generic code was added. * Improved tests. * Added some TODO for further simplifications in next PRs. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addressed Brian's comments. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Moved s/Labeled/SeriesLabels as per Krasi suggestion. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addressed Krasi's comments. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Second iteration of Krasi comments. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Another round of comments. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
2020-03-24 13:15:47 -07:00
set := storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge)
2017-04-04 02:09:11 -07:00
metrics := []labels.Labels{}
for set.Next() {
metrics = append(metrics, set.At().Labels())
}
if set.Err() != nil {
return apiFuncResult{nil, &apiError{errorExec, set.Err()}, warnings, nil}
2017-04-04 02:09:11 -07:00
}
return apiFuncResult{metrics, nil, warnings, nil}
}
func (api *API) dropSeries(r *http.Request) apiFuncResult {
return apiFuncResult{nil, &apiError{errorInternal, errors.New("not implemented")}, nil, nil}
}
2017-03-06 09:51:27 -08:00
// Target has the information for one target.
2016-12-02 04:31:43 -08:00
type Target struct {
// Labels before any processing.
DiscoveredLabels map[string]string `json:"discoveredLabels"`
2016-12-02 04:31:43 -08:00
// Any labels that are added to this target and its metrics.
Labels map[string]string `json:"labels"`
2016-12-02 04:31:43 -08:00
ScrapePool string `json:"scrapePool"`
ScrapeURL string `json:"scrapeUrl"`
GlobalURL string `json:"globalUrl"`
2016-12-02 04:31:43 -08:00
LastError string `json:"lastError"`
LastScrape time.Time `json:"lastScrape"`
LastScrapeDuration float64 `json:"lastScrapeDuration"`
Health scrape.TargetHealth `json:"health"`
2016-12-02 04:31:43 -08:00
}
// DroppedTarget has the information for one target that was dropped during relabelling.
type DroppedTarget struct {
// Labels before any processing.
DiscoveredLabels map[string]string `json:"discoveredLabels"`
}
2017-03-06 03:46:37 -08:00
// TargetDiscovery has all the active targets.
type TargetDiscovery struct {
ActiveTargets []*Target `json:"activeTargets"`
DroppedTargets []*DroppedTarget `json:"droppedTargets"`
}
// GlobalURLOptions contains fields used for deriving the global URL for local targets.
type GlobalURLOptions struct {
ListenAddress string
Host string
Scheme string
}
func getGlobalURL(u *url.URL, opts GlobalURLOptions) (*url.URL, error) {
host, port, err := net.SplitHostPort(u.Host)
if err != nil {
return u, err
}
for _, lhr := range LocalhostRepresentations {
if host == lhr {
_, ownPort, err := net.SplitHostPort(opts.ListenAddress)
if err != nil {
return u, err
}
if port == ownPort {
// Only in the case where the target is on localhost and its port is
// the same as the one we're listening on, we know for sure that
// we're monitoring our own process and that we need to change the
// scheme, hostname, and port to the externally reachable ones as
// well. We shouldn't need to touch the path at all, since if a
// path prefix is defined, the path under which we scrape ourselves
// should already contain the prefix.
u.Scheme = opts.Scheme
u.Host = opts.Host
} else {
// Otherwise, we only know that localhost is not reachable
// externally, so we replace only the hostname by the one in the
// external URL. It could be the wrong hostname for the service on
// this port, but it's still the best possible guess.
host, _, err := net.SplitHostPort(opts.Host)
if err != nil {
return u, err
}
u.Host = host + ":" + port
}
break
}
}
return u, nil
}
func (api *API) targets(r *http.Request) apiFuncResult {
sortKeys := func(targets map[string][]*scrape.Target) ([]string, int) {
var n int
keys := make([]string, 0, len(targets))
for k := range targets {
keys = append(keys, k)
n += len(targets[k])
}
sort.Strings(keys)
return keys, n
}
flatten := func(targets map[string][]*scrape.Target) []*scrape.Target {
keys, n := sortKeys(targets)
res := make([]*scrape.Target, 0, n)
for _, k := range keys {
res = append(res, targets[k]...)
2016-12-02 04:31:43 -08:00
}
return res
2016-12-02 04:31:43 -08:00
}
state := strings.ToLower(r.URL.Query().Get("state"))
showActive := state == "" || state == "any" || state == "active"
showDropped := state == "" || state == "any" || state == "dropped"
res := &TargetDiscovery{}
if showActive {
targetsActive := api.targetRetriever.TargetsActive()
activeKeys, numTargets := sortKeys(targetsActive)
res.ActiveTargets = make([]*Target, 0, numTargets)
for _, key := range activeKeys {
for _, target := range targetsActive[key] {
lastErrStr := ""
lastErr := target.LastError()
if lastErr != nil {
lastErrStr = lastErr.Error()
}
globalURL, err := getGlobalURL(target.URL(), api.globalURLOptions)
res.ActiveTargets = append(res.ActiveTargets, &Target{
DiscoveredLabels: target.DiscoveredLabels().Map(),
Labels: target.Labels().Map(),
ScrapePool: key,
ScrapeURL: target.URL().String(),
GlobalURL: globalURL.String(),
LastError: func() string {
if err == nil && lastErrStr == "" {
return ""
} else if err != nil {
return errors.Wrapf(err, lastErrStr).Error()
}
return lastErrStr
}(),
LastScrape: target.LastScrape(),
LastScrapeDuration: target.LastScrapeDuration().Seconds(),
Health: target.Health(),
})
}
}
} else {
res.ActiveTargets = []*Target{}
}
if showDropped {
tDropped := flatten(api.targetRetriever.TargetsDropped())
res.DroppedTargets = make([]*DroppedTarget, 0, len(tDropped))
for _, t := range tDropped {
res.DroppedTargets = append(res.DroppedTargets, &DroppedTarget{
DiscoveredLabels: t.DiscoveredLabels().Map(),
})
}
} else {
res.DroppedTargets = []*DroppedTarget{}
}
return apiFuncResult{res, nil, nil, nil}
2016-12-02 04:31:43 -08:00
}
func matchLabels(lset labels.Labels, matchers []*labels.Matcher) bool {
for _, m := range matchers {
if !m.Matches(lset.Get(m.Name)) {
return false
}
}
return true
}
func (api *API) targetMetadata(r *http.Request) apiFuncResult {
limit := -1
if s := r.FormValue("limit"); s != "" {
var err error
if limit, err = strconv.Atoi(s); err != nil {
return apiFuncResult{nil, &apiError{errorBadData, errors.New("limit must be a number")}, nil, nil}
}
}
matchTarget := r.FormValue("match_target")
var matchers []*labels.Matcher
var err error
if matchTarget != "" {
matchers, err = parser.ParseMetricSelector(matchTarget)
if err != nil {
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
}
metric := r.FormValue("metric")
res := []metricMetadata{}
for _, tt := range api.targetRetriever.TargetsActive() {
for _, t := range tt {
if limit >= 0 && len(res) >= limit {
break
}
// Filter targets that don't satisfy the label matchers.
if matchTarget != "" && !matchLabels(t.Labels(), matchers) {
continue
}
// If no metric is specified, get the full list for the target.
if metric == "" {
for _, md := range t.MetadataList() {
res = append(res, metricMetadata{
Target: t.Labels(),
Metric: md.Metric,
Type: md.Type,
Help: md.Help,
Unit: md.Unit,
})
}
continue
}
// Get metadata for the specified metric.
if md, ok := t.Metadata(metric); ok {
res = append(res, metricMetadata{
Target: t.Labels(),
Type: md.Type,
Help: md.Help,
Unit: md.Unit,
})
}
}
}
return apiFuncResult{res, nil, nil, nil}
}
type metricMetadata struct {
Target labels.Labels `json:"target"`
Metric string `json:"metric,omitempty"`
Type textparse.MetricType `json:"type"`
Help string `json:"help"`
Unit string `json:"unit"`
}
2017-03-06 09:51:27 -08:00
// AlertmanagerDiscovery has all the active Alertmanagers.
2017-01-13 01:20:11 -08:00
type AlertmanagerDiscovery struct {
ActiveAlertmanagers []*AlertmanagerTarget `json:"activeAlertmanagers"`
DroppedAlertmanagers []*AlertmanagerTarget `json:"droppedAlertmanagers"`
2017-01-13 01:20:11 -08:00
}
2017-03-06 09:51:27 -08:00
// AlertmanagerTarget has info on one AM.
2017-01-13 01:20:11 -08:00
type AlertmanagerTarget struct {
URL string `json:"url"`
}
func (api *API) alertmanagers(r *http.Request) apiFuncResult {
2017-01-13 01:20:11 -08:00
urls := api.alertmanagerRetriever.Alertmanagers()
droppedURLS := api.alertmanagerRetriever.DroppedAlertmanagers()
ams := &AlertmanagerDiscovery{ActiveAlertmanagers: make([]*AlertmanagerTarget, len(urls)), DroppedAlertmanagers: make([]*AlertmanagerTarget, len(droppedURLS))}
for i, url := range urls {
ams.ActiveAlertmanagers[i] = &AlertmanagerTarget{URL: url.String()}
2017-01-13 01:20:11 -08:00
}
for i, url := range droppedURLS {
ams.DroppedAlertmanagers[i] = &AlertmanagerTarget{URL: url.String()}
}
return apiFuncResult{ams, nil, nil, nil}
2017-01-13 01:20:11 -08:00
}
// AlertDiscovery has info for all active alerts.
type AlertDiscovery struct {
Alerts []*Alert `json:"alerts"`
}
// Alert has info for an alert.
type Alert struct {
Labels labels.Labels `json:"labels"`
Annotations labels.Labels `json:"annotations"`
State string `json:"state"`
ActiveAt *time.Time `json:"activeAt,omitempty"`
Value string `json:"value"`
}
func (api *API) alerts(r *http.Request) apiFuncResult {
alertingRules := api.rulesRetriever.AlertingRules()
alerts := []*Alert{}
for _, alertingRule := range alertingRules {
alerts = append(
alerts,
rulesAlertsToAPIAlerts(alertingRule.ActiveAlerts())...,
)
}
res := &AlertDiscovery{Alerts: alerts}
return apiFuncResult{res, nil, nil, nil}
}
func rulesAlertsToAPIAlerts(rulesAlerts []*rules.Alert) []*Alert {
apiAlerts := make([]*Alert, len(rulesAlerts))
for i, ruleAlert := range rulesAlerts {
apiAlerts[i] = &Alert{
Labels: ruleAlert.Labels,
Annotations: ruleAlert.Annotations,
State: ruleAlert.State.String(),
ActiveAt: &ruleAlert.ActiveAt,
Value: strconv.FormatFloat(ruleAlert.Value, 'e', -1, 64),
}
}
return apiAlerts
}
type metadata struct {
Type textparse.MetricType `json:"type"`
Help string `json:"help"`
Unit string `json:"unit"`
}
func (api *API) metricMetadata(r *http.Request) apiFuncResult {
metrics := map[string]map[metadata]struct{}{}
limit := -1
if s := r.FormValue("limit"); s != "" {
var err error
if limit, err = strconv.Atoi(s); err != nil {
return apiFuncResult{nil, &apiError{errorBadData, errors.New("limit must be a number")}, nil, nil}
}
}
metric := r.FormValue("metric")
for _, tt := range api.targetRetriever.TargetsActive() {
for _, t := range tt {
if metric == "" {
for _, mm := range t.MetadataList() {
m := metadata{Type: mm.Type, Help: mm.Help, Unit: mm.Unit}
ms, ok := metrics[mm.Metric]
if !ok {
ms = map[metadata]struct{}{}
metrics[mm.Metric] = ms
}
ms[m] = struct{}{}
}
continue
}
if md, ok := t.Metadata(metric); ok {
m := metadata{Type: md.Type, Help: md.Help, Unit: md.Unit}
ms, ok := metrics[md.Metric]
if !ok {
ms = map[metadata]struct{}{}
metrics[md.Metric] = ms
}
ms[m] = struct{}{}
}
}
}
// Put the elements from the pseudo-set into a slice for marshaling.
res := map[string][]metadata{}
for name, set := range metrics {
if limit >= 0 && len(res) >= limit {
break
}
s := []metadata{}
for metadata := range set {
s = append(s, metadata)
}
res[name] = s
}
return apiFuncResult{res, nil, nil, nil}
}
// RuleDiscovery has info for all rules
type RuleDiscovery struct {
RuleGroups []*RuleGroup `json:"groups"`
}
// RuleGroup has info for rules which are part of a group
type RuleGroup struct {
Name string `json:"name"`
File string `json:"file"`
// In order to preserve rule ordering, while exposing type (alerting or recording)
// specific properties, both alerting and recording rules are exposed in the
// same array.
Rules []rule `json:"rules"`
Interval float64 `json:"interval"`
EvaluationTime float64 `json:"evaluationTime"`
LastEvaluation time.Time `json:"lastEvaluation"`
}
type rule interface{}
type alertingRule struct {
React UI: Implement alerts page (#6402) * url filter rules param Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * address review changes Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * ui initial commit Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * improve ui Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * fix typo in component name Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * create query link + ui enhancements Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * add count to state labels Signed-off-by: blalov <boiskila@gmail.com> * put alerts table render in the right place Signed-off-by: blalov <boiskila@gmail.com> * refactoring Signed-off-by: blalov <boiskila@gmail.com> * fix rules endpoint test Signed-off-by: blalov <boiskila@gmail.com> * lint fixes Signed-off-by: blalov <boiskila@gmail.com> * test query params Signed-off-by: blalov <boiskila@gmail.com> * refactoring Signed-off-by: blalov <boiskila@gmail.com> * review changes Signed-off-by: blalov <boiskila@gmail.com> * adding down arrow as click indicator in Alert Signed-off-by: blalov <boiskila@gmail.com> * add period at the end of the comment Signed-off-by: blalov <boiskila@gmail.com> * review changes Signed-off-by: blalov <boiskila@gmail.com> * remove left-over css Signed-off-by: blalov <boiskila@gmail.com> * adding expand/collapse arrows on Alert Signed-off-by: blalov <boiskila@gmail.com> * create proper expression for alert name Signed-off-by: blalov <boiskila@gmail.com>
2019-12-09 14:42:59 -08:00
// State can be "pending", "firing", "inactive".
State string `json:"state"`
Name string `json:"name"`
Query string `json:"query"`
Duration float64 `json:"duration"`
Labels labels.Labels `json:"labels"`
Annotations labels.Labels `json:"annotations"`
Alerts []*Alert `json:"alerts"`
Health rules.RuleHealth `json:"health"`
LastError string `json:"lastError,omitempty"`
EvaluationTime float64 `json:"evaluationTime"`
LastEvaluation time.Time `json:"lastEvaluation"`
// Type of an alertingRule is always "alerting".
Type string `json:"type"`
}
type recordingRule struct {
Name string `json:"name"`
Query string `json:"query"`
Labels labels.Labels `json:"labels,omitempty"`
Health rules.RuleHealth `json:"health"`
LastError string `json:"lastError,omitempty"`
EvaluationTime float64 `json:"evaluationTime"`
LastEvaluation time.Time `json:"lastEvaluation"`
// Type of a recordingRule is always "recording".
Type string `json:"type"`
}
func (api *API) rules(r *http.Request) apiFuncResult {
ruleGroups := api.rulesRetriever.RuleGroups()
res := &RuleDiscovery{RuleGroups: make([]*RuleGroup, len(ruleGroups))}
React UI: Implement alerts page (#6402) * url filter rules param Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * address review changes Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * ui initial commit Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * improve ui Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * fix typo in component name Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * create query link + ui enhancements Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * add count to state labels Signed-off-by: blalov <boiskila@gmail.com> * put alerts table render in the right place Signed-off-by: blalov <boiskila@gmail.com> * refactoring Signed-off-by: blalov <boiskila@gmail.com> * fix rules endpoint test Signed-off-by: blalov <boiskila@gmail.com> * lint fixes Signed-off-by: blalov <boiskila@gmail.com> * test query params Signed-off-by: blalov <boiskila@gmail.com> * refactoring Signed-off-by: blalov <boiskila@gmail.com> * review changes Signed-off-by: blalov <boiskila@gmail.com> * adding down arrow as click indicator in Alert Signed-off-by: blalov <boiskila@gmail.com> * add period at the end of the comment Signed-off-by: blalov <boiskila@gmail.com> * review changes Signed-off-by: blalov <boiskila@gmail.com> * remove left-over css Signed-off-by: blalov <boiskila@gmail.com> * adding expand/collapse arrows on Alert Signed-off-by: blalov <boiskila@gmail.com> * create proper expression for alert name Signed-off-by: blalov <boiskila@gmail.com>
2019-12-09 14:42:59 -08:00
typeParam := strings.ToLower(r.URL.Query().Get("type"))
if typeParam != "" && typeParam != "alert" && typeParam != "record" {
err := errors.Errorf("invalid query parameter type='%v'", typeParam)
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
returnAlerts := typeParam == "" || typeParam == "alert"
returnRecording := typeParam == "" || typeParam == "record"
for i, grp := range ruleGroups {
apiRuleGroup := &RuleGroup{
Name: grp.Name(),
File: grp.File(),
Interval: grp.Interval().Seconds(),
Rules: []rule{},
EvaluationTime: grp.GetEvaluationDuration().Seconds(),
LastEvaluation: grp.GetEvaluationTimestamp(),
}
for _, r := range grp.Rules() {
var enrichedRule rule
lastError := ""
if r.LastError() != nil {
lastError = r.LastError().Error()
}
switch rule := r.(type) {
case *rules.AlertingRule:
React UI: Implement alerts page (#6402) * url filter rules param Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * address review changes Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * ui initial commit Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * improve ui Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * fix typo in component name Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * create query link + ui enhancements Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * add count to state labels Signed-off-by: blalov <boiskila@gmail.com> * put alerts table render in the right place Signed-off-by: blalov <boiskila@gmail.com> * refactoring Signed-off-by: blalov <boiskila@gmail.com> * fix rules endpoint test Signed-off-by: blalov <boiskila@gmail.com> * lint fixes Signed-off-by: blalov <boiskila@gmail.com> * test query params Signed-off-by: blalov <boiskila@gmail.com> * refactoring Signed-off-by: blalov <boiskila@gmail.com> * review changes Signed-off-by: blalov <boiskila@gmail.com> * adding down arrow as click indicator in Alert Signed-off-by: blalov <boiskila@gmail.com> * add period at the end of the comment Signed-off-by: blalov <boiskila@gmail.com> * review changes Signed-off-by: blalov <boiskila@gmail.com> * remove left-over css Signed-off-by: blalov <boiskila@gmail.com> * adding expand/collapse arrows on Alert Signed-off-by: blalov <boiskila@gmail.com> * create proper expression for alert name Signed-off-by: blalov <boiskila@gmail.com>
2019-12-09 14:42:59 -08:00
if !returnAlerts {
break
}
enrichedRule = alertingRule{
State: rule.State().String(),
Name: rule.Name(),
Query: rule.Query().String(),
Duration: rule.HoldDuration().Seconds(),
Labels: rule.Labels(),
Annotations: rule.Annotations(),
Alerts: rulesAlertsToAPIAlerts(rule.ActiveAlerts()),
Health: rule.Health(),
LastError: lastError,
EvaluationTime: rule.GetEvaluationDuration().Seconds(),
LastEvaluation: rule.GetEvaluationTimestamp(),
Type: "alerting",
}
case *rules.RecordingRule:
React UI: Implement alerts page (#6402) * url filter rules param Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * address review changes Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * ui initial commit Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * improve ui Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * fix typo in component name Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * create query link + ui enhancements Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * add count to state labels Signed-off-by: blalov <boiskila@gmail.com> * put alerts table render in the right place Signed-off-by: blalov <boiskila@gmail.com> * refactoring Signed-off-by: blalov <boiskila@gmail.com> * fix rules endpoint test Signed-off-by: blalov <boiskila@gmail.com> * lint fixes Signed-off-by: blalov <boiskila@gmail.com> * test query params Signed-off-by: blalov <boiskila@gmail.com> * refactoring Signed-off-by: blalov <boiskila@gmail.com> * review changes Signed-off-by: blalov <boiskila@gmail.com> * adding down arrow as click indicator in Alert Signed-off-by: blalov <boiskila@gmail.com> * add period at the end of the comment Signed-off-by: blalov <boiskila@gmail.com> * review changes Signed-off-by: blalov <boiskila@gmail.com> * remove left-over css Signed-off-by: blalov <boiskila@gmail.com> * adding expand/collapse arrows on Alert Signed-off-by: blalov <boiskila@gmail.com> * create proper expression for alert name Signed-off-by: blalov <boiskila@gmail.com>
2019-12-09 14:42:59 -08:00
if !returnRecording {
break
}
enrichedRule = recordingRule{
Name: rule.Name(),
Query: rule.Query().String(),
Labels: rule.Labels(),
Health: rule.Health(),
LastError: lastError,
EvaluationTime: rule.GetEvaluationDuration().Seconds(),
LastEvaluation: rule.GetEvaluationTimestamp(),
Type: "recording",
}
default:
err := errors.Errorf("failed to assert type of rule '%v'", rule.Name())
return apiFuncResult{nil, &apiError{errorInternal, err}, nil, nil}
}
React UI: Implement alerts page (#6402) * url filter rules param Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * address review changes Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * ui initial commit Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * improve ui Signed-off-by: blalov <boiskila@gmail.com> Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * fix typo in component name Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * create query link + ui enhancements Signed-off-by: Boyko Lalov <boiskila@gmail.com> Signed-off-by: blalov <boiskila@gmail.com> * add count to state labels Signed-off-by: blalov <boiskila@gmail.com> * put alerts table render in the right place Signed-off-by: blalov <boiskila@gmail.com> * refactoring Signed-off-by: blalov <boiskila@gmail.com> * fix rules endpoint test Signed-off-by: blalov <boiskila@gmail.com> * lint fixes Signed-off-by: blalov <boiskila@gmail.com> * test query params Signed-off-by: blalov <boiskila@gmail.com> * refactoring Signed-off-by: blalov <boiskila@gmail.com> * review changes Signed-off-by: blalov <boiskila@gmail.com> * adding down arrow as click indicator in Alert Signed-off-by: blalov <boiskila@gmail.com> * add period at the end of the comment Signed-off-by: blalov <boiskila@gmail.com> * review changes Signed-off-by: blalov <boiskila@gmail.com> * remove left-over css Signed-off-by: blalov <boiskila@gmail.com> * adding expand/collapse arrows on Alert Signed-off-by: blalov <boiskila@gmail.com> * create proper expression for alert name Signed-off-by: blalov <boiskila@gmail.com>
2019-12-09 14:42:59 -08:00
if enrichedRule != nil {
apiRuleGroup.Rules = append(apiRuleGroup.Rules, enrichedRule)
}
}
res.RuleGroups[i] = apiRuleGroup
}
return apiFuncResult{res, nil, nil, nil}
}
type prometheusConfig struct {
YAML string `json:"yaml"`
}
2019-11-02 08:53:32 -07:00
func (api *API) serveRuntimeInfo(r *http.Request) apiFuncResult {
status, err := api.runtimeInfo()
if err != nil {
return apiFuncResult{status, &apiError{errorInternal, err}, nil, nil}
}
return apiFuncResult{status, nil, nil, nil}
}
func (api *API) serveBuildInfo(r *http.Request) apiFuncResult {
return apiFuncResult{api.buildInfo, nil, nil, nil}
}
func (api *API) serveConfig(r *http.Request) apiFuncResult {
cfg := &prometheusConfig{
YAML: api.config().String(),
}
return apiFuncResult{cfg, nil, nil, nil}
}
func (api *API) serveFlags(r *http.Request) apiFuncResult {
return apiFuncResult{api.flagsMap, nil, nil, nil}
}
// stat holds the information about individual cardinality.
type stat struct {
Name string `json:"name"`
Value uint64 `json:"value"`
}
// tsdbStatus has information of cardinality statistics from postings.
type tsdbStatus struct {
SeriesCountByMetricName []stat `json:"seriesCountByMetricName"`
LabelValueCountByLabelName []stat `json:"labelValueCountByLabelName"`
MemoryInBytesByLabelName []stat `json:"memoryInBytesByLabelName"`
SeriesCountByLabelValuePair []stat `json:"seriesCountByLabelValuePair"`
}
func (api *API) serveTSDBStatus(r *http.Request) apiFuncResult {
db := api.db()
if db == nil {
return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("TSDB not ready")}, nil, nil}
}
convert := func(stats []index.Stat) []stat {
result := make([]stat, 0, len(stats))
for _, item := range stats {
item := stat{Name: item.Name, Value: item.Count}
result = append(result, item)
}
return result
}
posting := db.Head().PostingsCardinalityStats(model.MetricNameLabel)
response := tsdbStatus{
SeriesCountByMetricName: convert(posting.CardinalityMetricsStats),
LabelValueCountByLabelName: convert(posting.CardinalityLabelStats),
MemoryInBytesByLabelName: convert(posting.LabelValueStats),
SeriesCountByLabelValuePair: convert(posting.LabelValuePairsStats),
}
return apiFuncResult{response, nil, nil, nil}
}
2017-10-23 13:28:17 -07:00
func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
if err := api.remoteReadGate.Start(ctx); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
remoteReadQueries.Inc()
defer api.remoteReadGate.Done()
defer remoteReadQueries.Dec()
2017-10-23 13:28:17 -07:00
req, err := remote.DecodeReadRequest(r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
externalLabels := api.config().GlobalConfig.ExternalLabels.Map()
sortedExternalLabels := make([]prompb.Label, 0, len(externalLabels))
for name, value := range externalLabels {
sortedExternalLabels = append(sortedExternalLabels, prompb.Label{
Name: string(name),
Value: string(value),
})
2017-10-23 13:28:17 -07:00
}
sort.Slice(sortedExternalLabels, func(i, j int) bool {
return sortedExternalLabels[i].Name < sortedExternalLabels[j].Name
})
responseType, err := remote.NegotiateResponseType(req.AcceptedResponseTypes)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
switch responseType {
case prompb.ReadRequest_STREAMED_XOR_CHUNKS:
w.Header().Set("Content-Type", "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse")
f, ok := w.(http.Flusher)
if !ok {
http.Error(w, "internal http.ResponseWriter does not implement http.Flusher interface", http.StatusInternalServerError)
2017-10-23 13:28:17 -07:00
return
}
for i, query := range req.Queries {
err := api.remoteReadQuery(ctx, query, externalLabels, func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) error {
// The streaming API has to provide the series sorted.
set, _, err := querier.Select(true, hints, filteredMatchers...)
Don't sort postings if we only have one block. Sorting the heads postings can be quite slow. We only need sorted series when merging with another querier, so only sort then. This will make big queries that only touch the head faster, though queries that touch both the head and a block will still be the same speed. This probably won't help much with graphing unless the range is under an hour, however it should make most recording rules faster. Add gaurantee that remote read streaming produces sorted series. PromQL benchmarks for histograms show only 2-3% improvement, but they're only over 1k series. benchmark old ns/op new ns/op delta BenchmarkQuerierSelect/Head/1of1000000-4 1375486282 507657736 -63.09% BenchmarkQuerierSelect/Head/10of1000000-4 1387859004 507769850 -63.41% BenchmarkQuerierSelect/Head/100of1000000-4 1387087935 506029110 -63.52% BenchmarkQuerierSelect/Head/1000of1000000-4 1386869064 504521986 -63.62% BenchmarkQuerierSelect/Head/10000of1000000-4 1386213685 505210422 -63.55% BenchmarkQuerierSelect/Head/100000of1000000-4 1392754988 529842406 -61.96% BenchmarkQuerierSelect/Head/1000000of1000000-4 1569414722 725059506 -53.80% BenchmarkQuerierSelect/SortedHead/1of1000000-4 1381019902 1370495863 -0.76% BenchmarkQuerierSelect/SortedHead/10of1000000-4 1375696209 1366789468 -0.65% BenchmarkQuerierSelect/SortedHead/100of1000000-4 1386009422 1364519297 -1.55% BenchmarkQuerierSelect/SortedHead/1000of1000000-4 1377700532 1364486191 -0.96% BenchmarkQuerierSelect/SortedHead/10000of1000000-4 1383539536 1369545314 -1.01% BenchmarkQuerierSelect/SortedHead/100000of1000000-4 1410089163 1394731339 -1.09% BenchmarkQuerierSelect/SortedHead/1000000of1000000-4 1634744148 1581554956 -3.25% BenchmarkQuerierSelect/Block/1of1000000-4 881741242 879839470 -0.22% BenchmarkQuerierSelect/Block/10of1000000-4 880381562 882846038 +0.28% BenchmarkQuerierSelect/Block/100of1000000-4 887519357 881016916 -0.73% BenchmarkQuerierSelect/Block/1000of1000000-4 902194205 883433524 -2.08% BenchmarkQuerierSelect/Block/10000of1000000-4 892321964 885130170 -0.81% BenchmarkQuerierSelect/Block/100000of1000000-4 938604466 933527150 -0.54% BenchmarkQuerierSelect/Block/1000000of1000000-4 1313510845 1295881124 -1.34% benchmark old allocs new allocs delta BenchmarkQuerierSelect/Head/1of1000000-4 4000056 4000018 -0.00% BenchmarkQuerierSelect/Head/10of1000000-4 4000074 4000036 -0.00% BenchmarkQuerierSelect/Head/100of1000000-4 4000254 4000216 -0.00% BenchmarkQuerierSelect/Head/1000of1000000-4 4002054 4002016 -0.00% BenchmarkQuerierSelect/Head/10000of1000000-4 4020054 4020016 -0.00% BenchmarkQuerierSelect/Head/100000of1000000-4 4200054 4200016 -0.00% BenchmarkQuerierSelect/Head/1000000of1000000-4 6000054 6000016 -0.00% BenchmarkQuerierSelect/SortedHead/1of1000000-4 4000071 4000071 +0.00% BenchmarkQuerierSelect/SortedHead/10of1000000-4 4000089 4000089 +0.00% BenchmarkQuerierSelect/SortedHead/100of1000000-4 4000269 4000269 +0.00% BenchmarkQuerierSelect/SortedHead/1000of1000000-4 4002069 4002069 +0.00% BenchmarkQuerierSelect/SortedHead/10000of1000000-4 4020069 4020069 +0.00% BenchmarkQuerierSelect/SortedHead/100000of1000000-4 4200069 4200069 +0.00% BenchmarkQuerierSelect/SortedHead/1000000of1000000-4 6000069 6000069 +0.00% BenchmarkQuerierSelect/Block/1of1000000-4 6000023 6000022 -0.00% BenchmarkQuerierSelect/Block/10of1000000-4 6000059 6000058 -0.00% BenchmarkQuerierSelect/Block/100of1000000-4 6000419 6000418 -0.00% BenchmarkQuerierSelect/Block/1000of1000000-4 6004019 6004018 -0.00% BenchmarkQuerierSelect/Block/10000of1000000-4 6040019 6040018 -0.00% BenchmarkQuerierSelect/Block/100000of1000000-4 6400019 6400018 -0.00% BenchmarkQuerierSelect/Block/1000000of1000000-4 10000020 10000019 -0.00% benchmark old bytes new bytes delta BenchmarkQuerierSelect/Head/1of1000000-4 229192200 176001176 -23.21% BenchmarkQuerierSelect/Head/10of1000000-4 229193352 176002328 -23.21% BenchmarkQuerierSelect/Head/100of1000000-4 229204872 176013848 -23.21% BenchmarkQuerierSelect/Head/1000of1000000-4 229320072 176129048 -23.20% BenchmarkQuerierSelect/Head/10000of1000000-4 230472072 177281048 -23.08% BenchmarkQuerierSelect/Head/100000of1000000-4 241992072 188801048 -21.98% BenchmarkQuerierSelect/Head/1000000of1000000-4 357192072 304001048 -14.89% BenchmarkQuerierSelect/SortedHead/1of1000000-4 229193928 229193928 +0.00% BenchmarkQuerierSelect/SortedHead/10of1000000-4 229195080 229195080 +0.00% BenchmarkQuerierSelect/SortedHead/100of1000000-4 229206600 229206600 +0.00% BenchmarkQuerierSelect/SortedHead/1000of1000000-4 229321800 229321800 +0.00% BenchmarkQuerierSelect/SortedHead/10000of1000000-4 230473800 230473800 +0.00% BenchmarkQuerierSelect/SortedHead/100000of1000000-4 241993800 241993800 +0.00% BenchmarkQuerierSelect/SortedHead/1000000of1000000-4 357193800 357193800 +0.00% BenchmarkQuerierSelect/Block/1of1000000-4 227201516 227201500 -0.00% BenchmarkQuerierSelect/Block/10of1000000-4 227202924 227202908 -0.00% BenchmarkQuerierSelect/Block/100of1000000-4 227217036 227217020 -0.00% BenchmarkQuerierSelect/Block/1000of1000000-4 227358156 227358140 -0.00% BenchmarkQuerierSelect/Block/10000of1000000-4 228769356 228769340 -0.00% BenchmarkQuerierSelect/Block/100000of1000000-4 242881356 242881340 -0.00% BenchmarkQuerierSelect/Block/1000000of1000000-4 384001616 384001600 -0.00% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-17 03:21:44 -08:00
if err != nil {
return err
}
return remote.StreamChunkedReadResponses(
remote.NewChunkedWriter(w, f),
int64(i),
set,
sortedExternalLabels,
api.remoteReadMaxBytesInFrame,
)
})
if err != nil {
if httpErr, ok := err.(remote.HTTPError); ok {
http.Error(w, httpErr.Error(), httpErr.Status())
return
}
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
default:
w.Header().Set("Content-Type", "application/x-protobuf")
w.Header().Set("Content-Encoding", "snappy")
2017-10-23 13:28:17 -07:00
// On empty or unknown types in req.AcceptedResponseTypes we default to non streamed, raw samples response.
resp := prompb.ReadResponse{
Results: make([]*prompb.QueryResult, len(req.Queries)),
2017-10-23 13:28:17 -07:00
}
for i, query := range req.Queries {
err := api.remoteReadQuery(ctx, query, externalLabels, func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) error {
set, _, err := querier.Select(false, hints, filteredMatchers...)
Don't sort postings if we only have one block. Sorting the heads postings can be quite slow. We only need sorted series when merging with another querier, so only sort then. This will make big queries that only touch the head faster, though queries that touch both the head and a block will still be the same speed. This probably won't help much with graphing unless the range is under an hour, however it should make most recording rules faster. Add gaurantee that remote read streaming produces sorted series. PromQL benchmarks for histograms show only 2-3% improvement, but they're only over 1k series. benchmark old ns/op new ns/op delta BenchmarkQuerierSelect/Head/1of1000000-4 1375486282 507657736 -63.09% BenchmarkQuerierSelect/Head/10of1000000-4 1387859004 507769850 -63.41% BenchmarkQuerierSelect/Head/100of1000000-4 1387087935 506029110 -63.52% BenchmarkQuerierSelect/Head/1000of1000000-4 1386869064 504521986 -63.62% BenchmarkQuerierSelect/Head/10000of1000000-4 1386213685 505210422 -63.55% BenchmarkQuerierSelect/Head/100000of1000000-4 1392754988 529842406 -61.96% BenchmarkQuerierSelect/Head/1000000of1000000-4 1569414722 725059506 -53.80% BenchmarkQuerierSelect/SortedHead/1of1000000-4 1381019902 1370495863 -0.76% BenchmarkQuerierSelect/SortedHead/10of1000000-4 1375696209 1366789468 -0.65% BenchmarkQuerierSelect/SortedHead/100of1000000-4 1386009422 1364519297 -1.55% BenchmarkQuerierSelect/SortedHead/1000of1000000-4 1377700532 1364486191 -0.96% BenchmarkQuerierSelect/SortedHead/10000of1000000-4 1383539536 1369545314 -1.01% BenchmarkQuerierSelect/SortedHead/100000of1000000-4 1410089163 1394731339 -1.09% BenchmarkQuerierSelect/SortedHead/1000000of1000000-4 1634744148 1581554956 -3.25% BenchmarkQuerierSelect/Block/1of1000000-4 881741242 879839470 -0.22% BenchmarkQuerierSelect/Block/10of1000000-4 880381562 882846038 +0.28% BenchmarkQuerierSelect/Block/100of1000000-4 887519357 881016916 -0.73% BenchmarkQuerierSelect/Block/1000of1000000-4 902194205 883433524 -2.08% BenchmarkQuerierSelect/Block/10000of1000000-4 892321964 885130170 -0.81% BenchmarkQuerierSelect/Block/100000of1000000-4 938604466 933527150 -0.54% BenchmarkQuerierSelect/Block/1000000of1000000-4 1313510845 1295881124 -1.34% benchmark old allocs new allocs delta BenchmarkQuerierSelect/Head/1of1000000-4 4000056 4000018 -0.00% BenchmarkQuerierSelect/Head/10of1000000-4 4000074 4000036 -0.00% BenchmarkQuerierSelect/Head/100of1000000-4 4000254 4000216 -0.00% BenchmarkQuerierSelect/Head/1000of1000000-4 4002054 4002016 -0.00% BenchmarkQuerierSelect/Head/10000of1000000-4 4020054 4020016 -0.00% BenchmarkQuerierSelect/Head/100000of1000000-4 4200054 4200016 -0.00% BenchmarkQuerierSelect/Head/1000000of1000000-4 6000054 6000016 -0.00% BenchmarkQuerierSelect/SortedHead/1of1000000-4 4000071 4000071 +0.00% BenchmarkQuerierSelect/SortedHead/10of1000000-4 4000089 4000089 +0.00% BenchmarkQuerierSelect/SortedHead/100of1000000-4 4000269 4000269 +0.00% BenchmarkQuerierSelect/SortedHead/1000of1000000-4 4002069 4002069 +0.00% BenchmarkQuerierSelect/SortedHead/10000of1000000-4 4020069 4020069 +0.00% BenchmarkQuerierSelect/SortedHead/100000of1000000-4 4200069 4200069 +0.00% BenchmarkQuerierSelect/SortedHead/1000000of1000000-4 6000069 6000069 +0.00% BenchmarkQuerierSelect/Block/1of1000000-4 6000023 6000022 -0.00% BenchmarkQuerierSelect/Block/10of1000000-4 6000059 6000058 -0.00% BenchmarkQuerierSelect/Block/100of1000000-4 6000419 6000418 -0.00% BenchmarkQuerierSelect/Block/1000of1000000-4 6004019 6004018 -0.00% BenchmarkQuerierSelect/Block/10000of1000000-4 6040019 6040018 -0.00% BenchmarkQuerierSelect/Block/100000of1000000-4 6400019 6400018 -0.00% BenchmarkQuerierSelect/Block/1000000of1000000-4 10000020 10000019 -0.00% benchmark old bytes new bytes delta BenchmarkQuerierSelect/Head/1of1000000-4 229192200 176001176 -23.21% BenchmarkQuerierSelect/Head/10of1000000-4 229193352 176002328 -23.21% BenchmarkQuerierSelect/Head/100of1000000-4 229204872 176013848 -23.21% BenchmarkQuerierSelect/Head/1000of1000000-4 229320072 176129048 -23.20% BenchmarkQuerierSelect/Head/10000of1000000-4 230472072 177281048 -23.08% BenchmarkQuerierSelect/Head/100000of1000000-4 241992072 188801048 -21.98% BenchmarkQuerierSelect/Head/1000000of1000000-4 357192072 304001048 -14.89% BenchmarkQuerierSelect/SortedHead/1of1000000-4 229193928 229193928 +0.00% BenchmarkQuerierSelect/SortedHead/10of1000000-4 229195080 229195080 +0.00% BenchmarkQuerierSelect/SortedHead/100of1000000-4 229206600 229206600 +0.00% BenchmarkQuerierSelect/SortedHead/1000of1000000-4 229321800 229321800 +0.00% BenchmarkQuerierSelect/SortedHead/10000of1000000-4 230473800 230473800 +0.00% BenchmarkQuerierSelect/SortedHead/100000of1000000-4 241993800 241993800 +0.00% BenchmarkQuerierSelect/SortedHead/1000000of1000000-4 357193800 357193800 +0.00% BenchmarkQuerierSelect/Block/1of1000000-4 227201516 227201500 -0.00% BenchmarkQuerierSelect/Block/10of1000000-4 227202924 227202908 -0.00% BenchmarkQuerierSelect/Block/100of1000000-4 227217036 227217020 -0.00% BenchmarkQuerierSelect/Block/1000of1000000-4 227358156 227358140 -0.00% BenchmarkQuerierSelect/Block/10000of1000000-4 228769356 228769340 -0.00% BenchmarkQuerierSelect/Block/100000of1000000-4 242881356 242881340 -0.00% BenchmarkQuerierSelect/Block/1000000of1000000-4 384001616 384001600 -0.00% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-17 03:21:44 -08:00
if err != nil {
return err
}
2017-10-23 13:28:17 -07:00
resp.Results[i], err = remote.ToQueryResult(set, api.remoteReadSampleLimit)
2017-10-23 13:28:17 -07:00
if err != nil {
return err
}
for _, ts := range resp.Results[i].Timeseries {
ts.Labels = remote.MergeLabels(ts.Labels, sortedExternalLabels)
}
return nil
})
if err != nil {
if httpErr, ok := err.(remote.HTTPError); ok {
http.Error(w, httpErr.Error(), httpErr.Status())
2017-10-23 13:28:17 -07:00
return
}
http.Error(w, err.Error(), http.StatusInternalServerError)
return
2017-10-23 13:28:17 -07:00
}
}
if err := remote.EncodeReadResponse(&resp, w); err != nil {
2017-11-23 04:50:06 -08:00
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
}
// filterExtLabelsFromMatchers change equality matchers which match external labels
// to a matcher that looks for an empty label,
// as that label should not be present in the storage.
func filterExtLabelsFromMatchers(pbMatchers []*prompb.LabelMatcher, externalLabels map[string]string) ([]*labels.Matcher, error) {
matchers, err := remote.FromLabelMatchers(pbMatchers)
if err != nil {
return nil, err
}
filteredMatchers := make([]*labels.Matcher, 0, len(matchers))
for _, m := range matchers {
value := externalLabels[m.Name]
if m.Type == labels.MatchEqual && value == m.Value {
matcher, err := labels.NewMatcher(labels.MatchEqual, m.Name, "")
if err != nil {
return nil, err
}
filteredMatchers = append(filteredMatchers, matcher)
} else {
filteredMatchers = append(filteredMatchers, m)
2017-10-23 13:28:17 -07:00
}
}
2017-10-23 13:28:17 -07:00
return filteredMatchers, nil
}
func (api *API) remoteReadQuery(ctx context.Context, query *prompb.Query, externalLabels map[string]string, seriesHandleFn func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) error) error {
filteredMatchers, err := filterExtLabelsFromMatchers(query.Matchers, externalLabels)
if err != nil {
return err
}
querier, err := api.Queryable.Querier(ctx, query.StartTimestampMs, query.EndTimestampMs)
if err != nil {
return err
}
defer func() {
if err := querier.Close(); err != nil {
level.Warn(api.logger).Log("msg", "Error on querier close", "err", err.Error())
}
}()
var hints *storage.SelectHints
if query.Hints != nil {
hints = &storage.SelectHints{
Start: query.Hints.StartMs,
End: query.Hints.EndMs,
Step: query.Hints.StepMs,
Func: query.Hints.Func,
Grouping: query.Hints.Grouping,
Range: query.Hints.RangeMs,
By: query.Hints.By,
}
}
return seriesHandleFn(querier, hints, filteredMatchers)
2017-10-23 13:28:17 -07:00
}
func (api *API) deleteSeries(r *http.Request) apiFuncResult {
if !api.enableAdmin {
return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("admin APIs disabled")}, nil, nil}
}
db := api.db()
if db == nil {
return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("TSDB not ready")}, nil, nil}
}
if err := r.ParseForm(); err != nil {
return apiFuncResult{nil, &apiError{errorBadData, errors.Wrap(err, "error parsing form values")}, nil, nil}
}
if len(r.Form["match[]"]) == 0 {
return apiFuncResult{nil, &apiError{errorBadData, errors.New("no match[] parameter provided")}, nil, nil}
}
start, err := parseTimeParam(r, "start", minTime)
if err != nil {
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
end, err := parseTimeParam(r, "end", maxTime)
if err != nil {
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
for _, s := range r.Form["match[]"] {
matchers, err := parser.ParseMetricSelector(s)
if err != nil {
return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
}
if err := db.Delete(timestamp.FromTime(start), timestamp.FromTime(end), matchers...); err != nil {
return apiFuncResult{nil, &apiError{errorInternal, err}, nil, nil}
}
}
return apiFuncResult{nil, nil, nil, nil}
}
func (api *API) snapshot(r *http.Request) apiFuncResult {
if !api.enableAdmin {
return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("admin APIs disabled")}, nil, nil}
}
var (
skipHead bool
err error
)
if r.FormValue("skip_head") != "" {
skipHead, err = strconv.ParseBool(r.FormValue("skip_head"))
if err != nil {
return apiFuncResult{nil, &apiError{errorBadData, errors.Wrapf(err, "unable to parse boolean 'skip_head' argument")}, nil, nil}
}
}
db := api.db()
if db == nil {
return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("TSDB not ready")}, nil, nil}
}
var (
snapdir = filepath.Join(db.Dir(), "snapshots")
name = fmt.Sprintf("%s-%x",
time.Now().UTC().Format("20060102T150405Z0700"),
rand.Int())
dir = filepath.Join(snapdir, name)
)
if err := os.MkdirAll(dir, 0777); err != nil {
return apiFuncResult{nil, &apiError{errorInternal, errors.Wrap(err, "create snapshot directory")}, nil, nil}
}
if err := db.Snapshot(dir, !skipHead); err != nil {
return apiFuncResult{nil, &apiError{errorInternal, errors.Wrap(err, "create snapshot")}, nil, nil}
}
return apiFuncResult{struct {
Name string `json:"name"`
}{name}, nil, nil, nil}
}
func (api *API) cleanTombstones(r *http.Request) apiFuncResult {
if !api.enableAdmin {
return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("admin APIs disabled")}, nil, nil}
}
db := api.db()
if db == nil {
return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("TSDB not ready")}, nil, nil}
}
if err := db.CleanTombstones(); err != nil {
return apiFuncResult{nil, &apiError{errorInternal, err}, nil, nil}
}
return apiFuncResult{nil, nil, nil, nil}
}
func (api *API) respond(w http.ResponseWriter, data interface{}, warnings storage.Warnings) {
statusMessage := statusSuccess
var warningStrings []string
for _, warning := range warnings {
warningStrings = append(warningStrings, warning.Error())
}
json := jsoniter.ConfigCompatibleWithStandardLibrary
b, err := json.Marshal(&response{
Status: statusMessage,
Data: data,
Warnings: warningStrings,
})
if err != nil {
level.Error(api.logger).Log("msg", "error marshaling json response", "err", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if n, err := w.Write(b); err != nil {
level.Error(api.logger).Log("msg", "error writing response", "bytesWritten", n, "err", err)
}
}
func (api *API) respondError(w http.ResponseWriter, apiErr *apiError, data interface{}) {
json := jsoniter.ConfigCompatibleWithStandardLibrary
b, err := json.Marshal(&response{
Status: statusError,
ErrorType: apiErr.typ,
Error: apiErr.err.Error(),
Data: data,
})
2019-11-02 08:53:32 -07:00
if err != nil {
level.Error(api.logger).Log("msg", "error marshaling json response", "err", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var code int
switch apiErr.typ {
case errorBadData:
code = http.StatusBadRequest
case errorExec:
code = 422
case errorCanceled, errorTimeout:
code = http.StatusServiceUnavailable
case errorInternal:
code = http.StatusInternalServerError
case errorNotFound:
code = http.StatusNotFound
default:
code = http.StatusInternalServerError
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
if n, err := w.Write(b); err != nil {
level.Error(api.logger).Log("msg", "error writing response", "bytesWritten", n, "err", err)
}
}
func parseTimeParam(r *http.Request, paramName string, defaultValue time.Time) (time.Time, error) {
val := r.FormValue(paramName)
if val == "" {
return defaultValue, nil
}
result, err := parseTime(val)
if err != nil {
return time.Time{}, errors.Wrapf(err, "Invalid time value for '%s'", paramName)
}
return result, nil
}
func parseTime(s string) (time.Time, error) {
if t, err := strconv.ParseFloat(s, 64); err == nil {
s, ns := math.Modf(t)
ns = math.Round(ns*1000) / 1000
return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
}
if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
return t, nil
}
// Stdlib's time parser can only handle 4 digit years. As a workaround until
// that is fixed we want to at least support our own boundary times.
// Context: https://github.com/prometheus/client_golang/issues/614
// Upstream issue: https://github.com/golang/go/issues/20555
switch s {
case minTimeFormatted:
return minTime, nil
case maxTimeFormatted:
return maxTime, nil
}
return time.Time{}, errors.Errorf("cannot parse %q to a valid timestamp", s)
}
func parseDuration(s string) (time.Duration, error) {
if d, err := strconv.ParseFloat(s, 64); err == nil {
ts := d * float64(time.Second)
if ts > float64(math.MaxInt64) || ts < float64(math.MinInt64) {
return 0, errors.Errorf("cannot parse %q to a valid duration. It overflows int64", s)
}
return time.Duration(ts), nil
}
if d, err := model.ParseDuration(s); err == nil {
return time.Duration(d), nil
}
return 0, errors.Errorf("cannot parse %q to a valid duration", s)
}
func marshalPointJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
p := *((*promql.Point)(ptr))
stream.WriteArrayStart()
// Write out the timestamp as a float divided by 1000.
// This is ~3x faster than converting to a float.
t := p.T
if t < 0 {
stream.WriteRaw(`-`)
t = -t
}
stream.WriteInt64(t / 1000)
fraction := t % 1000
if fraction != 0 {
stream.WriteRaw(`.`)
if fraction < 100 {
stream.WriteRaw(`0`)
}
if fraction < 10 {
stream.WriteRaw(`0`)
}
stream.WriteInt64(fraction)
}
stream.WriteMore()
stream.WriteRaw(`"`)
// Taken from https://github.com/json-iterator/go/blob/master/stream_float.go#L71 as a workaround
// to https://github.com/json-iterator/go/issues/365 (jsoniter, to follow json standard, doesn't allow inf/nan).
buf := stream.Buffer()
abs := math.Abs(p.V)
fmt := byte('f')
// Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right.
if abs != 0 {
if abs < 1e-6 || abs >= 1e21 {
fmt = 'e'
}
}
buf = strconv.AppendFloat(buf, p.V, fmt, -1, 64)
stream.SetBuffer(buf)
stream.WriteRaw(`"`)
stream.WriteArrayEnd()
}
func marshalPointJSONIsEmpty(ptr unsafe.Pointer) bool {
return false
}