From 5602328c7ce724d9d0fb395147771f4fedabcb2f Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Mon, 30 Mar 2015 19:13:36 +0200 Subject: [PATCH] Refactor query evaluation. This copies the evaluation logic from the current rules/ package. The new engine handles the execution process from query string to final result. It provides query timeout and cancellation and general flexibility for future changes. functions.go: Add evaluation implementation. Slight changes to in/out data but not to the processing logic. quantile.go: No changes. analyzer.go: No changes. engine.go: Actually new part. Mainly consists of evaluation methods which were not changed. setup_test.go: Copy of rules/helpers_test.go to setup test storage. promql_test.go: Copy of rules/rules_test.go. --- Godeps/Godeps.json | 4 + .../src/golang.org/x/net/context/context.go | 447 +++++ .../golang.org/x/net/context/context_test.go | 575 ++++++ .../x/net/context/withtimeout_test.go | 26 + promql/analyzer.go | 170 ++ promql/ast.go | 2 +- promql/engine.go | 1219 ++++++++++++ promql/engine_test.go | 272 +++ promql/functions.go | 583 +++++- promql/parse.go | 2 +- promql/parse_test.go | 2 +- promql/printer.go | 45 +- promql/promql_test.go | 1656 +++++++++++++++++ promql/quantile.go | 106 ++ promql/setup_test.go | 486 +++++ 15 files changed, 5561 insertions(+), 34 deletions(-) create mode 100644 Godeps/_workspace/src/golang.org/x/net/context/context.go create mode 100644 Godeps/_workspace/src/golang.org/x/net/context/context_test.go create mode 100644 Godeps/_workspace/src/golang.org/x/net/context/withtimeout_test.go create mode 100644 promql/analyzer.go create mode 100644 promql/engine.go create mode 100644 promql/engine_test.go create mode 100644 promql/promql_test.go create mode 100644 promql/quantile.go create mode 100644 promql/setup_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1f3b264686..426f1e078f 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -7,6 +7,10 @@ "Comment": "null-5", "Rev": "75cd24fc2f2c2a2088577d12123ddee5f54e0675" }, + { + "ImportPath": "github.com/prometheus/procfs", + "Rev": "92faa308558161acab0ada1db048e9996ecec160" + }, { "ImportPath": "github.com/beorn7/perks/quantile", "Rev": "b965b613227fddccbfffe13eae360ed3fa822f8d" diff --git a/Godeps/_workspace/src/golang.org/x/net/context/context.go b/Godeps/_workspace/src/golang.org/x/net/context/context.go new file mode 100644 index 0000000000..ef2f3e86fe --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/context/context.go @@ -0,0 +1,447 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package context defines the Context type, which carries deadlines, +// cancelation signals, and other request-scoped values across API boundaries +// and between processes. +// +// Incoming requests to a server should create a Context, and outgoing calls to +// servers should accept a Context. The chain of function calls between must +// propagate the Context, optionally replacing it with a modified copy created +// using WithDeadline, WithTimeout, WithCancel, or WithValue. +// +// Programs that use Contexts should follow these rules to keep interfaces +// consistent across packages and enable static analysis tools to check context +// propagation: +// +// Do not store Contexts inside a struct type; instead, pass a Context +// explicitly to each function that needs it. The Context should be the first +// parameter, typically named ctx: +// +// func DoSomething(ctx context.Context, arg Arg) error { +// // ... use ctx ... +// } +// +// Do not pass a nil Context, even if a function permits it. Pass context.TODO +// if you are unsure about which Context to use. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +// +// The same Context may be passed to functions running in different goroutines; +// Contexts are safe for simultaneous use by multiple goroutines. +// +// See http://blog.golang.org/context for example code for a server that uses +// Contexts. +package context + +import ( + "errors" + "fmt" + "sync" + "time" +) + +// A Context carries a deadline, a cancelation signal, and other values across +// API boundaries. +// +// Context's methods may be called by multiple goroutines simultaneously. +type Context interface { + // Deadline returns the time when work done on behalf of this context + // should be canceled. Deadline returns ok==false when no deadline is + // set. Successive calls to Deadline return the same results. + Deadline() (deadline time.Time, ok bool) + + // Done returns a channel that's closed when work done on behalf of this + // context should be canceled. Done may return nil if this context can + // never be canceled. Successive calls to Done return the same value. + // + // WithCancel arranges for Done to be closed when cancel is called; + // WithDeadline arranges for Done to be closed when the deadline + // expires; WithTimeout arranges for Done to be closed when the timeout + // elapses. + // + // Done is provided for use in select statements: + // + // // Stream generates values with DoSomething and sends them to out + // // until DoSomething returns an error or ctx.Done is closed. + // func Stream(ctx context.Context, out <-chan Value) error { + // for { + // v, err := DoSomething(ctx) + // if err != nil { + // return err + // } + // select { + // case <-ctx.Done(): + // return ctx.Err() + // case out <- v: + // } + // } + // } + // + // See http://blog.golang.org/pipelines for more examples of how to use + // a Done channel for cancelation. + Done() <-chan struct{} + + // Err returns a non-nil error value after Done is closed. Err returns + // Canceled if the context was canceled or DeadlineExceeded if the + // context's deadline passed. No other values for Err are defined. + // After Done is closed, successive calls to Err return the same value. + Err() error + + // Value returns the value associated with this context for key, or nil + // if no value is associated with key. Successive calls to Value with + // the same key returns the same result. + // + // Use context values only for request-scoped data that transits + // processes and API boundaries, not for passing optional parameters to + // functions. + // + // A key identifies a specific value in a Context. Functions that wish + // to store values in Context typically allocate a key in a global + // variable then use that key as the argument to context.WithValue and + // Context.Value. A key can be any type that supports equality; + // packages should define keys as an unexported type to avoid + // collisions. + // + // Packages that define a Context key should provide type-safe accessors + // for the values stores using that key: + // + // // Package user defines a User type that's stored in Contexts. + // package user + // + // import "golang.org/x/net/context" + // + // // User is the type of value stored in the Contexts. + // type User struct {...} + // + // // key is an unexported type for keys defined in this package. + // // This prevents collisions with keys defined in other packages. + // type key int + // + // // userKey is the key for user.User values in Contexts. It is + // // unexported; clients use user.NewContext and user.FromContext + // // instead of using this key directly. + // var userKey key = 0 + // + // // NewContext returns a new Context that carries value u. + // func NewContext(ctx context.Context, u *User) context.Context { + // return context.WithValue(ctx, userKey, u) + // } + // + // // FromContext returns the User value stored in ctx, if any. + // func FromContext(ctx context.Context) (*User, bool) { + // u, ok := ctx.Value(userKey).(*User) + // return u, ok + // } + Value(key interface{}) interface{} +} + +// Canceled is the error returned by Context.Err when the context is canceled. +var Canceled = errors.New("context canceled") + +// DeadlineExceeded is the error returned by Context.Err when the context's +// deadline passes. +var DeadlineExceeded = errors.New("context deadline exceeded") + +// An emptyCtx is never canceled, has no values, and has no deadline. It is not +// struct{}, since vars of this type must have distinct addresses. +type emptyCtx int + +func (*emptyCtx) Deadline() (deadline time.Time, ok bool) { + return +} + +func (*emptyCtx) Done() <-chan struct{} { + return nil +} + +func (*emptyCtx) Err() error { + return nil +} + +func (*emptyCtx) Value(key interface{}) interface{} { + return nil +} + +func (e *emptyCtx) String() string { + switch e { + case background: + return "context.Background" + case todo: + return "context.TODO" + } + return "unknown empty Context" +} + +var ( + background = new(emptyCtx) + todo = new(emptyCtx) +) + +// Background returns a non-nil, empty Context. It is never canceled, has no +// values, and has no deadline. It is typically used by the main function, +// initialization, and tests, and as the top-level Context for incoming +// requests. +func Background() Context { + return background +} + +// TODO returns a non-nil, empty Context. Code should use context.TODO when +// it's unclear which Context to use or it's is not yet available (because the +// surrounding function has not yet been extended to accept a Context +// parameter). TODO is recognized by static analysis tools that determine +// whether Contexts are propagated correctly in a program. +func TODO() Context { + return todo +} + +// A CancelFunc tells an operation to abandon its work. +// A CancelFunc does not wait for the work to stop. +// After the first call, subsequent calls to a CancelFunc do nothing. +type CancelFunc func() + +// WithCancel returns a copy of parent with a new Done channel. The returned +// context's Done channel is closed when the returned cancel function is called +// or when the parent context's Done channel is closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { + c := newCancelCtx(parent) + propagateCancel(parent, &c) + return &c, func() { c.cancel(true, Canceled) } +} + +// newCancelCtx returns an initialized cancelCtx. +func newCancelCtx(parent Context) cancelCtx { + return cancelCtx{ + Context: parent, + done: make(chan struct{}), + } +} + +// propagateCancel arranges for child to be canceled when parent is. +func propagateCancel(parent Context, child canceler) { + if parent.Done() == nil { + return // parent is never canceled + } + if p, ok := parentCancelCtx(parent); ok { + p.mu.Lock() + if p.err != nil { + // parent has already been canceled + child.cancel(false, p.err) + } else { + if p.children == nil { + p.children = make(map[canceler]bool) + } + p.children[child] = true + } + p.mu.Unlock() + } else { + go func() { + select { + case <-parent.Done(): + child.cancel(false, parent.Err()) + case <-child.Done(): + } + }() + } +} + +// parentCancelCtx follows a chain of parent references until it finds a +// *cancelCtx. This function understands how each of the concrete types in this +// package represents its parent. +func parentCancelCtx(parent Context) (*cancelCtx, bool) { + for { + switch c := parent.(type) { + case *cancelCtx: + return c, true + case *timerCtx: + return &c.cancelCtx, true + case *valueCtx: + parent = c.Context + default: + return nil, false + } + } +} + +// removeChild removes a context from its parent. +func removeChild(parent Context, child canceler) { + p, ok := parentCancelCtx(parent) + if !ok { + return + } + p.mu.Lock() + if p.children != nil { + delete(p.children, child) + } + p.mu.Unlock() +} + +// A canceler is a context type that can be canceled directly. The +// implementations are *cancelCtx and *timerCtx. +type canceler interface { + cancel(removeFromParent bool, err error) + Done() <-chan struct{} +} + +// A cancelCtx can be canceled. When canceled, it also cancels any children +// that implement canceler. +type cancelCtx struct { + Context + + done chan struct{} // closed by the first cancel call. + + mu sync.Mutex + children map[canceler]bool // set to nil by the first cancel call + err error // set to non-nil by the first cancel call +} + +func (c *cancelCtx) Done() <-chan struct{} { + return c.done +} + +func (c *cancelCtx) Err() error { + c.mu.Lock() + defer c.mu.Unlock() + return c.err +} + +func (c *cancelCtx) String() string { + return fmt.Sprintf("%v.WithCancel", c.Context) +} + +// cancel closes c.done, cancels each of c's children, and, if +// removeFromParent is true, removes c from its parent's children. +func (c *cancelCtx) cancel(removeFromParent bool, err error) { + if err == nil { + panic("context: internal error: missing cancel error") + } + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return // already canceled + } + c.err = err + close(c.done) + for child := range c.children { + // NOTE: acquiring the child's lock while holding parent's lock. + child.cancel(false, err) + } + c.children = nil + c.mu.Unlock() + + if removeFromParent { + removeChild(c.Context, c) + } +} + +// WithDeadline returns a copy of the parent context with the deadline adjusted +// to be no later than d. If the parent's deadline is already earlier than d, +// WithDeadline(parent, d) is semantically equivalent to parent. The returned +// context's Done channel is closed when the deadline expires, when the returned +// cancel function is called, or when the parent context's Done channel is +// closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { + if cur, ok := parent.Deadline(); ok && cur.Before(deadline) { + // The current deadline is already sooner than the new one. + return WithCancel(parent) + } + c := &timerCtx{ + cancelCtx: newCancelCtx(parent), + deadline: deadline, + } + propagateCancel(parent, c) + d := deadline.Sub(time.Now()) + if d <= 0 { + c.cancel(true, DeadlineExceeded) // deadline has already passed + return c, func() { c.cancel(true, Canceled) } + } + c.mu.Lock() + defer c.mu.Unlock() + if c.err == nil { + c.timer = time.AfterFunc(d, func() { + c.cancel(true, DeadlineExceeded) + }) + } + return c, func() { c.cancel(true, Canceled) } +} + +// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to +// implement Done and Err. It implements cancel by stopping its timer then +// delegating to cancelCtx.cancel. +type timerCtx struct { + cancelCtx + timer *time.Timer // Under cancelCtx.mu. + + deadline time.Time +} + +func (c *timerCtx) Deadline() (deadline time.Time, ok bool) { + return c.deadline, true +} + +func (c *timerCtx) String() string { + return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now())) +} + +func (c *timerCtx) cancel(removeFromParent bool, err error) { + c.cancelCtx.cancel(false, err) + if removeFromParent { + // Remove this timerCtx from its parent cancelCtx's children. + removeChild(c.cancelCtx.Context, c) + } + c.mu.Lock() + if c.timer != nil { + c.timer.Stop() + c.timer = nil + } + c.mu.Unlock() +} + +// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete: +// +// func slowOperationWithTimeout(ctx context.Context) (Result, error) { +// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) +// defer cancel() // releases resources if slowOperation completes before timeout elapses +// return slowOperation(ctx) +// } +func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) { + return WithDeadline(parent, time.Now().Add(timeout)) +} + +// WithValue returns a copy of parent in which the value associated with key is +// val. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +func WithValue(parent Context, key interface{}, val interface{}) Context { + return &valueCtx{parent, key, val} +} + +// A valueCtx carries a key-value pair. It implements Value for that key and +// delegates all other calls to the embedded Context. +type valueCtx struct { + Context + key, val interface{} +} + +func (c *valueCtx) String() string { + return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val) +} + +func (c *valueCtx) Value(key interface{}) interface{} { + if c.key == key { + return c.val + } + return c.Context.Value(key) +} diff --git a/Godeps/_workspace/src/golang.org/x/net/context/context_test.go b/Godeps/_workspace/src/golang.org/x/net/context/context_test.go new file mode 100644 index 0000000000..faf67722a0 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/context/context_test.go @@ -0,0 +1,575 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package context + +import ( + "fmt" + "math/rand" + "runtime" + "strings" + "sync" + "testing" + "time" +) + +// otherContext is a Context that's not one of the types defined in context.go. +// This lets us test code paths that differ based on the underlying type of the +// Context. +type otherContext struct { + Context +} + +func TestBackground(t *testing.T) { + c := Background() + if c == nil { + t.Fatalf("Background returned nil") + } + select { + case x := <-c.Done(): + t.Errorf("<-c.Done() == %v want nothing (it should block)", x) + default: + } + if got, want := fmt.Sprint(c), "context.Background"; got != want { + t.Errorf("Background().String() = %q want %q", got, want) + } +} + +func TestTODO(t *testing.T) { + c := TODO() + if c == nil { + t.Fatalf("TODO returned nil") + } + select { + case x := <-c.Done(): + t.Errorf("<-c.Done() == %v want nothing (it should block)", x) + default: + } + if got, want := fmt.Sprint(c), "context.TODO"; got != want { + t.Errorf("TODO().String() = %q want %q", got, want) + } +} + +func TestWithCancel(t *testing.T) { + c1, cancel := WithCancel(Background()) + + if got, want := fmt.Sprint(c1), "context.Background.WithCancel"; got != want { + t.Errorf("c1.String() = %q want %q", got, want) + } + + o := otherContext{c1} + c2, _ := WithCancel(o) + contexts := []Context{c1, o, c2} + + for i, c := range contexts { + if d := c.Done(); d == nil { + t.Errorf("c[%d].Done() == %v want non-nil", i, d) + } + if e := c.Err(); e != nil { + t.Errorf("c[%d].Err() == %v want nil", i, e) + } + + select { + case x := <-c.Done(): + t.Errorf("<-c.Done() == %v want nothing (it should block)", x) + default: + } + } + + cancel() + time.Sleep(100 * time.Millisecond) // let cancelation propagate + + for i, c := range contexts { + select { + case <-c.Done(): + default: + t.Errorf("<-c[%d].Done() blocked, but shouldn't have", i) + } + if e := c.Err(); e != Canceled { + t.Errorf("c[%d].Err() == %v want %v", i, e, Canceled) + } + } +} + +func TestParentFinishesChild(t *testing.T) { + // Context tree: + // parent -> cancelChild + // parent -> valueChild -> timerChild + parent, cancel := WithCancel(Background()) + cancelChild, stop := WithCancel(parent) + defer stop() + valueChild := WithValue(parent, "key", "value") + timerChild, stop := WithTimeout(valueChild, 10000*time.Hour) + defer stop() + + select { + case x := <-parent.Done(): + t.Errorf("<-parent.Done() == %v want nothing (it should block)", x) + case x := <-cancelChild.Done(): + t.Errorf("<-cancelChild.Done() == %v want nothing (it should block)", x) + case x := <-timerChild.Done(): + t.Errorf("<-timerChild.Done() == %v want nothing (it should block)", x) + case x := <-valueChild.Done(): + t.Errorf("<-valueChild.Done() == %v want nothing (it should block)", x) + default: + } + + // The parent's children should contain the two cancelable children. + pc := parent.(*cancelCtx) + cc := cancelChild.(*cancelCtx) + tc := timerChild.(*timerCtx) + pc.mu.Lock() + if len(pc.children) != 2 || !pc.children[cc] || !pc.children[tc] { + t.Errorf("bad linkage: pc.children = %v, want %v and %v", + pc.children, cc, tc) + } + pc.mu.Unlock() + + if p, ok := parentCancelCtx(cc.Context); !ok || p != pc { + t.Errorf("bad linkage: parentCancelCtx(cancelChild.Context) = %v, %v want %v, true", p, ok, pc) + } + if p, ok := parentCancelCtx(tc.Context); !ok || p != pc { + t.Errorf("bad linkage: parentCancelCtx(timerChild.Context) = %v, %v want %v, true", p, ok, pc) + } + + cancel() + + pc.mu.Lock() + if len(pc.children) != 0 { + t.Errorf("pc.cancel didn't clear pc.children = %v", pc.children) + } + pc.mu.Unlock() + + // parent and children should all be finished. + check := func(ctx Context, name string) { + select { + case <-ctx.Done(): + default: + t.Errorf("<-%s.Done() blocked, but shouldn't have", name) + } + if e := ctx.Err(); e != Canceled { + t.Errorf("%s.Err() == %v want %v", name, e, Canceled) + } + } + check(parent, "parent") + check(cancelChild, "cancelChild") + check(valueChild, "valueChild") + check(timerChild, "timerChild") + + // WithCancel should return a canceled context on a canceled parent. + precanceledChild := WithValue(parent, "key", "value") + select { + case <-precanceledChild.Done(): + default: + t.Errorf("<-precanceledChild.Done() blocked, but shouldn't have") + } + if e := precanceledChild.Err(); e != Canceled { + t.Errorf("precanceledChild.Err() == %v want %v", e, Canceled) + } +} + +func TestChildFinishesFirst(t *testing.T) { + cancelable, stop := WithCancel(Background()) + defer stop() + for _, parent := range []Context{Background(), cancelable} { + child, cancel := WithCancel(parent) + + select { + case x := <-parent.Done(): + t.Errorf("<-parent.Done() == %v want nothing (it should block)", x) + case x := <-child.Done(): + t.Errorf("<-child.Done() == %v want nothing (it should block)", x) + default: + } + + cc := child.(*cancelCtx) + pc, pcok := parent.(*cancelCtx) // pcok == false when parent == Background() + if p, ok := parentCancelCtx(cc.Context); ok != pcok || (ok && pc != p) { + t.Errorf("bad linkage: parentCancelCtx(cc.Context) = %v, %v want %v, %v", p, ok, pc, pcok) + } + + if pcok { + pc.mu.Lock() + if len(pc.children) != 1 || !pc.children[cc] { + t.Errorf("bad linkage: pc.children = %v, cc = %v", pc.children, cc) + } + pc.mu.Unlock() + } + + cancel() + + if pcok { + pc.mu.Lock() + if len(pc.children) != 0 { + t.Errorf("child's cancel didn't remove self from pc.children = %v", pc.children) + } + pc.mu.Unlock() + } + + // child should be finished. + select { + case <-child.Done(): + default: + t.Errorf("<-child.Done() blocked, but shouldn't have") + } + if e := child.Err(); e != Canceled { + t.Errorf("child.Err() == %v want %v", e, Canceled) + } + + // parent should not be finished. + select { + case x := <-parent.Done(): + t.Errorf("<-parent.Done() == %v want nothing (it should block)", x) + default: + } + if e := parent.Err(); e != nil { + t.Errorf("parent.Err() == %v want nil", e) + } + } +} + +func testDeadline(c Context, wait time.Duration, t *testing.T) { + select { + case <-time.After(wait): + t.Fatalf("context should have timed out") + case <-c.Done(): + } + if e := c.Err(); e != DeadlineExceeded { + t.Errorf("c.Err() == %v want %v", e, DeadlineExceeded) + } +} + +func TestDeadline(t *testing.T) { + c, _ := WithDeadline(Background(), time.Now().Add(100*time.Millisecond)) + if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) { + t.Errorf("c.String() = %q want prefix %q", got, prefix) + } + testDeadline(c, 200*time.Millisecond, t) + + c, _ = WithDeadline(Background(), time.Now().Add(100*time.Millisecond)) + o := otherContext{c} + testDeadline(o, 200*time.Millisecond, t) + + c, _ = WithDeadline(Background(), time.Now().Add(100*time.Millisecond)) + o = otherContext{c} + c, _ = WithDeadline(o, time.Now().Add(300*time.Millisecond)) + testDeadline(c, 200*time.Millisecond, t) +} + +func TestTimeout(t *testing.T) { + c, _ := WithTimeout(Background(), 100*time.Millisecond) + if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) { + t.Errorf("c.String() = %q want prefix %q", got, prefix) + } + testDeadline(c, 200*time.Millisecond, t) + + c, _ = WithTimeout(Background(), 100*time.Millisecond) + o := otherContext{c} + testDeadline(o, 200*time.Millisecond, t) + + c, _ = WithTimeout(Background(), 100*time.Millisecond) + o = otherContext{c} + c, _ = WithTimeout(o, 300*time.Millisecond) + testDeadline(c, 200*time.Millisecond, t) +} + +func TestCanceledTimeout(t *testing.T) { + c, _ := WithTimeout(Background(), 200*time.Millisecond) + o := otherContext{c} + c, cancel := WithTimeout(o, 400*time.Millisecond) + cancel() + time.Sleep(100 * time.Millisecond) // let cancelation propagate + select { + case <-c.Done(): + default: + t.Errorf("<-c.Done() blocked, but shouldn't have") + } + if e := c.Err(); e != Canceled { + t.Errorf("c.Err() == %v want %v", e, Canceled) + } +} + +type key1 int +type key2 int + +var k1 = key1(1) +var k2 = key2(1) // same int as k1, different type +var k3 = key2(3) // same type as k2, different int + +func TestValues(t *testing.T) { + check := func(c Context, nm, v1, v2, v3 string) { + if v, ok := c.Value(k1).(string); ok == (len(v1) == 0) || v != v1 { + t.Errorf(`%s.Value(k1).(string) = %q, %t want %q, %t`, nm, v, ok, v1, len(v1) != 0) + } + if v, ok := c.Value(k2).(string); ok == (len(v2) == 0) || v != v2 { + t.Errorf(`%s.Value(k2).(string) = %q, %t want %q, %t`, nm, v, ok, v2, len(v2) != 0) + } + if v, ok := c.Value(k3).(string); ok == (len(v3) == 0) || v != v3 { + t.Errorf(`%s.Value(k3).(string) = %q, %t want %q, %t`, nm, v, ok, v3, len(v3) != 0) + } + } + + c0 := Background() + check(c0, "c0", "", "", "") + + c1 := WithValue(Background(), k1, "c1k1") + check(c1, "c1", "c1k1", "", "") + + if got, want := fmt.Sprint(c1), `context.Background.WithValue(1, "c1k1")`; got != want { + t.Errorf("c.String() = %q want %q", got, want) + } + + c2 := WithValue(c1, k2, "c2k2") + check(c2, "c2", "c1k1", "c2k2", "") + + c3 := WithValue(c2, k3, "c3k3") + check(c3, "c2", "c1k1", "c2k2", "c3k3") + + c4 := WithValue(c3, k1, nil) + check(c4, "c4", "", "c2k2", "c3k3") + + o0 := otherContext{Background()} + check(o0, "o0", "", "", "") + + o1 := otherContext{WithValue(Background(), k1, "c1k1")} + check(o1, "o1", "c1k1", "", "") + + o2 := WithValue(o1, k2, "o2k2") + check(o2, "o2", "c1k1", "o2k2", "") + + o3 := otherContext{c4} + check(o3, "o3", "", "c2k2", "c3k3") + + o4 := WithValue(o3, k3, nil) + check(o4, "o4", "", "c2k2", "") +} + +func TestAllocs(t *testing.T) { + bg := Background() + for _, test := range []struct { + desc string + f func() + limit float64 + gccgoLimit float64 + }{ + { + desc: "Background()", + f: func() { Background() }, + limit: 0, + gccgoLimit: 0, + }, + { + desc: fmt.Sprintf("WithValue(bg, %v, nil)", k1), + f: func() { + c := WithValue(bg, k1, nil) + c.Value(k1) + }, + limit: 3, + gccgoLimit: 3, + }, + { + desc: "WithTimeout(bg, 15*time.Millisecond)", + f: func() { + c, _ := WithTimeout(bg, 15*time.Millisecond) + <-c.Done() + }, + limit: 8, + gccgoLimit: 13, + }, + { + desc: "WithCancel(bg)", + f: func() { + c, cancel := WithCancel(bg) + cancel() + <-c.Done() + }, + limit: 5, + gccgoLimit: 8, + }, + { + desc: "WithTimeout(bg, 100*time.Millisecond)", + f: func() { + c, cancel := WithTimeout(bg, 100*time.Millisecond) + cancel() + <-c.Done() + }, + limit: 8, + gccgoLimit: 25, + }, + } { + limit := test.limit + if runtime.Compiler == "gccgo" { + // gccgo does not yet do escape analysis. + // TOOD(iant): Remove this when gccgo does do escape analysis. + limit = test.gccgoLimit + } + if n := testing.AllocsPerRun(100, test.f); n > limit { + t.Errorf("%s allocs = %f want %d", test.desc, n, int(limit)) + } + } +} + +func TestSimultaneousCancels(t *testing.T) { + root, cancel := WithCancel(Background()) + m := map[Context]CancelFunc{root: cancel} + q := []Context{root} + // Create a tree of contexts. + for len(q) != 0 && len(m) < 100 { + parent := q[0] + q = q[1:] + for i := 0; i < 4; i++ { + ctx, cancel := WithCancel(parent) + m[ctx] = cancel + q = append(q, ctx) + } + } + // Start all the cancels in a random order. + var wg sync.WaitGroup + wg.Add(len(m)) + for _, cancel := range m { + go func(cancel CancelFunc) { + cancel() + wg.Done() + }(cancel) + } + // Wait on all the contexts in a random order. + for ctx := range m { + select { + case <-ctx.Done(): + case <-time.After(1 * time.Second): + buf := make([]byte, 10<<10) + n := runtime.Stack(buf, true) + t.Fatalf("timed out waiting for <-ctx.Done(); stacks:\n%s", buf[:n]) + } + } + // Wait for all the cancel functions to return. + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(1 * time.Second): + buf := make([]byte, 10<<10) + n := runtime.Stack(buf, true) + t.Fatalf("timed out waiting for cancel functions; stacks:\n%s", buf[:n]) + } +} + +func TestInterlockedCancels(t *testing.T) { + parent, cancelParent := WithCancel(Background()) + child, cancelChild := WithCancel(parent) + go func() { + parent.Done() + cancelChild() + }() + cancelParent() + select { + case <-child.Done(): + case <-time.After(1 * time.Second): + buf := make([]byte, 10<<10) + n := runtime.Stack(buf, true) + t.Fatalf("timed out waiting for child.Done(); stacks:\n%s", buf[:n]) + } +} + +func TestLayersCancel(t *testing.T) { + testLayers(t, time.Now().UnixNano(), false) +} + +func TestLayersTimeout(t *testing.T) { + testLayers(t, time.Now().UnixNano(), true) +} + +func testLayers(t *testing.T, seed int64, testTimeout bool) { + rand.Seed(seed) + errorf := func(format string, a ...interface{}) { + t.Errorf(fmt.Sprintf("seed=%d: %s", seed, format), a...) + } + const ( + timeout = 200 * time.Millisecond + minLayers = 30 + ) + type value int + var ( + vals []*value + cancels []CancelFunc + numTimers int + ctx = Background() + ) + for i := 0; i < minLayers || numTimers == 0 || len(cancels) == 0 || len(vals) == 0; i++ { + switch rand.Intn(3) { + case 0: + v := new(value) + ctx = WithValue(ctx, v, v) + vals = append(vals, v) + case 1: + var cancel CancelFunc + ctx, cancel = WithCancel(ctx) + cancels = append(cancels, cancel) + case 2: + var cancel CancelFunc + ctx, cancel = WithTimeout(ctx, timeout) + cancels = append(cancels, cancel) + numTimers++ + } + } + checkValues := func(when string) { + for _, key := range vals { + if val := ctx.Value(key).(*value); key != val { + errorf("%s: ctx.Value(%p) = %p want %p", when, key, val, key) + } + } + } + select { + case <-ctx.Done(): + errorf("ctx should not be canceled yet") + default: + } + if s, prefix := fmt.Sprint(ctx), "context.Background."; !strings.HasPrefix(s, prefix) { + t.Errorf("ctx.String() = %q want prefix %q", s, prefix) + } + t.Log(ctx) + checkValues("before cancel") + if testTimeout { + select { + case <-ctx.Done(): + case <-time.After(timeout + timeout/10): + errorf("ctx should have timed out") + } + checkValues("after timeout") + } else { + cancel := cancels[rand.Intn(len(cancels))] + cancel() + select { + case <-ctx.Done(): + default: + errorf("ctx should be canceled") + } + checkValues("after cancel") + } +} + +func TestCancelRemoves(t *testing.T) { + checkChildren := func(when string, ctx Context, want int) { + if got := len(ctx.(*cancelCtx).children); got != want { + t.Errorf("%s: context has %d children, want %d", when, got, want) + } + } + + ctx, _ := WithCancel(Background()) + checkChildren("after creation", ctx, 0) + _, cancel := WithCancel(ctx) + checkChildren("with WithCancel child ", ctx, 1) + cancel() + checkChildren("after cancelling WithCancel child", ctx, 0) + + ctx, _ = WithCancel(Background()) + checkChildren("after creation", ctx, 0) + _, cancel = WithTimeout(ctx, 60*time.Minute) + checkChildren("with WithTimeout child ", ctx, 1) + cancel() + checkChildren("after cancelling WithTimeout child", ctx, 0) +} diff --git a/Godeps/_workspace/src/golang.org/x/net/context/withtimeout_test.go b/Godeps/_workspace/src/golang.org/x/net/context/withtimeout_test.go new file mode 100644 index 0000000000..a6754dc368 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/context/withtimeout_test.go @@ -0,0 +1,26 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package context_test + +import ( + "fmt" + "time" + + "golang.org/x/net/context" +) + +func ExampleWithTimeout() { + // Pass a context with a timeout to tell a blocking function that it + // should abandon its work after the timeout elapses. + ctx, _ := context.WithTimeout(context.Background(), 100*time.Millisecond) + select { + case <-time.After(200 * time.Millisecond): + fmt.Println("overslept") + case <-ctx.Done(): + fmt.Println(ctx.Err()) // prints "context deadline exceeded" + } + // Output: + // context deadline exceeded +} diff --git a/promql/analyzer.go b/promql/analyzer.go new file mode 100644 index 0000000000..a4d2521f99 --- /dev/null +++ b/promql/analyzer.go @@ -0,0 +1,170 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promql + +import ( + "errors" + "time" + + "golang.org/x/net/context" + + clientmodel "github.com/prometheus/client_golang/model" + + "github.com/prometheus/prometheus/storage/local" +) + +// An Analyzer traverses an expression and determines which data has to be requested +// from the storage. It is bound to a context that allows cancellation and timing out. +type Analyzer struct { + // The storage from which to query data. + Storage local.Storage + // The expression being analyzed. + Expr Expr + // The time range for evaluation of Expr. + Start, End clientmodel.Timestamp + + // The preload times for different query time offsets. + offsetPreloadTimes map[time.Duration]preloadTimes +} + +// preloadTimes tracks which instants or ranges to preload for a set of +// fingerprints. One of these structs is collected for each offset by the query +// analyzer. +type preloadTimes struct { + // Instants require single samples to be loaded along the entire query + // range, with intervals between the samples corresponding to the query + // resolution. + instants map[clientmodel.Fingerprint]struct{} + // Ranges require loading a range of samples at each resolution step, + // stretching backwards from the current evaluation timestamp. The length of + // the range into the past is given by the duration, as in "foo[5m]". + ranges map[clientmodel.Fingerprint]time.Duration +} + +// Analyze the provided expression and attach metrics and fingerprints to data-selecting +// AST nodes that are later used to preload the data from the storage. +func (a *Analyzer) Analyze(ctx context.Context) error { + a.offsetPreloadTimes = map[time.Duration]preloadTimes{} + + getPreloadTimes := func(offset time.Duration) preloadTimes { + if _, ok := a.offsetPreloadTimes[offset]; !ok { + a.offsetPreloadTimes[offset] = preloadTimes{ + instants: map[clientmodel.Fingerprint]struct{}{}, + ranges: map[clientmodel.Fingerprint]time.Duration{}, + } + } + return a.offsetPreloadTimes[offset] + } + + // Retrieve fingerprints and metrics for the required time range for + // each metric or matrix selector node. + Inspect(a.Expr, func(node Node) bool { + switch n := node.(type) { + case *VectorSelector: + pt := getPreloadTimes(n.Offset) + fpts := a.Storage.GetFingerprintsForLabelMatchers(n.LabelMatchers) + n.fingerprints = fpts + n.metrics = map[clientmodel.Fingerprint]clientmodel.COWMetric{} + n.iterators = map[clientmodel.Fingerprint]local.SeriesIterator{} + for _, fp := range fpts { + // Only add the fingerprint to the instants if not yet present in the + // ranges. Ranges always contain more points and span more time than + // instants for the same offset. + if _, alreadyInRanges := pt.ranges[fp]; !alreadyInRanges { + pt.instants[fp] = struct{}{} + } + n.metrics[fp] = a.Storage.GetMetricForFingerprint(fp) + } + case *MatrixSelector: + pt := getPreloadTimes(n.Offset) + fpts := a.Storage.GetFingerprintsForLabelMatchers(n.LabelMatchers) + n.fingerprints = fpts + n.metrics = map[clientmodel.Fingerprint]clientmodel.COWMetric{} + n.iterators = map[clientmodel.Fingerprint]local.SeriesIterator{} + for _, fp := range fpts { + if pt.ranges[fp] < n.Range { + pt.ranges[fp] = n.Range + // Delete the fingerprint from the instants. Ranges always contain more + // points and span more time than instants, so we don't need to track + // an instant for the same fingerprint, should we have one. + delete(pt.instants, fp) + } + n.metrics[fp] = a.Storage.GetMetricForFingerprint(fp) + } + } + return true + }) + + // Currently we do not return an error but we might place a context check in here + // or extend the stage in some other way. + return nil +} + +// Prepare the expression evaluation by preloading all required chunks from the storage +// and setting the respective storage iterators in the AST nodes. +func (a *Analyzer) Prepare(ctx context.Context) (local.Preloader, error) { + const env = "query preparation" + + if a.offsetPreloadTimes == nil { + return nil, errors.New("analysis must be performed before preparing query") + } + var err error + // The preloader must not be closed unless an error ocurred as closing + // unpins the preloaded chunks. + p := a.Storage.NewPreloader() + defer func() { + if err != nil { + p.Close() + } + }() + + // Preload all analyzed ranges. + for offset, pt := range a.offsetPreloadTimes { + if err = contextDone(ctx, env); err != nil { + return nil, err + } + + start := a.Start.Add(-offset) + end := a.End.Add(-offset) + for fp, rangeDuration := range pt.ranges { + err = p.PreloadRange(fp, start.Add(-rangeDuration), end, *stalenessDelta) + if err != nil { + return nil, err + } + } + for fp := range pt.instants { + err = p.PreloadRange(fp, start, end, *stalenessDelta) + if err != nil { + return nil, err + } + } + } + + // Attach storage iterators to AST nodes. + Inspect(a.Expr, func(node Node) bool { + switch n := node.(type) { + case *VectorSelector: + for _, fp := range n.fingerprints { + n.iterators[fp] = a.Storage.NewIterator(fp) + } + case *MatrixSelector: + for _, fp := range n.fingerprints { + n.iterators[fp] = a.Storage.NewIterator(fp) + } + } + return true + }) + + return p, nil +} diff --git a/promql/ast.go b/promql/ast.go index 861f10eb3c..fc7c26a758 100644 --- a/promql/ast.go +++ b/promql/ast.go @@ -180,7 +180,7 @@ type ParenExpr struct { // StringLiteral represents a string. type StringLiteral struct { - Str string + Val string } // UnaryExpr represents a unary operation on another expression. diff --git a/promql/engine.go b/promql/engine.go new file mode 100644 index 0000000000..dc3714ab02 --- /dev/null +++ b/promql/engine.go @@ -0,0 +1,1219 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promql + +import ( + "flag" + "fmt" + "io/ioutil" + "math" + "runtime" + "sort" + "sync" + "time" + + "golang.org/x/net/context" + + clientmodel "github.com/prometheus/client_golang/model" + + "github.com/prometheus/prometheus/stats" + "github.com/prometheus/prometheus/storage/local" + "github.com/prometheus/prometheus/storage/metric" +) + +var ( + stalenessDelta = flag.Duration("query.staleness-delta", 300*time.Second, "Staleness delta allowance during expression evaluations.") + defaultQueryTimeout = flag.Duration("query.timeout", 2*time.Minute, "Maximum time a query may take before being aborted.") +) + +// SampleStream is a stream of Values belonging to an attached COWMetric. +type SampleStream struct { + Metric clientmodel.COWMetric `json:"metric"` + Values metric.Values `json:"values"` +} + +// Sample is a single sample belonging to a COWMetric. +type Sample struct { + Metric clientmodel.COWMetric `json:"metric"` + Value clientmodel.SampleValue `json:"value"` + Timestamp clientmodel.Timestamp `json:"timestamp"` +} + +// Scalar is a scalar value evaluated at the set timestamp. +type Scalar struct { + Value clientmodel.SampleValue + Timestamp clientmodel.Timestamp +} + +func (s *Scalar) String() string { + return fmt.Sprintf("scalar: %v @[%v]", s.Value, s.Timestamp) +} + +// String is a string value evaluated at the set timestamp. +type String struct { + Value string + Timestamp clientmodel.Timestamp +} + +func (s *String) String() string { + return s.Value +} + +// Vector is basically only an alias for clientmodel.Samples, but the +// contract is that in a Vector, all Samples have the same timestamp. +type Vector []*Sample + +// Matrix is a slice of SampleStreams that implements sort.Interface and +// has a String method. +type Matrix []*SampleStream + +// Len implements sort.Interface. +func (matrix Matrix) Len() int { + return len(matrix) +} + +// Less implements sort.Interface. +func (matrix Matrix) Less(i, j int) bool { + return matrix[i].Metric.String() < matrix[j].Metric.String() +} + +// Swap implements sort.Interface. +func (matrix Matrix) Swap(i, j int) { + matrix[i], matrix[j] = matrix[j], matrix[i] +} + +// Value is a generic interface for values resulting from a query evaluation. +type Value interface { + Type() ExprType + String() string +} + +func (Matrix) Type() ExprType { return ExprMatrix } +func (Vector) Type() ExprType { return ExprVector } +func (*Scalar) Type() ExprType { return ExprScalar } +func (*String) Type() ExprType { return ExprString } + +// Result holds the resulting value of an execution or an error +// if any occurred. +type Result struct { + Err error + Value Value +} + +// Vector returns a vector if the result value is one. An error is returned if +// the result was an error or the result value is not a vector. +func (r *Result) Vector() (Vector, error) { + if r.Err != nil { + return nil, r.Err + } + v, ok := r.Value.(Vector) + if !ok { + return nil, fmt.Errorf("query result is not a vector") + } + return v, nil +} + +// Matrix returns a matrix. An error is returned if +// the result was an error or the result value is not a matrix. +func (r *Result) Matrix() (Matrix, error) { + if r.Err != nil { + return nil, r.Err + } + v, ok := r.Value.(Matrix) + if !ok { + return nil, fmt.Errorf("query result is not a matrix") + } + return v, nil +} + +// Scalar returns a scalar value. An error is returned if +// the result was an error or the result value is not a scalar. +func (r *Result) Scalar() (*Scalar, error) { + if r.Err != nil { + return nil, r.Err + } + v, ok := r.Value.(*Scalar) + if !ok { + return nil, fmt.Errorf("query result is not a scalar") + } + return v, nil +} + +func (r *Result) String() string { + if r.Err != nil { + return r.Err.Error() + } + if r.Value == nil { + return "" + } + return r.Value.String() +} + +type ( + // ErrQueryTimeout is returned if a query timed out during processing. + ErrQueryTimeout string + // ErrQueryCanceled is returned if a query was canceled during processing. + ErrQueryCanceled string + // ErrNoHandlers is returned if no handlers were registered for the + // execution of a statement. + ErrNoHandlers string +) + +func (e ErrQueryTimeout) Error() string { return fmt.Sprintf("query timed out in %s", e) } +func (e ErrQueryCanceled) Error() string { return fmt.Sprintf("query was canceled in %s", e) } +func (e ErrNoHandlers) Error() string { return fmt.Sprintf("no handlers registered to process %s", e) } + +// A Query is derived from an a raw query string and can be run against an engine +// it is associated with. +type Query interface { + // Exec processes the query and + Exec() *Result + // Statements returns the parsed statements of the query. + Statements() Statements + // Stats returns statistics about the lifetime of the query. + Stats() *stats.TimerGroup + // Cancel signals that a running query execution should be aborted. + Cancel() +} + +// query implements the Query interface. +type query struct { + // The original query string. + q string + // Statements of the parsed query. + stmts Statements + // On finished execution two bools indicating success of the execution + // are sent on the channel. + done chan bool + // Timer stats for the query execution. + stats *stats.TimerGroup + // Cancelation function for the query. + cancel func() + + // The engine against which the query is executed. + ng *Engine +} + +// Statements implements the Query interface. +func (q *query) Statements() Statements { + return q.stmts +} + +// Stats implements the Query interface. +func (q *query) Stats() *stats.TimerGroup { + return q.stats +} + +// Cancel implements the Query interface. +func (q *query) Cancel() { + if q.cancel != nil { + q.cancel() + } +} + +// Exec implements the Query interface. +func (q *query) Exec() *Result { + ctx, cancel := context.WithTimeout(q.ng.baseCtx, *defaultQueryTimeout) + q.cancel = cancel + + res, err := q.ng.exec(ctx, q) + return &Result{Err: err, Value: res} +} + +type ( + // AlertHandlers can be registered with an engine and are called on + // each executed alert statement. + AlertHandler func(context.Context, *AlertStmt) error + // RecordHandlers can be registered with an engine and are called on + // each executed record statement. + RecordHandler func(context.Context, *RecordStmt) error +) + +// contextDone returns an error if the context was canceled or timed out. +func contextDone(ctx context.Context, env string) error { + select { + case <-ctx.Done(): + err := ctx.Err() + switch err { + case context.Canceled: + return ErrQueryCanceled(env) + case context.DeadlineExceeded: + return ErrQueryTimeout(env) + default: + return err + } + default: + return nil + } +} + +// Engine handles the liftetime of queries from beginning to end. It is connected +// to a storage. +type Engine struct { + sync.RWMutex + + // The storage on which the engine operates. + storage local.Storage + + // The base context for all queries and its cancellation function. + baseCtx context.Context + cancelQueries func() + + // Handlers for the statements. + alertHandlers map[string]AlertHandler + recordHandlers map[string]RecordHandler +} + +// NewEngine returns a new engine. +func NewEngine(storage local.Storage) *Engine { + ctx, cancel := context.WithCancel(context.Background()) + return &Engine{ + storage: storage, + baseCtx: ctx, + cancelQueries: cancel, + alertHandlers: map[string]AlertHandler{}, + recordHandlers: map[string]RecordHandler{}, + } +} + +// Stop the engine and cancel all running queries. +func (ng *Engine) Stop() { + ng.cancelQueries() +} + +// NewQuery returns a new query of the given query string. +func (ng *Engine) NewQuery(qs string) (Query, error) { + stmts, err := ParseStmts("query", qs) + if err != nil { + return nil, err + } + query := &query{ + q: qs, + stmts: stmts, + ng: ng, + done: make(chan bool, 2), + stats: stats.NewTimerGroup(), + } + return query, nil +} + +// NewQueryFromFile reads a file and returns a query of statements it contains. +func (ng *Engine) NewQueryFromFile(filename string) (Query, error) { + content, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return ng.NewQuery(string(content)) +} + +// NewInstantQuery returns an evaluation query for the given expression at the given time. +func (ng *Engine) NewInstantQuery(es string, ts clientmodel.Timestamp) (Query, error) { + return ng.NewRangeQuery(es, ts, ts, 0) +} + +// NewRangeQuery returns an evaluation query for the given time range and with +// the resolution set by the interval. +func (ng *Engine) NewRangeQuery(qs string, start, end clientmodel.Timestamp, interval time.Duration) (Query, error) { + expr, err := ParseExpr("query", qs) + if err != nil { + return nil, err + } + es := &EvalStmt{ + Expr: expr, + Start: start, + End: end, + Interval: interval, + } + + query := &query{ + q: qs, + stmts: Statements{es}, + ng: ng, + done: make(chan bool, 2), + stats: stats.NewTimerGroup(), + } + return query, nil +} + +// exec executes all statements in the query. For evaluation statements only +// one statement per query is allowed, after which the execution returns. +func (ng *Engine) exec(ctx context.Context, q *query) (Value, error) { + const env = "query execution" + + // Cancel when execution is done or an error was raised. + defer q.cancel() + + // The base context might already be canceled (e.g. during shutdown). + if err := contextDone(ctx, env); err != nil { + return nil, err + } + + evalTimer := q.stats.GetTimer(stats.TotalEvalTime).Start() + defer evalTimer.Stop() + + ng.RLock() + alertHandlers := []AlertHandler{} + for _, h := range ng.alertHandlers { + alertHandlers = append(alertHandlers, h) + } + recordHandlers := []RecordHandler{} + for _, h := range ng.recordHandlers { + recordHandlers = append(recordHandlers, h) + } + ng.RUnlock() + + for _, stmt := range q.stmts { + switch s := stmt.(type) { + case *AlertStmt: + if len(alertHandlers) == 0 { + return nil, ErrNoHandlers("alert statement") + } + for _, h := range alertHandlers { + if err := contextDone(ctx, env); err != nil { + return nil, err + } + err := h(ctx, s) + if err != nil { + return nil, err + } + } + case *RecordStmt: + if len(recordHandlers) == 0 { + return nil, ErrNoHandlers("record statement") + } + for _, h := range recordHandlers { + if err := contextDone(ctx, env); err != nil { + return nil, err + } + err := h(ctx, s) + if err != nil { + return nil, err + } + } + case *EvalStmt: + // Currently, only one execution statement per query is allowed. + return ng.execEvalStmt(ctx, q, s) + + default: + panic(fmt.Errorf("statement of unknown type %T", stmt)) + } + } + return nil, nil +} + +// execEvalStmt evaluates the expression of an evaluation statement for the given time range. +func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *EvalStmt) (Value, error) { + prepareTimer := query.stats.GetTimer(stats.TotalQueryPreparationTime).Start() + analyzeTimer := query.stats.GetTimer(stats.QueryAnalysisTime).Start() + + // Only one execution statement per query is allowed. + analyzer := &Analyzer{ + Storage: ng.storage, + Expr: s.Expr, + Start: s.Start, + End: s.End, + } + err := analyzer.Analyze(ctx) + if err != nil { + analyzeTimer.Stop() + prepareTimer.Stop() + return nil, err + } + analyzeTimer.Stop() + + preloadTimer := query.stats.GetTimer(stats.PreloadTime).Start() + closer, err := analyzer.Prepare(ctx) + if err != nil { + preloadTimer.Stop() + prepareTimer.Stop() + return nil, err + } + defer closer.Close() + + preloadTimer.Stop() + prepareTimer.Stop() + + evalTimer := query.stats.GetTimer(stats.InnerEvalTime).Start() + // Instant evaluation. + if s.Start == s.End && s.Interval == 0 { + evaluator := &evaluator{ + Timestamp: s.Start, + ctx: ctx, + } + val, err := evaluator.Eval(s.Expr) + if err != nil { + return nil, err + } + + evalTimer.Stop() + return val, nil + } + + // Range evaluation. + sampleStreams := map[clientmodel.Fingerprint]*SampleStream{} + for ts := s.Start; !ts.After(s.End); ts = ts.Add(s.Interval) { + + if err := contextDone(ctx, "range evaluation"); err != nil { + return nil, err + } + + evaluator := &evaluator{ + Timestamp: ts, + ctx: ctx, + } + val, err := evaluator.Eval(s.Expr) + if err != nil { + return nil, err + } + vector, ok := val.(Vector) + if !ok { + return nil, fmt.Errorf("value for expression %q must be of type vector but is %s", s.Expr, val.Type()) + } + + for _, sample := range vector { + samplePair := metric.SamplePair{ + Value: sample.Value, + Timestamp: sample.Timestamp, + } + fp := sample.Metric.Metric.Fingerprint() + if sampleStreams[fp] == nil { + sampleStreams[fp] = &SampleStream{ + Metric: sample.Metric, + Values: metric.Values{samplePair}, + } + } else { + sampleStreams[fp].Values = append(sampleStreams[fp].Values, samplePair) + } + + } + } + evalTimer.Stop() + + if err := contextDone(ctx, "expression evaluation"); err != nil { + return nil, err + } + + appendTimer := query.stats.GetTimer(stats.ResultAppendTime).Start() + matrix := Matrix{} + for _, sampleStream := range sampleStreams { + matrix = append(matrix, sampleStream) + } + appendTimer.Stop() + + if err := contextDone(ctx, "expression evaluation"); err != nil { + return nil, err + } + + sortTimer := query.stats.GetTimer(stats.ResultSortTime).Start() + sort.Sort(matrix) + sortTimer.Stop() + + return matrix, nil +} + +// An evaluator evaluates given expressions at a fixed timestamp. It is attached to an +// engine through which it connects to a storage and reports errors. On timeout or +// cancellation of its context it terminates. +type evaluator struct { + ctx context.Context + + Timestamp clientmodel.Timestamp +} + +// fatalf causes a panic with the input formatted into an error. +func (ev *evaluator) errorf(format string, args ...interface{}) { + ev.error(fmt.Errorf(format, args...)) +} + +// fatal causes a panic with the given error. +func (ev *evaluator) error(err error) { + panic(err) +} + +// recover is the handler that turns panics into returns from the top level of evaluation. +func (ev *evaluator) recover(errp *error) { + e := recover() + if e != nil { + // Do not recover from runtime errors. + if _, ok := e.(runtime.Error); ok { + panic(e) + } + *errp = e.(error) + } +} + +// evalScalar attempts to evaluate e to a scalar value and errors otherwise. +func (ev *evaluator) evalScalar(e Expr) *Scalar { + val := ev.eval(e) + sv, ok := val.(*Scalar) + if !ok { + ev.errorf("expected scalar but got %s", val.Type()) + } + return sv +} + +// evalVector attempts to evaluate e to a vector value and errors otherwise. +func (ev *evaluator) evalVector(e Expr) Vector { + val := ev.eval(e) + vec, ok := val.(Vector) + if !ok { + ev.errorf("expected vector but got %s", val.Type()) + } + return vec +} + +// evalInt attempts to evaluate e into an integer and errors otherwise. +func (ev *evaluator) evalInt(e Expr) int { + sc := ev.evalScalar(e) + return int(sc.Value) +} + +// evalFloat attempts to evaluate e into a float and errors otherwise. +func (ev *evaluator) evalFloat(e Expr) float64 { + sc := ev.evalScalar(e) + return float64(sc.Value) +} + +// evalMatrix attempts to evaluate e into a matrix and errors otherwise. +func (ev *evaluator) evalMatrix(e Expr) Matrix { + val := ev.eval(e) + mat, ok := val.(Matrix) + if !ok { + ev.errorf("expected matrix but got %s", val.Type()) + } + return mat +} + +// evalMatrixBounds attempts to evaluate e to matrix boundaries and errors otherwise. +func (ev *evaluator) evalMatrixBounds(e Expr) Matrix { + ms, ok := e.(*MatrixSelector) + if !ok { + ev.errorf("matrix bounds can only be evaluated for matrix selectors, got %T", e) + } + return ev.matrixSelectorBounds(ms) +} + +// evalOneOf evaluates e and errors unless the result is of one of the given types. +func (ev *evaluator) evalOneOf(e Expr, t1, t2 ExprType) Value { + val := ev.eval(e) + if val.Type() != t1 && val.Type() != t2 { + ev.errorf("expected %s or %s but got %s", t1, t2, val.Type()) + } + return val +} + +func (ev *evaluator) Eval(expr Expr) (v Value, err error) { + defer ev.recover(&err) + return ev.eval(expr), nil +} + +// eval evaluates the given expression as the given AST expression node requires. +func (ev *evaluator) eval(expr Expr) Value { + // This is the top-level evaluation method. + // Thus, we check for timeout/cancellation here. + if err := contextDone(ev.ctx, "expression evaluation"); err != nil { + ev.error(err) + } + + switch e := expr.(type) { + case *AggregateExpr: + vector := ev.evalVector(e.Expr) + return ev.aggregation(e.Op, e.Grouping, e.KeepExtraLabels, vector) + + case *BinaryExpr: + lhs := ev.evalOneOf(e.LHS, ExprScalar, ExprVector) + rhs := ev.evalOneOf(e.RHS, ExprScalar, ExprVector) + + switch lt, rt := lhs.Type(), rhs.Type(); { + case lt == ExprScalar && rt == ExprScalar: + return &Scalar{ + Value: scalarBinop(e.Op, lhs.(*Scalar).Value, rhs.(*Scalar).Value), + Timestamp: ev.Timestamp, + } + + case lt == ExprVector && rt == ExprVector: + return ev.vectorBinop(e.Op, lhs.(Vector), rhs.(Vector), e.VectorMatching) + + case lt == ExprVector && rt == ExprScalar: + return ev.vectorScalarBinop(e.Op, lhs.(Vector), rhs.(*Scalar), false) + + case lt == ExprScalar && rt == ExprVector: + return ev.vectorScalarBinop(e.Op, rhs.(Vector), lhs.(*Scalar), true) + } + + case *Call: + return e.Func.Call(ev, e.Args) + + case *MatrixSelector: + return ev.matrixSelector(e) + + case *NumberLiteral: + return &Scalar{Value: e.Val, Timestamp: ev.Timestamp} + + case *ParenExpr: + return ev.eval(e.Expr) + + case *StringLiteral: + return &String{Value: e.Val, Timestamp: ev.Timestamp} + + case *UnaryExpr: + smpl := ev.evalScalar(e.Expr) + if e.Op == itemSUB { + smpl.Value = -smpl.Value + } + return smpl + + case *VectorSelector: + return ev.vectorSelector(e) + } + panic(fmt.Errorf("unhandled expression of type: %T", expr)) +} + +// vectorSelector evaluates a *VectorSelector expression. +func (ev *evaluator) vectorSelector(node *VectorSelector) Vector { + vec := Vector{} + for fp, it := range node.iterators { + sampleCandidates := it.GetValueAtTime(ev.Timestamp.Add(-node.Offset)) + samplePair := chooseClosestSample(sampleCandidates, ev.Timestamp.Add(-node.Offset)) + if samplePair != nil { + vec = append(vec, &Sample{ + Metric: node.metrics[fp], + Value: samplePair.Value, + Timestamp: ev.Timestamp, + }) + } + } + return vec +} + +// matrixSelector evaluates a *MatrixSelector expression. +func (ev *evaluator) matrixSelector(node *MatrixSelector) Matrix { + interval := metric.Interval{ + OldestInclusive: ev.Timestamp.Add(-node.Range - node.Offset), + NewestInclusive: ev.Timestamp.Add(-node.Offset), + } + + sampleStreams := make([]*SampleStream, 0, len(node.iterators)) + for fp, it := range node.iterators { + samplePairs := it.GetRangeValues(interval) + if len(samplePairs) == 0 { + continue + } + + if node.Offset != 0 { + for _, sp := range samplePairs { + sp.Timestamp = sp.Timestamp.Add(node.Offset) + } + } + + sampleStream := &SampleStream{ + Metric: node.metrics[fp], + Values: samplePairs, + } + sampleStreams = append(sampleStreams, sampleStream) + } + return Matrix(sampleStreams) +} + +// matrixSelectorBounds evaluates the boundaries of a *MatrixSelector. +func (ev *evaluator) matrixSelectorBounds(node *MatrixSelector) Matrix { + interval := metric.Interval{ + OldestInclusive: ev.Timestamp.Add(-node.Range - node.Offset), + NewestInclusive: ev.Timestamp.Add(-node.Offset), + } + + sampleStreams := make([]*SampleStream, 0, len(node.iterators)) + for fp, it := range node.iterators { + samplePairs := it.GetBoundaryValues(interval) + if len(samplePairs) == 0 { + continue + } + + sampleStream := &SampleStream{ + Metric: node.metrics[fp], + Values: samplePairs, + } + sampleStreams = append(sampleStreams, sampleStream) + } + return Matrix(sampleStreams) +} + +// vectorBinop evaluates a binary operation between two vector values. +func (ev *evaluator) vectorBinop(op itemType, lhs, rhs Vector, matching *VectorMatching) Vector { + result := make(Vector, 0, len(rhs)) + // The control flow below handles one-to-one or many-to-one matching. + // For one-to-many, swap sidedness and account for the swap when calculating + // values. + if matching.Card == CardOneToMany { + lhs, rhs = rhs, lhs + } + // All samples from the rhs hashed by the matching label/values. + rm := map[uint64]*Sample{} + // Maps the hash of the label values used for matching to the hashes of the label + // values of the include labels (if any). It is used to keep track of already + // inserted samples. + added := map[uint64][]uint64{} + + // Add all rhs samples to a map so we can easily find matches later. + for _, rs := range rhs { + hash := hashForMetric(rs.Metric.Metric, matching.On) + // The rhs is guaranteed to be the 'one' side. Having multiple samples + // with the same hash means that the matching is many-to-many, + // which is not supported. + if _, found := rm[hash]; matching.Card != CardManyToMany && found { + // Many-to-many matching not allowed. + ev.errorf("many-to-many matching not allowed") + } + // In many-to-many matching the entry is simply overwritten. It can thus only + // be used to check whether any matching rhs entry exists but not retrieve them all. + rm[hash] = rs + } + + // For all lhs samples find a respective rhs sample and perform + // the binary operation. + for _, ls := range lhs { + hash := hashForMetric(ls.Metric.Metric, matching.On) + // Any lhs sample we encounter in an OR operation belongs to the result. + if op == itemLOR { + ls.Metric = resultMetric(op, ls, nil, matching) + result = append(result, ls) + added[hash] = nil // Ensure matching rhs sample is not added later. + continue + } + + rs, found := rm[hash] // Look for a match in the rhs vector. + if !found { + continue + } + var value clientmodel.SampleValue + var keep bool + + if op == itemLAND { + value = ls.Value + keep = true + } else { + if _, exists := added[hash]; matching.Card == CardOneToOne && exists { + // Many-to-one matching must be explicit. + ev.errorf("many-to-one matching must be explicit") + } + // Account for potentially swapped sidedness. + vl, vr := ls.Value, rs.Value + if matching.Card == CardOneToMany { + vl, vr = vr, vl + } + value, keep = vectorElemBinop(op, vl, vr) + } + + if keep { + metric := resultMetric(op, ls, rs, matching) + // Check if the same label set has been added for a many-to-one matching before. + if matching.Card == CardManyToOne || matching.Card == CardOneToMany { + insHash := clientmodel.SignatureForLabels(metric.Metric, matching.Include) + if ihs, exists := added[hash]; exists { + for _, ih := range ihs { + if ih == insHash { + ev.errorf("metric with label set has already been matched") + } + } + added[hash] = append(ihs, insHash) + } else { + added[hash] = []uint64{insHash} + } + } + ns := &Sample{ + Metric: metric, + Value: value, + Timestamp: ev.Timestamp, + } + result = append(result, ns) + added[hash] = added[hash] // Set existance to true. + } + } + + // Add all remaining samples in the rhs in an OR operation if they + // have not been matched up with a lhs sample. + if op == itemLOR { + for hash, rs := range rm { + if _, exists := added[hash]; !exists { + rs.Metric = resultMetric(op, rs, nil, matching) + result = append(result, rs) + } + } + } + return result +} + +// vectorScalarBinop evaluates a binary operation between a vector and a scalar. +func (ev *evaluator) vectorScalarBinop(op itemType, lhs Vector, rhs *Scalar, swap bool) Vector { + vector := make(Vector, 0, len(lhs)) + + for _, lhsSample := range lhs { + lv, rv := lhsSample.Value, rhs.Value + // lhs always contains the vector. If the original position was different + // swap for calculating the value. + if swap { + lv, rv = rv, lv + } + value, keep := vectorElemBinop(op, lv, rv) + if keep { + lhsSample.Value = value + if shouldDropMetricName(op) { + lhsSample.Metric.Delete(clientmodel.MetricNameLabel) + } + vector = append(vector, lhsSample) + } + } + return vector +} + +// scalarBinop evaluates a binary operation between two scalars. +func scalarBinop(op itemType, lhs, rhs clientmodel.SampleValue) clientmodel.SampleValue { + switch op { + case itemADD: + return lhs + rhs + case itemSUB: + return lhs - rhs + case itemMUL: + return lhs * rhs + case itemDIV: + return lhs / rhs + case itemMOD: + if rhs != 0 { + return clientmodel.SampleValue(int(lhs) % int(rhs)) + } + return clientmodel.SampleValue(math.NaN()) + case itemEQL: + return btos(lhs == rhs) + case itemNEQ: + return btos(lhs != rhs) + case itemGTR: + return btos(lhs > rhs) + case itemLSS: + return btos(lhs < rhs) + case itemGTE: + return btos(lhs >= rhs) + case itemLTE: + return btos(lhs <= rhs) + } + panic(fmt.Errorf("operator %q not allowed for scalar operations", op)) +} + +// vectorElemBinop evaluates a binary operation between two vector elements. +func vectorElemBinop(op itemType, lhs, rhs clientmodel.SampleValue) (clientmodel.SampleValue, bool) { + switch op { + case itemADD: + return lhs + rhs, true + case itemSUB: + return lhs - rhs, true + case itemMUL: + return lhs * rhs, true + case itemDIV: + return lhs / rhs, true + case itemMOD: + if rhs != 0 { + return clientmodel.SampleValue(int(lhs) % int(rhs)), true + } + return clientmodel.SampleValue(math.NaN()), true + case itemEQL: + return lhs, lhs == rhs + case itemNEQ: + return lhs, lhs != rhs + case itemGTR: + return lhs, lhs > rhs + case itemLSS: + return lhs, lhs < rhs + case itemGTE: + return lhs, lhs >= rhs + case itemLTE: + return lhs, lhs <= rhs + } + panic(fmt.Errorf("operator %q not allowed for operations between vectors", op)) +} + +// labelIntersection returns the metric of common label/value pairs of two input metrics. +func labelIntersection(metric1, metric2 clientmodel.COWMetric) clientmodel.COWMetric { + for label, value := range metric1.Metric { + if metric2.Metric[label] != value { + metric1.Delete(label) + } + } + return metric1 +} + +type groupedAggregation struct { + labels clientmodel.COWMetric + value clientmodel.SampleValue + valuesSquaredSum clientmodel.SampleValue + groupCount int +} + +// aggregation evaluates an aggregation operation on a vector. +func (ev *evaluator) aggregation(op itemType, grouping clientmodel.LabelNames, keepExtra bool, vector Vector) Vector { + + result := map[uint64]*groupedAggregation{} + + for _, sample := range vector { + groupingKey := clientmodel.SignatureForLabels(sample.Metric.Metric, grouping) + + groupedResult, ok := result[groupingKey] + // Add a new group if it doesn't exist. + if !ok { + var m clientmodel.COWMetric + if keepExtra { + m = sample.Metric + m.Delete(clientmodel.MetricNameLabel) + } else { + m = clientmodel.COWMetric{ + Metric: clientmodel.Metric{}, + Copied: true, + } + for _, l := range grouping { + if v, ok := sample.Metric.Metric[l]; ok { + m.Set(l, v) + } + } + } + result[groupingKey] = &groupedAggregation{ + labels: m, + value: sample.Value, + valuesSquaredSum: sample.Value * sample.Value, + groupCount: 1, + } + continue + } + // Add the sample to the existing group. + if keepExtra { + groupedResult.labels = labelIntersection(groupedResult.labels, sample.Metric) + } + + switch op { + case itemSum: + groupedResult.value += sample.Value + case itemAvg: + groupedResult.value += sample.Value + groupedResult.groupCount++ + case itemMax: + if groupedResult.value < sample.Value { + groupedResult.value = sample.Value + } + case itemMin: + if groupedResult.value > sample.Value { + groupedResult.value = sample.Value + } + case itemCount: + groupedResult.groupCount++ + case itemStdvar, itemStddev: + groupedResult.value += sample.Value + groupedResult.valuesSquaredSum += sample.Value * sample.Value + groupedResult.groupCount++ + default: + panic(fmt.Errorf("expected aggregation operator but got %q", op)) + } + } + + // Construct the result vector from the aggregated groups. + resultVector := make(Vector, 0, len(result)) + + for _, aggr := range result { + switch op { + case itemAvg: + aggr.value = aggr.value / clientmodel.SampleValue(aggr.groupCount) + case itemCount: + aggr.value = clientmodel.SampleValue(aggr.groupCount) + case itemStdvar: + avg := float64(aggr.value) / float64(aggr.groupCount) + aggr.value = clientmodel.SampleValue(float64(aggr.valuesSquaredSum)/float64(aggr.groupCount) - avg*avg) + case itemStddev: + avg := float64(aggr.value) / float64(aggr.groupCount) + aggr.value = clientmodel.SampleValue(math.Sqrt(float64(aggr.valuesSquaredSum)/float64(aggr.groupCount) - avg*avg)) + default: + // For other aggregations, we already have the right value. + } + sample := &Sample{ + Metric: aggr.labels, + Value: aggr.value, + Timestamp: ev.Timestamp, + } + resultVector = append(resultVector, sample) + } + return resultVector +} + +// RegisterAlertHandler registers a new alert handler of the given name. +func (ng *Engine) RegisterAlertHandler(name string, h AlertHandler) { + ng.Lock() + ng.alertHandlers[name] = h + ng.Unlock() +} + +// RegisterRecordHandler registers a new record handler of the given name. +func (ng *Engine) RegisterRecordHandler(name string, h RecordHandler) { + ng.Lock() + ng.recordHandlers[name] = h + ng.Unlock() +} + +// UnregisterAlertHandler removes the alert handler with the given name. +func (ng *Engine) UnregisterAlertHandler(name string) { + ng.Lock() + delete(ng.alertHandlers, name) + ng.Unlock() +} + +// UnregisterRecordHandler removes the record handler with the given name. +func (ng *Engine) UnregisterRecordHandler(name string) { + ng.Lock() + delete(ng.recordHandlers, name) + ng.Unlock() +} + +// btos returns 1 if b is true, 0 otherwise. +func btos(b bool) clientmodel.SampleValue { + if b { + return 1 + } + return 0 +} + +// shouldDropMetricName returns whether the metric name should be dropped in the +// result of the op operation. +func shouldDropMetricName(op itemType) bool { + switch op { + case itemADD, itemSUB, itemDIV, itemMUL, itemMOD: + return true + default: + return false + } +} + +// resultMetric returns the metric for the given sample(s) based on the vector +// binary operation and the matching options. +func resultMetric(op itemType, ls, rs *Sample, matching *VectorMatching) clientmodel.COWMetric { + if len(matching.On) == 0 || op == itemLOR || op == itemLAND { + if shouldDropMetricName(op) { + ls.Metric.Delete(clientmodel.MetricNameLabel) + } + return ls.Metric + } + + m := clientmodel.Metric{} + for _, ln := range matching.On { + m[ln] = ls.Metric.Metric[ln] + } + + for _, ln := range matching.Include { + // Included labels from the `group_x` modifier are taken from the "many"-side. + v, ok := ls.Metric.Metric[ln] + if ok { + m[ln] = v + } + } + return clientmodel.COWMetric{false, m} +} + +// hashForMetric calculates a hash value for the given metric based on the matching +// options for the binary operation. +func hashForMetric(metric clientmodel.Metric, withLabels clientmodel.LabelNames) uint64 { + var labels clientmodel.LabelNames + + if len(withLabels) > 0 { + var match bool + for _, ln := range withLabels { + if _, match = metric[ln]; !match { + break + } + } + // If the metric does not contain the labels to match on, build the hash + // over the whole metric to give it a unique hash. + if !match { + labels = make(clientmodel.LabelNames, 0, len(metric)) + for ln := range metric { + labels = append(labels, ln) + } + } else { + labels = withLabels + } + } else { + labels = make(clientmodel.LabelNames, 0, len(metric)) + for ln := range metric { + if ln != clientmodel.MetricNameLabel { + labels = append(labels, ln) + } + } + } + return clientmodel.SignatureForLabels(metric, labels) +} + +// chooseClosestSample chooses the closest sample of a list of samples +// surrounding a given target time. If samples are found both before and after +// the target time, the sample value is interpolated between these. Otherwise, +// the single closest sample is returned verbatim. +func chooseClosestSample(samples metric.Values, timestamp clientmodel.Timestamp) *metric.SamplePair { + var closestBefore *metric.SamplePair + var closestAfter *metric.SamplePair + for _, candidate := range samples { + delta := candidate.Timestamp.Sub(timestamp) + // Samples before target time. + if delta < 0 { + // Ignore samples outside of staleness policy window. + if -delta > *stalenessDelta { + continue + } + // Ignore samples that are farther away than what we've seen before. + if closestBefore != nil && candidate.Timestamp.Before(closestBefore.Timestamp) { + continue + } + sample := candidate + closestBefore = &sample + } + + // Samples after target time. + if delta >= 0 { + // Ignore samples outside of staleness policy window. + if delta > *stalenessDelta { + continue + } + // Ignore samples that are farther away than samples we've seen before. + if closestAfter != nil && candidate.Timestamp.After(closestAfter.Timestamp) { + continue + } + sample := candidate + closestAfter = &sample + } + } + + switch { + case closestBefore != nil && closestAfter != nil: + return interpolateSamples(closestBefore, closestAfter, timestamp) + case closestBefore != nil: + return closestBefore + default: + return closestAfter + } +} + +// interpolateSamples interpolates a value at a target time between two +// provided sample pairs. +func interpolateSamples(first, second *metric.SamplePair, timestamp clientmodel.Timestamp) *metric.SamplePair { + dv := second.Value - first.Value + dt := second.Timestamp.Sub(first.Timestamp) + + dDt := dv / clientmodel.SampleValue(dt) + offset := clientmodel.SampleValue(timestamp.Sub(first.Timestamp)) + + return &metric.SamplePair{ + Value: first.Value + (offset * dDt), + Timestamp: timestamp, + } +} diff --git a/promql/engine_test.go b/promql/engine_test.go new file mode 100644 index 0000000000..35e9188af3 --- /dev/null +++ b/promql/engine_test.go @@ -0,0 +1,272 @@ +package promql + +import ( + "reflect" + "sync" + "testing" + "time" + + "golang.org/x/net/context" + + "github.com/prometheus/prometheus/storage/local" +) + +func TestQueryTimeout(t *testing.T) { + *defaultQueryTimeout = 5 * time.Millisecond + defer func() { + // Restore default query timeout + *defaultQueryTimeout = 2 * time.Minute + }() + + storage, closer := local.NewTestStorage(t, 1) + defer closer.Close() + + engine := NewEngine(storage) + defer engine.Stop() + + query, err := engine.NewQuery("foo = bar") + if err != nil { + t.Fatalf("error parsing query: %s", err) + } + + // Timeouts are not exact but checked in designated places. For example between + // invoking handlers. Thus, we reigster two handlers that take some time to ensure we check + // after exceeding the timeout. + // Should the implementation of this area change, the test might have to be adjusted. + engine.RegisterRecordHandler("test", func(context.Context, *RecordStmt) error { + time.Sleep(10 * time.Millisecond) + return nil + }) + engine.RegisterRecordHandler("test2", func(context.Context, *RecordStmt) error { + time.Sleep(10 * time.Millisecond) + return nil + }) + + res := query.Exec() + if res.Err == nil { + t.Fatalf("expected timeout error but got none") + } + if _, ok := res.Err.(ErrQueryTimeout); res.Err != nil && !ok { + t.Fatalf("expected timeout error but got: %s", res.Err) + } +} + +func TestQueryCancel(t *testing.T) { + storage, closer := local.NewTestStorage(t, 1) + defer closer.Close() + + engine := NewEngine(storage) + defer engine.Stop() + + query1, err := engine.NewQuery("foo = bar") + if err != nil { + t.Fatalf("error parsing query: %s", err) + } + query2, err := engine.NewQuery("foo = baz") + if err != nil { + t.Fatalf("error parsing query: %s", err) + } + + // As for timeouts, cancellation is only checked at designated points. We ensure + // that we reach one of those points using the same method. + engine.RegisterRecordHandler("test1", func(context.Context, *RecordStmt) error { + <-time.After(2 * time.Millisecond) + return nil + }) + engine.RegisterRecordHandler("test2", func(context.Context, *RecordStmt) error { + <-time.After(2 * time.Millisecond) + return nil + }) + + // Cancel query after starting it. + var wg sync.WaitGroup + var res *Result + + wg.Add(1) + go func() { + res = query1.Exec() + wg.Done() + }() + <-time.After(1 * time.Millisecond) + query1.Cancel() + wg.Wait() + + if res.Err == nil { + t.Fatalf("expected cancellation error for query1 but got none") + } + if _, ok := res.Err.(ErrQueryCanceled); res.Err != nil && !ok { + t.Fatalf("expected cancellation error for query1 but got: %s", res.Err) + } + + // Canceling query before starting it must have no effect. + query2.Cancel() + res = query2.Exec() + if res.Err != nil { + t.Fatalf("unexpeceted error on executing query2: %s", res.Err) + } +} + +func TestEngineShutdown(t *testing.T) { + storage, closer := local.NewTestStorage(t, 1) + defer closer.Close() + + engine := NewEngine(storage) + + query1, err := engine.NewQuery("foo = bar") + if err != nil { + t.Fatalf("error parsing query: %s", err) + } + query2, err := engine.NewQuery("foo = baz") + if err != nil { + t.Fatalf("error parsing query: %s", err) + } + + handlerExecutions := 0 + + // Shutdown engine on first handler execution. Should handler execution ever become + // concurrent this test has to be adjusted accordingly. + engine.RegisterRecordHandler("test", func(context.Context, *RecordStmt) error { + handlerExecutions++ + engine.Stop() + time.Sleep(10 * time.Millisecond) + return nil + }) + engine.RegisterRecordHandler("test2", func(context.Context, *RecordStmt) error { + handlerExecutions++ + engine.Stop() + time.Sleep(10 * time.Millisecond) + return nil + }) + + // Stopping the engine should cancel the base context. While setting up queries is + // still possible their context is canceled from the beginning and execution should + // terminate immediately. + + res := query1.Exec() + if res.Err == nil { + t.Fatalf("expected error on shutdown during query but got none") + } + if handlerExecutions != 1 { + t.Fatalf("expected only one handler to be executed before query cancellation but got %d executons", handlerExecutions) + } + + res2 := query2.Exec() + if res2.Err == nil { + t.Fatalf("expected error on querying shutdown engine but got none") + } + if handlerExecutions != 1 { + t.Fatalf("expected no handler execution for query after engine shutdown") + } + +} + +func TestAlertHandler(t *testing.T) { + storage, closer := local.NewTestStorage(t, 1) + defer closer.Close() + + engine := NewEngine(storage) + defer engine.Stop() + + qs := `ALERT Foo IF bar FOR 5m WITH {a="b"} SUMMARY "sum" DESCRIPTION "desc"` + + doQuery := func(expectFailure bool) *AlertStmt { + query, err := engine.NewQuery(qs) + if err != nil { + t.Fatalf("error parsing query: %s", err) + } + res := query.Exec() + if expectFailure && res.Err == nil { + t.Fatalf("expected error but got none.") + } + if res.Err != nil && !expectFailure { + t.Fatalf("error on executing alert query: %s", res.Err) + } + // That this alert statement is correct is tested elsewhere. + return query.Statements()[0].(*AlertStmt) + } + + // We expect an error if nothing is registered to handle the query. + alertStmt := doQuery(true) + + receivedCalls := 0 + + // Ensure that we receive the correct statement. + engine.RegisterAlertHandler("test", func(ctx context.Context, as *AlertStmt) error { + if !reflect.DeepEqual(alertStmt, as) { + t.Errorf("received alert statement did not match input: %q", qs) + t.Fatalf("no match\n\nexpected:\n%s\ngot: \n%s\n", Tree(alertStmt), Tree(as)) + } + receivedCalls++ + return nil + }) + + for i := 0; i < 10; i++ { + doQuery(false) + if receivedCalls != i+1 { + t.Fatalf("alert handler was not called on query execution") + } + } + + engine.UnregisterAlertHandler("test") + + // We must receive no further calls after unregistering. + doQuery(true) + if receivedCalls != 10 { + t.Fatalf("received calls after unregistering alert handler") + } +} + +func TestRecordHandler(t *testing.T) { + storage, closer := local.NewTestStorage(t, 1) + defer closer.Close() + + engine := NewEngine(storage) + defer engine.Stop() + + qs := `foo = bar` + + doQuery := func(expectFailure bool) *RecordStmt { + query, err := engine.NewQuery(qs) + if err != nil { + t.Fatalf("error parsing query: %s", err) + } + res := query.Exec() + if expectFailure && res.Err == nil { + t.Fatalf("expected error but got none.") + } + if res.Err != nil && !expectFailure { + t.Fatalf("error on executing record query: %s", res.Err) + } + return query.Statements()[0].(*RecordStmt) + } + + // We expect an error if nothing is registered to handle the query. + recordStmt := doQuery(true) + + receivedCalls := 0 + + // Ensure that we receive the correct statement. + engine.RegisterRecordHandler("test", func(ctx context.Context, rs *RecordStmt) error { + if !reflect.DeepEqual(recordStmt, rs) { + t.Errorf("received record statement did not match input: %q", qs) + t.Fatalf("no match\n\nexpected:\n%s\ngot: \n%s\n", Tree(recordStmt), Tree(rs)) + } + receivedCalls++ + return nil + }) + + for i := 0; i < 10; i++ { + doQuery(false) + if receivedCalls != i+1 { + t.Fatalf("record handler was not called on query execution") + } + } + + engine.UnregisterRecordHandler("test") + + // We must receive no further calls after unregistering. + doQuery(true) + if receivedCalls != 10 { + t.Fatalf("received calls after unregistering record handler") + } +} diff --git a/promql/functions.go b/promql/functions.go index 4d747ed75a..f1b9c0fd70 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -13,6 +13,18 @@ package promql +import ( + "container/heap" + "math" + "sort" + "strconv" + "time" + + clientmodel "github.com/prometheus/client_golang/model" + + "github.com/prometheus/prometheus/storage/metric" +) + // Function represents a function of the expression language and is // used by function nodes. type Function struct { @@ -20,7 +32,477 @@ type Function struct { ArgTypes []ExprType OptionalArgs int ReturnType ExprType - Call func() + Call func(ev *evaluator, args Expressions) Value +} + +// === time() clientmodel.SampleValue === +func funcTime(ev *evaluator, args Expressions) Value { + return &Scalar{ + Value: clientmodel.SampleValue(ev.Timestamp.Unix()), + Timestamp: ev.Timestamp, + } +} + +// === delta(matrix ExprMatrix, isCounter=0 ExprScalar) Vector === +func funcDelta(ev *evaluator, args Expressions) Value { + isCounter := len(args) >= 2 && ev.evalInt(args[1]) > 0 + resultVector := Vector{} + + // If we treat these metrics as counters, we need to fetch all values + // in the interval to find breaks in the timeseries' monotonicity. + // I.e. if a counter resets, we want to ignore that reset. + var matrixValue Matrix + if isCounter { + matrixValue = ev.evalMatrix(args[0]) + } else { + matrixValue = ev.evalMatrixBounds(args[0]) + } + for _, samples := range matrixValue { + // No sense in trying to compute a delta without at least two points. Drop + // this vector element. + if len(samples.Values) < 2 { + continue + } + + counterCorrection := clientmodel.SampleValue(0) + lastValue := clientmodel.SampleValue(0) + for _, sample := range samples.Values { + currentValue := sample.Value + if isCounter && currentValue < lastValue { + counterCorrection += lastValue - currentValue + } + lastValue = currentValue + } + resultValue := lastValue - samples.Values[0].Value + counterCorrection + + targetInterval := args[0].(*MatrixSelector).Range + sampledInterval := samples.Values[len(samples.Values)-1].Timestamp.Sub(samples.Values[0].Timestamp) + if sampledInterval == 0 { + // Only found one sample. Cannot compute a rate from this. + continue + } + // Correct for differences in target vs. actual delta interval. + // + // Above, we didn't actually calculate the delta for the specified target + // interval, but for an interval between the first and last found samples + // under the target interval, which will usually have less time between + // them. Depending on how many samples are found under a target interval, + // the delta results are distorted and temporal aliasing occurs (ugly + // bumps). This effect is corrected for below. + intervalCorrection := clientmodel.SampleValue(targetInterval) / clientmodel.SampleValue(sampledInterval) + resultValue *= intervalCorrection + + resultSample := &Sample{ + Metric: samples.Metric, + Value: resultValue, + Timestamp: ev.Timestamp, + } + resultSample.Metric.Delete(clientmodel.MetricNameLabel) + resultVector = append(resultVector, resultSample) + } + return resultVector +} + +// === rate(node ExprMatrix) Vector === +func funcRate(ev *evaluator, args Expressions) Value { + args = append(args, &NumberLiteral{1}) + vector := funcDelta(ev, args).(Vector) + + // TODO: could be other type of ExprMatrix in the future (right now, only + // MatrixSelector exists). Find a better way of getting the duration of a + // matrix, such as looking at the samples themselves. + interval := args[0].(*MatrixSelector).Range + for i := range vector { + vector[i].Value /= clientmodel.SampleValue(interval / time.Second) + } + return vector +} + +// === sort(node ExprVector) Vector === +func funcSort(ev *evaluator, args Expressions) Value { + byValueSorter := vectorByValueHeap(ev.evalVector(args[0])) + sort.Sort(byValueSorter) + return Vector(byValueSorter) +} + +// === sortDesc(node ExprVector) Vector === +func funcSortDesc(ev *evaluator, args Expressions) Value { + byValueSorter := vectorByValueHeap(ev.evalVector(args[0])) + sort.Sort(sort.Reverse(byValueSorter)) + return Vector(byValueSorter) +} + +// === topk(k ExprScalar, node ExprVector) Vector === +func funcTopk(ev *evaluator, args Expressions) Value { + k := ev.evalInt(args[0]) + if k < 1 { + return Vector{} + } + vector := ev.evalVector(args[1]) + + topk := make(vectorByValueHeap, 0, k) + + for _, el := range vector { + if len(topk) < k || topk[0].Value < el.Value { + if len(topk) == k { + heap.Pop(&topk) + } + heap.Push(&topk, el) + } + } + sort.Sort(sort.Reverse(topk)) + return Vector(topk) +} + +// === bottomk(k ExprScalar, node ExprVector) Vector === +func funcBottomk(ev *evaluator, args Expressions) Value { + k := ev.evalInt(args[0]) + if k < 1 { + return Vector{} + } + vector := ev.evalVector(args[1]) + + bottomk := make(vectorByValueHeap, 0, k) + bkHeap := reverseHeap{Interface: &bottomk} + + for _, el := range vector { + if len(bottomk) < k || bottomk[0].Value > el.Value { + if len(bottomk) == k { + heap.Pop(&bkHeap) + } + heap.Push(&bkHeap, el) + } + } + sort.Sort(bottomk) + return Vector(bottomk) +} + +// === drop_common_labels(node ExprVector) Vector === +func funcDropCommonLabels(ev *evaluator, args Expressions) Value { + vector := ev.evalVector(args[0]) + if len(vector) < 1 { + return Vector{} + } + common := clientmodel.LabelSet{} + for k, v := range vector[0].Metric.Metric { + // TODO(julius): Should we also drop common metric names? + if k == clientmodel.MetricNameLabel { + continue + } + common[k] = v + } + + for _, el := range vector[1:] { + for k, v := range common { + if el.Metric.Metric[k] != v { + // Deletion of map entries while iterating over them is safe. + // From http://golang.org/ref/spec#For_statements: + // "If map entries that have not yet been reached are deleted during + // iteration, the corresponding iteration values will not be produced." + delete(common, k) + } + } + } + + for _, el := range vector { + for k := range el.Metric.Metric { + if _, ok := common[k]; ok { + el.Metric.Delete(k) + } + } + } + return vector +} + +// === round(vector ExprVector, toNearest=1 Scalar) Vector === +func funcRound(ev *evaluator, args Expressions) Value { + // round returns a number rounded to toNearest. + // Ties are solved by rounding up. + toNearest := float64(1) + if len(args) >= 2 { + toNearest = ev.evalFloat(args[1]) + } + // Invert as it seems to cause fewer floating point accuracy issues. + toNearestInverse := 1.0 / toNearest + + vector := ev.evalVector(args[0]) + for _, el := range vector { + el.Metric.Delete(clientmodel.MetricNameLabel) + el.Value = clientmodel.SampleValue(math.Floor(float64(el.Value)*toNearestInverse+0.5) / toNearestInverse) + } + return vector +} + +// === scalar(node ExprVector) Scalar === +func funcScalar(ev *evaluator, args Expressions) Value { + v := ev.evalVector(args[0]) + if len(v) != 1 { + return &Scalar{clientmodel.SampleValue(math.NaN()), ev.Timestamp} + } + return &Scalar{clientmodel.SampleValue(v[0].Value), ev.Timestamp} +} + +// === count_scalar(vector ExprVector) model.SampleValue === +func funcCountScalar(ev *evaluator, args Expressions) Value { + return &Scalar{ + Value: clientmodel.SampleValue(len(ev.evalVector(args[0]))), + Timestamp: ev.Timestamp, + } +} + +func aggrOverTime(ev *evaluator, args Expressions, aggrFn func(metric.Values) clientmodel.SampleValue) Value { + matrix := ev.evalMatrix(args[0]) + resultVector := Vector{} + + for _, el := range matrix { + if len(el.Values) == 0 { + continue + } + + el.Metric.Delete(clientmodel.MetricNameLabel) + resultVector = append(resultVector, &Sample{ + Metric: el.Metric, + Value: aggrFn(el.Values), + Timestamp: ev.Timestamp, + }) + } + return resultVector +} + +// === avg_over_time(matrix ExprMatrix) Vector === +func funcAvgOverTime(ev *evaluator, args Expressions) Value { + return aggrOverTime(ev, args, func(values metric.Values) clientmodel.SampleValue { + var sum clientmodel.SampleValue + for _, v := range values { + sum += v.Value + } + return sum / clientmodel.SampleValue(len(values)) + }) +} + +// === count_over_time(matrix ExprMatrix) Vector === +func funcCountOverTime(ev *evaluator, args Expressions) Value { + return aggrOverTime(ev, args, func(values metric.Values) clientmodel.SampleValue { + return clientmodel.SampleValue(len(values)) + }) +} + +// === floor(vector ExprVector) Vector === +func funcFloor(ev *evaluator, args Expressions) Value { + vector := ev.evalVector(args[0]) + for _, el := range vector { + el.Metric.Delete(clientmodel.MetricNameLabel) + el.Value = clientmodel.SampleValue(math.Floor(float64(el.Value))) + } + return vector +} + +// === max_over_time(matrix ExprMatrix) Vector === +func funcMaxOverTime(ev *evaluator, args Expressions) Value { + return aggrOverTime(ev, args, func(values metric.Values) clientmodel.SampleValue { + max := math.Inf(-1) + for _, v := range values { + max = math.Max(max, float64(v.Value)) + } + return clientmodel.SampleValue(max) + }) +} + +// === min_over_time(matrix ExprMatrix) Vector === +func funcMinOverTime(ev *evaluator, args Expressions) Value { + return aggrOverTime(ev, args, func(values metric.Values) clientmodel.SampleValue { + min := math.Inf(1) + for _, v := range values { + min = math.Min(min, float64(v.Value)) + } + return clientmodel.SampleValue(min) + }) +} + +// === sum_over_time(matrix ExprMatrix) Vector === +func funcSumOverTime(ev *evaluator, args Expressions) Value { + return aggrOverTime(ev, args, func(values metric.Values) clientmodel.SampleValue { + var sum clientmodel.SampleValue + for _, v := range values { + sum += v.Value + } + return sum + }) +} + +// === abs(vector ExprVector) Vector === +func funcAbs(ev *evaluator, args Expressions) Value { + vector := ev.evalVector(args[0]) + for _, el := range vector { + el.Metric.Delete(clientmodel.MetricNameLabel) + el.Value = clientmodel.SampleValue(math.Abs(float64(el.Value))) + } + return vector +} + +// === absent(vector ExprVector) Vector === +func funcAbsent(ev *evaluator, args Expressions) Value { + if len(ev.evalVector(args[0])) > 0 { + return Vector{} + } + m := clientmodel.Metric{} + if vs, ok := args[0].(*VectorSelector); ok { + for _, matcher := range vs.LabelMatchers { + if matcher.Type == metric.Equal && matcher.Name != clientmodel.MetricNameLabel { + m[matcher.Name] = matcher.Value + } + } + } + return Vector{ + &Sample{ + Metric: clientmodel.COWMetric{ + Metric: m, + Copied: true, + }, + Value: 1, + Timestamp: ev.Timestamp, + }, + } +} + +// === ceil(vector ExprVector) Vector === +func funcCeil(ev *evaluator, args Expressions) Value { + vector := ev.evalVector(args[0]) + for _, el := range vector { + el.Metric.Delete(clientmodel.MetricNameLabel) + el.Value = clientmodel.SampleValue(math.Ceil(float64(el.Value))) + } + return vector +} + +// === exp(vector ExprVector) Vector === +func funcExp(ev *evaluator, args Expressions) Value { + vector := ev.evalVector(args[0]) + for _, el := range vector { + el.Metric.Delete(clientmodel.MetricNameLabel) + el.Value = clientmodel.SampleValue(math.Exp(float64(el.Value))) + } + return vector +} + +// === sqrt(vector VectorNode) Vector === +func funcSqrt(ev *evaluator, args Expressions) Value { + vector := ev.evalVector(args[0]) + for _, el := range vector { + el.Metric.Delete(clientmodel.MetricNameLabel) + el.Value = clientmodel.SampleValue(math.Sqrt(float64(el.Value))) + } + return vector +} + +// === ln(vector ExprVector) Vector === +func funcLn(ev *evaluator, args Expressions) Value { + vector := ev.evalVector(args[0]) + for _, el := range vector { + el.Metric.Delete(clientmodel.MetricNameLabel) + el.Value = clientmodel.SampleValue(math.Log(float64(el.Value))) + } + return vector +} + +// === log2(vector ExprVector) Vector === +func funcLog2(ev *evaluator, args Expressions) Value { + vector := ev.evalVector(args[0]) + for _, el := range vector { + el.Metric.Delete(clientmodel.MetricNameLabel) + el.Value = clientmodel.SampleValue(math.Log2(float64(el.Value))) + } + return vector +} + +// === log10(vector ExprVector) Vector === +func funcLog10(ev *evaluator, args Expressions) Value { + vector := ev.evalVector(args[0]) + for _, el := range vector { + el.Metric.Delete(clientmodel.MetricNameLabel) + el.Value = clientmodel.SampleValue(math.Log10(float64(el.Value))) + } + return vector +} + +// === deriv(node ExprMatrix) Vector === +func funcDeriv(ev *evaluator, args Expressions) Value { + resultVector := Vector{} + matrix := ev.evalMatrix(args[0]) + + for _, samples := range matrix { + // No sense in trying to compute a derivative without at least two points. + // Drop this vector element. + if len(samples.Values) < 2 { + continue + } + + // Least squares. + n := clientmodel.SampleValue(0) + sumY := clientmodel.SampleValue(0) + sumX := clientmodel.SampleValue(0) + sumXY := clientmodel.SampleValue(0) + sumX2 := clientmodel.SampleValue(0) + for _, sample := range samples.Values { + x := clientmodel.SampleValue(sample.Timestamp.UnixNano() / 1e9) + n += 1.0 + sumY += sample.Value + sumX += x + sumXY += x * sample.Value + sumX2 += x * x + } + numerator := sumXY - sumX*sumY/n + denominator := sumX2 - (sumX*sumX)/n + + resultValue := numerator / denominator + + resultSample := &Sample{ + Metric: samples.Metric, + Value: resultValue, + Timestamp: ev.Timestamp, + } + resultSample.Metric.Delete(clientmodel.MetricNameLabel) + resultVector = append(resultVector, resultSample) + } + return resultVector +} + +// === histogram_quantile(k ExprScalar, vector ExprVector) Vector === +func funcHistogramQuantile(ev *evaluator, args Expressions) Value { + q := clientmodel.SampleValue(ev.evalFloat(args[0])) + inVec := ev.evalVector(args[1]) + + outVec := Vector{} + signatureToMetricWithBuckets := map[uint64]*metricWithBuckets{} + for _, el := range inVec { + upperBound, err := strconv.ParseFloat( + string(el.Metric.Metric[clientmodel.BucketLabel]), 64, + ) + if err != nil { + // Oops, no bucket label or malformed label value. Skip. + // TODO(beorn7): Issue a warning somehow. + continue + } + signature := clientmodel.SignatureWithoutLabels(el.Metric.Metric, excludedLabels) + mb, ok := signatureToMetricWithBuckets[signature] + if !ok { + el.Metric.Delete(clientmodel.BucketLabel) + el.Metric.Delete(clientmodel.MetricNameLabel) + mb = &metricWithBuckets{el.Metric, nil} + signatureToMetricWithBuckets[signature] = mb + } + mb.buckets = append(mb.buckets, bucket{upperBound, el.Value}) + } + + for _, mb := range signatureToMetricWithBuckets { + outVec = append(outVec, &Sample{ + Metric: mb.metric, + Value: clientmodel.SampleValue(quantile(q, mb.buckets)), + Timestamp: ev.Timestamp, + }) + } + + return outVec } var functions = map[string]*Function{ @@ -28,164 +510,207 @@ var functions = map[string]*Function{ Name: "abs", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcAbs, }, "absent": { Name: "absent", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcAbsent, }, "avg_over_time": { Name: "avg_over_time", ArgTypes: []ExprType{ExprMatrix}, ReturnType: ExprVector, - Call: func() {}, + Call: funcAvgOverTime, }, "bottomk": { Name: "bottomk", ArgTypes: []ExprType{ExprScalar, ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcBottomk, }, "ceil": { Name: "ceil", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcCeil, }, "count_over_time": { Name: "count_over_time", ArgTypes: []ExprType{ExprMatrix}, ReturnType: ExprVector, - Call: func() {}, + Call: funcCountOverTime, }, "count_scalar": { Name: "count_scalar", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprScalar, - Call: func() {}, + Call: funcCountScalar, }, "delta": { Name: "delta", ArgTypes: []ExprType{ExprMatrix, ExprScalar}, OptionalArgs: 1, // The 2nd argument is deprecated. ReturnType: ExprVector, - Call: func() {}, + Call: funcDelta, }, "deriv": { Name: "deriv", ArgTypes: []ExprType{ExprMatrix}, ReturnType: ExprVector, - Call: func() {}, + Call: funcDeriv, }, "drop_common_labels": { Name: "drop_common_labels", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcDropCommonLabels, }, "exp": { Name: "exp", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcExp, }, "floor": { Name: "floor", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcFloor, }, "histogram_quantile": { Name: "histogram_quantile", ArgTypes: []ExprType{ExprScalar, ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcHistogramQuantile, }, "ln": { Name: "ln", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcLn, }, "log10": { Name: "log10", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcLog10, }, "log2": { Name: "log2", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcLog2, }, "max_over_time": { Name: "max_over_time", ArgTypes: []ExprType{ExprMatrix}, ReturnType: ExprVector, - Call: func() {}, + Call: funcMaxOverTime, }, "min_over_time": { Name: "min_over_time", ArgTypes: []ExprType{ExprMatrix}, ReturnType: ExprVector, - Call: func() {}, + Call: funcMinOverTime, }, "rate": { Name: "rate", ArgTypes: []ExprType{ExprMatrix}, ReturnType: ExprVector, - Call: func() {}, + Call: funcRate, }, "round": { Name: "round", ArgTypes: []ExprType{ExprVector, ExprScalar}, OptionalArgs: 1, ReturnType: ExprVector, - Call: func() {}, + Call: funcRound, }, "scalar": { Name: "scalar", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprScalar, - Call: func() {}, + Call: funcScalar, }, "sort": { Name: "sort", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcSort, }, "sort_desc": { Name: "sort_desc", ArgTypes: []ExprType{ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcSortDesc, + }, + "sqrt": { + Name: "sqrt", + ArgTypes: []ExprType{ExprVector}, + ReturnType: ExprVector, + Call: funcSqrt, }, "sum_over_time": { Name: "sum_over_time", ArgTypes: []ExprType{ExprMatrix}, ReturnType: ExprVector, - Call: func() {}, + Call: funcSumOverTime, }, "time": { Name: "time", ArgTypes: []ExprType{}, ReturnType: ExprScalar, - Call: func() {}, + Call: funcTime, }, "topk": { Name: "topk", ArgTypes: []ExprType{ExprScalar, ExprVector}, ReturnType: ExprVector, - Call: func() {}, + Call: funcTopk, }, } -// GetFunction returns a predefined Function object for the given name. -func GetFunction(name string) (*Function, bool) { +// getFunction returns a predefined Function object for the given name. +func getFunction(name string) (*Function, bool) { function, ok := functions[name] return function, ok } + +type vectorByValueHeap Vector + +func (s vectorByValueHeap) Len() int { + return len(s) +} + +func (s vectorByValueHeap) Less(i, j int) bool { + if math.IsNaN(float64(s[i].Value)) { + return true + } + return s[i].Value < s[j].Value +} + +func (s vectorByValueHeap) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s *vectorByValueHeap) Push(x interface{}) { + *s = append(*s, x.(*Sample)) +} + +func (s *vectorByValueHeap) Pop() interface{} { + old := *s + n := len(old) + el := old[n-1] + *s = old[0 : n-1] + return el +} + +type reverseHeap struct { + heap.Interface +} + +func (s reverseHeap) Less(i, j int) bool { + return s.Interface.Less(j, i) +} diff --git a/promql/parse.go b/promql/parse.go index a683782adc..ec388c4500 100644 --- a/promql/parse.go +++ b/promql/parse.go @@ -565,7 +565,7 @@ func (p *parser) aggrExpr() *AggregateExpr { func (p *parser) call(name string) *Call { const ctx = "function call" - fn, exist := GetFunction(name) + fn, exist := getFunction(name) if !exist { p.errorf("unknown function with name %q", name) } diff --git a/promql/parse_test.go b/promql/parse_test.go index fc990c0a12..49750859a0 100644 --- a/promql/parse_test.go +++ b/promql/parse_test.go @@ -1069,7 +1069,7 @@ func mustLabelMatcher(mt metric.MatchType, name clientmodel.LabelName, val clien } func mustGetFunction(name string) *Function { - f, ok := GetFunction(name) + f, ok := getFunction(name) if !ok { panic(fmt.Errorf("function %q does not exist", name)) } diff --git a/promql/printer.go b/promql/printer.go index 03833accb9..6576d23323 100644 --- a/promql/printer.go +++ b/promql/printer.go @@ -25,6 +25,47 @@ import ( "github.com/prometheus/prometheus/utility" ) +func (matrix Matrix) String() string { + metricStrings := make([]string, 0, len(matrix)) + for _, sampleStream := range matrix { + metricName, hasName := sampleStream.Metric.Metric[clientmodel.MetricNameLabel] + numLabels := len(sampleStream.Metric.Metric) + if hasName { + numLabels-- + } + labelStrings := make([]string, 0, numLabels) + for label, value := range sampleStream.Metric.Metric { + if label != clientmodel.MetricNameLabel { + labelStrings = append(labelStrings, fmt.Sprintf("%s=%q", label, value)) + } + } + sort.Strings(labelStrings) + valueStrings := make([]string, 0, len(sampleStream.Values)) + for _, value := range sampleStream.Values { + valueStrings = append(valueStrings, + fmt.Sprintf("\n%v @[%v]", value.Value, value.Timestamp)) + } + metricStrings = append(metricStrings, + fmt.Sprintf("%s{%s} => %s", + metricName, + strings.Join(labelStrings, ", "), + strings.Join(valueStrings, ", "))) + } + sort.Strings(metricStrings) + return strings.Join(metricStrings, "\n") +} + +func (vector Vector) String() string { + metricStrings := make([]string, 0, len(vector)) + for _, sample := range vector { + metricStrings = append(metricStrings, + fmt.Sprintf("%s => %v @[%v]", + sample.Metric, + sample.Value, sample.Timestamp)) + } + return strings.Join(metricStrings, "\n") +} + // Tree returns a string of the tree structure of the given node. func Tree(node Node) string { return tree(node, "") @@ -175,7 +216,7 @@ func (node *ParenExpr) String() string { } func (node *StringLiteral) String() string { - return fmt.Sprintf("%q", node.Str) + return fmt.Sprintf("%q", node.Val) } func (node *UnaryExpr) String() string { @@ -321,7 +362,7 @@ func (node *MatrixSelector) DotGraph() string { // DotGraph returns a DOT representation of the string literal. func (node *StringLiteral) DotGraph() string { - return fmt.Sprintf("%#p[label=\"'%q'\"];\n", node, node.Str) + return fmt.Sprintf("%#p[label=\"'%q'\"];\n", node, node.Val) } // DotGraph returns a DOT representation of the unary expression. diff --git a/promql/promql_test.go b/promql/promql_test.go new file mode 100644 index 0000000000..93af620d3e --- /dev/null +++ b/promql/promql_test.go @@ -0,0 +1,1656 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promql + +import ( + "fmt" + "math" + "regexp" + "strconv" + "strings" + "testing" + "time" + + clientmodel "github.com/prometheus/client_golang/model" + + "github.com/prometheus/prometheus/storage/local" + "github.com/prometheus/prometheus/storage/metric" + "github.com/prometheus/prometheus/utility/test" +) + +var ( + testEvalTime = testStartTime.Add(testSampleInterval * 10) + fixturesPath = "fixtures" + + reSample = regexp.MustCompile(`^(.*)(?: \=\>|:) (\-?\d+\.?\d*(?:e-?\d+)?|[+-]Inf|NaN) \@\[(\d+)\]$`) + minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64. +) + +const ( + epsilon = 0.000001 // Relative error allowed for sample values. +) + +func annotateWithTime(lines []string, timestamp clientmodel.Timestamp) []string { + annotatedLines := []string{} + for _, line := range lines { + annotatedLines = append(annotatedLines, fmt.Sprintf(line, timestamp)) + } + return annotatedLines +} + +func vectorComparisonString(expected []string, actual []string) string { + separator := "\n--------------\n" + return fmt.Sprintf("Expected:%v%v%v\nActual:%v%v%v ", + separator, + strings.Join(expected, "\n"), + separator, + separator, + strings.Join(actual, "\n"), + separator) +} + +// samplesAlmostEqual returns true if the two sample lines only differ by a +// small relative error in their sample value. +func samplesAlmostEqual(a, b string) bool { + if a == b { + // Fast path if strings are equal. + return true + } + aMatches := reSample.FindStringSubmatch(a) + if aMatches == nil { + panic(fmt.Errorf("sample %q did not match regular expression", a)) + } + bMatches := reSample.FindStringSubmatch(b) + if bMatches == nil { + panic(fmt.Errorf("sample %q did not match regular expression", b)) + } + if aMatches[1] != bMatches[1] { + return false // Labels don't match. + } + if aMatches[3] != bMatches[3] { + return false // Timestamps don't match. + } + // If we are here, we have the diff in the floats. + // We have to check if they are almost equal. + aVal, err := strconv.ParseFloat(aMatches[2], 64) + if err != nil { + panic(err) + } + bVal, err := strconv.ParseFloat(bMatches[2], 64) + if err != nil { + panic(err) + } + + // Cf. http://floating-point-gui.de/errors/comparison/ + if aVal == bVal { + return true + } + + diff := math.Abs(aVal - bVal) + + if aVal == 0 || bVal == 0 || diff < minNormal { + return diff < epsilon*minNormal + } + return diff/(math.Abs(aVal)+math.Abs(bVal)) < epsilon +} + +func newTestStorage(t testing.TB) (storage local.Storage, closer test.Closer) { + storage, closer = local.NewTestStorage(t, 1) + storeMatrix(storage, testMatrix) + return storage, closer +} + +func TestExpressions(t *testing.T) { + // Labels in expected output need to be alphabetically sorted. + expressionTests := []struct { + expr string + output []string + shouldFail bool + checkOrder bool + }{ + { + expr: `SUM(http_requests)`, + output: []string{`{} => 3600 @[%v]`}, + }, { + expr: `SUM(http_requests{instance="0"}) BY(job)`, + output: []string{ + `{job="api-server"} => 400 @[%v]`, + `{job="app-server"} => 1200 @[%v]`, + }, + }, { + expr: `SUM(http_requests{instance="0"}) BY(job) KEEPING_EXTRA`, + output: []string{ + `{instance="0", job="api-server"} => 400 @[%v]`, + `{instance="0", job="app-server"} => 1200 @[%v]`, + }, + }, { + expr: `SUM(http_requests) BY (job)`, + output: []string{ + `{job="api-server"} => 1000 @[%v]`, + `{job="app-server"} => 2600 @[%v]`, + }, + }, { + // Non-existent labels mentioned in BY-clauses shouldn't propagate to output. + expr: `SUM(http_requests) BY (job, nonexistent)`, + output: []string{ + `{job="api-server"} => 1000 @[%v]`, + `{job="app-server"} => 2600 @[%v]`, + }, + }, { + expr: ` + # Test comment. + SUM(http_requests) BY # comments shouldn't have any effect + (job) # another comment`, + output: []string{ + `{job="api-server"} => 1000 @[%v]`, + `{job="app-server"} => 2600 @[%v]`, + }, + }, { + expr: `COUNT(http_requests) BY (job)`, + output: []string{ + `{job="api-server"} => 4 @[%v]`, + `{job="app-server"} => 4 @[%v]`, + }, + }, { + expr: `SUM(http_requests) BY (job, group)`, + output: []string{ + `{group="canary", job="api-server"} => 700 @[%v]`, + `{group="canary", job="app-server"} => 1500 @[%v]`, + `{group="production", job="api-server"} => 300 @[%v]`, + `{group="production", job="app-server"} => 1100 @[%v]`, + }, + }, { + expr: `AVG(http_requests) BY (job)`, + output: []string{ + `{job="api-server"} => 250 @[%v]`, + `{job="app-server"} => 650 @[%v]`, + }, + }, { + expr: `MIN(http_requests) BY (job)`, + output: []string{ + `{job="api-server"} => 100 @[%v]`, + `{job="app-server"} => 500 @[%v]`, + }, + }, { + expr: `MAX(http_requests) BY (job)`, + output: []string{ + `{job="api-server"} => 400 @[%v]`, + `{job="app-server"} => 800 @[%v]`, + }, + }, { + expr: `SUM(http_requests) BY (job) - COUNT(http_requests) BY (job)`, + output: []string{ + `{job="api-server"} => 996 @[%v]`, + `{job="app-server"} => 2596 @[%v]`, + }, + }, { + expr: `2 - SUM(http_requests) BY (job)`, + output: []string{ + `{job="api-server"} => -998 @[%v]`, + `{job="app-server"} => -2598 @[%v]`, + }, + }, { + expr: `1000 / SUM(http_requests) BY (job)`, + output: []string{ + `{job="api-server"} => 1 @[%v]`, + `{job="app-server"} => 0.38461538461538464 @[%v]`, + }, + }, { + expr: `SUM(http_requests) BY (job) - 2`, + output: []string{ + `{job="api-server"} => 998 @[%v]`, + `{job="app-server"} => 2598 @[%v]`, + }, + }, { + expr: `SUM(http_requests) BY (job) % 3`, + output: []string{ + `{job="api-server"} => 1 @[%v]`, + `{job="app-server"} => 2 @[%v]`, + }, + }, { + expr: `SUM(http_requests) BY (job) / 0`, + output: []string{ + `{job="api-server"} => +Inf @[%v]`, + `{job="app-server"} => +Inf @[%v]`, + }, + }, { + expr: `SUM(http_requests) BY (job) > 1000`, + output: []string{ + `{job="app-server"} => 2600 @[%v]`, + }, + }, { + expr: `1000 < SUM(http_requests) BY (job)`, + output: []string{ + `{job="app-server"} => 1000 @[%v]`, + }, + }, { + expr: `SUM(http_requests) BY (job) <= 1000`, + output: []string{ + `{job="api-server"} => 1000 @[%v]`, + }, + }, { + expr: `SUM(http_requests) BY (job) != 1000`, + output: []string{ + `{job="app-server"} => 2600 @[%v]`, + }, + }, { + expr: `SUM(http_requests) BY (job) == 1000`, + output: []string{ + `{job="api-server"} => 1000 @[%v]`, + }, + }, { + expr: `SUM(http_requests) BY (job) + SUM(http_requests) BY (job)`, + output: []string{ + `{job="api-server"} => 2000 @[%v]`, + `{job="app-server"} => 5200 @[%v]`, + }, + }, { + expr: `http_requests{job="api-server", group="canary"}`, + output: []string{ + `http_requests{group="canary", instance="0", job="api-server"} => 300 @[%v]`, + `http_requests{group="canary", instance="1", job="api-server"} => 400 @[%v]`, + }, + }, { + expr: `http_requests{job="api-server", group="canary"} + rate(http_requests{job="api-server"}[5m]) * 5 * 60`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => 330 @[%v]`, + `{group="canary", instance="1", job="api-server"} => 440 @[%v]`, + }, + }, { + expr: `rate(http_requests[25m]) * 25 * 60`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => 150 @[%v]`, + `{group="canary", instance="0", job="app-server"} => 350 @[%v]`, + `{group="canary", instance="1", job="api-server"} => 200 @[%v]`, + `{group="canary", instance="1", job="app-server"} => 400 @[%v]`, + `{group="production", instance="0", job="api-server"} => 50 @[%v]`, + `{group="production", instance="0", job="app-server"} => 249.99999999999997 @[%v]`, + `{group="production", instance="1", job="api-server"} => 100 @[%v]`, + `{group="production", instance="1", job="app-server"} => 300 @[%v]`, + }, + }, { + expr: `delta(http_requests[25m], 1)`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => 150 @[%v]`, + `{group="canary", instance="0", job="app-server"} => 350 @[%v]`, + `{group="canary", instance="1", job="api-server"} => 200 @[%v]`, + `{group="canary", instance="1", job="app-server"} => 400 @[%v]`, + `{group="production", instance="0", job="api-server"} => 50 @[%v]`, + `{group="production", instance="0", job="app-server"} => 250 @[%v]`, + `{group="production", instance="1", job="api-server"} => 100 @[%v]`, + `{group="production", instance="1", job="app-server"} => 300 @[%v]`, + }, + }, + { + expr: `sort(http_requests)`, + output: []string{ + `http_requests{group="production", instance="0", job="api-server"} => 100 @[%v]`, + `http_requests{group="production", instance="1", job="api-server"} => 200 @[%v]`, + `http_requests{group="canary", instance="0", job="api-server"} => 300 @[%v]`, + `http_requests{group="canary", instance="1", job="api-server"} => 400 @[%v]`, + `http_requests{group="production", instance="0", job="app-server"} => 500 @[%v]`, + `http_requests{group="production", instance="1", job="app-server"} => 600 @[%v]`, + `http_requests{group="canary", instance="0", job="app-server"} => 700 @[%v]`, + `http_requests{group="canary", instance="1", job="app-server"} => 800 @[%v]`, + }, + checkOrder: true, + }, { + expr: `sort(0 / round(http_requests, 400) + http_requests)`, + output: []string{ + `{group="production", instance="0", job="api-server"} => NaN @[%v]`, + `{group="production", instance="1", job="api-server"} => 200 @[%v]`, + `{group="canary", instance="0", job="api-server"} => 300 @[%v]`, + `{group="canary", instance="1", job="api-server"} => 400 @[%v]`, + `{group="production", instance="0", job="app-server"} => 500 @[%v]`, + `{group="production", instance="1", job="app-server"} => 600 @[%v]`, + `{group="canary", instance="0", job="app-server"} => 700 @[%v]`, + `{group="canary", instance="1", job="app-server"} => 800 @[%v]`, + }, + checkOrder: true, + }, { + expr: `sort_desc(http_requests)`, + output: []string{ + `http_requests{group="canary", instance="1", job="app-server"} => 800 @[%v]`, + `http_requests{group="canary", instance="0", job="app-server"} => 700 @[%v]`, + `http_requests{group="production", instance="1", job="app-server"} => 600 @[%v]`, + `http_requests{group="production", instance="0", job="app-server"} => 500 @[%v]`, + `http_requests{group="canary", instance="1", job="api-server"} => 400 @[%v]`, + `http_requests{group="canary", instance="0", job="api-server"} => 300 @[%v]`, + `http_requests{group="production", instance="1", job="api-server"} => 200 @[%v]`, + `http_requests{group="production", instance="0", job="api-server"} => 100 @[%v]`, + }, + checkOrder: true, + }, + { + expr: `topk(3, http_requests)`, + output: []string{ + `http_requests{group="canary", instance="1", job="app-server"} => 800 @[%v]`, + `http_requests{group="canary", instance="0", job="app-server"} => 700 @[%v]`, + `http_requests{group="production", instance="1", job="app-server"} => 600 @[%v]`, + }, + checkOrder: true, + }, { + expr: `topk(5, http_requests{group="canary",job="app-server"})`, + output: []string{ + `http_requests{group="canary", instance="1", job="app-server"} => 800 @[%v]`, + `http_requests{group="canary", instance="0", job="app-server"} => 700 @[%v]`, + }, + checkOrder: true, + }, { + expr: `bottomk(3, http_requests)`, + output: []string{ + `http_requests{group="production", instance="0", job="api-server"} => 100 @[%v]`, + `http_requests{group="production", instance="1", job="api-server"} => 200 @[%v]`, + `http_requests{group="canary", instance="0", job="api-server"} => 300 @[%v]`, + }, + checkOrder: true, + }, { + expr: `bottomk(5, http_requests{group="canary",job="app-server"})`, + output: []string{ + `http_requests{group="canary", instance="0", job="app-server"} => 700 @[%v]`, + `http_requests{group="canary", instance="1", job="app-server"} => 800 @[%v]`, + }, + checkOrder: true, + }, + { + // Single-letter label names and values. + expr: `x{y="testvalue"}`, + output: []string{ + `x{y="testvalue"} => 100 @[%v]`, + }, + }, { + // Lower-cased aggregation operators should work too. + expr: `sum(http_requests) by (job) + min(http_requests) by (job) + max(http_requests) by (job) + avg(http_requests) by (job)`, + output: []string{ + `{job="app-server"} => 4550 @[%v]`, + `{job="api-server"} => 1750 @[%v]`, + }, + }, { + // Deltas should be adjusted for target interval vs. samples under target interval. + expr: `delta(http_requests{group="canary", instance="1", job="app-server"}[18m])`, + output: []string{`{group="canary", instance="1", job="app-server"} => 288 @[%v]`}, + }, { + // Deltas should perform the same operation when 2nd argument is 0. + expr: `delta(http_requests{group="canary", instance="1", job="app-server"}[18m], 0)`, + output: []string{`{group="canary", instance="1", job="app-server"} => 288 @[%v]`}, + }, { + // Rates should calculate per-second rates. + expr: `rate(http_requests{group="canary", instance="1", job="app-server"}[60m])`, + output: []string{`{group="canary", instance="1", job="app-server"} => 0.26666666666666666 @[%v]`}, + }, + { + // Deriv should return the same as rate in simple cases. + expr: `deriv(http_requests{group="canary", instance="1", job="app-server"}[60m])`, + output: []string{`{group="canary", instance="1", job="app-server"} => 0.26666666666666666 @[%v]`}, + }, + { + // Counter resets at in the middle of range are handled correctly by rate(). + expr: `rate(testcounter_reset_middle[60m])`, + output: []string{`{} => 0.03 @[%v]`}, + }, { + // Counter resets at end of range are ignored by rate(). + expr: `rate(testcounter_reset_end[5m])`, + output: []string{`{} => 0 @[%v]`}, + }, + { + // Deriv should return correct result. + expr: `deriv(testcounter_reset_middle[100m])`, + output: []string{`{} => 0.010606060606060607 @[%v]`}, + }, + { + // count_scalar for a non-empty vector should return scalar element count. + expr: `count_scalar(http_requests)`, + output: []string{`scalar: 8 @[%v]`}, + }, { + // count_scalar for an empty vector should return scalar 0. + expr: `count_scalar(nonexistent)`, + output: []string{`scalar: 0 @[%v]`}, + }, { + // Empty expressions shouldn't parse. + expr: ``, + shouldFail: true, + }, { + // Interval durations can't be in quotes. + expr: `http_requests["1m"]`, + shouldFail: true, + }, { + // Binop arguments need to be scalar or vector. + expr: `http_requests - http_requests[1m]`, + shouldFail: true, + }, { + expr: `http_requests{group!="canary"}`, + output: []string{ + `http_requests{group="production", instance="1", job="app-server"} => 600 @[%v]`, + `http_requests{group="production", instance="0", job="app-server"} => 500 @[%v]`, + `http_requests{group="production", instance="1", job="api-server"} => 200 @[%v]`, + `http_requests{group="production", instance="0", job="api-server"} => 100 @[%v]`, + }, + }, { + expr: `http_requests{job=~"server",group!="canary"}`, + output: []string{ + `http_requests{group="production", instance="1", job="app-server"} => 600 @[%v]`, + `http_requests{group="production", instance="0", job="app-server"} => 500 @[%v]`, + `http_requests{group="production", instance="1", job="api-server"} => 200 @[%v]`, + `http_requests{group="production", instance="0", job="api-server"} => 100 @[%v]`, + }, + }, { + expr: `http_requests{job!~"api",group!="canary"}`, + output: []string{ + `http_requests{group="production", instance="1", job="app-server"} => 600 @[%v]`, + `http_requests{group="production", instance="0", job="app-server"} => 500 @[%v]`, + }, + }, { + expr: `count_scalar(http_requests{job=~"^server$"})`, + output: []string{`scalar: 0 @[%v]`}, + }, { + expr: `http_requests{group="production",job=~"^api"}`, + output: []string{ + `http_requests{group="production", instance="0", job="api-server"} => 100 @[%v]`, + `http_requests{group="production", instance="1", job="api-server"} => 200 @[%v]`, + }, + }, + { + expr: `abs(-1 * http_requests{group="production",job="api-server"})`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 100 @[%v]`, + `{group="production", instance="1", job="api-server"} => 200 @[%v]`, + }, + }, + { + expr: `floor(0.004 * http_requests{group="production",job="api-server"})`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 0 @[%v]`, + `{group="production", instance="1", job="api-server"} => 0 @[%v]`, + }, + }, + { + expr: `ceil(0.004 * http_requests{group="production",job="api-server"})`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 1 @[%v]`, + `{group="production", instance="1", job="api-server"} => 1 @[%v]`, + }, + }, + { + expr: `round(0.004 * http_requests{group="production",job="api-server"})`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 0 @[%v]`, + `{group="production", instance="1", job="api-server"} => 1 @[%v]`, + }, + }, + { // Round should correctly handle negative numbers. + expr: `round(-1 * (0.004 * http_requests{group="production",job="api-server"}))`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 0 @[%v]`, + `{group="production", instance="1", job="api-server"} => -1 @[%v]`, + }, + }, + { // Round should round half up. + expr: `round(0.005 * http_requests{group="production",job="api-server"})`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 1 @[%v]`, + `{group="production", instance="1", job="api-server"} => 1 @[%v]`, + }, + }, + { + expr: `round(-1 * (0.005 * http_requests{group="production",job="api-server"}))`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 0 @[%v]`, + `{group="production", instance="1", job="api-server"} => -1 @[%v]`, + }, + }, + { + expr: `round(1 + 0.005 * http_requests{group="production",job="api-server"})`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 2 @[%v]`, + `{group="production", instance="1", job="api-server"} => 2 @[%v]`, + }, + }, + { + expr: `round(-1 * (1 + 0.005 * http_requests{group="production",job="api-server"}))`, + output: []string{ + `{group="production", instance="0", job="api-server"} => -1 @[%v]`, + `{group="production", instance="1", job="api-server"} => -2 @[%v]`, + }, + }, + { // Round should accept the number to round nearest to. + expr: `round(0.0005 * http_requests{group="production",job="api-server"}, 0.1)`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 0.1 @[%v]`, + `{group="production", instance="1", job="api-server"} => 0.1 @[%v]`, + }, + }, + { + expr: `round(2.1 + 0.0005 * http_requests{group="production",job="api-server"}, 0.1)`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 2.2 @[%v]`, + `{group="production", instance="1", job="api-server"} => 2.2 @[%v]`, + }, + }, + { + expr: `round(5.2 + 0.0005 * http_requests{group="production",job="api-server"}, 0.1)`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 5.3 @[%v]`, + `{group="production", instance="1", job="api-server"} => 5.3 @[%v]`, + }, + }, + { // Round should work correctly with negative numbers and multiple decimal places. + expr: `round(-1 * (5.2 + 0.0005 * http_requests{group="production",job="api-server"}), 0.1)`, + output: []string{ + `{group="production", instance="0", job="api-server"} => -5.2 @[%v]`, + `{group="production", instance="1", job="api-server"} => -5.3 @[%v]`, + }, + }, + { // Round should work correctly with big toNearests. + expr: `round(0.025 * http_requests{group="production",job="api-server"}, 5)`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 5 @[%v]`, + `{group="production", instance="1", job="api-server"} => 5 @[%v]`, + }, + }, + { + expr: `round(0.045 * http_requests{group="production",job="api-server"}, 5)`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 5 @[%v]`, + `{group="production", instance="1", job="api-server"} => 10 @[%v]`, + }, + }, + { + expr: `avg_over_time(http_requests{group="production",job="api-server"}[1h])`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 50 @[%v]`, + `{group="production", instance="1", job="api-server"} => 100 @[%v]`, + }, + }, + { + expr: `count_over_time(http_requests{group="production",job="api-server"}[1h])`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 11 @[%v]`, + `{group="production", instance="1", job="api-server"} => 11 @[%v]`, + }, + }, + { + expr: `max_over_time(http_requests{group="production",job="api-server"}[1h])`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 100 @[%v]`, + `{group="production", instance="1", job="api-server"} => 200 @[%v]`, + }, + }, + { + expr: `min_over_time(http_requests{group="production",job="api-server"}[1h])`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 0 @[%v]`, + `{group="production", instance="1", job="api-server"} => 0 @[%v]`, + }, + }, + { + expr: `sum_over_time(http_requests{group="production",job="api-server"}[1h])`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 550 @[%v]`, + `{group="production", instance="1", job="api-server"} => 1100 @[%v]`, + }, + }, + { + expr: `time()`, + output: []string{`scalar: 3000 @[%v]`}, + }, + { + expr: `drop_common_labels(http_requests{group="production",job="api-server"})`, + output: []string{ + `http_requests{instance="0"} => 100 @[%v]`, + `http_requests{instance="1"} => 200 @[%v]`, + }, + }, + { + expr: `{` + string(clientmodel.MetricNameLabel) + `=~".*"}`, + output: []string{ + `http_requests{group="canary", instance="0", job="api-server"} => 300 @[%v]`, + `http_requests{group="canary", instance="0", job="app-server"} => 700 @[%v]`, + `http_requests{group="canary", instance="1", job="api-server"} => 400 @[%v]`, + `http_requests{group="canary", instance="1", job="app-server"} => 800 @[%v]`, + `http_requests{group="production", instance="0", job="api-server"} => 100 @[%v]`, + `http_requests{group="production", instance="0", job="app-server"} => 500 @[%v]`, + `http_requests{group="production", instance="1", job="api-server"} => 200 @[%v]`, + `http_requests{group="production", instance="1", job="app-server"} => 600 @[%v]`, + `testcounter_reset_end => 0 @[%v]`, + `testcounter_reset_middle => 50 @[%v]`, + `x{y="testvalue"} => 100 @[%v]`, + `label_grouping_test{a="a", b="abb"} => 200 @[%v]`, + `label_grouping_test{a="aa", b="bb"} => 100 @[%v]`, + `testhistogram_bucket{le="0.1", start="positive"} => 50 @[%v]`, + `testhistogram_bucket{le=".2", start="positive"} => 70 @[%v]`, + `testhistogram_bucket{le="1e0", start="positive"} => 110 @[%v]`, + `testhistogram_bucket{le="+Inf", start="positive"} => 120 @[%v]`, + `testhistogram_bucket{le="-.2", start="negative"} => 10 @[%v]`, + `testhistogram_bucket{le="-0.1", start="negative"} => 20 @[%v]`, + `testhistogram_bucket{le="0.3", start="negative"} => 20 @[%v]`, + `testhistogram_bucket{le="+Inf", start="negative"} => 30 @[%v]`, + `request_duration_seconds_bucket{instance="ins1", job="job1", le="0.1"} => 10 @[%v]`, + `request_duration_seconds_bucket{instance="ins1", job="job1", le="0.2"} => 30 @[%v]`, + `request_duration_seconds_bucket{instance="ins1", job="job1", le="+Inf"} => 40 @[%v]`, + `request_duration_seconds_bucket{instance="ins2", job="job1", le="0.1"} => 20 @[%v]`, + `request_duration_seconds_bucket{instance="ins2", job="job1", le="0.2"} => 50 @[%v]`, + `request_duration_seconds_bucket{instance="ins2", job="job1", le="+Inf"} => 60 @[%v]`, + `request_duration_seconds_bucket{instance="ins1", job="job2", le="0.1"} => 30 @[%v]`, + `request_duration_seconds_bucket{instance="ins1", job="job2", le="0.2"} => 40 @[%v]`, + `request_duration_seconds_bucket{instance="ins1", job="job2", le="+Inf"} => 60 @[%v]`, + `request_duration_seconds_bucket{instance="ins2", job="job2", le="0.1"} => 40 @[%v]`, + `request_duration_seconds_bucket{instance="ins2", job="job2", le="0.2"} => 70 @[%v]`, + `request_duration_seconds_bucket{instance="ins2", job="job2", le="+Inf"} => 90 @[%v]`, + `vector_matching_a{l="x"} => 10 @[%v]`, + `vector_matching_a{l="y"} => 20 @[%v]`, + `vector_matching_b{l="x"} => 40 @[%v]`, + `cpu_count{instance="1", type="smp"} => 200 @[%v]`, + `cpu_count{instance="0", type="smp"} => 100 @[%v]`, + `cpu_count{instance="0", type="numa"} => 300 @[%v]`, + }, + }, + { + expr: `{job=~"server", job!~"api"}`, + output: []string{ + `http_requests{group="canary", instance="0", job="app-server"} => 700 @[%v]`, + `http_requests{group="canary", instance="1", job="app-server"} => 800 @[%v]`, + `http_requests{group="production", instance="0", job="app-server"} => 500 @[%v]`, + `http_requests{group="production", instance="1", job="app-server"} => 600 @[%v]`, + }, + }, + { + // Test alternative "by"-clause order. + expr: `sum by (group) (http_requests{job="api-server"})`, + output: []string{ + `{group="canary"} => 700 @[%v]`, + `{group="production"} => 300 @[%v]`, + }, + }, + { + // Test alternative "by"-clause order with "keeping_extra". + expr: `sum by (group) keeping_extra (http_requests{job="api-server"})`, + output: []string{ + `{group="canary", job="api-server"} => 700 @[%v]`, + `{group="production", job="api-server"} => 300 @[%v]`, + }, + }, + { + // Test both alternative "by"-clause orders in one expression. + // Public health warning: stick to one form within an expression (or even + // in an organization), or risk serious user confusion. + expr: `sum(sum by (group) keeping_extra (http_requests{job="api-server"})) by (job)`, + output: []string{ + `{job="api-server"} => 1000 @[%v]`, + }, + }, + { + expr: `http_requests{group="canary"} and http_requests{instance="0"}`, + output: []string{ + `http_requests{group="canary", instance="0", job="api-server"} => 300 @[%v]`, + `http_requests{group="canary", instance="0", job="app-server"} => 700 @[%v]`, + }, + }, + { + expr: `(http_requests{group="canary"} + 1) and http_requests{instance="0"}`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => 301 @[%v]`, + `{group="canary", instance="0", job="app-server"} => 701 @[%v]`, + }, + }, + { + expr: `(http_requests{group="canary"} + 1) and on(instance, job) http_requests{instance="0", group="production"}`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => 301 @[%v]`, + `{group="canary", instance="0", job="app-server"} => 701 @[%v]`, + }, + }, + { + expr: `(http_requests{group="canary"} + 1) and on(instance) http_requests{instance="0", group="production"}`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => 301 @[%v]`, + `{group="canary", instance="0", job="app-server"} => 701 @[%v]`, + }, + }, + { + expr: `http_requests{group="canary"} or http_requests{group="production"}`, + output: []string{ + `http_requests{group="canary", instance="0", job="api-server"} => 300 @[%v]`, + `http_requests{group="canary", instance="0", job="app-server"} => 700 @[%v]`, + `http_requests{group="canary", instance="1", job="api-server"} => 400 @[%v]`, + `http_requests{group="canary", instance="1", job="app-server"} => 800 @[%v]`, + `http_requests{group="production", instance="0", job="api-server"} => 100 @[%v]`, + `http_requests{group="production", instance="0", job="app-server"} => 500 @[%v]`, + `http_requests{group="production", instance="1", job="api-server"} => 200 @[%v]`, + `http_requests{group="production", instance="1", job="app-server"} => 600 @[%v]`, + }, + }, + { + // On overlap the rhs samples must be dropped. + expr: `(http_requests{group="canary"} + 1) or http_requests{instance="1"}`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => 301 @[%v]`, + `{group="canary", instance="0", job="app-server"} => 701 @[%v]`, + `{group="canary", instance="1", job="api-server"} => 401 @[%v]`, + `{group="canary", instance="1", job="app-server"} => 801 @[%v]`, + `http_requests{group="production", instance="1", job="api-server"} => 200 @[%v]`, + `http_requests{group="production", instance="1", job="app-server"} => 600 @[%v]`, + }, + }, + { + // Matching only on instance excludes everything that has instance=0/1 but includes + // entries without the instance label. + expr: `(http_requests{group="canary"} + 1) or on(instance) (http_requests or cpu_count or vector_matching_a)`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => 301 @[%v]`, + `{group="canary", instance="0", job="app-server"} => 701 @[%v]`, + `{group="canary", instance="1", job="api-server"} => 401 @[%v]`, + `{group="canary", instance="1", job="app-server"} => 801 @[%v]`, + `vector_matching_a{l="x"} => 10 @[%v]`, + `vector_matching_a{l="y"} => 20 @[%v]`, + }, + }, + { + expr: `http_requests{group="canary"} / on(instance,job) http_requests{group="production"}`, + output: []string{ + `{instance="0", job="api-server"} => 3 @[%v]`, + `{instance="0", job="app-server"} => 1.4 @[%v]`, + `{instance="1", job="api-server"} => 2 @[%v]`, + `{instance="1", job="app-server"} => 1.3333333333333333 @[%v]`, + }, + }, + { + // Include labels must guarantee uniquely identifiable time series. + expr: `http_requests{group="production"} / on(instance) group_left(group) cpu_count{type="smp"}`, + shouldFail: true, + }, + { + // Many-to-many matching is not allowed. + expr: `http_requests{group="production"} / on(instance) group_left(job,type) cpu_count`, + shouldFail: true, + }, + { + // Many-to-one matching must be explicit. + expr: `http_requests{group="production"} / on(instance) cpu_count{type="smp"}`, + shouldFail: true, + }, + { + expr: `http_requests{group="production"} / on(instance) group_left(job) cpu_count{type="smp"}`, + output: []string{ + `{instance="1", job="api-server"} => 1 @[%v]`, + `{instance="0", job="app-server"} => 5 @[%v]`, + `{instance="1", job="app-server"} => 3 @[%v]`, + `{instance="0", job="api-server"} => 1 @[%v]`, + }, + }, + { + // Ensure sidedness of grouping preserves operand sides. + expr: `cpu_count{type="smp"} / on(instance) group_right(job) http_requests{group="production"}`, + output: []string{ + `{instance="1", job="app-server"} => 0.3333333333333333 @[%v]`, + `{instance="0", job="app-server"} => 0.2 @[%v]`, + `{instance="1", job="api-server"} => 1 @[%v]`, + `{instance="0", job="api-server"} => 1 @[%v]`, + }, + }, + { + // Include labels from both sides. + expr: `http_requests{group="production"} / on(instance) group_left(job) cpu_count{type="smp"}`, + output: []string{ + `{instance="1", job="api-server"} => 1 @[%v]`, + `{instance="0", job="app-server"} => 5 @[%v]`, + `{instance="1", job="app-server"} => 3 @[%v]`, + `{instance="0", job="api-server"} => 1 @[%v]`, + }, + }, + { + expr: `http_requests{group="production"} < on(instance,job) http_requests{group="canary"}`, + output: []string{ + `{instance="1", job="app-server"} => 600 @[%v]`, + `{instance="0", job="app-server"} => 500 @[%v]`, + `{instance="1", job="api-server"} => 200 @[%v]`, + `{instance="0", job="api-server"} => 100 @[%v]`, + }, + }, + { + expr: `http_requests{group="production"} > on(instance,job) http_requests{group="canary"}`, + output: []string{}, + }, + { + expr: `http_requests{group="production"} == on(instance,job) http_requests{group="canary"}`, + output: []string{}, + }, + { + expr: `http_requests > on(instance) group_left(group,job) cpu_count{type="smp"}`, + output: []string{ + `{group="canary", instance="0", job="app-server"} => 700 @[%v]`, + `{group="canary", instance="1", job="app-server"} => 800 @[%v]`, + `{group="canary", instance="0", job="api-server"} => 300 @[%v]`, + `{group="canary", instance="1", job="api-server"} => 400 @[%v]`, + `{group="production", instance="0", job="app-server"} => 500 @[%v]`, + `{group="production", instance="1", job="app-server"} => 600 @[%v]`, + }, + }, + { + expr: `http_requests / on(instance) 3`, + shouldFail: true, + }, + { + expr: `3 / on(instance) http_requests_total`, + shouldFail: true, + }, + { + expr: `3 / on(instance) 3`, + shouldFail: true, + }, + { + // Missing label list for grouping mod. + expr: `http_requests{group="production"} / on(instance) group_left cpu_count{type="smp"}`, + shouldFail: true, + }, + { + // No group mod allowed for logical operations. + expr: `http_requests{group="production"} or on(instance) group_left(type) cpu_count{type="smp"}`, + shouldFail: true, + }, + { + // No group mod allowed for logical operations. + expr: `http_requests{group="production"} and on(instance) group_left(type) cpu_count{type="smp"}`, + shouldFail: true, + }, + { + // No duplicate use of label. + expr: `http_requests{group="production"} + on(instance) group_left(job,instance) cpu_count{type="smp"}`, + shouldFail: true, + }, + { + expr: `{l="x"} + on(__name__) {l="y"}`, + output: []string{ + `vector_matching_a => 30 @[%v]`, + }, + }, + { + expr: `absent(nonexistent)`, + output: []string{ + `{} => 1 @[%v]`, + }, + }, + { + expr: `absent(nonexistent{job="testjob", instance="testinstance", method=~".*"})`, + output: []string{ + `{instance="testinstance", job="testjob"} => 1 @[%v]`, + }, + }, + { + expr: `count_scalar(absent(http_requests))`, + output: []string{ + `scalar: 0 @[%v]`, + }, + }, + { + expr: `count_scalar(absent(sum(http_requests)))`, + output: []string{ + `scalar: 0 @[%v]`, + }, + }, + { + expr: `absent(sum(nonexistent{job="testjob", instance="testinstance"}))`, + output: []string{ + `{} => 1 @[%v]`, + }, + }, + { + expr: `http_requests{group="production",job="api-server"} offset 5m`, + output: []string{ + `http_requests{group="production", instance="0", job="api-server"} => 90 @[%v]`, + `http_requests{group="production", instance="1", job="api-server"} => 180 @[%v]`, + }, + }, + { + expr: `rate(http_requests{group="production",job="api-server"}[10m] offset 5m)`, + output: []string{ + `{group="production", instance="0", job="api-server"} => 0.03333333333333333 @[%v]`, + `{group="production", instance="1", job="api-server"} => 0.06666666666666667 @[%v]`, + }, + }, + { + expr: `rate(http_requests[10m]) offset 5m`, + shouldFail: true, + }, + { + expr: `sum(http_requests) offset 5m`, + shouldFail: true, + }, + // Regression test for missing separator byte in labelsToGroupingKey. + { + expr: `sum(label_grouping_test) by (a, b)`, + output: []string{ + `{a="a", b="abb"} => 200 @[%v]`, + `{a="aa", b="bb"} => 100 @[%v]`, + }, + }, + // Quantile too low. + { + expr: `histogram_quantile(-0.1, testhistogram_bucket)`, + output: []string{ + `{start="positive"} => -Inf @[%v]`, + `{start="negative"} => -Inf @[%v]`, + }, + }, + // Quantile too high. + { + expr: `histogram_quantile(1.01, testhistogram_bucket)`, + output: []string{ + `{start="positive"} => +Inf @[%v]`, + `{start="negative"} => +Inf @[%v]`, + }, + }, + // Quantile value in lowest bucket, which is positive. + { + expr: `histogram_quantile(0, testhistogram_bucket{start="positive"})`, + output: []string{ + `{start="positive"} => 0 @[%v]`, + }, + }, + // Quantile value in lowest bucket, which is negative. + { + expr: `histogram_quantile(0, testhistogram_bucket{start="negative"})`, + output: []string{ + `{start="negative"} => -0.2 @[%v]`, + }, + }, + // Quantile value in highest bucket. + { + expr: `histogram_quantile(1, testhistogram_bucket)`, + output: []string{ + `{start="positive"} => 1 @[%v]`, + `{start="negative"} => 0.3 @[%v]`, + }, + }, + // Finally some useful quantiles. + { + expr: `histogram_quantile(0.2, testhistogram_bucket)`, + output: []string{ + `{start="positive"} => 0.048 @[%v]`, + `{start="negative"} => -0.2 @[%v]`, + }, + }, + { + expr: `histogram_quantile(0.5, testhistogram_bucket)`, + output: []string{ + `{start="positive"} => 0.15 @[%v]`, + `{start="negative"} => -0.15 @[%v]`, + }, + }, + { + expr: `histogram_quantile(0.8, testhistogram_bucket)`, + output: []string{ + `{start="positive"} => 0.72 @[%v]`, + `{start="negative"} => 0.3 @[%v]`, + }, + }, + // More realistic with rates. + { + expr: `histogram_quantile(0.2, rate(testhistogram_bucket[5m]))`, + output: []string{ + `{start="positive"} => 0.048 @[%v]`, + `{start="negative"} => -0.2 @[%v]`, + }, + }, + { + expr: `histogram_quantile(0.5, rate(testhistogram_bucket[5m]))`, + output: []string{ + `{start="positive"} => 0.15 @[%v]`, + `{start="negative"} => -0.15 @[%v]`, + }, + }, + { + expr: `histogram_quantile(0.8, rate(testhistogram_bucket[5m]))`, + output: []string{ + `{start="positive"} => 0.72 @[%v]`, + `{start="negative"} => 0.3 @[%v]`, + }, + }, + // Aggregated histogram: Everything in one. + { + expr: `histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[5m])) by (le))`, + output: []string{ + `{} => 0.075 @[%v]`, + }, + }, + { + expr: `histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[5m])) by (le))`, + output: []string{ + `{} => 0.1277777777777778 @[%v]`, + }, + }, + // Aggregated histogram: Everything in one. Now with avg, which does not change anything. + { + expr: `histogram_quantile(0.3, avg(rate(request_duration_seconds_bucket[5m])) by (le))`, + output: []string{ + `{} => 0.075 @[%v]`, + }, + }, + { + expr: `histogram_quantile(0.5, avg(rate(request_duration_seconds_bucket[5m])) by (le))`, + output: []string{ + `{} => 0.12777777777777778 @[%v]`, + }, + }, + // Aggregated histogram: By job. + { + expr: `histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[5m])) by (le, instance))`, + output: []string{ + `{instance="ins1"} => 0.075 @[%v]`, + `{instance="ins2"} => 0.075 @[%v]`, + }, + }, + { + expr: `histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[5m])) by (le, instance))`, + output: []string{ + `{instance="ins1"} => 0.1333333333 @[%v]`, + `{instance="ins2"} => 0.125 @[%v]`, + }, + }, + // Aggregated histogram: By instance. + { + expr: `histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[5m])) by (le, job))`, + output: []string{ + `{job="job1"} => 0.1 @[%v]`, + `{job="job2"} => 0.0642857142857143 @[%v]`, + }, + }, + { + expr: `histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[5m])) by (le, job))`, + output: []string{ + `{job="job1"} => 0.14 @[%v]`, + `{job="job2"} => 0.1125 @[%v]`, + }, + }, + // Aggregated histogram: By job and instance. + { + expr: `histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[5m])) by (le, job, instance))`, + output: []string{ + `{instance="ins1", job="job1"} => 0.11 @[%v]`, + `{instance="ins2", job="job1"} => 0.09 @[%v]`, + `{instance="ins1", job="job2"} => 0.06 @[%v]`, + `{instance="ins2", job="job2"} => 0.0675 @[%v]`, + }, + }, + { + expr: `histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[5m])) by (le, job, instance))`, + output: []string{ + `{instance="ins1", job="job1"} => 0.15 @[%v]`, + `{instance="ins2", job="job1"} => 0.1333333333333333 @[%v]`, + `{instance="ins1", job="job2"} => 0.1 @[%v]`, + `{instance="ins2", job="job2"} => 0.1166666666666667 @[%v]`, + }, + }, + // The unaggregated histogram for comparison. Same result as the previous one. + { + expr: `histogram_quantile(0.3, rate(request_duration_seconds_bucket[5m]))`, + output: []string{ + `{instance="ins1", job="job1"} => 0.11 @[%v]`, + `{instance="ins2", job="job1"} => 0.09 @[%v]`, + `{instance="ins1", job="job2"} => 0.06 @[%v]`, + `{instance="ins2", job="job2"} => 0.0675 @[%v]`, + }, + }, + { + expr: `histogram_quantile(0.5, rate(request_duration_seconds_bucket[5m]))`, + output: []string{ + `{instance="ins1", job="job1"} => 0.15 @[%v]`, + `{instance="ins2", job="job1"} => 0.13333333333333333 @[%v]`, + `{instance="ins1", job="job2"} => 0.1 @[%v]`, + `{instance="ins2", job="job2"} => 0.11666666666666667 @[%v]`, + }, + }, + { + expr: `12.34e6`, + output: []string{`scalar: 12340000 @[%v]`}, + }, + { + expr: `12.34e+6`, + output: []string{`scalar: 12340000 @[%v]`}, + }, + { + expr: `12.34e-6`, + output: []string{`scalar: 0.00001234 @[%v]`}, + }, + { + expr: `1+1`, + output: []string{`scalar: 2 @[%v]`}, + }, + { + expr: `1-1`, + output: []string{`scalar: 0 @[%v]`}, + }, + { + expr: `1 - -1`, + output: []string{`scalar: 2 @[%v]`}, + }, + { + expr: `.2`, + output: []string{`scalar: 0.2 @[%v]`}, + }, + { + expr: `+0.2`, + output: []string{`scalar: 0.2 @[%v]`}, + }, + { + expr: `-0.2e-6`, + output: []string{`scalar: -0.0000002 @[%v]`}, + }, + { + expr: `+Inf`, + output: []string{`scalar: +Inf @[%v]`}, + }, + { + expr: `inF`, + output: []string{`scalar: +Inf @[%v]`}, + }, + { + expr: `-inf`, + output: []string{`scalar: -Inf @[%v]`}, + }, + { + expr: `NaN`, + output: []string{`scalar: NaN @[%v]`}, + }, + { + expr: `nan`, + output: []string{`scalar: NaN @[%v]`}, + }, + { + expr: `2.`, + output: []string{`scalar: 2 @[%v]`}, + }, + { + expr: `999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`, + shouldFail: true, + }, + { + expr: `1 / 0`, + output: []string{`scalar: +Inf @[%v]`}, + }, + { + expr: `-1 / 0`, + output: []string{`scalar: -Inf @[%v]`}, + }, + { + expr: `0 / 0`, + output: []string{`scalar: NaN @[%v]`}, + }, + { + expr: `1 % 0`, + output: []string{`scalar: NaN @[%v]`}, + }, + { + expr: `http_requests{group="canary", instance="0", job="api-server"} / 0`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => +Inf @[%v]`, + }, + }, + { + expr: `-1 * http_requests{group="canary", instance="0", job="api-server"} / 0`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => -Inf @[%v]`, + }, + }, + { + expr: `0 * http_requests{group="canary", instance="0", job="api-server"} / 0`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => NaN @[%v]`, + }, + }, + { + expr: `0 * http_requests{group="canary", instance="0", job="api-server"} % 0`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => NaN @[%v]`, + }, + }, + { + expr: `exp(vector_matching_a)`, + output: []string{ + `{l="x"} => 22026.465794806718 @[%v]`, + `{l="y"} => 485165195.4097903 @[%v]`, + }, + }, + { + expr: `exp(vector_matching_a - 10)`, + output: []string{ + `{l="y"} => 22026.465794806718 @[%v]`, + `{l="x"} => 1 @[%v]`, + }, + }, + { + expr: `exp(vector_matching_a - 20)`, + output: []string{ + `{l="x"} => 4.5399929762484854e-05 @[%v]`, + `{l="y"} => 1 @[%v]`, + }, + }, + { + expr: `ln(vector_matching_a)`, + output: []string{ + `{l="x"} => 2.302585092994046 @[%v]`, + `{l="y"} => 2.995732273553991 @[%v]`, + }, + }, + { + expr: `ln(vector_matching_a - 10)`, + output: []string{ + `{l="y"} => 2.302585092994046 @[%v]`, + `{l="x"} => -Inf @[%v]`, + }, + }, + { + expr: `ln(vector_matching_a - 20)`, + output: []string{ + `{l="y"} => -Inf @[%v]`, + `{l="x"} => NaN @[%v]`, + }, + }, + { + expr: `exp(ln(vector_matching_a))`, + output: []string{ + `{l="y"} => 20 @[%v]`, + `{l="x"} => 10 @[%v]`, + }, + }, + { + expr: `sqrt(vector_matching_a)`, + output: []string{ + `{l="x"} => 3.1622776601683795 @[%v]`, + `{l="y"} => 4.47213595499958 @[%v]`, + }, + }, + { + expr: `log2(vector_matching_a)`, + output: []string{ + `{l="x"} => 3.3219280948873626 @[%v]`, + `{l="y"} => 4.321928094887363 @[%v]`, + }, + }, + { + expr: `log2(vector_matching_a - 10)`, + output: []string{ + `{l="y"} => 3.3219280948873626 @[%v]`, + `{l="x"} => -Inf @[%v]`, + }, + }, + { + expr: `log2(vector_matching_a - 20)`, + output: []string{ + `{l="x"} => NaN @[%v]`, + `{l="y"} => -Inf @[%v]`, + }, + }, + { + expr: `log10(vector_matching_a)`, + output: []string{ + `{l="x"} => 1 @[%v]`, + `{l="y"} => 1.301029995663981 @[%v]`, + }, + }, + { + expr: `log10(vector_matching_a - 10)`, + output: []string{ + `{l="y"} => 1 @[%v]`, + `{l="x"} => -Inf @[%v]`, + }, + }, + { + expr: `log10(vector_matching_a - 20)`, + output: []string{ + `{l="x"} => NaN @[%v]`, + `{l="y"} => -Inf @[%v]`, + }, + }, + { + expr: `stddev(http_requests)`, + output: []string{ + `{} => 229.12878474779 @[%v]`, + }, + }, + { + expr: `stddev by (instance)(http_requests)`, + output: []string{ + `{instance="0"} => 223.60679774998 @[%v]`, + `{instance="1"} => 223.60679774998 @[%v]`, + }, + }, + { + expr: `stdvar(http_requests)`, + output: []string{ + `{} => 52500 @[%v]`, + }, + }, + { + expr: `stdvar by (instance)(http_requests)`, + output: []string{ + `{instance="0"} => 50000 @[%v]`, + `{instance="1"} => 50000 @[%v]`, + }, + }, + } + + storage, closer := newTestStorage(t) + defer closer.Close() + + engine := NewEngine(storage) + + for i, exprTest := range expressionTests { + expectedLines := annotateWithTime(exprTest.output, testEvalTime) + + query, err := engine.NewInstantQuery(exprTest.expr, testEvalTime) + + if err != nil { + if !exprTest.shouldFail { + t.Errorf("%d. Error during parsing: %v", i, err) + t.Errorf("%d. Expression: %v", i, exprTest.expr) + } + continue + } + + failed := false + + res := query.Exec() + if res.Err != nil { + if !exprTest.shouldFail { + t.Errorf("%d. Error evaluating query: %s", res.Err) + t.Errorf("%d. Expression: %v", i, exprTest.expr) + } + continue + } + if exprTest.shouldFail { + t.Errorf("%d. Expression should fail but did not", i) + continue + } + resultLines := strings.Split(res.String(), "\n") + // resultStr := ast.EvalToString(testExpr, testEvalTime, ast.Text, storage, stats.NewTimerGroup()) + // resultLines := strings.Split(resultStr, "\n") + + if len(exprTest.output) == 0 && strings.Trim(res.String(), "\n") == "" { + // expected and received empty vector, everything is fine + continue + } else if len(exprTest.output) != len(resultLines) { + t.Errorf("%d. Number of samples in expected and actual output don't match", i) + failed = true + } + + if exprTest.checkOrder { + for j, expectedSample := range expectedLines { + if resultLines[j] != expectedSample { + t.Errorf("%d.%d. Expected sample '%v', got '%v'", i, j, resultLines[j], expectedSample) + failed = true + } + } + } else { + for j, expectedSample := range expectedLines { + found := false + for _, actualSample := range resultLines { + if samplesAlmostEqual(actualSample, expectedSample) { + found = true + } + } + if !found { + t.Errorf("%d.%d. Couldn't find expected sample in output: '%v'", i, j, expectedSample) + failed = true + } + } + } + + if failed { + t.Errorf("%d. Expression: %v\n%v", i, exprTest.expr, vectorComparisonString(expectedLines, resultLines)) + } + + } +} + +func TestRangedEvaluationRegressions(t *testing.T) { + scenarios := []struct { + in Matrix + out Matrix + expr string + }{ + { + // Testing COWMetric behavior in drop_common_labels. + in: Matrix{ + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testmetric", + "testlabel": "1", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime, + Value: 1, + }, + { + Timestamp: testStartTime.Add(time.Hour), + Value: 1, + }, + }, + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testmetric", + "testlabel": "2", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime.Add(time.Hour), + Value: 2, + }, + }, + }, + }, + out: Matrix{ + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testmetric", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime, + Value: 1, + }, + }, + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testmetric", + "testlabel": "1", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime.Add(time.Hour), + Value: 1, + }, + }, + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testmetric", + "testlabel": "2", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime.Add(time.Hour), + Value: 2, + }, + }, + }, + }, + expr: "drop_common_labels(testmetric)", + }, + { + // Testing COWMetric behavior in vector aggregation. + in: Matrix{ + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testmetric", + "testlabel": "1", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime, + Value: 1, + }, + { + Timestamp: testStartTime.Add(time.Hour), + Value: 1, + }, + }, + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testmetric", + "testlabel": "2", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime, + Value: 2, + }, + }, + }, + }, + out: Matrix{ + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{}, + }, + Values: metric.Values{ + { + Timestamp: testStartTime, + Value: 3, + }, + }, + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + "testlabel": "1", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime.Add(time.Hour), + Value: 1, + }, + }, + }, + }, + expr: "sum(testmetric) keeping_extra", + }, + { + // Testing metric fingerprint grouping behavior. + in: Matrix{ + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testmetric", + "aa": "bb", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime, + Value: 1, + }, + }, + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testmetric", + "a": "abb", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime, + Value: 2, + }, + }, + }, + }, + out: Matrix{ + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testmetric", + "aa": "bb", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime, + Value: 1, + }, + }, + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testmetric", + "a": "abb", + }, + }, + Values: metric.Values{ + { + Timestamp: testStartTime, + Value: 2, + }, + }, + }, + }, + expr: "testmetric", + }, + } + + for i, s := range scenarios { + storage, closer := local.NewTestStorage(t, 1) + storeMatrix(storage, s.in) + + engine := NewEngine(storage) + query, err := engine.NewRangeQuery(s.expr, testStartTime, testStartTime.Add(time.Hour), time.Hour) + if err != nil { + t.Errorf("%d. Error in expression %q", i, s.expr) + t.Fatalf("%d. Error parsing expression: %v", i, err) + } + res := query.Exec() + if res.Err != nil { + t.Errorf("%d. Error in expression %q", i, s.expr) + t.Fatalf("%d. Error evaluating expression: %v", i, err) + } + + if res.String() != s.out.String() { + t.Errorf("%d. Error in expression %q", i, s.expr) + t.Fatalf("%d. Expression: %s\n\ngot:\n=====\n%v\n====\n\nwant:\n=====\n%v\n=====\n", i, s.expr, res.String(), s.out.String()) + } + + closer.Close() + } +} diff --git a/promql/quantile.go b/promql/quantile.go new file mode 100644 index 0000000000..11edad85fe --- /dev/null +++ b/promql/quantile.go @@ -0,0 +1,106 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promql + +import ( + "math" + "sort" + + clientmodel "github.com/prometheus/client_golang/model" +) + +// Helpers to calculate quantiles. + +// excludedLabels are the labels to exclude from signature calculation for +// quantiles. +var excludedLabels = map[clientmodel.LabelName]struct{}{ + clientmodel.MetricNameLabel: struct{}{}, + clientmodel.BucketLabel: struct{}{}, +} + +type bucket struct { + upperBound float64 + count clientmodel.SampleValue +} + +// buckets implements sort.Interface. +type buckets []bucket + +func (b buckets) Len() int { return len(b) } +func (b buckets) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b buckets) Less(i, j int) bool { return b[i].upperBound < b[j].upperBound } + +type metricWithBuckets struct { + metric clientmodel.COWMetric + buckets buckets +} + +// quantile calculates the quantile 'q' based on the given buckets. The buckets +// will be sorted by upperBound by this function (i.e. no sorting needed before +// calling this function). The quantile value is interpolated assuming a linear +// distribution within a bucket. However, if the quantile falls into the highest +// bucket, the upper bound of the 2nd highest bucket is returned. A natural +// lower bound of 0 is assumed if the upper bound of the lowest bucket is +// greater 0. In that case, interpolation in the lowest bucket happens linearly +// between 0 and the upper bound of the lowest bucket. However, if the lowest +// bucket has an upper bound less or equal 0, this upper bound is returned if +// the quantile falls into the lowest bucket. +// +// There are a number of special cases (once we have a way to report errors +// happening during evaluations of AST functions, we should report those +// explicitly): +// +// If 'buckets' has fewer than 2 elements, NaN is returned. +// +// If the highest bucket is not +Inf, NaN is returned. +// +// If q<0, -Inf is returned. +// +// If q>1, +Inf is returned. +func quantile(q clientmodel.SampleValue, buckets buckets) float64 { + if q < 0 { + return math.Inf(-1) + } + if q > 1 { + return math.Inf(+1) + } + if len(buckets) < 2 { + return math.NaN() + } + sort.Sort(buckets) + if !math.IsInf(buckets[len(buckets)-1].upperBound, +1) { + return math.NaN() + } + + rank := q * buckets[len(buckets)-1].count + b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank }) + + if b == len(buckets)-1 { + return buckets[len(buckets)-2].upperBound + } + if b == 0 && buckets[0].upperBound <= 0 { + return buckets[0].upperBound + } + var ( + bucketStart float64 + bucketEnd = buckets[b].upperBound + count = buckets[b].count + ) + if b > 0 { + bucketStart = buckets[b-1].upperBound + count -= buckets[b-1].count + rank -= buckets[b-1].count + } + return bucketStart + (bucketEnd-bucketStart)*float64(rank/count) +} diff --git a/promql/setup_test.go b/promql/setup_test.go new file mode 100644 index 0000000000..5796d7f874 --- /dev/null +++ b/promql/setup_test.go @@ -0,0 +1,486 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promql + +import ( + "time" + + clientmodel "github.com/prometheus/client_golang/model" + + "github.com/prometheus/prometheus/storage/local" + "github.com/prometheus/prometheus/storage/metric" +) + +var testSampleInterval = time.Duration(5) * time.Minute +var testStartTime = clientmodel.Timestamp(0) + +func getTestValueStream(startVal, endVal, stepVal clientmodel.SampleValue, startTime clientmodel.Timestamp) (resultValues metric.Values) { + currentTime := startTime + for currentVal := startVal; currentVal <= endVal; currentVal += stepVal { + sample := metric.SamplePair{ + Value: currentVal, + Timestamp: currentTime, + } + resultValues = append(resultValues, sample) + currentTime = currentTime.Add(testSampleInterval) + } + return resultValues +} + +func getTestVectorFromTestMatrix(matrix Matrix) Vector { + vector := Vector{} + for _, sampleStream := range matrix { + lastSample := sampleStream.Values[len(sampleStream.Values)-1] + vector = append(vector, &Sample{ + Metric: sampleStream.Metric, + Value: lastSample.Value, + Timestamp: lastSample.Timestamp, + }) + } + return vector +} + +func storeMatrix(storage local.Storage, matrix Matrix) { + pendingSamples := clientmodel.Samples{} + for _, sampleStream := range matrix { + for _, sample := range sampleStream.Values { + pendingSamples = append(pendingSamples, &clientmodel.Sample{ + Metric: sampleStream.Metric.Metric, + Value: sample.Value, + Timestamp: sample.Timestamp, + }) + } + } + for _, s := range pendingSamples { + storage.Append(s) + } + storage.WaitForIndexing() +} + +var testVector = getTestVectorFromTestMatrix(testMatrix) + +var testMatrix = Matrix{ + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "http_requests", + clientmodel.JobLabel: "api-server", + "instance": "0", + "group": "production", + }, + }, + Values: getTestValueStream(0, 100, 10, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "http_requests", + clientmodel.JobLabel: "api-server", + "instance": "1", + "group": "production", + }, + }, + Values: getTestValueStream(0, 200, 20, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "http_requests", + clientmodel.JobLabel: "api-server", + "instance": "0", + "group": "canary", + }, + }, + Values: getTestValueStream(0, 300, 30, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "http_requests", + clientmodel.JobLabel: "api-server", + "instance": "1", + "group": "canary", + }, + }, + Values: getTestValueStream(0, 400, 40, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "http_requests", + clientmodel.JobLabel: "app-server", + "instance": "0", + "group": "production", + }, + }, + Values: getTestValueStream(0, 500, 50, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "http_requests", + clientmodel.JobLabel: "app-server", + "instance": "1", + "group": "production", + }, + }, + Values: getTestValueStream(0, 600, 60, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "http_requests", + clientmodel.JobLabel: "app-server", + "instance": "0", + "group": "canary", + }, + }, + Values: getTestValueStream(0, 700, 70, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "http_requests", + clientmodel.JobLabel: "app-server", + "instance": "1", + "group": "canary", + }, + }, + Values: getTestValueStream(0, 800, 80, testStartTime), + }, + // Single-letter metric and label names. + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "x", + "y": "testvalue", + }, + }, + Values: getTestValueStream(0, 100, 10, testStartTime), + }, + // Counter reset in the middle of range. + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testcounter_reset_middle", + }, + }, + Values: append(getTestValueStream(0, 40, 10, testStartTime), getTestValueStream(0, 50, 10, testStartTime.Add(testSampleInterval*5))...), + }, + // Counter reset at the end of range. + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testcounter_reset_end", + }, + }, + Values: append(getTestValueStream(0, 90, 10, testStartTime), getTestValueStream(0, 0, 10, testStartTime.Add(testSampleInterval*10))...), + }, + // For label-key grouping regression test. + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "label_grouping_test", + "a": "aa", + "b": "bb", + }, + }, + Values: getTestValueStream(0, 100, 10, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "label_grouping_test", + "a": "a", + "b": "abb", + }, + }, + Values: getTestValueStream(0, 200, 20, testStartTime), + }, + // Two histograms with 4 buckets each (*_sum and *_count not included, + // only buckets). Lowest bucket for one histogram < 0, for the other > + // 0. They have the same name, just separated by label. Not useful in + // practice, but can happen (if clients change bucketing), and the + // server has to cope with it. + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testhistogram_bucket", + "le": "0.1", + "start": "positive", + }, + }, + Values: getTestValueStream(0, 50, 5, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testhistogram_bucket", + "le": ".2", + "start": "positive", + }, + }, + Values: getTestValueStream(0, 70, 7, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testhistogram_bucket", + "le": "1e0", + "start": "positive", + }, + }, + Values: getTestValueStream(0, 110, 11, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testhistogram_bucket", + "le": "+Inf", + "start": "positive", + }, + }, + Values: getTestValueStream(0, 120, 12, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testhistogram_bucket", + "le": "-.2", + "start": "negative", + }, + }, + Values: getTestValueStream(0, 10, 1, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testhistogram_bucket", + "le": "-0.1", + "start": "negative", + }, + }, + Values: getTestValueStream(0, 20, 2, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testhistogram_bucket", + "le": "0.3", + "start": "negative", + }, + }, + Values: getTestValueStream(0, 20, 2, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "testhistogram_bucket", + "le": "+Inf", + "start": "negative", + }, + }, + Values: getTestValueStream(0, 30, 3, testStartTime), + }, + // Now a more realistic histogram per job and instance to test aggregation. + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job1", + "instance": "ins1", + "le": "0.1", + }, + }, + Values: getTestValueStream(0, 10, 1, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job1", + "instance": "ins1", + "le": "0.2", + }, + }, + Values: getTestValueStream(0, 30, 3, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job1", + "instance": "ins1", + "le": "+Inf", + }, + }, + Values: getTestValueStream(0, 40, 4, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job1", + "instance": "ins2", + "le": "0.1", + }, + }, + Values: getTestValueStream(0, 20, 2, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job1", + "instance": "ins2", + "le": "0.2", + }, + }, + Values: getTestValueStream(0, 50, 5, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job1", + "instance": "ins2", + "le": "+Inf", + }, + }, + Values: getTestValueStream(0, 60, 6, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job2", + "instance": "ins1", + "le": "0.1", + }, + }, + Values: getTestValueStream(0, 30, 3, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job2", + "instance": "ins1", + "le": "0.2", + }, + }, + Values: getTestValueStream(0, 40, 4, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job2", + "instance": "ins1", + "le": "+Inf", + }, + }, + Values: getTestValueStream(0, 60, 6, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job2", + "instance": "ins2", + "le": "0.1", + }, + }, + Values: getTestValueStream(0, 40, 4, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job2", + "instance": "ins2", + "le": "0.2", + }, + }, + Values: getTestValueStream(0, 70, 7, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "request_duration_seconds_bucket", + clientmodel.JobLabel: "job2", + "instance": "ins2", + "le": "+Inf", + }, + }, + Values: getTestValueStream(0, 90, 9, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "vector_matching_a", + "l": "x", + }, + }, + Values: getTestValueStream(0, 100, 1, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "vector_matching_a", + "l": "y", + }, + }, + Values: getTestValueStream(0, 100, 2, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "vector_matching_b", + "l": "x", + }, + }, + Values: getTestValueStream(0, 100, 4, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "cpu_count", + "instance": "0", + "type": "numa", + }, + }, + Values: getTestValueStream(0, 500, 30, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "cpu_count", + "instance": "0", + "type": "smp", + }, + }, + Values: getTestValueStream(0, 200, 10, testStartTime), + }, + { + Metric: clientmodel.COWMetric{ + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "cpu_count", + "instance": "1", + "type": "smp", + }, + }, + Values: getTestValueStream(0, 200, 20, testStartTime), + }, +}