Merge pull request #8487 from pschou/dev_neg_offset

allow negative offset

cmd/prometheus/main.go

@@ -117,7 +117,8 @@ type flagConfig struct {
 	featureList []string
 	// These options are extracted from featureList
 	// for ease of use.
-	enablePromQLAtModifier bool
+	enablePromQLAtModifier     bool
+	enablePromQLNegativeOffset bool
 
 	prometheusURL   string
 	corsRegexString string
@@ -134,6 +135,9 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 			case "promql-at-modifier":
 				c.enablePromQLAtModifier = true
 				level.Info(logger).Log("msg", "Experimental promql-at-modifier enabled")
+			case "promql-negative-offset":
+				c.enablePromQLNegativeOffset = true
+				level.Info(logger).Log("msg", "Experimental promql-negative-offset enabled")
 			case "remote-write-receiver":
 				c.web.RemoteWriteReceiver = true
 				level.Info(logger).Log("msg", "Experimental remote-write-receiver enabled")
@@ -441,6 +445,7 @@ func main() {
 			LookbackDelta:            time.Duration(cfg.lookbackDelta),
 			NoStepSubqueryIntervalFn: noStepSubqueryInterval.Get,
 			EnableAtModifier:         cfg.enablePromQLAtModifier,
+			EnableNegativeOffset:     cfg.enablePromQLNegativeOffset,
 		}
 
 		queryEngine = promql.NewEngine(opts)

docs/disabled_features.md

@@ -18,6 +18,20 @@ They may be enabled by default in future versions.
 The `@` modifier lets you specify the evaluation time for instant vector selectors,
 range vector selectors, and subqueries. More details can be found [here](querying/basics.md#modifier).
 
+## Negative offset in PromQL
+
+This negative offset is disabled by default since it breaks the invariant
+that PromQL does not look ahead of the evaluation time for samples.
+
+`--enable-feature=promql-negative-offset`
+
+In contrast to the positive offset modifier, the negative offset modifier lets
+one shift a vector into the future.  An example in which one may want to use a
+negative offset is reviewing past data and making temporal comparisons with
+more recent data.
+
+More details can be found [here](querying/basics.md#offset-modifier).
+
 ## Remote Write Receiver
 
 `--enable-feature=remote-write-receiver`

docs/querying/basics.md

@@ -204,6 +204,15 @@ The same works for range vectors. This returns the 5-minute rate that
 
     rate(http_requests_total[5m] offset 1w)
 
+For comparisons with temporal shifts forward in time, a negative offset
+can be specified:
+
+    rate(http_requests_total[5m] offset -1w)
+
+This feature is enabled by setting `--enable-feature=promql-negative-offset`
+flag. See [disabled features](../disabled_features.md) for more details about
+this flag.
+
 ### @ modifier
 
 The `@` modifier allows changing the evaluation time for individual instant

promql/engine.go

@@ -211,6 +211,9 @@ type EngineOpts struct {
 
 	// EnableAtModifier if true enables @ modifier. Disabled otherwise.
 	EnableAtModifier bool
+
+	// EnableNegativeOffset if true enables negative (-) offset values. Disabled otherwise.
+	EnableNegativeOffset bool
 }
 
 // Engine handles the lifetime of queries from beginning to end.
@@ -226,6 +229,7 @@ type Engine struct {
 	lookbackDelta            time.Duration
 	noStepSubqueryIntervalFn func(rangeMillis int64) int64
 	enableAtModifier         bool
+	enableNegativeOffset     bool
 }
 
 // NewEngine returns a new engine.
@@ -307,6 +311,7 @@ func NewEngine(opts EngineOpts) *Engine {
 		lookbackDelta:            opts.LookbackDelta,
 		noStepSubqueryIntervalFn: opts.NoStepSubqueryIntervalFn,
 		enableAtModifier:         opts.EnableAtModifier,
+		enableNegativeOffset:     opts.EnableNegativeOffset,
 	}
 }
 
@@ -388,34 +393,53 @@ func (ng *Engine) newQuery(q storage.Queryable, expr parser.Expr, start, end tim
 }
 
 var ErrValidationAtModifierDisabled = errors.New("@ modifier is disabled")
+var ErrValidationNegativeOffsetDisabled = errors.New("negative offset is disabled")
 
 func (ng *Engine) validateOpts(expr parser.Expr) error {
-	if ng.enableAtModifier {
+	if ng.enableAtModifier && ng.enableNegativeOffset {
 		return nil
 	}
 
+	var atModifierUsed, negativeOffsetUsed bool
+
 	var validationErr error
 	parser.Inspect(expr, func(node parser.Node, path []parser.Node) error {
 		switch n := node.(type) {
 		case *parser.VectorSelector:
 			if n.Timestamp != nil || n.StartOrEnd == parser.START || n.StartOrEnd == parser.END {
-				validationErr = ErrValidationAtModifierDisabled
-				return validationErr
+				atModifierUsed = true
+			}
+			if n.OriginalOffset < 0 {
+				negativeOffsetUsed = true
 			}
 
 		case *parser.MatrixSelector:
 			vs := n.VectorSelector.(*parser.VectorSelector)
 			if vs.Timestamp != nil || vs.StartOrEnd == parser.START || vs.StartOrEnd == parser.END {
-				validationErr = ErrValidationAtModifierDisabled
-				return validationErr
+				atModifierUsed = true
+			}
+			if vs.OriginalOffset < 0 {
+				negativeOffsetUsed = true
 			}
 
 		case *parser.SubqueryExpr:
 			if n.Timestamp != nil || n.StartOrEnd == parser.START || n.StartOrEnd == parser.END {
-				validationErr = ErrValidationAtModifierDisabled
-				return validationErr
+				atModifierUsed = true
+			}
+			if n.OriginalOffset < 0 {
+				negativeOffsetUsed = true
 			}
 		}
+
+		if atModifierUsed && !ng.enableAtModifier {
+			validationErr = ErrValidationAtModifierDisabled
+			return validationErr
+		}
+		if negativeOffsetUsed && !ng.enableNegativeOffset {
+			validationErr = ErrValidationNegativeOffsetDisabled
+			return validationErr
+		}
+
 		return nil
 	})
 

promql/engine_test.go

@@ -2299,6 +2299,18 @@ func TestEngineOptsValidation(t *testing.T) {
 		}, {
 			opts:  EngineOpts{EnableAtModifier: true},
 			query: "rate(metric[1h:1m] @ end())",
+		}, {
+			opts:  EngineOpts{EnableNegativeOffset: false},
+			query: "metric offset -1s", fail: true, expError: ErrValidationNegativeOffsetDisabled,
+		}, {
+			opts:  EngineOpts{EnableNegativeOffset: true},
+			query: "metric offset -1s",
+		}, {
+			opts:  EngineOpts{EnableAtModifier: true, EnableNegativeOffset: true},
+			query: "metric @ 100 offset -2m",
+		}, {
+			opts:  EngineOpts{EnableAtModifier: true, EnableNegativeOffset: true},
+			query: "metric offset -2m @ 100",
 		},
 	}
 

promql/parser/generated_parser.y

@@ -386,6 +386,11 @@ offset_expr: expr OFFSET duration
                         yylex.(*parser).addOffset($1, $3)
                         $$ = $1
                         }
+                | expr OFFSET SUB duration
+                        {
+                        yylex.(*parser).addOffset($1, -$4)
+                        $$ = $1
+                        }
                 | expr OFFSET error
                         { yylex.(*parser).unexpected("offset", "duration"); $$ = $1 }
                 ;

promql/parser/parse_test.go

@@ -1362,6 +1362,19 @@ var testExpr = []struct {
 				End:   13,
 			},
 		},
+	}, {
+		input: "foo offset -7m",
+		expected: &VectorSelector{
+			Name:           "foo",
+			OriginalOffset: -7 * time.Minute,
+			LabelMatchers: []*labels.Matcher{
+				MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
+			},
+			PosRange: PositionRange{
+				Start: 0,
+				End:   14,
+			},
+		},
 	}, {
 		input: `foo OFFSET 1h30m`,
 		expected: &VectorSelector{

promql/parser/printer.go

@@ -116,8 +116,10 @@ func (node *MatrixSelector) String() string {
 	// Copy the Vector selector before changing the offset
 	vecSelector := *node.VectorSelector.(*VectorSelector)
 	offset := ""
-	if vecSelector.OriginalOffset != time.Duration(0) {
+	if vecSelector.OriginalOffset > time.Duration(0) {
 		offset = fmt.Sprintf(" offset %s", model.Duration(vecSelector.OriginalOffset))
+	} else if vecSelector.OriginalOffset < time.Duration(0) {
+		offset = fmt.Sprintf(" offset -%s", model.Duration(-vecSelector.OriginalOffset))
 	}
 	at := ""
 	if vecSelector.Timestamp != nil {
@@ -147,8 +149,10 @@ func (node *SubqueryExpr) String() string {
 		step = model.Duration(node.Step).String()
 	}
 	offset := ""
-	if node.OriginalOffset != time.Duration(0) {
+	if node.OriginalOffset > time.Duration(0) {
 		offset = fmt.Sprintf(" offset %s", model.Duration(node.OriginalOffset))
+	} else if node.OriginalOffset < time.Duration(0) {
+		offset = fmt.Sprintf(" offset -%s", model.Duration(-node.OriginalOffset))
 	}
 	at := ""
 	if node.Timestamp != nil {
@@ -187,8 +191,10 @@ func (node *VectorSelector) String() string {
 		labelStrings = append(labelStrings, matcher.String())
 	}
 	offset := ""
-	if node.OriginalOffset != time.Duration(0) {
+	if node.OriginalOffset > time.Duration(0) {
 		offset = fmt.Sprintf(" offset %s", model.Duration(node.OriginalOffset))
+	} else if node.OriginalOffset < time.Duration(0) {
+		offset = fmt.Sprintf(" offset -%s", model.Duration(-node.OriginalOffset))
 	}
 	at := ""
 	if node.Timestamp != nil {

promql/parser/printer_test.go

@@ -77,12 +77,18 @@ func TestExprString(t *testing.T) {
 		{
 			in: `a offset 1m`,
 		},
+		{
+			in: `a offset -7m`,
+		},
 		{
 			in: `a{c="d"}[5m] offset 1m`,
 		},
 		{
 			in: `a[5m] offset 1m`,
 		},
+		{
+			in: `a[12m] offset -3m`,
+		},
 		{
 			in: `a[1h:5m] offset 1m`,
 		},

web/api/v1/api.go

@@ -354,6 +354,8 @@ func (api *API) query(r *http.Request) (result apiFuncResult) {
 	qry, err := api.QueryEngine.NewInstantQuery(api.Queryable, r.FormValue("query"), ts)
 	if err == promql.ErrValidationAtModifierDisabled {
 		err = errors.New("@ modifier is disabled, use --enable-feature=promql-at-modifier to enable it")
+	} else if err == promql.ErrValidationNegativeOffsetDisabled {
+		err = errors.New("negative offset is disabled, use --enable-feature=promql-negative-offset to enable it")
 	}
 	if err != nil {
 		return invalidParamError(err, "query")
@@ -432,6 +434,8 @@ func (api *API) queryRange(r *http.Request) (result apiFuncResult) {
 	qry, err := api.QueryEngine.NewRangeQuery(api.Queryable, r.FormValue("query"), start, end, step)
 	if err == promql.ErrValidationAtModifierDisabled {
 		err = errors.New("@ modifier is disabled, use --enable-feature=promql-at-modifier to enable it")
+	} else if err == promql.ErrValidationNegativeOffsetDisabled {
+		err = errors.New("negative offset is disabled, use --enable-feature=promql-negative-offset to enable it")
 	}
 	if err != nil {
 		return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}