Support extended durations in promtool unit tests (Fixes #6285) (#6297)

* Fixed evaluation_time duration parsing in promtool unit tests (Fixes #6285)

Signed-off-by: Jordan Neufeld <jordan@neufeldtech.com>

cmd/promtool/testdata/alerts.yml

@@ -0,0 +1,13 @@
+# This is the rules file.
+
+groups:
+  - name: example
+    rules:
+      - alert: InstanceDown
+        expr: up == 0
+        for: 5m
+        labels:
+          severity: page
+        annotations:
+          summary: "Instance {{ $labels.instance }} down"
+          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."

cmd/promtool/testdata/unittest.yml

@@ -0,0 +1,21 @@
+rule_files:
+  - alerts.yml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: 'up{job="prometheus", instance="localhost:9090"}'
+        values: "0+0x1440"
+    alert_rule_test:
+      - eval_time: 1d
+        alertname: InstanceDown
+        exp_alerts:
+          - exp_labels:
+              severity: page
+              instance: localhost:9090
+              job: prometheus
+            exp_annotations:
+              summary: "Instance localhost:9090 down"
+              description: "localhost:9090 of job prometheus has been down for more than 5 minutes."

cmd/promtool/unittest.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v2"
 
+	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
@@ -76,15 +77,16 @@ func ruleUnitTest(filename string) []error {
 	}
 
 	if unitTestInp.EvaluationInterval == 0 {
-		unitTestInp.EvaluationInterval = 1 * time.Minute
+		unitTestInp.EvaluationInterval = model.Duration(1 * time.Minute)
 	}
 
 	// Bounds for evaluating the rules.
 	mint := time.Unix(0, 0).UTC()
 	maxd := unitTestInp.maxEvalTime()
 	maxt := mint.Add(maxd)
+	evalInterval := time.Duration(unitTestInp.EvaluationInterval)
 	// Rounding off to nearest Eval time (> maxt).
-	maxt = maxt.Add(unitTestInp.EvaluationInterval / 2).Round(unitTestInp.EvaluationInterval)
+	maxt = maxt.Add(evalInterval / 2).Round(evalInterval)
 
 	// Giving number for groups mentioned in the file for ordering.
 	// Lower number group should be evaluated before higher number group.
@@ -99,7 +101,7 @@ func ruleUnitTest(filename string) []error {
 	// Testing.
 	var errs []error
 	for _, t := range unitTestInp.Tests {
-		ers := t.test(mint, maxt, unitTestInp.EvaluationInterval, groupOrderMap,
+		ers := t.test(mint, maxt, evalInterval, groupOrderMap,
 			unitTestInp.RuleFiles...)
 		if ers != nil {
 			errs = append(errs, ers...)
@@ -114,10 +116,10 @@ func ruleUnitTest(filename string) []error {
 
 // unitTestFile holds the contents of a single unit test file.
 type unitTestFile struct {
-	RuleFiles          []string      `yaml:"rule_files"`
+	RuleFiles          []string       `yaml:"rule_files"`
-	EvaluationInterval time.Duration `yaml:"evaluation_interval,omitempty"`
+	EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"`
-	GroupEvalOrder     []string      `yaml:"group_eval_order"`
+	GroupEvalOrder     []string       `yaml:"group_eval_order"`
-	Tests              []testGroup   `yaml:"tests"`
+	Tests              []testGroup    `yaml:"tests"`
 }
 
 func (utf *unitTestFile) maxEvalTime() time.Duration {
@@ -157,7 +159,7 @@ func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {
 
 // testGroup is a group of input series and tests associated with it.
 type testGroup struct {
-	Interval        time.Duration    `yaml:"interval"`
+	Interval        model.Duration   `yaml:"interval"`
 	InputSeries     []series         `yaml:"input_series"`
 	AlertRuleTests  []alertTestCase  `yaml:"alert_rule_test,omitempty"`
 	PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"`
@@ -182,7 +184,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
 		Logger:     log.NewNopLogger(),
 	}
 	m := rules.NewManager(opts)
-	groupsMap, ers := m.LoadGroups(tg.Interval, tg.ExternalLabels, ruleFiles...)
+	groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, ruleFiles...)
 	if ers != nil {
 		return ers
 	}
@@ -193,11 +195,11 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
 	// This avoids storing them in memory, as the number of evals might be high.
 
 	// All the `eval_time` for which we have unit tests for alerts.
-	alertEvalTimesMap := map[time.Duration]struct{}{}
+	alertEvalTimesMap := map[model.Duration]struct{}{}
 	// Map of all the eval_time+alertname combination present in the unit tests.
-	alertsInTest := make(map[time.Duration]map[string]struct{})
+	alertsInTest := make(map[model.Duration]map[string]struct{})
 	// Map of all the unit tests for given eval_time.
-	alertTests := make(map[time.Duration][]alertTestCase)
+	alertTests := make(map[model.Duration][]alertTestCase)
 	for _, alert := range tg.AlertRuleTests {
 		alertEvalTimesMap[alert.EvalTime] = struct{}{}
 
@@ -208,7 +210,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
 
 		alertTests[alert.EvalTime] = append(alertTests[alert.EvalTime], alert)
 	}
-	alertEvalTimes := make([]time.Duration, 0, len(alertEvalTimesMap))
+	alertEvalTimes := make([]model.Duration, 0, len(alertEvalTimesMap))
 	for k := range alertEvalTimesMap {
 		alertEvalTimes = append(alertEvalTimes, k)
 	}
@@ -242,8 +244,8 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
 		}
 
 		for {
-			if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= alertEvalTimes[curr] &&
+			if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= time.Duration(alertEvalTimes[curr]) &&
-				alertEvalTimes[curr] < ts.Add(evalInterval).Sub(mint)) {
+				time.Duration(alertEvalTimes[curr]) < ts.Add(time.Duration(evalInterval)).Sub(mint)) {
 				break
 			}
 
@@ -322,7 +324,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
 	// Checking promql expressions.
 Outer:
 	for _, testCase := range tg.PromqlExprTests {
-		got, err := query(suite.Context(), testCase.Expr, mint.Add(testCase.EvalTime),
+		got, err := query(suite.Context(), testCase.Expr, mint.Add(time.Duration(testCase.EvalTime)),
 			suite.QueryEngine(), suite.Queryable())
 		if err != nil {
 			errs = append(errs, errors.Errorf("    expr: %q, time: %s, err: %s", testCase.Expr,
@@ -373,15 +375,15 @@ Outer:
 
 // seriesLoadingString returns the input series in PromQL notation.
 func (tg *testGroup) seriesLoadingString() string {
-	result := ""
+
-	result += "load " + shortDuration(tg.Interval) + "\n"
+	result := fmt.Sprintf("load %v\n", shortDuration(tg.Interval))
 	for _, is := range tg.InputSeries {
-		result += "  " + is.Series + " " + is.Values + "\n"
+		result += fmt.Sprintf("  %v %v\n", is.Series, is.Values)
 	}
 	return result
 }
 
-func shortDuration(d time.Duration) string {
+func shortDuration(d model.Duration) string {
 	s := d.String()
 	if strings.HasSuffix(s, "m0s") {
 		s = s[:len(s)-2]
@@ -407,7 +409,7 @@ func orderedGroups(groupsMap map[string]*rules.Group, groupOrderMap map[string]i
 
 // maxEvalTime returns the max eval time among all alert and promql unit tests.
 func (tg *testGroup) maxEvalTime() time.Duration {
-	var maxd time.Duration
+	var maxd model.Duration
 	for _, alert := range tg.AlertRuleTests {
 		if alert.EvalTime > maxd {
 			maxd = alert.EvalTime
@@ -418,7 +420,7 @@ func (tg *testGroup) maxEvalTime() time.Duration {
 			maxd = pet.EvalTime
 		}
 	}
-	return maxd
+	return time.Duration(maxd)
 }
 
 func query(ctx context.Context, qs string, t time.Time, engine *promql.Engine, qu storage.Queryable) (promql.Vector, error) {
@@ -483,9 +485,9 @@ type series struct {
 }
 
 type alertTestCase struct {
-	EvalTime  time.Duration `yaml:"eval_time"`
+	EvalTime  model.Duration `yaml:"eval_time"`
-	Alertname string        `yaml:"alertname"`
+	Alertname string         `yaml:"alertname"`
-	ExpAlerts []alert       `yaml:"exp_alerts"`
+	ExpAlerts []alert        `yaml:"exp_alerts"`
 }
 
 type alert struct {
@@ -494,9 +496,9 @@ type alert struct {
 }
 
 type promqlTestCase struct {
-	Expr       string        `yaml:"expr"`
+	Expr       string         `yaml:"expr"`
-	EvalTime   time.Duration `yaml:"eval_time"`
+	EvalTime   model.Duration `yaml:"eval_time"`
-	ExpSamples []sample      `yaml:"exp_samples"`
+	ExpSamples []sample       `yaml:"exp_samples"`
 }
 
 type sample struct {

cmd/promtool/unittest_test.go

@@ -0,0 +1,42 @@
+// Copyright 2018 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import "testing"
+
+func TestRulesUnitTest(t *testing.T) {
+	type args struct {
+		files []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want int
+	}{
+		{
+			name: "Passing Unit Tests",
+			args: args{
+				files: []string{"./testdata/unittest.yml"},
+			},
+			want: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := RulesUnitTest(tt.args.files...); got != tt.want {
+				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}