Merge pull request #8440 from mishamo/master

Add optional name property to testgroup for better test failure output

cmd/promtool/testdata/failing.yml

@@ -1,6 +1,7 @@
 tests:
   # Simple failing test.
   - interval: 1m
+    name: "Failing test"
     input_series:
       - series: test
         values: '0'

cmd/promtool/unittest.go

@@ -146,6 +146,7 @@ type testGroup struct {
 	AlertRuleTests  []alertTestCase  `yaml:"alert_rule_test,omitempty"`
 	PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"`
 	ExternalLabels  labels.Labels    `yaml:"external_labels,omitempty"`
+	TestGroupName   string           `yaml:"name,omitempty"`
 }
 
 // test performs the unit tests.
@@ -299,16 +300,29 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 					})
 				}
 
+				var sb strings.Builder
 				if gotAlerts.Len() != expAlerts.Len() {
-					errs = append(errs, errors.Errorf("    alertname:%s, time:%s, \n        exp:%#v, \n        got:%#v",
-						testcase.Alertname, testcase.EvalTime.String(), expAlerts.String(), gotAlerts.String()))
+					if tg.TestGroupName != "" {
+						fmt.Fprintf(&sb, "    name: %s,\n", tg.TestGroupName)
+					}
+					fmt.Fprintf(&sb, "    alertname:%s, time:%s, \n", testcase.Alertname, testcase.EvalTime.String())
+					fmt.Fprintf(&sb, "        exp:%#v, \n", expAlerts.String())
+					fmt.Fprintf(&sb, "        got:%#v", gotAlerts.String())
+
+					errs = append(errs, errors.New(sb.String()))
 				} else {
 					sort.Sort(gotAlerts)
 					sort.Sort(expAlerts)
 
 					if !reflect.DeepEqual(expAlerts, gotAlerts) {
-						errs = append(errs, errors.Errorf("    alertname:%s, time:%s, \n        exp:%#v, \n        got:%#v",
-							testcase.Alertname, testcase.EvalTime.String(), expAlerts.String(), gotAlerts.String()))
+						if tg.TestGroupName != "" {
+							fmt.Fprintf(&sb, "    name: %s,\n", tg.TestGroupName)
+						}
+						fmt.Fprintf(&sb, "    alertname:%s, time:%s, \n", testcase.Alertname, testcase.EvalTime.String())
+						fmt.Fprintf(&sb, "        exp:%#v, \n", expAlerts.String())
+						fmt.Fprintf(&sb, "        got:%#v", gotAlerts.String())
+
+						errs = append(errs, errors.New(sb.String()))
 					}
 				}
 			}

docs/configuration/unit_testing_rules.md

@@ -44,6 +44,9 @@ interval: <duration>
 input_series:
   [ - <series> ]
 
+# Name of the test group
+[ name: <string> ]
+
 # Unit tests for the above data.
 
 # Unit tests for alerting rules. We consider the alerting rules from the input file.