promtool: JUnit-Format XML Test Results (#14506)

* Junit compatible output

Signed-off-by: Kushal Shukla <kushalshukla110@gmail.com>

cmd/promtool/main.go

@@ -204,6 +204,7 @@ func main() {
 	pushMetricsHeaders := pushMetricsCmd.Flag("header", "Prometheus remote write header.").StringMap()
 
 	testCmd := app.Command("test", "Unit testing.")
+	junitOutFile := testCmd.Flag("junit", "File path to store JUnit XML test results.").OpenFile(os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
 	testRulesCmd := testCmd.Command("rules", "Unit tests for rules.")
 	testRulesRun := testRulesCmd.Flag("run", "If set, will only run test groups whose names match the regular expression. Can be specified multiple times.").Strings()
 	testRulesFiles := testRulesCmd.Arg(
@@ -378,7 +379,11 @@ func main() {
 		os.Exit(QueryLabels(serverURL, httpRoundTripper, *queryLabelsMatch, *queryLabelsName, *queryLabelsBegin, *queryLabelsEnd, p))
 
 	case testRulesCmd.FullCommand():
-		os.Exit(RulesUnitTest(
+		results := io.Discard
+		if *junitOutFile != nil {
+			results = *junitOutFile
+		}
+		os.Exit(RulesUnitTestResult(results,
 			promqltest.LazyLoaderOpts{
 				EnableAtModifier:     true,
 				EnableNegativeOffset: true,

cmd/promtool/unittest.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"sort"
@@ -29,9 +30,10 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/grafana/regexp"
 	"github.com/nsf/jsondiff"
-	"github.com/prometheus/common/model"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/common/model"
+
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql"
@@ -39,12 +41,18 @@ import (
 	"github.com/prometheus/prometheus/promql/promqltest"
 	"github.com/prometheus/prometheus/rules"
 	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/util/junitxml"
 )
 
 // RulesUnitTest does unit testing of rules based on the unit testing files provided.
 // More info about the file format can be found in the docs.
 func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
+	return RulesUnitTestResult(io.Discard, queryOpts, runStrings, diffFlag, files...)
+}
+
+func RulesUnitTestResult(results io.Writer, queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
 	failed := false
+	junit := &junitxml.JUnitXML{}
 
 	var run *regexp.Regexp
 	if runStrings != nil {
@@ -52,7 +60,7 @@ func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, dif
 	}
 
 	for _, f := range files {
-		if errs := ruleUnitTest(f, queryOpts, run, diffFlag); errs != nil {
+		if errs := ruleUnitTest(f, queryOpts, run, diffFlag, junit.Suite(f)); errs != nil {
 			fmt.Fprintln(os.Stderr, "  FAILED:")
 			for _, e := range errs {
 				fmt.Fprintln(os.Stderr, e.Error())
@@ -64,25 +72,30 @@ func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, dif
 		}
 		fmt.Println()
 	}
+	err := junit.WriteXML(results)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to write JUnit XML: %s\n", err)
+	}
 	if failed {
 		return failureExitCode
 	}
 	return successExitCode
 }
 
-func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error {
+func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool, ts *junitxml.TestSuite) []error {
-	fmt.Println("Unit Testing: ", filename)
-
 	b, err := os.ReadFile(filename)
 	if err != nil {
+		ts.Abort(err)
 		return []error{err}
 	}
 
 	var unitTestInp unitTestFile
 	if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil {
+		ts.Abort(err)
 		return []error{err}
 	}
 	if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil {
+		ts.Abort(err)
 		return []error{err}
 	}
 
@@ -91,29 +104,38 @@ func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *reg
 	}
 
 	evalInterval := time.Duration(unitTestInp.EvaluationInterval)
-
+	ts.Settime(time.Now().Format("2006-01-02T15:04:05"))
 	// Giving number for groups mentioned in the file for ordering.
 	// Lower number group should be evaluated before higher number group.
 	groupOrderMap := make(map[string]int)
 	for i, gn := range unitTestInp.GroupEvalOrder {
 		if _, ok := groupOrderMap[gn]; ok {
-			return []error{fmt.Errorf("group name repeated in evaluation order: %s", gn)}
+			err := fmt.Errorf("group name repeated in evaluation order: %s", gn)
+			ts.Abort(err)
+			return []error{err}
 		}
 		groupOrderMap[gn] = i
 	}
 
 	// Testing.
 	var errs []error
-	for _, t := range unitTestInp.Tests {
+	for i, t := range unitTestInp.Tests {
 		if !matchesRun(t.TestGroupName, run) {
 			continue
 		}
-
+		testname := t.TestGroupName
+		if testname == "" {
+			testname = fmt.Sprintf("unnamed#%d", i)
+		}
+		tc := ts.Case(testname)
 		if t.Interval == 0 {
 			t.Interval = unitTestInp.EvaluationInterval
 		}
 		ers := t.test(evalInterval, groupOrderMap, queryOpts, diffFlag, unitTestInp.RuleFiles...)
 		if ers != nil {
+			for _, e := range ers {
+				tc.Fail(e.Error())
+			}
 			errs = append(errs, ers...)
 		}
 	}

cmd/promtool/unittest_test.go

@@ -14,11 +14,15 @@
 package main
 
 import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/promql/promqltest"
+	"github.com/prometheus/prometheus/util/junitxml"
 )
 
 func TestRulesUnitTest(t *testing.T) {
@@ -125,13 +129,59 @@ func TestRulesUnitTest(t *testing.T) {
 			want: 0,
 		},
 	}
+	reuseFiles := []string{}
+	reuseCount := [2]int{}
 	for _, tt := range tests {
+		if (tt.queryOpts == promqltest.LazyLoaderOpts{
+			EnableNegativeOffset: true,
+		} || tt.queryOpts == promqltest.LazyLoaderOpts{
+			EnableAtModifier: true,
+		}) {
+			reuseFiles = append(reuseFiles, tt.args.files...)
+			reuseCount[tt.want] += len(tt.args.files)
+		}
 		t.Run(tt.name, func(t *testing.T) {
 			if got := RulesUnitTest(tt.queryOpts, nil, false, tt.args.files...); got != tt.want {
 				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
 			}
 		})
 	}
+	t.Run("Junit xml output ", func(t *testing.T) {
+		var buf bytes.Buffer
+		if got := RulesUnitTestResult(&buf, promqltest.LazyLoaderOpts{}, nil, false, reuseFiles...); got != 1 {
+			t.Errorf("RulesUnitTestResults() = %v, want 1", got)
+		}
+		var test junitxml.JUnitXML
+		output := buf.Bytes()
+		err := xml.Unmarshal(output, &test)
+		if err != nil {
+			fmt.Println("error in decoding XML:", err)
+			return
+		}
+		var total int
+		var passes int
+		var failures int
+		var cases int
+		total = len(test.Suites)
+		if total != len(reuseFiles) {
+			t.Errorf("JUnit output had %d testsuite elements; expected %d\n", total, len(reuseFiles))
+		}
+
+		for _, i := range test.Suites {
+			if i.FailureCount == 0 {
+				passes++
+			} else {
+				failures++
+			}
+			cases += len(i.Cases)
+		}
+		if total != passes+failures {
+			t.Errorf("JUnit output mismatch: Total testsuites (%d) does not equal the sum of passes (%d) and failures (%d).", total, passes, failures)
+		}
+		if cases < total {
+			t.Errorf("JUnit output had %d suites without test cases\n", total-cases)
+		}
+	})
 }
 
 func TestRulesUnitTestRun(t *testing.T) {

docs/command-line/promtool.md

@@ -442,6 +442,15 @@ Unit testing.
 
 
 
+#### Flags
+
+| Flag | Description |
+| --- | --- |
+| <code class="text-nowrap">--junit</code> | File path to store JUnit XML test results. |
+
+
+
+
 ##### `promtool test rules`
 
 Unit tests for rules.

util/junitxml/junitxml.go

@@ -0,0 +1,81 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package junitxml
+
+import (
+	"encoding/xml"
+	"io"
+)
+
+type JUnitXML struct {
+	XMLName xml.Name     `xml:"testsuites"`
+	Suites  []*TestSuite `xml:"testsuite"`
+}
+
+type TestSuite struct {
+	Name         string      `xml:"name,attr"`
+	TestCount    int         `xml:"tests,attr"`
+	FailureCount int         `xml:"failures,attr"`
+	ErrorCount   int         `xml:"errors,attr"`
+	SkippedCount int         `xml:"skipped,attr"`
+	Timestamp    string      `xml:"timestamp,attr"`
+	Cases        []*TestCase `xml:"testcase"`
+}
+type TestCase struct {
+	Name     string   `xml:"name,attr"`
+	Failures []string `xml:"failure,omitempty"`
+	Error    string   `xml:"error,omitempty"`
+}
+
+func (j *JUnitXML) WriteXML(h io.Writer) error {
+	return xml.NewEncoder(h).Encode(j)
+}
+
+func (j *JUnitXML) Suite(name string) *TestSuite {
+	ts := &TestSuite{Name: name}
+	j.Suites = append(j.Suites, ts)
+	return ts
+}
+
+func (ts *TestSuite) Fail(f string) {
+	ts.FailureCount++
+	curt := ts.lastCase()
+	curt.Failures = append(curt.Failures, f)
+}
+
+func (ts *TestSuite) lastCase() *TestCase {
+	if len(ts.Cases) == 0 {
+		ts.Case("unknown")
+	}
+	return ts.Cases[len(ts.Cases)-1]
+}
+
+func (ts *TestSuite) Case(name string) *TestSuite {
+	j := &TestCase{
+		Name: name,
+	}
+	ts.Cases = append(ts.Cases, j)
+	ts.TestCount++
+	return ts
+}
+
+func (ts *TestSuite) Settime(name string) {
+	ts.Timestamp = name
+}
+
+func (ts *TestSuite) Abort(e error) {
+	ts.ErrorCount++
+	curt := ts.lastCase()
+	curt.Error = e.Error()
+}

util/junitxml/junitxml_test.go

@@ -0,0 +1,66 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package junitxml
+
+import (
+	"bytes"
+	"encoding/xml"
+	"errors"
+	"testing"
+)
+
+func TestJunitOutput(t *testing.T) {
+	var buf bytes.Buffer
+	var test JUnitXML
+	x := FakeTestSuites()
+	if err := x.WriteXML(&buf); err != nil {
+		t.Fatalf("Failed to encode XML: %v", err)
+	}
+
+	output := buf.Bytes()
+
+	err := xml.Unmarshal(output, &test)
+	if err != nil {
+		t.Errorf("Unmarshal failed with error: %v", err)
+	}
+	var total int
+	var cases int
+	total = len(test.Suites)
+	if total != 3 {
+		t.Errorf("JUnit output had %d testsuite elements; expected 3\n", total)
+	}
+	for _, i := range test.Suites {
+		cases += len(i.Cases)
+	}
+
+	if cases != 7 {
+		t.Errorf("JUnit output had %d testcase; expected 7\n", cases)
+	}
+}
+
+func FakeTestSuites() *JUnitXML {
+	ju := &JUnitXML{}
+	good := ju.Suite("all good")
+	good.Case("alpha")
+	good.Case("beta")
+	good.Case("gamma")
+	mixed := ju.Suite("mixed")
+	mixed.Case("good")
+	bad := mixed.Case("bad")
+	bad.Fail("once")
+	bad.Fail("twice")
+	mixed.Case("ugly").Abort(errors.New("buggy"))
+	ju.Suite("fast").Fail("fail early")
+	return ju
+}