Merge pull request #949 from prometheus/fabxc/pql-rt

Handle parser runtime panics gracefully

promql/parse.go

@@ -20,7 +20,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/prometheus/log"
+
 	clientmodel "github.com/prometheus/client_golang/model"
+
 	"github.com/prometheus/prometheus/storage/metric"
 	"github.com/prometheus/prometheus/util/strutil"
 )
@@ -323,10 +326,16 @@ func (p *parser) recover(errp *error) {
 	e := recover()
 	if e != nil {
 		if _, ok := e.(runtime.Error); ok {
-			panic(e)
-		}
+			// Print the stack trace but do not inhibit the running application.
+			buf := make([]byte, 64<<10)
+			buf = buf[:runtime.Stack(buf, false)]
+
+			log.Errorf("parser panic: %v\n%s", e, buf)
+			*errp = fmt.Errorf("unexpected error")
+		} else {
 			*errp = e.(error)
 		}
+	}
 	return
 }
 

promql/parse_test.go

@@ -1354,3 +1354,30 @@ func TestParseSeries(t *testing.T) {
 		}
 	}
 }
+
+func TestRecoverRuntime(t *testing.T) {
+	var p *parser
+	var err error
+	defer p.recover(&err)
+
+	// Cause a runtime panic.
+	var a []int
+	a[123] = 1
+
+	if err.Error() != "unexpected error" {
+		t.Fatalf("wrong error message: %q, expected %q", err, "unexpected error")
+	}
+}
+
+func TestRecoverError(t *testing.T) {
+	var p *parser
+	var err error
+	defer p.recover(&err)
+
+	e := fmt.Errorf("custom error")
+	panic(e)
+
+	if err.Error() != e.Error() {
+		t.Fatalf("wrong error message: %q, expected %q", err, e)
+	}
+}