prometheus/template/template.go

404 lines
11 KiB
Go
Raw Permalink Normal View History

// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2015-05-28 12:22:08 -07:00
package template
import (
"bytes"
"context"
"errors"
"fmt"
html_template "html/template"
"math"
"net"
"net/url"
"sort"
"strings"
text_template "text/template"
"time"
"github.com/grafana/regexp"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
common_templates "github.com/prometheus/common/helpers/templates"
"github.com/prometheus/prometheus/promql"
2015-05-29 04:30:30 -07:00
"github.com/prometheus/prometheus/util/strutil"
)
var (
templateTextExpansionFailures = prometheus.NewCounter(prometheus.CounterOpts{
Name: "prometheus_template_text_expansion_failures_total",
Help: "The total number of template text expansion failures.",
})
templateTextExpansionTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "prometheus_template_text_expansions_total",
Help: "The total number of template text expansions.",
})
errNaNOrInf = errors.New("value is NaN or Inf")
)
func init() {
prometheus.MustRegister(templateTextExpansionFailures)
prometheus.MustRegister(templateTextExpansionTotal)
}
// A version of vector that's easier to use from templates.
type sample struct {
Labels map[string]string
Value interface{}
}
type queryResult []*sample
type queryResultByLabelSorter struct {
results queryResult
by string
}
func (q queryResultByLabelSorter) Len() int {
return len(q.results)
}
func (q queryResultByLabelSorter) Less(i, j int) bool {
return q.results[i].Labels[q.by] < q.results[j].Labels[q.by]
}
func (q queryResultByLabelSorter) Swap(i, j int) {
q.results[i], q.results[j] = q.results[j], q.results[i]
}
// QueryFunc executes a PromQL query at the given time.
type QueryFunc func(context.Context, string, time.Time) (promql.Vector, error)
func query(ctx context.Context, q string, ts time.Time, queryFn QueryFunc) (queryResult, error) {
vector, err := queryFn(ctx, q, ts)
if err != nil {
return nil, err
}
// promql.Vector is hard to work with in templates, so convert to
// base data types.
// TODO(fabxc): probably not true anymore after type rework.
result := make(queryResult, len(vector))
for n, v := range vector {
s := sample{
promql: Separate `Point` into `FPoint` and `HPoint` In other words: Instead of having a “polymorphous” `Point` that can either contain a float value or a histogram value, use an `FPoint` for floats and an `HPoint` for histograms. This seemingly small change has a _lot_ of repercussions throughout the codebase. The idea here is to avoid the increase in size of `Point` arrays that happened after native histograms had been added. The higher-level data structures (`Sample`, `Series`, etc.) are still “polymorphous”. The same idea could be applied to them, but at each step the trade-offs needed to be evaluated. The idea with this change is to do the minimum necessary to get back to pre-histogram performance for functions that do not touch histograms. Here are comparisons for the `changes` function. The test data doesn't include histograms yet. Ideally, there would be no change in the benchmark result at all. First runtime v2.39 compared to directly prior to this commit: ``` name old time/op new time/op delta RangeQuery/expr=changes(a_one[1d]),steps=1-16 391µs ± 2% 542µs ± 1% +38.58% (p=0.000 n=9+8) RangeQuery/expr=changes(a_one[1d]),steps=10-16 452µs ± 2% 617µs ± 2% +36.48% (p=0.000 n=10+10) RangeQuery/expr=changes(a_one[1d]),steps=100-16 1.12ms ± 1% 1.36ms ± 2% +21.58% (p=0.000 n=8+10) RangeQuery/expr=changes(a_one[1d]),steps=1000-16 7.83ms ± 1% 8.94ms ± 1% +14.21% (p=0.000 n=10+10) RangeQuery/expr=changes(a_ten[1d]),steps=1-16 2.98ms ± 0% 3.30ms ± 1% +10.67% (p=0.000 n=9+10) RangeQuery/expr=changes(a_ten[1d]),steps=10-16 3.66ms ± 1% 4.10ms ± 1% +11.82% (p=0.000 n=10+10) RangeQuery/expr=changes(a_ten[1d]),steps=100-16 10.5ms ± 0% 11.8ms ± 1% +12.50% (p=0.000 n=8+10) RangeQuery/expr=changes(a_ten[1d]),steps=1000-16 77.6ms ± 1% 87.4ms ± 1% +12.63% (p=0.000 n=9+9) RangeQuery/expr=changes(a_hundred[1d]),steps=1-16 30.4ms ± 2% 32.8ms ± 1% +8.01% (p=0.000 n=10+10) RangeQuery/expr=changes(a_hundred[1d]),steps=10-16 37.1ms ± 2% 40.6ms ± 2% +9.64% (p=0.000 n=10+10) RangeQuery/expr=changes(a_hundred[1d]),steps=100-16 105ms ± 1% 117ms ± 1% +11.69% (p=0.000 n=10+10) RangeQuery/expr=changes(a_hundred[1d]),steps=1000-16 783ms ± 3% 876ms ± 1% +11.83% (p=0.000 n=9+10) ``` And then runtime v2.39 compared to after this commit: ``` name old time/op new time/op delta RangeQuery/expr=changes(a_one[1d]),steps=1-16 391µs ± 2% 547µs ± 1% +39.84% (p=0.000 n=9+8) RangeQuery/expr=changes(a_one[1d]),steps=10-16 452µs ± 2% 616µs ± 2% +36.15% (p=0.000 n=10+10) RangeQuery/expr=changes(a_one[1d]),steps=100-16 1.12ms ± 1% 1.26ms ± 1% +12.20% (p=0.000 n=8+10) RangeQuery/expr=changes(a_one[1d]),steps=1000-16 7.83ms ± 1% 7.95ms ± 1% +1.59% (p=0.000 n=10+8) RangeQuery/expr=changes(a_ten[1d]),steps=1-16 2.98ms ± 0% 3.38ms ± 2% +13.49% (p=0.000 n=9+10) RangeQuery/expr=changes(a_ten[1d]),steps=10-16 3.66ms ± 1% 4.02ms ± 1% +9.80% (p=0.000 n=10+9) RangeQuery/expr=changes(a_ten[1d]),steps=100-16 10.5ms ± 0% 10.8ms ± 1% +3.08% (p=0.000 n=8+10) RangeQuery/expr=changes(a_ten[1d]),steps=1000-16 77.6ms ± 1% 78.1ms ± 1% +0.58% (p=0.035 n=9+10) RangeQuery/expr=changes(a_hundred[1d]),steps=1-16 30.4ms ± 2% 33.5ms ± 4% +10.18% (p=0.000 n=10+10) RangeQuery/expr=changes(a_hundred[1d]),steps=10-16 37.1ms ± 2% 40.0ms ± 1% +7.98% (p=0.000 n=10+10) RangeQuery/expr=changes(a_hundred[1d]),steps=100-16 105ms ± 1% 107ms ± 1% +1.92% (p=0.000 n=10+10) RangeQuery/expr=changes(a_hundred[1d]),steps=1000-16 783ms ± 3% 775ms ± 1% -1.02% (p=0.019 n=9+9) ``` In summary, the runtime doesn't really improve with this change for queries with just a few steps. For queries with many steps, this commit essentially reinstates the old performance. This is good because the many-step queries are the one that matter most (longest absolute runtime). In terms of allocations, though, this commit doesn't make a dent at all (numbers not shown). The reason is that most of the allocations happen in the sampleRingIterator (in the storage package), which has to be addressed in a separate commit. Signed-off-by: beorn7 <beorn@grafana.com>
2022-10-28 07:58:40 -07:00
Value: v.F,
Labels: v.Metric.Map(),
}
if v.H != nil {
s.Value = v.H
}
result[n] = &s
}
return result, nil
}
// Expander executes templates in text or HTML mode with a common set of Prometheus template functions.
type Expander struct {
text string
name string
data interface{}
funcMap text_template.FuncMap
options []string
}
// NewTemplateExpander returns a template expander ready to use.
func NewTemplateExpander(
ctx context.Context,
text string,
name string,
data interface{},
timestamp model.Time,
queryFunc QueryFunc,
externalURL *url.URL,
options []string,
) *Expander {
if options == nil {
options = []string{"missingkey=zero"}
}
return &Expander{
text: text,
name: name,
data: data,
funcMap: text_template.FuncMap{
"query": func(q string) (queryResult, error) {
return query(ctx, q, timestamp.Time(), queryFunc)
},
"first": func(v queryResult) (*sample, error) {
if len(v) > 0 {
return v[0], nil
}
return nil, errors.New("first() called on vector with no elements")
},
"label": func(label string, s *sample) string {
return s.Labels[label]
},
"value": func(s *sample) interface{} {
return s.Value
},
"strvalue": func(s *sample) string {
return s.Labels["__value__"]
},
"args": func(args ...interface{}) map[string]interface{} {
result := make(map[string]interface{})
for i, a := range args {
result[fmt.Sprintf("arg%d", i)] = a
}
return result
},
"reReplaceAll": func(pattern, repl, text string) string {
re := regexp.MustCompile(pattern)
return re.ReplaceAllString(text, repl)
},
"safeHtml": func(text string) html_template.HTML {
return html_template.HTML(text)
},
"match": regexp.MatchString,
lint: Revamp our linting rules, mostly around doc comments Several things done here: - Set `max-issues-per-linter` to 0 so that we actually see all linter warnings and not just 50 per linter. (As we also set `max-same-issues` to 0, I assume this was the intention from the beginning.) - Stop using the golangci-lint default excludes (by setting `exclude-use-default: false`. Those are too generous and don't match our style conventions. (I have re-added some of the excludes explicitly in this commit. See below.) - Re-add the `errcheck` exclusion we have used so far via the defaults. - Exclude the signature requirement `govet` has for `Seek` methods because we use non-standard `Seek` methods a lot. (But we keep other requirements, while the default excludes completely disabled the check for common method segnatures.) - Exclude warnings about missing doc comments on exported symbols. (We used to be pretty adamant about doc comments, but stopped that at some point in the past. By now, we have about 500 missing doc comments. We may consider reintroducing this check, but that's outside of the scope of this commit. The default excludes of golangci-lint essentially ignore doc comments completely.) - By stop using the default excludes, we now get warnings back on malformed doc comments. That's the most impactful change in this commit. It does not enforce doc comments (again), but _if_ there is a doc comment, it has to have the recommended form. (Most of the changes in this commit are fixing this form.) - Improve wording/spelling of some comments in .golangci.yml, and remove an outdated comment. - Leave `package-comments` inactive, but add a TODO asking if we should change that. - Add a new sub-linter `comment-spacings` (and fix corresponding comments), which avoids missing spaces after the leading `//`. Signed-off-by: beorn7 <beorn@grafana.com>
2024-08-22 04:59:36 -07:00
"title": strings.Title, //nolint:staticcheck // TODO(beorn7): Need to come up with a replacement using the cases package.
"toUpper": strings.ToUpper,
"toLower": strings.ToLower,
"graphLink": strutil.GraphLinkForExpression,
"tableLink": strutil.TableLinkForExpression,
"sortByLabel": func(label string, v queryResult) queryResult {
sorter := queryResultByLabelSorter{v[:], label}
sort.Stable(sorter)
return v
},
"stripPort": func(hostPort string) string {
host, _, err := net.SplitHostPort(hostPort)
if err != nil {
return hostPort
}
return host
},
"stripDomain": func(hostPort string) string {
host, port, err := net.SplitHostPort(hostPort)
if err != nil {
host = hostPort
}
ip := net.ParseIP(host)
if ip != nil {
return hostPort
}
host = strings.Split(host, ".")[0]
if port != "" {
return net.JoinHostPort(host, port)
}
return host
},
"humanize": func(i interface{}) (string, error) {
v, err := common_templates.ConvertToFloat(i)
if err != nil {
return "", err
}
if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
return fmt.Sprintf("%.4g", v), nil
}
if math.Abs(v) >= 1 {
prefix := ""
for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
if math.Abs(v) < 1000 {
break
}
prefix = p
v /= 1000
}
return fmt.Sprintf("%.4g%s", v, prefix), nil
}
prefix := ""
for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
if math.Abs(v) >= 1 {
break
}
prefix = p
v *= 1000
}
return fmt.Sprintf("%.4g%s", v, prefix), nil
},
"humanize1024": func(i interface{}) (string, error) {
v, err := common_templates.ConvertToFloat(i)
if err != nil {
return "", err
}
if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
return fmt.Sprintf("%.4g", v), nil
}
prefix := ""
for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
if math.Abs(v) < 1024 {
break
}
prefix = p
v /= 1024
}
return fmt.Sprintf("%.4g%s", v, prefix), nil
},
"humanizeDuration": common_templates.HumanizeDuration,
"humanizePercentage": func(i interface{}) (string, error) {
v, err := common_templates.ConvertToFloat(i)
if err != nil {
return "", err
}
return fmt.Sprintf("%.4g%%", v*100), nil
},
"humanizeTimestamp": common_templates.HumanizeTimestamp,
"toTime": func(i interface{}) (*time.Time, error) {
v, err := common_templates.ConvertToFloat(i)
if err != nil {
return nil, err
}
return floatToTime(v)
},
"pathPrefix": func() string {
return externalURL.Path
},
"externalURL": func() string {
return externalURL.String()
},
"parseDuration": func(d string) (float64, error) {
v, err := model.ParseDuration(d)
if err != nil {
return 0, err
}
return float64(time.Duration(v)) / float64(time.Second), nil
},
},
options: options,
}
}
// AlertTemplateData returns the interface to be used in expanding the template.
func AlertTemplateData(labels, externalLabels map[string]string, externalURL string, smpl promql.Sample) interface{} {
res := struct {
Labels map[string]string
ExternalLabels map[string]string
ExternalURL string
Value interface{}
}{
Labels: labels,
ExternalLabels: externalLabels,
ExternalURL: externalURL,
Value: smpl.F,
}
if smpl.H != nil {
res.Value = smpl.H
}
return res
}
// Funcs adds the functions in fm to the Expander's function map.
// Existing functions will be overwritten in case of conflict.
func (te Expander) Funcs(fm text_template.FuncMap) {
for k, v := range fm {
te.funcMap[k] = v
}
}
// Expand expands a template in text (non-HTML) mode.
func (te Expander) Expand() (result string, resultErr error) {
// It'd better to have no alert description than to kill the whole process
// if there's a bug in the template.
defer func() {
if r := recover(); r != nil {
var ok bool
resultErr, ok = r.(error)
if !ok {
resultErr = fmt.Errorf("panic expanding template %v: %v", te.name, r)
}
}
if resultErr != nil {
templateTextExpansionFailures.Inc()
}
}()
templateTextExpansionTotal.Inc()
tmpl := text_template.New(te.name).Funcs(te.funcMap)
tmpl.Option(te.options...)
tmpl, err := tmpl.Parse(te.text)
if err != nil {
return "", fmt.Errorf("error parsing template %v: %w", te.name, err)
}
var buffer bytes.Buffer
err = tmpl.Execute(&buffer, te.data)
if err != nil {
return "", fmt.Errorf("error executing template %v: %w", te.name, err)
}
return buffer.String(), nil
}
// ExpandHTML expands a template with HTML escaping, with templates read from the given files.
func (te Expander) ExpandHTML(templateFiles []string) (result string, resultErr error) {
defer func() {
if r := recover(); r != nil {
var ok bool
resultErr, ok = r.(error)
if !ok {
resultErr = fmt.Errorf("panic expanding template %s: %v", te.name, r)
}
}
}()
//nolint:unconvert // Before Go 1.19 conversion from text_template to html_template is mandatory
tmpl := html_template.New(te.name).Funcs(html_template.FuncMap(te.funcMap))
tmpl.Option(te.options...)
tmpl.Funcs(html_template.FuncMap{
"tmpl": func(name string, data interface{}) (html_template.HTML, error) {
var buffer bytes.Buffer
err := tmpl.ExecuteTemplate(&buffer, name, data)
return html_template.HTML(buffer.String()), err
},
})
tmpl, err := tmpl.Parse(te.text)
if err != nil {
return "", fmt.Errorf("error parsing template %v: %w", te.name, err)
}
if len(templateFiles) > 0 {
_, err = tmpl.ParseFiles(templateFiles...)
if err != nil {
return "", fmt.Errorf("error parsing template files for %v: %w", te.name, err)
}
}
var buffer bytes.Buffer
err = tmpl.Execute(&buffer, te.data)
if err != nil {
return "", fmt.Errorf("error executing template %v: %w", te.name, err)
}
return buffer.String(), nil
}
// ParseTest parses the templates and returns the error if any.
func (te Expander) ParseTest() error {
_, err := text_template.New(te.name).Funcs(te.funcMap).Option("missingkey=zero").Parse(te.text)
if err != nil {
return err
}
return nil
}
func floatToTime(v float64) (*time.Time, error) {
if math.IsNaN(v) || math.IsInf(v, 0) {
return nil, errNaNOrInf
}
timestamp := v * 1e9
if timestamp > math.MaxInt64 || timestamp < math.MinInt64 {
return nil, fmt.Errorf("%v cannot be represented as a nanoseconds timestamp since it overflows int64", v)
}
t := model.TimeFromUnixNano(int64(timestamp)).Time().UTC()
return &t, nil
}