prometheus/storage/remote/codec.go

377 lines
9.3 KiB
Go
Raw Normal View History

// Copyright 2017 The Prometheus Authors
2017-10-23 06:44:57 -07:00
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"fmt"
"io"
2017-10-23 06:44:57 -07:00
"io/ioutil"
"net/http"
"sort"
2017-10-23 06:44:57 -07:00
"github.com/gogo/protobuf/proto"
"github.com/golang/snappy"
"github.com/pkg/errors"
2017-10-23 06:44:57 -07:00
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/prompb"
2017-10-23 13:28:17 -07:00
"github.com/prometheus/prometheus/storage"
2017-10-23 06:44:57 -07:00
)
// decodeReadLimit is the maximum size of a read request body in bytes.
const decodeReadLimit = 32 * 1024 * 1024
type HTTPError struct {
msg string
status int
}
func (e HTTPError) Error() string {
return e.msg
}
func (e HTTPError) Status() int {
return e.status
}
2017-10-23 06:44:57 -07:00
// DecodeReadRequest reads a remote.Request from a http.Request.
func DecodeReadRequest(r *http.Request) (*prompb.ReadRequest, error) {
compressed, err := ioutil.ReadAll(io.LimitReader(r.Body, decodeReadLimit))
2017-10-23 06:44:57 -07:00
if err != nil {
return nil, err
}
reqBuf, err := snappy.Decode(nil, compressed)
if err != nil {
return nil, err
}
var req prompb.ReadRequest
if err := proto.Unmarshal(reqBuf, &req); err != nil {
return nil, err
}
return &req, nil
}
// EncodeReadResponse writes a remote.Response to a http.ResponseWriter.
func EncodeReadResponse(resp *prompb.ReadResponse, w http.ResponseWriter) error {
data, err := proto.Marshal(resp)
if err != nil {
return err
}
w.Header().Set("Content-Type", "application/x-protobuf")
w.Header().Set("Content-Encoding", "snappy")
compressed := snappy.Encode(nil, data)
_, err = w.Write(compressed)
return err
}
// ToQuery builds a Query proto.
func ToQuery(from, to int64, matchers []*labels.Matcher, p *storage.SelectParams) (*prompb.Query, error) {
2017-10-23 06:44:57 -07:00
ms, err := toLabelMatchers(matchers)
if err != nil {
return nil, err
}
var rp *prompb.ReadHints
if p != nil {
rp = &prompb.ReadHints{
StepMs: p.Step,
Func: p.Func,
StartMs: p.Start,
EndMs: p.End,
}
}
2017-10-23 06:44:57 -07:00
return &prompb.Query{
StartTimestampMs: from,
EndTimestampMs: to,
Matchers: ms,
Hints: rp,
2017-10-23 06:44:57 -07:00
}, nil
}
// FromQuery unpacks a Query proto.
func FromQuery(req *prompb.Query) (int64, int64, []*labels.Matcher, *storage.SelectParams, error) {
2017-10-23 06:44:57 -07:00
matchers, err := fromLabelMatchers(req.Matchers)
if err != nil {
return 0, 0, nil, nil, err
2017-10-23 06:44:57 -07:00
}
var selectParams *storage.SelectParams
if req.Hints != nil {
selectParams = &storage.SelectParams{
Start: req.Hints.StartMs,
End: req.Hints.EndMs,
Step: req.Hints.StepMs,
Func: req.Hints.Func,
}
}
return req.StartTimestampMs, req.EndTimestampMs, matchers, selectParams, nil
2017-10-23 06:44:57 -07:00
}
// ToQueryResult builds a QueryResult proto.
func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, error) {
numSamples := 0
2017-10-23 06:44:57 -07:00
resp := &prompb.QueryResult{}
2017-10-23 13:28:17 -07:00
for ss.Next() {
series := ss.At()
iter := series.Iterator()
samples := []prompb.Sample{}
2017-10-23 13:28:17 -07:00
for iter.Next() {
numSamples++
if sampleLimit > 0 && numSamples > sampleLimit {
return nil, HTTPError{
msg: fmt.Sprintf("exceeded sample limit (%d)", sampleLimit),
status: http.StatusBadRequest,
}
}
2017-10-23 13:28:17 -07:00
ts, val := iter.At()
samples = append(samples, prompb.Sample{
2017-10-23 13:28:17 -07:00
Timestamp: ts,
Value: val,
2017-10-23 06:44:57 -07:00
})
}
2017-10-23 13:28:17 -07:00
if err := iter.Err(); err != nil {
return nil, err
}
resp.Timeseries = append(resp.Timeseries, &prompb.TimeSeries{
Labels: labelsToLabelsProto(series.Labels()),
Samples: samples,
})
}
if err := ss.Err(); err != nil {
return nil, err
2017-10-23 06:44:57 -07:00
}
2017-10-23 13:28:17 -07:00
return resp, nil
2017-10-23 06:44:57 -07:00
}
// FromQueryResult unpacks a QueryResult proto.
2017-10-23 13:28:17 -07:00
func FromQueryResult(res *prompb.QueryResult) storage.SeriesSet {
series := make([]storage.Series, 0, len(res.Timeseries))
for _, ts := range res.Timeseries {
labels := labelProtosToLabels(ts.Labels)
if err := validateLabelsAndMetricName(labels); err != nil {
return errSeriesSet{err: err}
2017-10-23 06:44:57 -07:00
}
2017-10-23 13:28:17 -07:00
series = append(series, &concreteSeries{
labels: labels,
samples: ts.Samples,
})
2017-10-23 06:44:57 -07:00
}
2017-10-23 13:28:17 -07:00
sort.Sort(byLabel(series))
return &concreteSeriesSet{
series: series,
}
}
type byLabel []storage.Series
func (a byLabel) Len() int { return len(a) }
func (a byLabel) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byLabel) Less(i, j int) bool { return labels.Compare(a[i].Labels(), a[j].Labels()) < 0 }
2017-10-23 13:28:17 -07:00
// errSeriesSet implements storage.SeriesSet, just returning an error.
type errSeriesSet struct {
err error
}
2017-10-23 06:44:57 -07:00
2017-10-23 13:28:17 -07:00
func (errSeriesSet) Next() bool {
return false
}
func (errSeriesSet) At() storage.Series {
return nil
}
func (e errSeriesSet) Err() error {
return e.err
}
// concreteSeriesSet implements storage.SeriesSet.
type concreteSeriesSet struct {
cur int
series []storage.Series
}
func (c *concreteSeriesSet) Next() bool {
c.cur++
return c.cur-1 < len(c.series)
}
func (c *concreteSeriesSet) At() storage.Series {
return c.series[c.cur-1]
}
func (c *concreteSeriesSet) Err() error {
return nil
}
2018-04-08 02:51:54 -07:00
// concreteSeries implements storage.Series.
2017-10-23 13:28:17 -07:00
type concreteSeries struct {
labels labels.Labels
samples []prompb.Sample
2017-10-23 13:28:17 -07:00
}
func (c *concreteSeries) Labels() labels.Labels {
return labels.New(c.labels...)
2017-10-23 13:28:17 -07:00
}
func (c *concreteSeries) Iterator() storage.SeriesIterator {
return newConcreteSeriersIterator(c)
}
// concreteSeriesIterator implements storage.SeriesIterator.
type concreteSeriesIterator struct {
cur int
series *concreteSeries
}
func newConcreteSeriersIterator(series *concreteSeries) storage.SeriesIterator {
return &concreteSeriesIterator{
cur: -1,
series: series,
}
}
// Seek implements storage.SeriesIterator.
func (c *concreteSeriesIterator) Seek(t int64) bool {
c.cur = sort.Search(len(c.series.samples), func(n int) bool {
return c.series.samples[n].Timestamp >= t
})
return c.cur < len(c.series.samples)
}
// At implements storage.SeriesIterator.
func (c *concreteSeriesIterator) At() (t int64, v float64) {
s := c.series.samples[c.cur]
return s.Timestamp, s.Value
}
// Next implements storage.SeriesIterator.
func (c *concreteSeriesIterator) Next() bool {
c.cur++
return c.cur < len(c.series.samples)
}
// Err implements storage.SeriesIterator.
func (c *concreteSeriesIterator) Err() error {
return nil
}
// validateLabelsAndMetricName validates the label names/values and metric names returned from remote read.
func validateLabelsAndMetricName(ls labels.Labels) error {
for _, l := range ls {
if l.Name == labels.MetricName && !model.IsValidMetricName(model.LabelValue(l.Value)) {
return errors.Errorf("invalid metric name: %v", l.Value)
2017-10-23 13:28:17 -07:00
}
if !model.LabelName(l.Name).IsValid() {
return errors.Errorf("invalid label name: %v", l.Name)
2017-10-23 13:28:17 -07:00
}
if !model.LabelValue(l.Value).IsValid() {
return errors.Errorf("invalid label value: %v", l.Value)
2017-10-23 13:28:17 -07:00
}
}
return nil
2017-10-23 06:44:57 -07:00
}
func toLabelMatchers(matchers []*labels.Matcher) ([]*prompb.LabelMatcher, error) {
pbMatchers := make([]*prompb.LabelMatcher, 0, len(matchers))
for _, m := range matchers {
var mType prompb.LabelMatcher_Type
switch m.Type {
case labels.MatchEqual:
mType = prompb.LabelMatcher_EQ
case labels.MatchNotEqual:
mType = prompb.LabelMatcher_NEQ
case labels.MatchRegexp:
mType = prompb.LabelMatcher_RE
case labels.MatchNotRegexp:
mType = prompb.LabelMatcher_NRE
default:
return nil, errors.New("invalid matcher type")
2017-10-23 06:44:57 -07:00
}
pbMatchers = append(pbMatchers, &prompb.LabelMatcher{
Type: mType,
Name: m.Name,
Value: m.Value,
})
}
return pbMatchers, nil
}
func fromLabelMatchers(matchers []*prompb.LabelMatcher) ([]*labels.Matcher, error) {
result := make([]*labels.Matcher, 0, len(matchers))
for _, matcher := range matchers {
var mtype labels.MatchType
switch matcher.Type {
case prompb.LabelMatcher_EQ:
mtype = labels.MatchEqual
case prompb.LabelMatcher_NEQ:
mtype = labels.MatchNotEqual
case prompb.LabelMatcher_RE:
mtype = labels.MatchRegexp
case prompb.LabelMatcher_NRE:
mtype = labels.MatchNotRegexp
default:
return nil, errors.New("invalid matcher type")
2017-10-23 06:44:57 -07:00
}
matcher, err := labels.NewMatcher(mtype, matcher.Name, matcher.Value)
if err != nil {
return nil, err
}
result = append(result, matcher)
}
return result, nil
}
// LabelProtosToMetric unpack a []*prompb.Label to a model.Metric
func LabelProtosToMetric(labelPairs []*prompb.Label) model.Metric {
2017-10-23 06:44:57 -07:00
metric := make(model.Metric, len(labelPairs))
for _, l := range labelPairs {
metric[model.LabelName(l.Name)] = model.LabelValue(l.Value)
}
return metric
}
func labelProtosToLabels(labelPairs []prompb.Label) labels.Labels {
result := make(labels.Labels, 0, len(labelPairs))
for _, l := range labelPairs {
result = append(result, labels.Label{
Name: l.Name,
Value: l.Value,
})
}
sort.Sort(result)
return result
}
func labelsToLabelsProto(labels labels.Labels) []prompb.Label {
result := make([]prompb.Label, 0, len(labels))
2017-10-23 13:28:17 -07:00
for _, l := range labels {
result = append(result, prompb.Label{
Name: interner.intern(l.Name),
Value: interner.intern(l.Value),
2017-10-23 13:28:17 -07:00
})
}
return result
}