mirror of
https://github.com/prometheus/prometheus.git
synced 2024-12-24 21:24:05 -08:00
[ENHANCEMENT] remote storage:Add default api implementation of remote write
Signed-off-by: fuling <fuling.lgz@alibaba-inc.com>
This commit is contained in:
parent
ff40af2671
commit
72475b8a0c
|
@ -284,7 +284,7 @@ func main() {
|
||||||
Default("2m").SetValue(&cfg.queryTimeout)
|
Default("2m").SetValue(&cfg.queryTimeout)
|
||||||
|
|
||||||
a.Flag("query.max-concurrency", "Maximum number of queries executed concurrently.").
|
a.Flag("query.max-concurrency", "Maximum number of queries executed concurrently.").
|
||||||
Default("20").IntVar(&cfg.queryConcurrency)
|
Default("2000").IntVar(&cfg.queryConcurrency)
|
||||||
|
|
||||||
a.Flag("query.max-samples", "Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return.").
|
a.Flag("query.max-samples", "Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return.").
|
||||||
Default("50000000").IntVar(&cfg.queryMaxSamples)
|
Default("50000000").IntVar(&cfg.queryMaxSamples)
|
||||||
|
|
|
@ -16,6 +16,7 @@ package v1
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
"math"
|
"math"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"net"
|
"net"
|
||||||
|
@ -27,11 +28,14 @@ import (
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
"github.com/go-kit/kit/log"
|
"github.com/go-kit/kit/log"
|
||||||
"github.com/go-kit/kit/log/level"
|
"github.com/go-kit/kit/log/level"
|
||||||
|
"github.com/gogo/protobuf/proto"
|
||||||
|
"github.com/golang/snappy"
|
||||||
jsoniter "github.com/json-iterator/go"
|
jsoniter "github.com/json-iterator/go"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
@ -71,7 +75,6 @@ const (
|
||||||
type errorType string
|
type errorType string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
errorNone errorType = ""
|
|
||||||
errorTimeout errorType = "timeout"
|
errorTimeout errorType = "timeout"
|
||||||
errorCanceled errorType = "canceled"
|
errorCanceled errorType = "canceled"
|
||||||
errorExec errorType = "execution"
|
errorExec errorType = "execution"
|
||||||
|
@ -172,6 +175,9 @@ type TSDBAdminStats interface {
|
||||||
// them using the provided storage and query engine.
|
// them using the provided storage and query engine.
|
||||||
type API struct {
|
type API struct {
|
||||||
Queryable storage.SampleAndChunkQueryable
|
Queryable storage.SampleAndChunkQueryable
|
||||||
|
Appendable storage.Appendable
|
||||||
|
refs map[string]uint64
|
||||||
|
refsLock *sync.RWMutex
|
||||||
QueryEngine *promql.Engine
|
QueryEngine *promql.Engine
|
||||||
|
|
||||||
targetRetriever func(context.Context) TargetRetriever
|
targetRetriever func(context.Context) TargetRetriever
|
||||||
|
@ -226,7 +232,10 @@ func NewAPI(
|
||||||
) *API {
|
) *API {
|
||||||
return &API{
|
return &API{
|
||||||
QueryEngine: qe,
|
QueryEngine: qe,
|
||||||
|
refs: make(map[string]uint64),
|
||||||
|
refsLock: &sync.RWMutex{},
|
||||||
Queryable: q,
|
Queryable: q,
|
||||||
|
Appendable: q,
|
||||||
targetRetriever: tr,
|
targetRetriever: tr,
|
||||||
alertmanagerRetriever: ar,
|
alertmanagerRetriever: ar,
|
||||||
|
|
||||||
|
@ -309,6 +318,7 @@ func (api *API) Register(r *route.Router) {
|
||||||
r.Get("/status/flags", wrap(api.serveFlags))
|
r.Get("/status/flags", wrap(api.serveFlags))
|
||||||
r.Get("/status/tsdb", wrap(api.serveTSDBStatus))
|
r.Get("/status/tsdb", wrap(api.serveTSDBStatus))
|
||||||
r.Post("/read", api.ready(http.HandlerFunc(api.remoteRead)))
|
r.Post("/read", api.ready(http.HandlerFunc(api.remoteRead)))
|
||||||
|
r.Post("/write", api.ready(http.HandlerFunc(api.remoteWrite)))
|
||||||
|
|
||||||
r.Get("/alerts", wrap(api.alerts))
|
r.Get("/alerts", wrap(api.alerts))
|
||||||
r.Get("/rules", wrap(api.rules))
|
r.Get("/rules", wrap(api.rules))
|
||||||
|
@ -667,7 +677,7 @@ func (api *API) series(r *http.Request) (result apiFuncResult) {
|
||||||
return invalidParamError(err, "match[]")
|
return invalidParamError(err, "match[]")
|
||||||
}
|
}
|
||||||
|
|
||||||
q, err := api.Queryable.Querier(r.Context(), timestamp.FromTime(start), timestamp.FromTime(end))
|
q, err := api.Queryable.Querier(r.Context(), timestamp.FromTime(end.Add(time.Minute*-5)), timestamp.FromTime(end))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return apiFuncResult{nil, &apiError{errorExec, err}, nil, nil}
|
return apiFuncResult{nil, &apiError{errorExec, err}, nil, nil}
|
||||||
}
|
}
|
||||||
|
@ -1496,6 +1506,84 @@ func (api *API) remoteReadStreamedXORChunks(ctx context.Context, w http.Response
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (api *API) remoteWrite(w http.ResponseWriter, r *http.Request) {
|
||||||
|
compressed, err := ioutil.ReadAll(r.Body)
|
||||||
|
if err != nil {
|
||||||
|
level.Error(api.logger).Log("msg", "Read error", "err", err.Error())
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
reqBuf, err := snappy.Decode(nil, compressed)
|
||||||
|
if err != nil {
|
||||||
|
level.Error(api.logger).Log("msg", "Decode error", "err", err.Error())
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req prompb.WriteRequest
|
||||||
|
if err := proto.Unmarshal(reqBuf, &req); err != nil {
|
||||||
|
level.Error(api.logger).Log("msg", "Unmarshal error", "err", err.Error())
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = api.write(&req)
|
||||||
|
if err != nil {
|
||||||
|
level.Error(api.logger).Log("msg", "Api write", "err", err.Error())
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (api *API) write(req *prompb.WriteRequest) error {
|
||||||
|
var err error = nil
|
||||||
|
app := api.Appendable.Appender()
|
||||||
|
defer func() { //TODO:clear api.refs cache
|
||||||
|
if err != nil {
|
||||||
|
app.Rollback()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err = app.Commit(); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
for _, ts := range req.Timeseries {
|
||||||
|
tsLabels := make(labels.Labels, 0, len(ts.Labels))
|
||||||
|
for _, l := range ts.Labels {
|
||||||
|
tsLabels = append(tsLabels, labels.Label{Name: l.Name, Value: l.Value})
|
||||||
|
}
|
||||||
|
sort.Sort(tsLabels)
|
||||||
|
tsLabelsKey := tsLabels.String()
|
||||||
|
for _, s := range ts.Samples {
|
||||||
|
api.refsLock.RLock()
|
||||||
|
ref, ok := api.refs[tsLabelsKey]
|
||||||
|
api.refsLock.RUnlock()
|
||||||
|
if ok {
|
||||||
|
err = app.AddFast(ref, s.Timestamp, s.Value)
|
||||||
|
if err != nil && strings.Contains(err.Error(), "unknown series") {
|
||||||
|
//
|
||||||
|
} else {
|
||||||
|
switch err {
|
||||||
|
case nil:
|
||||||
|
case storage.ErrOutOfOrderSample:
|
||||||
|
//level.Error(api.logger).Log("msg", "AddFast fail .Out of order sample", "err", err, "series", tsLabelsKey, "Timestamp", s.Timestamp, "Value", s.Value)
|
||||||
|
default:
|
||||||
|
level.Error(api.logger).Log("msg", "AddFast fail .unexpected error", "err", err, "series", tsLabelsKey, "Timestamp", s.Timestamp, "Value", s.Value)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ref, err = app.Add(tsLabels, s.Timestamp, s.Value)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
api.refsLock.Lock()
|
||||||
|
api.refs[tsLabelsKey] = ref
|
||||||
|
api.refsLock.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// filterExtLabelsFromMatchers change equality matchers which match external labels
|
// filterExtLabelsFromMatchers change equality matchers which match external labels
|
||||||
// to a matcher that looks for an empty label,
|
// to a matcher that looks for an empty label,
|
||||||
// as that label should not be present in the storage.
|
// as that label should not be present in the storage.
|
||||||
|
|
|
@ -29,6 +29,7 @@ import (
|
||||||
"runtime"
|
"runtime"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -60,6 +61,10 @@ import (
|
||||||
"github.com/prometheus/prometheus/util/teststorage"
|
"github.com/prometheus/prometheus/util/teststorage"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
errorNone errorType = ""
|
||||||
|
)
|
||||||
|
|
||||||
// testMetaStore satisfies the scrape.MetricMetadataStore interface.
|
// testMetaStore satisfies the scrape.MetricMetadataStore interface.
|
||||||
// It is used to inject specific metadata as part of a test case.
|
// It is used to inject specific metadata as part of a test case.
|
||||||
type testMetaStore struct {
|
type testMetaStore struct {
|
||||||
|
@ -2249,6 +2254,55 @@ func TestStreamReadEndpoint(t *testing.T) {
|
||||||
}, results)
|
}, results)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSampledWriteEndpoint(t *testing.T) {
|
||||||
|
|
||||||
|
samples := []prompb.TimeSeries{
|
||||||
|
{
|
||||||
|
Labels: []prompb.Label{
|
||||||
|
{Name: "__name__", Value: "test_metric1"},
|
||||||
|
{Name: "b", Value: "c"},
|
||||||
|
{Name: "baz", Value: "qux"},
|
||||||
|
{Name: "d", Value: "e"},
|
||||||
|
{Name: "foo", Value: "bar"},
|
||||||
|
},
|
||||||
|
Samples: []prompb.Sample{{Value: 1, Timestamp: 0}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Labels: []prompb.Label{
|
||||||
|
{Name: "__name__", Value: "test_metric1"},
|
||||||
|
{Name: "b", Value: "c"},
|
||||||
|
{Name: "baz", Value: "qux"},
|
||||||
|
{Name: "d", Value: "e"},
|
||||||
|
{Name: "foo", Value: "bar"},
|
||||||
|
},
|
||||||
|
Samples: []prompb.Sample{{Value: 2, Timestamp: 1}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
req := &prompb.WriteRequest{
|
||||||
|
Timeseries: samples,
|
||||||
|
}
|
||||||
|
|
||||||
|
suite, err := promql.NewTest(t, `
|
||||||
|
load 1m
|
||||||
|
test_metric1{foo="bar",baz="qux"} 1
|
||||||
|
`)
|
||||||
|
testutil.Ok(t, err)
|
||||||
|
|
||||||
|
defer suite.Close()
|
||||||
|
|
||||||
|
err = suite.Run()
|
||||||
|
testutil.Ok(t, err)
|
||||||
|
|
||||||
|
api := &API{
|
||||||
|
Appendable: suite.Storage(),
|
||||||
|
refs: make(map[string]uint64),
|
||||||
|
refsLock: &sync.RWMutex{},
|
||||||
|
}
|
||||||
|
err = api.write(req)
|
||||||
|
testutil.Ok(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
type fakeDB struct {
|
type fakeDB struct {
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue