mirror of
https://github.com/prometheus/prometheus.git
synced 2024-09-22 16:57:32 -07:00
ec08c9a391
This gives up on the idea to communicate throuh the Append() call (by either not returning as it is now or returning an error as suggested/explored elsewhere). Here I have added a Throttled() call, which has the advantage that it can be called before a whole _batch_ of Append()'s. Scrapes will happen completely or not at all. Same for rule group evaluations. That's a highly desired behavior (as discussed elsewhere). The code is even simpler now as the whole ingestion buffer could be removed. Logging of throttled mode has been streamlined and will create at most one message per minute.
147 lines
3.8 KiB
Go
147 lines
3.8 KiB
Go
// Copyright 2015 The Prometheus Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package remote
|
|
|
|
import (
|
|
"net/url"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/common/model"
|
|
|
|
influx "github.com/influxdb/influxdb/client"
|
|
|
|
"github.com/prometheus/prometheus/config"
|
|
"github.com/prometheus/prometheus/storage/remote/graphite"
|
|
"github.com/prometheus/prometheus/storage/remote/influxdb"
|
|
"github.com/prometheus/prometheus/storage/remote/opentsdb"
|
|
)
|
|
|
|
// Storage collects multiple remote storage queues.
|
|
type Storage struct {
|
|
queues []*StorageQueueManager
|
|
externalLabels model.LabelSet
|
|
mtx sync.RWMutex
|
|
}
|
|
|
|
// ApplyConfig updates the status state as the new config requires.
|
|
// Returns true on success.
|
|
func (s *Storage) ApplyConfig(conf *config.Config) bool {
|
|
s.mtx.Lock()
|
|
defer s.mtx.Unlock()
|
|
|
|
s.externalLabels = conf.GlobalConfig.ExternalLabels
|
|
return true
|
|
}
|
|
|
|
// New returns a new remote Storage.
|
|
func New(o *Options) *Storage {
|
|
s := &Storage{}
|
|
if o.GraphiteAddress != "" {
|
|
c := graphite.NewClient(
|
|
o.GraphiteAddress, o.GraphiteTransport,
|
|
o.StorageTimeout, o.GraphitePrefix)
|
|
s.queues = append(s.queues, NewStorageQueueManager(c, 100*1024))
|
|
}
|
|
if o.OpentsdbURL != "" {
|
|
c := opentsdb.NewClient(o.OpentsdbURL, o.StorageTimeout)
|
|
s.queues = append(s.queues, NewStorageQueueManager(c, 100*1024))
|
|
}
|
|
if o.InfluxdbURL != nil {
|
|
conf := influx.Config{
|
|
URL: *o.InfluxdbURL,
|
|
Username: o.InfluxdbUsername,
|
|
Password: o.InfluxdbPassword,
|
|
Timeout: o.StorageTimeout,
|
|
}
|
|
c := influxdb.NewClient(conf, o.InfluxdbDatabase, o.InfluxdbRetentionPolicy)
|
|
prometheus.MustRegister(c)
|
|
s.queues = append(s.queues, NewStorageQueueManager(c, 100*1024))
|
|
}
|
|
if len(s.queues) == 0 {
|
|
return nil
|
|
}
|
|
return s
|
|
}
|
|
|
|
// Options contains configuration parameters for a remote storage.
|
|
type Options struct {
|
|
StorageTimeout time.Duration
|
|
InfluxdbURL *url.URL
|
|
InfluxdbRetentionPolicy string
|
|
InfluxdbUsername string
|
|
InfluxdbPassword string
|
|
InfluxdbDatabase string
|
|
OpentsdbURL string
|
|
GraphiteAddress string
|
|
GraphiteTransport string
|
|
GraphitePrefix string
|
|
}
|
|
|
|
// Run starts the background processing of the storage queues.
|
|
func (s *Storage) Run() {
|
|
for _, q := range s.queues {
|
|
go q.Run()
|
|
}
|
|
}
|
|
|
|
// Stop the background processing of the storage queues.
|
|
func (s *Storage) Stop() {
|
|
for _, q := range s.queues {
|
|
q.Stop()
|
|
}
|
|
}
|
|
|
|
// Append implements storage.SampleAppender.
|
|
func (s *Storage) Append(smpl *model.Sample) {
|
|
s.mtx.RLock()
|
|
|
|
var snew model.Sample
|
|
snew = *smpl
|
|
snew.Metric = smpl.Metric.Clone()
|
|
|
|
for ln, lv := range s.externalLabels {
|
|
if _, ok := smpl.Metric[ln]; !ok {
|
|
snew.Metric[ln] = lv
|
|
}
|
|
}
|
|
s.mtx.RUnlock()
|
|
|
|
for _, q := range s.queues {
|
|
q.Append(&snew)
|
|
}
|
|
}
|
|
|
|
// NeedsThrottling implements storage.SampleAppender. It will always return
|
|
// false as a remote storage drops samples on the floor if backlogging instead
|
|
// of asking for throttling.
|
|
func (s *Storage) NeedsThrottling() bool {
|
|
return false
|
|
}
|
|
|
|
// Describe implements prometheus.Collector.
|
|
func (s *Storage) Describe(ch chan<- *prometheus.Desc) {
|
|
for _, q := range s.queues {
|
|
q.Describe(ch)
|
|
}
|
|
}
|
|
|
|
// Collect implements prometheus.Collector.
|
|
func (s *Storage) Collect(ch chan<- prometheus.Metric) {
|
|
for _, q := range s.queues {
|
|
q.Collect(ch)
|
|
}
|
|
}
|