mirror of
https://github.com/prometheus/prometheus.git
synced 2024-12-26 06:04:05 -08:00
Merge pull request #357 from prometheus/refactor/storage/index-pipelines
Depointerize storage conf. and chain ingester.
This commit is contained in:
commit
83fb0a9a2d
|
@ -129,8 +129,8 @@ func (n *NotificationHandler) sendNotifications(reqs NotificationReqs) error {
|
|||
"Description": interpolateMessage(req.Description, req.Labels, req.Value),
|
||||
"Labels": req.Labels,
|
||||
"Payload": map[string]interface{}{
|
||||
"Value": req.Value,
|
||||
"ActiveSince": req.ActiveSince,
|
||||
"Value": req.Value,
|
||||
"ActiveSince": req.ActiveSince,
|
||||
"GeneratorUrl": n.prometheusUrl,
|
||||
"AlertingRule": req.RuleString,
|
||||
},
|
||||
|
|
|
@ -202,17 +202,26 @@ func (t *target) Scrape(earliest time.Time, results chan<- *extraction.Result) e
|
|||
|
||||
const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,application/json;schema=prometheus/telemetry;version=0.0.2;q=0.2,*/*;q=0.1`
|
||||
|
||||
type channelIngester chan<- *extraction.Result
|
||||
|
||||
func (i channelIngester) Ingest(r *extraction.Result) error {
|
||||
i <- r
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type extendLabelsIngester struct {
|
||||
baseLabels clientmodel.LabelSet
|
||||
results chan<- *extraction.Result
|
||||
|
||||
i extraction.Ingester
|
||||
}
|
||||
|
||||
func (i *extendLabelsIngester) Ingest(r *extraction.Result) error {
|
||||
for _, s := range r.Samples {
|
||||
s.Metric.MergeFromLabelSet(i.baseLabels, clientmodel.ExporterLabelPrefix)
|
||||
}
|
||||
i.results <- r
|
||||
return nil
|
||||
|
||||
return i.i.Ingest(r)
|
||||
}
|
||||
|
||||
func (t *target) scrape(timestamp time.Time, results chan<- *extraction.Result) (err error) {
|
||||
|
@ -261,10 +270,11 @@ func (t *target) scrape(timestamp time.Time, results chan<- *extraction.Result)
|
|||
|
||||
ingester := &extendLabelsIngester{
|
||||
baseLabels: baseLabels,
|
||||
results: results,
|
||||
|
||||
i: channelIngester(results),
|
||||
}
|
||||
processOptions := &extraction.ProcessOptions{
|
||||
Timestamp: timestamp,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
return processor.ProcessSingle(buf, ingester, processOptions)
|
||||
}
|
||||
|
|
|
@ -28,10 +28,10 @@ import (
|
|||
|
||||
func TestTargetScrapeUpdatesState(t *testing.T) {
|
||||
testTarget := target{
|
||||
scheduler: literalScheduler{},
|
||||
state: UNKNOWN,
|
||||
address: "bad schema",
|
||||
httpClient: utility.NewDeadlineClient(0),
|
||||
scheduler: literalScheduler{},
|
||||
state: UNKNOWN,
|
||||
address: "bad schema",
|
||||
httpClient: utility.NewDeadlineClient(0),
|
||||
}
|
||||
testTarget.Scrape(time.Time{}, make(chan *extraction.Result, 2))
|
||||
if testTarget.state != UNREACHABLE {
|
||||
|
@ -44,7 +44,7 @@ func TestTargetRecordScrapeHealth(t *testing.T) {
|
|||
scheduler: literalScheduler{},
|
||||
address: "http://example.url",
|
||||
baseLabels: clientmodel.LabelSet{clientmodel.JobLabel: "testjob"},
|
||||
httpClient: utility.NewDeadlineClient(0),
|
||||
httpClient: utility.NewDeadlineClient(0),
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
|
|
@ -101,8 +101,8 @@ func (i *LevelDBFingerprintMetricIndex) Prune() (bool, error) {
|
|||
return false, nil
|
||||
}
|
||||
|
||||
func NewLevelDBFingerprintMetricIndex(o *LevelDBFingerprintMetricIndexOptions) (*LevelDBFingerprintMetricIndex, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
|
||||
func NewLevelDBFingerprintMetricIndex(o LevelDBFingerprintMetricIndexOptions) (*LevelDBFingerprintMetricIndex, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -201,8 +201,8 @@ type LevelDBLabelNameFingerprintIndexOptions struct {
|
|||
leveldb.LevelDBOptions
|
||||
}
|
||||
|
||||
func NewLevelLabelNameFingerprintIndex(o *LevelDBLabelNameFingerprintIndexOptions) (*LevelDBLabelNameFingerprintIndex, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
|
||||
func NewLevelLabelNameFingerprintIndex(o LevelDBLabelNameFingerprintIndexOptions) (*LevelDBLabelNameFingerprintIndex, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -313,8 +313,8 @@ func (i *LevelDBLabelPairFingerprintIndex) State() *raw.DatabaseState {
|
|||
return i.p.State()
|
||||
}
|
||||
|
||||
func NewLevelDBLabelSetFingerprintIndex(o *LevelDBLabelSetFingerprintIndexOptions) (*LevelDBLabelPairFingerprintIndex, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
|
||||
func NewLevelDBLabelSetFingerprintIndex(o LevelDBLabelSetFingerprintIndexOptions) (*LevelDBLabelPairFingerprintIndex, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -382,8 +382,8 @@ type LevelDBMetricMembershipIndexOptions struct {
|
|||
leveldb.LevelDBOptions
|
||||
}
|
||||
|
||||
func NewLevelDBMetricMembershipIndex(o *LevelDBMetricMembershipIndexOptions) (*LevelDBMetricMembershipIndex, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
|
||||
func NewLevelDBMetricMembershipIndex(o LevelDBMetricMembershipIndexOptions) (*LevelDBMetricMembershipIndex, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -128,7 +128,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
|
|||
"Label Names and Value Pairs by Fingerprint",
|
||||
func() {
|
||||
var err error
|
||||
emission.fingerprintToMetrics, err = NewLevelDBFingerprintMetricIndex(&LevelDBFingerprintMetricIndexOptions{
|
||||
emission.fingerprintToMetrics, err = NewLevelDBFingerprintMetricIndex(LevelDBFingerprintMetricIndexOptions{
|
||||
LevelDBOptions: leveldb.LevelDBOptions{
|
||||
Name: "Metrics by Fingerprint",
|
||||
Purpose: "Index",
|
||||
|
@ -143,13 +143,12 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
|
|||
"Samples by Fingerprint",
|
||||
func() {
|
||||
var err error
|
||||
o := &leveldb.LevelDBOptions{
|
||||
emission.MetricSamples, err = leveldb.NewLevelDBPersistence(leveldb.LevelDBOptions{
|
||||
Name: "Samples",
|
||||
Purpose: "Timeseries",
|
||||
Path: baseDirectory + "/samples_by_fingerprint",
|
||||
CacheSizeBytes: *fingerprintsToLabelPairCacheSize,
|
||||
}
|
||||
emission.MetricSamples, err = leveldb.NewLevelDBPersistence(o)
|
||||
})
|
||||
workers.MayFail(err)
|
||||
},
|
||||
},
|
||||
|
@ -157,7 +156,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
|
|||
"High Watermarks by Fingerprint",
|
||||
func() {
|
||||
var err error
|
||||
emission.MetricHighWatermarks, err = NewLevelDBHighWatermarker(&LevelDBHighWatermarkerOptions{
|
||||
emission.MetricHighWatermarks, err = NewLevelDBHighWatermarker(LevelDBHighWatermarkerOptions{
|
||||
LevelDBOptions: leveldb.LevelDBOptions{
|
||||
Name: "High Watermarks",
|
||||
Purpose: "The youngest sample in the database per metric.",
|
||||
|
@ -171,7 +170,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
|
|||
"Fingerprints by Label Name",
|
||||
func() {
|
||||
var err error
|
||||
emission.labelNameToFingerprints, err = NewLevelLabelNameFingerprintIndex(&LevelDBLabelNameFingerprintIndexOptions{
|
||||
emission.labelNameToFingerprints, err = NewLevelLabelNameFingerprintIndex(LevelDBLabelNameFingerprintIndexOptions{
|
||||
LevelDBOptions: leveldb.LevelDBOptions{
|
||||
Name: "Fingerprints by Label Name",
|
||||
Purpose: "Index",
|
||||
|
@ -186,7 +185,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
|
|||
"Fingerprints by Label Name and Value Pair",
|
||||
func() {
|
||||
var err error
|
||||
emission.labelSetToFingerprints, err = NewLevelDBLabelSetFingerprintIndex(&LevelDBLabelSetFingerprintIndexOptions{
|
||||
emission.labelSetToFingerprints, err = NewLevelDBLabelSetFingerprintIndex(LevelDBLabelSetFingerprintIndexOptions{
|
||||
LevelDBOptions: leveldb.LevelDBOptions{
|
||||
Name: "Fingerprints by Label Pair",
|
||||
Purpose: "Index",
|
||||
|
@ -202,7 +201,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
|
|||
func() {
|
||||
var err error
|
||||
emission.metricMembershipIndex, err = NewLevelDBMetricMembershipIndex(
|
||||
&LevelDBMetricMembershipIndexOptions{
|
||||
LevelDBMetricMembershipIndexOptions{
|
||||
LevelDBOptions: leveldb.LevelDBOptions{
|
||||
Name: "Metric Membership",
|
||||
Purpose: "Index",
|
||||
|
@ -217,7 +216,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
|
|||
"Sample Curation Remarks",
|
||||
func() {
|
||||
var err error
|
||||
emission.CurationRemarks, err = NewLevelDBCurationRemarker(&LevelDBCurationRemarkerOptions{
|
||||
emission.CurationRemarks, err = NewLevelDBCurationRemarker(LevelDBCurationRemarkerOptions{
|
||||
LevelDBOptions: leveldb.LevelDBOptions{
|
||||
Name: "Sample Curation Remarks",
|
||||
Purpose: "Ledger of Progress for Various Curators",
|
||||
|
|
|
@ -845,7 +845,7 @@ func TestCuratorCompactionProcessor(t *testing.T) {
|
|||
sampleDirectory := fixture.NewPreparer(t).Prepare("sample", fixture.NewCassetteFactory(scenario.in.sampleGroups))
|
||||
defer sampleDirectory.Close()
|
||||
|
||||
curatorStates, err := NewLevelDBCurationRemarker(&LevelDBCurationRemarkerOptions{
|
||||
curatorStates, err := NewLevelDBCurationRemarker(LevelDBCurationRemarkerOptions{
|
||||
LevelDBOptions: leveldb.LevelDBOptions{
|
||||
Path: curatorDirectory.Path(),
|
||||
},
|
||||
|
@ -854,7 +854,7 @@ func TestCuratorCompactionProcessor(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
watermarkStates, err := NewLevelDBHighWatermarker(&LevelDBHighWatermarkerOptions{
|
||||
watermarkStates, err := NewLevelDBHighWatermarker(LevelDBHighWatermarkerOptions{
|
||||
LevelDBOptions: leveldb.LevelDBOptions{
|
||||
Path: watermarkDirectory.Path(),
|
||||
},
|
||||
|
@ -864,7 +864,7 @@ func TestCuratorCompactionProcessor(t *testing.T) {
|
|||
}
|
||||
defer watermarkStates.Close()
|
||||
|
||||
samples, err := leveldb.NewLevelDBPersistence(&leveldb.LevelDBOptions{
|
||||
samples, err := leveldb.NewLevelDBPersistence(leveldb.LevelDBOptions{
|
||||
Path: sampleDirectory.Path(),
|
||||
})
|
||||
if err != nil {
|
||||
|
@ -1370,7 +1370,7 @@ func TestCuratorDeletionProcessor(t *testing.T) {
|
|||
sampleDirectory := fixture.NewPreparer(t).Prepare("sample", fixture.NewCassetteFactory(scenario.in.sampleGroups))
|
||||
defer sampleDirectory.Close()
|
||||
|
||||
curatorStates, err := NewLevelDBCurationRemarker(&LevelDBCurationRemarkerOptions{
|
||||
curatorStates, err := NewLevelDBCurationRemarker(LevelDBCurationRemarkerOptions{
|
||||
LevelDBOptions: leveldb.LevelDBOptions{
|
||||
Path: curatorDirectory.Path(),
|
||||
},
|
||||
|
@ -1380,7 +1380,7 @@ func TestCuratorDeletionProcessor(t *testing.T) {
|
|||
}
|
||||
defer curatorStates.Close()
|
||||
|
||||
watermarkStates, err := NewLevelDBHighWatermarker(&LevelDBHighWatermarkerOptions{
|
||||
watermarkStates, err := NewLevelDBHighWatermarker(LevelDBHighWatermarkerOptions{
|
||||
LevelDBOptions: leveldb.LevelDBOptions{
|
||||
Path: watermarkDirectory.Path(),
|
||||
},
|
||||
|
@ -1390,7 +1390,7 @@ func TestCuratorDeletionProcessor(t *testing.T) {
|
|||
}
|
||||
defer watermarkStates.Close()
|
||||
|
||||
samples, err := leveldb.NewLevelDBPersistence(&leveldb.LevelDBOptions{Path: sampleDirectory.Path()})
|
||||
samples, err := leveldb.NewLevelDBPersistence(leveldb.LevelDBOptions{Path: sampleDirectory.Path()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
|
@ -254,8 +254,8 @@ type LevelDBHighWatermarkerOptions struct {
|
|||
leveldb.LevelDBOptions
|
||||
}
|
||||
|
||||
func NewLevelDBHighWatermarker(o *LevelDBHighWatermarkerOptions) (*LevelDBHighWatermarker, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
|
||||
func NewLevelDBHighWatermarker(o LevelDBHighWatermarkerOptions) (*LevelDBHighWatermarker, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -323,8 +323,8 @@ func (w *LevelDBCurationRemarker) Update(pair *curationKey, t time.Time) error {
|
|||
})
|
||||
}
|
||||
|
||||
func NewLevelDBCurationRemarker(o *LevelDBCurationRemarkerOptions) (*LevelDBCurationRemarker, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
|
||||
func NewLevelDBCurationRemarker(o LevelDBCurationRemarkerOptions) (*LevelDBCurationRemarker, error) {
|
||||
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -48,8 +48,8 @@ type LevelDBIndexOptions struct {
|
|||
leveldb.LevelDBOptions
|
||||
}
|
||||
|
||||
func NewLevelDBMembershipIndex(o *LevelDBIndexOptions) (i *LevelDBMembershipIndex, err error) {
|
||||
leveldbPersistence, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
|
||||
func NewLevelDBMembershipIndex(o LevelDBIndexOptions) (i *LevelDBMembershipIndex, err error) {
|
||||
leveldbPersistence, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -184,7 +184,7 @@ type LevelDBOptions struct {
|
|||
Compression Compression
|
||||
}
|
||||
|
||||
func NewLevelDBPersistence(o *LevelDBOptions) (*LevelDBPersistence, error) {
|
||||
func NewLevelDBPersistence(o LevelDBOptions) (*LevelDBPersistence, error) {
|
||||
options := levigo.NewOptions()
|
||||
options.SetCreateIfMissing(true)
|
||||
options.SetParanoidChecks(o.UseParanoidChecks)
|
||||
|
|
|
@ -61,11 +61,10 @@ type (
|
|||
|
||||
func (p preparer) Prepare(n string, f FixtureFactory) (t test.TemporaryDirectory) {
|
||||
t = test.NewTemporaryDirectory(n, p.tester)
|
||||
o := &leveldb.LevelDBOptions{
|
||||
persistence, err := leveldb.NewLevelDBPersistence(leveldb.LevelDBOptions{
|
||||
Path: t.Path(),
|
||||
CacheSizeBytes: cacheCapacity,
|
||||
}
|
||||
persistence, err := leveldb.NewLevelDBPersistence(o)
|
||||
})
|
||||
if err != nil {
|
||||
defer t.Close()
|
||||
p.tester.Fatal(err)
|
||||
|
|
Loading…
Reference in a new issue