mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-25 13:44:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #357 from prometheus/refactor/storage/index-pipelines

Browse source 
Depointerize storage conf. and chain ingester.
This commit is contained in:
juliusv 2013-08-12 08:17:03 -07:00
parent c453445390 a5141e4d0a
commit 83fb0a9a2d
10 changed files with 53 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
notification/notification.go
View file

@ -129,8 +129,8 @@ func (n *NotificationHandler) sendNotifications(reqs NotificationReqs) error {
"Description": interpolateMessage(req.Description, req.Labels, req.Value),
"Labels": req.Labels,
"Payload": map[string]interface{}{
"Value": req.Value,
"ActiveSince": req.ActiveSince,
"Value": req.Value,
"ActiveSince": req.ActiveSince,
"GeneratorUrl": n.prometheusUrl,
"AlertingRule": req.RuleString,
},

20
retrieval/target.go
View file

@ -202,17 +202,26 @@ func (t *target) Scrape(earliest time.Time, results chan<- *extraction.Result) e
const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,application/json;schema=prometheus/telemetry;version=0.0.2;q=0.2,*/*;q=0.1`
type channelIngester chan<- *extraction.Result
func (i channelIngester) Ingest(r *extraction.Result) error {
i <- r
return nil
}
type extendLabelsIngester struct {
baseLabels clientmodel.LabelSet
results chan<- *extraction.Result
i extraction.Ingester
}
func (i *extendLabelsIngester) Ingest(r *extraction.Result) error {
for _, s := range r.Samples {
s.Metric.MergeFromLabelSet(i.baseLabels, clientmodel.ExporterLabelPrefix)
}
i.results <- r
return nil
return i.i.Ingest(r)
}
func (t *target) scrape(timestamp time.Time, results chan<- *extraction.Result) (err error) {
@ -261,10 +270,11 @@ func (t *target) scrape(timestamp time.Time, results chan<- *extraction.Result)
ingester := &extendLabelsIngester{
baseLabels: baseLabels,
results: results,
i: channelIngester(results),
}
processOptions := &extraction.ProcessOptions{
Timestamp: timestamp,
Timestamp: timestamp,
}
return processor.ProcessSingle(buf, ingester, processOptions)
}

10
retrieval/target_test.go
View file

@ -28,10 +28,10 @@ import (
func TestTargetScrapeUpdatesState(t *testing.T) {
testTarget := target{
scheduler: literalScheduler{},
state: UNKNOWN,
address: "bad schema",
httpClient: utility.NewDeadlineClient(0),
scheduler: literalScheduler{},
state: UNKNOWN,
address: "bad schema",
httpClient: utility.NewDeadlineClient(0),
}
testTarget.Scrape(time.Time{}, make(chan *extraction.Result, 2))
if testTarget.state != UNREACHABLE {
@ -44,7 +44,7 @@ func TestTargetRecordScrapeHealth(t *testing.T) {
scheduler: literalScheduler{},
address: "http://example.url",
baseLabels: clientmodel.LabelSet{clientmodel.JobLabel: "testjob"},
httpClient: utility.NewDeadlineClient(0),
httpClient: utility.NewDeadlineClient(0),
}
now := time.Now()

16
storage/metric/index.go
View file

@ -101,8 +101,8 @@ func (i *LevelDBFingerprintMetricIndex) Prune() (bool, error) {
return false, nil
}
func NewLevelDBFingerprintMetricIndex(o *LevelDBFingerprintMetricIndexOptions) (*LevelDBFingerprintMetricIndex, error) {
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
func NewLevelDBFingerprintMetricIndex(o LevelDBFingerprintMetricIndexOptions) (*LevelDBFingerprintMetricIndex, error) {
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
if err != nil {
return nil, err
}
@ -201,8 +201,8 @@ type LevelDBLabelNameFingerprintIndexOptions struct {
leveldb.LevelDBOptions
}
func NewLevelLabelNameFingerprintIndex(o *LevelDBLabelNameFingerprintIndexOptions) (*LevelDBLabelNameFingerprintIndex, error) {
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
func NewLevelLabelNameFingerprintIndex(o LevelDBLabelNameFingerprintIndexOptions) (*LevelDBLabelNameFingerprintIndex, error) {
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
if err != nil {
return nil, err
}
@ -313,8 +313,8 @@ func (i *LevelDBLabelPairFingerprintIndex) State() *raw.DatabaseState {
return i.p.State()
}
func NewLevelDBLabelSetFingerprintIndex(o *LevelDBLabelSetFingerprintIndexOptions) (*LevelDBLabelPairFingerprintIndex, error) {
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
func NewLevelDBLabelSetFingerprintIndex(o LevelDBLabelSetFingerprintIndexOptions) (*LevelDBLabelPairFingerprintIndex, error) {
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
if err != nil {
return nil, err
}
@ -382,8 +382,8 @@ type LevelDBMetricMembershipIndexOptions struct {
leveldb.LevelDBOptions
}
func NewLevelDBMetricMembershipIndex(o *LevelDBMetricMembershipIndexOptions) (*LevelDBMetricMembershipIndex, error) {
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
func NewLevelDBMetricMembershipIndex(o LevelDBMetricMembershipIndexOptions) (*LevelDBMetricMembershipIndex, error) {
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
if err != nil {
return nil, err
}

17
storage/metric/leveldb.go
View file

@ -128,7 +128,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
"Label Names and Value Pairs by Fingerprint",
func() {
var err error
emission.fingerprintToMetrics, err = NewLevelDBFingerprintMetricIndex(&LevelDBFingerprintMetricIndexOptions{
emission.fingerprintToMetrics, err = NewLevelDBFingerprintMetricIndex(LevelDBFingerprintMetricIndexOptions{
LevelDBOptions: leveldb.LevelDBOptions{
Name: "Metrics by Fingerprint",
Purpose: "Index",
@ -143,13 +143,12 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
"Samples by Fingerprint",
func() {
var err error
o := &leveldb.LevelDBOptions{
emission.MetricSamples, err = leveldb.NewLevelDBPersistence(leveldb.LevelDBOptions{
Name: "Samples",
Purpose: "Timeseries",
Path: baseDirectory + "/samples_by_fingerprint",
CacheSizeBytes: *fingerprintsToLabelPairCacheSize,
}
emission.MetricSamples, err = leveldb.NewLevelDBPersistence(o)
})
workers.MayFail(err)
},
},
@ -157,7 +156,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
"High Watermarks by Fingerprint",
func() {
var err error
emission.MetricHighWatermarks, err = NewLevelDBHighWatermarker(&LevelDBHighWatermarkerOptions{
emission.MetricHighWatermarks, err = NewLevelDBHighWatermarker(LevelDBHighWatermarkerOptions{
LevelDBOptions: leveldb.LevelDBOptions{
Name: "High Watermarks",
Purpose: "The youngest sample in the database per metric.",
@ -171,7 +170,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
"Fingerprints by Label Name",
func() {
var err error
emission.labelNameToFingerprints, err = NewLevelLabelNameFingerprintIndex(&LevelDBLabelNameFingerprintIndexOptions{
emission.labelNameToFingerprints, err = NewLevelLabelNameFingerprintIndex(LevelDBLabelNameFingerprintIndexOptions{
LevelDBOptions: leveldb.LevelDBOptions{
Name: "Fingerprints by Label Name",
Purpose: "Index",
@ -186,7 +185,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
"Fingerprints by Label Name and Value Pair",
func() {
var err error
emission.labelSetToFingerprints, err = NewLevelDBLabelSetFingerprintIndex(&LevelDBLabelSetFingerprintIndexOptions{
emission.labelSetToFingerprints, err = NewLevelDBLabelSetFingerprintIndex(LevelDBLabelSetFingerprintIndexOptions{
LevelDBOptions: leveldb.LevelDBOptions{
Name: "Fingerprints by Label Pair",
Purpose: "Index",
@ -202,7 +201,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
func() {
var err error
emission.metricMembershipIndex, err = NewLevelDBMetricMembershipIndex(
&LevelDBMetricMembershipIndexOptions{
LevelDBMetricMembershipIndexOptions{
LevelDBOptions: leveldb.LevelDBOptions{
Name: "Metric Membership",
Purpose: "Index",
@ -217,7 +216,7 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
"Sample Curation Remarks",
func() {
var err error
emission.CurationRemarks, err = NewLevelDBCurationRemarker(&LevelDBCurationRemarkerOptions{
emission.CurationRemarks, err = NewLevelDBCurationRemarker(LevelDBCurationRemarkerOptions{
LevelDBOptions: leveldb.LevelDBOptions{
Name: "Sample Curation Remarks",
Purpose: "Ledger of Progress for Various Curators",

12
storage/metric/processor_test.go
View file

@ -845,7 +845,7 @@ func TestCuratorCompactionProcessor(t *testing.T) {
sampleDirectory := fixture.NewPreparer(t).Prepare("sample", fixture.NewCassetteFactory(scenario.in.sampleGroups))
defer sampleDirectory.Close()
curatorStates, err := NewLevelDBCurationRemarker(&LevelDBCurationRemarkerOptions{
curatorStates, err := NewLevelDBCurationRemarker(LevelDBCurationRemarkerOptions{
LevelDBOptions: leveldb.LevelDBOptions{
Path: curatorDirectory.Path(),
},
@ -854,7 +854,7 @@ func TestCuratorCompactionProcessor(t *testing.T) {
t.Fatal(err)
}
watermarkStates, err := NewLevelDBHighWatermarker(&LevelDBHighWatermarkerOptions{
watermarkStates, err := NewLevelDBHighWatermarker(LevelDBHighWatermarkerOptions{
LevelDBOptions: leveldb.LevelDBOptions{
Path: watermarkDirectory.Path(),
},
@ -864,7 +864,7 @@ func TestCuratorCompactionProcessor(t *testing.T) {
}
defer watermarkStates.Close()
samples, err := leveldb.NewLevelDBPersistence(&leveldb.LevelDBOptions{
samples, err := leveldb.NewLevelDBPersistence(leveldb.LevelDBOptions{
Path: sampleDirectory.Path(),
})
if err != nil {
@ -1370,7 +1370,7 @@ func TestCuratorDeletionProcessor(t *testing.T) {
sampleDirectory := fixture.NewPreparer(t).Prepare("sample", fixture.NewCassetteFactory(scenario.in.sampleGroups))
defer sampleDirectory.Close()
curatorStates, err := NewLevelDBCurationRemarker(&LevelDBCurationRemarkerOptions{
curatorStates, err := NewLevelDBCurationRemarker(LevelDBCurationRemarkerOptions{
LevelDBOptions: leveldb.LevelDBOptions{
Path: curatorDirectory.Path(),
},
@ -1380,7 +1380,7 @@ func TestCuratorDeletionProcessor(t *testing.T) {
}
defer curatorStates.Close()
watermarkStates, err := NewLevelDBHighWatermarker(&LevelDBHighWatermarkerOptions{
watermarkStates, err := NewLevelDBHighWatermarker(LevelDBHighWatermarkerOptions{
LevelDBOptions: leveldb.LevelDBOptions{
Path: watermarkDirectory.Path(),
},
@ -1390,7 +1390,7 @@ func TestCuratorDeletionProcessor(t *testing.T) {
}
defer watermarkStates.Close()
samples, err := leveldb.NewLevelDBPersistence(&leveldb.LevelDBOptions{Path: sampleDirectory.Path()})
samples, err := leveldb.NewLevelDBPersistence(leveldb.LevelDBOptions{Path: sampleDirectory.Path()})
if err != nil {
t.Fatal(err)
}

8
storage/metric/watermark.go
View file

@ -254,8 +254,8 @@ type LevelDBHighWatermarkerOptions struct {
leveldb.LevelDBOptions
}
func NewLevelDBHighWatermarker(o *LevelDBHighWatermarkerOptions) (*LevelDBHighWatermarker, error) {
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
func NewLevelDBHighWatermarker(o LevelDBHighWatermarkerOptions) (*LevelDBHighWatermarker, error) {
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
if err != nil {
return nil, err
}
@ -323,8 +323,8 @@ func (w *LevelDBCurationRemarker) Update(pair *curationKey, t time.Time) error {
})
}
func NewLevelDBCurationRemarker(o *LevelDBCurationRemarkerOptions) (*LevelDBCurationRemarker, error) {
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
func NewLevelDBCurationRemarker(o LevelDBCurationRemarkerOptions) (*LevelDBCurationRemarker, error) {
s, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
if err != nil {
return nil, err
}

4
storage/raw/index/leveldb/leveldb.go
View file

@ -48,8 +48,8 @@ type LevelDBIndexOptions struct {
leveldb.LevelDBOptions
}
func NewLevelDBMembershipIndex(o *LevelDBIndexOptions) (i *LevelDBMembershipIndex, err error) {
leveldbPersistence, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
func NewLevelDBMembershipIndex(o LevelDBIndexOptions) (i *LevelDBMembershipIndex, err error) {
leveldbPersistence, err := leveldb.NewLevelDBPersistence(o.LevelDBOptions)
if err != nil {
return nil, err
}

2
storage/raw/leveldb/leveldb.go
View file

@ -184,7 +184,7 @@ type LevelDBOptions struct {
Compression Compression
}
func NewLevelDBPersistence(o *LevelDBOptions) (*LevelDBPersistence, error) {
func NewLevelDBPersistence(o LevelDBOptions) (*LevelDBPersistence, error) {
options := levigo.NewOptions()
options.SetCreateIfMissing(true)
options.SetParanoidChecks(o.UseParanoidChecks)

5
storage/raw/leveldb/test/fixtures.go
View file

@ -61,11 +61,10 @@ type (
func (p preparer) Prepare(n string, f FixtureFactory) (t test.TemporaryDirectory) {
t = test.NewTemporaryDirectory(n, p.tester)
o := &leveldb.LevelDBOptions{
persistence, err := leveldb.NewLevelDBPersistence(leveldb.LevelDBOptions{
Path: t.Path(),
CacheSizeBytes: cacheCapacity,
}
persistence, err := leveldb.NewLevelDBPersistence(o)
})
if err != nil {
defer t.Close()
p.tester.Fatal(err)