2013-06-06 01:09:16 -07:00
|
|
|
// Copyright 2013 Prometheus Team
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package metric
|
|
|
|
|
|
|
|
import (
|
|
|
|
"container/list"
|
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
2013-06-25 05:02:27 -07:00
|
|
|
"code.google.com/p/goprotobuf/proto"
|
|
|
|
|
|
|
|
clientmodel "github.com/prometheus/client_golang/model"
|
|
|
|
|
|
|
|
dto "github.com/prometheus/prometheus/model/generated"
|
2013-08-05 00:25:47 -07:00
|
|
|
|
|
|
|
"github.com/prometheus/prometheus/storage"
|
|
|
|
"github.com/prometheus/prometheus/storage/raw"
|
|
|
|
"github.com/prometheus/prometheus/storage/raw/leveldb"
|
2013-06-06 01:09:16 -07:00
|
|
|
)
|
|
|
|
|
2013-06-25 05:02:27 -07:00
|
|
|
// unsafe.Sizeof(watermarks{})
|
2013-06-06 01:09:16 -07:00
|
|
|
const elementSize = 24
|
|
|
|
|
|
|
|
type Bytes uint64
|
|
|
|
|
|
|
|
// WatermarkCache is a thread-safe LRU cache for fingerprint watermark
|
|
|
|
// state.
|
|
|
|
type WatermarkCache struct {
|
|
|
|
mu sync.Mutex
|
|
|
|
|
|
|
|
list *list.List
|
2013-06-25 05:02:27 -07:00
|
|
|
table map[clientmodel.Fingerprint]*list.Element
|
2013-06-06 01:09:16 -07:00
|
|
|
|
|
|
|
size Bytes
|
|
|
|
|
|
|
|
allowance Bytes
|
|
|
|
}
|
|
|
|
|
2013-06-25 05:02:27 -07:00
|
|
|
type watermarks struct {
|
2013-06-06 01:09:16 -07:00
|
|
|
High time.Time
|
|
|
|
}
|
|
|
|
|
2013-06-25 05:02:27 -07:00
|
|
|
func (w *watermarks) load(d *dto.MetricHighWatermark) {
|
|
|
|
w.High = time.Unix(d.GetTimestamp(), 0).UTC()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (w *watermarks) dump(d *dto.MetricHighWatermark) {
|
|
|
|
d.Reset()
|
|
|
|
|
|
|
|
d.Timestamp = proto.Int64(w.High.Unix())
|
|
|
|
}
|
|
|
|
|
2013-06-06 01:09:16 -07:00
|
|
|
type entry struct {
|
2013-06-25 05:02:27 -07:00
|
|
|
fingerprint *clientmodel.Fingerprint
|
|
|
|
watermarks *watermarks
|
2013-06-06 01:09:16 -07:00
|
|
|
accessed time.Time
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewWatermarkCache(allowance Bytes) *WatermarkCache {
|
|
|
|
return &WatermarkCache{
|
|
|
|
list: list.New(),
|
2013-06-25 05:02:27 -07:00
|
|
|
table: map[clientmodel.Fingerprint]*list.Element{},
|
2013-06-06 01:09:16 -07:00
|
|
|
allowance: allowance,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-06-25 05:02:27 -07:00
|
|
|
func (lru *WatermarkCache) Get(f *clientmodel.Fingerprint) (v *watermarks, ok bool) {
|
2013-06-06 01:09:16 -07:00
|
|
|
lru.mu.Lock()
|
|
|
|
defer lru.mu.Unlock()
|
|
|
|
|
|
|
|
element, ok := lru.table[*f]
|
|
|
|
if !ok {
|
|
|
|
return nil, false
|
|
|
|
}
|
|
|
|
|
|
|
|
lru.moveToFront(element)
|
|
|
|
|
|
|
|
return element.Value.(*entry).watermarks, true
|
|
|
|
}
|
|
|
|
|
2013-06-25 05:02:27 -07:00
|
|
|
func (lru *WatermarkCache) Set(f *clientmodel.Fingerprint, w *watermarks) {
|
2013-06-06 01:09:16 -07:00
|
|
|
lru.mu.Lock()
|
|
|
|
defer lru.mu.Unlock()
|
|
|
|
|
|
|
|
if element, ok := lru.table[*f]; ok {
|
|
|
|
lru.updateInplace(element, w)
|
|
|
|
} else {
|
|
|
|
lru.addNew(f, w)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-06-25 05:02:27 -07:00
|
|
|
func (lru *WatermarkCache) SetIfAbsent(f *clientmodel.Fingerprint, w *watermarks) {
|
2013-06-06 01:09:16 -07:00
|
|
|
lru.mu.Lock()
|
|
|
|
defer lru.mu.Unlock()
|
|
|
|
|
|
|
|
if element, ok := lru.table[*f]; ok {
|
|
|
|
lru.moveToFront(element)
|
|
|
|
} else {
|
|
|
|
lru.addNew(f, w)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-06-25 05:02:27 -07:00
|
|
|
func (lru *WatermarkCache) Delete(f *clientmodel.Fingerprint) bool {
|
2013-06-06 01:09:16 -07:00
|
|
|
lru.mu.Lock()
|
|
|
|
defer lru.mu.Unlock()
|
|
|
|
|
|
|
|
element, ok := lru.table[*f]
|
|
|
|
if !ok {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
lru.list.Remove(element)
|
|
|
|
delete(lru.table, *f)
|
|
|
|
lru.size -= elementSize
|
|
|
|
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
func (lru *WatermarkCache) Clear() {
|
|
|
|
lru.mu.Lock()
|
|
|
|
defer lru.mu.Unlock()
|
|
|
|
|
|
|
|
lru.list.Init()
|
2013-06-25 05:02:27 -07:00
|
|
|
lru.table = map[clientmodel.Fingerprint]*list.Element{}
|
2013-06-06 01:09:16 -07:00
|
|
|
lru.size = 0
|
|
|
|
}
|
|
|
|
|
2013-06-25 05:02:27 -07:00
|
|
|
func (lru *WatermarkCache) updateInplace(e *list.Element, w *watermarks) {
|
2013-06-06 09:14:47 -07:00
|
|
|
e.Value.(*entry).watermarks = w
|
2013-06-06 01:09:16 -07:00
|
|
|
lru.moveToFront(e)
|
|
|
|
lru.checkCapacity()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (lru *WatermarkCache) moveToFront(e *list.Element) {
|
|
|
|
lru.list.MoveToFront(e)
|
|
|
|
e.Value.(*entry).accessed = time.Now()
|
|
|
|
}
|
|
|
|
|
2013-06-25 05:02:27 -07:00
|
|
|
func (lru *WatermarkCache) addNew(f *clientmodel.Fingerprint, w *watermarks) {
|
2013-06-06 01:09:16 -07:00
|
|
|
lru.table[*f] = lru.list.PushFront(&entry{
|
|
|
|
fingerprint: f,
|
|
|
|
watermarks: w,
|
|
|
|
accessed: time.Now(),
|
|
|
|
})
|
|
|
|
lru.size += elementSize
|
|
|
|
lru.checkCapacity()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (lru *WatermarkCache) checkCapacity() {
|
|
|
|
for lru.size > lru.allowance {
|
|
|
|
delElem := lru.list.Back()
|
|
|
|
delWatermarks := delElem.Value.(*entry)
|
|
|
|
lru.list.Remove(delElem)
|
|
|
|
delete(lru.table, *delWatermarks.fingerprint)
|
|
|
|
lru.size -= elementSize
|
|
|
|
}
|
|
|
|
}
|
2013-08-05 00:25:47 -07:00
|
|
|
|
|
|
|
type FingerprintHighWatermarkMapping map[clientmodel.Fingerprint]time.Time
|
|
|
|
|
|
|
|
type HighWatermarker interface {
|
|
|
|
raw.ForEacher
|
|
|
|
|
|
|
|
UpdateBatch(FingerprintHighWatermarkMapping) error
|
|
|
|
Get(*clientmodel.Fingerprint) (t time.Time, ok bool, err error)
|
|
|
|
Close() error
|
|
|
|
State() string
|
|
|
|
Size() (uint64, bool, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
type leveldbHighWatermarker struct {
|
|
|
|
p *leveldb.LevelDBPersistence
|
|
|
|
}
|
|
|
|
|
|
|
|
func (w *leveldbHighWatermarker) Get(f *clientmodel.Fingerprint) (t time.Time, ok bool, err error) {
|
|
|
|
k := new(dto.Fingerprint)
|
|
|
|
dumpFingerprint(k, f)
|
|
|
|
v := new(dto.MetricHighWatermark)
|
|
|
|
ok, err = w.p.Get(k, v)
|
|
|
|
if err != nil {
|
|
|
|
return t, ok, err
|
|
|
|
}
|
|
|
|
if !ok {
|
|
|
|
return t, ok, err
|
|
|
|
}
|
|
|
|
t = time.Unix(v.GetTimestamp(), 0)
|
|
|
|
return t, true, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (w *leveldbHighWatermarker) UpdateBatch(m FingerprintHighWatermarkMapping) error {
|
|
|
|
batch := leveldb.NewBatch()
|
|
|
|
defer batch.Close()
|
|
|
|
|
|
|
|
for fp, t := range m {
|
|
|
|
existing, present, err := w.Get(&fp)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
k := new(dto.Fingerprint)
|
|
|
|
dumpFingerprint(k, &fp)
|
|
|
|
v := new(dto.MetricHighWatermark)
|
|
|
|
if !present {
|
|
|
|
v.Timestamp = proto.Int64(t.Unix())
|
|
|
|
batch.Put(k, v)
|
|
|
|
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// BUG(matt): Repace this with watermark management.
|
|
|
|
if t.After(existing) {
|
|
|
|
v.Timestamp = proto.Int64(t.Unix())
|
|
|
|
batch.Put(k, v)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return w.p.Commit(batch)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *leveldbHighWatermarker) ForEach(d storage.RecordDecoder, f storage.RecordFilter, o storage.RecordOperator) (bool, error) {
|
|
|
|
return i.p.ForEach(d, f, o)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *leveldbHighWatermarker) Close() error {
|
|
|
|
i.p.Close()
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *leveldbHighWatermarker) State() string {
|
|
|
|
return i.p.State()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *leveldbHighWatermarker) Size() (uint64, bool, error) {
|
|
|
|
s, err := i.p.ApproximateSize()
|
|
|
|
return s, true, err
|
|
|
|
}
|
|
|
|
|
|
|
|
type LevelDBHighWatermarkerOptions struct {
|
|
|
|
leveldb.LevelDBOptions
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewLevelDBHighWatermarker(o *LevelDBHighWatermarkerOptions) (HighWatermarker, error) {
|
|
|
|
s, err := leveldb.NewLevelDBPersistence(&o.LevelDBOptions)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return &leveldbHighWatermarker{
|
|
|
|
p: s,
|
|
|
|
}, nil
|
|
|
|
}
|