From 99881ded63eb9650a10366a3bfb86a4fd9a1307b Mon Sep 17 00:00:00 2001 From: beorn7 Date: Thu, 2 Jun 2016 19:18:00 +0200 Subject: [PATCH 1/2] Make the number of fingerprint mutexes configurable With a lot of series accessed in a short timeframe (by a query, a large scrape, checkpointing, ...), there is actually quite a significant amount of lock contention if something similar is running at the same time. In those cases, the number of locks needs to be increased. On the same front, as our fingerprints don't have a lot of entropy, I introduced some additional shuffling. With the current state, anly changes in the least singificant bits of a FP would matter. --- cmd/prometheus/config.go | 4 ++++ storage/local/locker.go | 14 +++++++++++--- storage/local/storage.go | 3 ++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/cmd/prometheus/config.go b/cmd/prometheus/config.go index a8ac2a34a..dbbd7ee39 100644 --- a/cmd/prometheus/config.go +++ b/cmd/prometheus/config.go @@ -162,6 +162,10 @@ func init() { &index.LabelPairFingerprintsCacheSize, "storage.local.index-cache-size.label-pair-to-fingerprints", index.LabelPairFingerprintsCacheSize, "The size in bytes for the label pair to fingerprints index cache.", ) + cfg.fs.IntVar( + &cfg.storage.NumMutexes, "storage.local.num-fingerprint-mutexes", 4096, + "The number of mutexes used for fingerprint locking.", + ) // Remote storage. cfg.fs.StringVar( diff --git a/storage/local/locker.go b/storage/local/locker.go index 8ad30cc55..18fbacb9d 100644 --- a/storage/local/locker.go +++ b/storage/local/locker.go @@ -37,8 +37,12 @@ type fingerprintLocker struct { numFpMtxs uint } -// newFingerprintLocker returns a new fingerprintLocker ready for use. +// newFingerprintLocker returns a new fingerprintLocker ready for use. At least +// 1024 preallocated mutexes are used, even if preallocatedMutexes is lower. func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker { + if preallocatedMutexes < 1024 { + preallocatedMutexes = 1024 + } return &fingerprintLocker{ make([]sync.Mutex, preallocatedMutexes), uint(preallocatedMutexes), @@ -47,10 +51,14 @@ func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker { // Lock locks the given fingerprint. func (l *fingerprintLocker) Lock(fp model.Fingerprint) { - l.fpMtxs[uint(fp)%l.numFpMtxs].Lock() + l.fpMtxs[hashFP(fp)%l.numFpMtxs].Lock() } // Unlock unlocks the given fingerprint. func (l *fingerprintLocker) Unlock(fp model.Fingerprint) { - l.fpMtxs[uint(fp)%l.numFpMtxs].Unlock() + l.fpMtxs[hashFP(fp)%l.numFpMtxs].Unlock() +} + +func hashFP(fp model.Fingerprint) uint { + return uint(fp ^ (fp >> 32) ^ (fp >> 16)) } diff --git a/storage/local/storage.go b/storage/local/storage.go index 85c50d089..fb97dd603 100644 --- a/storage/local/storage.go +++ b/storage/local/storage.go @@ -184,13 +184,14 @@ type MemorySeriesStorageOptions struct { PedanticChecks bool // If dirty, perform crash-recovery checks on each series file. SyncStrategy SyncStrategy // Which sync strategy to apply to series files. MinShrinkRatio float64 // Minimum ratio a series file has to shrink during truncation. + NumMutexes int // Number of mutexes used for stochastic fingerprint locking. } // NewMemorySeriesStorage returns a newly allocated Storage. Storage.Serve still // has to be called to start the storage. func NewMemorySeriesStorage(o *MemorySeriesStorageOptions) Storage { s := &memorySeriesStorage{ - fpLocker: newFingerprintLocker(1024), + fpLocker: newFingerprintLocker(o.NumMutexes), options: o, From b274c7aaa7314ec800ecf3233c378b1365ace20e Mon Sep 17 00:00:00 2001 From: beorn7 Date: Fri, 3 Jun 2016 12:34:01 +0200 Subject: [PATCH 2/2] Update doc comments --- storage/local/locker.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/storage/local/locker.go b/storage/local/locker.go index 18fbacb9d..1628ec127 100644 --- a/storage/local/locker.go +++ b/storage/local/locker.go @@ -22,16 +22,13 @@ import ( // fingerprintLocker allows locking individual fingerprints. To limit the number // of mutexes needed for that, only a fixed number of mutexes are // allocated. Fingerprints to be locked are assigned to those pre-allocated -// mutexes by their value. (Note that fingerprints are calculated by a hash -// function, so that an approximately equal distribution over the mutexes is -// expected, even without additional hashing of the fingerprint value.) -// Collisions are not detected. If two fingerprints get assigned to the same -// mutex, only one of them can be locked at the same time. As long as the number -// of pre-allocated mutexes is much larger than the number of goroutines -// requiring a fingerprint lock concurrently, the loss in efficiency is -// small. However, a goroutine must never lock more than one fingerprint at the -// same time. (In that case a collision would try to acquire the same mutex -// twice). +// mutexes by their value. Collisions are not detected. If two fingerprints get +// assigned to the same mutex, only one of them can be locked at the same +// time. As long as the number of pre-allocated mutexes is much larger than the +// number of goroutines requiring a fingerprint lock concurrently, the loss in +// efficiency is small. However, a goroutine must never lock more than one +// fingerprint at the same time. (In that case a collision would try to acquire +// the same mutex twice). type fingerprintLocker struct { fpMtxs []sync.Mutex numFpMtxs uint @@ -59,6 +56,13 @@ func (l *fingerprintLocker) Unlock(fp model.Fingerprint) { l.fpMtxs[hashFP(fp)%l.numFpMtxs].Unlock() } +// hashFP simply moves entropy from the most significant 48 bits of the +// fingerprint into the least significant 16 bits (by XORing) so that a simple +// MOD on the result can be used to pick a mutex while still making use of +// changes in more significant bits of the fingerprint. (The fast fingerprinting +// function we use is prone to only change a few bits for similar metrics. We +// really want to make use of every change in the fingerprint to vary mutex +// selection.) func hashFP(fp model.Fingerprint) uint { return uint(fp ^ (fp >> 32) ^ (fp >> 16)) }