diff --git a/cmd/prometheus/config.go b/cmd/prometheus/config.go index a8ac2a34a..dbbd7ee39 100644 --- a/cmd/prometheus/config.go +++ b/cmd/prometheus/config.go @@ -162,6 +162,10 @@ func init() { &index.LabelPairFingerprintsCacheSize, "storage.local.index-cache-size.label-pair-to-fingerprints", index.LabelPairFingerprintsCacheSize, "The size in bytes for the label pair to fingerprints index cache.", ) + cfg.fs.IntVar( + &cfg.storage.NumMutexes, "storage.local.num-fingerprint-mutexes", 4096, + "The number of mutexes used for fingerprint locking.", + ) // Remote storage. cfg.fs.StringVar( diff --git a/storage/local/locker.go b/storage/local/locker.go index 8ad30cc55..1628ec127 100644 --- a/storage/local/locker.go +++ b/storage/local/locker.go @@ -22,23 +22,24 @@ import ( // fingerprintLocker allows locking individual fingerprints. To limit the number // of mutexes needed for that, only a fixed number of mutexes are // allocated. Fingerprints to be locked are assigned to those pre-allocated -// mutexes by their value. (Note that fingerprints are calculated by a hash -// function, so that an approximately equal distribution over the mutexes is -// expected, even without additional hashing of the fingerprint value.) -// Collisions are not detected. If two fingerprints get assigned to the same -// mutex, only one of them can be locked at the same time. As long as the number -// of pre-allocated mutexes is much larger than the number of goroutines -// requiring a fingerprint lock concurrently, the loss in efficiency is -// small. However, a goroutine must never lock more than one fingerprint at the -// same time. (In that case a collision would try to acquire the same mutex -// twice). +// mutexes by their value. Collisions are not detected. If two fingerprints get +// assigned to the same mutex, only one of them can be locked at the same +// time. As long as the number of pre-allocated mutexes is much larger than the +// number of goroutines requiring a fingerprint lock concurrently, the loss in +// efficiency is small. However, a goroutine must never lock more than one +// fingerprint at the same time. (In that case a collision would try to acquire +// the same mutex twice). type fingerprintLocker struct { fpMtxs []sync.Mutex numFpMtxs uint } -// newFingerprintLocker returns a new fingerprintLocker ready for use. +// newFingerprintLocker returns a new fingerprintLocker ready for use. At least +// 1024 preallocated mutexes are used, even if preallocatedMutexes is lower. func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker { + if preallocatedMutexes < 1024 { + preallocatedMutexes = 1024 + } return &fingerprintLocker{ make([]sync.Mutex, preallocatedMutexes), uint(preallocatedMutexes), @@ -47,10 +48,21 @@ func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker { // Lock locks the given fingerprint. func (l *fingerprintLocker) Lock(fp model.Fingerprint) { - l.fpMtxs[uint(fp)%l.numFpMtxs].Lock() + l.fpMtxs[hashFP(fp)%l.numFpMtxs].Lock() } // Unlock unlocks the given fingerprint. func (l *fingerprintLocker) Unlock(fp model.Fingerprint) { - l.fpMtxs[uint(fp)%l.numFpMtxs].Unlock() + l.fpMtxs[hashFP(fp)%l.numFpMtxs].Unlock() +} + +// hashFP simply moves entropy from the most significant 48 bits of the +// fingerprint into the least significant 16 bits (by XORing) so that a simple +// MOD on the result can be used to pick a mutex while still making use of +// changes in more significant bits of the fingerprint. (The fast fingerprinting +// function we use is prone to only change a few bits for similar metrics. We +// really want to make use of every change in the fingerprint to vary mutex +// selection.) +func hashFP(fp model.Fingerprint) uint { + return uint(fp ^ (fp >> 32) ^ (fp >> 16)) } diff --git a/storage/local/storage.go b/storage/local/storage.go index 85c50d089..fb97dd603 100644 --- a/storage/local/storage.go +++ b/storage/local/storage.go @@ -184,13 +184,14 @@ type MemorySeriesStorageOptions struct { PedanticChecks bool // If dirty, perform crash-recovery checks on each series file. SyncStrategy SyncStrategy // Which sync strategy to apply to series files. MinShrinkRatio float64 // Minimum ratio a series file has to shrink during truncation. + NumMutexes int // Number of mutexes used for stochastic fingerprint locking. } // NewMemorySeriesStorage returns a newly allocated Storage. Storage.Serve still // has to be called to start the storage. func NewMemorySeriesStorage(o *MemorySeriesStorageOptions) Storage { s := &memorySeriesStorage{ - fpLocker: newFingerprintLocker(1024), + fpLocker: newFingerprintLocker(o.NumMutexes), options: o,