Merge pull request #1701 from prometheus/beorn7/lock

Make the number of fingerprint mutexes configurable

cmd/prometheus/config.go

@@ -162,6 +162,10 @@ func init() {
 		&index.LabelPairFingerprintsCacheSize, "storage.local.index-cache-size.label-pair-to-fingerprints", index.LabelPairFingerprintsCacheSize,
 		"The size in bytes for the label pair to fingerprints index cache.",
 	)
+	cfg.fs.IntVar(
+		&cfg.storage.NumMutexes, "storage.local.num-fingerprint-mutexes", 4096,
+		"The number of mutexes used for fingerprint locking.",
+	)
 
 	// Remote storage.
 	cfg.fs.StringVar(

storage/local/locker.go

@@ -22,23 +22,24 @@ import (
 // fingerprintLocker allows locking individual fingerprints. To limit the number
 // of mutexes needed for that, only a fixed number of mutexes are
 // allocated. Fingerprints to be locked are assigned to those pre-allocated
-// mutexes by their value. (Note that fingerprints are calculated by a hash
-// function, so that an approximately equal distribution over the mutexes is
-// expected, even without additional hashing of the fingerprint value.)
-// Collisions are not detected. If two fingerprints get assigned to the same
-// mutex, only one of them can be locked at the same time. As long as the number
-// of pre-allocated mutexes is much larger than the number of goroutines
-// requiring a fingerprint lock concurrently, the loss in efficiency is
-// small. However, a goroutine must never lock more than one fingerprint at the
-// same time. (In that case a collision would try to acquire the same mutex
-// twice).
+// mutexes by their value. Collisions are not detected. If two fingerprints get
+// assigned to the same mutex, only one of them can be locked at the same
+// time. As long as the number of pre-allocated mutexes is much larger than the
+// number of goroutines requiring a fingerprint lock concurrently, the loss in
+// efficiency is small. However, a goroutine must never lock more than one
+// fingerprint at the same time. (In that case a collision would try to acquire
+// the same mutex twice).
 type fingerprintLocker struct {
 	fpMtxs    []sync.Mutex
 	numFpMtxs uint
 }
 
-// newFingerprintLocker returns a new fingerprintLocker ready for use.
+// newFingerprintLocker returns a new fingerprintLocker ready for use.  At least
+// 1024 preallocated mutexes are used, even if preallocatedMutexes is lower.
 func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker {
+	if preallocatedMutexes < 1024 {
+		preallocatedMutexes = 1024
+	}
 	return &fingerprintLocker{
 		make([]sync.Mutex, preallocatedMutexes),
 		uint(preallocatedMutexes),
@@ -47,10 +48,21 @@ func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker {
 
 // Lock locks the given fingerprint.
 func (l *fingerprintLocker) Lock(fp model.Fingerprint) {
-	l.fpMtxs[uint(fp)%l.numFpMtxs].Lock()
+	l.fpMtxs[hashFP(fp)%l.numFpMtxs].Lock()
 }
 
 // Unlock unlocks the given fingerprint.
 func (l *fingerprintLocker) Unlock(fp model.Fingerprint) {
-	l.fpMtxs[uint(fp)%l.numFpMtxs].Unlock()
+	l.fpMtxs[hashFP(fp)%l.numFpMtxs].Unlock()
+}
+
+// hashFP simply moves entropy from the most significant 48 bits of the
+// fingerprint into the least significant 16 bits (by XORing) so that a simple
+// MOD on the result can be used to pick a mutex while still making use of
+// changes in more significant bits of the fingerprint. (The fast fingerprinting
+// function we use is prone to only change a few bits for similar metrics. We
+// really want to make use of every change in the fingerprint to vary mutex
+// selection.)
+func hashFP(fp model.Fingerprint) uint {
+	return uint(fp ^ (fp >> 32) ^ (fp >> 16))
 }

storage/local/storage.go

@@ -184,13 +184,14 @@ type MemorySeriesStorageOptions struct {
 	PedanticChecks             bool          // If dirty, perform crash-recovery checks on each series file.
 	SyncStrategy               SyncStrategy  // Which sync strategy to apply to series files.
 	MinShrinkRatio             float64       // Minimum ratio a series file has to shrink during truncation.
+	NumMutexes                 int           // Number of mutexes used for stochastic fingerprint locking.
 }
 
 // NewMemorySeriesStorage returns a newly allocated Storage. Storage.Serve still
 // has to be called to start the storage.
 func NewMemorySeriesStorage(o *MemorySeriesStorageOptions) Storage {
 	s := &memorySeriesStorage{
-		fpLocker: newFingerprintLocker(1024),
+		fpLocker: newFingerprintLocker(o.NumMutexes),
 
 		options: o,