From 99881ded63eb9650a10366a3bfb86a4fd9a1307b Mon Sep 17 00:00:00 2001
From: beorn7 <beorn@soundcloud.com>
Date: Thu, 2 Jun 2016 19:18:00 +0200
Subject: [PATCH] Make the number of fingerprint mutexes configurable

With a lot of series accessed in a short timeframe (by a query, a
large scrape, checkpointing, ...), there is actually quite a
significant amount of lock contention if something similar is running
at the same time.

In those cases, the number of locks needs to be increased.

On the same front, as our fingerprints don't have a lot of entropy, I
introduced some additional shuffling. With the current state, anly
changes in the least singificant bits of a FP would matter.
---
 cmd/prometheus/config.go |  4 ++++
 storage/local/locker.go  | 14 +++++++++++---
 storage/local/storage.go |  3 ++-
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/cmd/prometheus/config.go b/cmd/prometheus/config.go
index a8ac2a34a0..dbbd7ee394 100644
--- a/cmd/prometheus/config.go
+++ b/cmd/prometheus/config.go
@@ -162,6 +162,10 @@ func init() {
 		&index.LabelPairFingerprintsCacheSize, "storage.local.index-cache-size.label-pair-to-fingerprints", index.LabelPairFingerprintsCacheSize,
 		"The size in bytes for the label pair to fingerprints index cache.",
 	)
+	cfg.fs.IntVar(
+		&cfg.storage.NumMutexes, "storage.local.num-fingerprint-mutexes", 4096,
+		"The number of mutexes used for fingerprint locking.",
+	)
 
 	// Remote storage.
 	cfg.fs.StringVar(
diff --git a/storage/local/locker.go b/storage/local/locker.go
index 8ad30cc558..18fbacb9dd 100644
--- a/storage/local/locker.go
+++ b/storage/local/locker.go
@@ -37,8 +37,12 @@ type fingerprintLocker struct {
 	numFpMtxs uint
 }
 
-// newFingerprintLocker returns a new fingerprintLocker ready for use.
+// newFingerprintLocker returns a new fingerprintLocker ready for use.  At least
+// 1024 preallocated mutexes are used, even if preallocatedMutexes is lower.
 func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker {
+	if preallocatedMutexes < 1024 {
+		preallocatedMutexes = 1024
+	}
 	return &fingerprintLocker{
 		make([]sync.Mutex, preallocatedMutexes),
 		uint(preallocatedMutexes),
@@ -47,10 +51,14 @@ func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker {
 
 // Lock locks the given fingerprint.
 func (l *fingerprintLocker) Lock(fp model.Fingerprint) {
-	l.fpMtxs[uint(fp)%l.numFpMtxs].Lock()
+	l.fpMtxs[hashFP(fp)%l.numFpMtxs].Lock()
 }
 
 // Unlock unlocks the given fingerprint.
 func (l *fingerprintLocker) Unlock(fp model.Fingerprint) {
-	l.fpMtxs[uint(fp)%l.numFpMtxs].Unlock()
+	l.fpMtxs[hashFP(fp)%l.numFpMtxs].Unlock()
+}
+
+func hashFP(fp model.Fingerprint) uint {
+	return uint(fp ^ (fp >> 32) ^ (fp >> 16))
 }
diff --git a/storage/local/storage.go b/storage/local/storage.go
index 85c50d0890..fb97dd6033 100644
--- a/storage/local/storage.go
+++ b/storage/local/storage.go
@@ -184,13 +184,14 @@ type MemorySeriesStorageOptions struct {
 	PedanticChecks             bool          // If dirty, perform crash-recovery checks on each series file.
 	SyncStrategy               SyncStrategy  // Which sync strategy to apply to series files.
 	MinShrinkRatio             float64       // Minimum ratio a series file has to shrink during truncation.
+	NumMutexes                 int           // Number of mutexes used for stochastic fingerprint locking.
 }
 
 // NewMemorySeriesStorage returns a newly allocated Storage. Storage.Serve still
 // has to be called to start the storage.
 func NewMemorySeriesStorage(o *MemorySeriesStorageOptions) Storage {
 	s := &memorySeriesStorage{
-		fpLocker: newFingerprintLocker(1024),
+		fpLocker: newFingerprintLocker(o.NumMutexes),
 
 		options: o,