mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-01-12 06:17:27 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

remote_write: round desired shards up before check

Browse source 
Previously we would reject an increase from 2 to 2.5 as being
within 30%; by rounding up first we see this as an increase from 2 to 3.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
This commit is contained in:
Bryan Boreham 2021-08-30 13:39:16 +01:00
parent 6d01ce8c4d
commit 954c0e8020
2 changed files with 5 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
storage/remote/queue_manager.go
View file

@ -859,11 +859,13 @@ func (t *QueueManager) calculateDesiredShards() int {
) )
level.Debug(t.logger).Log("msg", "QueueManager.updateShardsLoop", level.Debug(t.logger).Log("msg", "QueueManager.updateShardsLoop",
"lowerBound", lowerBound, "desiredShards", desiredShards, "upperBound", upperBound) "lowerBound", lowerBound, "desiredShards", desiredShards, "upperBound", upperBound)
desiredShards = math.Ceil(desiredShards) // Round up to be on the safe side.
if lowerBound <= desiredShards && desiredShards <= upperBound { if lowerBound <= desiredShards && desiredShards <= upperBound {
return t.numShards return t.numShards
} }
numShards := int(math.Ceil(desiredShards)) numShards := int(desiredShards)
// Do not downshard if we are more than ten seconds back. // Do not downshard if we are more than ten seconds back.
if numShards < t.numShards && delay > 10.0 { if numShards < t.numShards && delay > 10.0 {
level.Debug(t.logger).Log("msg", "Not downsharding due to being too far behind") level.Debug(t.logger).Log("msg", "Not downsharding due to being too far behind")

4
storage/remote/queue_manager_test.go
View file

@ -1025,7 +1025,7 @@ func TestCalculateDesiredShardsDetail(t *testing.T) {
dataIn: 10, dataIn: 10,
dataOut: 10, dataOut: 10,
dataOutDuration: 1.2, dataOutDuration: 1.2,
expectedShards: 1, // no reaction - less than 30% change expectedShards: 2, // 1.2 is rounded up to 2.
}, },
{ {
name: "bigger slowdown", name: "bigger slowdown",
@ -1033,7 +1033,7 @@ func TestCalculateDesiredShardsDetail(t *testing.T) {
dataIn: 10, dataIn: 10,
dataOut: 10, dataOut: 10,
dataOutDuration: 1.4, dataOutDuration: 1.4,
expectedShards: 2, // now it shards up expectedShards: 2,
}, },
{ {
name: "speed up", name: "speed up",