mirror of
https://github.com/prometheus/prometheus.git
synced 2024-09-21 00:07:36 -07:00
Merge pull request #9534 from prometheus/beorn7/encoding
Remove code duplication in xor encoding.
This commit is contained in:
Björn Rabenstein
GitHub
commit
39be2df10d
|
|
@ -16,7 +16,6 @@ package chunkenc
|
||||||
import (
|
import (
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"math"
|
"math"
|
||||||
"math/bits"
|
|
||||||
|
|
||||||
"github.com/prometheus/prometheus/model/histogram"
|
"github.com/prometheus/prometheus/model/histogram"
|
||||||
"github.com/prometheus/prometheus/pkg/value"
|
"github.com/prometheus/prometheus/pkg/value"
|
||||||
|
|
@ -534,41 +533,7 @@ func (a *HistogramAppender) Recode(
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *HistogramAppender) writeSumDelta(v float64) {
|
func (a *HistogramAppender) writeSumDelta(v float64) {
|
||||||
vDelta := math.Float64bits(v) ^ math.Float64bits(a.sum)
|
a.leading, a.trailing = xorWrite(a.b, v, a.sum, a.leading, a.trailing)
|
||||||
|
|
||||||
if vDelta == 0 {
|
|
||||||
a.b.writeBit(zero)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.b.writeBit(one)
|
|
||||||
|
|
||||||
leading := uint8(bits.LeadingZeros64(vDelta))
|
|
||||||
trailing := uint8(bits.TrailingZeros64(vDelta))
|
|
||||||
|
|
||||||
// Clamp number of leading zeros to avoid overflow when encoding.
|
|
||||||
if leading >= 32 {
|
|
||||||
leading = 31
|
|
||||||
}
|
|
||||||
|
|
||||||
if a.leading != 0xff && leading >= a.leading && trailing >= a.trailing {
|
|
||||||
a.b.writeBit(zero)
|
|
||||||
a.b.writeBits(vDelta>>a.trailing, 64-int(a.leading)-int(a.trailing))
|
|
||||||
} else {
|
|
||||||
a.leading, a.trailing = leading, trailing
|
|
||||||
|
|
||||||
a.b.writeBit(one)
|
|
||||||
a.b.writeBits(uint64(leading), 5)
|
|
||||||
|
|
||||||
// Note that if leading == trailing == 0, then sigbits == 64.
|
|
||||||
// But that value doesn't actually fit into the 6 bits we have.
|
|
||||||
// Luckily, we never need to encode 0 significant bits, since
|
|
||||||
// that would put us in the other case (vdelta == 0). So
|
|
||||||
// instead we write out a 0 and adjust it back to 64 on
|
|
||||||
// unpacking.
|
|
||||||
sigbits := 64 - leading - trailing
|
|
||||||
a.b.writeBits(uint64(sigbits), 6)
|
|
||||||
a.b.writeBits(vDelta>>trailing, int(sigbits))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type histogramIterator struct {
|
type histogramIterator struct {
|
||||||
|
|
@ -873,69 +838,11 @@ func (it *histogramIterator) Next() bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (it *histogramIterator) readSum() bool {
|
func (it *histogramIterator) readSum() bool {
|
||||||
bit, err := it.br.readBitFast()
|
sum, leading, trailing, err := xorRead(&it.br, it.sum, it.leading, it.trailing)
|
||||||
if err != nil {
|
|
||||||
bit, err = it.br.readBit()
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
it.err = err
|
it.err = err
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
it.sum, it.leading, it.trailing = sum, leading, trailing
|
||||||
if bit == zero {
|
|
||||||
return true // it.sum = it.sum
|
|
||||||
}
|
|
||||||
|
|
||||||
bit, err = it.br.readBitFast()
|
|
||||||
if err != nil {
|
|
||||||
bit, err = it.br.readBit()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
it.err = err
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if bit == zero {
|
|
||||||
// Reuse leading/trailing zero bits.
|
|
||||||
// it.leading, it.trailing = it.leading, it.trailing
|
|
||||||
} else {
|
|
||||||
bits, err := it.br.readBitsFast(5)
|
|
||||||
if err != nil {
|
|
||||||
bits, err = it.br.readBits(5)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
it.err = err
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
it.leading = uint8(bits)
|
|
||||||
|
|
||||||
bits, err = it.br.readBitsFast(6)
|
|
||||||
if err != nil {
|
|
||||||
bits, err = it.br.readBits(6)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
it.err = err
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
mbits := uint8(bits)
|
|
||||||
// 0 significant bits here means we overflowed and we actually
|
|
||||||
// need 64; see comment in encoder.
|
|
||||||
if mbits == 0 {
|
|
||||||
mbits = 64
|
|
||||||
}
|
|
||||||
it.trailing = 64 - it.leading - mbits
|
|
||||||
}
|
|
||||||
|
|
||||||
mbits := 64 - it.leading - it.trailing
|
|
||||||
bits, err := it.br.readBitsFast(mbits)
|
|
||||||
if err != nil {
|
|
||||||
bits, err = it.br.readBits(mbits)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
it.err = err
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
vbits := math.Float64bits(it.sum)
|
|
||||||
vbits ^= bits << it.trailing
|
|
||||||
it.sum = math.Float64frombits(vbits)
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -218,38 +218,7 @@ func bitRange(x int64, nbits uint8) bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *xorAppender) writeVDelta(v float64) {
|
func (a *xorAppender) writeVDelta(v float64) {
|
||||||
vDelta := math.Float64bits(v) ^ math.Float64bits(a.v)
|
a.leading, a.trailing = xorWrite(a.b, v, a.v, a.leading, a.trailing)
|
||||||
|
|
||||||
if vDelta == 0 {
|
|
||||||
a.b.writeBit(zero)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.b.writeBit(one)
|
|
||||||
|
|
||||||
leading := uint8(bits.LeadingZeros64(vDelta))
|
|
||||||
trailing := uint8(bits.TrailingZeros64(vDelta))
|
|
||||||
|
|
||||||
// Clamp number of leading zeros to avoid overflow when encoding.
|
|
||||||
if leading >= 32 {
|
|
||||||
leading = 31
|
|
||||||
}
|
|
||||||
|
|
||||||
if a.leading != 0xff && leading >= a.leading && trailing >= a.trailing {
|
|
||||||
a.b.writeBit(zero)
|
|
||||||
a.b.writeBits(vDelta>>a.trailing, 64-int(a.leading)-int(a.trailing))
|
|
||||||
} else {
|
|
||||||
a.leading, a.trailing = leading, trailing
|
|
||||||
|
|
||||||
a.b.writeBit(one)
|
|
||||||
a.b.writeBits(uint64(leading), 5)
|
|
||||||
|
|
||||||
// Note that if leading == trailing == 0, then sigbits == 64. But that value doesn't actually fit into the 6 bits we have.
|
|
||||||
// Luckily, we never need to encode 0 significant bits, since that would put us in the other case (vdelta == 0).
|
|
||||||
// So instead we write out a 0 and adjust it back to 64 on unpacking.
|
|
||||||
sigbits := 64 - leading - trailing
|
|
||||||
a.b.writeBits(uint64(sigbits), 6)
|
|
||||||
a.b.writeBits(vDelta>>trailing, int(sigbits))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type xorIterator struct {
|
type xorIterator struct {
|
||||||
|
|
@ -407,70 +376,120 @@ func (it *xorIterator) Next() bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (it *xorIterator) readValue() bool {
|
func (it *xorIterator) readValue() bool {
|
||||||
bit, err := it.br.readBitFast()
|
val, leading, trailing, err := xorRead(&it.br, it.val, it.leading, it.trailing)
|
||||||
if err != nil {
|
|
||||||
bit, err = it.br.readBit()
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
it.err = err
|
it.err = err
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
it.val, it.leading, it.trailing = val, leading, trailing
|
||||||
if bit == zero {
|
|
||||||
// it.val = it.val
|
|
||||||
} else {
|
|
||||||
bit, err := it.br.readBitFast()
|
|
||||||
if err != nil {
|
|
||||||
bit, err = it.br.readBit()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
it.err = err
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if bit == zero {
|
|
||||||
// reuse leading/trailing zero bits
|
|
||||||
// it.leading, it.trailing = it.leading, it.trailing
|
|
||||||
} else {
|
|
||||||
bits, err := it.br.readBitsFast(5)
|
|
||||||
if err != nil {
|
|
||||||
bits, err = it.br.readBits(5)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
it.err = err
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
it.leading = uint8(bits)
|
|
||||||
|
|
||||||
bits, err = it.br.readBitsFast(6)
|
|
||||||
if err != nil {
|
|
||||||
bits, err = it.br.readBits(6)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
it.err = err
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
mbits := uint8(bits)
|
|
||||||
// 0 significant bits here means we overflowed and we actually need 64; see comment in encoder
|
|
||||||
if mbits == 0 {
|
|
||||||
mbits = 64
|
|
||||||
}
|
|
||||||
it.trailing = 64 - it.leading - mbits
|
|
||||||
}
|
|
||||||
|
|
||||||
mbits := 64 - it.leading - it.trailing
|
|
||||||
bits, err := it.br.readBitsFast(mbits)
|
|
||||||
if err != nil {
|
|
||||||
bits, err = it.br.readBits(mbits)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
it.err = err
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
vbits := math.Float64bits(it.val)
|
|
||||||
vbits ^= bits << it.trailing
|
|
||||||
it.val = math.Float64frombits(vbits)
|
|
||||||
}
|
|
||||||
|
|
||||||
it.numRead++
|
it.numRead++
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func xorWrite(
|
||||||
|
b *bstream,
|
||||||
|
newValue, currentValue float64,
|
||||||
|
currentLeading, currentTrailing uint8,
|
||||||
|
) (newLeading, newTrailing uint8) {
|
||||||
|
delta := math.Float64bits(newValue) ^ math.Float64bits(currentValue)
|
||||||
|
|
||||||
|
if delta == 0 {
|
||||||
|
b.writeBit(zero)
|
||||||
|
return currentLeading, currentTrailing
|
||||||
|
}
|
||||||
|
b.writeBit(one)
|
||||||
|
|
||||||
|
newLeading = uint8(bits.LeadingZeros64(delta))
|
||||||
|
newTrailing = uint8(bits.TrailingZeros64(delta))
|
||||||
|
|
||||||
|
// Clamp number of leading zeros to avoid overflow when encoding.
|
||||||
|
if newLeading >= 32 {
|
||||||
|
newLeading = 31
|
||||||
|
}
|
||||||
|
|
||||||
|
if currentLeading != 0xff && newLeading >= currentLeading && newTrailing >= currentTrailing {
|
||||||
|
// In this case, we stick with the current leading/trailing.
|
||||||
|
b.writeBit(zero)
|
||||||
|
b.writeBits(delta>>currentTrailing, 64-int(currentLeading)-int(currentTrailing))
|
||||||
|
return currentLeading, currentTrailing
|
||||||
|
}
|
||||||
|
|
||||||
|
b.writeBit(one)
|
||||||
|
b.writeBits(uint64(newLeading), 5)
|
||||||
|
|
||||||
|
// Note that if newLeading == newTrailing == 0, then sigbits == 64. But
|
||||||
|
// that value doesn't actually fit into the 6 bits we have. Luckily, we
|
||||||
|
// never need to encode 0 significant bits, since that would put us in
|
||||||
|
// the other case (vdelta == 0). So instead we write out a 0 and adjust
|
||||||
|
// it back to 64 on unpacking.
|
||||||
|
sigbits := 64 - newLeading - newTrailing
|
||||||
|
b.writeBits(uint64(sigbits), 6)
|
||||||
|
b.writeBits(delta>>newTrailing, int(sigbits))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func xorRead(
|
||||||
|
br *bstreamReader, currentValue float64, currentLeading, currentTrailing uint8,
|
||||||
|
) (newValue float64, newLeading, newTrailing uint8, err error) {
|
||||||
|
var bit bit
|
||||||
|
var bits uint64
|
||||||
|
|
||||||
|
bit, err = br.readBitFast()
|
||||||
|
if err != nil {
|
||||||
|
bit, err = br.readBit()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if bit == zero {
|
||||||
|
return currentValue, currentLeading, currentTrailing, nil
|
||||||
|
}
|
||||||
|
bit, err = br.readBitFast()
|
||||||
|
if err != nil {
|
||||||
|
bit, err = br.readBit()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if bit == zero {
|
||||||
|
// Reuse leading/trailing zero bits.
|
||||||
|
newLeading, newTrailing = currentLeading, currentTrailing
|
||||||
|
} else {
|
||||||
|
bits, err = br.readBitsFast(5)
|
||||||
|
if err != nil {
|
||||||
|
bits, err = br.readBits(5)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
newLeading = uint8(bits)
|
||||||
|
|
||||||
|
bits, err = br.readBitsFast(6)
|
||||||
|
if err != nil {
|
||||||
|
bits, err = br.readBits(6)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
mbits := uint8(bits)
|
||||||
|
// 0 significant bits here means we overflowed and we actually
|
||||||
|
// need 64; see comment in xrWrite.
|
||||||
|
if mbits == 0 {
|
||||||
|
mbits = 64
|
||||||
|
}
|
||||||
|
newTrailing = 64 - newLeading - mbits
|
||||||
|
}
|
||||||
|
|
||||||
|
mbits := 64 - newLeading - newTrailing
|
||||||
|
bits, err = br.readBitsFast(mbits)
|
||||||
|
if err != nil {
|
||||||
|
bits, err = br.readBits(mbits)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
vbits := math.Float64bits(currentValue)
|
||||||
|
vbits ^= bits << newTrailing
|
||||||
|
newValue = math.Float64frombits(vbits)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue