mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-11-09 23:24:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Add checksums and magic number to the tombstones.

Browse source 
Signed-off-by: Goutham Veeramachaneni <cs14btech11014@iith.ac.in>
This commit is contained in:
Goutham Veeramachaneni 2017-05-23 17:27:45 +05:30
parent 3eb4119ab1
commit 9d388941ef
No known key found for this signature in database
GPG key ID: F1C217E8E9023CAD
2 changed files with 101 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
Documentation/format/tombstones.md Normal file
View file

@ -0,0 +1,55 @@
# Tombstones Disk Format
The following describes the format of a tombstones file, which is the directory of a block.
The last 8 bytes specifies the offset to the start of Stones section.
The stones section is 0 padded to a multiple of 4 for fast scans.
```
┌────────────────────────────┬─────────────────────┐
│ magic(0x130BA30) <4b> │ version(1) <1 byte> │
├────────────────────────────┴─────────────────────┤
│ ┌──────────────────────────────────────────────┐ │
│ │ Ranges 1 │ │
│ ├──────────────────────────────────────────────┤ │
│ │ ... │ │
│ ├──────────────────────────────────────────────┤ │
│ │ Ranges N │ │
│ ├──────────────────────────────────────────────┤ │
│ │ Stones │ │
│ ├──────────────────────────────────────────────┤ │
│ │ Ref(stones start)<8b> │ │
│ └──────────────────────────────────────────────┘ │
└──────────────────────────────────────────────────┘
```
# Ranges
```
┌──────────────────────────────────────────┐
│ ┌──────────────────────────────────────┐ │
│ │ #ranges <uvarint> │ │
│ ├───────────────────┬──────────────────┤ │
│ │ mint <varint64> │ maxt <varint64> │ │
│ ├───────────────────┴──────────────────┤ │
│ │ . . . │ │
│ ├──────────────────────────────────────┤ │
│ │ CRC32 <4b> │ │
│ └──────────────────────────────────────┘ │
└──────────────────────────────────────────┘
```
# Stones
```
┌──────────────────────────────────────────┐
│ ┌──────────────────────────────────────┐ │
│ │ #stones <4b> │ │
│ ├───────────────────┬──────────────────┤ │
│ │ ref <4b> │ offset <8b> │ │
│ ├───────────────────┴──────────────────┤ │
│ │ . . . │ │
│ └──────────────────────────────────────┘ │
└──────────────────────────────────────────┘
```
The offset here is the offset to the relevant ranges.

55
tombstones.go
View file

@ -2,6 +2,8 @@ package tsdb
import (
"encoding/binary"
"fmt"
"hash/crc32"
"io/ioutil"
"os"
"path/filepath"
@ -10,6 +12,13 @@ import (
const tombstoneFilename = "tombstones"
const (
// MagicTombstone is 4 bytes at the head of a tombstone file.
MagicTombstone = 0x130BA30
tombstoneFormatV1 = 1
)
func readTombstoneFile(dir string) (TombstoneReader, error) {
return newTombStoneReader(dir)
}
@ -17,6 +26,7 @@ func readTombstoneFile(dir string) (TombstoneReader, error) {
func writeTombstoneFile(dir string, tr TombstoneReader) error {
path := filepath.Join(dir, tombstoneFilename)
tmp := path + ".tmp"
hash := crc32.New(crc32.MakeTable(crc32.Castagnoli))
f, err := os.Create(tmp)
if err != nil {
@ -28,6 +38,16 @@ func writeTombstoneFile(dir string, tr TombstoneReader) error {
pos := int64(0)
buf := encbuf{b: make([]byte, 2*binary.MaxVarintLen64)}
buf.reset()
// Write the meta.
buf.putBE32(MagicTombstone)
buf.putByte(tombstoneFormatV1)
n, err := f.Write(buf.get())
if err != nil {
return err
}
pos += int64(n)
for tr.Next() {
s := tr.At()
@ -36,29 +56,40 @@ func writeTombstoneFile(dir string, tr TombstoneReader) error {
// Write the ranges.
buf.reset()
buf.putVarint64(int64(len(s.intervals)))
buf.putUvarint(len(s.intervals))
n, err := f.Write(buf.get())
if err != nil {
return err
}
pos += int64(n)
buf.reset()
for _, r := range s.intervals {
buf.reset()
buf.putVarint64(r.mint)
buf.putVarint64(r.maxt)
n, err = f.Write(buf.get())
if err != nil {
return err
}
pos += int64(n)
}
buf.putHash(hash)
n, err = f.Write(buf.get())
if err != nil {
return err
}
pos += int64(n)
}
if err := tr.Err(); err != nil {
return err
}
// Write the offset table.
// Pad first.
if p := 4 - (int(pos) % 4); p != 0 {
if _, err := f.Write(make([]byte, p)); err != nil {
return err
}
pos += int64(p)
}
buf.reset()
buf.putBE32int(len(refs))
if _, err := f.Write(buf.get()); err != nil {
@ -123,8 +154,13 @@ func newTombStoneReader(dir string) (*tombstoneReader, error) {
return nil, err
}
d := &decbuf{b: b}
if mg := d.be32(); mg != MagicTombstone {
return nil, fmt.Errorf("invalid magic number %x", mg)
}
offsetBytes := b[len(b)-8:]
d := &decbuf{b: offsetBytes}
d = &decbuf{b: offsetBytes}
off := d.be64int64()
if err := d.err(); err != nil {
return nil, err
@ -158,7 +194,7 @@ func (t *tombstoneReader) Next() bool {
off := d.be64int64()
d = &decbuf{b: t.b[off:]}
numRanges := d.varint64()
numRanges := d.uvarint()
if err := d.err(); err != nil {
t.err = err
return false
@ -176,6 +212,7 @@ func (t *tombstoneReader) Next() bool {
dranges = append(dranges, interval{mint, maxt})
}
// TODO(gouthamve): Verify checksum.
t.stones = t.stones[12:]
t.cur = stone{ref: ref, intervals: dranges}
return true