From 77eb9fb56e25a7474555ac99fafa2fc70fa0e99e Mon Sep 17 00:00:00 2001 From: Shubheksha Jalan Date: Fri, 5 Jan 2018 18:28:09 +0530 Subject: [PATCH 1/7] add padding for series offsets --- index/index.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/index/index.go b/index/index.go index ad6dbb218..760b3b9d9 100644 --- a/index/index.go +++ b/index/index.go @@ -180,11 +180,11 @@ func (w *Writer) write(bufs ...[]byte) error { if err != nil { return err } - // For now the index file must not grow beyond 4GiB. Some of the fixed-sized + // For now the index file must not grow beyond 64GiB. Some of the fixed-sized // offset references in v1 are only 4 bytes large. // Once we move to compressed/varint representations in those areas, this limitation // can be lifted. - if w.pos > math.MaxUint32 { + if w.pos > 16*math.MaxUint32 { return errors.Errorf("exceeding max size of 4GiB") } } @@ -250,6 +250,7 @@ func (w *Writer) writeMeta() error { return w.write(w.buf1.get()) } +//AddSeries adds the series one at a time along with its chunks. func (w *Writer) AddSeries(ref uint64, lset labels.Labels, chunks ...chunks.Meta) error { if err := w.ensureStage(idxStageSeries); err != nil { return err @@ -261,7 +262,8 @@ func (w *Writer) AddSeries(ref uint64, lset labels.Labels, chunks ...chunks.Meta if _, ok := w.seriesOffsets[ref]; ok { return errors.Errorf("series with reference %d already added", ref) } - w.seriesOffsets[ref] = w.pos + w.addPadding(16) + w.seriesOffsets[ref] = w.pos / 16 w.buf2.reset() w.buf2.putUvarint(len(lset)) @@ -854,7 +856,7 @@ func (r *Reader) LabelIndices() ([][]string, error) { // Series the series with the given ID and writes its labels and chunks into lbls and chks. func (r *Reader) Series(id uint64, lbls *labels.Labels, chks *[]chunks.Meta) error { - d := r.decbufUvarintAt(int(id)) + d := r.decbufUvarintAt(int(16 * id)) if d.err() != nil { return d.err() } From 129773b41a565fde5156301e37f9a87158030443 Mon Sep 17 00:00:00 2001 From: Shubheksha Jalan Date: Mon, 8 Jan 2018 23:03:35 +0530 Subject: [PATCH 2/7] merge BlockMeta and blockMeta struct, implement v1 and v2 index readers --- block.go | 18 ++++--- index/index.go | 128 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 127 insertions(+), 19 deletions(-) diff --git a/block.go b/block.go index 400cdac78..3dd49b511 100644 --- a/block.go +++ b/block.go @@ -151,6 +151,9 @@ type BlockMeta struct { // Information on compactions the block was created from. Compaction BlockMetaCompaction `json:"compaction"` + + // Version of the index format + Version int `json:"version"` } // BlockStats contains stats about contents of a block. @@ -176,12 +179,6 @@ const ( flagStd = 1 ) -type blockMeta struct { - Version int `json:"version"` - - *BlockMeta -} - const indexFilename = "index" const metaFilename = "meta.json" @@ -193,16 +190,16 @@ func readMetaFile(dir string) (*BlockMeta, error) { if err != nil { return nil, err } - var m blockMeta + var m BlockMeta if err := json.Unmarshal(b, &m); err != nil { return nil, err } - if m.Version != 1 { + if m.Version != 1 && m.Version != 2 { return nil, errors.Errorf("unexpected meta file version %d", m.Version) } - return m.BlockMeta, nil + return &m, nil } func writeMetaFile(dir string, meta *BlockMeta) error { @@ -219,7 +216,8 @@ func writeMetaFile(dir string, meta *BlockMeta) error { enc.SetIndent("", "\t") var merr MultiError - if merr.Add(enc.Encode(&blockMeta{Version: 1, BlockMeta: meta})); merr.Err() != nil { + meta.Version = 2 + if merr.Add(enc.Encode(meta)); merr.Err() != nil { merr.Add(f.Close()) return merr.Err() } diff --git a/index/index.go b/index/index.go index 760b3b9d9..2d76970d1 100644 --- a/index/index.go +++ b/index/index.go @@ -533,7 +533,7 @@ type Reader struct { // the block has been unmapped. symbols map[uint32]string - dec *DecoderV1 + dec Decoder crc32 hash.Hash32 } @@ -566,7 +566,15 @@ func (b realByteSlice) Sub(start, end int) ByteSlice { // NewReader returns a new IndexReader on the given byte slice. func NewReader(b ByteSlice) (*Reader, error) { - return newReader(b, nil) + return newReader(b, nil, nil) +} + +func NewReaderV1(b ByteSlice, c io.Closer, d DecoderV1) (*Reader, error) { + return newReader(b, c, d) +} + +func NewReaderV2(b ByteSlice, c io.Closer, d DecoderV1) (*Reader, error) { + return newReader(b, c, d) } // NewFileReader returns a new index reader against the given index file. @@ -575,10 +583,10 @@ func NewFileReader(path string) (*Reader, error) { if err != nil { return nil, err } - return newReader(realByteSlice(f.Bytes()), f) + return newReader(realByteSlice(f.Bytes()), f, nil) } -func newReader(b ByteSlice, c io.Closer) (*Reader, error) { +func newReader(b ByteSlice, c io.Closer, d Decoder) (*Reader, error) { r := &Reader{ b: b, c: c, @@ -957,15 +965,23 @@ func (t *serializedStringTuples) At(i int) ([]string, error) { return res, nil } -// DecoderV1 provides decoding methods for the v1 index file format. +// Decoder provides decoding methods for the v1 and v2 index file format. // // It currently does not contain decoding methods for all entry types but can be extended // by them if there's demand. +type Decoder interface { + lookupSymbol(o uint32) (string, error) + SetSymbolTable(t map[uint32]string) + Postings(b []byte) (int, Postings, error) + Series(b []byte, lbls *labels.Labels, chks *[]chunks.Meta) error +} + +// DecoderV1 is used for dedcoding V1 of the index file. type DecoderV1 struct { symbols map[uint32]string } -func (dec *DecoderV1) lookupSymbol(o uint32) (string, error) { +func (dec DecoderV1) lookupSymbol(o uint32) (string, error) { s, ok := dec.symbols[o] if !ok { return "", errors.Errorf("unknown symbol offset %d", o) @@ -975,12 +991,12 @@ func (dec *DecoderV1) lookupSymbol(o uint32) (string, error) { // SetSymbolTable set the symbol table to be used for lookups when decoding series // and label indices -func (dec *DecoderV1) SetSymbolTable(t map[uint32]string) { +func (dec DecoderV1) SetSymbolTable(t map[uint32]string) { dec.symbols = t } // Postings returns a postings list for b and its number of elements. -func (dec *DecoderV1) Postings(b []byte) (int, Postings, error) { +func (dec DecoderV1) Postings(b []byte) (int, Postings, error) { d := decbuf{b: b} n := d.be32int() l := d.get() @@ -988,7 +1004,101 @@ func (dec *DecoderV1) Postings(b []byte) (int, Postings, error) { } // Series decodes a series entry from the given byte slice into lset and chks. -func (dec *DecoderV1) Series(b []byte, lbls *labels.Labels, chks *[]chunks.Meta) error { +func (dec DecoderV1) Series(b []byte, lbls *labels.Labels, chks *[]chunks.Meta) error { + *lbls = (*lbls)[:0] + *chks = (*chks)[:0] + + d := decbuf{b: b} + + k := int(d.uvarint()) + + for i := 0; i < k; i++ { + lno := uint32(d.uvarint()) + lvo := uint32(d.uvarint()) + + if d.err() != nil { + return errors.Wrap(d.err(), "read series label offsets") + } + + ln, err := dec.lookupSymbol(lno) + if err != nil { + return errors.Wrap(err, "lookup label name") + } + lv, err := dec.lookupSymbol(lvo) + if err != nil { + return errors.Wrap(err, "lookup label value") + } + + *lbls = append(*lbls, labels.Label{Name: ln, Value: lv}) + } + + // Read the chunks meta data. + k = int(d.uvarint()) + + if k == 0 { + return nil + } + + t0 := d.varint64() + maxt := int64(d.uvarint64()) + t0 + ref0 := int64(d.uvarint64()) + + *chks = append(*chks, chunks.Meta{ + Ref: uint64(ref0), + MinTime: t0, + MaxTime: maxt, + }) + t0 = maxt + + for i := 1; i < k; i++ { + mint := int64(d.uvarint64()) + t0 + maxt := int64(d.uvarint64()) + mint + + ref0 += d.varint64() + t0 = maxt + + if d.err() != nil { + return errors.Wrapf(d.err(), "read meta for chunk %d", i) + } + + *chks = append(*chks, chunks.Meta{ + Ref: uint64(ref0), + MinTime: mint, + MaxTime: maxt, + }) + } + return d.err() +} + +// DecoderV2 is used for dedcoding V2 of the index file. +type DecoderV2 struct { + symbols map[uint32]string +} + +func (dec *DecoderV2) lookupSymbol(o uint32) (string, error) { + s, ok := dec.symbols[o] + if !ok { + return "", errors.Errorf("unknown symbol offset %d", o) + } + return s, nil +} + +// SetSymbolTable set the symbol table to be used for lookups when decoding series +// and label indices +func (dec *DecoderV2) SetSymbolTable(t map[uint32]string) { + dec.symbols = t +} + +// Postings returns a postings list for b and its number of elements. +func (dec *DecoderV2) Postings(b []byte) (int, Postings, error) { + d := decbuf{b: b} + n := d.be32int() + l := d.get() + return n, newBigEndianPostings(l), d.err() +} + +// Series decodes a series entry from the given byte slice into lset and chks. +func (dec DecoderV2) Series(b []byte, lbls *labels.Labels, chks *[]chunks.Meta) error { *lbls = (*lbls)[:0] *chks = (*chks)[:0] From 44052bc9374a3198d1f56e4750b0e3dcccb5c45c Mon Sep 17 00:00:00 2001 From: Shubheksha Jalan Date: Thu, 11 Jan 2018 00:49:16 +0530 Subject: [PATCH 3/7] Make version for index format configurable --- block.go | 4 +- compact.go | 1 + index/index.go | 154 ++++++++++--------------------------------------- 3 files changed, 32 insertions(+), 127 deletions(-) diff --git a/block.go b/block.go index 3dd49b511..35466c29e 100644 --- a/block.go +++ b/block.go @@ -216,7 +216,7 @@ func writeMetaFile(dir string, meta *BlockMeta) error { enc.SetIndent("", "\t") var merr MultiError - meta.Version = 2 + if merr.Add(enc.Encode(meta)); merr.Err() != nil { merr.Add(f.Close()) return merr.Err() @@ -253,7 +253,7 @@ func OpenBlock(dir string, pool chunkenc.Pool) (*Block, error) { if err != nil { return nil, err } - ir, err := index.NewFileReader(filepath.Join(dir, "index")) + ir, err := index.NewFileReader(filepath.Join(dir, "index"), meta.Version) if err != nil { return nil, err } diff --git a/compact.go b/compact.go index 5c532759f..f96e64394 100644 --- a/compact.go +++ b/compact.go @@ -428,6 +428,7 @@ func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blocks ...BlockRe } indexw, err := index.NewWriter(filepath.Join(tmp, indexFilename)) + meta.Version = indexw.Version if err != nil { return errors.Wrap(err, "open index writer") } diff --git a/index/index.go b/index/index.go index 2d76970d1..3b7a09c39 100644 --- a/index/index.go +++ b/index/index.go @@ -98,7 +98,7 @@ func newCRC32() hash.Hash32 { return crc32.New(castagnoliTable) } -// indexWriter implements the IndexWriter interface for the standard +// Writer implements the IndexWriter interface for the standard // serialization format. type Writer struct { f *os.File @@ -122,6 +122,8 @@ type Writer struct { lastSeries labels.Labels crc32 hash.Hash + + Version int } type indexTOC struct { @@ -185,7 +187,7 @@ func (w *Writer) write(bufs ...[]byte) error { // Once we move to compressed/varint representations in those areas, this limitation // can be lifted. if w.pos > 16*math.MaxUint32 { - return errors.Errorf("exceeding max size of 4GiB") + return errors.Errorf("exceeding max size of 64GiB") } } return nil @@ -250,7 +252,7 @@ func (w *Writer) writeMeta() error { return w.write(w.buf1.get()) } -//AddSeries adds the series one at a time along with its chunks. +// AddSeries adds the series one at a time along with its chunks. func (w *Writer) AddSeries(ref uint64, lset labels.Labels, chunks ...chunks.Meta) error { if err := w.ensureStage(idxStageSeries); err != nil { return err @@ -264,6 +266,7 @@ func (w *Writer) AddSeries(ref uint64, lset labels.Labels, chunks ...chunks.Meta } w.addPadding(16) w.seriesOffsets[ref] = w.pos / 16 + w.Version = 2 w.buf2.reset() w.buf2.putUvarint(len(lset)) @@ -533,9 +536,11 @@ type Reader struct { // the block has been unmapped. symbols map[uint32]string - dec Decoder + dec *Decoder crc32 hash.Hash32 + + version int } var ( @@ -565,28 +570,24 @@ func (b realByteSlice) Sub(start, end int) ByteSlice { } // NewReader returns a new IndexReader on the given byte slice. -func NewReader(b ByteSlice) (*Reader, error) { - return newReader(b, nil, nil) +func NewReader(b ByteSlice, v int) (*Reader, error) { + return newReader(b, nil, v) } -func NewReaderV1(b ByteSlice, c io.Closer, d DecoderV1) (*Reader, error) { - return newReader(b, c, d) -} - -func NewReaderV2(b ByteSlice, c io.Closer, d DecoderV1) (*Reader, error) { - return newReader(b, c, d) +func NewReaderV1(b ByteSlice, c io.Closer, v int) (*Reader, error) { + return newReader(b, c, v) } // NewFileReader returns a new index reader against the given index file. -func NewFileReader(path string) (*Reader, error) { +func NewFileReader(path string, v int) (*Reader, error) { f, err := fileutil.OpenMmapFile(path) if err != nil { return nil, err } - return newReader(realByteSlice(f.Bytes()), f, nil) + return newReader(realByteSlice(f.Bytes()), f, v) } -func newReader(b ByteSlice, c io.Closer, d Decoder) (*Reader, error) { +func newReader(b ByteSlice, c io.Closer, v int) (*Reader, error) { r := &Reader{ b: b, c: c, @@ -594,6 +595,7 @@ func newReader(b ByteSlice, c io.Closer, d Decoder) (*Reader, error) { labels: map[string]uint32{}, postings: map[labels.Label]uint32{}, crc32: newCRC32(), + version: v, } // Verify magic number. if b.Len() < 4 { @@ -632,7 +634,7 @@ func newReader(b ByteSlice, c io.Closer, d Decoder) (*Reader, error) { return nil, errors.Wrap(err, "read postings table") } - r.dec = &DecoderV1{symbols: r.symbols} + r.dec = &Decoder{symbols: r.symbols} return r, nil } @@ -862,9 +864,13 @@ func (r *Reader) LabelIndices() ([][]string, error) { return res, nil } -// Series the series with the given ID and writes its labels and chunks into lbls and chks. +// Reads the series with the given ID and writes its labels and chunks into lbls and chks. func (r *Reader) Series(id uint64, lbls *labels.Labels, chks *[]chunks.Meta) error { - d := r.decbufUvarintAt(int(16 * id)) + offset := id + if r.version == 2 { + offset = 16 * id + } + d := r.decbufUvarintAt(int(offset)) if d.err() != nil { return d.err() } @@ -969,19 +975,11 @@ func (t *serializedStringTuples) At(i int) ([]string, error) { // // It currently does not contain decoding methods for all entry types but can be extended // by them if there's demand. -type Decoder interface { - lookupSymbol(o uint32) (string, error) - SetSymbolTable(t map[uint32]string) - Postings(b []byte) (int, Postings, error) - Series(b []byte, lbls *labels.Labels, chks *[]chunks.Meta) error -} - -// DecoderV1 is used for dedcoding V1 of the index file. -type DecoderV1 struct { +type Decoder struct { symbols map[uint32]string } -func (dec DecoderV1) lookupSymbol(o uint32) (string, error) { +func (dec *Decoder) lookupSymbol(o uint32) (string, error) { s, ok := dec.symbols[o] if !ok { return "", errors.Errorf("unknown symbol offset %d", o) @@ -991,12 +989,12 @@ func (dec DecoderV1) lookupSymbol(o uint32) (string, error) { // SetSymbolTable set the symbol table to be used for lookups when decoding series // and label indices -func (dec DecoderV1) SetSymbolTable(t map[uint32]string) { +func (dec *Decoder) SetSymbolTable(t map[uint32]string) { dec.symbols = t } // Postings returns a postings list for b and its number of elements. -func (dec DecoderV1) Postings(b []byte) (int, Postings, error) { +func (dec *Decoder) Postings(b []byte) (int, Postings, error) { d := decbuf{b: b} n := d.be32int() l := d.get() @@ -1004,101 +1002,7 @@ func (dec DecoderV1) Postings(b []byte) (int, Postings, error) { } // Series decodes a series entry from the given byte slice into lset and chks. -func (dec DecoderV1) Series(b []byte, lbls *labels.Labels, chks *[]chunks.Meta) error { - *lbls = (*lbls)[:0] - *chks = (*chks)[:0] - - d := decbuf{b: b} - - k := int(d.uvarint()) - - for i := 0; i < k; i++ { - lno := uint32(d.uvarint()) - lvo := uint32(d.uvarint()) - - if d.err() != nil { - return errors.Wrap(d.err(), "read series label offsets") - } - - ln, err := dec.lookupSymbol(lno) - if err != nil { - return errors.Wrap(err, "lookup label name") - } - lv, err := dec.lookupSymbol(lvo) - if err != nil { - return errors.Wrap(err, "lookup label value") - } - - *lbls = append(*lbls, labels.Label{Name: ln, Value: lv}) - } - - // Read the chunks meta data. - k = int(d.uvarint()) - - if k == 0 { - return nil - } - - t0 := d.varint64() - maxt := int64(d.uvarint64()) + t0 - ref0 := int64(d.uvarint64()) - - *chks = append(*chks, chunks.Meta{ - Ref: uint64(ref0), - MinTime: t0, - MaxTime: maxt, - }) - t0 = maxt - - for i := 1; i < k; i++ { - mint := int64(d.uvarint64()) + t0 - maxt := int64(d.uvarint64()) + mint - - ref0 += d.varint64() - t0 = maxt - - if d.err() != nil { - return errors.Wrapf(d.err(), "read meta for chunk %d", i) - } - - *chks = append(*chks, chunks.Meta{ - Ref: uint64(ref0), - MinTime: mint, - MaxTime: maxt, - }) - } - return d.err() -} - -// DecoderV2 is used for dedcoding V2 of the index file. -type DecoderV2 struct { - symbols map[uint32]string -} - -func (dec *DecoderV2) lookupSymbol(o uint32) (string, error) { - s, ok := dec.symbols[o] - if !ok { - return "", errors.Errorf("unknown symbol offset %d", o) - } - return s, nil -} - -// SetSymbolTable set the symbol table to be used for lookups when decoding series -// and label indices -func (dec *DecoderV2) SetSymbolTable(t map[uint32]string) { - dec.symbols = t -} - -// Postings returns a postings list for b and its number of elements. -func (dec *DecoderV2) Postings(b []byte) (int, Postings, error) { - d := decbuf{b: b} - n := d.be32int() - l := d.get() - return n, newBigEndianPostings(l), d.err() -} - -// Series decodes a series entry from the given byte slice into lset and chks. -func (dec DecoderV2) Series(b []byte, lbls *labels.Labels, chks *[]chunks.Meta) error { +func (dec *Decoder) Series(b []byte, lbls *labels.Labels, chks *[]chunks.Meta) error { *lbls = (*lbls)[:0] *chks = (*chks)[:0] From 510dc17066fef873342ccecfe095e03d8d29524b Mon Sep 17 00:00:00 2001 From: Shubheksha Jalan Date: Thu, 11 Jan 2018 01:03:20 +0530 Subject: [PATCH 4/7] fix tests, set version when initializing the index writer --- block_test.go | 2 +- index/index.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/block_test.go b/block_test.go index 690cd07ad..4ebb9685d 100644 --- a/block_test.go +++ b/block_test.go @@ -42,7 +42,7 @@ func TestSetCompactionFailed(t *testing.T) { func createEmptyBlock(t *testing.T, dir string) *Block { testutil.Ok(t, os.MkdirAll(dir, 0777)) - testutil.Ok(t, writeMetaFile(dir, &BlockMeta{})) + testutil.Ok(t, writeMetaFile(dir, &BlockMeta{Version: 2})) ir, err := index.NewWriter(filepath.Join(dir, indexFilename)) testutil.Ok(t, err) diff --git a/index/index.go b/index/index.go index 3b7a09c39..7decaa517 100644 --- a/index/index.go +++ b/index/index.go @@ -168,6 +168,8 @@ func NewWriter(fn string) (*Writer, error) { symbols: make(map[string]uint32, 1<<13), seriesOffsets: make(map[uint64]uint64, 1<<16), crc32: newCRC32(), + + Version: 2, } if err := iw.writeMeta(); err != nil { return nil, err From 7b27dc2109b5bc1c0e2adef0cc6eb70f3c31af4b Mon Sep 17 00:00:00 2001 From: Shubheksha Jalan Date: Thu, 11 Jan 2018 13:20:42 +0530 Subject: [PATCH 5/7] address review comments, fix index tests --- block.go | 2 +- index/index.go | 19 +++++++++---------- index/index_test.go | 8 ++++---- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/block.go b/block.go index 35466c29e..2dab0c32d 100644 --- a/block.go +++ b/block.go @@ -152,7 +152,7 @@ type BlockMeta struct { // Information on compactions the block was created from. Compaction BlockMetaCompaction `json:"compaction"` - // Version of the index format + // Version of the index format. Version int `json:"version"` } diff --git a/index/index.go b/index/index.go index 7decaa517..f49b9236e 100644 --- a/index/index.go +++ b/index/index.go @@ -268,7 +268,6 @@ func (w *Writer) AddSeries(ref uint64, lset labels.Labels, chunks ...chunks.Meta } w.addPadding(16) w.seriesOffsets[ref] = w.pos / 16 - w.Version = 2 w.buf2.reset() w.buf2.putUvarint(len(lset)) @@ -572,24 +571,24 @@ func (b realByteSlice) Sub(start, end int) ByteSlice { } // NewReader returns a new IndexReader on the given byte slice. -func NewReader(b ByteSlice, v int) (*Reader, error) { - return newReader(b, nil, v) +func NewReader(b ByteSlice, version int) (*Reader, error) { + return newReader(b, nil, version) } -func NewReaderV1(b ByteSlice, c io.Closer, v int) (*Reader, error) { - return newReader(b, c, v) +func NewReaderV1(b ByteSlice, c io.Closer, version int) (*Reader, error) { + return newReader(b, c, version) } // NewFileReader returns a new index reader against the given index file. -func NewFileReader(path string, v int) (*Reader, error) { +func NewFileReader(path string, version int) (*Reader, error) { f, err := fileutil.OpenMmapFile(path) if err != nil { return nil, err } - return newReader(realByteSlice(f.Bytes()), f, v) + return newReader(realByteSlice(f.Bytes()), f, version) } -func newReader(b ByteSlice, c io.Closer, v int) (*Reader, error) { +func newReader(b ByteSlice, c io.Closer, version int) (*Reader, error) { r := &Reader{ b: b, c: c, @@ -597,7 +596,7 @@ func newReader(b ByteSlice, c io.Closer, v int) (*Reader, error) { labels: map[string]uint32{}, postings: map[labels.Label]uint32{}, crc32: newCRC32(), - version: v, + version: version, } // Verify magic number. if b.Len() < 4 { @@ -866,7 +865,7 @@ func (r *Reader) LabelIndices() ([][]string, error) { return res, nil } -// Reads the series with the given ID and writes its labels and chunks into lbls and chks. +// Series reads the series with the given ID and writes its labels and chunks into lbls and chks. func (r *Reader) Series(id uint64, lbls *labels.Labels, chks *[]chunks.Meta) error { offset := id if r.version == 2 { diff --git a/index/index_test.go b/index/index_test.go index 83b6ef657..65ef9b6e6 100644 --- a/index/index_test.go +++ b/index/index_test.go @@ -160,7 +160,7 @@ func TestIndexRW_Create_Open(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, iw.Close()) - ir, err := NewFileReader(fn) + ir, err := NewFileReader(fn, 1) testutil.Ok(t, err) testutil.Ok(t, ir.Close()) @@ -170,7 +170,7 @@ func TestIndexRW_Create_Open(t *testing.T) { _, err = f.WriteAt([]byte{0, 0}, 0) testutil.Ok(t, err) - _, err = NewFileReader(dir) + _, err = NewFileReader(dir, 1) testutil.NotOk(t, err) } @@ -213,7 +213,7 @@ func TestIndexRW_Postings(t *testing.T) { testutil.Ok(t, iw.Close()) - ir, err := NewFileReader(fn) + ir, err := NewFileReader(fn, 2) testutil.Ok(t, err) p, err := ir.Postings("a", "1") @@ -331,7 +331,7 @@ func TestPersistence_index_e2e(t *testing.T) { err = iw.Close() testutil.Ok(t, err) - ir, err := NewFileReader(filepath.Join(dir, "index")) + ir, err := NewFileReader(filepath.Join(dir, "index"), 2) testutil.Ok(t, err) for p := range mi.postings { From faecfac2fa90c14f8d4cd1d3a1b478dcc81df318 Mon Sep 17 00:00:00 2001 From: Shubheksha Jalan Date: Thu, 11 Jan 2018 14:06:49 +0530 Subject: [PATCH 6/7] update docs, rename docs folder, add error condition --- {Documentation => docs}/format/chunks.md | 0 {Documentation => docs}/format/index.md | 2 +- {Documentation => docs}/format/tombstones.md | 0 index/index.go | 8 ++++---- 4 files changed, 5 insertions(+), 5 deletions(-) rename {Documentation => docs}/format/chunks.md (100%) rename {Documentation => docs}/format/index.md (98%) rename {Documentation => docs}/format/tombstones.md (100%) diff --git a/Documentation/format/chunks.md b/docs/format/chunks.md similarity index 100% rename from Documentation/format/chunks.md rename to docs/format/chunks.md diff --git a/Documentation/format/index.md b/docs/format/index.md similarity index 98% rename from Documentation/format/index.md rename to docs/format/index.md index ce3799907..48366ef8d 100644 --- a/Documentation/format/index.md +++ b/docs/format/index.md @@ -65,7 +65,7 @@ Strings are referenced by pointing to the beginning of their length field. The s ### Series The section contains a sequence of series that hold the label set of the series as well as its chunks within the block. The series are sorted lexicographically by their label sets. -The file offset to the beginning of a series serves as the series' ID in all subsequent references. Thereby, a sorted list of series IDs implies a lexicographically sorted list of series label sets. +Each series section is aligned to 16 bytes. The ID for a series is the `offset/16`. This serves as the series' ID in all subsequent references. Thereby, a sorted list of series IDs implies a lexicographically sorted list of series label sets. ``` ┌───────────────────────────────────────┐ diff --git a/Documentation/format/tombstones.md b/docs/format/tombstones.md similarity index 100% rename from Documentation/format/tombstones.md rename to docs/format/tombstones.md diff --git a/index/index.go b/index/index.go index f49b9236e..98ddb8da8 100644 --- a/index/index.go +++ b/index/index.go @@ -575,12 +575,12 @@ func NewReader(b ByteSlice, version int) (*Reader, error) { return newReader(b, nil, version) } -func NewReaderV1(b ByteSlice, c io.Closer, version int) (*Reader, error) { - return newReader(b, c, version) -} - // NewFileReader returns a new index reader against the given index file. func NewFileReader(path string, version int) (*Reader, error) { + if version != 1 && version != 2 { + return nil, errors.Errorf("unexpected file version %d", version) + + } f, err := fileutil.OpenMmapFile(path) if err != nil { return nil, err From 618bceadb4a44ff8073605c8c983f469e0ccf081 Mon Sep 17 00:00:00 2001 From: Shubheksha Jalan Date: Sat, 13 Jan 2018 00:36:20 +0530 Subject: [PATCH 7/7] move version check into newReader() --- index/index.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/index/index.go b/index/index.go index 98ddb8da8..df8d7b5fe 100644 --- a/index/index.go +++ b/index/index.go @@ -577,10 +577,6 @@ func NewReader(b ByteSlice, version int) (*Reader, error) { // NewFileReader returns a new index reader against the given index file. func NewFileReader(path string, version int) (*Reader, error) { - if version != 1 && version != 2 { - return nil, errors.Errorf("unexpected file version %d", version) - - } f, err := fileutil.OpenMmapFile(path) if err != nil { return nil, err @@ -598,6 +594,11 @@ func newReader(b ByteSlice, c io.Closer, version int) (*Reader, error) { crc32: newCRC32(), version: version, } + + if version != 1 && version != 2 { + return nil, errors.Errorf("unexpected file version %d", version) + + } // Verify magic number. if b.Len() < 4 { return nil, errors.Wrap(errInvalidSize, "index header")