2017-04-10 11:59:45 -07:00
|
|
|
// Copyright 2017 The Prometheus Authors
|
2017-06-06 05:45:54 -07:00
|
|
|
|
2017-04-10 11:59:45 -07:00
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2016-12-08 08:43:10 -08:00
|
|
|
package tsdb
|
|
|
|
|
2016-12-14 23:31:26 -08:00
|
|
|
import (
|
2017-01-19 02:22:47 -08:00
|
|
|
"encoding/json"
|
2019-02-08 03:35:32 -08:00
|
|
|
"io"
|
2016-12-14 23:31:26 -08:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2017-10-09 06:21:46 -07:00
|
|
|
"sync"
|
2016-12-22 06:54:39 -08:00
|
|
|
|
2021-06-11 09:17:59 -07:00
|
|
|
"github.com/go-kit/log"
|
|
|
|
"github.com/go-kit/log/level"
|
2017-02-27 01:46:15 -08:00
|
|
|
"github.com/oklog/ulid"
|
2016-12-22 06:54:39 -08:00
|
|
|
"github.com/pkg/errors"
|
2022-09-30 07:33:56 -07:00
|
|
|
"golang.org/x/exp/slices"
|
2020-10-22 02:00:08 -07:00
|
|
|
|
2021-11-08 06:23:17 -08:00
|
|
|
"github.com/prometheus/prometheus/model/labels"
|
2021-11-06 03:10:04 -07:00
|
|
|
"github.com/prometheus/prometheus/storage"
|
2019-08-13 01:34:14 -07:00
|
|
|
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
|
|
|
"github.com/prometheus/prometheus/tsdb/chunks"
|
|
|
|
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
|
|
|
|
"github.com/prometheus/prometheus/tsdb/fileutil"
|
|
|
|
"github.com/prometheus/prometheus/tsdb/index"
|
2019-09-19 02:15:41 -07:00
|
|
|
"github.com/prometheus/prometheus/tsdb/tombstones"
|
2016-12-08 08:43:10 -08:00
|
|
|
)
|
|
|
|
|
2017-11-30 06:34:49 -08:00
|
|
|
// IndexWriter serializes the index for a block of series data.
|
|
|
|
// The methods must be called in the order they are specified in.
|
|
|
|
type IndexWriter interface {
|
2022-05-25 09:01:45 -07:00
|
|
|
// AddSymbol registers a single symbol.
|
|
|
|
// Symbols must be registered in sorted order.
|
Stream symbols during compaction. (#6468)
Rather than buffer up symbols in RAM, do it one by one
during compaction. Then use the reader's symbol handling
for symbol lookups during the rest of the index write.
There is some slowdown in compaction, due to having to look through a file
rather than a hash lookup. This is noise to the overall cost of compacting
series with thousands of samples though.
benchmark old ns/op new ns/op delta
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 539917175 675341565 +25.08%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 2441815993 2477453524 +1.46%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 3978543559 3922909687 -1.40%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 8430219716 8586610007 +1.86%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 1786424591 1909552782 +6.89%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 5328998202 6020839950 +12.98%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 10085059958 11085278690 +9.92%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 25497010155 27018079806 +5.97%
BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 2427391406 2817217987 +16.06%
BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 2592965497 2538805050 -2.09%
BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 2437388343 2668012858 +9.46%
BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 2317095324 2787423966 +20.30%
BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 2600239857 2096973860 -19.35%
benchmark old allocs new allocs delta
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 500851 470794 -6.00%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 821527 791451 -3.66%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 1141562 1111508 -2.63%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 2141576 2111504 -1.40%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 871466 841424 -3.45%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 1941428 1911415 -1.55%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 3071573 3041510 -0.98%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 6771648 6741509 -0.45%
BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 731493 824888 +12.77%
BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 793918 887311 +11.76%
BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 811842 905204 +11.50%
BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 832244 925081 +11.16%
BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 921553 1019162 +10.59%
benchmark old bytes new bytes delta
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 40532648 35698276 -11.93%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 60340216 53409568 -11.49%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 81087336 72065552 -11.13%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 142485576 120878544 -15.16%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 208661368 203831136 -2.31%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 347345904 340484696 -1.98%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 585185856 576244648 -1.53%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 1357641792 1358966528 +0.10%
BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 126486664 119666744 -5.39%
BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 122323192 115117224 -5.89%
BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 126404504 119469864 -5.49%
BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 119047832 112230408 -5.73%
BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 136576016 116634800 -14.60%
Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2019-12-17 11:49:54 -08:00
|
|
|
AddSymbol(sym string) error
|
2017-11-30 06:34:49 -08:00
|
|
|
|
|
|
|
// AddSeries populates the index writer with a series and its offsets
|
|
|
|
// of chunks that the index can reference.
|
2020-07-31 08:03:02 -07:00
|
|
|
// Implementations may require series to be insert in strictly increasing order by
|
|
|
|
// their labels. The reference numbers are used to resolve entries in postings lists
|
|
|
|
// that are added later.
|
2021-11-06 03:10:04 -07:00
|
|
|
AddSeries(ref storage.SeriesRef, l labels.Labels, chunks ...chunks.Meta) error
|
2017-11-30 06:34:49 -08:00
|
|
|
|
|
|
|
// Close writes any finalization and closes the resources associated with
|
|
|
|
// the underlying writer.
|
|
|
|
Close() error
|
|
|
|
}
|
|
|
|
|
|
|
|
// IndexReader provides reading access of serialized index data.
|
|
|
|
type IndexReader interface {
|
Stream symbols during compaction. (#6468)
Rather than buffer up symbols in RAM, do it one by one
during compaction. Then use the reader's symbol handling
for symbol lookups during the rest of the index write.
There is some slowdown in compaction, due to having to look through a file
rather than a hash lookup. This is noise to the overall cost of compacting
series with thousands of samples though.
benchmark old ns/op new ns/op delta
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 539917175 675341565 +25.08%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 2441815993 2477453524 +1.46%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 3978543559 3922909687 -1.40%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 8430219716 8586610007 +1.86%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 1786424591 1909552782 +6.89%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 5328998202 6020839950 +12.98%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 10085059958 11085278690 +9.92%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 25497010155 27018079806 +5.97%
BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 2427391406 2817217987 +16.06%
BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 2592965497 2538805050 -2.09%
BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 2437388343 2668012858 +9.46%
BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 2317095324 2787423966 +20.30%
BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 2600239857 2096973860 -19.35%
benchmark old allocs new allocs delta
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 500851 470794 -6.00%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 821527 791451 -3.66%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 1141562 1111508 -2.63%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 2141576 2111504 -1.40%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 871466 841424 -3.45%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 1941428 1911415 -1.55%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 3071573 3041510 -0.98%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 6771648 6741509 -0.45%
BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 731493 824888 +12.77%
BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 793918 887311 +11.76%
BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 811842 905204 +11.50%
BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 832244 925081 +11.16%
BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 921553 1019162 +10.59%
benchmark old bytes new bytes delta
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 40532648 35698276 -11.93%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 60340216 53409568 -11.49%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 81087336 72065552 -11.13%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 142485576 120878544 -15.16%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 208661368 203831136 -2.31%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 347345904 340484696 -1.98%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 585185856 576244648 -1.53%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 1357641792 1358966528 +0.10%
BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 126486664 119666744 -5.39%
BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 122323192 115117224 -5.89%
BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 126404504 119469864 -5.49%
BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 119047832 112230408 -5.73%
BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 136576016 116634800 -14.60%
Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2019-12-17 11:49:54 -08:00
|
|
|
// Symbols return an iterator over sorted string symbols that may occur in
|
|
|
|
// series' labels and indices. It is not safe to use the returned strings
|
|
|
|
// beyond the lifetime of the index reader.
|
|
|
|
Symbols() index.StringIter
|
2017-11-30 06:34:49 -08:00
|
|
|
|
2020-06-25 06:10:29 -07:00
|
|
|
// SortedLabelValues returns sorted possible label values.
|
2021-02-09 09:38:35 -08:00
|
|
|
SortedLabelValues(name string, matchers ...*labels.Matcher) ([]string, error)
|
2020-06-25 06:10:29 -07:00
|
|
|
|
|
|
|
// LabelValues returns possible label values which may not be sorted.
|
2021-02-09 09:38:35 -08:00
|
|
|
LabelValues(name string, matchers ...*labels.Matcher) ([]string, error)
|
2017-11-30 06:34:49 -08:00
|
|
|
|
Reduce memory used by postings offset table.
Rather than keeping the offset of each postings list, instead
keep the nth offset of the offset of the posting list. As postings
list offsets have always been sorted, we can then get to the closest
entry before the one we want an iterate forwards.
I haven't done much tuning on the 32 number, it was chosen to try
not to read through more than a 4k page of data.
Switch to a bulk interface for fetching postings. Use it to avoid having
to re-read parts of the posting offset table when querying lots of it.
For a index with what BenchmarkHeadPostingForMatchers uses RAM
for r.postings drops from 3.79MB to 80.19kB or about 48x.
Bytes allocated go down by 30%, and suprisingly CPU usage drops by
4-6% for typical queries too.
benchmark old ns/op new ns/op delta
BenchmarkPostingsForMatchers/Block/n="1"-4 35231 36673 +4.09%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 563380 540627 -4.04%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 536782 534186 -0.48%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 533990 541550 +1.42%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 113374598 117969608 +4.05%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 146329884 139651442 -4.56%
BenchmarkPostingsForMatchers/Block/i=~""-4 50346510 44961127 -10.70%
BenchmarkPostingsForMatchers/Block/i!=""-4 41261550 35356165 -14.31%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112544418 116904010 +3.87%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 112487086 116864918 +3.89%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 41094758 35457904 -13.72%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 41906372 36151473 -13.73%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 147262414 140424800 -4.64%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 28615629 27872072 -2.60%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 147117177 140462403 -4.52%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 175096826 167902298 -4.11%
benchmark old allocs new allocs delta
BenchmarkPostingsForMatchers/Block/n="1"-4 4 6 +50.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 7 11 +57.14%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 7 11 +57.14%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 17 +13.33%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 100010 100012 +0.00%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 200069 200040 -0.01%
BenchmarkPostingsForMatchers/Block/i=~""-4 200072 200045 -0.01%
BenchmarkPostingsForMatchers/Block/i!=""-4 200070 200041 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 100013 100017 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 100017 100023 +0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 200073 200046 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 200075 200050 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 200074 200049 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 111165 111150 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 200078 200055 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 311282 311238 -0.01%
benchmark old bytes new bytes delta
BenchmarkPostingsForMatchers/Block/n="1"-4 264 296 +12.12%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 360 424 +17.78%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 360 424 +17.78%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 520 552 +6.15%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 1600461 1600482 +0.00%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 24900801 17259077 -30.69%
BenchmarkPostingsForMatchers/Block/i=~""-4 24900836 17259151 -30.69%
BenchmarkPostingsForMatchers/Block/i!=""-4 24900760 17259048 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1600557 1600621 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1600717 1600813 +0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 24900856 17259176 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 24900952 17259304 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 24900993 17259333 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3788311 3142630 -17.04%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 24901137 17259509 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 28693086 20405680 -28.88%
Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2019-12-05 10:27:40 -08:00
|
|
|
// Postings returns the postings list iterator for the label pairs.
|
2017-11-30 06:34:49 -08:00
|
|
|
// The Postings here contain the offsets to the series inside the index.
|
Reduce memory used by postings offset table.
Rather than keeping the offset of each postings list, instead
keep the nth offset of the offset of the posting list. As postings
list offsets have always been sorted, we can then get to the closest
entry before the one we want an iterate forwards.
I haven't done much tuning on the 32 number, it was chosen to try
not to read through more than a 4k page of data.
Switch to a bulk interface for fetching postings. Use it to avoid having
to re-read parts of the posting offset table when querying lots of it.
For a index with what BenchmarkHeadPostingForMatchers uses RAM
for r.postings drops from 3.79MB to 80.19kB or about 48x.
Bytes allocated go down by 30%, and suprisingly CPU usage drops by
4-6% for typical queries too.
benchmark old ns/op new ns/op delta
BenchmarkPostingsForMatchers/Block/n="1"-4 35231 36673 +4.09%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 563380 540627 -4.04%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 536782 534186 -0.48%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 533990 541550 +1.42%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 113374598 117969608 +4.05%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 146329884 139651442 -4.56%
BenchmarkPostingsForMatchers/Block/i=~""-4 50346510 44961127 -10.70%
BenchmarkPostingsForMatchers/Block/i!=""-4 41261550 35356165 -14.31%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112544418 116904010 +3.87%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 112487086 116864918 +3.89%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 41094758 35457904 -13.72%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 41906372 36151473 -13.73%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 147262414 140424800 -4.64%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 28615629 27872072 -2.60%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 147117177 140462403 -4.52%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 175096826 167902298 -4.11%
benchmark old allocs new allocs delta
BenchmarkPostingsForMatchers/Block/n="1"-4 4 6 +50.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 7 11 +57.14%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 7 11 +57.14%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 17 +13.33%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 100010 100012 +0.00%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 200069 200040 -0.01%
BenchmarkPostingsForMatchers/Block/i=~""-4 200072 200045 -0.01%
BenchmarkPostingsForMatchers/Block/i!=""-4 200070 200041 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 100013 100017 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 100017 100023 +0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 200073 200046 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 200075 200050 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 200074 200049 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 111165 111150 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 200078 200055 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 311282 311238 -0.01%
benchmark old bytes new bytes delta
BenchmarkPostingsForMatchers/Block/n="1"-4 264 296 +12.12%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 360 424 +17.78%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 360 424 +17.78%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 520 552 +6.15%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 1600461 1600482 +0.00%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 24900801 17259077 -30.69%
BenchmarkPostingsForMatchers/Block/i=~""-4 24900836 17259151 -30.69%
BenchmarkPostingsForMatchers/Block/i!=""-4 24900760 17259048 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1600557 1600621 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1600717 1600813 +0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 24900856 17259176 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 24900952 17259304 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 24900993 17259333 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3788311 3142630 -17.04%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 24901137 17259509 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 28693086 20405680 -28.88%
Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2019-12-05 10:27:40 -08:00
|
|
|
// Found IDs are not strictly required to point to a valid Series, e.g.
|
|
|
|
// during background garbage collections. Input values must be sorted.
|
|
|
|
Postings(name string, values ...string) (index.Postings, error)
|
2017-11-30 06:34:49 -08:00
|
|
|
|
|
|
|
// SortedPostings returns a postings list that is reordered to be sorted
|
|
|
|
// by the label set of the underlying series.
|
|
|
|
SortedPostings(index.Postings) index.Postings
|
|
|
|
|
2022-12-15 10:19:15 -08:00
|
|
|
// Series populates the given builder and chunk metas for the series identified
|
2017-11-30 06:34:49 -08:00
|
|
|
// by the reference.
|
2020-03-16 14:52:02 -07:00
|
|
|
// Returns storage.ErrNotFound if the ref does not resolve to a known series.
|
2022-12-15 10:19:15 -08:00
|
|
|
Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error
|
2017-11-30 06:34:49 -08:00
|
|
|
|
2018-11-07 07:52:41 -08:00
|
|
|
// LabelNames returns all the unique label names present in the index in sorted order.
|
2021-07-20 05:38:08 -07:00
|
|
|
LabelNames(matchers ...*labels.Matcher) ([]string, error)
|
2018-11-07 07:52:41 -08:00
|
|
|
|
2021-02-09 09:38:35 -08:00
|
|
|
// LabelValueFor returns label value for the given label name in the series referred to by ID.
|
|
|
|
// If the series couldn't be found or the series doesn't have the requested label a
|
|
|
|
// storage.ErrNotFound is returned as error.
|
2021-11-06 03:10:04 -07:00
|
|
|
LabelValueFor(id storage.SeriesRef, label string) (string, error)
|
2021-02-09 09:38:35 -08:00
|
|
|
|
2021-07-20 05:38:08 -07:00
|
|
|
// LabelNamesFor returns all the label names for the series referred to by IDs.
|
|
|
|
// The names returned are sorted.
|
2021-11-06 03:10:04 -07:00
|
|
|
LabelNamesFor(ids ...storage.SeriesRef) ([]string, error)
|
2021-07-20 05:38:08 -07:00
|
|
|
|
2017-12-22 00:43:34 -08:00
|
|
|
// Close releases the underlying resources of the reader.
|
2017-11-30 06:34:49 -08:00
|
|
|
Close() error
|
|
|
|
}
|
|
|
|
|
|
|
|
// ChunkWriter serializes a time block of chunked series data.
|
|
|
|
type ChunkWriter interface {
|
|
|
|
// WriteChunks writes several chunks. The Chunk field of the ChunkMetas
|
|
|
|
// must be populated.
|
|
|
|
// After returning successfully, the Ref fields in the ChunkMetas
|
|
|
|
// are set and can be used to retrieve the chunks from the written data.
|
|
|
|
WriteChunks(chunks ...chunks.Meta) error
|
|
|
|
|
|
|
|
// Close writes any required finalization and closes the resources
|
|
|
|
// associated with the underlying writer.
|
|
|
|
Close() error
|
|
|
|
}
|
|
|
|
|
|
|
|
// ChunkReader provides reading access of serialized time series data.
|
|
|
|
type ChunkReader interface {
|
|
|
|
// Chunk returns the series data chunk with the given reference.
|
2022-09-20 10:05:50 -07:00
|
|
|
Chunk(meta chunks.Meta) (chunkenc.Chunk, error)
|
2017-11-30 06:34:49 -08:00
|
|
|
|
|
|
|
// Close releases all underlying resources of the reader.
|
|
|
|
Close() error
|
|
|
|
}
|
|
|
|
|
2017-10-02 06:56:57 -07:00
|
|
|
// BlockReader provides reading access to a data block.
|
2017-08-28 15:39:17 -07:00
|
|
|
type BlockReader interface {
|
2020-03-25 12:13:47 -07:00
|
|
|
// Index returns an IndexReader over the block's data.
|
|
|
|
Index() (IndexReader, error)
|
2017-01-19 02:22:47 -08:00
|
|
|
|
2017-04-28 06:41:42 -07:00
|
|
|
// Chunks returns a ChunkReader over the block's data.
|
2017-10-09 06:21:46 -07:00
|
|
|
Chunks() (ChunkReader, error)
|
2017-01-19 02:22:47 -08:00
|
|
|
|
2019-09-19 02:15:41 -07:00
|
|
|
// Tombstones returns a tombstones.Reader over the block's deleted data.
|
|
|
|
Tombstones() (tombstones.Reader, error)
|
2019-02-14 05:29:41 -08:00
|
|
|
|
2019-07-23 01:04:48 -07:00
|
|
|
// Meta provides meta information about the block reader.
|
|
|
|
Meta() BlockMeta
|
2020-10-12 14:15:40 -07:00
|
|
|
|
|
|
|
// Size returns the number of bytes that the block takes up on disk.
|
|
|
|
Size() int64
|
2016-12-13 06:26:58 -08:00
|
|
|
}
|
2016-12-08 08:43:10 -08:00
|
|
|
|
2017-01-19 02:22:47 -08:00
|
|
|
// BlockMeta provides meta information about a block.
|
|
|
|
type BlockMeta struct {
|
2017-02-27 01:46:15 -08:00
|
|
|
// Unique identifier for the block and its contents. Changes on compaction.
|
|
|
|
ULID ulid.ULID `json:"ulid"`
|
|
|
|
|
2017-01-19 05:01:38 -08:00
|
|
|
// MinTime and MaxTime specify the time range all samples
|
2017-02-01 06:29:48 -08:00
|
|
|
// in the block are in.
|
|
|
|
MinTime int64 `json:"minTime"`
|
|
|
|
MaxTime int64 `json:"maxTime"`
|
2017-01-07 09:02:17 -08:00
|
|
|
|
2017-01-28 23:11:47 -08:00
|
|
|
// Stats about the contents of the block.
|
2017-06-07 00:52:20 -07:00
|
|
|
Stats BlockStats `json:"stats,omitempty"`
|
2017-01-19 10:45:52 -08:00
|
|
|
|
2017-01-28 23:11:47 -08:00
|
|
|
// Information on compactions the block was created from.
|
2017-06-07 00:52:20 -07:00
|
|
|
Compaction BlockMetaCompaction `json:"compaction"`
|
2018-01-08 09:33:35 -08:00
|
|
|
|
2018-01-10 23:50:42 -08:00
|
|
|
// Version of the index format.
|
2018-01-08 09:33:35 -08:00
|
|
|
Version int `json:"version"`
|
2017-06-07 00:52:20 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// BlockStats contains stats about contents of a block.
|
|
|
|
type BlockStats struct {
|
|
|
|
NumSamples uint64 `json:"numSamples,omitempty"`
|
|
|
|
NumSeries uint64 `json:"numSeries,omitempty"`
|
|
|
|
NumChunks uint64 `json:"numChunks,omitempty"`
|
|
|
|
NumTombstones uint64 `json:"numTombstones,omitempty"`
|
|
|
|
}
|
|
|
|
|
2018-06-27 06:47:11 -07:00
|
|
|
// BlockDesc describes a block by ULID and time range.
|
|
|
|
type BlockDesc struct {
|
|
|
|
ULID ulid.ULID `json:"ulid"`
|
|
|
|
MinTime int64 `json:"minTime"`
|
|
|
|
MaxTime int64 `json:"maxTime"`
|
|
|
|
}
|
|
|
|
|
2017-06-07 00:52:20 -07:00
|
|
|
// BlockMetaCompaction holds information about compactions a block went through.
|
|
|
|
type BlockMetaCompaction struct {
|
|
|
|
// Maximum number of compaction cycles any source block has
|
|
|
|
// gone through.
|
2017-08-09 02:10:29 -07:00
|
|
|
Level int `json:"level"`
|
2017-06-07 00:52:20 -07:00
|
|
|
// ULIDs of all source head blocks that went into the block.
|
|
|
|
Sources []ulid.ULID `json:"sources,omitempty"`
|
2019-01-18 00:35:16 -08:00
|
|
|
// Indicates that during compaction it resulted in a block without any samples
|
2020-10-19 08:27:08 -07:00
|
|
|
// so it should be deleted on the next reloadBlocks.
|
2019-01-18 00:35:16 -08:00
|
|
|
Deletable bool `json:"deletable,omitempty"`
|
2018-06-27 09:05:21 -07:00
|
|
|
// Short descriptions of the direct blocks that were used to create
|
|
|
|
// this block.
|
2018-06-27 06:47:11 -07:00
|
|
|
Parents []BlockDesc `json:"parents,omitempty"`
|
2017-11-21 03:15:02 -08:00
|
|
|
Failed bool `json:"failed,omitempty"`
|
2022-09-20 10:05:50 -07:00
|
|
|
// Additional information about the compaction, for example, block created from out-of-order chunks.
|
|
|
|
Hints []string `json:"hints,omitempty"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func (bm *BlockMetaCompaction) SetOutOfOrder() {
|
|
|
|
if bm.containsHint(CompactionHintFromOutOfOrder) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
bm.Hints = append(bm.Hints, CompactionHintFromOutOfOrder)
|
2022-10-01 09:55:40 -07:00
|
|
|
slices.Sort(bm.Hints)
|
2022-09-20 10:05:50 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
func (bm *BlockMetaCompaction) FromOutOfOrder() bool {
|
|
|
|
return bm.containsHint(CompactionHintFromOutOfOrder)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (bm *BlockMetaCompaction) containsHint(hint string) bool {
|
|
|
|
for _, h := range bm.Hints {
|
|
|
|
if h == hint {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
2016-12-19 13:29:49 -08:00
|
|
|
}
|
|
|
|
|
2021-10-22 01:06:44 -07:00
|
|
|
const (
|
|
|
|
indexFilename = "index"
|
|
|
|
metaFilename = "meta.json"
|
|
|
|
metaVersion1 = 1
|
2022-09-20 10:05:50 -07:00
|
|
|
|
|
|
|
// CompactionHintFromOutOfOrder is a hint noting that the block
|
|
|
|
// was created from out-of-order chunks.
|
|
|
|
CompactionHintFromOutOfOrder = "from-out-of-order"
|
2021-10-22 01:06:44 -07:00
|
|
|
)
|
2017-01-19 02:22:47 -08:00
|
|
|
|
2017-11-30 06:34:49 -08:00
|
|
|
func chunkDir(dir string) string { return filepath.Join(dir, "chunks") }
|
|
|
|
|
2019-06-24 08:42:29 -07:00
|
|
|
func readMetaFile(dir string) (*BlockMeta, int64, error) {
|
2022-04-27 02:24:36 -07:00
|
|
|
b, err := os.ReadFile(filepath.Join(dir, metaFilename))
|
2017-01-19 05:01:38 -08:00
|
|
|
if err != nil {
|
2019-06-24 08:42:29 -07:00
|
|
|
return nil, 0, err
|
2017-01-19 05:01:38 -08:00
|
|
|
}
|
2018-01-08 09:33:35 -08:00
|
|
|
var m BlockMeta
|
2017-01-19 05:01:38 -08:00
|
|
|
|
|
|
|
if err := json.Unmarshal(b, &m); err != nil {
|
2019-06-24 08:42:29 -07:00
|
|
|
return nil, 0, err
|
2017-01-19 05:01:38 -08:00
|
|
|
}
|
2019-09-09 02:28:01 -07:00
|
|
|
if m.Version != metaVersion1 {
|
2019-06-24 08:42:29 -07:00
|
|
|
return nil, 0, errors.Errorf("unexpected meta file version %d", m.Version)
|
2017-01-19 05:01:38 -08:00
|
|
|
}
|
|
|
|
|
2019-06-24 08:42:29 -07:00
|
|
|
return &m, int64(len(b)), nil
|
2017-01-19 05:01:38 -08:00
|
|
|
}
|
|
|
|
|
2019-06-24 08:42:29 -07:00
|
|
|
func writeMetaFile(logger log.Logger, dir string, meta *BlockMeta) (int64, error) {
|
2019-09-09 02:28:01 -07:00
|
|
|
meta.Version = metaVersion1
|
2018-02-09 04:11:03 -08:00
|
|
|
|
2017-03-01 08:19:57 -08:00
|
|
|
// Make any changes to the file appear atomic.
|
|
|
|
path := filepath.Join(dir, metaFilename)
|
|
|
|
tmp := path + ".tmp"
|
2019-04-26 01:27:36 -07:00
|
|
|
defer func() {
|
|
|
|
if err := os.RemoveAll(tmp); err != nil {
|
|
|
|
level.Error(logger).Log("msg", "remove tmp file", "err", err.Error())
|
|
|
|
}
|
|
|
|
}()
|
2017-03-01 08:19:57 -08:00
|
|
|
|
|
|
|
f, err := os.Create(tmp)
|
2017-01-19 05:01:38 -08:00
|
|
|
if err != nil {
|
2019-06-24 08:42:29 -07:00
|
|
|
return 0, err
|
2017-01-19 05:01:38 -08:00
|
|
|
}
|
|
|
|
|
2019-06-24 08:42:29 -07:00
|
|
|
jsonMeta, err := json.MarshalIndent(meta, "", "\t")
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
2017-01-19 05:01:38 -08:00
|
|
|
|
2019-06-24 08:42:29 -07:00
|
|
|
n, err := f.Write(jsonMeta)
|
|
|
|
if err != nil {
|
2020-10-28 08:24:58 -07:00
|
|
|
return 0, tsdb_errors.NewMulti(err, f.Close()).Err()
|
2017-01-19 05:01:38 -08:00
|
|
|
}
|
2019-06-24 08:42:29 -07:00
|
|
|
|
2019-04-01 06:23:45 -07:00
|
|
|
// Force the kernel to persist the file on disk to avoid data loss if the host crashes.
|
2019-06-24 08:42:29 -07:00
|
|
|
if err := f.Sync(); err != nil {
|
2020-10-28 08:24:58 -07:00
|
|
|
return 0, tsdb_errors.NewMulti(err, f.Close()).Err()
|
2019-04-01 06:23:45 -07:00
|
|
|
}
|
2017-01-19 05:01:38 -08:00
|
|
|
if err := f.Close(); err != nil {
|
2019-06-24 08:42:29 -07:00
|
|
|
return 0, err
|
2017-01-19 05:01:38 -08:00
|
|
|
}
|
2019-06-24 08:42:29 -07:00
|
|
|
return int64(n), fileutil.Replace(tmp, path)
|
2017-01-19 05:01:38 -08:00
|
|
|
}
|
|
|
|
|
2017-10-25 15:12:13 -07:00
|
|
|
// Block represents a directory of time series data covering a continuous time range.
|
2017-10-09 06:21:46 -07:00
|
|
|
type Block struct {
|
|
|
|
mtx sync.RWMutex
|
|
|
|
closing bool
|
|
|
|
pendingReaders sync.WaitGroup
|
|
|
|
|
2017-03-20 02:21:21 -07:00
|
|
|
dir string
|
|
|
|
meta BlockMeta
|
|
|
|
|
2018-09-08 11:28:36 -07:00
|
|
|
// Symbol Table Size in bytes.
|
2019-09-30 08:54:55 -07:00
|
|
|
// We maintain this variable to avoid recalculation every time.
|
2018-09-08 11:28:36 -07:00
|
|
|
symbolTableSize uint64
|
|
|
|
|
2017-11-13 04:32:24 -08:00
|
|
|
chunkr ChunkReader
|
|
|
|
indexr IndexReader
|
2019-09-19 02:15:41 -07:00
|
|
|
tombstones tombstones.Reader
|
2019-04-26 01:27:36 -07:00
|
|
|
|
|
|
|
logger log.Logger
|
2019-06-24 08:42:29 -07:00
|
|
|
|
|
|
|
numBytesChunks int64
|
|
|
|
numBytesIndex int64
|
|
|
|
numBytesTombstone int64
|
|
|
|
numBytesMeta int64
|
2017-03-20 02:21:21 -07:00
|
|
|
}
|
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
// OpenBlock opens the block in the directory. It can be passed a chunk pool, which is used
|
|
|
|
// to instantiate chunk structs.
|
2019-01-30 01:40:40 -08:00
|
|
|
func OpenBlock(logger log.Logger, dir string, pool chunkenc.Pool) (pb *Block, err error) {
|
2019-01-16 02:03:52 -08:00
|
|
|
if logger == nil {
|
|
|
|
logger = log.NewNopLogger()
|
|
|
|
}
|
2019-02-08 03:35:32 -08:00
|
|
|
var closers []io.Closer
|
|
|
|
defer func() {
|
|
|
|
if err != nil {
|
2020-10-28 08:24:58 -07:00
|
|
|
err = tsdb_errors.NewMulti(err, tsdb_errors.CloseAll(closers)).Err()
|
2019-02-08 03:35:32 -08:00
|
|
|
}
|
|
|
|
}()
|
2019-06-24 08:42:29 -07:00
|
|
|
meta, sizeMeta, err := readMetaFile(dir)
|
2017-01-19 10:45:52 -08:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2016-12-14 23:31:26 -08:00
|
|
|
|
2017-11-30 06:34:49 -08:00
|
|
|
cr, err := chunks.NewDirReader(chunkDir(dir), pool)
|
2016-12-14 23:31:26 -08:00
|
|
|
if err != nil {
|
2017-02-23 01:50:22 -08:00
|
|
|
return nil, err
|
2016-12-14 23:31:26 -08:00
|
|
|
}
|
2019-02-08 03:35:32 -08:00
|
|
|
closers = append(closers, cr)
|
|
|
|
|
2019-01-29 00:32:32 -08:00
|
|
|
ir, err := index.NewFileReader(filepath.Join(dir, indexFilename))
|
2016-12-14 23:31:26 -08:00
|
|
|
if err != nil {
|
2017-02-24 22:24:20 -08:00
|
|
|
return nil, err
|
2016-12-14 23:31:26 -08:00
|
|
|
}
|
2019-02-08 03:35:32 -08:00
|
|
|
closers = append(closers, ir)
|
2019-01-30 01:40:40 -08:00
|
|
|
|
2019-09-19 02:15:41 -07:00
|
|
|
tr, sizeTomb, err := tombstones.ReadTombstones(dir)
|
2017-05-16 07:18:28 -07:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2019-02-08 03:35:32 -08:00
|
|
|
closers = append(closers, tr)
|
2017-05-18 23:22:15 -07:00
|
|
|
|
2019-01-30 01:40:40 -08:00
|
|
|
pb = &Block{
|
2019-06-24 08:42:29 -07:00
|
|
|
dir: dir,
|
|
|
|
meta: *meta,
|
|
|
|
chunkr: cr,
|
|
|
|
indexr: ir,
|
|
|
|
tombstones: tr,
|
|
|
|
symbolTableSize: ir.SymbolTableSize(),
|
|
|
|
logger: logger,
|
|
|
|
numBytesChunks: cr.Size(),
|
|
|
|
numBytesIndex: ir.Size(),
|
|
|
|
numBytesTombstone: sizeTomb,
|
|
|
|
numBytesMeta: sizeMeta,
|
2016-12-14 23:31:26 -08:00
|
|
|
}
|
|
|
|
return pb, nil
|
|
|
|
}
|
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
// Close closes the on-disk block. It blocks as long as there are readers reading from the block.
|
|
|
|
func (pb *Block) Close() error {
|
|
|
|
pb.mtx.Lock()
|
|
|
|
pb.closing = true
|
|
|
|
pb.mtx.Unlock()
|
|
|
|
|
|
|
|
pb.pendingReaders.Wait()
|
|
|
|
|
2020-10-28 08:24:58 -07:00
|
|
|
return tsdb_errors.NewMulti(
|
|
|
|
pb.chunkr.Close(),
|
|
|
|
pb.indexr.Close(),
|
|
|
|
pb.tombstones.Close(),
|
|
|
|
).Err()
|
2016-12-14 23:31:26 -08:00
|
|
|
}
|
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
func (pb *Block) String() string {
|
2017-05-18 07:09:30 -07:00
|
|
|
return pb.meta.ULID.String()
|
2017-03-20 02:41:43 -07:00
|
|
|
}
|
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
// Dir returns the directory of the block.
|
|
|
|
func (pb *Block) Dir() string { return pb.dir }
|
|
|
|
|
|
|
|
// Meta returns meta information about the block.
|
|
|
|
func (pb *Block) Meta() BlockMeta { return pb.meta }
|
|
|
|
|
2019-02-14 05:29:41 -08:00
|
|
|
// MinTime returns the min time of the meta.
|
|
|
|
func (pb *Block) MinTime() int64 { return pb.meta.MinTime }
|
|
|
|
|
|
|
|
// MaxTime returns the max time of the meta.
|
|
|
|
func (pb *Block) MaxTime() int64 { return pb.meta.MaxTime }
|
|
|
|
|
2019-01-16 02:03:52 -08:00
|
|
|
// Size returns the number of bytes that the block takes up.
|
2019-06-24 08:42:29 -07:00
|
|
|
func (pb *Block) Size() int64 {
|
|
|
|
return pb.numBytesChunks + pb.numBytesIndex + pb.numBytesTombstone + pb.numBytesMeta
|
|
|
|
}
|
2019-01-16 02:03:52 -08:00
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
// ErrClosing is returned when a block is in the process of being closed.
|
|
|
|
var ErrClosing = errors.New("block is closing")
|
|
|
|
|
|
|
|
func (pb *Block) startRead() error {
|
|
|
|
pb.mtx.RLock()
|
|
|
|
defer pb.mtx.RUnlock()
|
|
|
|
|
|
|
|
if pb.closing {
|
|
|
|
return ErrClosing
|
|
|
|
}
|
|
|
|
pb.pendingReaders.Add(1)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Index returns a new IndexReader against the block data.
|
2020-03-25 12:13:47 -07:00
|
|
|
func (pb *Block) Index() (IndexReader, error) {
|
2017-10-09 06:21:46 -07:00
|
|
|
if err := pb.startRead(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2018-01-16 09:22:50 -08:00
|
|
|
return blockIndexReader{ir: pb.indexr, b: pb}, nil
|
2017-10-09 06:21:46 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Chunks returns a new ChunkReader against the block data.
|
|
|
|
func (pb *Block) Chunks() (ChunkReader, error) {
|
|
|
|
if err := pb.startRead(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return blockChunkReader{ChunkReader: pb.chunkr, b: pb}, nil
|
2017-05-16 07:18:28 -07:00
|
|
|
}
|
2016-12-15 07:14:33 -08:00
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
// Tombstones returns a new TombstoneReader against the block data.
|
2019-09-19 02:15:41 -07:00
|
|
|
func (pb *Block) Tombstones() (tombstones.Reader, error) {
|
2017-10-09 06:21:46 -07:00
|
|
|
if err := pb.startRead(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2019-09-19 02:15:41 -07:00
|
|
|
return blockTombstoneReader{Reader: pb.tombstones, b: pb}, nil
|
2017-10-09 06:21:46 -07:00
|
|
|
}
|
|
|
|
|
2018-09-08 11:28:36 -07:00
|
|
|
// GetSymbolTableSize returns the Symbol Table Size in the index of this block.
|
|
|
|
func (pb *Block) GetSymbolTableSize() uint64 {
|
|
|
|
return pb.symbolTableSize
|
|
|
|
}
|
|
|
|
|
2017-11-21 03:15:02 -08:00
|
|
|
func (pb *Block) setCompactionFailed() error {
|
|
|
|
pb.meta.Compaction.Failed = true
|
2019-06-24 08:42:29 -07:00
|
|
|
n, err := writeMetaFile(pb.logger, pb.dir, &pb.meta)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
pb.numBytesMeta = n
|
|
|
|
return nil
|
2017-11-21 03:15:02 -08:00
|
|
|
}
|
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
type blockIndexReader struct {
|
2018-01-16 09:22:50 -08:00
|
|
|
ir IndexReader
|
|
|
|
b *Block
|
|
|
|
}
|
|
|
|
|
Stream symbols during compaction. (#6468)
Rather than buffer up symbols in RAM, do it one by one
during compaction. Then use the reader's symbol handling
for symbol lookups during the rest of the index write.
There is some slowdown in compaction, due to having to look through a file
rather than a hash lookup. This is noise to the overall cost of compacting
series with thousands of samples though.
benchmark old ns/op new ns/op delta
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 539917175 675341565 +25.08%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 2441815993 2477453524 +1.46%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 3978543559 3922909687 -1.40%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 8430219716 8586610007 +1.86%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 1786424591 1909552782 +6.89%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 5328998202 6020839950 +12.98%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 10085059958 11085278690 +9.92%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 25497010155 27018079806 +5.97%
BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 2427391406 2817217987 +16.06%
BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 2592965497 2538805050 -2.09%
BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 2437388343 2668012858 +9.46%
BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 2317095324 2787423966 +20.30%
BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 2600239857 2096973860 -19.35%
benchmark old allocs new allocs delta
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 500851 470794 -6.00%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 821527 791451 -3.66%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 1141562 1111508 -2.63%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 2141576 2111504 -1.40%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 871466 841424 -3.45%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 1941428 1911415 -1.55%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 3071573 3041510 -0.98%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 6771648 6741509 -0.45%
BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 731493 824888 +12.77%
BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 793918 887311 +11.76%
BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 811842 905204 +11.50%
BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 832244 925081 +11.16%
BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 921553 1019162 +10.59%
benchmark old bytes new bytes delta
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 40532648 35698276 -11.93%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 60340216 53409568 -11.49%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 81087336 72065552 -11.13%
BenchmarkCompaction/type=normal,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 142485576 120878544 -15.16%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=101-4 208661368 203831136 -2.31%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=1001-4 347345904 340484696 -1.98%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=2001-4 585185856 576244648 -1.53%
BenchmarkCompaction/type=vertical,blocks=4,series=10000,samplesPerSeriesPerBlock=5001-4 1357641792 1358966528 +0.10%
BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 126486664 119666744 -5.39%
BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 122323192 115117224 -5.89%
BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 126404504 119469864 -5.49%
BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 119047832 112230408 -5.73%
BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 136576016 116634800 -14.60%
Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2019-12-17 11:49:54 -08:00
|
|
|
func (r blockIndexReader) Symbols() index.StringIter {
|
|
|
|
return r.ir.Symbols()
|
2018-01-16 09:22:50 -08:00
|
|
|
}
|
|
|
|
|
2021-02-09 09:38:35 -08:00
|
|
|
func (r blockIndexReader) SortedLabelValues(name string, matchers ...*labels.Matcher) ([]string, error) {
|
|
|
|
var st []string
|
|
|
|
var err error
|
|
|
|
|
|
|
|
if len(matchers) == 0 {
|
|
|
|
st, err = r.ir.SortedLabelValues(name)
|
|
|
|
} else {
|
|
|
|
st, err = r.LabelValues(name, matchers...)
|
|
|
|
if err == nil {
|
2022-09-30 07:33:56 -07:00
|
|
|
slices.Sort(st)
|
2021-02-09 09:38:35 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-25 06:10:29 -07:00
|
|
|
return st, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
|
|
|
|
}
|
|
|
|
|
2021-02-09 09:38:35 -08:00
|
|
|
func (r blockIndexReader) LabelValues(name string, matchers ...*labels.Matcher) ([]string, error) {
|
|
|
|
if len(matchers) == 0 {
|
|
|
|
st, err := r.ir.LabelValues(name)
|
|
|
|
return st, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
|
|
|
|
}
|
|
|
|
|
2021-07-20 05:38:08 -07:00
|
|
|
return labelValuesWithMatchers(r.ir, name, matchers...)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (r blockIndexReader) LabelNames(matchers ...*labels.Matcher) ([]string, error) {
|
|
|
|
if len(matchers) == 0 {
|
|
|
|
return r.b.LabelNames()
|
|
|
|
}
|
|
|
|
|
|
|
|
return labelNamesWithMatchers(r.ir, matchers...)
|
2018-01-16 09:22:50 -08:00
|
|
|
}
|
|
|
|
|
Reduce memory used by postings offset table.
Rather than keeping the offset of each postings list, instead
keep the nth offset of the offset of the posting list. As postings
list offsets have always been sorted, we can then get to the closest
entry before the one we want an iterate forwards.
I haven't done much tuning on the 32 number, it was chosen to try
not to read through more than a 4k page of data.
Switch to a bulk interface for fetching postings. Use it to avoid having
to re-read parts of the posting offset table when querying lots of it.
For a index with what BenchmarkHeadPostingForMatchers uses RAM
for r.postings drops from 3.79MB to 80.19kB or about 48x.
Bytes allocated go down by 30%, and suprisingly CPU usage drops by
4-6% for typical queries too.
benchmark old ns/op new ns/op delta
BenchmarkPostingsForMatchers/Block/n="1"-4 35231 36673 +4.09%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 563380 540627 -4.04%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 536782 534186 -0.48%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 533990 541550 +1.42%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 113374598 117969608 +4.05%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 146329884 139651442 -4.56%
BenchmarkPostingsForMatchers/Block/i=~""-4 50346510 44961127 -10.70%
BenchmarkPostingsForMatchers/Block/i!=""-4 41261550 35356165 -14.31%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112544418 116904010 +3.87%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 112487086 116864918 +3.89%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 41094758 35457904 -13.72%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 41906372 36151473 -13.73%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 147262414 140424800 -4.64%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 28615629 27872072 -2.60%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 147117177 140462403 -4.52%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 175096826 167902298 -4.11%
benchmark old allocs new allocs delta
BenchmarkPostingsForMatchers/Block/n="1"-4 4 6 +50.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 7 11 +57.14%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 7 11 +57.14%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 17 +13.33%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 100010 100012 +0.00%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 200069 200040 -0.01%
BenchmarkPostingsForMatchers/Block/i=~""-4 200072 200045 -0.01%
BenchmarkPostingsForMatchers/Block/i!=""-4 200070 200041 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 100013 100017 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 100017 100023 +0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 200073 200046 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 200075 200050 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 200074 200049 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 111165 111150 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 200078 200055 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 311282 311238 -0.01%
benchmark old bytes new bytes delta
BenchmarkPostingsForMatchers/Block/n="1"-4 264 296 +12.12%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 360 424 +17.78%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 360 424 +17.78%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 520 552 +6.15%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 1600461 1600482 +0.00%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 24900801 17259077 -30.69%
BenchmarkPostingsForMatchers/Block/i=~""-4 24900836 17259151 -30.69%
BenchmarkPostingsForMatchers/Block/i!=""-4 24900760 17259048 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1600557 1600621 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1600717 1600813 +0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 24900856 17259176 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 24900952 17259304 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 24900993 17259333 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3788311 3142630 -17.04%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 24901137 17259509 -30.69%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 28693086 20405680 -28.88%
Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2019-12-05 10:27:40 -08:00
|
|
|
func (r blockIndexReader) Postings(name string, values ...string) (index.Postings, error) {
|
|
|
|
p, err := r.ir.Postings(name, values...)
|
2019-07-02 03:25:29 -07:00
|
|
|
if err != nil {
|
|
|
|
return p, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
|
|
|
|
}
|
|
|
|
return p, nil
|
2018-01-16 09:22:50 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
func (r blockIndexReader) SortedPostings(p index.Postings) index.Postings {
|
|
|
|
return r.ir.SortedPostings(p)
|
|
|
|
}
|
|
|
|
|
2022-12-15 10:19:15 -08:00
|
|
|
func (r blockIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error {
|
|
|
|
if err := r.ir.Series(ref, builder, chks); err != nil {
|
2019-07-02 03:25:29 -07:00
|
|
|
return errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
|
|
|
|
}
|
|
|
|
return nil
|
2018-01-16 09:22:50 -08:00
|
|
|
}
|
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
func (r blockIndexReader) Close() error {
|
|
|
|
r.b.pendingReaders.Done()
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-02-09 09:38:35 -08:00
|
|
|
// LabelValueFor returns label value for the given label name in the series referred to by ID.
|
2021-11-06 03:10:04 -07:00
|
|
|
func (r blockIndexReader) LabelValueFor(id storage.SeriesRef, label string) (string, error) {
|
2021-02-09 09:38:35 -08:00
|
|
|
return r.ir.LabelValueFor(id, label)
|
|
|
|
}
|
|
|
|
|
2021-07-20 05:38:08 -07:00
|
|
|
// LabelNamesFor returns all the label names for the series referred to by IDs.
|
|
|
|
// The names returned are sorted.
|
2021-11-06 03:10:04 -07:00
|
|
|
func (r blockIndexReader) LabelNamesFor(ids ...storage.SeriesRef) ([]string, error) {
|
2021-07-20 05:38:08 -07:00
|
|
|
return r.ir.LabelNamesFor(ids...)
|
|
|
|
}
|
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
type blockTombstoneReader struct {
|
2019-09-19 02:15:41 -07:00
|
|
|
tombstones.Reader
|
2017-10-09 06:21:46 -07:00
|
|
|
b *Block
|
|
|
|
}
|
|
|
|
|
|
|
|
func (r blockTombstoneReader) Close() error {
|
|
|
|
r.b.pendingReaders.Done()
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
type blockChunkReader struct {
|
|
|
|
ChunkReader
|
|
|
|
b *Block
|
|
|
|
}
|
|
|
|
|
|
|
|
func (r blockChunkReader) Close() error {
|
|
|
|
r.b.pendingReaders.Done()
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Delete matching series between mint and maxt in the block.
|
2019-11-18 11:53:33 -08:00
|
|
|
func (pb *Block) Delete(mint, maxt int64, ms ...*labels.Matcher) error {
|
2017-10-09 06:21:46 -07:00
|
|
|
pb.mtx.Lock()
|
|
|
|
defer pb.mtx.Unlock()
|
|
|
|
|
|
|
|
if pb.closing {
|
|
|
|
return ErrClosing
|
|
|
|
}
|
|
|
|
|
2017-12-17 10:08:21 -08:00
|
|
|
p, err := PostingsForMatchers(pb.indexr, ms...)
|
2017-11-13 03:16:58 -08:00
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "select series")
|
|
|
|
}
|
2017-05-14 02:06:26 -07:00
|
|
|
|
|
|
|
ir := pb.indexr
|
|
|
|
|
|
|
|
// Choose only valid postings which have chunks in the time-range.
|
2019-09-19 02:15:41 -07:00
|
|
|
stones := tombstones.NewMemTombstones()
|
2017-05-14 02:06:26 -07:00
|
|
|
|
2017-11-30 06:34:49 -08:00
|
|
|
var chks []chunks.Meta
|
2022-06-28 08:03:26 -07:00
|
|
|
var builder labels.ScratchBuilder
|
2017-08-05 04:31:48 -07:00
|
|
|
|
2017-05-14 02:06:26 -07:00
|
|
|
Outer:
|
|
|
|
for p.Next() {
|
2022-12-15 10:19:15 -08:00
|
|
|
err := ir.Series(p.At(), &builder, &chks)
|
2017-05-15 10:58:14 -07:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-05-14 02:06:26 -07:00
|
|
|
|
2017-08-05 04:31:48 -07:00
|
|
|
for _, chk := range chks {
|
2018-07-02 01:23:36 -07:00
|
|
|
if chk.OverlapsClosedInterval(mint, maxt) {
|
2018-02-07 05:43:21 -08:00
|
|
|
// Delete only until the current values and not beyond.
|
2017-08-05 04:31:48 -07:00
|
|
|
tmin, tmax := clampInterval(mint, maxt, chks[0].MinTime, chks[len(chks)-1].MaxTime)
|
2019-09-19 02:15:41 -07:00
|
|
|
stones.AddInterval(p.At(), tombstones.Interval{Mint: tmin, Maxt: tmax})
|
2017-05-15 10:58:14 -07:00
|
|
|
continue Outer
|
2017-05-14 02:06:26 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if p.Err() != nil {
|
|
|
|
return p.Err()
|
|
|
|
}
|
|
|
|
|
2021-11-06 03:10:04 -07:00
|
|
|
err = pb.tombstones.Iter(func(id storage.SeriesRef, ivs tombstones.Intervals) error {
|
2017-11-13 04:32:24 -08:00
|
|
|
for _, iv := range ivs {
|
2019-09-19 02:15:41 -07:00
|
|
|
stones.AddInterval(id, iv)
|
2017-11-13 04:32:24 -08:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return err
|
2017-05-23 22:54:24 -07:00
|
|
|
}
|
2017-11-13 04:32:24 -08:00
|
|
|
pb.tombstones = stones
|
2018-09-27 04:43:22 -07:00
|
|
|
pb.meta.Stats.NumTombstones = pb.tombstones.Total()
|
2017-05-14 02:06:26 -07:00
|
|
|
|
2019-09-19 02:15:41 -07:00
|
|
|
n, err := tombstones.WriteFile(pb.logger, pb.dir, pb.tombstones)
|
2019-06-24 08:42:29 -07:00
|
|
|
if err != nil {
|
2017-05-23 05:07:04 -07:00
|
|
|
return err
|
|
|
|
}
|
2019-06-24 08:42:29 -07:00
|
|
|
pb.numBytesTombstone = n
|
|
|
|
n, err = writeMetaFile(pb.logger, pb.dir, &pb.meta)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
pb.numBytesMeta = n
|
|
|
|
return nil
|
2017-05-15 10:58:14 -07:00
|
|
|
}
|
2017-05-14 02:06:26 -07:00
|
|
|
|
2018-05-30 19:09:30 -07:00
|
|
|
// CleanTombstones will remove the tombstones and rewrite the block (only if there are any tombstones).
|
|
|
|
// If there was a rewrite, then it returns the ULID of the new block written, else nil.
|
2021-02-16 21:32:43 -08:00
|
|
|
// If the resultant block is empty (tombstones covered the whole block), then it deletes the new block and return nil UID.
|
|
|
|
// It returns a boolean indicating if the parent block can be deleted safely of not.
|
|
|
|
func (pb *Block) CleanTombstones(dest string, c Compactor) (*ulid.ULID, bool, error) {
|
2017-11-29 23:45:23 -08:00
|
|
|
numStones := 0
|
|
|
|
|
2021-11-06 03:10:04 -07:00
|
|
|
if err := pb.tombstones.Iter(func(id storage.SeriesRef, ivs tombstones.Intervals) error {
|
2018-04-03 05:47:02 -07:00
|
|
|
numStones += len(ivs)
|
2017-11-29 23:45:23 -08:00
|
|
|
return nil
|
2018-09-20 01:33:52 -07:00
|
|
|
}); err != nil {
|
|
|
|
// This should never happen, as the iteration function only returns nil.
|
|
|
|
panic(err)
|
|
|
|
}
|
2017-11-29 23:45:23 -08:00
|
|
|
if numStones == 0 {
|
2021-02-16 21:32:43 -08:00
|
|
|
return nil, false, nil
|
2017-11-22 04:34:50 -08:00
|
|
|
}
|
|
|
|
|
2018-06-27 06:47:11 -07:00
|
|
|
meta := pb.Meta()
|
|
|
|
uid, err := c.Write(dest, pb, pb.meta.MinTime, pb.meta.MaxTime, &meta)
|
2018-05-30 19:09:30 -07:00
|
|
|
if err != nil {
|
2021-02-16 21:32:43 -08:00
|
|
|
return nil, false, err
|
2017-11-22 04:34:50 -08:00
|
|
|
}
|
2021-02-16 21:32:43 -08:00
|
|
|
|
|
|
|
return &uid, true, nil
|
2017-11-22 04:34:50 -08:00
|
|
|
}
|
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
// Snapshot creates snapshot of the block into dir.
|
|
|
|
func (pb *Block) Snapshot(dir string) error {
|
2017-06-06 05:45:54 -07:00
|
|
|
blockDir := filepath.Join(dir, pb.meta.ULID.String())
|
2021-10-22 01:06:44 -07:00
|
|
|
if err := os.MkdirAll(blockDir, 0o777); err != nil {
|
2017-06-06 05:45:54 -07:00
|
|
|
return errors.Wrap(err, "create snapshot block dir")
|
|
|
|
}
|
|
|
|
|
|
|
|
chunksDir := chunkDir(blockDir)
|
2021-10-22 01:06:44 -07:00
|
|
|
if err := os.MkdirAll(chunksDir, 0o777); err != nil {
|
2017-06-06 05:45:54 -07:00
|
|
|
return errors.Wrap(err, "create snapshot chunk dir")
|
|
|
|
}
|
|
|
|
|
|
|
|
// Hardlink meta, index and tombstones
|
2017-06-06 07:53:20 -07:00
|
|
|
for _, fname := range []string{
|
|
|
|
metaFilename,
|
|
|
|
indexFilename,
|
2019-09-19 02:15:41 -07:00
|
|
|
tombstones.TombstonesFilename,
|
2017-06-06 07:53:20 -07:00
|
|
|
} {
|
2017-06-06 05:45:54 -07:00
|
|
|
if err := os.Link(filepath.Join(pb.dir, fname), filepath.Join(blockDir, fname)); err != nil {
|
|
|
|
return errors.Wrapf(err, "create snapshot %s", fname)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Hardlink the chunks
|
|
|
|
curChunkDir := chunkDir(pb.dir)
|
2022-04-27 02:24:36 -07:00
|
|
|
files, err := os.ReadDir(curChunkDir)
|
2017-06-06 05:45:54 -07:00
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "ReadDir the current chunk dir")
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, f := range files {
|
|
|
|
err := os.Link(filepath.Join(curChunkDir, f.Name()), filepath.Join(chunksDir, f.Name()))
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "hardlink a chunk")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-11-14 08:40:01 -08:00
|
|
|
// OverlapsClosedInterval returns true if the block overlaps [mint, maxt].
|
2018-07-02 01:23:36 -07:00
|
|
|
func (pb *Block) OverlapsClosedInterval(mint, maxt int64) bool {
|
|
|
|
// The block itself is a half-open interval
|
|
|
|
// [pb.meta.MinTime, pb.meta.MaxTime).
|
|
|
|
return pb.meta.MinTime <= maxt && mint < pb.meta.MaxTime
|
|
|
|
}
|
|
|
|
|
2018-11-07 07:52:41 -08:00
|
|
|
// LabelNames returns all the unique label names present in the Block in sorted order.
|
|
|
|
func (pb *Block) LabelNames() ([]string, error) {
|
|
|
|
return pb.indexr.LabelNames()
|
|
|
|
}
|
|
|
|
|
2017-05-26 08:56:31 -07:00
|
|
|
func clampInterval(a, b, mint, maxt int64) (int64, int64) {
|
|
|
|
if a < mint {
|
|
|
|
a = mint
|
|
|
|
}
|
|
|
|
if b > maxt {
|
|
|
|
b = maxt
|
|
|
|
}
|
|
|
|
return a, b
|
|
|
|
}
|