2015-01-21 11:07:45 -08:00
|
|
|
// Copyright 2014 The Prometheus Authors
|
2014-09-19 09:18:44 -07:00
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2014-09-16 06:47:24 -07:00
|
|
|
package local
|
2014-06-06 02:55:53 -07:00
|
|
|
|
|
|
|
import (
|
2016-02-19 09:35:30 -08:00
|
|
|
"time"
|
|
|
|
|
2014-10-07 10:11:24 -07:00
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
2015-08-20 08:18:46 -07:00
|
|
|
"github.com/prometheus/common/model"
|
2015-01-09 02:04:20 -08:00
|
|
|
|
2014-06-06 02:55:53 -07:00
|
|
|
"github.com/prometheus/prometheus/storage/metric"
|
|
|
|
)
|
|
|
|
|
2014-09-19 09:18:44 -07:00
|
|
|
// Storage ingests and manages samples, along with various indexes. All methods
|
2015-03-14 19:36:15 -07:00
|
|
|
// are goroutine-safe. Storage implements storage.SampleAppender.
|
2014-06-06 02:55:53 -07:00
|
|
|
type Storage interface {
|
2014-10-07 10:11:24 -07:00
|
|
|
prometheus.Collector
|
2015-03-14 19:36:15 -07:00
|
|
|
// Append stores a sample in the Storage. Multiple samples for the same
|
|
|
|
// fingerprint need to be submitted in chronological order, from oldest
|
|
|
|
// to newest. When Append has returned, the appended sample might not be
|
|
|
|
// queryable immediately. (Use WaitForIndexing to wait for complete
|
2015-06-23 06:58:43 -07:00
|
|
|
// processing.) The implementation might remove labels with empty value
|
|
|
|
// from the provided Sample as those labels are considered equivalent to
|
|
|
|
// a label not present at all.
|
2016-02-02 05:01:44 -08:00
|
|
|
Append(*model.Sample) error
|
2016-01-27 10:07:46 -08:00
|
|
|
// NeedsThrottling returns true if the Storage has too many chunks in memory
|
|
|
|
// already or has too many chunks waiting for persistence.
|
|
|
|
NeedsThrottling() bool
|
2014-06-06 02:55:53 -07:00
|
|
|
// NewPreloader returns a new Preloader which allows preloading and pinning
|
|
|
|
// series data into memory for use within a query.
|
|
|
|
NewPreloader() Preloader
|
2016-03-08 15:09:42 -08:00
|
|
|
// MetricsForLabelMatchers returns the metrics from storage that satisfy
|
|
|
|
// the given label matchers. At least one label matcher must be
|
|
|
|
// specified that does not match the empty string. The times from and
|
|
|
|
// through are hints for the storage to optimize the search. The storage
|
|
|
|
// MAY exclude metrics that have no samples in the specified interval
|
|
|
|
// from the returned map. In doubt, specify model.Earliest for from and
|
|
|
|
// model.Latest for through.
|
|
|
|
MetricsForLabelMatchers(from, through model.Time, matchers ...*metric.LabelMatcher) map[model.Fingerprint]metric.Metric
|
|
|
|
// LastSampleForFingerprint returns the last sample that has been
|
|
|
|
// ingested for the provided fingerprint. If this instance of the
|
2016-02-19 07:46:11 -08:00
|
|
|
// Storage has never ingested a sample for the provided fingerprint (or
|
|
|
|
// the last ingestion is so long ago that the series has been archived),
|
2016-03-08 15:09:42 -08:00
|
|
|
// ZeroSample is returned.
|
|
|
|
LastSampleForFingerprint(model.Fingerprint) model.Sample
|
2014-06-06 02:55:53 -07:00
|
|
|
// Get all of the label values that are associated with a given label name.
|
2015-08-20 08:18:46 -07:00
|
|
|
LabelValuesForLabelName(model.LabelName) model.LabelValues
|
2016-01-12 03:38:18 -08:00
|
|
|
// Drop all time series associated with the given fingerprints.
|
2015-08-20 08:18:46 -07:00
|
|
|
DropMetricsForFingerprints(...model.Fingerprint)
|
2014-10-24 11:27:27 -07:00
|
|
|
// Run the various maintenance loops in goroutines. Returns when the
|
|
|
|
// storage is ready to use. Keeps everything running in the background
|
2014-11-20 12:03:51 -08:00
|
|
|
// until Stop is called.
|
2015-05-18 10:26:28 -07:00
|
|
|
Start() error
|
2014-10-24 11:27:27 -07:00
|
|
|
// Stop shuts down the Storage gracefully, flushes all pending
|
|
|
|
// operations, stops all maintenance loops,and frees all resources.
|
|
|
|
Stop() error
|
2014-10-08 07:22:54 -07:00
|
|
|
// WaitForIndexing returns once all samples in the storage are
|
2015-05-20 10:13:06 -07:00
|
|
|
// indexed. Indexing is needed for FingerprintsForLabelMatchers and
|
|
|
|
// LabelValuesForLabelName and may lag behind.
|
2014-10-08 07:22:54 -07:00
|
|
|
WaitForIndexing()
|
2014-06-06 02:55:53 -07:00
|
|
|
}
|
|
|
|
|
2015-05-04 11:16:01 -07:00
|
|
|
// SeriesIterator enables efficient access of sample values in a series. Its
|
|
|
|
// methods are not goroutine-safe. A SeriesIterator iterates over a snapshot of
|
|
|
|
// a series, i.e. it is safe to continue using a SeriesIterator after or during
|
|
|
|
// modifying the corresponding series, but the iterator will represent the state
|
Handle errors caused by data corruption more gracefully
This requires all the panic calls upon unexpected data to be converted
into errors returned. This pollute the function signatures quite
lot. Well, this is Go...
The ideas behind this are the following:
- panic only if it's a programming error. Data corruptions happen, and
they are not programming errors.
- If we detect a data corruption, we "quarantine" the series,
essentially removing it from the database and putting its data into
a separate directory for forensics.
- Failure during writing to a series file is not considered corruption
automatically. It will call setDirty, though, so that a
crashrecovery upon the next restart will commence and check for
that.
- Series quarantining and setDirty calls are logged and counted in
metrics, but are hidden from the user of the interfaces in
interface.go, whith the notable exception of Append(). The reasoning
is that we treat corruption by removing the corrupted series, i.e. a
query for it will return no results on its next call anyway, so
return no results right now. In the case of Append(), we want to
tell the user that no data has been appended, though.
Minor side effects:
- Now consistently using filepath.* instead of path.*.
- Introduced structured logging where I touched it. This makes things
less consistent, but a complete change to structured logging would
be out of scope for this PR.
2016-02-25 03:23:42 -08:00
|
|
|
// of the series prior to the modification.
|
2014-06-06 02:55:53 -07:00
|
|
|
type SeriesIterator interface {
|
Streamline series iterator creation
This will fix issue #1035 and will also help to make issue #1264 less
bad.
The fundamental problem in the current code:
In the preload phase, we quite accurately determine which chunks will
be used for the query being executed. However, in the subsequent step
of creating series iterators, the created iterators are referencing
_all_ in-memory chunks in their series, even the un-pinned ones. In
iterator creation, we copy a pointer to each in-memory chunk of a
series into the iterator. While this creates a certain amount of
allocation churn, the worst thing about it is that copying the chunk
pointer out of the chunkDesc requires a mutex acquisition. (Remember
that the iterator will also reference un-pinned chunks, so we need to
acquire the mutex to protect against concurrent eviction.) The worst
case happens if a series doesn't even contain any relevant samples for
the query time range. We notice that during preloading but then we
will still create a series iterator for it. But even for series that
do contain relevant samples, the overhead is quite bad for instant
queries that retrieve a single sample from each series, but still go
through all the effort of series iterator creation. All of that is
particularly bad if a series has many in-memory chunks.
This commit addresses the problem from two sides:
First, it merges preloading and iterator creation into one step,
i.e. the preload call returns an iterator for exactly the preloaded
chunks.
Second, the required mutex acquisition in chunkDesc has been greatly
reduced. That was enabled by a side effect of the first step, which is
that the iterator is only referencing pinned chunks, so there is no
risk of concurrent eviction anymore, and chunks can be accessed
without mutex acquisition.
To simplify the code changes for the above, the long-planned change of
ValueAtTime to ValueAtOrBefore time was performed at the same
time. (It should have been done first, but it kind of accidentally
happened while I was in the middle of writing the series iterator
changes. Sorry for that.) So far, we actively filtered the up to two
values that were returned by ValueAtTime, i.e. we invested work to
retrieve up to two values, and then we invested more work to throw one
of them away.
The SeriesIterator.BoundaryValues method can be removed once #1401 is
fixed. But I really didn't want to load even more changes into this
PR.
Benchmarks:
The BenchmarkFuzz.* benchmarks run 83% faster (i.e. about six times
faster) and allocate 95% fewer bytes. The reason for that is that the
benchmark reads one sample after another from the time series and
creates a new series iterator for each sample read.
To find out how much these improvements matter in practice, I have
mirrored a beefy Prometheus server at SoundCloud that suffers from
both issues #1035 and #1264. To reach steady state that would be
comparable, the server needs to run for 15d. So far, it has run for
1d. The test server currently has only half as many memory time series
and 60% of the memory chunks the main server has. The 90th percentile
rule evaluation cycle time is ~11s on the main server and only ~3s on
the test server. However, these numbers might get much closer over
time.
In addition to performance improvements, this commit removes about 150
LOC.
2016-02-16 09:47:50 -08:00
|
|
|
// Gets the value that is closest before the given time. In case a value
|
2016-03-02 04:45:17 -08:00
|
|
|
// exists at precisely the given time, that value is returned. If no
|
2016-02-19 07:46:11 -08:00
|
|
|
// applicable value exists, ZeroSamplePair is returned.
|
Streamline series iterator creation
This will fix issue #1035 and will also help to make issue #1264 less
bad.
The fundamental problem in the current code:
In the preload phase, we quite accurately determine which chunks will
be used for the query being executed. However, in the subsequent step
of creating series iterators, the created iterators are referencing
_all_ in-memory chunks in their series, even the un-pinned ones. In
iterator creation, we copy a pointer to each in-memory chunk of a
series into the iterator. While this creates a certain amount of
allocation churn, the worst thing about it is that copying the chunk
pointer out of the chunkDesc requires a mutex acquisition. (Remember
that the iterator will also reference un-pinned chunks, so we need to
acquire the mutex to protect against concurrent eviction.) The worst
case happens if a series doesn't even contain any relevant samples for
the query time range. We notice that during preloading but then we
will still create a series iterator for it. But even for series that
do contain relevant samples, the overhead is quite bad for instant
queries that retrieve a single sample from each series, but still go
through all the effort of series iterator creation. All of that is
particularly bad if a series has many in-memory chunks.
This commit addresses the problem from two sides:
First, it merges preloading and iterator creation into one step,
i.e. the preload call returns an iterator for exactly the preloaded
chunks.
Second, the required mutex acquisition in chunkDesc has been greatly
reduced. That was enabled by a side effect of the first step, which is
that the iterator is only referencing pinned chunks, so there is no
risk of concurrent eviction anymore, and chunks can be accessed
without mutex acquisition.
To simplify the code changes for the above, the long-planned change of
ValueAtTime to ValueAtOrBefore time was performed at the same
time. (It should have been done first, but it kind of accidentally
happened while I was in the middle of writing the series iterator
changes. Sorry for that.) So far, we actively filtered the up to two
values that were returned by ValueAtTime, i.e. we invested work to
retrieve up to two values, and then we invested more work to throw one
of them away.
The SeriesIterator.BoundaryValues method can be removed once #1401 is
fixed. But I really didn't want to load even more changes into this
PR.
Benchmarks:
The BenchmarkFuzz.* benchmarks run 83% faster (i.e. about six times
faster) and allocate 95% fewer bytes. The reason for that is that the
benchmark reads one sample after another from the time series and
creates a new series iterator for each sample read.
To find out how much these improvements matter in practice, I have
mirrored a beefy Prometheus server at SoundCloud that suffers from
both issues #1035 and #1264. To reach steady state that would be
comparable, the server needs to run for 15d. So far, it has run for
1d. The test server currently has only half as many memory time series
and 60% of the memory chunks the main server has. The 90th percentile
rule evaluation cycle time is ~11s on the main server and only ~3s on
the test server. However, these numbers might get much closer over
time.
In addition to performance improvements, this commit removes about 150
LOC.
2016-02-16 09:47:50 -08:00
|
|
|
ValueAtOrBeforeTime(model.Time) model.SamplePair
|
2014-09-16 06:47:24 -07:00
|
|
|
// Gets all values contained within a given interval.
|
2015-08-22 05:52:35 -07:00
|
|
|
RangeValues(metric.Interval) []model.SamplePair
|
2014-06-06 02:55:53 -07:00
|
|
|
}
|
|
|
|
|
Streamline series iterator creation
This will fix issue #1035 and will also help to make issue #1264 less
bad.
The fundamental problem in the current code:
In the preload phase, we quite accurately determine which chunks will
be used for the query being executed. However, in the subsequent step
of creating series iterators, the created iterators are referencing
_all_ in-memory chunks in their series, even the un-pinned ones. In
iterator creation, we copy a pointer to each in-memory chunk of a
series into the iterator. While this creates a certain amount of
allocation churn, the worst thing about it is that copying the chunk
pointer out of the chunkDesc requires a mutex acquisition. (Remember
that the iterator will also reference un-pinned chunks, so we need to
acquire the mutex to protect against concurrent eviction.) The worst
case happens if a series doesn't even contain any relevant samples for
the query time range. We notice that during preloading but then we
will still create a series iterator for it. But even for series that
do contain relevant samples, the overhead is quite bad for instant
queries that retrieve a single sample from each series, but still go
through all the effort of series iterator creation. All of that is
particularly bad if a series has many in-memory chunks.
This commit addresses the problem from two sides:
First, it merges preloading and iterator creation into one step,
i.e. the preload call returns an iterator for exactly the preloaded
chunks.
Second, the required mutex acquisition in chunkDesc has been greatly
reduced. That was enabled by a side effect of the first step, which is
that the iterator is only referencing pinned chunks, so there is no
risk of concurrent eviction anymore, and chunks can be accessed
without mutex acquisition.
To simplify the code changes for the above, the long-planned change of
ValueAtTime to ValueAtOrBefore time was performed at the same
time. (It should have been done first, but it kind of accidentally
happened while I was in the middle of writing the series iterator
changes. Sorry for that.) So far, we actively filtered the up to two
values that were returned by ValueAtTime, i.e. we invested work to
retrieve up to two values, and then we invested more work to throw one
of them away.
The SeriesIterator.BoundaryValues method can be removed once #1401 is
fixed. But I really didn't want to load even more changes into this
PR.
Benchmarks:
The BenchmarkFuzz.* benchmarks run 83% faster (i.e. about six times
faster) and allocate 95% fewer bytes. The reason for that is that the
benchmark reads one sample after another from the time series and
creates a new series iterator for each sample read.
To find out how much these improvements matter in practice, I have
mirrored a beefy Prometheus server at SoundCloud that suffers from
both issues #1035 and #1264. To reach steady state that would be
comparable, the server needs to run for 15d. So far, it has run for
1d. The test server currently has only half as many memory time series
and 60% of the memory chunks the main server has. The 90th percentile
rule evaluation cycle time is ~11s on the main server and only ~3s on
the test server. However, these numbers might get much closer over
time.
In addition to performance improvements, this commit removes about 150
LOC.
2016-02-16 09:47:50 -08:00
|
|
|
// A Preloader preloads series data necessary for a query into memory, pins it
|
|
|
|
// until released via Close(), and returns an iterator for the pinned data. Its
|
|
|
|
// methods are generally not goroutine-safe.
|
2014-06-06 02:55:53 -07:00
|
|
|
type Preloader interface {
|
2014-10-15 06:53:05 -07:00
|
|
|
PreloadRange(
|
2015-08-20 08:18:46 -07:00
|
|
|
fp model.Fingerprint,
|
2016-03-08 15:09:42 -08:00
|
|
|
from, through model.Time,
|
Handle errors caused by data corruption more gracefully
This requires all the panic calls upon unexpected data to be converted
into errors returned. This pollute the function signatures quite
lot. Well, this is Go...
The ideas behind this are the following:
- panic only if it's a programming error. Data corruptions happen, and
they are not programming errors.
- If we detect a data corruption, we "quarantine" the series,
essentially removing it from the database and putting its data into
a separate directory for forensics.
- Failure during writing to a series file is not considered corruption
automatically. It will call setDirty, though, so that a
crashrecovery upon the next restart will commence and check for
that.
- Series quarantining and setDirty calls are logged and counted in
metrics, but are hidden from the user of the interfaces in
interface.go, whith the notable exception of Append(). The reasoning
is that we treat corruption by removing the corrupted series, i.e. a
query for it will return no results on its next call anyway, so
return no results right now. In the case of Append(), we want to
tell the user that no data has been appended, though.
Minor side effects:
- Now consistently using filepath.* instead of path.*.
- Introduced structured logging where I touched it. This makes things
less consistent, but a complete change to structured logging would
be out of scope for this PR.
2016-02-25 03:23:42 -08:00
|
|
|
) SeriesIterator
|
2016-02-19 09:35:30 -08:00
|
|
|
PreloadInstant(
|
|
|
|
fp model.Fingerprint,
|
|
|
|
timestamp model.Time, stalenessDelta time.Duration,
|
Handle errors caused by data corruption more gracefully
This requires all the panic calls upon unexpected data to be converted
into errors returned. This pollute the function signatures quite
lot. Well, this is Go...
The ideas behind this are the following:
- panic only if it's a programming error. Data corruptions happen, and
they are not programming errors.
- If we detect a data corruption, we "quarantine" the series,
essentially removing it from the database and putting its data into
a separate directory for forensics.
- Failure during writing to a series file is not considered corruption
automatically. It will call setDirty, though, so that a
crashrecovery upon the next restart will commence and check for
that.
- Series quarantining and setDirty calls are logged and counted in
metrics, but are hidden from the user of the interfaces in
interface.go, whith the notable exception of Append(). The reasoning
is that we treat corruption by removing the corrupted series, i.e. a
query for it will return no results on its next call anyway, so
return no results right now. In the case of Append(), we want to
tell the user that no data has been appended, though.
Minor side effects:
- Now consistently using filepath.* instead of path.*.
- Introduced structured logging where I touched it. This makes things
less consistent, but a complete change to structured logging would
be out of scope for this PR.
2016-02-25 03:23:42 -08:00
|
|
|
) SeriesIterator
|
2014-06-06 02:55:53 -07:00
|
|
|
// Close unpins any previously requested series data from memory.
|
|
|
|
Close()
|
|
|
|
}
|
2016-02-19 07:46:11 -08:00
|
|
|
|
|
|
|
// ZeroSamplePair is the pseudo zero-value of model.SamplePair used by the local
|
2016-03-08 15:09:42 -08:00
|
|
|
// package to signal a non-existing sample pair. It is a SamplePair with
|
|
|
|
// timestamp model.Earliest and value 0.0. Note that the natural zero value of
|
|
|
|
// SamplePair has a timestamp of 0, which is possible to appear in a real
|
|
|
|
// SamplePair and thus not suitable to signal a non-existing SamplePair.
|
2016-02-19 07:46:11 -08:00
|
|
|
var ZeroSamplePair = model.SamplePair{Timestamp: model.Earliest}
|
2016-03-08 15:09:42 -08:00
|
|
|
|
|
|
|
// ZeroSample is the pseudo zero-value of model.Sample used by the local package
|
|
|
|
// to signal a non-existing sample. It is a Sample with timestamp
|
|
|
|
// model.Earliest, value 0.0, and metric nil. Note that the natural zero value
|
|
|
|
// of Sample has a timestamp of 0, which is possible to appear in a real
|
|
|
|
// Sample and thus not suitable to signal a non-existing Sample.
|
|
|
|
var ZeroSample = model.Sample{Timestamp: model.Earliest}
|