mirror of
https://github.com/prometheus/prometheus.git
synced 2025-02-02 08:31:11 -08:00
Merge branch 'main' into cedwards/nhcb-wal-wbl
Some checks are pending
CI / Go tests (push) Waiting to run
CI / More Go tests (push) Waiting to run
CI / Go tests with previous Go version (push) Waiting to run
CI / UI tests (push) Waiting to run
CI / Go tests on Windows (push) Waiting to run
CI / Mixins tests (push) Waiting to run
CI / Build Prometheus for common architectures (0) (push) Waiting to run
CI / Build Prometheus for common architectures (1) (push) Waiting to run
CI / Build Prometheus for common architectures (2) (push) Waiting to run
CI / Build Prometheus for all architectures (0) (push) Waiting to run
CI / Build Prometheus for all architectures (1) (push) Waiting to run
CI / Build Prometheus for all architectures (10) (push) Waiting to run
CI / Build Prometheus for all architectures (11) (push) Waiting to run
CI / Build Prometheus for all architectures (2) (push) Waiting to run
CI / Build Prometheus for all architectures (3) (push) Waiting to run
CI / Build Prometheus for all architectures (4) (push) Waiting to run
CI / Build Prometheus for all architectures (5) (push) Waiting to run
CI / Build Prometheus for all architectures (6) (push) Waiting to run
CI / Build Prometheus for all architectures (7) (push) Waiting to run
CI / Build Prometheus for all architectures (8) (push) Waiting to run
CI / Build Prometheus for all architectures (9) (push) Waiting to run
CI / Report status of build Prometheus for all architectures (push) Blocked by required conditions
CI / Check generated parser (push) Waiting to run
CI / golangci-lint (push) Waiting to run
CI / fuzzing (push) Waiting to run
CI / codeql (push) Waiting to run
CI / Publish main branch artifacts (push) Blocked by required conditions
CI / Publish release artefacts (push) Blocked by required conditions
CI / Publish UI on npm Registry (push) Blocked by required conditions
Some checks are pending
CI / Go tests (push) Waiting to run
CI / More Go tests (push) Waiting to run
CI / Go tests with previous Go version (push) Waiting to run
CI / UI tests (push) Waiting to run
CI / Go tests on Windows (push) Waiting to run
CI / Mixins tests (push) Waiting to run
CI / Build Prometheus for common architectures (0) (push) Waiting to run
CI / Build Prometheus for common architectures (1) (push) Waiting to run
CI / Build Prometheus for common architectures (2) (push) Waiting to run
CI / Build Prometheus for all architectures (0) (push) Waiting to run
CI / Build Prometheus for all architectures (1) (push) Waiting to run
CI / Build Prometheus for all architectures (10) (push) Waiting to run
CI / Build Prometheus for all architectures (11) (push) Waiting to run
CI / Build Prometheus for all architectures (2) (push) Waiting to run
CI / Build Prometheus for all architectures (3) (push) Waiting to run
CI / Build Prometheus for all architectures (4) (push) Waiting to run
CI / Build Prometheus for all architectures (5) (push) Waiting to run
CI / Build Prometheus for all architectures (6) (push) Waiting to run
CI / Build Prometheus for all architectures (7) (push) Waiting to run
CI / Build Prometheus for all architectures (8) (push) Waiting to run
CI / Build Prometheus for all architectures (9) (push) Waiting to run
CI / Report status of build Prometheus for all architectures (push) Blocked by required conditions
CI / Check generated parser (push) Waiting to run
CI / golangci-lint (push) Waiting to run
CI / fuzzing (push) Waiting to run
CI / codeql (push) Waiting to run
CI / Publish main branch artifacts (push) Blocked by required conditions
CI / Publish release artefacts (push) Blocked by required conditions
CI / Publish UI on npm Registry (push) Blocked by required conditions
# Conflicts: # tsdb/docs/format/wal.md
This commit is contained in:
György Krajcsovits
commit
efdd0880c1
64
RELEASE.md
64
RELEASE.md
|
|
@ -5,61 +5,15 @@ This page describes the release process and the currently planned schedule for u
|
|||
## Release schedule
|
||||
|
||||
Release cadence of first pre-releases being cut is 6 weeks.
|
||||
Please see [the v2.55 RELEASE.md](https://github.com/prometheus/prometheus/blob/release-2.55/RELEASE.md) for the v2 release series schedule.
|
||||
|
||||
| release series | date of first pre-release (year-month-day) | release shepherd |
|
||||
|----------------|--------------------------------------------|---------------------------------------------|
|
||||
| v2.4 | 2018-09-06 | Goutham Veeramachaneni (GitHub: @gouthamve) |
|
||||
| v2.5 | 2018-10-24 | Frederic Branczyk (GitHub: @brancz) |
|
||||
| v2.6 | 2018-12-05 | Simon Pasquier (GitHub: @simonpasquier) |
|
||||
| v2.7 | 2019-01-16 | Goutham Veeramachaneni (GitHub: @gouthamve) |
|
||||
| v2.8 | 2019-02-27 | Ganesh Vernekar (GitHub: @codesome) |
|
||||
| v2.9 | 2019-04-10 | Brian Brazil (GitHub: @brian-brazil) |
|
||||
| v2.10 | 2019-05-22 | Björn Rabenstein (GitHub: @beorn7) |
|
||||
| v2.11 | 2019-07-03 | Frederic Branczyk (GitHub: @brancz) |
|
||||
| v2.12 | 2019-08-14 | Julius Volz (GitHub: @juliusv) |
|
||||
| v2.13 | 2019-09-25 | Krasi Georgiev (GitHub: @krasi-georgiev) |
|
||||
| v2.14 | 2019-11-06 | Chris Marchbanks (GitHub: @csmarchbanks) |
|
||||
| v2.15 | 2019-12-18 | Bartek Plotka (GitHub: @bwplotka) |
|
||||
| v2.16 | 2020-01-29 | Callum Styan (GitHub: @cstyan) |
|
||||
| v2.17 | 2020-03-11 | Julien Pivotto (GitHub: @roidelapluie) |
|
||||
| v2.18 | 2020-04-22 | Bartek Plotka (GitHub: @bwplotka) |
|
||||
| v2.19 | 2020-06-03 | Ganesh Vernekar (GitHub: @codesome) |
|
||||
| v2.20 | 2020-07-15 | Björn Rabenstein (GitHub: @beorn7) |
|
||||
| v2.21 | 2020-08-26 | Julien Pivotto (GitHub: @roidelapluie) |
|
||||
| v2.22 | 2020-10-07 | Frederic Branczyk (GitHub: @brancz) |
|
||||
| v2.23 | 2020-11-18 | Ganesh Vernekar (GitHub: @codesome) |
|
||||
| v2.24 | 2020-12-30 | Björn Rabenstein (GitHub: @beorn7) |
|
||||
| v2.25 | 2021-02-10 | Julien Pivotto (GitHub: @roidelapluie) |
|
||||
| v2.26 | 2021-03-24 | Bartek Plotka (GitHub: @bwplotka) |
|
||||
| v2.27 | 2021-05-05 | Chris Marchbanks (GitHub: @csmarchbanks) |
|
||||
| v2.28 | 2021-06-16 | Julius Volz (GitHub: @juliusv) |
|
||||
| v2.29 | 2021-07-28 | Frederic Branczyk (GitHub: @brancz) |
|
||||
| v2.30 | 2021-09-08 | Ganesh Vernekar (GitHub: @codesome) |
|
||||
| v2.31 | 2021-10-20 | Julien Pivotto (GitHub: @roidelapluie) |
|
||||
| v2.32 | 2021-12-01 | Julius Volz (GitHub: @juliusv) |
|
||||
| v2.33 | 2022-01-12 | Björn Rabenstein (GitHub: @beorn7) |
|
||||
| v2.34 | 2022-02-23 | Chris Marchbanks (GitHub: @csmarchbanks) |
|
||||
| v2.35 | 2022-04-06 | Augustin Husson (GitHub: @nexucis) |
|
||||
| v2.36 | 2022-05-18 | Matthias Loibl (GitHub: @metalmatze) |
|
||||
| v2.37 LTS | 2022-06-29 | Julien Pivotto (GitHub: @roidelapluie) |
|
||||
| v2.38 | 2022-08-10 | Julius Volz (GitHub: @juliusv) |
|
||||
| v2.39 | 2022-09-21 | Ganesh Vernekar (GitHub: @codesome) |
|
||||
| v2.40 | 2022-11-02 | Ganesh Vernekar (GitHub: @codesome) |
|
||||
| v2.41 | 2022-12-14 | Julien Pivotto (GitHub: @roidelapluie) |
|
||||
| v2.42 | 2023-01-25 | Kemal Akkoyun (GitHub: @kakkoyun) |
|
||||
| v2.43 | 2023-03-08 | Julien Pivotto (GitHub: @roidelapluie) |
|
||||
| v2.44 | 2023-04-19 | Bryan Boreham (GitHub: @bboreham) |
|
||||
| v2.45 LTS | 2023-05-31 | Jesus Vazquez (Github: @jesusvazquez) |
|
||||
| v2.46 | 2023-07-12 | Julien Pivotto (GitHub: @roidelapluie) |
|
||||
| v2.47 | 2023-08-23 | Bryan Boreham (GitHub: @bboreham) |
|
||||
| v2.48 | 2023-10-04 | Levi Harrison (GitHub: @LeviHarrison) |
|
||||
| v2.49 | 2023-12-05 | Bartek Plotka (GitHub: @bwplotka) |
|
||||
| v2.50 | 2024-01-16 | Augustin Husson (GitHub: @nexucis) |
|
||||
| v2.51 | 2024-03-07 | Bryan Boreham (GitHub: @bboreham) |
|
||||
| v2.52 | 2024-04-22 | Arthur Silva Sens (GitHub: @ArthurSens) |
|
||||
| v2.53 LTS | 2024-06-03 | George Krajcsovits (GitHub: @krajorama) |
|
||||
| v2.54 | 2024-07-17 | Bryan Boreham (GitHub: @bboreham) |
|
||||
| v2.55 | 2024-09-17 | Bryan Boreham (GitHub: @bboreham) |
|
||||
| release series | date of first pre-release (year-month-day) | release shepherd |
|
||||
|----------------|--------------------------------------------|-----------------------------------|
|
||||
| v3.0 | 2024-11-14 | Jan Fajerski (GitHub: @jan--f) |
|
||||
| v3.1 | 2024-12-17 | Bryan Boreham (GitHub: @bboreham) |
|
||||
| v3.2 | 2025-01-28 | Jan Fajerski (GitHub: @jan--f) |
|
||||
| v3.3 | 2025-03-11 | Ayoub Mrini (Github: @machine424) |
|
||||
| v3.4 | 2025-04-22 | **volunteer welcome** |
|
||||
|
||||
If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice.
|
||||
|
||||
|
|
@ -204,7 +158,7 @@ Then release with `git tag-release`.
|
|||
|
||||
Signing a tag with a GPG key is appreciated, but in case you can't add a GPG key to your Github account using the following [procedure](https://help.github.com/articles/generating-a-gpg-key/), you can replace the `-s` flag by `-a` flag of the `git tag` command to only annotate the tag without signing.
|
||||
|
||||
Once a tag is created, the release process through CircleCI will be triggered for this tag and Circle CI will draft the GitHub release using the `prombot` account.
|
||||
Once a tag is created, the release process through Github Actions will be triggered for this tag and Github Actions will draft the GitHub release using the `prombot` account.
|
||||
|
||||
Finally, wait for the build step for the tag to finish. The point here is to wait for tarballs to be uploaded to the Github release and the container images to be pushed to the Docker Hub and Quay.io. Once that has happened, click _Publish release_, which will make the release publicly visible and create a GitHub notification.
|
||||
**Note:** for a release candidate version ensure the _This is a pre-release_ box is checked when drafting the release in the Github UI. The CI job should take care of this but it's a good idea to double check before clicking _Publish release_.`
|
||||
|
|
|
|||
|
|
@ -1688,7 +1688,7 @@ func (s *shards) updateMetrics(_ context.Context, err error, sampleCount, exempl
|
|||
s.enqueuedHistograms.Sub(int64(histogramCount))
|
||||
}
|
||||
|
||||
// sendSamples to the remote storage with backoff for recoverable errors.
|
||||
// sendSamplesWithBackoff to the remote storage with backoff for recoverable errors.
|
||||
func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.TimeSeries, sampleCount, exemplarCount, histogramCount, metadataCount int, pBuf *proto.Buffer, buf *[]byte, enc Compression) (WriteResponseStats, error) {
|
||||
// Build the WriteRequest with no metadata.
|
||||
req, highest, lowest, err := buildWriteRequest(s.qm.logger, samples, nil, pBuf, buf, nil, enc)
|
||||
|
|
@ -1802,7 +1802,7 @@ func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.Ti
|
|||
return accumulatedStats, err
|
||||
}
|
||||
|
||||
// sendV2Samples to the remote storage with backoff for recoverable errors.
|
||||
// sendV2SamplesWithBackoff to the remote storage with backoff for recoverable errors.
|
||||
func (s *shards) sendV2SamplesWithBackoff(ctx context.Context, samples []writev2.TimeSeries, labels []string, sampleCount, exemplarCount, histogramCount, metadataCount int, pBuf, buf *[]byte, enc Compression) (WriteResponseStats, error) {
|
||||
// Build the WriteRequest with no metadata.
|
||||
req, highest, lowest, err := buildV2WriteRequest(s.qm.logger, samples, labels, pBuf, buf, nil, enc)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,32 @@
|
|||
# WAL Disk Format
|
||||
|
||||
This document describes the official Prometheus WAL format.
|
||||
|
||||
The write ahead log operates in segments that are numbered and sequential,
|
||||
e.g. `000000`, `000001`, `000002`, etc., and are limited to 128MB by default.
|
||||
A segment is written to in pages of 32KB. Only the last page of the most recent segment
|
||||
and are limited to 128MB by default.
|
||||
|
||||
## Segment filename
|
||||
|
||||
The sequence number is captured in the segment filename,
|
||||
e.g. `000000`, `000001`, `000002`, etc. The first unsigned integer represents
|
||||
the sequence number of the segment, typically encoded with six digits.
|
||||
|
||||
## Segment encoding
|
||||
|
||||
This section describes the segment encoding.
|
||||
|
||||
A segment encodes an array of records. It does not contain any header. A segment
|
||||
is written to pages of 32KB. Only the last page of the most recent segment
|
||||
may be partial. A WAL record is an opaque byte slice that gets split up into sub-records
|
||||
should it exceed the remaining space of the current page. Records are never split across
|
||||
segment boundaries. If a single record exceeds the default segment size, a segment with
|
||||
a larger size will be created.
|
||||
|
||||
The encoding of pages is largely borrowed from [LevelDB's/RocksDB's write ahead log.](https://github.com/facebook/rocksdb/wiki/Write-Ahead-Log-File-Format)
|
||||
|
||||
Notable deviations are that the record fragment is encoded as:
|
||||
### Records encoding
|
||||
|
||||
Each record fragment is encoded as:
|
||||
|
||||
```
|
||||
┌───────────┬──────────┬────────────┬──────────────┐
|
||||
|
|
@ -17,7 +34,8 @@ Notable deviations are that the record fragment is encoded as:
|
|||
└───────────┴──────────┴────────────┴──────────────┘
|
||||
```
|
||||
|
||||
The initial type byte is made up of three components: a 3-bit reserved field, a 1-bit zstd compression flag, a 1-bit snappy compression flag, and a 3-bit type flag.
|
||||
The initial type byte is made up of three components: a 3-bit reserved field,
|
||||
a 1-bit zstd compression flag, a 1-bit snappy compression flag, and a 3-bit type flag.
|
||||
|
||||
```
|
||||
┌─────────────────┬──────────────────┬────────────────────┬──────────────────┐
|
||||
|
|
@ -25,7 +43,7 @@ The initial type byte is made up of three components: a 3-bit reserved field, a
|
|||
└─────────────────┴──────────────────┴────────────────────┴──────────────────┘
|
||||
```
|
||||
|
||||
The lowest 3 bits within this flag represent the record type as follows:
|
||||
The lowest 3 bits within the type flag represent the record type as follows:
|
||||
|
||||
* `0`: rest of page will be empty
|
||||
* `1`: a full record encoded in a single fragment
|
||||
|
|
@ -33,11 +51,16 @@ The lowest 3 bits within this flag represent the record type as follows:
|
|||
* `3`: middle fragment of a record
|
||||
* `4`: final fragment of a record
|
||||
|
||||
## Record encoding
|
||||
After the type byte, 2-byte length and then 4-byte checksum of the following data are encoded.
|
||||
|
||||
The records written to the write ahead log are encoded as follows:
|
||||
All float values are represented using the [IEEE 754 format](https://en.wikipedia.org/wiki/IEEE_754).
|
||||
|
||||
### Series records
|
||||
### Record types
|
||||
|
||||
In the following sections, all the known record types are described. New types,
|
||||
can be added in the future.
|
||||
|
||||
#### Series records
|
||||
|
||||
Series records encode the labels that identifies a series and its unique ID.
|
||||
|
||||
|
|
@ -58,7 +81,7 @@ Series records encode the labels that identifies a series and its unique ID.
|
|||
└────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Sample records
|
||||
#### Sample records
|
||||
|
||||
Sample records encode samples as a list of triples `(series_id, timestamp, value)`.
|
||||
Series reference and timestamp are encoded as deltas w.r.t the first sample.
|
||||
|
|
@ -79,7 +102,7 @@ The first sample record begins at the second row.
|
|||
└──────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Tombstone records
|
||||
#### Tombstone records
|
||||
|
||||
Tombstone records encode tombstones as a list of triples `(series_id, min_time, max_time)`
|
||||
and specify an interval for which samples of a series got deleted.
|
||||
|
|
@ -95,9 +118,9 @@ and specify an interval for which samples of a series got deleted.
|
|||
└─────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Exemplar records
|
||||
#### Exemplar records
|
||||
|
||||
Exemplar records encode exemplars as a list of triples `(series_id, timestamp, value)`
|
||||
Exemplar records encode exemplars as a list of triples `(series_id, timestamp, value)`
|
||||
plus the length of the labels list, and all the labels.
|
||||
The first row stores the starting id and the starting timestamp.
|
||||
Series reference and timestamp are encoded as deltas w.r.t the first exemplar.
|
||||
|
|
@ -127,7 +150,7 @@ See: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/Op
|
|||
└──────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Metadata records
|
||||
#### Metadata records
|
||||
|
||||
Metadata records encode the metadata updates associated with a series.
|
||||
|
||||
|
|
@ -156,193 +179,172 @@ Metadata records encode the metadata updates associated with a series.
|
|||
└────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Native histogram records
|
||||
#### Histogram records
|
||||
|
||||
Native histogram records are encoded as a list of histogram samples.
|
||||
Series reference and timestamp are encoded as deltas w.r.t the first histogram sample.
|
||||
The first row stores the starting id and the starting timestamp.
|
||||
The first native histogram sample record begins at the second row.
|
||||
Histogram records encode the integer and float native histogram samples.
|
||||
|
||||
There are several different types of native histogram samples.
|
||||
|
||||
Integer histogram encoding:
|
||||
A record with the integer native histograms with the exponential bucketing:
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ type = 9 <1b> │
|
||||
├──────────────────────────────────────────────────────────────────┤
|
||||
│ ┌────────────────────┬───────────────────────────┐ │
|
||||
│ │ id <8b> │ timestamp <8b> │ │
|
||||
│ └────────────────────┴───────────────────────────┘ │
|
||||
│ ┌────────────────────┬───────────────────────────┬ │
|
||||
│ │ id_delta <uvarint> │ timestamp_delta <uvarint> │ │
|
||||
│ ├────────────────────┴────┬──────────────────────┤ │
|
||||
│ │ counter_reset_hint <8b> │ schema <uvarint> │ │
|
||||
│ ├──────────────────────┬──┴──────────────────────┤ │
|
||||
│ │ zero_threshold <8b> │ zero_count <uvarint> │ │
|
||||
│ ├──────────────────────┴┬────────────────────────┤ │
|
||||
│ │ count <uvarint> │ sum <uvariant> │ │
|
||||
│ ├───────────────────────┴────────┬───────────────┴─────────────┐ │
|
||||
│ │ len(positive_spans) <uvarint> │ offset <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ length <uvarint> │ │
|
||||
│ │ ├─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(negative_spans) <uvarint> │ offset <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ length <uvarint> │ │
|
||||
│ │ ├─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(positive_buckets) <uvarint>│ bucket_count <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(negative_buckets) <uvarint>│ bucket_count <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ └────────────────────────────────┴─────────────────────────────┘ │
|
||||
│ . . . │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
┌───────────────────────────────────────────────────────────────────────┐
|
||||
│ type = 7 <1b> │
|
||||
├───────────────────────────────────────────────────────────────────────┤
|
||||
│ ┌────────────────────┬───────────────────────────┐ │
|
||||
│ │ id <8b> │ timestamp <8b> │ │
|
||||
│ └────────────────────┴───────────────────────────┘ │
|
||||
│ ┌────────────────────┬──────────────────────────────────────────────┐ │
|
||||
│ │ id_delta <uvarint> │ timestamp_delta <uvarint> │ │
|
||||
│ ├────────────────────┴────┬─────────────────────────────────────────┤ │
|
||||
│ │ counter_reset_hint <1b> │ schema <varint> │ │
|
||||
│ ├─────────────────────────┴────┬────────────────────────────────────┤ │
|
||||
│ │ zero_threshold (float) <8b> │ zero_count <uvarint> │ │
|
||||
│ ├─────────────────┬────────────┴────────────────────────────────────┤ │
|
||||
│ │ count <uvarint> │ sum (float) <8b> │ │
|
||||
│ ├─────────────────┴─────────────────────────────────────────────────┤ │
|
||||
│ │ positive_spans_num <uvarint> │ │
|
||||
│ ├─────────────────────────────────┬─────────────────────────────────┤ │
|
||||
│ │ positive_span_offset_1 <varint> │ positive_span_len_1 <uvarint32> │ │
|
||||
│ ├─────────────────────────────────┴─────────────────────────────────┤ │
|
||||
│ │ . . . │ │
|
||||
│ ├───────────────────────────────────────────────────────────────────┤ │
|
||||
│ │ negative_spans_num <uvarint> │ │
|
||||
│ ├───────────────────────────────┬───────────────────────────────────┤ │
|
||||
│ │ negative_span_offset <varint> │ negative_span_len <uvarint32> │ │
|
||||
│ ├───────────────────────────────┴───────────────────────────────────┤ │
|
||||
│ │ . . . │ │
|
||||
│ ├───────────────────────────────────────────────────────────────────┤ │
|
||||
│ │ positive_bkts_num <uvarint> │ │
|
||||
│ ├─────────────────────────┬───────┬─────────────────────────────────┤ │
|
||||
│ │ positive_bkt_1 <varint> │ . . . │ positive_bkt_n <varint> │ │
|
||||
│ ├─────────────────────────┴───────┴─────────────────────────────────┤ │
|
||||
│ │ negative_bkts_num <uvarint> │ │
|
||||
│ ├─────────────────────────┬───────┬─────────────────────────────────┤ │
|
||||
│ │ negative_bkt_1 <varint> │ . . . │ negative_bkt_n <varint> │ │
|
||||
│ └─────────────────────────┴───────┴─────────────────────────────────┘ │
|
||||
│ . . . │
|
||||
└───────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
There are also integer histograms that have custom buckets, which will always
|
||||
have a schem of 053. Custom bucket native histograms additionally encode
|
||||
a field that specifies the custom values:
|
||||
A record with the float native histograms with the exponential bucketing:
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ type = 9 <1b> │
|
||||
├──────────────────────────────────────────────────────────────────┤
|
||||
│ ┌────────────────────┬───────────────────────────┐ │
|
||||
│ │ id <8b> │ timestamp <8b> │ │
|
||||
│ └────────────────────┴───────────────────────────┘ │
|
||||
│ ┌────────────────────┬───────────────────────────┬ │
|
||||
│ │ id_delta <uvarint> │ timestamp_delta <uvarint> │ │
|
||||
│ ├────────────────────┴────┬──────────────────────┤ │
|
||||
│ │ counter_reset_hint <8b> │ schema <uvarint> │ │
|
||||
│ ├──────────────────────┬──┴──────────────────────┤ │
|
||||
│ │ zero_threshold <8b> │ zero_count <uvarint> │ │
|
||||
│ ├──────────────────────┴┬────────────────────────┤ │
|
||||
│ │ count <uvarint> │ sum <uvariant> │ │
|
||||
│ ├───────────────────────┴────────┬───────────────┴─────────────┐ │
|
||||
│ │ len(positive_spans) <uvarint> │ offset <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ length <uvarint> │ │
|
||||
│ │ ├─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(negative_spans) <uvarint> │ offset <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ length <uvarint> │ │
|
||||
│ │ ├─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(positive_buckets) <uvarint>│ bucket_count <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(negative_buckets) <uvarint>│ bucket_count <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(custom_values) <uvarint> │ value <8b> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ └────────────────────────────────┴─────────────────────────────┘ │
|
||||
│ . . . │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
┌───────────────────────────────────────────────────────────────────────┐
|
||||
│ type = 8 <1b> │
|
||||
├───────────────────────────────────────────────────────────────────────┤
|
||||
│ ┌────────────────────┬───────────────────────────┐ │
|
||||
│ │ id <8b> │ timestamp <8b> │ │
|
||||
│ └────────────────────┴───────────────────────────┘ │
|
||||
│ ┌────────────────────┬──────────────────────────────────────────────┐ │
|
||||
│ │ id_delta <uvarint> │ timestamp_delta <uvarint> │ │
|
||||
│ ├────────────────────┴────┬─────────────────────────────────────────┤ │
|
||||
│ │ counter_reset_hint <1b> │ schema <varint> │ │
|
||||
│ ├─────────────────────────┴────┬────────────────────────────────────┤ │
|
||||
│ │ zero_threshold (float) <8b> │ zero_count (float) <8b> │ │
|
||||
│ ├────────────────────┬─────────┴────────────────────────────────────┤ │
|
||||
│ │ count (float) <8b> │ sum (float) <8b> │ │
|
||||
│ ├────────────────────┴──────────────────────────────────────────────┤ │
|
||||
│ │ positive_spans_num <uvarint> │ │
|
||||
│ ├─────────────────────────────────┬─────────────────────────────────┤ │
|
||||
│ │ positive_span_offset_1 <varint> │ positive_span_len_1 <uvarint32> │ │
|
||||
│ ├─────────────────────────────────┴─────────────────────────────────┤ │
|
||||
│ │ . . . │ │
|
||||
│ ├───────────────────────────────────────────────────────────────────┤ │
|
||||
│ │ negative_spans_num <uvarint> │ │
|
||||
│ ├───────────────────────────────┬───────────────────────────────────┤ │
|
||||
│ │ negative_span_offset <varint> │ negative_span_len <uvarint32> │ │
|
||||
│ ├───────────────────────────────┴───────────────────────────────────┤ │
|
||||
│ │ . . . │ │
|
||||
│ ├───────────────────────────────────────────────────────────────────┤ │
|
||||
│ │ positive_bkts_num <uvarint> │ │
|
||||
│ ├─────────────────────────────┬───────┬─────────────────────────────┤ │
|
||||
│ │ positive_bkt_1 (float) <8b> │ . . . │ positive_bkt_n (float) <8b> │ │
|
||||
│ ├─────────────────────────────┴───────┴─────────────────────────────┤ │
|
||||
│ │ negative_bkts_num <uvarint> │ │
|
||||
│ ├─────────────────────────────┬───────┬─────────────────────────────┤ │
|
||||
│ │ negative_bkt_1 (float) <8b> │ . . . │ negative_bkt_n (float) <8b> │ │
|
||||
│ └─────────────────────────────┴───────┴─────────────────────────────┘ │
|
||||
│ . . . │
|
||||
└───────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
(Note: negative spans and negative buckets will be empty for custom bucket native histograms.)
|
||||
|
||||
Float histogram encoding:
|
||||
A record with the integer native histograms with the custom bucketing, also known as NHCB.
|
||||
This record format is backwards compatible with type 7.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ type = 10 <1b> │
|
||||
├──────────────────────────────────────────────────────────────────┤
|
||||
│ ┌────────────────────┬───────────────────────────┐ │
|
||||
│ │ id <8b> │ timestamp <8b> │ │
|
||||
│ └────────────────────┴───────────────────────────┘ │
|
||||
│ ┌────────────────────┬───────────────────────────┐ │
|
||||
│ │ id_delta <uvarint> │ timestamp_delta <uvarint> │ │
|
||||
│ ├────────────────────┴────┬──────────────────────┤ │
|
||||
│ │ counter_reset_hint <8b> │ schema <uvarint> │ │
|
||||
│ ├──────────────────────┬──┴──────────────────────┤ │
|
||||
│ │ zero_threshold <8b> │ zero_count <8b> │ │
|
||||
│ ├──────────────────────┴┬────────────────────────┤ │
|
||||
│ │ count <8b> │ sum <8b> │ │
|
||||
│ ├───────────────────────┴────────┬───────────────┴─────────────┐ │
|
||||
│ │ len(positive_spans) <uvarint> │ offset <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ length <uvarint> │ │
|
||||
│ │ ├─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(negative_spans) <uvarint> │ offset <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ length <uvarint> │ │
|
||||
│ │ ├─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(positive_buckets) <uvarint>│ bucket_count <8b> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(negative_buckets) <uvarint>│ bucket_count <8b> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ └────────────────────────────────┴─────────────────────────────┘ │
|
||||
│ . . . │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
┌───────────────────────────────────────────────────────────────────────┐
|
||||
│ type = 9 <1b> │
|
||||
├───────────────────────────────────────────────────────────────────────┤
|
||||
│ ┌────────────────────┬───────────────────────────┐ │
|
||||
│ │ id <8b> │ timestamp <8b> │ │
|
||||
│ └────────────────────┴───────────────────────────┘ │
|
||||
│ ┌────────────────────┬──────────────────────────────────────────────┐ │
|
||||
│ │ id_delta <uvarint> │ timestamp_delta <uvarint> │ │
|
||||
│ ├────────────────────┴────┬─────────────────────────────────────────┤ │
|
||||
│ │ counter_reset_hint <1b> │ schema <varint> │ │
|
||||
│ ├─────────────────────────┴────┬────────────────────────────────────┤ │
|
||||
│ │ zero_threshold (float) <8b> │ zero_count <uvarint> │ │
|
||||
│ ├─────────────────┬────────────┴────────────────────────────────────┤ │
|
||||
│ │ count <uvarint> │ sum (float) <8b> │ │
|
||||
│ ├─────────────────┴─────────────────────────────────────────────────┤ │
|
||||
│ │ positive_spans_num <uvarint> │ │
|
||||
│ ├─────────────────────────────────┬─────────────────────────────────┤ │
|
||||
│ │ positive_span_offset_1 <varint> │ positive_span_len_1 <uvarint32> │ │
|
||||
│ ├─────────────────────────────────┴─────────────────────────────────┤ │
|
||||
│ │ . . . │ │
|
||||
│ ├───────────────────────────────────────────────────────────────────┤ │
|
||||
│ │ negative_spans_num <uvarint> = 0 │ │
|
||||
│ ├───────────────────────────────────────────────────────────────────┤ │
|
||||
│ │ positive_bkts_num <uvarint> │ │
|
||||
│ ├─────────────────────────┬───────┬─────────────────────────────────┤ │
|
||||
│ │ positive_bkt_1 <varint> │ . . . │ positive_bkt_n <varint> │ │
|
||||
│ ├─────────────────────────┴───────┴─────────────────────────────────┤ │
|
||||
│ │ negative_bkts_num <uvarint> = 0 │ │
|
||||
│ ├───────────────────────────────────────────────────────────────────┤ │
|
||||
│ │ custom_values_num <uvarint> │ │
|
||||
│ ├─────────────────────────────┬───────┬─────────────────────────────┤ │
|
||||
│ │ custom_value_1 (float) <8b> │ . . . │ custom_value_n (float) <8b> │ │
|
||||
│ └─────────────────────────────┴───────┴─────────────────────────────┘ │
|
||||
│ . . . │
|
||||
└───────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
There are also float histograms with custom buckets.
|
||||
A record with the float native histograms with the custom bucketing, also known as NHCB.
|
||||
This record format is backwards compatible with type 8.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ type = 10 <1b> │
|
||||
├──────────────────────────────────────────────────────────────────┤
|
||||
│ ┌────────────────────┬───────────────────────────┐ │
|
||||
│ │ id <8b> │ timestamp <8b> │ │
|
||||
│ └────────────────────┴───────────────────────────┘ │
|
||||
│ ┌────────────────────┬───────────────────────────┐ │
|
||||
│ │ id_delta <uvarint> │ timestamp_delta <uvarint> │ │
|
||||
│ ├────────────────────┴────┬──────────────────────┤ │
|
||||
│ │ counter_reset_hint <8b> │ schema <uvarint> │ │
|
||||
│ ├──────────────────────┬──┴──────────────────────┤ │
|
||||
│ │ zero_threshold <8b> │ zero_count <8b> │ │
|
||||
│ ├──────────────────────┴┬────────────────────────┤ │
|
||||
│ │ count <8b> │ sum <8b> │ │
|
||||
│ ├───────────────────────┴────────┬───────────────┴─────────────┐ │
|
||||
│ │ len(positive_spans) <uvarint> │ offset <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ length <uvarint> │ │
|
||||
│ │ ├─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(negative_spans) <uvarint> │ offset <uvarint> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ length <uvarint> │ │
|
||||
│ │ ├─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(positive_buckets) <uvarint>│ bucket_count <8b> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(negative_buckets) <uvarint>│ bucket_count <8b> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ len(custom_values) <uvarint> │ value <8b> │ │
|
||||
│ ├────────────────────────────────┼─────────────────────────────┤ │
|
||||
│ │ │ . . . │ │
|
||||
│ └────────────────────────────────┴─────────────────────────────┘ │
|
||||
│ . . . │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
┌───────────────────────────────────────────────────────────────────────┐
|
||||
│ type = 10 <1b> │
|
||||
├───────────────────────────────────────────────────────────────────────┤
|
||||
│ ┌────────────────────┬───────────────────────────┐ │
|
||||
│ │ id <8b> │ timestamp <8b> │ │
|
||||
│ └────────────────────┴───────────────────────────┘ │
|
||||
│ ┌────────────────────┬──────────────────────────────────────────────┐ │
|
||||
│ │ id_delta <uvarint> │ timestamp_delta <uvarint> │ │
|
||||
│ ├────────────────────┴────┬─────────────────────────────────────────┤ │
|
||||
│ │ counter_reset_hint <1b> │ schema <varint> │ │
|
||||
│ ├─────────────────────────┴────┬────────────────────────────────────┤ │
|
||||
│ │ zero_threshold (float) <8b> │ zero_count (float) <8b> │ │
|
||||
│ ├────────────────────┬─────────┴────────────────────────────────────┤ │
|
||||
│ │ count (float) <8b> │ sum (float) <8b> │ │
|
||||
│ ├────────────────────┴──────────────────────────────────────────────┤ │
|
||||
│ │ positive_spans_num <uvarint> │ │
|
||||
│ ├─────────────────────────────────┬─────────────────────────────────┤ │
|
||||
│ │ positive_span_offset_1 <varint> │ positive_span_len_1 <uvarint32> │ │
|
||||
│ ├─────────────────────────────────┴─────────────────────────────────┤ │
|
||||
│ │ . . . │ │
|
||||
│ ├───────────────────────────────────────────────────────────────────┤ │
|
||||
│ │ negative_spans_num <uvarint> = 0 │ │
|
||||
│ ├───────────────────────────────────────────────────────────────────┤ │
|
||||
│ │ positive_bkts_num <uvarint> │ │
|
||||
│ ├─────────────────────────────┬───────┬─────────────────────────────┤ │
|
||||
│ │ positive_bkt_1 (float) <8b> │ . . . │ positive_bkt_n (float) <8b> │ │
|
||||
│ ├─────────────────────────────┴───────┴─────────────────────────────┤ │
|
||||
│ │ negative_bkts_num <uvarint> = 0 │ │
|
||||
│ ├───────────────────────────────────────────────────────────────────┤ │
|
||||
│ │ custom_values_num <uvarint> │ │
|
||||
│ ├─────────────────────────────┬───────┬─────────────────────────────┤ │
|
||||
│ │ custom_value_1 (float) <8b> │ . . . │ custom_value_n (float) <8b> │ │
|
||||
│ └─────────────────────────────┴───────┴─────────────────────────────┘ │
|
||||
│ . . . │
|
||||
└───────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
(Note: negative spans and negative buckets will also be empty for custom bucket float native histograms.)
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ type Deduper struct {
|
|||
next *slog.Logger
|
||||
repeat time.Duration
|
||||
quit chan struct{}
|
||||
mtx sync.RWMutex
|
||||
mtx *sync.RWMutex
|
||||
seen map[string]time.Time
|
||||
}
|
||||
|
||||
|
|
@ -43,6 +43,7 @@ func Dedupe(next *slog.Logger, repeat time.Duration) *Deduper {
|
|||
next: next,
|
||||
repeat: repeat,
|
||||
quit: make(chan struct{}),
|
||||
mtx: new(sync.RWMutex),
|
||||
seen: map[string]time.Time{},
|
||||
}
|
||||
go d.run()
|
||||
|
|
@ -88,6 +89,7 @@ func (d *Deduper) WithAttrs(attrs []slog.Attr) slog.Handler {
|
|||
repeat: d.repeat,
|
||||
quit: d.quit,
|
||||
seen: d.seen,
|
||||
mtx: d.mtx,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -103,6 +105,7 @@ func (d *Deduper) WithGroup(name string) slog.Handler {
|
|||
repeat: d.repeat,
|
||||
quit: d.quit,
|
||||
seen: d.seen,
|
||||
mtx: d.mtx,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -56,3 +56,27 @@ func TestDedupe(t *testing.T) {
|
|||
}
|
||||
require.Len(t, lines, 2)
|
||||
}
|
||||
|
||||
func TestDedupeConcurrent(t *testing.T) {
|
||||
d := Dedupe(promslog.New(&promslog.Config{}), 250*time.Millisecond)
|
||||
dlog := slog.New(d)
|
||||
defer d.Stop()
|
||||
|
||||
concurrentWriteFunc := func() {
|
||||
go func() {
|
||||
dlog1 := dlog.With("writer", 1)
|
||||
for i := 0; i < 10; i++ {
|
||||
dlog1.With("foo", "bar").Info("test", "hello", "world")
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
dlog2 := dlog.With("writer", 2)
|
||||
for i := 0; i < 10; i++ {
|
||||
dlog2.With("foo", "bar").Info("test", "hello", "world")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
require.NotPanics(t, func() { concurrentWriteFunc() })
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue