prometheus/.circleci/config.yml
Callum Styan 8fd73b1d28
Add Exemplar Remote Write support (#8296)
* Write exemplars to the WAL and send them over remote write.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Update example for exemplars, print data in a more obvious format.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Add metrics for remote write of exemplars.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Fix incorrect slices passed to send in remote write.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* We need to unregister the new metrics.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Address review comments

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Order of exemplar append vs write exemplar to WAL needs to change.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Several fixes to prevent sending uninitialized or incorrect samples with an exemplar. Fix dropping exemplar for missing series. Add tests for queue_manager sending exemplars

Signed-off-by: Martin Disibio <mdisibio@gmail.com>

* Store both samples and exemplars in the same timeseries buffer to remove the alloc when building final request, keep sub-slices in separate buffers for re-use

Signed-off-by: Martin Disibio <mdisibio@gmail.com>

* Condense sample/exemplar delivery tests to parameterized sub-tests

Signed-off-by: Martin Disibio <mdisibio@gmail.com>

* Rename test methods for clarity now that they also handle exemplars

Signed-off-by: Martin Disibio <mdisibio@gmail.com>

* Rename counter variable. Fix instances where metrics were not updated correctly

Signed-off-by: Martin Disibio <mdisibio@gmail.com>

* Add exemplars to LoadWAL benchmark

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* last exemplars timestamp metric needs to convert value to seconds with
ms precision

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Process exemplar records in a separate go routine when loading the WAL.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Address review comments related to clarifying comments and variable
names. Also refactor sample/exemplar to enqueue prompb types.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Regenerate types proto with comments, update protoc version again.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Put remote write of exemplars behind a feature flag.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Address some of Ganesh's review comments.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Move exemplar remote write feature flag to a config file field.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Address Bartek's review comments.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Don't allocate exemplar buffers in queue_manager if we're not going to
send exemplars over remote write.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Add ValidateExemplar function, validate exemplars when appending to head
and log them all to WAL before adding them to exemplar storage.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Address more reivew comments from Ganesh.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Add exemplar total label length check.

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Address a few last review comments

Signed-off-by: Callum Styan <callumstyan@gmail.com>

Co-authored-by: Martin Disibio <mdisibio@gmail.com>
2021-05-06 13:53:52 -07:00

174 lines
4.6 KiB
YAML

---
version: 2.1
orbs:
prometheus: prometheus/prometheus@0.11.0
go: circleci/go@0.2.0
win: circleci/windows@2.3.0
executors:
# Whenever the Go version is updated here, .promu.yml
# should also be updated.
golang:
docker:
- image: circleci/golang:1.16-node
golang_115:
docker:
- image: circleci/golang:1.15-node
jobs:
test:
executor: golang
steps:
- prometheus/setup_environment
- go/load-cache:
key: v1
- restore_cache:
keys:
- v3-npm-deps-{{ checksum "web/ui/react-app/yarn.lock" }}
- v3-npm-deps-
- run:
command: make
environment:
# Run garbage collection more aggressively to avoid getting OOMed during the lint phase.
GOGC: "20"
# By default Go uses GOMAXPROCS but a Circle CI executor has many
# cores (> 30) while the CPU and RAM resources are throttled. If we
# don't limit this to the number of allocated cores, the job is
# likely to get OOMed and killed.
GOOPTS: "-p 2"
GOMAXPROCS: "2"
GO111MODULE: "on"
- prometheus/check_proto:
version: "3.15.8"
- prometheus/store_artifact:
file: prometheus
- prometheus/store_artifact:
file: promtool
- go/save-cache:
key: v1
- save_cache:
key: v3-npm-deps-{{ checksum "web/ui/react-app/yarn.lock" }}
paths:
- /home/circleci/.cache/yarn
- store_test_results:
path: test-results
test_windows:
executor:
name: win/default
shell: powershell
working_directory: /go/src/github.com/prometheus/prometheus
steps:
- checkout
- run:
# Temporary workaround until circleci updates go.
command: |
choco upgrade -y golang
- run:
command:
refreshenv
- run:
command: |
$env:GOARCH=""; $env:GOOS=""; cd web/ui; go generate
cd ../..
$TestTargets = go list ./... | Where-Object { $_ -NotMatch "(github.com/prometheus/prometheus/discovery.*|github.com/prometheus/prometheus/config|github.com/prometheus/prometheus/web)"}
go test $TestTargets -vet=off -v
environment:
GOGC: "20"
GOOPTS: "-p 2"
test_tsdb_go115:
executor: golang_115
steps:
- checkout
- run: go test ./tsdb/...
test_mixins:
executor: golang
steps:
- checkout
- run: go install ./cmd/promtool/.
- run:
command: go install -mod=readonly github.com/google/go-jsonnet/cmd/jsonnet github.com/google/go-jsonnet/cmd/jsonnetfmt github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb
working_directory: ~/project/documentation/prometheus-mixin
- run:
command: make clean
working_directory: ~/project/documentation/prometheus-mixin
- run:
command: jb install
working_directory: ~/project/documentation/prometheus-mixin
- run:
command: make
working_directory: ~/project/documentation/prometheus-mixin
- run:
command: git diff --exit-code
working_directory: ~/project/documentation/prometheus-mixin
repo_sync:
executor: golang
steps:
- checkout
- run: mkdir -v -p "${PATH%%:*}" && curl -sL --fail https://github.com/mikefarah/yq/releases/download/v4.6.3/yq_linux_amd64 -o "${PATH%%:*}/yq" && chmod -v +x "${PATH%%:*}/yq"
- run: sha256sum -c <(echo "c4343783c3361495c0d6d1eb742bba7432aa65e13e9fb8d7e201d544bcf14246 ${PATH%%:*}/yq")
- run: ./scripts/sync_repo_files.sh
workflows:
version: 2
prometheus:
jobs:
- test:
filters:
tags:
only: /.*/
- test_tsdb_go115:
filters:
tags:
only: /.*/
- test_mixins:
filters:
tags:
only: /.*/
- test_windows:
filters:
tags:
only: /.*/
- prometheus/build:
name: build
parallelism: 12
filters:
tags:
only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
- prometheus/publish_main:
context: org-context
requires:
- test
- build
filters:
branches:
only: main
image: circleci/golang:1-node
- prometheus/publish_release:
context: org-context
requires:
- test
- build
filters:
tags:
only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
branches:
ignore: /.*/
image: circleci/golang:1-node
nightly:
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- main
jobs:
- repo_sync:
context: org-context