Merge branch 'main' into prometheus-2023-04-03-3923e83

This commit is contained in:
Oleg Zaytsev 2023-04-13 09:15:24 +02:00
commit 4086a5f042
89 changed files with 9076 additions and 1850 deletions

View file

@ -0,0 +1,74 @@
---
name: Bug report
description: Create a report to help us improve.
body:
- type: markdown
attributes:
value: |
Thank you for opening a bug report for Prometheus.
Please do *NOT* ask support questions in Github issues.
If your issue is not a feature request or bug report use our [community support](https://prometheus.io/community/).
There is also [commercial support](https://prometheus.io/support-training/) available.
- type: textarea
attributes:
label: What did you do?
description: Please provide steps for us to reproduce this issue.
validations:
required: true
- type: textarea
attributes:
label: What did you expect to see?
- type: textarea
attributes:
label: What did you see instead? Under which circumstances?
validations:
required: true
- type: markdown
attributes:
value: |
## Environment
- type: input
attributes:
label: System information
description: insert output of `uname -srm` here, or operating system version
placeholder: e.g. Linux 5.16.15 x86_64
- type: textarea
attributes:
label: Prometheus version
description: Insert output of `prometheus --version` here.
render: text
placeholder: |
e.g. prometheus, version 2.23.0 (branch: HEAD, revision: 26d89b4b0776fe4cd5a3656dfa520f119a375273)
build user: root@37609b3a0a21
build date: 20201126-10:56:17
go version: go1.15.5
platform: linux/amd64
- type: textarea
attributes:
label: Prometheus configuration file
description: Insert relevant configuration here. Don't forget to remove secrets.
render: yaml
- type: textarea
attributes:
label: Alertmanager version
description: Insert output of `alertmanager --version` here (if relevant to the issue).
render: text
placeholder: |
e.g. alertmanager, version 0.22.2 (branch: HEAD, revision: 44f8adc06af5101ad64bd8b9c8b18273f2922051)
build user: root@b595c7f32520
build date: 20210602-07:50:37
go version: go1.16.4
platform: linux/amd64
- type: textarea
attributes:
label: Alertmanager configuration file
description: Insert relevant configuration here. Don't forget to remove secrets.
render: yaml
- type: textarea
attributes:
label: Logs
description: Insert Prometheus and Alertmanager logs relevant to the issue here.
render: text

View file

@ -0,0 +1,8 @@
blank_issues_enabled: false
contact_links:
- name: Prometheus Community Support
url: https://prometheus.io/community/
about: If you need help or support, please request help here.
- name: Commercial Support & Training
url: https://prometheus.io/support-training/
about: If you want commercial support or training, vendors are listed here.

View file

@ -0,0 +1,26 @@
---
name: Feature request
about: Suggest an idea for this project.
title: ''
labels: ''
assignees: ''
---
<!--
Please do *NOT* ask support questions in Github issues.
If your issue is not a feature request or bug report use our
community support.
https://prometheus.io/community/
There is also commercial support available.
https://prometheus.io/support-training/
-->
## Proposal
**Use case. Why is this important?**
*“Nice to have” is not a good use case. :)*

View file

@ -0,0 +1,17 @@
<!--
Don't forget!
- Please sign CNCF's Developer Certificate of Origin and sign-off your commits by adding the -s / --sign-off flag to `git commit`. See https://github.com/apps/dco for more information.
- If the PR adds or changes a behaviour or fixes a bug of an exported API it would need a unit/e2e test.
- Where possible use only exported APIs for tests to simplify the review and make it as close as possible to an actual library usage.
- No tests are needed for internal implementation changes.
- Performance improvements would need a benchmark test to prove it.
- All exposed objects should have a comment.
- All comments should start with a capital letter and end with a full stop.
-->

22
.github/.github/dependabot.yml vendored Normal file
View file

@ -0,0 +1,22 @@
version: 2
updates:
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: "monthly"
- package-ecosystem: "gomod"
directory: "/documentation/examples/remote_storage"
schedule:
interval: "monthly"
- package-ecosystem: "npm"
directory: "/web/ui"
schedule:
interval: "monthly"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
- package-ecosystem: "docker"
directory: "/"
schedule:
interval: "monthly"

56
.github/.github/stale.yml vendored Normal file
View file

@ -0,0 +1,56 @@
# Configuration for probot-stale - https://github.com/probot/stale
# Number of days of inactivity before an Issue or Pull Request becomes stale
daysUntilStale: 60
# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
daysUntilClose: false
# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
onlyLabels: []
# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
exemptLabels:
- keepalive
# Set to true to ignore issues in a project (defaults to false)
exemptProjects: false
# Set to true to ignore issues in a milestone (defaults to false)
exemptMilestones: false
# Set to true to ignore issues with an assignee (defaults to false)
exemptAssignees: false
# Label to use when marking as stale
staleLabel: stale
# Comment to post when marking as stale. Set to `false` to disable
markComment: false
# Comment to post when removing the stale label.
# unmarkComment: >
# Your comment here.
# Comment to post when closing a stale Issue or Pull Request.
# closeComment: >
# Your comment here.
# Limit the number of actions per hour, from 1-30. Default is 30
limitPerRun: 30
# Limit to only `issues` or `pulls`
only: pulls
# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
# pulls:
# daysUntilStale: 30
# markComment: >
# This pull request has been automatically marked as stale because it has not had
# recent activity. It will be closed if no further activity occurs. Thank you
# for your contributions.
# issues:
# exemptLabels:
# - confirmed

View file

@ -0,0 +1,30 @@
name: golangci-lint
on:
push:
paths:
- "go.sum"
- "go.mod"
- "**.go"
- "scripts/errcheck_excludes.txt"
- ".github/workflows/golangci-lint.yml"
- ".golangci.yml"
pull_request:
jobs:
golangci:
name: lint
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: install Go
uses: actions/setup-go@v2
with:
go-version: 1.18.x
- name: Install snmp_exporter/generator dependencies
run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
if: github.repository == 'prometheus/snmp_exporter'
- name: Lint
uses: golangci/golangci-lint-action@v3.2.0
with:
version: v1.45.2

22
.github/.github/workflows/lock.yml vendored Normal file
View file

@ -0,0 +1,22 @@
name: 'Lock Threads'
on:
schedule:
- cron: '13 23 * * *'
workflow_dispatch:
permissions:
issues: write
concurrency:
group: lock
jobs:
action:
runs-on: ubuntu-latest
steps:
- uses: dessant/lock-threads@v3
with:
process-only: 'issues'
issue-inactive-days: '180'
github-token: ${{ secrets.PROMBOT_LOCKTHREADS_TOKEN }}

30
.github/.github/workflows/test.yml vendored Normal file
View file

@ -0,0 +1,30 @@
name: ci
on:
push:
branches: [main]
pull_request:
jobs:
test:
runs-on: ubuntu-20.04
steps:
- name: Upgrade golang
run: |
cd /tmp
wget https://dl.google.com/go/go1.18.7.linux-amd64.tar.gz
tar -zxvf go1.18.7.linux-amd64.tar.gz
sudo rm -fr /usr/local/go
sudo mv /tmp/go /usr/local/go
cd -
ls -l /usr/bin/go
- name: Checkout Repo
uses: actions/checkout@v2
# This file would normally be created by `make assets`, here we just
# mock it because the file is required for the tests to pass.
- name: Mock building of necessary react file
run: mkdir web/ui/static/react && touch web/ui/static/react/index.html
- name: Run Tests
run: GO=/usr/local/go/bin/go make common-test

View file

@ -0,0 +1,44 @@
name: ui_build_and_release
on:
pull_request:
push:
branches:
- main
tags:
- "v0.[0-9]+.[0-9]+*"
jobs:
release:
name: release
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install nodejs
uses: actions/setup-node@v3
with:
node-version-file: "web/ui/.nvmrc"
- uses: actions/cache@v3.0.4
with:
path: ~/.npm
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
- name: Check libraries version
## This step is verifying that the version of each package is matching the tag
if: ${{ github.event_name == 'push' && startsWith(github.ref_name, 'v') }}
run: ./scripts/ui_release.sh --check-package "${{ github.ref_name }}"
- name: build
run: make assets
- name: Copy files before publishing libs
run: ./scripts/ui_release.sh --copy
- name: Publish dry-run libraries
if: ${{ github.event_name == 'pull_request' || github.ref_name == 'main' }}
run: ./scripts/ui_release.sh --publish dry-run
- name: Publish libraries
if: ${{ github.event_name == 'push' && startsWith(github.ref_name, 'v') }}
run: ./scripts/ui_release.sh --publish
env:
# The setup-node action writes an .npmrc file with this env variable
# as the placeholder for the auth token
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

7
.github/CODEOWNERS vendored
View file

@ -1,7 +0,0 @@
/web/ui @juliusv
/web/ui/module @juliusv @nexucis
/storage/remote @csmarchbanks @cstyan @bwplotka @tomwilkie
/discovery/kubernetes @brancz
/tsdb @codesome
/promql @roidelapluie
/cmd/promtool @dgl

View file

@ -1,22 +0,0 @@
name: buf.build
on:
pull_request:
paths:
- ".github/workflows/buf-lint.yml"
- "**.proto"
jobs:
buf:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: bufbuild/buf-setup-action@v1.16.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
- uses: bufbuild/buf-lint-action@v1
with:
input: 'prompb'
- uses: bufbuild/buf-breaking-action@v1
with:
input: 'prompb'
against: 'https://github.com/prometheus/prometheus.git#branch=main,ref=HEAD,subdir=prompb'

View file

@ -1,25 +0,0 @@
name: buf.build
on:
push:
branches:
- main
jobs:
buf:
name: lint and publish
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: bufbuild/buf-setup-action@v1.16.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
- uses: bufbuild/buf-lint-action@v1
with:
input: 'prompb'
- uses: bufbuild/buf-breaking-action@v1
with:
input: 'prompb'
against: 'https://github.com/prometheus/prometheus.git#branch=main,ref=HEAD~1,subdir=prompb'
- uses: bufbuild/buf-push-action@v1
with:
input: 'prompb'
buf_token: ${{ secrets.BUF_TOKEN }}

View file

@ -1,223 +0,0 @@
---
name: CI
on:
pull_request:
push:
jobs:
test_go:
name: Go tests
runs-on: ubuntu-latest
# Whenever the Go version is updated here, .promu.yml
# should also be updated.
container:
image: quay.io/prometheus/golang-builder:1.20-base
steps:
- uses: actions/checkout@v3
- uses: prometheus/promci@v0.1.0
- uses: ./.github/promci/actions/setup_environment
- run: make GO_ONLY=1 SKIP_GOLANGCI_LINT=1
- run: go test ./tsdb/ -test.tsdb-isolation=false
- run: go test --tags=stringlabels ./...
- run: GOARCH=386 go test ./cmd/prometheus
- run: make -C documentation/examples/remote_storage
- run: make -C documentation/examples
- uses: ./.github/promci/actions/check_proto
with:
version: "3.15.8"
test_ui:
name: UI tests
runs-on: ubuntu-latest
# Whenever the Go version is updated here, .promu.yml
# should also be updated.
container:
image: quay.io/prometheus/golang-builder:1.20-base
steps:
- uses: actions/checkout@v3
- uses: prometheus/promci@v0.1.0
- uses: ./.github/promci/actions/setup_environment
with:
enable_go: false
enable_npm: true
- run: make assets-tarball
- run: make ui-lint
- run: make ui-test
- uses: ./.github/promci/actions/save_artifacts
with:
directory: .tarballs
test_windows:
name: Go tests on Windows
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: '>=1.20 <1.21'
- run: |
$TestTargets = go list ./... | Where-Object { $_ -NotMatch "(github.com/prometheus/prometheus/discovery.*|github.com/prometheus/prometheus/config|github.com/prometheus/prometheus/web)"}
go test $TestTargets -vet=off -v
shell: powershell
test_golang_oldest:
name: Go tests with previous Go version
runs-on: ubuntu-latest
# The go verson in this image should be N-1 wrt test_go.
container:
image: quay.io/prometheus/golang-builder:1.18-base
steps:
- uses: actions/checkout@v3
- run: make build
- run: go test ./tsdb/...
- run: go test ./tsdb/ -test.tsdb-isolation=false
test_mixins:
name: Mixins tests
runs-on: ubuntu-latest
# Whenever the Go version is updated here, .promu.yml
# should also be updated.
container:
image: quay.io/prometheus/golang-builder:1.19-base
steps:
- uses: actions/checkout@v3
- run: go install ./cmd/promtool/.
- run: go install github.com/google/go-jsonnet/cmd/jsonnet@latest
- run: go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest
- run: go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest
- run: make -C documentation/prometheus-mixin clean
- run: make -C documentation/prometheus-mixin jb_install
- run: make -C documentation/prometheus-mixin
- run: git diff --exit-code
build:
name: Build Prometheus for common architectures
runs-on: ubuntu-latest
if: |
!(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.'))
&&
!(github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-'))
&&
!(github.event_name == 'push' && github.event.ref == 'refs/heads/main')
strategy:
matrix:
thread: [ 0, 1, 2 ]
steps:
- uses: actions/checkout@v3
- uses: prometheus/promci@v0.1.0
- uses: ./.github/promci/actions/build
with:
promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386"
parallelism: 3
thread: ${{ matrix.thread }}
build_all:
name: Build Prometheus for all architectures
runs-on: ubuntu-latest
if: |
(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.'))
||
(github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-'))
||
(github.event_name == 'push' && github.event.ref == 'refs/heads/main')
strategy:
matrix:
thread: [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ]
# Whenever the Go version is updated here, .promu.yml
# should also be updated.
steps:
- uses: actions/checkout@v3
- uses: prometheus/promci@v0.1.0
- uses: ./.github/promci/actions/build
with:
parallelism: 12
thread: ${{ matrix.thread }}
golangci:
name: golangci-lint
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: 1.20.x
- name: Install snmp_exporter/generator dependencies
run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
if: github.repository == 'prometheus/snmp_exporter'
- name: Lint
uses: golangci/golangci-lint-action@v3.4.0
with:
version: v1.51.2
fuzzing:
uses: ./.github/workflows/fuzzing.yml
if: github.event_name == 'pull_request'
codeql:
uses: ./.github/workflows/codeql-analysis.yml
publish_main:
name: Publish main branch artifacts
runs-on: ubuntu-latest
needs: [test_ui, test_go, test_windows, golangci, codeql, build_all]
if: github.event_name == 'push' && github.event.ref == 'refs/heads/main'
steps:
- uses: actions/checkout@v3
- uses: prometheus/promci@v0.1.0
- uses: ./.github/promci/actions/publish_main
with:
docker_hub_login: ${{ secrets.docker_hub_login }}
docker_hub_password: ${{ secrets.docker_hub_password }}
quay_io_login: ${{ secrets.quay_io_login }}
quay_io_password: ${{ secrets.quay_io_password }}
publish_release:
name: Publish release arfefacts
runs-on: ubuntu-latest
needs: [test_ui, test_go, test_windows, golangci, codeql, build_all]
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')
steps:
- uses: actions/checkout@v3
- uses: prometheus/promci@v0.1.0
- uses: ./.github/promci/actions/publish_release
with:
docker_hub_login: ${{ secrets.docker_hub_login }}
docker_hub_password: ${{ secrets.docker_hub_password }}
quay_io_login: ${{ secrets.quay_io_login }}
quay_io_password: ${{ secrets.quay_io_password }}
github_token: ${{ secrets.PROMBOT_GITHUB_TOKEN }}
publish_ui_release:
name: Publish UI on npm Registry
runs-on: ubuntu-latest
needs: [test_ui, codeql]
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: prometheus/promci@v0.1.0
- name: Install nodejs
uses: actions/setup-node@v3
with:
node-version-file: "web/ui/.nvmrc"
registry-url: "https://registry.npmjs.org"
- uses: actions/cache@v3.3.1
with:
path: ~/.npm
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
- name: Check libraries version
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')
run: ./scripts/ui_release.sh --check-package "$(echo ${{ github.ref_name }}|sed s/v2/v0/)"
- name: build
run: make assets
- name: Copy files before publishing libs
run: ./scripts/ui_release.sh --copy
- name: Publish dry-run libraries
if: "!(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.'))"
run: ./scripts/ui_release.sh --publish dry-run
- name: Publish libraries
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')
run: ./scripts/ui_release.sh --publish
env:
# The setup-node action writes an .npmrc file with this env variable
# as the placeholder for the auth token
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

View file

@ -1,37 +0,0 @@
---
name: "CodeQL"
on:
workflow_call:
schedule:
- cron: "26 14 * * 1"
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
security-events: write
strategy:
fail-fast: false
matrix:
language: ["go", "javascript"]
steps:
- name: Checkout repository
uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: '>=1.20 <1.21'
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
- name: Autobuild
uses: github/codeql-action/autobuild@v2
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2

View file

@ -1,58 +0,0 @@
on:
repository_dispatch:
types: [funcbench_start]
name: Funcbench Workflow
jobs:
run_funcbench:
name: Running funcbench
if: github.event.action == 'funcbench_start'
runs-on: ubuntu-latest
env:
AUTH_FILE: ${{ secrets.TEST_INFRA_PROVIDER_AUTH }}
BRANCH: ${{ github.event.client_payload.BRANCH }}
BENCH_FUNC_REGEX: ${{ github.event.client_payload.BENCH_FUNC_REGEX }}
PACKAGE_PATH: ${{ github.event.client_payload.PACKAGE_PATH }}
GITHUB_TOKEN: ${{ secrets.PROMBOT_GITHUB_TOKEN }}
GITHUB_ORG: prometheus
GITHUB_REPO: prometheus
GITHUB_STATUS_TARGET_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
LAST_COMMIT_SHA: ${{ github.event.client_payload.LAST_COMMIT_SHA }}
GKE_PROJECT_ID: macro-mile-203600
PR_NUMBER: ${{ github.event.client_payload.PR_NUMBER }}
PROVIDER: gke
ZONE: europe-west3-a
steps:
- name: Update status to pending
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"pending","context":"funcbench-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Prepare nodepool
uses: docker://prominfra/funcbench:master
with:
entrypoint: "docker_entrypoint"
args: make deploy
- name: Delete all resources
if: always()
uses: docker://prominfra/funcbench:master
with:
entrypoint: "docker_entrypoint"
args: make clean
- name: Update status to failure
if: failure()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"failure","context":"funcbench-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Update status to success
if: success()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"success","context":"funcbench-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"

View file

@ -1,25 +0,0 @@
name: CIFuzz
on:
workflow_call:
jobs:
Fuzzing:
runs-on: ubuntu-latest
steps:
- name: Build Fuzzers
id: build
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
with:
oss-fuzz-project-name: "prometheus"
dry-run: false
- name: Run Fuzzers
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
with:
oss-fuzz-project-name: "prometheus"
fuzz-seconds: 600
dry-run: false
- name: Upload Crash
uses: actions/upload-artifact@v3
if: failure() && steps.build.outcome == 'success'
with:
name: artifacts
path: ./out/artifacts

30
.github/workflows/golangci-lint.yml vendored Normal file
View file

@ -0,0 +1,30 @@
name: golangci-lint
on:
push:
paths:
- "go.sum"
- "go.mod"
- "**.go"
- "scripts/errcheck_excludes.txt"
- ".github/workflows/golangci-lint.yml"
- ".golangci.yml"
pull_request:
jobs:
golangci:
name: lint
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: install Go
uses: actions/setup-go@v2
with:
go-version: '<1.19'
- name: Install snmp_exporter/generator dependencies
run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
if: github.repository == 'prometheus/snmp_exporter'
- name: Lint
uses: golangci/golangci-lint-action@v3.3.1
with:
version: v1.50.1

View file

@ -1,126 +0,0 @@
on:
repository_dispatch:
types: [prombench_start, prombench_restart, prombench_stop]
name: Prombench Workflow
env:
AUTH_FILE: ${{ secrets.TEST_INFRA_PROVIDER_AUTH }}
CLUSTER_NAME: test-infra
DOMAIN_NAME: prombench.prometheus.io
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_ORG: prometheus
GITHUB_REPO: prometheus
GITHUB_STATUS_TARGET_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
LAST_COMMIT_SHA: ${{ github.event.client_payload.LAST_COMMIT_SHA }}
GKE_PROJECT_ID: macro-mile-203600
PR_NUMBER: ${{ github.event.client_payload.PR_NUMBER }}
PROVIDER: gke
RELEASE: ${{ github.event.client_payload.RELEASE }}
ZONE: europe-west3-a
jobs:
benchmark_start:
name: Benchmark Start
if: github.event.action == 'prombench_start'
runs-on: ubuntu-latest
steps:
- name: Update status to pending
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"pending", "context": "prombench-status-update-start", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Run make deploy to start test
id: make_deploy
uses: docker://prominfra/prombench:master
with:
args: >-
until make all_nodes_deleted; do echo "waiting for nodepools to be deleted"; sleep 10; done;
make deploy;
- name: Update status to failure
if: failure()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"failure", "context": "prombench-status-update-start", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Update status to success
if: success()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"success", "context": "prombench-status-update-start", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
benchmark_cancel:
name: Benchmark Cancel
if: github.event.action == 'prombench_stop'
runs-on: ubuntu-latest
steps:
- name: Update status to pending
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"pending", "context": "prombench-status-update-cancel", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Run make clean to stop test
id: make_clean
uses: docker://prominfra/prombench:master
with:
args: >-
until make all_nodes_running; do echo "waiting for nodepools to be created"; sleep 10; done;
make clean;
- name: Update status to failure
if: failure()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"failure", "context": "prombench-status-update-cancel", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Update status to success
if: success()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"success", "context": "prombench-status-update-cancel", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
benchmark_restart:
name: Benchmark Restart
if: github.event.action == 'prombench_restart'
runs-on: ubuntu-latest
steps:
- name: Update status to pending
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"pending", "context": "prombench-status-update-restart", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Run make clean then make deploy to restart test
id: make_restart
uses: docker://prominfra/prombench:master
with:
args: >-
until make all_nodes_running; do echo "waiting for nodepools to be created"; sleep 10; done;
make clean;
until make all_nodes_deleted; do echo "waiting for nodepools to be deleted"; sleep 10; done;
make deploy;
- name: Update status to failure
if: failure()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"failure", "context": "prombench-status-update-restart", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
- name: Update status to success
if: success()
run: >-
curl -i -X POST
-H "Authorization: Bearer $GITHUB_TOKEN"
-H "Content-Type: application/json"
--data '{"state":"success", "context": "prombench-status-update-restart", "target_url": "'$GITHUB_STATUS_TARGET_URL'"}'
"https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"

View file

@ -1,15 +0,0 @@
---
name: Sync repo files
on:
schedule:
- cron: '44 17 * * *'
jobs:
repo_sync:
runs-on: ubuntu-latest
container:
image: quay.io/prometheus/golang-builder
steps:
- uses: actions/checkout@v3
- run: ./scripts/sync_repo_files.sh
env:
GITHUB_TOKEN: ${{ secrets.PROMBOT_GITHUB_TOKEN }}

17
.github/workflows/sync-fork.yml vendored Normal file
View file

@ -0,0 +1,17 @@
name: sync fork with upstream
on:
schedule:
- cron: '11 8 * * 1' # 8:11 UTC on Monday
workflow_dispatch: # for manual testing
jobs:
sync-fork-pr:
runs-on: ubuntu-latest
steps:
- uses: tgymnich/fork-sync@v1.7
with:
owner: grafana
base: main
head: main

30
.github/workflows/test.yml vendored Normal file
View file

@ -0,0 +1,30 @@
name: ci
on:
push:
branches: [main]
pull_request:
jobs:
test:
runs-on: ubuntu-20.04
steps:
- name: Upgrade golang
run: |
cd /tmp
wget https://dl.google.com/go/go1.18.7.linux-amd64.tar.gz
tar -zxvf go1.18.7.linux-amd64.tar.gz
sudo rm -fr /usr/local/go
sudo mv /tmp/go /usr/local/go
cd -
ls -l /usr/bin/go
- name: Checkout Repo
uses: actions/checkout@v2
# This file would normally be created by `make assets`, here we just
# mock it because the file is required for the tests to pass.
- name: Mock building of necessary react file
run: mkdir web/ui/static/react && touch web/ui/static/react/index.html
- name: Run Tests
run: GO=/usr/local/go/bin/go make common-test

View file

@ -0,0 +1,44 @@
name: ui_build_and_release
on:
pull_request:
push:
branches:
- main
tags:
- "v0.[0-9]+.[0-9]+*"
jobs:
release:
name: release
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install nodejs
uses: actions/setup-node@v3
with:
node-version-file: "web/ui/.nvmrc"
- uses: actions/cache@v3.0.4
with:
path: ~/.npm
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
- name: Check libraries version
## This step is verifying that the version of each package is matching the tag
if: ${{ github.event_name == 'push' && startsWith(github.ref_name, 'v') }}
run: ./scripts/ui_release.sh --check-package "${{ github.ref_name }}"
- name: build
run: make assets
- name: Copy files before publishing libs
run: ./scripts/ui_release.sh --copy
- name: Publish dry-run libraries
if: ${{ github.event_name == 'pull_request' || github.ref_name == 'main' }}
run: ./scripts/ui_release.sh --publish dry-run
- name: Publish libraries
if: ${{ github.event_name == 'push' && startsWith(github.ref_name, 'v') }}
run: ./scripts/ui_release.sh --publish
env:
# The setup-node action writes an .npmrc file with this env variable
# as the placeholder for the auth token
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

View file

@ -100,6 +100,8 @@ ifeq ($(GOHOSTARCH),amd64)
endif
endif
test-flags += -timeout 20m
# This rule is used to forward a target like "build" to "common-build". This
# allows a new "build" target to be defined in a Makefile which includes this
# one and override "common-build" without override warnings.

89
cmd/compact/main.go Normal file
View file

@ -0,0 +1,89 @@
package main
import (
"context"
"flag"
"log"
"os"
"os/signal"
"runtime/pprof"
"syscall"
golog "github.com/go-kit/log"
"github.com/prometheus/prometheus/tsdb"
)
func main() {
var (
outputDir string
shardCount int
cpuProf string
segmentSizeMB int64
maxClosingBlocks int
symbolFlushers int
openConcurrency int
)
flag.StringVar(&outputDir, "output-dir", ".", "Output directory for new block(s)")
flag.StringVar(&cpuProf, "cpuprofile", "", "Where to store CPU profile (it not empty)")
flag.IntVar(&shardCount, "shard-count", 1, "Number of shards for splitting")
flag.Int64Var(&segmentSizeMB, "segment-file-size", 512, "Size of segment file")
flag.IntVar(&maxClosingBlocks, "max-closing-blocks", 2, "Number of blocks that can close at once during split compaction")
flag.IntVar(&symbolFlushers, "symbol-flushers", 4, "Number of symbol flushers used during split compaction")
flag.IntVar(&openConcurrency, "open-concurrency", 4, "Number of goroutines used when opening blocks")
flag.Parse()
logger := golog.NewLogfmtLogger(os.Stderr)
var blockDirs []string
for _, d := range flag.Args() {
s, err := os.Stat(d)
if err != nil {
panic(err)
}
if !s.IsDir() {
log.Fatalln("not a directory: ", d)
}
blockDirs = append(blockDirs, d)
}
if len(blockDirs) == 0 {
log.Fatalln("no blocks to compact")
}
if cpuProf != "" {
f, err := os.Create(cpuProf)
if err != nil {
log.Fatalln(err)
}
log.Println("writing to", cpuProf)
err = pprof.StartCPUProfile(f)
if err != nil {
log.Fatalln(err)
}
defer pprof.StopCPUProfile()
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
c, err := tsdb.NewLeveledCompactorWithChunkSize(ctx, nil, logger, []int64{0}, nil, segmentSizeMB*1024*1024, nil, true)
if err != nil {
log.Fatalln("creating compator", err)
}
opts := tsdb.DefaultLeveledCompactorConcurrencyOptions()
opts.MaxClosingBlocks = maxClosingBlocks
opts.SymbolsFlushersCount = symbolFlushers
opts.MaxOpeningBlocks = openConcurrency
c.SetConcurrencyOptions(opts)
_, err = c.CompactWithSplitting(outputDir, blockDirs, nil, uint64(shardCount))
if err != nil {
log.Fatalln("compacting", err)
}
}

3
go.mod
View file

@ -28,6 +28,7 @@ require (
github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd
github.com/grpc-ecosystem/grpc-gateway v1.16.0
github.com/hashicorp/consul/api v1.20.0
github.com/hashicorp/golang-lru/v2 v2.0.2
github.com/hashicorp/nomad/api v0.0.0-20230308192510-48e7d70fcd4b
github.com/hetznercloud/hcloud-go v1.41.0
github.com/ionos-cloud/sdk-go/v6 v6.1.5
@ -160,7 +161,7 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect

2
go.sum
View file

@ -444,6 +444,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4=
github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU=
github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=

View file

@ -118,3 +118,22 @@ func (m *Matcher) GetRegexString() string {
}
return m.re.GetRegexString()
}
// SetMatches returns a set of equality matchers for the current regex matchers if possible.
// For examples the regexp `a(b|f)` will returns "ab" and "af".
// Returns nil if we can't replace the regexp by only equality matchers.
func (m *Matcher) SetMatches() []string {
if m.re == nil {
return nil
}
return m.re.SetMatches()
}
// Prefix returns the required prefix of the value to match, if possible.
// It will be empty if it's an equality matcher or if the prefix can't be determined.
func (m *Matcher) Prefix() string {
if m.re == nil {
return ""
}
return m.re.prefix
}

View file

@ -14,13 +14,14 @@
package labels
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func mustNewMatcher(t *testing.T, mType MatchType, value string) *Matcher {
m, err := NewMatcher(mType, "", value)
m, err := NewMatcher(mType, "test_label_name", value)
require.NoError(t, err)
return m
}
@ -81,6 +82,21 @@ func TestMatcher(t *testing.T) {
value: "foo-bar",
match: false,
},
{
matcher: mustNewMatcher(t, MatchRegexp, "$*bar"),
value: "foo-bar",
match: false,
},
{
matcher: mustNewMatcher(t, MatchRegexp, "bar^+"),
value: "foo-bar",
match: false,
},
{
matcher: mustNewMatcher(t, MatchRegexp, "$+bar"),
value: "foo-bar",
match: false,
},
}
for _, test := range tests {
@ -118,6 +134,58 @@ func TestInverse(t *testing.T) {
}
}
func TestPrefix(t *testing.T) {
for i, tc := range []struct {
matcher *Matcher
prefix string
}{
{
matcher: mustNewMatcher(t, MatchEqual, "abc"),
prefix: "",
},
{
matcher: mustNewMatcher(t, MatchNotEqual, "abc"),
prefix: "",
},
{
matcher: mustNewMatcher(t, MatchRegexp, "abc.+"),
prefix: "abc",
},
{
matcher: mustNewMatcher(t, MatchRegexp, "abcd|abc.+"),
prefix: "abc",
},
{
matcher: mustNewMatcher(t, MatchNotRegexp, "abcd|abc.+"),
prefix: "abc",
},
{
matcher: mustNewMatcher(t, MatchRegexp, "abc(def|ghj)|ab|a."),
prefix: "a",
},
{
matcher: mustNewMatcher(t, MatchRegexp, "foo.+bar|foo.*baz"),
prefix: "foo",
},
{
matcher: mustNewMatcher(t, MatchRegexp, "abc|.*"),
prefix: "",
},
{
matcher: mustNewMatcher(t, MatchRegexp, "abc|def"),
prefix: "",
},
{
matcher: mustNewMatcher(t, MatchRegexp, ".+def"),
prefix: "",
},
} {
t.Run(fmt.Sprintf("%d: %s", i, tc.matcher), func(t *testing.T) {
require.Equal(t, tc.prefix, tc.matcher.Prefix())
})
}
}
func BenchmarkMatchType_String(b *testing.B) {
for i := 0; i <= b.N; i++ {
_ = MatchType(i % int(MatchNotRegexp+1)).String()

View file

@ -18,52 +18,369 @@ import (
"github.com/grafana/regexp"
"github.com/grafana/regexp/syntax"
lru "github.com/hashicorp/golang-lru/v2"
)
const (
maxSetMatches = 256
// The minimum number of alternate values a regex should have to trigger
// the optimization done by optimizeEqualStringMatchers() and so use a map
// to match values instead of iterating over a list. This value has
// been computed running BenchmarkOptimizeEqualStringMatchers.
minEqualMultiStringMatcherMapThreshold = 16
)
var fastRegexMatcherCache *lru.Cache[string, *FastRegexMatcher]
func init() {
// Ignore error because it can only return error if size is invalid,
// but we're using an hardcoded size here.
fastRegexMatcherCache, _ = lru.New[string, *FastRegexMatcher](10000)
}
type FastRegexMatcher struct {
// Under some conditions, re is nil because the expression is never parsed.
// We store the original string to be able to return it in GetRegexString().
reString string
re *regexp.Regexp
prefix string
suffix string
contains string
setMatches []string
stringMatcher StringMatcher
prefix string
suffix string
contains string
// matchString is the "compiled" function to run by MatchString().
matchString func(string) bool
}
func NewFastRegexMatcher(v string) (*FastRegexMatcher, error) {
re, err := regexp.Compile("^(?:" + v + ")$")
// Check the cache.
if matcher, ok := fastRegexMatcherCache.Get(v); ok {
return matcher, nil
}
// Create a new matcher.
matcher, err := newFastRegexMatcherWithoutCache(v)
if err != nil {
return nil, err
}
parsed, err := syntax.Parse(v, syntax.Perl)
if err != nil {
return nil, err
}
// Cache it.
fastRegexMatcherCache.Add(v, matcher)
return matcher, nil
}
func newFastRegexMatcherWithoutCache(v string) (*FastRegexMatcher, error) {
m := &FastRegexMatcher{
re: re,
reString: v,
}
if parsed.Op == syntax.OpConcat {
m.prefix, m.suffix, m.contains = optimizeConcatRegex(parsed)
m.stringMatcher, m.setMatches = optimizeAlternatingLiterals(v)
if m.stringMatcher != nil {
// If we already have a string matcher, we don't need to parse the regex
// or compile the matchString function. This also avoids the behavior in
// compileMatchStringFunction where it prefers to use setMatches when
// available, even if the string matcher is faster.
m.matchString = m.stringMatcher.Matches
} else {
parsed, err := syntax.Parse(v, syntax.Perl)
if err != nil {
return nil, err
}
// Simplify the syntax tree to run faster.
parsed = parsed.Simplify()
m.re, err = regexp.Compile("^(?:" + parsed.String() + ")$")
if err != nil {
return nil, err
}
if parsed.Op == syntax.OpConcat {
m.prefix, m.suffix, m.contains = optimizeConcatRegex(parsed)
}
if matches, caseSensitive := findSetMatches(parsed); caseSensitive {
m.setMatches = matches
}
m.stringMatcher = stringMatcherFromRegexp(parsed)
m.matchString = m.compileMatchStringFunction()
}
return m, nil
}
// compileMatchStringFunction returns the function to run by MatchString().
func (m *FastRegexMatcher) compileMatchStringFunction() func(string) bool {
// If the only optimization available is the string matcher, then we can just run it.
if len(m.setMatches) == 0 && m.prefix == "" && m.suffix == "" && m.contains == "" && m.stringMatcher != nil {
return m.stringMatcher.Matches
}
return func(s string) bool {
if len(m.setMatches) != 0 {
for _, match := range m.setMatches {
if match == s {
return true
}
}
return false
}
if m.prefix != "" && !strings.HasPrefix(s, m.prefix) {
return false
}
if m.suffix != "" && !strings.HasSuffix(s, m.suffix) {
return false
}
if m.contains != "" && !strings.Contains(s, m.contains) {
return false
}
if m.stringMatcher != nil {
return m.stringMatcher.Matches(s)
}
return m.re.MatchString(s)
}
}
// isOptimized returns true if any fast-path optimization is applied to the
// regex matcher.
//
//nolint:unused
func (m *FastRegexMatcher) isOptimized() bool {
return len(m.setMatches) > 0 || m.stringMatcher != nil || m.prefix != "" || m.suffix != "" || m.contains != ""
}
// findSetMatches extract equality matches from a regexp.
// Returns nil if we can't replace the regexp by only equality matchers or the regexp contains
// a mix of case sensitive and case insensitive matchers.
func findSetMatches(re *syntax.Regexp) (matches []string, caseSensitive bool) {
clearBeginEndText(re)
return findSetMatchesInternal(re, "")
}
func findSetMatchesInternal(re *syntax.Regexp, base string) (matches []string, caseSensitive bool) {
switch re.Op {
case syntax.OpBeginText:
// Correctly handling the begin text operator inside a regex is tricky,
// so in this case we fallback to the regex engine.
return nil, false
case syntax.OpEndText:
// Correctly handling the end text operator inside a regex is tricky,
// so in this case we fallback to the regex engine.
return nil, false
case syntax.OpLiteral:
return []string{base + string(re.Rune)}, isCaseSensitive(re)
case syntax.OpEmptyMatch:
if base != "" {
return []string{base}, isCaseSensitive(re)
}
case syntax.OpAlternate:
return findSetMatchesFromAlternate(re, base)
case syntax.OpCapture:
clearCapture(re)
return findSetMatchesInternal(re, base)
case syntax.OpConcat:
return findSetMatchesFromConcat(re, base)
case syntax.OpCharClass:
if len(re.Rune)%2 != 0 {
return nil, false
}
var matches []string
var totalSet int
for i := 0; i+1 < len(re.Rune); i = i + 2 {
totalSet += int(re.Rune[i+1]-re.Rune[i]) + 1
}
// limits the total characters that can be used to create matches.
// In some case like negation [^0-9] a lot of possibilities exists and that
// can create thousands of possible matches at which points we're better off using regexp.
if totalSet > maxSetMatches {
return nil, false
}
for i := 0; i+1 < len(re.Rune); i = i + 2 {
lo, hi := re.Rune[i], re.Rune[i+1]
for c := lo; c <= hi; c++ {
matches = append(matches, base+string(c))
}
}
return matches, isCaseSensitive(re)
default:
return nil, false
}
return nil, false
}
func findSetMatchesFromConcat(re *syntax.Regexp, base string) (matches []string, matchesCaseSensitive bool) {
if len(re.Sub) == 0 {
return nil, false
}
clearCapture(re.Sub...)
matches = []string{base}
for i := 0; i < len(re.Sub); i++ {
var newMatches []string
for j, b := range matches {
m, caseSensitive := findSetMatchesInternal(re.Sub[i], b)
if m == nil {
return nil, false
}
if tooManyMatches(newMatches, m...) {
return nil, false
}
// All matches must have the same case sensitivity. If it's the first set of matches
// returned, we store its sensitivity as the expected case, and then we'll check all
// other ones.
if i == 0 && j == 0 {
matchesCaseSensitive = caseSensitive
}
if matchesCaseSensitive != caseSensitive {
return nil, false
}
newMatches = append(newMatches, m...)
}
matches = newMatches
}
return matches, matchesCaseSensitive
}
func findSetMatchesFromAlternate(re *syntax.Regexp, base string) (matches []string, matchesCaseSensitive bool) {
for i, sub := range re.Sub {
found, caseSensitive := findSetMatchesInternal(sub, base)
if found == nil {
return nil, false
}
if tooManyMatches(matches, found...) {
return nil, false
}
// All matches must have the same case sensitivity. If it's the first set of matches
// returned, we store its sensitivity as the expected case, and then we'll check all
// other ones.
if i == 0 {
matchesCaseSensitive = caseSensitive
}
if matchesCaseSensitive != caseSensitive {
return nil, false
}
matches = append(matches, found...)
}
return matches, matchesCaseSensitive
}
// clearCapture removes capture operation as they are not used for matching.
func clearCapture(regs ...*syntax.Regexp) {
for _, r := range regs {
if r.Op == syntax.OpCapture {
*r = *r.Sub[0]
}
}
}
// clearBeginEndText removes the begin and end text from the regexp. Prometheus regexp are anchored to the beginning and end of the string.
func clearBeginEndText(re *syntax.Regexp) {
// Do not clear begin/end text from an alternate operator because it could
// change the actual regexp properties.
if re.Op == syntax.OpAlternate {
return
}
if len(re.Sub) == 0 {
return
}
if len(re.Sub) == 1 {
if re.Sub[0].Op == syntax.OpBeginText || re.Sub[0].Op == syntax.OpEndText {
// We need to remove this element. Since it's the only one, we convert into a matcher of an empty string.
// OpEmptyMatch is regexp's nop operator.
re.Op = syntax.OpEmptyMatch
re.Sub = nil
return
}
}
if re.Sub[0].Op == syntax.OpBeginText {
re.Sub = re.Sub[1:]
}
if re.Sub[len(re.Sub)-1].Op == syntax.OpEndText {
re.Sub = re.Sub[:len(re.Sub)-1]
}
}
// isCaseInsensitive tells if a regexp is case insensitive.
// The flag should be check at each level of the syntax tree.
func isCaseInsensitive(reg *syntax.Regexp) bool {
return (reg.Flags & syntax.FoldCase) != 0
}
// isCaseSensitive tells if a regexp is case sensitive.
// The flag should be check at each level of the syntax tree.
func isCaseSensitive(reg *syntax.Regexp) bool {
return !isCaseInsensitive(reg)
}
// tooManyMatches guards against creating too many set matches
func tooManyMatches(matches []string, new ...string) bool {
return len(matches)+len(new) > maxSetMatches
}
func (m *FastRegexMatcher) MatchString(s string) bool {
if m.prefix != "" && !strings.HasPrefix(s, m.prefix) {
return false
}
if m.suffix != "" && !strings.HasSuffix(s, m.suffix) {
return false
}
if m.contains != "" && !strings.Contains(s, m.contains) {
return false
}
return m.re.MatchString(s)
return m.matchString(s)
}
func (m *FastRegexMatcher) SetMatches() []string {
return m.setMatches
}
func (m *FastRegexMatcher) GetRegexString() string {
return m.re.String()
return m.reString
}
// optimizeAlternatingLiterals optimizes a regex of the form
//
// `literal1|literal2|literal3|...`
//
// this function returns an optimized StringMatcher or nil if the regex
// cannot be optimized in this way, and a list of setMatches up to maxSetMatches
func optimizeAlternatingLiterals(s string) (StringMatcher, []string) {
if len(s) == 0 {
return emptyStringMatcher{}, nil
}
estimatedAlternates := strings.Count(s, "|") + 1
// If there are no alternates, check if the string is a literal
if estimatedAlternates == 1 {
if regexp.QuoteMeta(s) == s {
return &equalStringMatcher{s: s, caseSensitive: true}, nil
}
return nil, nil
}
multiMatcher := newEqualMultiStringMatcher(true, estimatedAlternates)
for end := strings.IndexByte(s, '|'); end > -1; end = strings.IndexByte(s, '|') {
// Split the string into the next literal and the remainder
subMatch := s[:end]
s = s[end+1:]
// break if any of the submatches are not literals
if regexp.QuoteMeta(subMatch) != subMatch {
return nil, nil
}
multiMatcher.add(subMatch)
}
// break if the remainder is not a literal
if regexp.QuoteMeta(s) != s {
return nil, nil
}
multiMatcher.add(s)
return multiMatcher, multiMatcher.setMatches()
}
// optimizeConcatRegex returns literal prefix/suffix text that can be safely
@ -106,3 +423,408 @@ func optimizeConcatRegex(r *syntax.Regexp) (prefix, suffix, contains string) {
return
}
// StringMatcher is a matcher that matches a string in place of a regular expression.
type StringMatcher interface {
Matches(s string) bool
}
// stringMatcherFromRegexp attempts to replace a common regexp with a string matcher.
// It returns nil if the regexp is not supported.
// For examples, it will replace `.*foo` with `foo.*` and `.*foo.*` with `(?i)foo`.
func stringMatcherFromRegexp(re *syntax.Regexp) StringMatcher {
clearBeginEndText(re)
m := stringMatcherFromRegexpInternal(re)
m = optimizeEqualStringMatchers(m, minEqualMultiStringMatcherMapThreshold)
return m
}
func stringMatcherFromRegexpInternal(re *syntax.Regexp) StringMatcher {
clearCapture(re)
switch re.Op {
case syntax.OpBeginText:
// Correctly handling the begin text operator inside a regex is tricky,
// so in this case we fallback to the regex engine.
return nil
case syntax.OpEndText:
// Correctly handling the end text operator inside a regex is tricky,
// so in this case we fallback to the regex engine.
return nil
case syntax.OpPlus:
if re.Sub[0].Op != syntax.OpAnyChar && re.Sub[0].Op != syntax.OpAnyCharNotNL {
return nil
}
return &anyNonEmptyStringMatcher{
matchNL: re.Sub[0].Op == syntax.OpAnyChar,
}
case syntax.OpStar:
if re.Sub[0].Op != syntax.OpAnyChar && re.Sub[0].Op != syntax.OpAnyCharNotNL {
return nil
}
// If the newline is valid, than this matcher literally match any string (even empty).
if re.Sub[0].Op == syntax.OpAnyChar {
return trueMatcher{}
}
// Any string is fine (including an empty one), as far as it doesn't contain any newline.
return anyStringWithoutNewlineMatcher{}
case syntax.OpEmptyMatch:
return emptyStringMatcher{}
case syntax.OpLiteral:
return &equalStringMatcher{
s: string(re.Rune),
caseSensitive: !isCaseInsensitive(re),
}
case syntax.OpAlternate:
or := make([]StringMatcher, 0, len(re.Sub))
for _, sub := range re.Sub {
m := stringMatcherFromRegexpInternal(sub)
if m == nil {
return nil
}
or = append(or, m)
}
return orStringMatcher(or)
case syntax.OpConcat:
clearCapture(re.Sub...)
if len(re.Sub) == 0 {
return emptyStringMatcher{}
}
if len(re.Sub) == 1 {
return stringMatcherFromRegexpInternal(re.Sub[0])
}
var left, right StringMatcher
// Let's try to find if there's a first and last any matchers.
if re.Sub[0].Op == syntax.OpPlus || re.Sub[0].Op == syntax.OpStar {
left = stringMatcherFromRegexpInternal(re.Sub[0])
if left == nil {
return nil
}
re.Sub = re.Sub[1:]
}
if re.Sub[len(re.Sub)-1].Op == syntax.OpPlus || re.Sub[len(re.Sub)-1].Op == syntax.OpStar {
right = stringMatcherFromRegexpInternal(re.Sub[len(re.Sub)-1])
if right == nil {
return nil
}
re.Sub = re.Sub[:len(re.Sub)-1]
}
matches, matchesCaseSensitive := findSetMatchesInternal(re, "")
if len(matches) == 0 {
return nil
}
if left == nil && right == nil {
// if there's no any matchers on both side it's a concat of literals
or := make([]StringMatcher, 0, len(matches))
for _, match := range matches {
or = append(or, &equalStringMatcher{
s: match,
caseSensitive: matchesCaseSensitive,
})
}
return orStringMatcher(or)
}
// We found literals in the middle. We can triggered the fast path only if
// the matches are case sensitive because containsStringMatcher doesn't
// support case insensitive.
if matchesCaseSensitive {
return &containsStringMatcher{
substrings: matches,
left: left,
right: right,
}
}
}
return nil
}
// containsStringMatcher matches a string if it contains any of the substrings.
// If left and right are not nil, it's a contains operation where left and right must match.
// If left is nil, it's a hasPrefix operation and right must match.
// Finally if right is nil it's a hasSuffix operation and left must match.
type containsStringMatcher struct {
substrings []string
left StringMatcher
right StringMatcher
}
func (m *containsStringMatcher) Matches(s string) bool {
for _, substr := range m.substrings {
if m.right != nil && m.left != nil {
searchStartPos := 0
for {
pos := strings.Index(s[searchStartPos:], substr)
if pos < 0 {
break
}
// Since we started searching from searchStartPos, we have to add that offset
// to get the actual position of the substring inside the text.
pos += searchStartPos
// If both the left and right matchers match, then we can stop searching because
// we've found a match.
if m.left.Matches(s[:pos]) && m.right.Matches(s[pos+len(substr):]) {
return true
}
// Continue searching for another occurrence of the substring inside the text.
searchStartPos = pos + 1
}
} else if m.left != nil {
// If we have to check for characters on the left then we need to match a suffix.
if strings.HasSuffix(s, substr) && m.left.Matches(s[:len(s)-len(substr)]) {
return true
}
} else if m.right != nil {
if strings.HasPrefix(s, substr) && m.right.Matches(s[len(substr):]) {
return true
}
}
}
return false
}
// emptyStringMatcher matches an empty string.
type emptyStringMatcher struct{}
func (m emptyStringMatcher) Matches(s string) bool {
return len(s) == 0
}
// orStringMatcher matches any of the sub-matchers.
type orStringMatcher []StringMatcher
func (m orStringMatcher) Matches(s string) bool {
for _, matcher := range m {
if matcher.Matches(s) {
return true
}
}
return false
}
// equalStringMatcher matches a string exactly and support case insensitive.
type equalStringMatcher struct {
s string
caseSensitive bool
}
func (m *equalStringMatcher) Matches(s string) bool {
if m.caseSensitive {
return m.s == s
}
return strings.EqualFold(m.s, s)
}
type multiStringMatcherBuilder interface {
StringMatcher
add(s string)
setMatches() []string
}
func newEqualMultiStringMatcher(caseSensitive bool, estimatedSize int) multiStringMatcherBuilder {
// If the estimated size is low enough, it's faster to use a slice instead of a map.
if estimatedSize < minEqualMultiStringMatcherMapThreshold {
return &equalMultiStringSliceMatcher{caseSensitive: caseSensitive, values: make([]string, 0, estimatedSize)}
}
return &equalMultiStringMapMatcher{
values: make(map[string]struct{}, estimatedSize),
caseSensitive: caseSensitive,
}
}
// equalMultiStringSliceMatcher matches a string exactly against a slice of valid values.
type equalMultiStringSliceMatcher struct {
values []string
caseSensitive bool
}
func (m *equalMultiStringSliceMatcher) add(s string) {
m.values = append(m.values, s)
}
func (m *equalMultiStringSliceMatcher) setMatches() []string {
return m.values
}
func (m *equalMultiStringSliceMatcher) Matches(s string) bool {
if m.caseSensitive {
for _, v := range m.values {
if s == v {
return true
}
}
} else {
for _, v := range m.values {
if strings.EqualFold(s, v) {
return true
}
}
}
return false
}
// equalMultiStringMapMatcher matches a string exactly against a map of valid values.
type equalMultiStringMapMatcher struct {
// values contains values to match a string against. If the matching is case insensitive,
// the values here must be lowercase.
values map[string]struct{}
caseSensitive bool
}
func (m *equalMultiStringMapMatcher) add(s string) {
if !m.caseSensitive {
s = strings.ToLower(s)
}
m.values[s] = struct{}{}
}
func (m *equalMultiStringMapMatcher) setMatches() []string {
if len(m.values) >= maxSetMatches {
return nil
}
matches := make([]string, 0, len(m.values))
for s := range m.values {
matches = append(matches, s)
}
return matches
}
func (m *equalMultiStringMapMatcher) Matches(s string) bool {
if !m.caseSensitive {
s = strings.ToLower(s)
}
_, ok := m.values[s]
return ok
}
// anyStringWithoutNewlineMatcher is a stringMatcher which matches any string
// (including an empty one) as far as it doesn't contain any newline character.
type anyStringWithoutNewlineMatcher struct{}
func (m anyStringWithoutNewlineMatcher) Matches(s string) bool {
// We need to make sure it doesn't contain a newline. Since the newline is
// an ASCII character, we can use strings.IndexByte().
return strings.IndexByte(s, '\n') == -1
}
// anyNonEmptyStringMatcher is a stringMatcher which matches any non-empty string.
type anyNonEmptyStringMatcher struct {
matchNL bool
}
func (m *anyNonEmptyStringMatcher) Matches(s string) bool {
if m.matchNL {
// It's OK if the string contains a newline so we just need to make
// sure it's non-empty.
return len(s) > 0
}
// We need to make sure it non-empty and doesn't contain a newline.
// Since the newline is an ASCII character, we can use strings.IndexByte().
return len(s) > 0 && strings.IndexByte(s, '\n') == -1
}
// trueMatcher is a stringMatcher which matches any string (always returns true).
type trueMatcher struct{}
func (m trueMatcher) Matches(_ string) bool {
return true
}
// optimizeEqualStringMatchers optimize a specific case where all matchers are made by an
// alternation (orStringMatcher) of strings checked for equality (equalStringMatcher). In
// this specific case, when we have many strings to match against we can use a map instead
// of iterating over the list of strings.
func optimizeEqualStringMatchers(input StringMatcher, threshold int) StringMatcher {
var (
caseSensitive bool
caseSensitiveSet bool
numValues int
)
// Analyse the input StringMatcher to count the number of occurrences
// and ensure all of them have the same case sensitivity.
analyseCallback := func(matcher *equalStringMatcher) bool {
// Ensure we don't have mixed case sensitivity.
if caseSensitiveSet && caseSensitive != matcher.caseSensitive {
return false
} else if !caseSensitiveSet {
caseSensitive = matcher.caseSensitive
caseSensitiveSet = true
}
numValues++
return true
}
if !findEqualStringMatchers(input, analyseCallback) {
return input
}
// If the number of values found is less than the threshold, then we should skip the optimization.
if numValues < threshold {
return input
}
// Parse again the input StringMatcher to extract all values and storing them.
// We can skip the case sensitivity check because we've already checked it and
// if the code reach this point then it means all matchers have the same case sensitivity.
multiMatcher := newEqualMultiStringMatcher(caseSensitive, numValues)
// Ignore the return value because we already iterated over the input StringMatcher
// and it was all good.
findEqualStringMatchers(input, func(matcher *equalStringMatcher) bool {
multiMatcher.add(matcher.s)
return true
})
return multiMatcher
}
// findEqualStringMatchers analyze the input StringMatcher and calls the callback for each
// equalStringMatcher found. Returns true if and only if the input StringMatcher is *only*
// composed by an alternation of equalStringMatcher.
func findEqualStringMatchers(input StringMatcher, callback func(matcher *equalStringMatcher) bool) bool {
orInput, ok := input.(orStringMatcher)
if !ok {
return false
}
for _, m := range orInput {
switch casted := m.(type) {
case orStringMatcher:
if !findEqualStringMatchers(m, callback) {
return false
}
case *equalStringMatcher:
if !callback(casted) {
return false
}
default:
// It's not an equal string matcher, so we have to stop searching
// cause this optimization can't be applied.
return false
}
}
return true
}

File diff suppressed because one or more lines are too long

32
model/labels/sharding.go Normal file
View file

@ -0,0 +1,32 @@
package labels
import (
"github.com/cespare/xxhash/v2"
)
// StableHash is a labels hashing implementation which is guaranteed to not change over time.
// This function should be used whenever labels hashing backward compatibility must be guaranteed.
func StableHash(ls Labels) uint64 {
// Use xxhash.Sum64(b) for fast path as it's faster.
b := make([]byte, 0, 1024)
for i, v := range ls {
if len(b)+len(v.Name)+len(v.Value)+2 >= cap(b) {
// If labels entry is 1KB+ do not allocate whole entry.
h := xxhash.New()
_, _ = h.Write(b)
for _, v := range ls[i:] {
_, _ = h.WriteString(v.Name)
_, _ = h.Write(seps)
_, _ = h.WriteString(v.Value)
_, _ = h.Write(seps)
}
return h.Sum64()
}
b = append(b, v.Name...)
b = append(b, seps[0])
b = append(b, v.Value...)
b = append(b, seps[0])
}
return xxhash.Sum64(b)
}

View file

@ -135,10 +135,13 @@ func (g *RuleGroups) Validate(node ruleGroups) (errs []error) {
// RuleGroup is a list of sequentially evaluated recording and alerting rules.
type RuleGroup struct {
Name string `yaml:"name"`
Interval model.Duration `yaml:"interval,omitempty"`
Limit int `yaml:"limit,omitempty"`
Rules []RuleNode `yaml:"rules"`
Name string `yaml:"name"`
Interval model.Duration `yaml:"interval,omitempty"`
EvaluationDelay *model.Duration `yaml:"evaluation_delay,omitempty"`
Limit int `yaml:"limit,omitempty"`
Rules []RuleNode `yaml:"rules"`
SourceTenants []string `yaml:"source_tenants,omitempty"`
AlignEvaluationTimeOnInterval bool `yaml:"align_evaluation_time_on_interval,omitempty"`
}
// Rule describes an alerting or recording rule.

View file

@ -36,6 +36,7 @@ groups:
- name: my-another-name
interval: 30s # defaults to global interval
source_tenants: [tenant-1]
rules:
- alert: HighErrors
expr: |

View file

@ -3105,6 +3105,26 @@ func TestRangeQuery(t *testing.T) {
End: time.Unix(120, 0),
Interval: 1 * time.Minute,
},
{
Name: "short-circuit",
Load: `load 30s
foo{job="1"} 1+1x4
bar{job="2"} 1+1x4`,
Query: `foo > 2 or bar`,
Result: Matrix{
Series{
Points: []Point{{V: 1, T: 0}, {V: 3, T: 60000}, {V: 5, T: 120000}},
Metric: labels.FromStrings("__name__", "bar", "job", "2"),
},
Series{
Points: []Point{{V: 3, T: 60000}, {V: 5, T: 120000}},
Metric: labels.FromStrings("__name__", "foo", "job", "1"),
},
},
Start: time.Unix(0, 0),
End: time.Unix(120, 0),
Interval: 1 * time.Minute,
},
}
for _, c := range cases {
t.Run(c.Name, func(t *testing.T) {

View file

@ -3565,7 +3565,32 @@ func TestParseExpressions(t *testing.T) {
if !test.fail {
require.NoError(t, err)
require.Equal(t, test.expected, expr, "error on input '%s'", test.input)
expected := test.expected
// The FastRegexMatcher introduced in mimir-prometheus is not comparable with
// a deep equal, so only compare its String() version.
if actualVector, ok := expr.(*VectorSelector); ok {
require.IsType(t, &VectorSelector{}, test.expected, "error on input '%s'", test.input)
expectedVector := test.expected.(*VectorSelector)
require.Len(t, actualVector.LabelMatchers, len(expectedVector.LabelMatchers), "error on input '%s'", test.input)
for i := 0; i < len(actualVector.LabelMatchers); i++ {
expectedMatcher := expectedVector.LabelMatchers[i].String()
actualMatcher := actualVector.LabelMatchers[i].String()
require.Equal(t, expectedMatcher, actualMatcher, "unexpected label matcher '%s' on input '%s'", actualMatcher, test.input)
}
// Make a shallow copy of the expected expr (because the test cases are defined in a global variable)
// and then reset the LabelMatcher to not compared them with the following deep equal.
expectedCopy := *expectedVector
expectedCopy.LabelMatchers = nil
expected = &expectedCopy
actualVector.LabelMatchers = nil
}
require.Equal(t, expected, expr, "error on input '%s'", test.input)
} else {
require.Error(t, err)
require.Contains(t, err.Error(), test.errMsg, "unexpected error on input '%s', expected '%s', got '%s'", test.input, test.errMsg, err.Error())

View file

@ -321,10 +321,10 @@ const resolvedRetention = 15 * time.Minute
// Eval evaluates the rule expression and then creates pending alerts and fires
// or removes previously pending alerts accordingly.
func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc, externalURL *url.URL, limit int) (promql.Vector, error) {
func (r *AlertingRule) Eval(ctx context.Context, evalDelay time.Duration, ts time.Time, query QueryFunc, externalURL *url.URL, limit int) (promql.Vector, error) {
ctx = NewOriginContext(ctx, NewRuleDetail(r))
res, err := query(ctx, r.vector.String(), ts)
res, err := query(ctx, r.vector.String(), ts.Add(-evalDelay))
if err != nil {
return nil, err
}
@ -456,8 +456,8 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc,
}
if r.restored.Load() {
vec = append(vec, r.sample(a, ts))
vec = append(vec, r.forStateSample(a, ts, float64(a.ActiveAt.Unix())))
vec = append(vec, r.sample(a, ts.Add(-evalDelay)))
vec = append(vec, r.forStateSample(a, ts.Add(-evalDelay), float64(a.ActiveAt.Unix())))
}
}
@ -533,7 +533,7 @@ func (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay
if interval > resendDelay {
delta = interval
}
alert.ValidUntil = ts.Add(4 * delta)
alert.ValidUntil = ts.Add(5 * delta)
anew := *alert
// The notifier re-uses the labels slice, hence make a copy.
anew.Labels = alert.Labels.Copy()

View file

@ -158,7 +158,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) {
t.Logf("case %d", i)
evalTime := baseTime.Add(time.Duration(i) * time.Minute)
result[0].Point.T = timestamp.FromTime(evalTime)
res, err := rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
res, err := rule.Eval(suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
require.NoError(t, err)
var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples.
@ -175,7 +175,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) {
require.Equal(t, result, filteredRes)
}
evalTime := baseTime.Add(time.Duration(len(results)) * time.Minute)
res, err := rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
res, err := rule.Eval(suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
require.NoError(t, err)
require.Equal(t, 0, len(res))
}
@ -246,7 +246,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples.
res, err := ruleWithoutExternalLabels.Eval(
suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0,
suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0,
)
require.NoError(t, err)
for _, smpl := range res {
@ -260,7 +260,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
}
res, err = ruleWithExternalLabels.Eval(
suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0,
suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0,
)
require.NoError(t, err)
for _, smpl := range res {
@ -342,7 +342,7 @@ func TestAlertingRuleExternalURLInTemplate(t *testing.T) {
var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples.
res, err := ruleWithoutExternalURL.Eval(
suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0,
suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0,
)
require.NoError(t, err)
for _, smpl := range res {
@ -356,7 +356,7 @@ func TestAlertingRuleExternalURLInTemplate(t *testing.T) {
}
res, err = ruleWithExternalURL.Eval(
suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0,
suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0,
)
require.NoError(t, err)
for _, smpl := range res {
@ -414,7 +414,7 @@ func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) {
var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples.
res, err := rule.Eval(
suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0,
suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0,
)
require.NoError(t, err)
for _, smpl := range res {
@ -484,7 +484,7 @@ instance: {{ $v.Labels.instance }}, value: {{ printf "%.0f" $v.Value }};
close(getDoneCh)
}()
_, err = ruleWithQueryInTemplate.Eval(
suite.Context(), evalTime, slowQueryFunc, nil, 0,
suite.Context(), 0, evalTime, slowQueryFunc, nil, 0,
)
require.NoError(t, err)
}
@ -536,7 +536,7 @@ func TestAlertingRuleDuplicate(t *testing.T) {
"",
true, log.NewNopLogger(),
)
_, err := rule.Eval(ctx, now, EngineQueryFunc(engine, storage), nil, 0)
_, err := rule.Eval(ctx, 0, now, EngineQueryFunc(engine, storage), nil, 0)
require.Error(t, err)
require.EqualError(t, err, "vector contains metrics with the same labelset after applying alert labels")
}
@ -587,7 +587,7 @@ func TestAlertingRuleLimit(t *testing.T) {
evalTime := time.Unix(0, 0)
for _, test := range tests {
_, err := rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, test.limit)
_, err := rule.Eval(suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, test.limit)
if err != nil {
require.EqualError(t, err, test.err)
} else if test.err != "" {
@ -819,7 +819,7 @@ func TestKeepFiringFor(t *testing.T) {
t.Logf("case %d", i)
evalTime := baseTime.Add(time.Duration(i) * time.Minute)
result[0].Point.T = timestamp.FromTime(evalTime)
res, err := rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
res, err := rule.Eval(suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
require.NoError(t, err)
var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples.
@ -836,7 +836,7 @@ func TestKeepFiringFor(t *testing.T) {
require.Equal(t, result, filteredRes)
}
evalTime := baseTime.Add(time.Duration(len(results)) * time.Minute)
res, err := rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
res, err := rule.Eval(suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
require.NoError(t, err)
require.Equal(t, 0, len(res))
}
@ -876,7 +876,7 @@ func TestPendingAndKeepFiringFor(t *testing.T) {
baseTime := time.Unix(0, 0)
result.Point.T = timestamp.FromTime(baseTime)
res, err := rule.Eval(suite.Context(), baseTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
res, err := rule.Eval(suite.Context(), 0, baseTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
require.NoError(t, err)
require.Len(t, res, 2)
@ -891,7 +891,7 @@ func TestPendingAndKeepFiringFor(t *testing.T) {
}
evalTime := baseTime.Add(time.Minute)
res, err = rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
res, err = rule.Eval(suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
require.NoError(t, err)
require.Equal(t, 0, len(res))
}
@ -925,7 +925,7 @@ func TestAlertingEvalWithOrigin(t *testing.T) {
true, log.NewNopLogger(),
)
_, err = rule.Eval(ctx, now, func(ctx context.Context, qs string, _ time.Time) (promql.Vector, error) {
_, err = rule.Eval(ctx, 0, now, func(ctx context.Context, qs string, _ time.Time) (promql.Vector, error) {
detail = FromOriginContext(ctx)
return nil, nil
}, nil, 0)

View file

@ -0,0 +1,5 @@
groups:
- name: test
rules:
- alert: test
expr: sum by (job)(rate(http_requests_total[5m]))

View file

@ -0,0 +1,5 @@
groups:
- name: test
rules:
- alert: test_2
expr: sum by (job)(rate(http_requests_total[5m]))

View file

@ -0,0 +1,27 @@
groups:
- name: aligned
align_evaluation_time_on_interval: true
interval: 5m
rules:
- record: job:http_requests:rate5m
expr: sum by (job)(rate(http_requests_total[5m]))
- name: aligned_with_crazy_interval
align_evaluation_time_on_interval: true
interval: 1m27s
rules:
- record: job:http_requests:rate5m
expr: sum by (job)(rate(http_requests_total[5m]))
- name: unaligned_default
interval: 5m
rules:
- record: job:http_requests:rate5m
expr: sum by (job)(rate(http_requests_total[5m]))
- name: unaligned_explicit
interval: 5m
align_evaluation_time_on_interval: false
rules:
- record: job:http_requests:rate5m
expr: sum by (job)(rate(http_requests_total[5m]))

View file

@ -0,0 +1,6 @@
groups:
- name: test
rules:
- record: job:http_requests:rate5m
expr: sum by (job)(rate(http_requests_total[5m]))
source_tenants: [tenant-1, tenant-2]

View file

@ -217,8 +217,9 @@ type Rule interface {
Name() string
// Labels of the rule.
Labels() labels.Labels
// eval evaluates the rule, including any associated recording or alerting actions.
Eval(context.Context, time.Time, QueryFunc, *url.URL, int) (promql.Vector, error)
// Eval evaluates the rule, including any associated recording or alerting actions.
// The duration passed is the evaluation delay.
Eval(context.Context, time.Duration, time.Time, QueryFunc, *url.URL, int) (promql.Vector, error)
// String returns a human-readable string representation of the rule.
String() string
// Query returns the rule query expression.
@ -246,8 +247,10 @@ type Group struct {
name string
file string
interval time.Duration
evaluationDelay *time.Duration
limit int
rules []Rule
sourceTenants []string
seriesInPreviousEval []map[string]labels.Labels // One per Rule.
staleSeries []labels.Labels
opts *ManagerOptions
@ -266,7 +269,8 @@ type Group struct {
metrics *Metrics
ruleGroupPostProcessFunc RuleGroupPostProcessFunc
ruleGroupPostProcessFunc RuleGroupPostProcessFunc
alignEvaluationTimeOnInterval bool
}
// This function will be used before each rule group evaluation if not nil.
@ -274,14 +278,17 @@ type Group struct {
type RuleGroupPostProcessFunc func(g *Group, lastEvalTimestamp time.Time, log log.Logger) error
type GroupOptions struct {
Name, File string
Interval time.Duration
Limit int
Rules []Rule
ShouldRestore bool
Opts *ManagerOptions
done chan struct{}
RuleGroupPostProcessFunc RuleGroupPostProcessFunc
Name, File string
Interval time.Duration
Limit int
Rules []Rule
SourceTenants []string
ShouldRestore bool
Opts *ManagerOptions
EvaluationDelay *time.Duration
done chan struct{}
RuleGroupPostProcessFunc RuleGroupPostProcessFunc
AlignEvaluationTimeOnInterval bool
}
// NewGroup makes a new Group with the given name, options, and rules.
@ -303,20 +310,23 @@ func NewGroup(o GroupOptions) *Group {
metrics.GroupInterval.WithLabelValues(key).Set(o.Interval.Seconds())
return &Group{
name: o.Name,
file: o.File,
interval: o.Interval,
limit: o.Limit,
rules: o.Rules,
shouldRestore: o.ShouldRestore,
opts: o.Opts,
seriesInPreviousEval: make([]map[string]labels.Labels, len(o.Rules)),
done: make(chan struct{}),
managerDone: o.done,
terminated: make(chan struct{}),
logger: log.With(o.Opts.Logger, "file", o.File, "group", o.Name),
metrics: metrics,
ruleGroupPostProcessFunc: o.RuleGroupPostProcessFunc,
name: o.Name,
file: o.File,
interval: o.Interval,
evaluationDelay: o.EvaluationDelay,
limit: o.Limit,
rules: o.Rules,
shouldRestore: o.ShouldRestore,
opts: o.Opts,
sourceTenants: o.SourceTenants,
seriesInPreviousEval: make([]map[string]labels.Labels, len(o.Rules)),
done: make(chan struct{}),
managerDone: o.done,
terminated: make(chan struct{}),
logger: log.With(o.Opts.Logger, "file", o.File, "group", o.Name),
metrics: metrics,
ruleGroupPostProcessFunc: o.RuleGroupPostProcessFunc,
alignEvaluationTimeOnInterval: o.AlignEvaluationTimeOnInterval,
}
}
@ -341,6 +351,10 @@ func (g *Group) Interval() time.Duration { return g.interval }
// Limit returns the group's limit.
func (g *Group) Limit() int { return g.limit }
// SourceTenants returns the source tenants for the group.
// If it's empty or nil, then the owning user/tenant is considered to be the source tenant.
func (g *Group) SourceTenants() []string { return g.sourceTenants }
func (g *Group) run(ctx context.Context) {
defer close(g.terminated)
@ -535,11 +549,13 @@ func (g *Group) setLastEvaluation(ts time.Time) {
// EvalTimestamp returns the immediately preceding consistently slotted evaluation time.
func (g *Group) EvalTimestamp(startTime int64) time.Time {
var (
var offset int64
if !g.alignEvaluationTimeOnInterval {
offset = int64(g.hash() % uint64(g.interval))
adjNow = startTime - offset
base = adjNow - (adjNow % int64(g.interval))
)
}
adjNow := startTime - offset
base := adjNow - (adjNow % int64(g.interval))
return time.Unix(0, base+offset).UTC()
}
@ -604,6 +620,7 @@ func (g *Group) CopyState(from *Group) {
// Eval runs a single evaluation cycle in which all rules are evaluated sequentially.
func (g *Group) Eval(ctx context.Context, ts time.Time) {
var samplesTotal float64
evaluationDelay := g.EvaluationDelay()
for i, rule := range g.rules {
select {
case <-g.done:
@ -625,7 +642,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
g.metrics.EvalTotal.WithLabelValues(GroupKey(g.File(), g.Name())).Inc()
vector, err := rule.Eval(ctx, ts, g.opts.QueryFunc, g.opts.ExternalURL, g.Limit())
vector, err := rule.Eval(ctx, evaluationDelay, ts, g.opts.QueryFunc, g.opts.ExternalURL, g.Limit())
if err != nil {
rule.SetHealth(HealthBad)
rule.SetLastError(err)
@ -714,7 +731,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
for metric, lset := range g.seriesInPreviousEval[i] {
if _, ok := seriesReturned[metric]; !ok {
// Series no longer exposed, mark it stale.
_, err = app.Append(0, lset, timestamp.FromTime(ts), math.Float64frombits(value.StaleNaN))
_, err = app.Append(0, lset, timestamp.FromTime(ts.Add(-evaluationDelay)), math.Float64frombits(value.StaleNaN))
unwrappedErr := errors.Unwrap(err)
if unwrappedErr == nil {
unwrappedErr = err
@ -739,14 +756,25 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
g.cleanupStaleSeries(ctx, ts)
}
func (g *Group) EvaluationDelay() time.Duration {
if g.evaluationDelay != nil {
return *g.evaluationDelay
}
if g.opts.DefaultEvaluationDelay != nil {
return g.opts.DefaultEvaluationDelay()
}
return time.Duration(0)
}
func (g *Group) cleanupStaleSeries(ctx context.Context, ts time.Time) {
if len(g.staleSeries) == 0 {
return
}
app := g.opts.Appendable.Appender(ctx)
evaluationDelay := g.EvaluationDelay()
for _, s := range g.staleSeries {
// Rule that produced series no longer configured, mark it stale.
_, err := app.Append(0, s, timestamp.FromTime(ts), math.Float64frombits(value.StaleNaN))
_, err := app.Append(0, s, timestamp.FromTime(ts.Add(-evaluationDelay)), math.Float64frombits(value.StaleNaN))
unwrappedErr := errors.Unwrap(err)
if unwrappedErr == nil {
unwrappedErr = err
@ -901,11 +929,37 @@ func (g *Group) Equals(ng *Group) bool {
return false
}
if g.alignEvaluationTimeOnInterval != ng.alignEvaluationTimeOnInterval {
return false
}
for i, gr := range g.rules {
if gr.String() != ng.rules[i].String() {
return false
}
}
{
// compare source tenants
if len(g.sourceTenants) != len(ng.sourceTenants) {
return false
}
copyAndSort := func(x []string) []string {
copied := make([]string, len(x))
copy(copied, x)
sort.Strings(copied)
return copied
}
ngSourceTenantsCopy := copyAndSort(ng.sourceTenants)
gSourceTenantsCopy := copyAndSort(g.sourceTenants)
for i := range ngSourceTenantsCopy {
if gSourceTenantsCopy[i] != ngSourceTenantsCopy[i] {
return false
}
}
}
return true
}
@ -925,20 +979,30 @@ type Manager struct {
// NotifyFunc sends notifications about a set of alerts generated by the given expression.
type NotifyFunc func(ctx context.Context, expr string, alerts ...*Alert)
type ContextWrapFunc func(ctx context.Context, g *Group) context.Context
// ManagerOptions bundles options for the Manager.
type ManagerOptions struct {
ExternalURL *url.URL
QueryFunc QueryFunc
NotifyFunc NotifyFunc
Context context.Context
Appendable storage.Appendable
Queryable storage.Queryable
Logger log.Logger
Registerer prometheus.Registerer
OutageTolerance time.Duration
ForGracePeriod time.Duration
ResendDelay time.Duration
GroupLoader GroupLoader
ExternalURL *url.URL
QueryFunc QueryFunc
NotifyFunc NotifyFunc
Context context.Context
// GroupEvaluationContextFunc will be called to wrap Context based on the group being evaluated.
// Will be skipped if nil.
GroupEvaluationContextFunc ContextWrapFunc
Appendable storage.Appendable
Queryable storage.Queryable
Logger log.Logger
Registerer prometheus.Registerer
OutageTolerance time.Duration
ForGracePeriod time.Duration
ResendDelay time.Duration
GroupLoader GroupLoader
DefaultEvaluationDelay func() time.Duration
// AlwaysRestoreAlertState forces all new or changed groups in calls to Update to restore.
// Useful when you know you will be adding alerting rules after the manager has already started.
AlwaysRestoreAlertState bool
Metrics *Metrics
}
@ -1032,11 +1096,16 @@ func (m *Manager) Update(interval time.Duration, files []string, externalLabels
newg.CopyState(oldg)
}
wg.Done()
ctx := m.opts.Context
if m.opts.GroupEvaluationContextFunc != nil {
ctx = m.opts.GroupEvaluationContextFunc(ctx, newg)
}
// Wait with starting evaluation until the rule manager
// is told to run. This is necessary to avoid running
// queries against a bootstrapping storage.
<-m.block
newg.run(m.opts.Context)
newg.run(ctx)
}(newg)
}
@ -1089,7 +1158,7 @@ func (m *Manager) LoadGroups(
) (map[string]*Group, []error) {
groups := make(map[string]*Group)
shouldRestore := !m.restored
shouldRestore := !m.restored || m.opts.AlwaysRestoreAlertState
for _, fn := range filenames {
rgs, errs := m.opts.GroupLoader.Load(fn)
@ -1120,7 +1189,7 @@ func (m *Manager) LoadGroups(
labels.FromMap(r.Annotations),
externalLabels,
externalURL,
m.restored,
!shouldRestore,
log.With(m.logger, "alert", r.Alert),
))
continue
@ -1133,15 +1202,18 @@ func (m *Manager) LoadGroups(
}
groups[GroupKey(fn, rg.Name)] = NewGroup(GroupOptions{
Name: rg.Name,
File: fn,
Interval: itv,
Limit: rg.Limit,
Rules: rules,
ShouldRestore: shouldRestore,
Opts: m.opts,
done: m.done,
RuleGroupPostProcessFunc: ruleGroupPostProcessFunc,
Name: rg.Name,
File: fn,
Interval: itv,
Limit: rg.Limit,
Rules: rules,
SourceTenants: rg.SourceTenants,
ShouldRestore: shouldRestore,
Opts: m.opts,
EvaluationDelay: (*time.Duration)(rg.EvaluationDelay),
done: m.done,
RuleGroupPostProcessFunc: ruleGroupPostProcessFunc,
AlignEvaluationTimeOnInterval: rg.AlignEvaluationTimeOnInterval,
})
}
}

File diff suppressed because it is too large Load diff

View file

@ -30,7 +30,7 @@ type unknownRule struct{}
func (u unknownRule) Name() string { return "" }
func (u unknownRule) Labels() labels.Labels { return labels.EmptyLabels() }
func (u unknownRule) Eval(ctx context.Context, time time.Time, queryFunc QueryFunc, url *url.URL, i int) (promql.Vector, error) {
func (u unknownRule) Eval(ctx context.Context, evalDelay time.Duration, time time.Time, queryFunc QueryFunc, url *url.URL, i int) (promql.Vector, error) {
return nil, nil
}
func (u unknownRule) String() string { return "" }

View file

@ -72,10 +72,10 @@ func (rule *RecordingRule) Labels() labels.Labels {
}
// Eval evaluates the rule and then overrides the metric names and labels accordingly.
func (rule *RecordingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc, _ *url.URL, limit int) (promql.Vector, error) {
func (rule *RecordingRule) Eval(ctx context.Context, evalDelay time.Duration, ts time.Time, query QueryFunc, _ *url.URL, limit int) (promql.Vector, error) {
ctx = NewOriginContext(ctx, NewRuleDetail(rule))
vector, err := query(ctx, rule.vector.String(), ts)
vector, err := query(ctx, rule.vector.String(), ts.Add(-evalDelay))
if err != nil {
return nil, err
}

View file

@ -121,7 +121,7 @@ func TestRuleEval(t *testing.T) {
for _, scenario := range ruleEvalTestScenarios {
t.Run(scenario.name, func(t *testing.T) {
rule := NewRecordingRule("test_rule", scenario.expr, scenario.ruleLabels)
result, err := rule.Eval(suite.Context(), ruleEvaluationTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
result, err := rule.Eval(suite.Context(), 0, ruleEvaluationTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
require.NoError(t, err)
require.Equal(t, scenario.expected, result)
})
@ -141,7 +141,7 @@ func BenchmarkRuleEval(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := rule.Eval(suite.Context(), ruleEvaluationTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
_, err := rule.Eval(suite.Context(), 0, ruleEvaluationTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
if err != nil {
require.NoError(b, err)
}
@ -170,7 +170,7 @@ func TestRuleEvalDuplicate(t *testing.T) {
expr, _ := parser.ParseExpr(`vector(0) or label_replace(vector(0),"test","x","","")`)
rule := NewRecordingRule("foo", expr, labels.FromStrings("test", "test"))
_, err := rule.Eval(ctx, now, EngineQueryFunc(engine, storage), nil, 0)
_, err := rule.Eval(ctx, 0, now, EngineQueryFunc(engine, storage), nil, 0)
require.Error(t, err)
require.EqualError(t, err, "vector contains metrics with the same labelset after applying rule labels")
}
@ -215,7 +215,7 @@ func TestRecordingRuleLimit(t *testing.T) {
evalTime := time.Unix(0, 0)
for _, test := range tests {
_, err := rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, test.limit)
_, err := rule.Eval(suite.Context(), 0, evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, test.limit)
if err != nil {
require.EqualError(t, err, test.err)
} else if test.err != "" {
@ -243,7 +243,7 @@ func TestRecordingEvalWithOrigin(t *testing.T) {
require.NoError(t, err)
rule := NewRecordingRule(name, expr, lbs)
_, err = rule.Eval(ctx, now, func(ctx context.Context, qs string, _ time.Time) (promql.Vector, error) {
_, err = rule.Eval(ctx, 0, now, func(ctx context.Context, qs string, _ time.Time) (promql.Vector, error) {
detail = FromOriginContext(ctx)
return nil, nil
}, nil, 0)

View file

@ -193,6 +193,9 @@ type SelectHints struct {
By bool // Indicate whether it is without or by.
Range int64 // Range vector selector range in milliseconds.
ShardIndex uint64 // Current shard index (starts from 0 and up to ShardCount-1).
ShardCount uint64 // Total number of shards (0 means sharding is disabled).
// DisableTrimming allows to disable trimming of matching series chunks based on query Start and End time.
// When disabled, the result may contain samples outside the queried time range but Select() performances
// may be improved.

175
tsdb/async_block_writer.go Normal file
View file

@ -0,0 +1,175 @@
package tsdb
import (
"context"
"github.com/pkg/errors"
"go.uber.org/atomic"
"golang.org/x/sync/semaphore"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb/chunkenc"
"github.com/prometheus/prometheus/tsdb/chunks"
)
var errAsyncBlockWriterNotRunning = errors.New("asyncBlockWriter doesn't run anymore")
// asyncBlockWriter runs a background goroutine that writes series and chunks to the block asynchronously.
// All calls on asyncBlockWriter must be done from single goroutine, it is not safe for concurrent usage from multiple goroutines.
type asyncBlockWriter struct {
chunkPool chunkenc.Pool // Where to return chunks after writing.
chunkw ChunkWriter
indexw IndexWriter
closeSemaphore *semaphore.Weighted
seriesChan chan seriesToWrite
finishedCh chan asyncBlockWriterResult
closed bool
result asyncBlockWriterResult
}
type asyncBlockWriterResult struct {
stats BlockStats
err error
}
type seriesToWrite struct {
lbls labels.Labels
chks []chunks.Meta
}
func newAsyncBlockWriter(chunkPool chunkenc.Pool, chunkw ChunkWriter, indexw IndexWriter, closeSema *semaphore.Weighted) *asyncBlockWriter {
bw := &asyncBlockWriter{
chunkPool: chunkPool,
chunkw: chunkw,
indexw: indexw,
seriesChan: make(chan seriesToWrite, 64),
finishedCh: make(chan asyncBlockWriterResult, 1),
closeSemaphore: closeSema,
}
go bw.loop()
return bw
}
// loop doing the writes. Return value is only used by defer statement, and is sent to the channel,
// before closing it.
func (bw *asyncBlockWriter) loop() (res asyncBlockWriterResult) {
defer func() {
bw.finishedCh <- res
close(bw.finishedCh)
}()
stats := BlockStats{}
ref := storage.SeriesRef(0)
for sw := range bw.seriesChan {
if err := bw.chunkw.WriteChunks(sw.chks...); err != nil {
return asyncBlockWriterResult{err: errors.Wrap(err, "write chunks")}
}
if err := bw.indexw.AddSeries(ref, sw.lbls, sw.chks...); err != nil {
return asyncBlockWriterResult{err: errors.Wrap(err, "add series")}
}
stats.NumChunks += uint64(len(sw.chks))
stats.NumSeries++
for _, chk := range sw.chks {
stats.NumSamples += uint64(chk.Chunk.NumSamples())
}
for _, chk := range sw.chks {
if err := bw.chunkPool.Put(chk.Chunk); err != nil {
return asyncBlockWriterResult{err: errors.Wrap(err, "put chunk")}
}
}
ref++
}
err := bw.closeSemaphore.Acquire(context.Background(), 1)
if err != nil {
return asyncBlockWriterResult{err: errors.Wrap(err, "failed to acquire semaphore before closing writers")}
}
defer bw.closeSemaphore.Release(1)
// If everything went fine with writing so far, close writers.
if err := bw.chunkw.Close(); err != nil {
return asyncBlockWriterResult{err: errors.Wrap(err, "closing chunk writer")}
}
if err := bw.indexw.Close(); err != nil {
return asyncBlockWriterResult{err: errors.Wrap(err, "closing index writer")}
}
return asyncBlockWriterResult{stats: stats}
}
func (bw *asyncBlockWriter) addSeries(lbls labels.Labels, chks []chunks.Meta) error {
select {
case bw.seriesChan <- seriesToWrite{lbls: lbls, chks: chks}:
return nil
case result, ok := <-bw.finishedCh:
if ok {
bw.result = result
}
// If the writer isn't running anymore because of an error occurred in loop()
// then we should return that error too, otherwise it may be never reported
// and we'll never know the actual root cause.
if bw.result.err != nil {
return errors.Wrap(bw.result.err, errAsyncBlockWriterNotRunning.Error())
}
return errAsyncBlockWriterNotRunning
}
}
func (bw *asyncBlockWriter) closeAsync() {
if !bw.closed {
bw.closed = true
close(bw.seriesChan)
}
}
func (bw *asyncBlockWriter) waitFinished() (BlockStats, error) {
// Wait for flusher to finish.
result, ok := <-bw.finishedCh
if ok {
bw.result = result
}
return bw.result.stats, bw.result.err
}
type preventDoubleCloseIndexWriter struct {
IndexWriter
closed atomic.Bool
}
func newPreventDoubleCloseIndexWriter(iw IndexWriter) *preventDoubleCloseIndexWriter {
return &preventDoubleCloseIndexWriter{IndexWriter: iw}
}
func (p *preventDoubleCloseIndexWriter) Close() error {
if p.closed.CompareAndSwap(false, true) {
return p.IndexWriter.Close()
}
return nil
}
type preventDoubleCloseChunkWriter struct {
ChunkWriter
closed atomic.Bool
}
func newPreventDoubleCloseChunkWriter(cw ChunkWriter) *preventDoubleCloseChunkWriter {
return &preventDoubleCloseChunkWriter{ChunkWriter: cw}
}
func (p *preventDoubleCloseChunkWriter) Close() error {
if p.closed.CompareAndSwap(false, true) {
return p.ChunkWriter.Close()
}
return nil
}

View file

@ -20,6 +20,7 @@ import (
"os"
"path/filepath"
"sync"
"time"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
@ -75,10 +76,21 @@ type IndexReader interface {
// during background garbage collections.
Postings(name string, values ...string) (index.Postings, error)
// PostingsForMatchers assembles a single postings iterator based on the given matchers.
// The resulting postings are not ordered by series.
// If concurrent hint is set to true, call will be optimized for a (most likely) concurrent call with same matchers,
// avoiding same calculations twice, however this implementation may lead to a worse performance when called once.
PostingsForMatchers(concurrent bool, ms ...*labels.Matcher) (index.Postings, error)
// SortedPostings returns a postings list that is reordered to be sorted
// by the label set of the underlying series.
SortedPostings(index.Postings) index.Postings
// ShardedPostings returns a postings list filtered by the provided shardIndex
// out of shardCount. For a given posting, its shard MUST be computed hashing
// the series labels mod shardCount, using a hash function which is consistent over time.
ShardedPostings(p index.Postings, shardIndex, shardCount uint64) index.Postings
// Series populates the given builder and chunk metas for the series identified
// by the reference.
// Returns storage.ErrNotFound if the ref does not resolve to a known series.
@ -158,6 +170,9 @@ type BlockMeta struct {
// Version of the index format.
Version int `json:"version"`
// OutOfOrder is true if the block was directly created from out-of-order samples.
OutOfOrder bool `json:"out_of_order"`
}
// BlockStats contains stats about contents of a block.
@ -308,6 +323,11 @@ type Block struct {
// OpenBlock opens the block in the directory. It can be passed a chunk pool, which is used
// to instantiate chunk structs.
func OpenBlock(logger log.Logger, dir string, pool chunkenc.Pool) (pb *Block, err error) {
return OpenBlockWithOptions(logger, dir, pool, nil, defaultPostingsForMatchersCacheTTL, defaultPostingsForMatchersCacheSize, false)
}
// OpenBlockWithOptions is like OpenBlock but allows to pass a cache provider and sharding function.
func OpenBlockWithOptions(logger log.Logger, dir string, pool chunkenc.Pool, cache index.ReaderCacheProvider, postingsCacheTTL time.Duration, postingsCacheSize int, postingsCacheForce bool) (pb *Block, err error) {
if logger == nil {
logger = log.NewNopLogger()
}
@ -328,10 +348,12 @@ func OpenBlock(logger log.Logger, dir string, pool chunkenc.Pool) (pb *Block, er
}
closers = append(closers, cr)
ir, err := index.NewFileReader(filepath.Join(dir, indexFilename))
indexReader, err := index.NewFileReaderWithOptions(filepath.Join(dir, indexFilename), cache)
if err != nil {
return nil, err
}
pfmc := NewPostingsForMatchersCache(postingsCacheTTL, postingsCacheSize, postingsCacheForce)
ir := indexReaderWithPostingsForMatchers{indexReader, pfmc}
closers = append(closers, ir)
tr, sizeTomb, err := tombstones.ReadTombstones(dir)
@ -495,10 +517,18 @@ func (r blockIndexReader) Postings(name string, values ...string) (index.Posting
return p, nil
}
func (r blockIndexReader) PostingsForMatchers(concurrent bool, ms ...*labels.Matcher) (index.Postings, error) {
return r.ir.PostingsForMatchers(concurrent, ms...)
}
func (r blockIndexReader) SortedPostings(p index.Postings) index.Postings {
return r.ir.SortedPostings(p)
}
func (r blockIndexReader) ShardedPostings(p index.Postings, shardIndex, shardCount uint64) index.Postings {
return r.ir.ShardedPostings(p, shardIndex, shardCount)
}
func (r blockIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error {
if err := r.ir.Series(ref, builder, chks); err != nil {
return errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
@ -551,7 +581,7 @@ func (pb *Block) Delete(mint, maxt int64, ms ...*labels.Matcher) error {
return ErrClosing
}
p, err := PostingsForMatchers(pb.indexr, ms...)
p, err := pb.indexr.PostingsForMatchers(false, ms...)
if err != nil {
return errors.Wrap(err, "select series")
}

View file

@ -312,7 +312,7 @@ func TestBlockSize(t *testing.T) {
require.NoError(t, err)
require.Equal(t, expAfterDelete, actAfterDelete, "after a delete reported block size doesn't match actual disk size")
c, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{0}, nil, nil)
c, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{0}, nil, nil, true)
require.NoError(t, err)
blockDirAfterCompact, err := c.Compact(tmpdir, []string{blockInit.Dir()}, nil)
require.NoError(t, err)
@ -349,6 +349,9 @@ func TestReadIndexFormatV1(t *testing.T) {
blockDir := filepath.Join("testdata", "index_format_v1")
block, err := OpenBlock(nil, blockDir, nil)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, block.Close())
})
q, err := NewBlockQuerier(block, 0, 1000)
require.NoError(t, err)
@ -487,7 +490,7 @@ func createBlock(tb testing.TB, dir string, series []storage.Series) string {
}
func createBlockFromHead(tb testing.TB, dir string, head *Head) string {
compactor, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil, nil)
compactor, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil, nil, true)
require.NoError(tb, err)
require.NoError(tb, os.MkdirAll(dir, 0o777))

View file

@ -100,7 +100,7 @@ func (w *BlockWriter) Flush(ctx context.Context) (ulid.ULID, error) {
nil,
w.logger,
[]int64{w.blockSize},
chunkenc.NewPool(), nil)
chunkenc.NewPool(), nil, true)
if err != nil {
return ulid.ULID{}, errors.Wrap(err, "create leveled compactor")
}

View file

@ -249,7 +249,7 @@ func (c *chunkWriteQueue) queueIsEmpty() bool {
}
func (c *chunkWriteQueue) queueIsFull() bool {
// When the queue is full and blocked on the writer the chunkRefMap has one more job than the cap of the jobCh
// When the queue is full and blocked on the writer the chunkRefMap has one more job than the capacity of the queue
// because one job is currently being processed and blocked in the writer.
return c.queueSize() == c.jobs.maxSize+1
}
@ -258,7 +258,7 @@ func (c *chunkWriteQueue) queueSize() int {
c.chunkRefMapMtx.Lock()
defer c.chunkRefMapMtx.Unlock()
// Looking at chunkRefMap instead of jobCh because the job is popped from the chan before it has
// been fully processed, it remains in the chunkRefMap until the processing is complete.
// Looking at chunkRefMap instead of jobs queue because the job is popped from the queue before it has
// been fully processed, but it remains in the chunkRefMap until the processing is complete.
return len(c.chunkRefMap)
}

View file

@ -178,7 +178,6 @@ func TestChunkWriteQueue_WrappingAroundSizeLimit(t *testing.T) {
// Wait until all jobs have been processed.
callbackWg.Wait()
require.Eventually(t, q.queueIsEmpty, 500*time.Millisecond, 50*time.Millisecond)
}

View file

@ -534,11 +534,12 @@ func (cdm *ChunkDiskMapper) writeChunk(seriesRef HeadSeriesRef, mint, maxt int64
}
// CutNewFile makes that a new file will be created the next time a chunk is written.
func (cdm *ChunkDiskMapper) CutNewFile() {
func (cdm *ChunkDiskMapper) CutNewFile() error {
cdm.evtlPosMtx.Lock()
defer cdm.evtlPosMtx.Unlock()
cdm.evtlPos.cutFileOnNextChunk()
return nil
}
func (cdm *ChunkDiskMapper) IsQueueEmpty() bool {
@ -940,7 +941,7 @@ func (cdm *ChunkDiskMapper) Truncate(fileNo uint32) error {
// There is a known race condition here because between the check of curFileSize() and the call to CutNewFile()
// a new file could already be cut, this is acceptable because it will simply result in an empty file which
// won't do any harm.
cdm.CutNewFile()
errs.Add(cdm.CutNewFile())
}
pendingDeletes, err := cdm.deleteFiles(removedFiles)
errs.Add(err)

View file

@ -15,12 +15,12 @@ package chunks
import (
"encoding/binary"
"errors"
"math/rand"
"os"
"strconv"
"testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/tsdb/chunkenc"
@ -121,7 +121,7 @@ func TestChunkDiskMapper_WriteChunk_Chunk_IterateChunks(t *testing.T) {
}
}
addChunks(100)
hrw.CutNewFile()
require.NoError(t, hrw.CutNewFile())
addChunks(10) // For chunks in in-memory buffer.
}
@ -159,7 +159,7 @@ func TestChunkDiskMapper_WriteChunk_Chunk_IterateChunks(t *testing.T) {
expData := expectedData[idx]
require.Equal(t, expData.seriesRef, seriesRef)
require.Equal(t, expData.chunkRef, chunkRef)
require.Equal(t, expData.maxt, maxt)
require.Equal(t, expData.mint, mint)
require.Equal(t, expData.maxt, maxt)
require.Equal(t, expData.numSamples, numSamples)
require.Equal(t, expData.isOOO, isOOO)
@ -174,6 +174,44 @@ func TestChunkDiskMapper_WriteChunk_Chunk_IterateChunks(t *testing.T) {
require.Equal(t, len(expectedData), idx)
}
func TestChunkDiskMapper_WriteUnsupportedChunk_Chunk_IterateChunks(t *testing.T) {
hrw := createChunkDiskMapper(t, "")
defer func() {
require.NoError(t, hrw.Close())
}()
ucSeriesRef, ucChkRef, ucMint, ucMaxt, uchunk := writeUnsupportedChunk(t, 0, hrw)
// Checking on-disk bytes for the first file.
require.Equal(t, 1, len(hrw.mmappedChunkFiles), "expected 1 mmapped file, got %d", len(hrw.mmappedChunkFiles))
require.Equal(t, len(hrw.mmappedChunkFiles), len(hrw.closers))
// Testing IterateAllChunks method.
dir := hrw.dir.Name()
require.NoError(t, hrw.Close())
hrw = createChunkDiskMapper(t, dir)
require.NoError(t, hrw.IterateAllChunks(func(seriesRef HeadSeriesRef, chunkRef ChunkDiskMapperRef, mint, maxt int64, numSamples uint16, encoding chunkenc.Encoding, isOOO bool) error {
t.Helper()
require.Equal(t, ucSeriesRef, seriesRef)
require.Equal(t, ucChkRef, chunkRef)
require.Equal(t, ucMint, mint)
require.Equal(t, ucMaxt, maxt)
require.Equal(t, uchunk.Encoding(), encoding) // Asserts that the encoding is EncUnsupportedXOR
actChunk, err := hrw.Chunk(chunkRef)
// The chunk encoding is unknown so Chunk() should fail but us the caller
// are ok with that. Above we asserted that the encoding we expected was
// EncUnsupportedXOR
require.NotNil(t, err)
require.Contains(t, err.Error(), "invalid chunk encoding \"<unknown>\"")
require.Nil(t, actChunk)
return nil
}))
}
// TestChunkDiskMapper_Truncate tests
// * If truncation is happening properly based on the time passed.
// * The active file is not deleted even if the passed time makes it eligible to be deleted.
@ -218,7 +256,7 @@ func TestChunkDiskMapper_Truncate(t *testing.T) {
// Create segments 1 to 7.
for i := 1; i <= 7; i++ {
hrw.CutNewFile()
require.NoError(t, hrw.CutNewFile())
addChunk()
}
verifyFiles([]int{1, 2, 3, 4, 5, 6, 7})
@ -411,7 +449,7 @@ func TestHeadReadWriter_ReadRepairOnEmptyLastFile(t *testing.T) {
nonEmptyFile := func() {
t.Helper()
hrw.CutNewFile()
require.NoError(t, hrw.CutNewFile())
addChunk()
}
@ -512,6 +550,17 @@ func randomChunk(t *testing.T) chunkenc.Chunk {
return chunk
}
func randomUnsupportedChunk(t *testing.T) chunkenc.Chunk {
chunk := newUnsupportedChunk()
len := rand.Int() % 120
app, err := chunk.Appender()
require.NoError(t, err)
for i := 0; i < len; i++ {
app.Append(rand.Int63(), rand.Float64())
}
return chunk
}
func createChunk(t *testing.T, idx int, hrw *ChunkDiskMapper) (seriesRef HeadSeriesRef, chunkRef ChunkDiskMapperRef, mint, maxt int64, chunk chunkenc.Chunk, isOOO bool) {
var err error
seriesRef = HeadSeriesRef(rand.Int63())
@ -529,3 +578,36 @@ func createChunk(t *testing.T, idx int, hrw *ChunkDiskMapper) (seriesRef HeadSer
<-awaitCb
return
}
func writeUnsupportedChunk(t *testing.T, idx int, hrw *ChunkDiskMapper) (seriesRef HeadSeriesRef, chunkRef ChunkDiskMapperRef, mint, maxt int64, chunk chunkenc.Chunk) {
var err error
seriesRef = HeadSeriesRef(rand.Int63())
mint = int64((idx)*1000 + 1)
maxt = int64((idx + 1) * 1000)
chunk = randomUnsupportedChunk(t)
awaitCb := make(chan struct{})
chunkRef = hrw.WriteChunk(seriesRef, mint, maxt, chunk, false, func(cbErr error) {
require.NoError(t, err)
close(awaitCb)
})
<-awaitCb
return
}
const (
UnsupportedMask = 0b01000000
EncUnsupportedXOR = chunkenc.EncXOR | UnsupportedMask
)
// unsupportedChunk holds a XORChunk and overrides the Encoding() method.
type unsupportedChunk struct {
*chunkenc.XORChunk
}
func newUnsupportedChunk() *unsupportedChunk {
return &unsupportedChunk{chunkenc.NewXORChunk()}
}
func (c *unsupportedChunk) Encoding() chunkenc.Encoding {
return EncUnsupportedXOR
}

File diff suppressed because it is too large Load diff

View file

@ -15,6 +15,7 @@ package tsdb
import (
"context"
crand "crypto/rand"
"fmt"
"math"
"math/rand"
@ -30,6 +31,7 @@ import (
"github.com/pkg/errors"
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/require"
"golang.org/x/sync/semaphore"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
@ -37,6 +39,7 @@ import (
"github.com/prometheus/prometheus/tsdb/chunkenc"
"github.com/prometheus/prometheus/tsdb/chunks"
"github.com/prometheus/prometheus/tsdb/fileutil"
"github.com/prometheus/prometheus/tsdb/index"
"github.com/prometheus/prometheus/tsdb/tombstones"
"github.com/prometheus/prometheus/tsdb/tsdbutil"
)
@ -163,7 +166,7 @@ func TestNoPanicFor0Tombstones(t *testing.T) {
},
}
c, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{50}, nil, nil)
c, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{50}, nil, nil, true)
require.NoError(t, err)
c.plan(metas)
@ -177,7 +180,7 @@ func TestLeveledCompactor_plan(t *testing.T) {
180,
540,
1620,
}, nil, nil)
}, nil, nil, true)
require.NoError(t, err)
cases := map[string]struct {
@ -386,7 +389,7 @@ func TestRangeWithFailedCompactionWontGetSelected(t *testing.T) {
240,
720,
2160,
}, nil, nil)
}, nil, nil, true)
require.NoError(t, err)
cases := []struct {
@ -430,20 +433,35 @@ func TestRangeWithFailedCompactionWontGetSelected(t *testing.T) {
}
func TestCompactionFailWillCleanUpTempDir(t *testing.T) {
compactor, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{
compactor, err := NewLeveledCompactorWithChunkSize(context.Background(), nil, log.NewNopLogger(), []int64{
20,
60,
240,
720,
2160,
}, nil, nil)
}, nil, chunks.DefaultChunkSegmentSize, nil, true)
require.NoError(t, err)
tmpdir := t.TempDir()
require.Error(t, compactor.write(tmpdir, &BlockMeta{}, erringBReader{}))
_, err = os.Stat(filepath.Join(tmpdir, BlockMeta{}.ULID.String()) + tmpForCreationBlockDirSuffix)
require.True(t, os.IsNotExist(err), "directory is not cleaned up")
shardedBlocks := []shardedBlock{
{meta: &BlockMeta{ULID: ulid.MustNew(ulid.Now(), crand.Reader)}},
{meta: &BlockMeta{ULID: ulid.MustNew(ulid.Now(), crand.Reader)}},
{meta: &BlockMeta{ULID: ulid.MustNew(ulid.Now(), crand.Reader)}},
}
require.Error(t, compactor.write(tmpdir, shardedBlocks, erringBReader{}))
// We rely on the fact that blockDir and tmpDir will be updated by compactor.write.
for _, b := range shardedBlocks {
require.NotEmpty(t, b.tmpDir)
_, err = os.Stat(b.tmpDir)
require.True(t, os.IsNotExist(err), "tmp directory is not cleaned up")
require.NotEmpty(t, b.blockDir)
_, err = os.Stat(b.blockDir)
require.True(t, os.IsNotExist(err), "block directory is not cleaned up")
}
}
func metaRange(name string, mint, maxt int64, stats *BlockStats) dirMeta {
@ -485,6 +503,189 @@ func samplesForRange(minTime, maxTime int64, maxSamplesPerChunk int) (ret [][]sa
return ret
}
func TestCompaction_CompactWithSplitting(t *testing.T) {
seriesCounts := []int{10, 1234}
shardCounts := []uint64{1, 13}
for _, series := range seriesCounts {
dir, err := os.MkdirTemp("", "compact")
require.NoError(t, err)
defer func() {
require.NoError(t, os.RemoveAll(dir))
}()
ranges := [][2]int64{{0, 5000}, {3000, 8000}, {6000, 11000}, {9000, 14000}}
// Generate blocks.
var blockDirs []string
var openBlocks []*Block
for _, r := range ranges {
block, err := OpenBlock(nil, createBlock(t, dir, genSeries(series, 10, r[0], r[1])), nil)
require.NoError(t, err)
defer func() {
require.NoError(t, block.Close())
}()
openBlocks = append(openBlocks, block)
blockDirs = append(blockDirs, block.Dir())
}
for _, shardCount := range shardCounts {
t.Run(fmt.Sprintf("series=%d, shards=%d", series, shardCount), func(t *testing.T) {
c, err := NewLeveledCompactorWithChunkSize(context.Background(), nil, log.NewNopLogger(), []int64{0}, nil, chunks.DefaultChunkSegmentSize, nil, true)
require.NoError(t, err)
blockIDs, err := c.CompactWithSplitting(dir, blockDirs, openBlocks, shardCount)
require.NoError(t, err)
require.Equal(t, shardCount, uint64(len(blockIDs)))
// Verify resulting blocks. We will iterate over all series in all blocks, and check two things:
// 1) Make sure that each series in the block belongs to the block (based on sharding).
// 2) Verify that total number of series over all blocks is correct.
totalSeries := uint64(0)
ts := uint64(0)
for shardIndex, blockID := range blockIDs {
// Some blocks may be empty, they will have zero block ID.
if blockID == (ulid.ULID{}) {
continue
}
// All blocks have the same timestamp.
if ts == 0 {
ts = blockID.Time()
} else {
require.Equal(t, ts, blockID.Time())
}
// Symbols found in series.
seriesSymbols := map[string]struct{}{}
// We always expect to find "" symbol in the symbols table even if it's not in the series.
// Head compaction always includes it, and then it survives additional non-sharded compactions.
// Our splitting compaction preserves it too.
seriesSymbols[""] = struct{}{}
block, err := OpenBlock(log.NewNopLogger(), filepath.Join(dir, blockID.String()), nil)
require.NoError(t, err)
defer func() {
require.NoError(t, block.Close())
}()
totalSeries += block.Meta().Stats.NumSeries
idxr, err := block.Index()
require.NoError(t, err)
defer func() {
require.NoError(t, idxr.Close())
}()
k, v := index.AllPostingsKey()
p, err := idxr.Postings(k, v)
require.NoError(t, err)
var lbls labels.ScratchBuilder
for p.Next() {
ref := p.At()
require.NoError(t, idxr.Series(ref, &lbls, nil))
require.Equal(t, uint64(shardIndex), labels.StableHash(lbls.Labels())%shardCount)
// Collect all symbols used by series.
lbls.Labels().Range(func(l labels.Label) {
seriesSymbols[l.Name] = struct{}{}
seriesSymbols[l.Value] = struct{}{}
})
}
require.NoError(t, p.Err())
// Check that all symbols in symbols table are actually used by series.
symIt := idxr.Symbols()
for symIt.Next() {
w := symIt.At()
_, ok := seriesSymbols[w]
require.True(t, ok, "not found in series: '%s'", w)
delete(seriesSymbols, w)
}
// Check that symbols table covered all symbols found from series.
require.Equal(t, 0, len(seriesSymbols))
}
require.Equal(t, uint64(series), totalSeries)
// Source blocks are *not* deletable.
for _, b := range openBlocks {
require.False(t, b.meta.Compaction.Deletable)
}
})
}
}
}
func TestCompaction_CompactEmptyBlocks(t *testing.T) {
dir, err := os.MkdirTemp("", "compact")
require.NoError(t, err)
defer func() {
require.NoError(t, os.RemoveAll(dir))
}()
ranges := [][2]int64{{0, 5000}, {3000, 8000}, {6000, 11000}, {9000, 14000}}
// Generate blocks.
var blockDirs []string
for _, r := range ranges {
// Generate blocks using index and chunk writer. CreateBlock would not return valid block for 0 series.
id := ulid.MustNew(ulid.Now(), crand.Reader)
m := &BlockMeta{
ULID: id,
MinTime: r[0],
MaxTime: r[1],
Compaction: BlockMetaCompaction{Level: 1, Sources: []ulid.ULID{id}},
Version: metaVersion1,
}
bdir := filepath.Join(dir, id.String())
require.NoError(t, os.Mkdir(bdir, 0o777))
require.NoError(t, os.Mkdir(chunkDir(bdir), 0o777))
_, err := writeMetaFile(log.NewNopLogger(), bdir, m)
require.NoError(t, err)
iw, err := index.NewWriter(context.Background(), filepath.Join(bdir, indexFilename))
require.NoError(t, err)
require.NoError(t, iw.AddSymbol("hello"))
require.NoError(t, iw.AddSymbol("world"))
require.NoError(t, iw.Close())
blockDirs = append(blockDirs, bdir)
}
c, err := NewLeveledCompactorWithChunkSize(context.Background(), nil, log.NewNopLogger(), []int64{0}, nil, chunks.DefaultChunkSegmentSize, nil, true)
require.NoError(t, err)
blockIDs, err := c.CompactWithSplitting(dir, blockDirs, nil, 5)
require.NoError(t, err)
// There are no output blocks.
for _, b := range blockIDs {
require.Equal(t, ulid.ULID{}, b)
}
// All source blocks are now marked for deletion.
for _, b := range blockDirs {
meta, _, err := readMetaFile(b)
require.NoError(t, err)
require.True(t, meta.Compaction.Deletable)
}
}
func TestCompaction_populateBlock(t *testing.T) {
for _, tc := range []struct {
title string
@ -497,7 +698,7 @@ func TestCompaction_populateBlock(t *testing.T) {
{
title: "Populate block from empty input should return error.",
inputSeriesSamples: [][]seriesSamples{},
expErr: errors.New("cannot populate block from no readers"),
expErr: errors.New("cannot populate block(s) from no readers"),
},
{
// Populate from single block without chunks. We expect these kind of series being ignored.
@ -941,7 +1142,7 @@ func TestCompaction_populateBlock(t *testing.T) {
blocks = append(blocks, &mockBReader{ir: ir, cr: cr, mint: mint, maxt: maxt})
}
c, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{0}, nil, nil)
c, err := NewLeveledCompactorWithChunkSize(context.Background(), nil, nil, []int64{0}, nil, chunks.DefaultChunkSegmentSize, nil, true)
require.NoError(t, err)
meta := &BlockMeta{
@ -953,7 +1154,8 @@ func TestCompaction_populateBlock(t *testing.T) {
}
iw := &mockIndexWriter{}
err = c.populateBlock(blocks, meta, iw, nopChunkWriter{})
ob := shardedBlock{meta: meta, indexw: iw, chunkw: nopChunkWriter{}}
err = c.populateBlock(blocks, meta.MinTime, meta.MaxTime, []shardedBlock{ob})
if tc.expErr != nil {
require.Error(t, err)
require.Equal(t, tc.expErr.Error(), err.Error())
@ -1062,7 +1264,7 @@ func BenchmarkCompaction(b *testing.B) {
blockDirs = append(blockDirs, block.Dir())
}
c, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{0}, nil, nil)
c, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{0}, nil, nil, true)
require.NoError(b, err)
b.ResetTimer()
@ -1302,6 +1504,121 @@ func TestDeleteCompactionBlockAfterFailedReload(t *testing.T) {
}
}
func TestOpenBlocksForCompaction(t *testing.T) {
dir := t.TempDir()
const blocks = 5
var blockDirs []string
for ix := 0; ix < blocks; ix++ {
d := createBlock(t, dir, genSeries(100, 10, 0, 5000))
blockDirs = append(blockDirs, d)
}
// Open subset of blocks first.
const blocksToOpen = 2
opened, toClose, err := openBlocksForCompaction(blockDirs[:blocksToOpen], nil, log.NewNopLogger(), nil, 10)
for _, b := range toClose {
defer func(b *Block) { require.NoError(t, b.Close()) }(b)
}
require.NoError(t, err)
checkBlocks(t, opened, blockDirs[:blocksToOpen]...)
checkBlocks(t, toClose, blockDirs[:blocksToOpen]...)
// Open all blocks, but provide previously opened blocks.
opened2, toClose2, err := openBlocksForCompaction(blockDirs, opened, log.NewNopLogger(), nil, 10)
for _, b := range toClose2 {
defer func(b *Block) { require.NoError(t, b.Close()) }(b)
}
require.NoError(t, err)
checkBlocks(t, opened2, blockDirs...)
checkBlocks(t, toClose2, blockDirs[blocksToOpen:]...)
}
func TestOpenBlocksForCompactionErrorsNoMeta(t *testing.T) {
dir := t.TempDir()
const blocks = 5
var blockDirs []string
for ix := 0; ix < blocks; ix++ {
d := createBlock(t, dir, genSeries(100, 10, 0, 5000))
blockDirs = append(blockDirs, d)
if ix == 3 {
blockDirs = append(blockDirs, path.Join(dir, "invalid-block"))
}
}
// open block[0]
b0, err := OpenBlock(log.NewNopLogger(), blockDirs[0], nil)
require.NoError(t, err)
defer func() { require.NoError(t, b0.Close()) }()
_, toClose, err := openBlocksForCompaction(blockDirs, []*Block{b0}, log.NewNopLogger(), nil, 10)
require.Error(t, err)
// We didn't get to opening more blocks, because we found invalid dir, so there is nothing to close.
require.Empty(t, toClose)
}
func TestOpenBlocksForCompactionErrorsMissingIndex(t *testing.T) {
dir := t.TempDir()
const blocks = 5
var blockDirs []string
for ix := 0; ix < blocks; ix++ {
d := createBlock(t, dir, genSeries(100, 10, 0, 5000))
blockDirs = append(blockDirs, d)
if ix == 3 {
require.NoError(t, os.Remove(path.Join(d, indexFilename)))
}
}
// open block[1]
b1, err := OpenBlock(log.NewNopLogger(), blockDirs[1], nil)
require.NoError(t, err)
defer func() { require.NoError(t, b1.Close()) }()
// We use concurrency = 1 to simplify the test.
// Block[0] will be opened correctly.
// Block[1] is already opened.
// Block[2] will be opened correctly.
// Block[3] is invalid and will cause error.
// Block[4] will not be opened at all.
opened, toClose, err := openBlocksForCompaction(blockDirs, []*Block{b1}, log.NewNopLogger(), nil, 1)
for _, b := range toClose {
defer func(b *Block) { require.NoError(t, b.Close()) }(b)
}
require.Error(t, err)
checkBlocks(t, opened, blockDirs[0:3]...)
checkBlocks(t, toClose, blockDirs[0], blockDirs[2])
}
// Check that blocks match IDs from directories.
func checkBlocks(t *testing.T, blocks []*Block, dirs ...string) {
t.Helper()
blockIDs := map[string]struct{}{}
for _, b := range blocks {
blockIDs[b.Meta().ULID.String()] = struct{}{}
}
dirBlockIDs := map[string]struct{}{}
for _, d := range dirs {
m, _, err := readMetaFile(d)
require.NoError(t, err)
dirBlockIDs[m.ULID.String()] = struct{}{}
}
require.Equal(t, blockIDs, dirBlockIDs)
}
func TestHeadCompactionWithHistograms(t *testing.T) {
for _, floatTest := range []bool{true, false} {
t.Run(fmt.Sprintf("float=%t", floatTest), func(t *testing.T) {
@ -1400,7 +1717,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) {
// Compaction.
mint := head.MinTime()
maxt := head.MaxTime() + 1 // Block intervals are half-open: [b.MinTime, b.MaxTime).
compactor, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{DefaultBlockDuration}, chunkenc.NewPool(), nil)
compactor, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{DefaultBlockDuration}, chunkenc.NewPool(), nil, true)
require.NoError(t, err)
id, err := compactor.Write(head.opts.ChunkDirRoot, head, mint, maxt, nil)
require.NoError(t, err)
@ -1548,7 +1865,7 @@ func TestSparseHistogramSpaceSavings(t *testing.T) {
// Sparse head compaction.
mint := sparseHead.MinTime()
maxt := sparseHead.MaxTime() + 1 // Block intervals are half-open: [b.MinTime, b.MaxTime).
compactor, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{DefaultBlockDuration}, chunkenc.NewPool(), nil)
compactor, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{DefaultBlockDuration}, chunkenc.NewPool(), nil, true)
require.NoError(t, err)
sparseULID, err = compactor.Write(sparseHead.opts.ChunkDirRoot, sparseHead, mint, maxt, nil)
require.NoError(t, err)
@ -1599,7 +1916,7 @@ func TestSparseHistogramSpaceSavings(t *testing.T) {
// Old head compaction.
mint := oldHead.MinTime()
maxt := oldHead.MaxTime() + 1 // Block intervals are half-open: [b.MinTime, b.MaxTime).
compactor, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{DefaultBlockDuration}, chunkenc.NewPool(), nil)
compactor, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{DefaultBlockDuration}, chunkenc.NewPool(), nil, true)
require.NoError(t, err)
oldULID, err = compactor.Write(oldHead.opts.ChunkDirRoot, oldHead, mint, maxt, nil)
require.NoError(t, err)
@ -1770,3 +2087,306 @@ func TestCompactBlockMetas(t *testing.T) {
}
require.Equal(t, expected, output)
}
func TestLeveledCompactor_plan_overlapping_disabled(t *testing.T) {
// This mimics our default ExponentialBlockRanges with min block size equals to 20.
compactor, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{
20,
60,
180,
540,
1620,
}, nil, nil, false)
require.NoError(t, err)
cases := map[string]struct {
metas []dirMeta
expected []string
}{
"Outside Range": {
metas: []dirMeta{
metaRange("1", 0, 20, nil),
},
expected: nil,
},
"We should wait for four blocks of size 20 to appear before compacting.": {
metas: []dirMeta{
metaRange("1", 0, 20, nil),
metaRange("2", 20, 40, nil),
},
expected: nil,
},
`We should wait for a next block of size 20 to appear before compacting
the existing ones. We have three, but we ignore the fresh one from WAl`: {
metas: []dirMeta{
metaRange("1", 0, 20, nil),
metaRange("2", 20, 40, nil),
metaRange("3", 40, 60, nil),
},
expected: nil,
},
"Block to fill the entire parent range appeared should be compacted": {
metas: []dirMeta{
metaRange("1", 0, 20, nil),
metaRange("2", 20, 40, nil),
metaRange("3", 40, 60, nil),
metaRange("4", 60, 80, nil),
},
expected: []string{"1", "2", "3"},
},
`Block for the next parent range appeared with gap with size 20. Nothing will happen in the first one
anymore but we ignore fresh one still, so no compaction`: {
metas: []dirMeta{
metaRange("1", 0, 20, nil),
metaRange("2", 20, 40, nil),
metaRange("3", 60, 80, nil),
},
expected: nil,
},
`Block for the next parent range appeared, and we have a gap with size 20 between second and third block.
We will not get this missed gap anymore and we should compact just these two.`: {
metas: []dirMeta{
metaRange("1", 0, 20, nil),
metaRange("2", 20, 40, nil),
metaRange("3", 60, 80, nil),
metaRange("4", 80, 100, nil),
},
expected: []string{"1", "2"},
},
"We have 20, 20, 20, 60, 60 range blocks. '5' is marked as fresh one": {
metas: []dirMeta{
metaRange("1", 0, 20, nil),
metaRange("2", 20, 40, nil),
metaRange("3", 40, 60, nil),
metaRange("4", 60, 120, nil),
metaRange("5", 120, 180, nil),
},
expected: []string{"1", "2", "3"},
},
"We have 20, 60, 20, 60, 240 range blocks. We can compact 20 + 60 + 60": {
metas: []dirMeta{
metaRange("2", 20, 40, nil),
metaRange("4", 60, 120, nil),
metaRange("5", 960, 980, nil), // Fresh one.
metaRange("6", 120, 180, nil),
metaRange("7", 720, 960, nil),
},
expected: []string{"2", "4", "6"},
},
"Do not select large blocks that have many tombstones when there is no fresh block": {
metas: []dirMeta{
metaRange("1", 0, 540, &BlockStats{
NumSeries: 10,
NumTombstones: 3,
}),
},
expected: nil,
},
"Select large blocks that have many tombstones when fresh appears": {
metas: []dirMeta{
metaRange("1", 0, 540, &BlockStats{
NumSeries: 10,
NumTombstones: 3,
}),
metaRange("2", 540, 560, nil),
},
expected: []string{"1"},
},
"For small blocks, do not compact tombstones, even when fresh appears.": {
metas: []dirMeta{
metaRange("1", 0, 60, &BlockStats{
NumSeries: 10,
NumTombstones: 3,
}),
metaRange("2", 60, 80, nil),
},
expected: nil,
},
`Regression test: we were stuck in a compact loop where we always recompacted
the same block when tombstones and series counts were zero`: {
metas: []dirMeta{
metaRange("1", 0, 540, &BlockStats{
NumSeries: 0,
NumTombstones: 0,
}),
metaRange("2", 540, 560, nil),
},
expected: nil,
},
`Regression test: we were wrongly assuming that new block is fresh from WAL when its ULID is newest.
We need to actually look on max time instead.
With previous, wrong approach "8" block was ignored, so we were wrongly compacting 5 and 7 and introducing
block overlaps`: {
metas: []dirMeta{
metaRange("5", 0, 360, nil),
metaRange("6", 540, 560, nil), // Fresh one.
metaRange("7", 360, 420, nil),
metaRange("8", 420, 540, nil),
},
expected: []string{"7", "8"},
},
// |--------------|
// |----------------|
// |--------------|
"Overlapping blocks 1": {
metas: []dirMeta{
metaRange("1", 0, 20, nil),
metaRange("2", 19, 40, nil),
metaRange("3", 40, 60, nil),
},
expected: nil,
},
// |--------------|
// |--------------|
// |--------------|
"Overlapping blocks 2": {
metas: []dirMeta{
metaRange("1", 0, 20, nil),
metaRange("2", 20, 40, nil),
metaRange("3", 30, 50, nil),
},
expected: nil,
},
// |--------------|
// |---------------------|
// |--------------|
"Overlapping blocks 3": {
metas: []dirMeta{
metaRange("1", 0, 20, nil),
metaRange("2", 10, 40, nil),
metaRange("3", 30, 50, nil),
},
expected: nil,
},
// |--------------|
// |--------------------------------|
// |--------------|
// |--------------|
"Overlapping blocks 4": {
metas: []dirMeta{
metaRange("5", 0, 360, nil),
metaRange("6", 340, 560, nil),
metaRange("7", 360, 420, nil),
metaRange("8", 420, 540, nil),
},
expected: nil,
},
// |--------------|
// |--------------|
// |--------------|
// |--------------|
"Overlapping blocks 5": {
metas: []dirMeta{
metaRange("1", 0, 10, nil),
metaRange("2", 9, 20, nil),
metaRange("3", 30, 40, nil),
metaRange("4", 39, 50, nil),
},
expected: nil,
},
}
for title, c := range cases {
if !t.Run(title, func(t *testing.T) {
res, err := compactor.plan(c.metas)
require.NoError(t, err)
require.Equal(t, c.expected, res)
}) {
return
}
}
}
func TestAsyncBlockWriterSuccess(t *testing.T) {
cw, err := chunks.NewWriter(t.TempDir())
require.NoError(t, err)
const series = 100
// prepare index, add all symbols
iw, err := index.NewWriter(context.Background(), filepath.Join(t.TempDir(), indexFilename))
require.NoError(t, err)
require.NoError(t, iw.AddSymbol("__name__"))
for ix := 0; ix < series; ix++ {
s := fmt.Sprintf("s_%3d", ix)
require.NoError(t, iw.AddSymbol(s))
}
// async block writer expects index writer ready to receive series.
abw := newAsyncBlockWriter(chunkenc.NewPool(), cw, iw, semaphore.NewWeighted(int64(1)))
for ix := 0; ix < series; ix++ {
s := fmt.Sprintf("s_%3d", ix)
require.NoError(t, abw.addSeries(labels.FromStrings("__name__", s), []chunks.Meta{{Chunk: randomChunk(t), MinTime: 0, MaxTime: math.MaxInt64}}))
}
// signal that no more series are coming
abw.closeAsync()
// We can do this repeatedly.
abw.closeAsync()
abw.closeAsync()
// wait for result
stats, err := abw.waitFinished()
require.NoError(t, err)
require.Equal(t, uint64(series), stats.NumSeries)
require.Equal(t, uint64(series), stats.NumChunks)
// We get the same result on subsequent calls to waitFinished.
for i := 0; i < 5; i++ {
newstats, err := abw.waitFinished()
require.NoError(t, err)
require.Equal(t, stats, newstats)
// We can call close async again, as long as it's on the same goroutine.
abw.closeAsync()
}
}
func TestAsyncBlockWriterFailure(t *testing.T) {
cw, err := chunks.NewWriter(t.TempDir())
require.NoError(t, err)
// We don't write symbols to this index writer, so adding series next will fail.
iw, err := index.NewWriter(context.Background(), filepath.Join(t.TempDir(), indexFilename))
require.NoError(t, err)
// async block writer expects index writer ready to receive series.
abw := newAsyncBlockWriter(chunkenc.NewPool(), cw, iw, semaphore.NewWeighted(int64(1)))
// Adding single series doesn't fail, as it just puts it onto the queue.
require.NoError(t, abw.addSeries(labels.FromStrings("__name__", "test"), []chunks.Meta{{Chunk: randomChunk(t), MinTime: 0, MaxTime: math.MaxInt64}}))
// Signal that no more series are coming.
abw.closeAsync()
// We can do this repeatedly.
abw.closeAsync()
abw.closeAsync()
// Wait for result, this time we get error due to missing symbols.
_, err = abw.waitFinished()
require.Error(t, err)
require.ErrorContains(t, err, "unknown symbol")
// We get the same error on each repeated call to waitFinished.
for i := 0; i < 5; i++ {
_, nerr := abw.waitFinished()
require.Equal(t, err, nerr)
// We can call close async again, as long as it's on the same goroutine.
abw.closeAsync()
}
}
func randomChunk(t *testing.T) chunkenc.Chunk {
chunk := chunkenc.NewXORChunk()
l := rand.Int() % 120
app, err := chunk.Appender()
require.NoError(t, err)
for i := 0; i < l; i++ {
app.Append(rand.Int63(), rand.Float64())
}
return chunk
}

View file

@ -44,6 +44,8 @@ import (
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
"github.com/prometheus/prometheus/tsdb/fileutil"
_ "github.com/prometheus/prometheus/tsdb/goversion" // Load the package into main to make sure minium Go version is met.
"github.com/prometheus/prometheus/tsdb/hashcache"
"github.com/prometheus/prometheus/tsdb/index"
"github.com/prometheus/prometheus/tsdb/tsdbutil"
"github.com/prometheus/prometheus/tsdb/wlog"
)
@ -70,19 +72,26 @@ var ErrNotReady = errors.New("TSDB not ready")
// millisecond precision timestamps.
func DefaultOptions() *Options {
return &Options{
WALSegmentSize: wlog.DefaultSegmentSize,
MaxBlockChunkSegmentSize: chunks.DefaultChunkSegmentSize,
RetentionDuration: int64(15 * 24 * time.Hour / time.Millisecond),
MinBlockDuration: DefaultBlockDuration,
MaxBlockDuration: DefaultBlockDuration,
NoLockfile: false,
AllowOverlappingCompaction: true,
WALCompression: false,
StripeSize: DefaultStripeSize,
HeadChunksWriteBufferSize: chunks.DefaultWriteBufferSize,
IsolationDisabled: defaultIsolationDisabled,
HeadChunksWriteQueueSize: chunks.DefaultWriteQueueSize,
OutOfOrderCapMax: DefaultOutOfOrderCapMax,
WALSegmentSize: wlog.DefaultSegmentSize,
MaxBlockChunkSegmentSize: chunks.DefaultChunkSegmentSize,
RetentionDuration: int64(15 * 24 * time.Hour / time.Millisecond),
MinBlockDuration: DefaultBlockDuration,
MaxBlockDuration: DefaultBlockDuration,
NoLockfile: false,
AllowOverlappingCompaction: true,
WALCompression: false,
StripeSize: DefaultStripeSize,
HeadChunksWriteBufferSize: chunks.DefaultWriteBufferSize,
IsolationDisabled: defaultIsolationDisabled,
HeadChunksEndTimeVariance: 0,
HeadChunksWriteQueueSize: chunks.DefaultWriteQueueSize,
OutOfOrderCapMax: DefaultOutOfOrderCapMax,
HeadPostingsForMatchersCacheTTL: defaultPostingsForMatchersCacheTTL,
HeadPostingsForMatchersCacheSize: defaultPostingsForMatchersCacheSize,
HeadPostingsForMatchersCacheForce: false,
BlockPostingsForMatchersCacheTTL: defaultPostingsForMatchersCacheTTL,
BlockPostingsForMatchersCacheSize: defaultPostingsForMatchersCacheSize,
BlockPostingsForMatchersCacheForce: false,
}
}
@ -146,6 +155,10 @@ type Options struct {
// HeadChunksWriteBufferSize configures the write buffer size used by the head chunks mapper.
HeadChunksWriteBufferSize int
// HeadChunksEndTimeVariance is how much variance (between 0 and 1) should be applied to the chunk end time,
// to spread chunks writing across time. Doesn't apply to the last chunk of the chunk range. 0 to disable variance.
HeadChunksEndTimeVariance float64
// HeadChunksWriteQueueSize configures the size of the chunk write queue used in the head chunks mapper.
HeadChunksWriteQueueSize int
@ -171,6 +184,10 @@ type Options struct {
// Disables isolation between reads and in-flight appends.
IsolationDisabled bool
// SeriesHashCache specifies the series hash cache used when querying shards via Querier.Select().
// If nil, the cache won't be used.
SeriesHashCache *hashcache.SeriesHashCache
// EnableNativeHistograms enables the ingestion of native histograms.
EnableNativeHistograms bool
@ -182,6 +199,29 @@ type Options struct {
// OutOfOrderCapMax is maximum capacity for OOO chunks (in samples).
// If it is <=0, the default value is assumed.
OutOfOrderCapMax int64
// HeadPostingsForMatchersCacheTTL is the TTL of the postings for matchers cache in the Head.
// If it's 0, the cache will only deduplicate in-flight requests, deleting the results once the first request has finished.
HeadPostingsForMatchersCacheTTL time.Duration
// HeadPostingsForMatchersCacheSize is the maximum size of cached postings for matchers elements in the Head.
// It's ignored when HeadPostingsForMatchersCacheTTL is 0.
HeadPostingsForMatchersCacheSize int
// HeadPostingsForMatchersCacheForce forces the usage of postings for matchers cache for all calls on Head and OOOHead regardless of the `concurrent` param.
HeadPostingsForMatchersCacheForce bool
// BlockPostingsForMatchersCacheTTL is the TTL of the postings for matchers cache of each compacted block.
// If it's 0, the cache will only deduplicate in-flight requests, deleting the results once the first request has finished.
BlockPostingsForMatchersCacheTTL time.Duration
// BlockPostingsForMatchersCacheSize is the maximum size of cached postings for matchers elements in each compacted block.
// It's ignored when BlockPostingsForMatchersCacheTTL is 0.
BlockPostingsForMatchersCacheSize int
// BlockPostingsForMatchersCacheForce forces the usage of postings for matchers cache for all calls on compacted blocks
// regardless of the `concurrent` param.
BlockPostingsForMatchersCacheForce bool
}
type BlocksToDeleteFunc func(blocks []*Block) map[ulid.ULID]struct{}
@ -436,6 +476,7 @@ func (db *DBReadOnly) FlushWAL(dir string) (returnErr error) {
ExponentialBlockRanges(DefaultOptions().MinBlockDuration, 3, 5),
chunkenc.NewPool(),
nil,
false,
)
if err != nil {
return errors.Wrap(err, "create leveled compactor")
@ -545,7 +586,7 @@ func (db *DBReadOnly) Blocks() ([]BlockReader, error) {
return nil, ErrClosed
default:
}
loadable, corrupted, err := openBlocks(db.logger, db.dir, nil, nil)
loadable, corrupted, err := openBlocks(db.logger, db.dir, nil, nil, nil, defaultPostingsForMatchersCacheTTL, defaultPostingsForMatchersCacheSize, false)
if err != nil {
return nil, err
}
@ -631,6 +672,9 @@ func validateOpts(opts *Options, rngs []int64) (*Options, []int64) {
if opts.HeadChunksWriteBufferSize <= 0 {
opts.HeadChunksWriteBufferSize = chunks.DefaultWriteBufferSize
}
if opts.HeadChunksEndTimeVariance <= 0 {
opts.HeadChunksEndTimeVariance = 0
}
if opts.HeadChunksWriteQueueSize < 0 {
opts.HeadChunksWriteQueueSize = chunks.DefaultWriteQueueSize
}
@ -740,7 +784,7 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
}
ctx, cancel := context.WithCancel(context.Background())
db.compactor, err = NewLeveledCompactorWithChunkSize(ctx, r, l, rngs, db.chunkPool, opts.MaxBlockChunkSegmentSize, nil)
db.compactor, err = NewLeveledCompactorWithChunkSize(ctx, r, l, rngs, db.chunkPool, opts.MaxBlockChunkSegmentSize, nil, opts.AllowOverlappingCompaction)
if err != nil {
cancel()
return nil, errors.Wrap(err, "create leveled compactor")
@ -777,6 +821,7 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
headOpts.ChunkDirRoot = dir
headOpts.ChunkPool = db.chunkPool
headOpts.ChunkWriteBufferSize = opts.HeadChunksWriteBufferSize
headOpts.ChunkEndTimeVariance = opts.HeadChunksEndTimeVariance
headOpts.ChunkWriteQueueSize = opts.HeadChunksWriteQueueSize
headOpts.StripeSize = opts.StripeSize
headOpts.SeriesCallback = opts.SeriesLifecycleCallback
@ -786,6 +831,9 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
headOpts.EnableNativeHistograms.Store(opts.EnableNativeHistograms)
headOpts.OutOfOrderTimeWindow.Store(opts.OutOfOrderTimeWindow)
headOpts.OutOfOrderCapMax.Store(opts.OutOfOrderCapMax)
headOpts.PostingsForMatchersCacheTTL = opts.HeadPostingsForMatchersCacheTTL
headOpts.PostingsForMatchersCacheSize = opts.HeadPostingsForMatchersCacheSize
headOpts.PostingsForMatchersCacheForce = opts.HeadPostingsForMatchersCacheForce
if opts.WALReplayConcurrency > 0 {
headOpts.WALReplayConcurrency = opts.WALReplayConcurrency
}
@ -1324,7 +1372,7 @@ func (db *DB) reloadBlocks() (err error) {
db.mtx.Lock()
defer db.mtx.Unlock()
loadable, corrupted, err := openBlocks(db.logger, db.dir, db.blocks, db.chunkPool)
loadable, corrupted, err := openBlocks(db.logger, db.dir, db.blocks, db.chunkPool, db.opts.SeriesHashCache, db.opts.BlockPostingsForMatchersCacheTTL, db.opts.BlockPostingsForMatchersCacheSize, db.opts.BlockPostingsForMatchersCacheForce)
if err != nil {
return err
}
@ -1392,7 +1440,7 @@ func (db *DB) reloadBlocks() (err error) {
blockMetas = append(blockMetas, b.Meta())
}
if overlaps := OverlappingBlocks(blockMetas); len(overlaps) > 0 {
level.Warn(db.logger).Log("msg", "Overlapping blocks found during reloadBlocks", "detail", overlaps.String())
level.Debug(db.logger).Log("msg", "Overlapping blocks found during reloadBlocks", "detail", overlaps.String())
}
// Append blocks to old, deletable blocks, so we can close them.
@ -1407,7 +1455,7 @@ func (db *DB) reloadBlocks() (err error) {
return nil
}
func openBlocks(l log.Logger, dir string, loaded []*Block, chunkPool chunkenc.Pool) (blocks []*Block, corrupted map[ulid.ULID]error, err error) {
func openBlocks(l log.Logger, dir string, loaded []*Block, chunkPool chunkenc.Pool, cache *hashcache.SeriesHashCache, postingsCacheTTL time.Duration, postingsCacheSize int, postingsCacheForce bool) (blocks []*Block, corrupted map[ulid.ULID]error, err error) {
bDirs, err := blockDirs(dir)
if err != nil {
return nil, nil, errors.Wrap(err, "find blocks")
@ -1424,7 +1472,12 @@ func openBlocks(l log.Logger, dir string, loaded []*Block, chunkPool chunkenc.Po
// See if we already have the block in memory or open it otherwise.
block, open := getBlock(loaded, meta.ULID)
if !open {
block, err = OpenBlock(l, bDir, chunkPool)
var cacheProvider index.ReaderCacheProvider
if cache != nil {
cacheProvider = cache.GetBlockCacheProvider(meta.ULID.String())
}
block, err = OpenBlockWithOptions(l, bDir, chunkPool, cacheProvider, postingsCacheTTL, postingsCacheSize, postingsCacheForce)
if err != nil {
corrupted[meta.ULID] = err
continue
@ -1672,7 +1725,7 @@ func (db *DB) inOrderBlocksMaxTime() (maxt int64, ok bool) {
maxt, ok = int64(math.MinInt64), false
// If blocks are overlapping, last block might not have the max time. So check all blocks.
for _, b := range db.Blocks() {
if !b.meta.Compaction.FromOutOfOrder() && b.meta.MaxTime > maxt {
if !b.meta.OutOfOrder && !b.meta.Compaction.FromOutOfOrder() && b.meta.MaxTime > maxt {
ok = true
maxt = b.meta.MaxTime
}
@ -1923,6 +1976,16 @@ func (db *DB) ChunkQuerier(_ context.Context, mint, maxt int64) (storage.ChunkQu
return storage.NewMergeChunkQuerier(blockQueriers, nil, storage.NewCompactingChunkSeriesMerger(storage.ChainedSeriesMerge)), nil
}
// UnorderedChunkQuerier returns a new chunk querier over the data partition for the given time range.
// The chunks can be overlapping and not sorted.
func (db *DB) UnorderedChunkQuerier(_ context.Context, mint, maxt int64) (storage.ChunkQuerier, error) {
blockQueriers, err := db.blockChunkQuerierForRange(mint, maxt)
if err != nil {
return nil, err
}
return storage.NewMergeChunkQuerier(blockQueriers, nil, storage.NewConcatenatingChunkSeriesMerger()), nil
}
func (db *DB) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) {
return db.head.exemplars.ExemplarQuerier(ctx)
}

View file

@ -5328,13 +5328,13 @@ func TestOOOMmapCorruption(t *testing.T) {
addSamples(120, 120, false)
// Second m-map file. We will corrupt this file. Sample 120 goes into this new file.
db.head.chunkDiskMapper.CutNewFile()
require.NoError(t, db.head.chunkDiskMapper.CutNewFile())
// More OOO samples.
addSamples(200, 230, false)
addSamples(240, 255, false)
db.head.chunkDiskMapper.CutNewFile()
require.NoError(t, db.head.chunkDiskMapper.CutNewFile())
addSamples(260, 290, false)
verifySamples := func(expSamples []tsdbutil.Sample) {

View file

@ -0,0 +1,192 @@
package hashcache
import (
"sync"
"go.uber.org/atomic"
"github.com/prometheus/prometheus/storage"
)
const (
numGenerations = 4
// approxBytesPerEntry is the estimated memory footprint (in bytes) of 1 cache
// entry, measured with TestSeriesHashCache_MeasureApproximateSizePerEntry().
approxBytesPerEntry = 28
)
// SeriesHashCache is a bounded cache mapping the per-block series ID with
// its labels hash.
type SeriesHashCache struct {
maxEntriesPerGeneration uint64
generationsMx sync.RWMutex
generations [numGenerations]cacheGeneration
}
func NewSeriesHashCache(maxBytes uint64) *SeriesHashCache {
maxEntriesPerGeneration := maxBytes / approxBytesPerEntry / numGenerations
if maxEntriesPerGeneration < 1 {
maxEntriesPerGeneration = 1
}
c := &SeriesHashCache{maxEntriesPerGeneration: maxEntriesPerGeneration}
// Init generations.
for idx := 0; idx < numGenerations; idx++ {
c.generations[idx].blocks = &sync.Map{}
c.generations[idx].length = atomic.NewUint64(0)
}
return c
}
// GetBlockCache returns a reference to the series hash cache for the provided blockID.
// The returned cache reference should be retained only for a short period (ie. the duration
// of the execution of 1 single query).
func (c *SeriesHashCache) GetBlockCache(blockID string) *BlockSeriesHashCache {
blockCache := &BlockSeriesHashCache{}
c.generationsMx.RLock()
defer c.generationsMx.RUnlock()
// Trigger a garbage collection if the current generation reached the max size.
if c.generations[0].length.Load() >= c.maxEntriesPerGeneration {
c.generationsMx.RUnlock()
c.gc()
c.generationsMx.RLock()
}
for idx := 0; idx < numGenerations; idx++ {
gen := c.generations[idx]
if value, ok := gen.blocks.Load(blockID); ok {
blockCache.generations[idx] = value.(*blockCacheGeneration)
continue
}
// Create a new per-block cache only for the current generation.
// If the cache for the older generation doesn't exist, then its
// value will be null and skipped when reading.
if idx == 0 {
value, _ := gen.blocks.LoadOrStore(blockID, newBlockCacheGeneration(gen.length))
blockCache.generations[idx] = value.(*blockCacheGeneration)
}
}
return blockCache
}
// GetBlockCacheProvider returns a cache provider bounded to the provided blockID.
func (c *SeriesHashCache) GetBlockCacheProvider(blockID string) *BlockSeriesHashCacheProvider {
return NewBlockSeriesHashCacheProvider(c, blockID)
}
func (c *SeriesHashCache) gc() {
c.generationsMx.Lock()
defer c.generationsMx.Unlock()
// Make sure no other goroutines already GCed the current generation.
if c.generations[0].length.Load() < c.maxEntriesPerGeneration {
return
}
// Shift the current generation to old.
for idx := numGenerations - 2; idx >= 0; idx-- {
c.generations[idx+1] = c.generations[idx]
}
// Initialise a new empty current generation.
c.generations[0] = cacheGeneration{
blocks: &sync.Map{},
length: atomic.NewUint64(0),
}
}
// cacheGeneration holds a multi-blocks cache generation.
type cacheGeneration struct {
// blocks maps the block ID with blockCacheGeneration.
blocks *sync.Map
// Keeps track of the number of items added to the cache. This counter
// is passed to each blockCacheGeneration belonging to this generation.
length *atomic.Uint64
}
// blockCacheGeneration holds a per-block cache generation.
type blockCacheGeneration struct {
// hashes maps per-block series ID with its hash.
hashesMx sync.RWMutex
hashes map[storage.SeriesRef]uint64
// Keeps track of the number of items added to the cache. This counter is
// shared with all blockCacheGeneration in the "parent" cacheGeneration.
length *atomic.Uint64
}
func newBlockCacheGeneration(length *atomic.Uint64) *blockCacheGeneration {
return &blockCacheGeneration{
hashes: make(map[storage.SeriesRef]uint64),
length: length,
}
}
type BlockSeriesHashCache struct {
generations [numGenerations]*blockCacheGeneration
}
// Fetch the hash of the given seriesID from the cache and returns a boolean
// whether the series was found in the cache or not.
func (c *BlockSeriesHashCache) Fetch(seriesID storage.SeriesRef) (uint64, bool) {
// Look for it in all generations, starting from the most recent one (index 0).
for idx := 0; idx < numGenerations; idx++ {
gen := c.generations[idx]
// Skip if the cache doesn't exist for this generation.
if gen == nil {
continue
}
gen.hashesMx.RLock()
value, ok := gen.hashes[seriesID]
gen.hashesMx.RUnlock()
if ok {
return value, true
}
}
return 0, false
}
// Store the hash of the given seriesID in the cache.
func (c *BlockSeriesHashCache) Store(seriesID storage.SeriesRef, hash uint64) {
// Store it in the most recent generation (index 0).
gen := c.generations[0]
gen.hashesMx.Lock()
gen.hashes[seriesID] = hash
gen.hashesMx.Unlock()
gen.length.Add(1)
}
type BlockSeriesHashCacheProvider struct {
cache *SeriesHashCache
blockID string
}
// NewBlockSeriesHashCacheProvider makes a new BlockSeriesHashCacheProvider.
func NewBlockSeriesHashCacheProvider(cache *SeriesHashCache, blockID string) *BlockSeriesHashCacheProvider {
return &BlockSeriesHashCacheProvider{
cache: cache,
blockID: blockID,
}
}
// SeriesHashCache returns a reference to the cache bounded to block provided
// to NewBlockSeriesHashCacheProvider().
func (p *BlockSeriesHashCacheProvider) SeriesHashCache() *BlockSeriesHashCache {
return p.cache.GetBlockCache(p.blockID)
}

View file

@ -0,0 +1,137 @@
package hashcache
import (
"crypto/rand"
"fmt"
"runtime"
"strconv"
"sync"
"testing"
"github.com/oklog/ulid"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/storage"
)
func TestSeriesHashCache(t *testing.T) {
// Set the max cache size to store at most 1 entry per generation,
// so that we test the GC logic too.
c := NewSeriesHashCache(numGenerations * approxBytesPerEntry)
block1 := c.GetBlockCache("1")
assertFetch(t, block1, 1, 0, false)
block1.Store(1, 100)
assertFetch(t, block1, 1, 100, true)
block2 := c.GetBlockCache("2")
assertFetch(t, block2, 1, 0, false)
block2.Store(1, 1000)
assertFetch(t, block2, 1, 1000, true)
block3 := c.GetBlockCache("3")
assertFetch(t, block1, 1, 100, true)
assertFetch(t, block2, 1, 1000, true)
assertFetch(t, block3, 1, 0, false)
// Get again the block caches.
block1 = c.GetBlockCache("1")
block2 = c.GetBlockCache("2")
block3 = c.GetBlockCache("3")
assertFetch(t, block1, 1, 100, true)
assertFetch(t, block2, 1, 1000, true)
assertFetch(t, block3, 1, 0, false)
}
func TestSeriesHashCache_MeasureApproximateSizePerEntry(t *testing.T) {
// This test measures the approximate size (in bytes) per cache entry.
// We only take in account the memory used by the map, which is the largest amount.
const numEntries = 100000
c := NewSeriesHashCache(1024 * 1024 * 1024)
b := c.GetBlockCache(ulid.MustNew(0, rand.Reader).String())
before := runtime.MemStats{}
runtime.ReadMemStats(&before)
// Preallocate the map in order to not account for re-allocations
// since we want to measure the heap utilization and not allocations.
b.generations[0].hashes = make(map[storage.SeriesRef]uint64, numEntries)
for i := uint64(0); i < numEntries; i++ {
b.Store(storage.SeriesRef(i), i)
}
after := runtime.MemStats{}
runtime.ReadMemStats(&after)
t.Logf("approximate size per entry: %d bytes", (after.TotalAlloc-before.TotalAlloc)/numEntries)
require.Equal(t, uint64(approxBytesPerEntry), (after.TotalAlloc-before.TotalAlloc)/numEntries, "approxBytesPerEntry constant is out date")
}
func TestSeriesHashCache_Concurrency(t *testing.T) {
const (
concurrency = 100
numIterations = 10000
numBlocks = 10
)
// Set the max cache size to store at most 10 entries per generation,
// so that we stress test the GC too.
c := NewSeriesHashCache(10 * numGenerations * approxBytesPerEntry)
wg := sync.WaitGroup{}
wg.Add(concurrency)
for i := 0; i < concurrency; i++ {
go func() {
defer wg.Done()
for n := 0; n < numIterations; n++ {
blockID := strconv.Itoa(n % numBlocks)
blockCache := c.GetBlockCache(blockID)
blockCache.Store(storage.SeriesRef(n), uint64(n))
actual, ok := blockCache.Fetch(storage.SeriesRef(n))
require.True(t, ok)
require.Equal(t, uint64(n), actual)
}
}()
}
wg.Wait()
}
func BenchmarkSeriesHashCache_StoreAndFetch(b *testing.B) {
for _, numBlocks := range []int{1, 10, 100, 1000, 10000} {
b.Run(fmt.Sprintf("blocks=%d", numBlocks), func(b *testing.B) {
c := NewSeriesHashCache(1024 * 1024)
// In this benchmark we assume the usage pattern is calling Fetch() and Store() will be
// orders of magnitude more frequent than GetBlockCache(), so we call GetBlockCache() just
// once per block.
blockCaches := make([]*BlockSeriesHashCache, numBlocks)
for idx := 0; idx < numBlocks; idx++ {
blockCaches[idx] = c.GetBlockCache(strconv.Itoa(idx))
}
// In this benchmark we assume the ratio between Store() and Fetch() is 1:10.
storeOps := (b.N / 10) + 1
for n := 0; n < b.N; n++ {
if n < storeOps {
blockCaches[n%numBlocks].Store(storage.SeriesRef(n), uint64(n))
} else {
blockCaches[n%numBlocks].Fetch(storage.SeriesRef(n % storeOps))
}
}
})
}
}
func assertFetch(t *testing.T, c *BlockSeriesHashCache, seriesID storage.SeriesRef, expectedValue uint64, expectedOk bool) {
actualValue, actualOk := c.Fetch(seriesID)
require.Equal(t, expectedValue, actualValue)
require.Equal(t, expectedOk, actualOk)
}

View file

@ -64,6 +64,20 @@ var (
defaultWALReplayConcurrency = runtime.GOMAXPROCS(0)
)
// chunkDiskMapper is a temporary interface while we transition from
// 0 size queue to queue based chunk disk mapper.
type chunkDiskMapper interface {
CutNewFile() (returnErr error)
IterateAllChunks(f func(seriesRef chunks.HeadSeriesRef, chunkRef chunks.ChunkDiskMapperRef, mint, maxt int64, numSamples uint16, encoding chunkenc.Encoding, isOOO bool) error) (err error)
Truncate(fileNo uint32) error
DeleteCorrupted(originalErr error) error
Size() (int64, error)
Close() error
Chunk(ref chunks.ChunkDiskMapperRef) (chunkenc.Chunk, error)
WriteChunk(seriesRef chunks.HeadSeriesRef, mint, maxt int64, chk chunkenc.Chunk, isOOO bool, callback func(err error)) (chkRef chunks.ChunkDiskMapperRef)
IsQueueEmpty() bool
}
// Head handles reads and writes of time series data within a time window.
type Head struct {
chunkRange atomic.Int64
@ -101,6 +115,7 @@ type Head struct {
// TODO(codesome): Extend MemPostings to return only OOOPostings, Set OOOStatus, ... Like an additional map of ooo postings.
postings *index.MemPostings // Postings lists for terms.
pfmc *PostingsForMatchersCache
tombstones *tombstones.MemTombstones
@ -111,7 +126,7 @@ type Head struct {
lastPostingsStatsCall time.Duration // Last posting stats call (PostingsCardinalityStats()) time for caching.
// chunkDiskMapper is used to write and read Head chunks to/from disk.
chunkDiskMapper *chunks.ChunkDiskMapper
chunkDiskMapper chunkDiskMapper
chunkSnapshotMtx sync.Mutex
@ -148,6 +163,7 @@ type HeadOptions struct {
ChunkDirRoot string
ChunkPool chunkenc.Pool
ChunkWriteBufferSize int
ChunkEndTimeVariance float64
ChunkWriteQueueSize int
// StripeSize sets the number of entries in the hash map, it must be a power of 2.
@ -160,6 +176,10 @@ type HeadOptions struct {
IsolationDisabled bool
PostingsForMatchersCacheTTL time.Duration
PostingsForMatchersCacheSize int
PostingsForMatchersCacheForce bool
// Maximum number of CPUs that can simultaneously processes WAL replay.
// The default value is GOMAXPROCS.
// If it is set to a negative value or zero, the default value is used.
@ -173,15 +193,19 @@ const (
func DefaultHeadOptions() *HeadOptions {
ho := &HeadOptions{
ChunkRange: DefaultBlockDuration,
ChunkDirRoot: "",
ChunkPool: chunkenc.NewPool(),
ChunkWriteBufferSize: chunks.DefaultWriteBufferSize,
ChunkWriteQueueSize: chunks.DefaultWriteQueueSize,
StripeSize: DefaultStripeSize,
SeriesCallback: &noopSeriesLifecycleCallback{},
IsolationDisabled: defaultIsolationDisabled,
WALReplayConcurrency: defaultWALReplayConcurrency,
ChunkRange: DefaultBlockDuration,
ChunkDirRoot: "",
ChunkPool: chunkenc.NewPool(),
ChunkWriteBufferSize: chunks.DefaultWriteBufferSize,
ChunkEndTimeVariance: 0,
ChunkWriteQueueSize: chunks.DefaultWriteQueueSize,
StripeSize: DefaultStripeSize,
SeriesCallback: &noopSeriesLifecycleCallback{},
IsolationDisabled: defaultIsolationDisabled,
PostingsForMatchersCacheTTL: defaultPostingsForMatchersCacheTTL,
PostingsForMatchersCacheSize: defaultPostingsForMatchersCacheSize,
PostingsForMatchersCacheForce: false,
WALReplayConcurrency: defaultWALReplayConcurrency,
}
ho.OutOfOrderCapMax.Store(DefaultOutOfOrderCapMax)
return ho
@ -247,6 +271,8 @@ func NewHead(r prometheus.Registerer, l log.Logger, wal, wbl *wlog.WL, opts *Hea
},
stats: stats,
reg: r,
pfmc: NewPostingsForMatchersCache(opts.PostingsForMatchersCacheTTL, opts.PostingsForMatchersCacheSize, opts.PostingsForMatchersCacheForce),
}
if err := h.resetInMemoryState(); err != nil {
return nil, err
@ -490,6 +516,7 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
m.checkpointCreationTotal,
m.mmapChunkCorruptionTotal,
m.snapshotReplayErrorTotal,
m.oooHistogram,
// Metrics bound to functions and not needed in tests
// can be created and registered on the spot.
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
@ -1407,7 +1434,7 @@ func (h *Head) Delete(mint, maxt int64, ms ...*labels.Matcher) error {
ir := h.indexRange(mint, maxt)
p, err := PostingsForMatchers(ir, ms...)
p, err := ir.PostingsForMatchers(false, ms...)
if err != nil {
return errors.Wrap(err, "select series")
}
@ -1594,7 +1621,7 @@ func (h *Head) getOrCreate(hash uint64, lset labels.Labels) (*memSeries, bool, e
func (h *Head) getOrCreateWithID(id chunks.HeadSeriesRef, hash uint64, lset labels.Labels) (*memSeries, bool, error) {
s, created, err := h.series.getOrSet(hash, lset, func() *memSeries {
return newMemSeries(lset, id, h.opts.IsolationDisabled)
return newMemSeries(lset, id, labels.StableHash(lset), h.opts.ChunkEndTimeVariance, h.opts.IsolationDisabled)
})
if err != nil {
return nil, false, err
@ -1885,6 +1912,9 @@ type memSeries struct {
lset labels.Labels
meta *metadata.Metadata
// Series labels hash to use for sharding purposes.
shardHash uint64
// Immutable chunks on disk that have not yet gone into a block, in order of ascending time stamps.
// When compaction runs, chunks get moved into a block and all pointers are shifted like so:
//
@ -1902,6 +1932,10 @@ type memSeries struct {
mmMaxTime int64 // Max time of any mmapped chunk, only used during WAL replay.
// chunkEndTimeVariance is how much variance (between 0 and 1) should be applied to the chunk end time,
// to spread chunks writing across time. Doesn't apply to the last chunk of the chunk range. 0 to disable variance.
chunkEndTimeVariance float64
nextAt int64 // Timestamp at which to cut the next chunk.
// We keep the last value here (in addition to appending it to the chunk) so we can check for duplicates.
@ -1930,11 +1964,13 @@ type memSeriesOOOFields struct {
firstOOOChunkID chunks.HeadChunkID // HeadOOOChunkID for oooMmappedChunks[0].
}
func newMemSeries(lset labels.Labels, id chunks.HeadSeriesRef, isolationDisabled bool) *memSeries {
func newMemSeries(lset labels.Labels, id chunks.HeadSeriesRef, shardHash uint64, chunkEndTimeVariance float64, isolationDisabled bool) *memSeries {
s := &memSeries{
lset: lset,
ref: id,
nextAt: math.MinInt64,
lset: lset,
ref: id,
nextAt: math.MinInt64,
chunkEndTimeVariance: chunkEndTimeVariance,
shardHash: shardHash,
}
if !isolationDisabled {
s.txs = newTxRing(4)

View file

@ -187,6 +187,13 @@ func (h *Head) AppendableMinValidTime() (int64, bool) {
return h.appendableMinValidTime(), true
}
func min(a, b int64) int64 {
if a < b {
return a
}
return b
}
func max(a, b int64) int64 {
if a > b {
return a
@ -1085,7 +1092,7 @@ func (a *headAppender) Commit() (err error) {
}
// insert is like append, except it inserts. Used for OOO samples.
func (s *memSeries) insert(t int64, v float64, chunkDiskMapper *chunks.ChunkDiskMapper, oooCapMax int64) (inserted, chunkCreated bool, mmapRef chunks.ChunkDiskMapperRef) {
func (s *memSeries) insert(t int64, v float64, chunkDiskMapper chunkDiskMapper, oooCapMax int64) (inserted, chunkCreated bool, mmapRef chunks.ChunkDiskMapperRef) {
if s.ooo == nil {
s.ooo = &memSeriesOOOFields{}
}
@ -1112,7 +1119,7 @@ func (s *memSeries) insert(t int64, v float64, chunkDiskMapper *chunks.ChunkDisk
// the appendID for isolation. (The appendID can be zero, which results in no
// isolation for this append.)
// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock.
func (s *memSeries) append(t int64, v float64, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper, chunkRange int64) (sampleInOrder, chunkCreated bool) {
func (s *memSeries) append(t int64, v float64, appendID uint64, chunkDiskMapper chunkDiskMapper, chunkRange int64) (sampleInOrder, chunkCreated bool) {
c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncXOR, chunkDiskMapper, chunkRange)
if !sampleInOrder {
return sampleInOrder, chunkCreated
@ -1134,7 +1141,7 @@ func (s *memSeries) append(t int64, v float64, appendID uint64, chunkDiskMapper
// appendHistogram adds the histogram.
// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock.
func (s *memSeries) appendHistogram(t int64, h *histogram.Histogram, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper, chunkRange int64) (sampleInOrder, chunkCreated bool) {
func (s *memSeries) appendHistogram(t int64, h *histogram.Histogram, appendID uint64, chunkDiskMapper chunkDiskMapper, chunkRange int64) (sampleInOrder, chunkCreated bool) {
// Head controls the execution of recoding, so that we own the proper
// chunk reference afterwards. We check for Appendable from appender before
// appendPreprocessor because in case it ends up creating a new chunk,
@ -1227,7 +1234,7 @@ func (s *memSeries) appendHistogram(t int64, h *histogram.Histogram, appendID ui
// appendFloatHistogram adds the float histogram.
// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock.
func (s *memSeries) appendFloatHistogram(t int64, fh *histogram.FloatHistogram, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper, chunkRange int64) (sampleInOrder, chunkCreated bool) {
func (s *memSeries) appendFloatHistogram(t int64, fh *histogram.FloatHistogram, appendID uint64, chunkDiskMapper chunkDiskMapper, chunkRange int64) (sampleInOrder, chunkCreated bool) {
// Head controls the execution of recoding, so that we own the proper
// chunk reference afterwards. We check for Appendable from appender before
// appendPreprocessor because in case it ends up creating a new chunk,
@ -1322,7 +1329,7 @@ func (s *memSeries) appendFloatHistogram(t int64, fh *histogram.FloatHistogram,
// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock.
// This should be called only when appending data.
func (s *memSeries) appendPreprocessor(
t int64, e chunkenc.Encoding, chunkDiskMapper *chunks.ChunkDiskMapper, chunkRange int64,
t int64, e chunkenc.Encoding, chunkDiskMapper chunkDiskMapper, chunkRange int64,
) (c *memChunk, sampleInOrder, chunkCreated bool) {
// Based on Gorilla white papers this offers near-optimal compression ratio
// so anything bigger that this has diminishing returns and increases
@ -1366,7 +1373,10 @@ func (s *memSeries) appendPreprocessor(
// the remaining chunks in the current chunk range.
// At latest it must happen at the timestamp set when the chunk was cut.
if numSamples == samplesPerChunk/4 {
s.nextAt = computeChunkEndTime(c.minTime, c.maxTime, s.nextAt)
maxNextAt := s.nextAt
s.nextAt = computeChunkEndTime(c.minTime, c.maxTime, maxNextAt)
s.nextAt = addJitterToChunkEndTime(s.shardHash, c.minTime, s.nextAt, maxNextAt, s.chunkEndTimeVariance)
}
// If numSamples > samplesPerChunk*2 then our previous prediction was invalid,
// most likely because samples rate has changed and now they are arriving more frequently.
@ -1394,8 +1404,32 @@ func computeChunkEndTime(start, cur, max int64) int64 {
return start + (max-start)/n
}
// addJitterToChunkEndTime return chunk's nextAt applying a jitter based on the provided expected variance.
// The variance is applied to the estimated chunk duration (nextAt - chunkMinTime); the returned updated chunk
// end time is guaranteed to be between "chunkDuration - (chunkDuration*(variance/2))" to
// "chunkDuration + chunkDuration*(variance/2)", and never greater than maxNextAt.
func addJitterToChunkEndTime(seriesHash uint64, chunkMinTime, nextAt, maxNextAt int64, variance float64) int64 {
if variance <= 0 {
return nextAt
}
// Do not apply the jitter if the chunk is expected to be the last one of the chunk range.
if nextAt >= maxNextAt {
return nextAt
}
// Compute the variance to apply to the chunk end time. The variance is based on the series hash so that
// different TSDBs ingesting the same exact samples (e.g. in a distributed system like Mimir) will have
// the same chunks for a given period.
chunkDuration := nextAt - chunkMinTime
chunkDurationMaxVariance := int64(float64(chunkDuration) * variance)
chunkDurationVariance := int64(seriesHash % uint64(chunkDurationMaxVariance))
return min(maxNextAt, nextAt+chunkDurationVariance-(chunkDurationMaxVariance/2))
}
func (s *memSeries) cutNewHeadChunk(
mint int64, e chunkenc.Encoding, chunkDiskMapper *chunks.ChunkDiskMapper, chunkRange int64,
mint int64, e chunkenc.Encoding, chunkDiskMapper chunkDiskMapper, chunkRange int64,
) *memChunk {
s.mmapCurrentHeadChunk(chunkDiskMapper)
@ -1428,7 +1462,7 @@ func (s *memSeries) cutNewHeadChunk(
// cutNewOOOHeadChunk cuts a new OOO chunk and m-maps the old chunk.
// The caller must ensure that s.ooo is not nil.
func (s *memSeries) cutNewOOOHeadChunk(mint int64, chunkDiskMapper *chunks.ChunkDiskMapper) (*oooHeadChunk, chunks.ChunkDiskMapperRef) {
func (s *memSeries) cutNewOOOHeadChunk(mint int64, chunkDiskMapper chunkDiskMapper) (*oooHeadChunk, chunks.ChunkDiskMapperRef) {
ref := s.mmapCurrentOOOHeadChunk(chunkDiskMapper)
s.ooo.oooHeadChunk = &oooHeadChunk{
@ -1440,7 +1474,7 @@ func (s *memSeries) cutNewOOOHeadChunk(mint int64, chunkDiskMapper *chunks.Chunk
return s.ooo.oooHeadChunk, ref
}
func (s *memSeries) mmapCurrentOOOHeadChunk(chunkDiskMapper *chunks.ChunkDiskMapper) chunks.ChunkDiskMapperRef {
func (s *memSeries) mmapCurrentOOOHeadChunk(chunkDiskMapper chunkDiskMapper) chunks.ChunkDiskMapperRef {
if s.ooo == nil || s.ooo.oooHeadChunk == nil {
// There is no head chunk, so nothing to m-map here.
return 0
@ -1457,7 +1491,7 @@ func (s *memSeries) mmapCurrentOOOHeadChunk(chunkDiskMapper *chunks.ChunkDiskMap
return chunkRef
}
func (s *memSeries) mmapCurrentHeadChunk(chunkDiskMapper *chunks.ChunkDiskMapper) {
func (s *memSeries) mmapCurrentHeadChunk(chunkDiskMapper chunkDiskMapper) {
if s.headChunk == nil || s.headChunk.chunk.NumSamples() == 0 {
// There is no head chunk, so nothing to m-map here.
return

66
tsdb/head_append_test.go Normal file
View file

@ -0,0 +1,66 @@
package tsdb
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestAddJitterToChunkEndTime_ShouldHonorMaxVarianceAndMaxNextAt(t *testing.T) {
chunkMinTime := int64(10)
nextAt := int64(95)
maxNextAt := int64(100)
variance := 0.2
// Compute the expected max variance.
expectedMaxVariance := int64(float64(nextAt-chunkMinTime) * variance)
for seriesHash := uint64(0); seriesHash < 1000; seriesHash++ {
actual := addJitterToChunkEndTime(seriesHash, chunkMinTime, nextAt, maxNextAt, variance)
require.GreaterOrEqual(t, actual, nextAt-(expectedMaxVariance/2))
require.LessOrEqual(t, actual, maxNextAt)
}
}
func TestAddJitterToChunkEndTime_Distribution(t *testing.T) {
chunkMinTime := int64(0)
nextAt := int64(50)
maxNextAt := int64(100)
variance := 0.2
numSeries := uint64(1000)
// Compute the expected max variance.
expectedMaxVariance := int64(float64(nextAt-chunkMinTime) * variance)
// Keep track of the distribution of the applied variance.
varianceDistribution := map[int64]int64{}
for seriesHash := uint64(0); seriesHash < numSeries; seriesHash++ {
actual := addJitterToChunkEndTime(seriesHash, chunkMinTime, nextAt, maxNextAt, variance)
require.GreaterOrEqual(t, actual, nextAt-(expectedMaxVariance/2))
require.LessOrEqual(t, actual, nextAt+(expectedMaxVariance/2))
require.LessOrEqual(t, actual, maxNextAt)
variance := nextAt - actual
varianceDistribution[variance]++
}
// Ensure a uniform distribution.
for variance, count := range varianceDistribution {
require.Equalf(t, int64(numSeries)/expectedMaxVariance, count, "variance = %d", variance)
}
}
func TestAddJitterToChunkEndTime_ShouldNotApplyJitterToTheLastChunkOfTheRange(t *testing.T) {
// Since the jitter could also be 0, we try it for multiple series.
for seriesHash := uint64(0); seriesHash < 10; seriesHash++ {
require.Equal(t, int64(200), addJitterToChunkEndTime(seriesHash, 150, 200, 200, 0.2))
}
}
func TestAddJitterToChunkEndTime_ShouldNotApplyJitterIfDisabled(t *testing.T) {
// Since the jitter could also be 0, we try it for multiple series.
for seriesHash := uint64(0); seriesHash < 10; seriesHash++ {
require.Equal(t, int64(130), addJitterToChunkEndTime(seriesHash, 100, 130, 200, 0))
}
}

View file

@ -121,6 +121,10 @@ func (h *headIndexReader) Postings(name string, values ...string) (index.Posting
}
}
func (h *headIndexReader) PostingsForMatchers(concurrent bool, ms ...*labels.Matcher) (index.Postings, error) {
return h.head.pfmc.PostingsForMatchers(h, concurrent, ms...)
}
func (h *headIndexReader) SortedPostings(p index.Postings) index.Postings {
series := make([]*memSeries, 0, 128)
@ -149,6 +153,27 @@ func (h *headIndexReader) SortedPostings(p index.Postings) index.Postings {
return index.NewListPostings(ep)
}
func (h *headIndexReader) ShardedPostings(p index.Postings, shardIndex, shardCount uint64) index.Postings {
out := make([]storage.SeriesRef, 0, 128)
for p.Next() {
s := h.head.series.getByID(chunks.HeadSeriesRef(p.At()))
if s == nil {
level.Debug(h.head.logger).Log("msg", "Looked up series not found")
continue
}
// Check if the series belong to the shard.
if s.shardHash%shardCount != shardIndex {
continue
}
out = append(out, storage.SeriesRef(s.ref))
}
return index.NewListPostings(out)
}
// Series returns the series for the given reference.
func (h *headIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error {
s := h.head.series.getByID(chunks.HeadSeriesRef(ref))
@ -159,6 +184,10 @@ func (h *headIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchB
}
builder.Assign(s.lset)
if chks == nil {
return nil
}
s.Lock()
defer s.Unlock()
@ -342,7 +371,7 @@ func (h *headChunkReader) chunk(meta chunks.Meta, copyLastChunk bool) (chunkenc.
// chunk returns the chunk for the HeadChunkID from memory or by m-mapping it from the disk.
// If headChunk is false, it means that the returned *memChunk
// (and not the chunkenc.Chunk inside it) can be garbage collected after its usage.
func (s *memSeries) chunk(id chunks.HeadChunkID, chunkDiskMapper *chunks.ChunkDiskMapper, memChunkPool *sync.Pool) (chunk *memChunk, headChunk bool, err error) {
func (s *memSeries) chunk(id chunks.HeadChunkID, cdm chunkDiskMapper, memChunkPool *sync.Pool) (chunk *memChunk, headChunk bool, err error) {
// ix represents the index of chunk in the s.mmappedChunks slice. The chunk id's are
// incremented by 1 when new chunk is created, hence (id - firstChunkID) gives the slice index.
// The max index for the s.mmappedChunks slice can be len(s.mmappedChunks)-1, hence if the ix
@ -358,7 +387,7 @@ func (s *memSeries) chunk(id chunks.HeadChunkID, chunkDiskMapper *chunks.ChunkDi
}
return s.headChunk, true, nil
}
chk, err := chunkDiskMapper.Chunk(s.mmappedChunks[ix].ref)
chk, err := cdm.Chunk(s.mmappedChunks[ix].ref)
if err != nil {
if _, ok := err.(*chunks.CorruptionErr); ok {
panic(err)
@ -378,7 +407,7 @@ func (s *memSeries) chunk(id chunks.HeadChunkID, chunkDiskMapper *chunks.ChunkDi
// chunks in the OOOHead.
// This function is not thread safe unless the caller holds a lock.
// The caller must ensure that s.ooo is not nil.
func (s *memSeries) oooMergedChunk(meta chunks.Meta, cdm *chunks.ChunkDiskMapper, mint, maxt int64) (chunk *mergedOOOChunks, err error) {
func (s *memSeries) oooMergedChunk(meta chunks.Meta, cdm chunkDiskMapper, mint, maxt int64) (chunk *mergedOOOChunks, err error) {
_, cid := chunks.HeadChunkRef(meta.Ref).Unpack()
// ix represents the index of chunk in the s.mmappedChunks slice. The chunk meta's are

View file

@ -284,7 +284,8 @@ func BenchmarkLoadWAL(b *testing.B) {
require.NoError(b, err)
for k := 0; k < c.batches*c.seriesPerBatch; k++ {
// Create one mmapped chunk per series, with one sample at the given time.
s := newMemSeries(labels.Labels{}, chunks.HeadSeriesRef(k)*101, defaultIsolationDisabled)
lbls := labels.Labels{}
s := newMemSeries(lbls, chunks.HeadSeriesRef(k)*101, labels.StableHash(lbls), 0, defaultIsolationDisabled)
s.append(c.mmappedChunkT, 42, 0, chunkDiskMapper, c.mmappedChunkT)
s.mmapCurrentHeadChunk(chunkDiskMapper)
}
@ -378,9 +379,9 @@ func TestHead_HighConcurrencyReadAndWrite(t *testing.T) {
workerReadyWg.Add(writeConcurrency + readConcurrency)
// Start the write workers.
for wid := 0; wid < writeConcurrency; wid++ {
for workerID := 0; workerID < writeConcurrency; workerID++ {
// Create copy of workerID to be used by worker routine.
workerID := wid
workerID := workerID
g.Go(func() error {
// The label sets which this worker will write.
@ -422,9 +423,9 @@ func TestHead_HighConcurrencyReadAndWrite(t *testing.T) {
readerTsCh := make(chan uint64)
// Start the read workers.
for wid := 0; wid < readConcurrency; wid++ {
for workerID := 0; workerID < readConcurrency; workerID++ {
// Create copy of threadID to be used by worker routine.
workerID := wid
workerID := workerID
g.Go(func() error {
querySeriesRef := (seriesCnt / readConcurrency) * workerID
@ -451,7 +452,7 @@ func TestHead_HighConcurrencyReadAndWrite(t *testing.T) {
}
if len(samples) != 1 {
return false, fmt.Errorf("expected 1 series, got %d", len(samples))
return false, fmt.Errorf("expected 1 sample, got %d", len(samples))
}
series := lbls.String()
@ -806,7 +807,8 @@ func TestMemSeries_truncateChunks(t *testing.T) {
},
}
s := newMemSeries(labels.FromStrings("a", "b"), 1, defaultIsolationDisabled)
lbls := labels.FromStrings("a", "b")
s := newMemSeries(lbls, 1, labels.StableHash(lbls), 0, defaultIsolationDisabled)
for i := 0; i < 4000; i += 5 {
ok, _ := s.append(int64(i), float64(i), 0, chunkDiskMapper, chunkRange)
@ -1337,7 +1339,8 @@ func TestMemSeries_append(t *testing.T) {
}()
const chunkRange = 500
s := newMemSeries(labels.Labels{}, 1, defaultIsolationDisabled)
lbls := labels.Labels{}
s := newMemSeries(lbls, 1, labels.StableHash(lbls), 0, defaultIsolationDisabled)
// Add first two samples at the very end of a chunk range and the next two
// on and after it.
@ -1391,7 +1394,8 @@ func TestMemSeries_appendHistogram(t *testing.T) {
}()
chunkRange := int64(1000)
s := newMemSeries(labels.Labels{}, 1, defaultIsolationDisabled)
lbls := labels.Labels{}
s := newMemSeries(lbls, 1, labels.StableHash(lbls), 0, defaultIsolationDisabled)
histograms := tsdbutil.GenerateTestHistograms(4)
histogramWithOneMoreBucket := histograms[3].Copy()
@ -1447,7 +1451,8 @@ func TestMemSeries_append_atVariableRate(t *testing.T) {
})
chunkRange := DefaultBlockDuration
s := newMemSeries(labels.Labels{}, 1, defaultIsolationDisabled)
lbls := labels.Labels{}
s := newMemSeries(lbls, 1, labels.StableHash(lbls), 0, defaultIsolationDisabled)
// At this slow rate, we will fill the chunk in two block durations.
slowRate := (DefaultBlockDuration * 2) / samplesPerChunk
@ -1800,7 +1805,7 @@ func TestHeadReadWriterRepair(t *testing.T) {
ok, chunkCreated = s.append(int64(i*chunkRange)+chunkRange-1, float64(i*chunkRange), 0, h.chunkDiskMapper, chunkRange)
require.True(t, ok, "series append failed")
require.False(t, chunkCreated, "chunk was created")
h.chunkDiskMapper.CutNewFile()
require.NoError(t, h.chunkDiskMapper.CutNewFile())
}
require.NoError(t, h.Close())
@ -2477,6 +2482,67 @@ func TestHeadLabelNamesWithMatchers(t *testing.T) {
}
}
func TestHeadShardedPostings(t *testing.T) {
head, _ := newTestHead(t, 1000, false, false)
defer func() {
require.NoError(t, head.Close())
}()
// Append some series.
app := head.Appender(context.Background())
for i := 0; i < 100; i++ {
_, err := app.Append(0, labels.FromStrings("unique", fmt.Sprintf("value%d", i), "const", "1"), 100, 0)
require.NoError(t, err)
}
require.NoError(t, app.Commit())
ir := head.indexRange(0, 200)
// List all postings for a given label value. This is what we expect to get
// in output from all shards.
p, err := ir.Postings("const", "1")
require.NoError(t, err)
var expected []storage.SeriesRef
for p.Next() {
expected = append(expected, p.At())
}
require.NoError(t, p.Err())
require.Greater(t, len(expected), 0)
// Query the same postings for each shard.
const shardCount = uint64(4)
actualShards := make(map[uint64][]storage.SeriesRef)
actualPostings := make([]storage.SeriesRef, 0, len(expected))
for shardIndex := uint64(0); shardIndex < shardCount; shardIndex++ {
p, err = ir.Postings("const", "1")
require.NoError(t, err)
p = ir.ShardedPostings(p, shardIndex, shardCount)
for p.Next() {
ref := p.At()
actualShards[shardIndex] = append(actualShards[shardIndex], ref)
actualPostings = append(actualPostings, ref)
}
require.NoError(t, p.Err())
}
// We expect the postings merged out of shards is the exact same of the non sharded ones.
require.ElementsMatch(t, expected, actualPostings)
// We expect the series in each shard are the expected ones.
for shardIndex, ids := range actualShards {
for _, id := range ids {
var lbls labels.ScratchBuilder
require.NoError(t, ir.Series(id, &lbls, nil))
require.Equal(t, shardIndex, labels.StableHash(lbls.Labels())%shardCount)
}
}
}
func TestErrReuseAppender(t *testing.T) {
head, _ := newTestHead(t, 1000, false, false)
defer func() {
@ -2609,7 +2675,8 @@ func TestIteratorSeekIntoBuffer(t *testing.T) {
}()
const chunkRange = 500
s := newMemSeries(labels.Labels{}, 1, defaultIsolationDisabled)
lbls := labels.Labels{}
s := newMemSeries(lbls, 1, labels.StableHash(lbls), 0, defaultIsolationDisabled)
for i := 0; i < 7; i++ {
ok, _ := s.append(int64(i), float64(i), 0, chunkDiskMapper, chunkRange)

View file

@ -501,6 +501,8 @@ func (h *Head) resetSeriesWithMMappedChunks(mSeries *memSeries, mmc, oooMmc []*m
}
// Any samples replayed till now would already be compacted. Resetting the head chunk.
// We do not reset oooHeadChunk because that is being replayed from a different WAL
// and has not been replayed here.
mSeries.nextAt = 0
mSeries.headChunk = nil
mSeries.app = nil

View file

@ -37,6 +37,7 @@ import (
"github.com/prometheus/prometheus/tsdb/encoding"
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
"github.com/prometheus/prometheus/tsdb/fileutil"
"github.com/prometheus/prometheus/tsdb/hashcache"
)
const (
@ -1062,6 +1063,10 @@ type StringIter interface {
Err() error
}
type ReaderCacheProvider interface {
SeriesHashCache() *hashcache.BlockSeriesHashCache
}
type Reader struct {
b ByteSlice
toc *TOC
@ -1082,6 +1087,9 @@ type Reader struct {
dec *Decoder
version int
// Provides a cache mapping series labels hash by series ID.
cacheProvider ReaderCacheProvider
}
type postingOffset struct {
@ -1112,16 +1120,26 @@ func (b realByteSlice) Sub(start, end int) ByteSlice {
// NewReader returns a new index reader on the given byte slice. It automatically
// handles different format versions.
func NewReader(b ByteSlice) (*Reader, error) {
return newReader(b, io.NopCloser(nil))
return newReader(b, io.NopCloser(nil), nil)
}
// NewReaderWithCache is like NewReader but allows to pass a cache provider.
func NewReaderWithCache(b ByteSlice, cacheProvider ReaderCacheProvider) (*Reader, error) {
return newReader(b, io.NopCloser(nil), cacheProvider)
}
// NewFileReader returns a new index reader against the given index file.
func NewFileReader(path string) (*Reader, error) {
return NewFileReaderWithOptions(path, nil)
}
// NewFileReaderWithOptions is like NewFileReader but allows to pass a cache provider and sharding function.
func NewFileReaderWithOptions(path string, cacheProvider ReaderCacheProvider) (*Reader, error) {
f, err := fileutil.OpenMmapFile(path)
if err != nil {
return nil, err
}
r, err := newReader(realByteSlice(f.Bytes()), f)
r, err := newReader(realByteSlice(f.Bytes()), f, cacheProvider)
if err != nil {
return nil, tsdb_errors.NewMulti(
err,
@ -1132,11 +1150,12 @@ func NewFileReader(path string) (*Reader, error) {
return r, nil
}
func newReader(b ByteSlice, c io.Closer) (*Reader, error) {
func newReader(b ByteSlice, c io.Closer, cacheProvider ReaderCacheProvider) (*Reader, error) {
r := &Reader{
b: b,
c: c,
postings: map[string][]postingOffset{},
b: b,
c: c,
postings: map[string][]postingOffset{},
cacheProvider: cacheProvider,
}
// Verify header.
@ -1718,6 +1737,57 @@ func (r *Reader) SortedPostings(p Postings) Postings {
return p
}
// ShardedPostings returns a postings list filtered by the provided shardIndex out of shardCount.
func (r *Reader) ShardedPostings(p Postings, shardIndex, shardCount uint64) Postings {
var (
out = make([]storage.SeriesRef, 0, 128)
bufLbls = labels.ScratchBuilder{}
)
// Request the cache each time because the cache implementation requires
// that the cache reference is retained for a short period.
var seriesHashCache *hashcache.BlockSeriesHashCache
if r.cacheProvider != nil {
seriesHashCache = r.cacheProvider.SeriesHashCache()
}
for p.Next() {
id := p.At()
var (
hash uint64
ok bool
)
// Check if the hash is cached.
if seriesHashCache != nil {
hash, ok = seriesHashCache.Fetch(id)
}
if !ok {
// Get the series labels (no chunks).
err := r.Series(id, &bufLbls, nil)
if err != nil {
return ErrPostings(errors.Errorf("series %d not found", id))
}
hash = labels.StableHash(bufLbls.Labels())
if seriesHashCache != nil {
seriesHashCache.Store(id, hash)
}
}
// Check if the series belong to the shard.
if hash%shardCount != shardIndex {
continue
}
out = append(out, id)
}
return NewListPostings(out)
}
// Size returns the size of an index file.
func (r *Reader) Size() int64 {
return int64(r.b.Len())
@ -1840,7 +1910,9 @@ func (dec *Decoder) LabelValueFor(b []byte, label string) (string, error) {
// Previous contents of builder can be overwritten - make sure you copy before retaining.
func (dec *Decoder) Series(b []byte, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error {
builder.Reset()
*chks = (*chks)[:0]
if chks != nil {
*chks = (*chks)[:0]
}
d := encoding.Decbuf{B: b}
@ -1866,6 +1938,11 @@ func (dec *Decoder) Series(b []byte, builder *labels.ScratchBuilder, chks *[]chu
builder.Add(ln, lv)
}
// Skip reading chunks metadata if chks is nil.
if chks == nil {
return d.Err()
}
// Read the chunks meta data.
k = d.Uvarint()

View file

@ -32,6 +32,7 @@ import (
"github.com/prometheus/prometheus/tsdb/chunkenc"
"github.com/prometheus/prometheus/tsdb/chunks"
"github.com/prometheus/prometheus/tsdb/encoding"
"github.com/prometheus/prometheus/tsdb/hashcache"
"github.com/prometheus/prometheus/util/testutil"
)
@ -240,6 +241,63 @@ func TestIndexRW_Postings(t *testing.T) {
"b": {"1", "2", "3", "4"},
}, labelIndices)
// Test ShardedPostings() with and without series hash cache.
for _, cacheEnabled := range []bool{false, true} {
t.Run(fmt.Sprintf("ShardedPostings() cache enabled: %v", cacheEnabled), func(t *testing.T) {
var cache ReaderCacheProvider
if cacheEnabled {
cache = hashcache.NewSeriesHashCache(1024 * 1024 * 1024).GetBlockCacheProvider("test")
}
ir, err := NewFileReaderWithOptions(fn, cache)
require.NoError(t, err)
// List all postings for a given label value. This is what we expect to get
// in output from all shards.
p, err = ir.Postings("a", "1")
require.NoError(t, err)
var expected []storage.SeriesRef
for p.Next() {
expected = append(expected, p.At())
}
require.NoError(t, p.Err())
require.Greater(t, len(expected), 0)
// Query the same postings for each shard.
const shardCount = uint64(4)
actualShards := make(map[uint64][]storage.SeriesRef)
actualPostings := make([]storage.SeriesRef, 0, len(expected))
for shardIndex := uint64(0); shardIndex < shardCount; shardIndex++ {
p, err = ir.Postings("a", "1")
require.NoError(t, err)
p = ir.ShardedPostings(p, shardIndex, shardCount)
for p.Next() {
ref := p.At()
actualShards[shardIndex] = append(actualShards[shardIndex], ref)
actualPostings = append(actualPostings, ref)
}
require.NoError(t, p.Err())
}
// We expect the postings merged out of shards is the exact same of the non sharded ones.
require.ElementsMatch(t, expected, actualPostings)
// We expect the series in each shard are the expected ones.
for shardIndex, ids := range actualShards {
for _, id := range ids {
var lbls labels.ScratchBuilder
require.NoError(t, ir.Series(id, &lbls, nil))
require.Equal(t, shardIndex, labels.StableHash(lbls.Labels())%shardCount)
}
}
})
}
require.NoError(t, ir.Close())
}
@ -547,6 +605,60 @@ func TestSymbols(t *testing.T) {
require.NoError(t, iter.Err())
}
func BenchmarkReader_ShardedPostings(b *testing.B) {
const (
numSeries = 10000
numShards = 16
)
dir, err := os.MkdirTemp("", "benchmark_reader_sharded_postings")
require.NoError(b, err)
defer func() {
require.NoError(b, os.RemoveAll(dir))
}()
// Generate an index.
fn := filepath.Join(dir, indexFilename)
iw, err := NewWriter(context.Background(), fn)
require.NoError(b, err)
for i := 1; i <= numSeries; i++ {
require.NoError(b, iw.AddSymbol(fmt.Sprintf("%10d", i)))
}
require.NoError(b, iw.AddSymbol("const"))
require.NoError(b, iw.AddSymbol("unique"))
for i := 1; i <= numSeries; i++ {
require.NoError(b, iw.AddSeries(storage.SeriesRef(i),
labels.FromStrings("const", fmt.Sprintf("%10d", 1), "unique", fmt.Sprintf("%10d", i))))
}
require.NoError(b, iw.Close())
for _, cacheEnabled := range []bool{true, false} {
b.Run(fmt.Sprintf("cached enabled: %v", cacheEnabled), func(b *testing.B) {
var cache ReaderCacheProvider
if cacheEnabled {
cache = hashcache.NewSeriesHashCache(1024 * 1024 * 1024).GetBlockCacheProvider("test")
}
// Create a reader to read back all postings from the index.
ir, err := NewFileReaderWithOptions(fn, cache)
require.NoError(b, err)
b.ResetTimer()
for n := 0; n < b.N; n++ {
allPostings, err := ir.Postings("const", fmt.Sprintf("%10d", 1))
require.NoError(b, err)
ir.ShardedPostings(allPostings, uint64(n%numShards), numShards)
}
})
}
}
func TestDecoder_Postings_WrongInput(t *testing.T) {
_, _, err := (&Decoder{}).Postings([]byte("the cake is a lie"))
require.Error(t, err)

View file

@ -836,6 +836,28 @@ func (it *bigEndianPostings) Err() error {
return nil
}
// PostingsCloner takes an existing Postings and allows independently clone them.
type PostingsCloner struct {
ids []storage.SeriesRef
err error
}
// NewPostingsCloner takes an existing Postings and allows independently clone them.
// The instance provided shouldn't have been used before (no Next() calls should have been done)
// and it shouldn't be used once provided to the PostingsCloner.
func NewPostingsCloner(p Postings) *PostingsCloner {
ids, err := ExpandPostings(p)
return &PostingsCloner{ids: ids, err: err}
}
// Clone returns another independent Postings instance.
func (c *PostingsCloner) Clone() Postings {
if c.err != nil {
return ErrPostings(c.err)
}
return newListPostings(c.ids...)
}
// FindIntersectingPostings checks the intersection of p and candidates[i] for each i in candidates,
// if intersection is non empty, then i is added to the indexes returned.
// Returned indexes are not sorted.

View file

@ -946,6 +946,129 @@ func TestMemPostings_Delete(t *testing.T) {
require.Equal(t, 0, len(expanded), "expected empty postings, got %v", expanded)
}
func TestPostingsCloner(t *testing.T) {
for _, tc := range []struct {
name string
check func(testing.TB, *PostingsCloner)
}{
{
name: "seek beyond highest value of postings, then other clone seeks higher",
check: func(t testing.TB, pc *PostingsCloner) {
p1 := pc.Clone()
require.False(t, p1.Seek(9))
require.Equal(t, storage.SeriesRef(0), p1.At())
p2 := pc.Clone()
require.False(t, p2.Seek(10))
require.Equal(t, storage.SeriesRef(0), p2.At())
},
},
{
name: "seek beyond highest value of postings, then other clone seeks lower",
check: func(t testing.TB, pc *PostingsCloner) {
p1 := pc.Clone()
require.False(t, p1.Seek(9))
require.Equal(t, storage.SeriesRef(0), p1.At())
p2 := pc.Clone()
require.True(t, p2.Seek(2))
require.Equal(t, storage.SeriesRef(2), p2.At())
},
},
{
name: "seek to posting with value 3 or higher",
check: func(t testing.TB, pc *PostingsCloner) {
p := pc.Clone()
require.True(t, p.Seek(3))
require.Equal(t, storage.SeriesRef(4), p.At())
require.True(t, p.Seek(4))
require.Equal(t, storage.SeriesRef(4), p.At())
},
},
{
name: "seek alternatively on different postings",
check: func(t testing.TB, pc *PostingsCloner) {
p1 := pc.Clone()
require.True(t, p1.Seek(1))
require.Equal(t, storage.SeriesRef(1), p1.At())
p2 := pc.Clone()
require.True(t, p2.Seek(2))
require.Equal(t, storage.SeriesRef(2), p2.At())
p3 := pc.Clone()
require.True(t, p3.Seek(4))
require.Equal(t, storage.SeriesRef(4), p3.At())
p4 := pc.Clone()
require.True(t, p4.Seek(5))
require.Equal(t, storage.SeriesRef(8), p4.At())
require.True(t, p1.Seek(3))
require.Equal(t, storage.SeriesRef(4), p1.At())
require.True(t, p1.Seek(4))
require.Equal(t, storage.SeriesRef(4), p1.At())
},
},
{
name: "iterate through the postings",
check: func(t testing.TB, pc *PostingsCloner) {
p1 := pc.Clone()
p2 := pc.Clone()
// both one step
require.True(t, p1.Next())
require.Equal(t, storage.SeriesRef(1), p1.At())
require.True(t, p2.Next())
require.Equal(t, storage.SeriesRef(1), p2.At())
require.True(t, p1.Next())
require.Equal(t, storage.SeriesRef(2), p1.At())
require.True(t, p1.Next())
require.Equal(t, storage.SeriesRef(4), p1.At())
require.True(t, p1.Next())
require.Equal(t, storage.SeriesRef(8), p1.At())
require.False(t, p1.Next())
require.True(t, p2.Next())
require.Equal(t, storage.SeriesRef(2), p2.At())
require.True(t, p2.Next())
require.Equal(t, storage.SeriesRef(4), p2.At())
},
},
{
name: "at before call of next shouldn't panic",
check: func(t testing.TB, pc *PostingsCloner) {
p := pc.Clone()
require.Equal(t, storage.SeriesRef(0), p.At())
},
},
{
name: "ensure a failed seek doesn't allow more next calls",
check: func(t testing.TB, pc *PostingsCloner) {
p := pc.Clone()
require.False(t, p.Seek(9))
require.Equal(t, storage.SeriesRef(0), p.At())
require.False(t, p.Next())
require.Equal(t, storage.SeriesRef(0), p.At())
},
},
} {
t.Run(tc.name, func(t *testing.T) {
pc := NewPostingsCloner(newListPostings(1, 2, 4, 8))
tc.check(t, pc)
})
}
t.Run("cloning an err postings", func(t *testing.T) {
expectedErr := fmt.Errorf("foobar")
pc := NewPostingsCloner(ErrPostings(expectedErr))
p := pc.Clone()
require.False(t, p.Next())
require.Equal(t, expectedErr, p.Err())
})
}
func TestFindIntersectingPostings(t *testing.T) {
t.Run("multiple intersections", func(t *testing.T) {
p := NewListPostings([]storage.SeriesRef{10, 15, 20, 25, 30, 35, 40, 45, 50})

View file

@ -151,6 +151,12 @@ func (oh *OOOHeadIndexReader) series(ref storage.SeriesRef, builder *labels.Scra
return nil
}
// PostingsForMatchers needs to be overridden so that the right IndexReader
// implementation gets passed down to the PostingsForMatchers call.
func (oh *OOOHeadIndexReader) PostingsForMatchers(concurrent bool, ms ...*labels.Matcher) (index.Postings, error) {
return oh.head.pfmc.PostingsForMatchers(oh, concurrent, ms...)
}
// LabelValues needs to be overridden from the headIndexReader implementation due
// to the check that happens at the beginning where we make sure that the query
// interval overlaps with the head minooot and maxooot.
@ -416,6 +422,10 @@ func (ir *OOOCompactionHeadIndexReader) SortedPostings(p index.Postings) index.P
return p
}
func (ir *OOOCompactionHeadIndexReader) ShardedPostings(p index.Postings, shardIndex, shardCount uint64) index.Postings {
return ir.ch.oooIR.ShardedPostings(p, shardIndex, shardCount)
}
func (ir *OOOCompactionHeadIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error {
return ir.ch.oooIR.series(ref, builder, chks, ir.ch.lastMmapRef)
}

View file

@ -0,0 +1,205 @@
package tsdb
import (
"container/list"
"strings"
"sync"
"time"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/tsdb/index"
)
const (
defaultPostingsForMatchersCacheTTL = 10 * time.Second
defaultPostingsForMatchersCacheSize = 100
)
// IndexPostingsReader is a subset of IndexReader methods, the minimum required to evaluate PostingsForMatchers
type IndexPostingsReader interface {
// LabelValues returns possible label values which may not be sorted.
LabelValues(name string, matchers ...*labels.Matcher) ([]string, error)
// Postings returns the postings list iterator for the label pairs.
// The Postings here contain the offsets to the series inside the index.
// Found IDs are not strictly required to point to a valid Series, e.g.
// during background garbage collections. Input values must be sorted.
Postings(name string, values ...string) (index.Postings, error)
}
// NewPostingsForMatchersCache creates a new PostingsForMatchersCache.
// If `ttl` is 0, then it only deduplicates in-flight requests.
// If `force` is true, then all requests go through cache, regardless of the `concurrent` param provided.
func NewPostingsForMatchersCache(ttl time.Duration, cacheSize int, force bool) *PostingsForMatchersCache {
b := &PostingsForMatchersCache{
calls: &sync.Map{},
cached: list.New(),
ttl: ttl,
cacheSize: cacheSize,
force: force,
timeNow: time.Now,
postingsForMatchers: PostingsForMatchers,
}
return b
}
// PostingsForMatchersCache caches PostingsForMatchers call results when the concurrent hint is passed in or force is true.
type PostingsForMatchersCache struct {
calls *sync.Map
cachedMtx sync.RWMutex
cached *list.List
ttl time.Duration
cacheSize int
force bool
// timeNow is the time.Now that can be replaced for testing purposes
timeNow func() time.Time
// postingsForMatchers can be replaced for testing purposes
postingsForMatchers func(ix IndexPostingsReader, ms ...*labels.Matcher) (index.Postings, error)
}
func (c *PostingsForMatchersCache) PostingsForMatchers(ix IndexPostingsReader, concurrent bool, ms ...*labels.Matcher) (index.Postings, error) {
if !concurrent && !c.force {
return c.postingsForMatchers(ix, ms...)
}
c.expire()
return c.postingsForMatchersPromise(ix, ms)()
}
func (c *PostingsForMatchersCache) postingsForMatchersPromise(ix IndexPostingsReader, ms []*labels.Matcher) func() (index.Postings, error) {
var (
wg sync.WaitGroup
cloner *index.PostingsCloner
outerErr error
)
wg.Add(1)
promise := func() (index.Postings, error) {
wg.Wait()
if outerErr != nil {
return nil, outerErr
}
return cloner.Clone(), nil
}
key := matchersKey(ms)
oldPromise, loaded := c.calls.LoadOrStore(key, promise)
if loaded {
return oldPromise.(func() (index.Postings, error))
}
defer wg.Done()
if postings, err := c.postingsForMatchers(ix, ms...); err != nil {
outerErr = err
} else {
cloner = index.NewPostingsCloner(postings)
}
c.created(key, c.timeNow())
return promise
}
type postingsForMatchersCachedCall struct {
key string
ts time.Time
}
func (c *PostingsForMatchersCache) expire() {
if c.ttl <= 0 {
return
}
c.cachedMtx.RLock()
if !c.shouldEvictHead() {
c.cachedMtx.RUnlock()
return
}
c.cachedMtx.RUnlock()
c.cachedMtx.Lock()
defer c.cachedMtx.Unlock()
for c.shouldEvictHead() {
c.evictHead()
}
}
// shouldEvictHead returns true if cache head should be evicted, either because it's too old,
// or because the cache has too many elements
// should be called while read lock is held on cachedMtx
func (c *PostingsForMatchersCache) shouldEvictHead() bool {
if c.cached.Len() > c.cacheSize {
return true
}
h := c.cached.Front()
if h == nil {
return false
}
ts := h.Value.(*postingsForMatchersCachedCall).ts
return c.timeNow().Sub(ts) >= c.ttl
}
func (c *PostingsForMatchersCache) evictHead() {
front := c.cached.Front()
oldest := front.Value.(*postingsForMatchersCachedCall)
c.calls.Delete(oldest.key)
c.cached.Remove(front)
}
// created has to be called when returning from the PostingsForMatchers call that creates the promise.
// the ts provided should be the call time.
func (c *PostingsForMatchersCache) created(key string, ts time.Time) {
if c.ttl <= 0 {
c.calls.Delete(key)
return
}
c.cachedMtx.Lock()
defer c.cachedMtx.Unlock()
c.cached.PushBack(&postingsForMatchersCachedCall{
key: key,
ts: ts,
})
}
// matchersKey provides a unique string key for the given matchers slice
// NOTE: different orders of matchers will produce different keys,
// but it's unlikely that we'll receive same matchers in different orders at the same time
func matchersKey(ms []*labels.Matcher) string {
const (
typeLen = 2
sepLen = 1
)
var size int
for _, m := range ms {
size += len(m.Name) + len(m.Value) + typeLen + sepLen
}
sb := strings.Builder{}
sb.Grow(size)
for _, m := range ms {
sb.WriteString(m.Name)
sb.WriteString(m.Type.String())
sb.WriteString(m.Value)
sb.WriteByte(0)
}
key := sb.String()
return key
}
// indexReaderWithPostingsForMatchers adapts an index.Reader to be an IndexReader by adding the PostingsForMatchers method
type indexReaderWithPostingsForMatchers struct {
*index.Reader
pfmc *PostingsForMatchersCache
}
func (ir indexReaderWithPostingsForMatchers) PostingsForMatchers(concurrent bool, ms ...*labels.Matcher) (index.Postings, error) {
return ir.pfmc.PostingsForMatchers(ir, concurrent, ms...)
}
var _ IndexReader = indexReaderWithPostingsForMatchers{}

View file

@ -0,0 +1,313 @@
package tsdb
import (
"fmt"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/tsdb/index"
)
func TestPostingsForMatchersCache(t *testing.T) {
const testCacheSize = 5
// newPostingsForMatchersCache tests the NewPostingsForMatcherCache constructor, but overrides the postingsForMatchers func
newPostingsForMatchersCache := func(ttl time.Duration, pfm func(ix IndexPostingsReader, ms ...*labels.Matcher) (index.Postings, error), timeMock *timeNowMock, force bool) *PostingsForMatchersCache {
c := NewPostingsForMatchersCache(ttl, testCacheSize, force)
if c.postingsForMatchers == nil {
t.Fatalf("NewPostingsForMatchersCache() didn't assign postingsForMatchers func")
}
c.postingsForMatchers = pfm
c.timeNow = timeMock.timeNow
return c
}
t.Run("happy case one call", func(t *testing.T) {
for _, concurrent := range []bool{true, false} {
t.Run(fmt.Sprintf("concurrent=%t", concurrent), func(t *testing.T) {
expectedMatchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}
expectedPostingsErr := fmt.Errorf("failed successfully")
c := newPostingsForMatchersCache(defaultPostingsForMatchersCacheTTL, func(ix IndexPostingsReader, ms ...*labels.Matcher) (index.Postings, error) {
require.IsType(t, indexForPostingsMock{}, ix, "Incorrect IndexPostingsReader was provided to PostingsForMatchers, expected the mock, was given %v (%T)", ix, ix)
require.Equal(t, expectedMatchers, ms, "Wrong label matchers provided, expected %v, got %v", expectedMatchers, ms)
return index.ErrPostings(expectedPostingsErr), nil
}, &timeNowMock{}, false)
p, err := c.PostingsForMatchers(indexForPostingsMock{}, concurrent, expectedMatchers...)
require.NoError(t, err)
require.NotNil(t, p)
require.Equal(t, p.Err(), expectedPostingsErr, "Expected ErrPostings with err %q, got %T with err %q", expectedPostingsErr, p, p.Err())
})
}
})
t.Run("err returned", func(t *testing.T) {
expectedMatchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}
expectedErr := fmt.Errorf("failed successfully")
c := newPostingsForMatchersCache(defaultPostingsForMatchersCacheTTL, func(ix IndexPostingsReader, ms ...*labels.Matcher) (index.Postings, error) {
return nil, expectedErr
}, &timeNowMock{}, false)
_, err := c.PostingsForMatchers(indexForPostingsMock{}, true, expectedMatchers...)
require.Equal(t, expectedErr, err)
})
t.Run("happy case multiple concurrent calls: two same one different", func(t *testing.T) {
for _, cacheEnabled := range []bool{true, false} {
t.Run(fmt.Sprintf("cacheEnabled=%t", cacheEnabled), func(t *testing.T) {
for _, forced := range []bool{true, false} {
concurrent := !forced
t.Run(fmt.Sprintf("forced=%t", forced), func(t *testing.T) {
calls := [][]*labels.Matcher{
{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}, // 1
{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}, // 1 same
{labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar")}, // 2: different match type
{labels.MustNewMatcher(labels.MatchEqual, "diff", "bar")}, // 3: different name
{labels.MustNewMatcher(labels.MatchEqual, "foo", "diff")}, // 4: different value
{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"), labels.MustNewMatcher(labels.MatchEqual, "boo", "bam")}, // 5
{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"), labels.MustNewMatcher(labels.MatchEqual, "boo", "bam")}, // 5 same
}
// we'll identify results by each call's error, and the error will be the string value of the first matcher
matchersString := func(ms []*labels.Matcher) string {
s := strings.Builder{}
for i, m := range ms {
if i > 0 {
s.WriteByte(',')
}
s.WriteString(m.String())
}
return s.String()
}
expectedResults := make([]string, len(calls))
for i, c := range calls {
expectedResults[i] = c[0].String()
}
expectedPostingsForMatchersCalls := 5
// we'll block all the calls until we receive the exact amount. if we receive more, WaitGroup will panic
called := make(chan struct{}, expectedPostingsForMatchersCalls)
release := make(chan struct{})
var ttl time.Duration
if cacheEnabled {
ttl = defaultPostingsForMatchersCacheTTL
}
c := newPostingsForMatchersCache(ttl, func(ix IndexPostingsReader, ms ...*labels.Matcher) (index.Postings, error) {
select {
case called <- struct{}{}:
default:
}
<-release
return nil, fmt.Errorf(matchersString(ms))
}, &timeNowMock{}, forced)
results := make([]string, len(calls))
resultsWg := sync.WaitGroup{}
resultsWg.Add(len(calls))
// perform all calls
for i := 0; i < len(calls); i++ {
go func(i int) {
_, err := c.PostingsForMatchers(indexForPostingsMock{}, concurrent, calls[i]...)
results[i] = err.Error()
resultsWg.Done()
}(i)
}
// wait until all calls arrive to the mocked function
for i := 0; i < expectedPostingsForMatchersCalls; i++ {
<-called
}
// let them all return
close(release)
// wait for the results
resultsWg.Wait()
// check that we got correct results
for i, c := range calls {
require.Equal(t, matchersString(c), results[i], "Call %d should have returned error %q, but got %q instead", i, matchersString(c), results[i])
}
})
}
})
}
})
t.Run("with concurrent==false, result is not cached", func(t *testing.T) {
expectedMatchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}
var call int
c := newPostingsForMatchersCache(defaultPostingsForMatchersCacheTTL, func(ix IndexPostingsReader, ms ...*labels.Matcher) (index.Postings, error) {
call++
return index.ErrPostings(fmt.Errorf("result from call %d", call)), nil
}, &timeNowMock{}, false)
// first call, fills the cache
p, err := c.PostingsForMatchers(indexForPostingsMock{}, false, expectedMatchers...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 1")
// second call within the ttl (we didn't advance the time), should call again because concurrent==false
p, err = c.PostingsForMatchers(indexForPostingsMock{}, false, expectedMatchers...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 2")
})
t.Run("with cache disabled, result is not cached", func(t *testing.T) {
expectedMatchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}
var call int
c := newPostingsForMatchersCache(0, func(ix IndexPostingsReader, ms ...*labels.Matcher) (index.Postings, error) {
call++
return index.ErrPostings(fmt.Errorf("result from call %d", call)), nil
}, &timeNowMock{}, false)
// first call, fills the cache
p, err := c.PostingsForMatchers(indexForPostingsMock{}, true, expectedMatchers...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 1")
// second call within the ttl (we didn't advance the time), should call again because concurrent==false
p, err = c.PostingsForMatchers(indexForPostingsMock{}, true, expectedMatchers...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 2")
})
t.Run("cached value is returned, then it expires", func(t *testing.T) {
timeNow := &timeNowMock{}
expectedMatchers := []*labels.Matcher{
labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"),
}
var call int
c := newPostingsForMatchersCache(defaultPostingsForMatchersCacheTTL, func(ix IndexPostingsReader, ms ...*labels.Matcher) (index.Postings, error) {
call++
return index.ErrPostings(fmt.Errorf("result from call %d", call)), nil
}, timeNow, false)
// first call, fills the cache
p, err := c.PostingsForMatchers(indexForPostingsMock{}, true, expectedMatchers...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 1")
timeNow.advance(defaultPostingsForMatchersCacheTTL / 2)
// second call within the ttl, should use the cache
p, err = c.PostingsForMatchers(indexForPostingsMock{}, true, expectedMatchers...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 1")
timeNow.advance(defaultPostingsForMatchersCacheTTL / 2)
// third call is after ttl (exactly), should call again
p, err = c.PostingsForMatchers(indexForPostingsMock{}, true, expectedMatchers...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 2")
})
t.Run("cached value is evicted because cache exceeds max size", func(t *testing.T) {
timeNow := &timeNowMock{}
calls := make([][]*labels.Matcher, testCacheSize)
for i := range calls {
calls[i] = []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "matchers", fmt.Sprintf("%d", i))}
}
callsPerMatchers := map[string]int{}
c := newPostingsForMatchersCache(defaultPostingsForMatchersCacheTTL, func(ix IndexPostingsReader, ms ...*labels.Matcher) (index.Postings, error) {
k := matchersKey(ms)
callsPerMatchers[k]++
return index.ErrPostings(fmt.Errorf("result from call %d", callsPerMatchers[k])), nil
}, timeNow, false)
// each one of the first testCacheSize calls is cached properly
for _, matchers := range calls {
// first call
p, err := c.PostingsForMatchers(indexForPostingsMock{}, true, matchers...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 1")
// cached value
p, err = c.PostingsForMatchers(indexForPostingsMock{}, true, matchers...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 1")
}
// one extra call is made, which is cached properly, but evicts the first cached value
someExtraMatchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}
// first call
p, err := c.PostingsForMatchers(indexForPostingsMock{}, true, someExtraMatchers...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 1")
// cached value
p, err = c.PostingsForMatchers(indexForPostingsMock{}, true, someExtraMatchers...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 1")
// make first call again, it's calculated again
p, err = c.PostingsForMatchers(indexForPostingsMock{}, true, calls[0]...)
require.NoError(t, err)
require.EqualError(t, p.Err(), "result from call 2")
})
}
type indexForPostingsMock struct{}
func (idx indexForPostingsMock) LabelValues(name string, matchers ...*labels.Matcher) ([]string, error) {
panic("implement me")
}
func (idx indexForPostingsMock) Postings(name string, values ...string) (index.Postings, error) {
panic("implement me")
}
// timeNowMock offers a mockable time.Now() implementation
// empty value is ready to be used, and it should not be copied (use a reference)
type timeNowMock struct {
sync.Mutex
now time.Time
}
// timeNow can be used as a mocked replacement for time.Now()
func (t *timeNowMock) timeNow() time.Time {
t.Lock()
defer t.Unlock()
if t.now.IsZero() {
t.now = time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC)
}
return t.now
}
// advance advances the mocked time.Now() value
func (t *timeNowMock) advance(d time.Duration) {
t.Lock()
defer t.Unlock()
if t.now.IsZero() {
t.now = time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC)
}
t.now = t.now.Add(d)
}
func BenchmarkMatchersKey(b *testing.B) {
const totalMatchers = 10
const matcherSets = 100
sets := make([][]*labels.Matcher, matcherSets)
for i := 0; i < matcherSets; i++ {
for j := 0; j < totalMatchers; j++ {
sets[i] = append(sets[i], labels.MustNewMatcher(labels.MatchType(j%4), fmt.Sprintf("%d_%d", i*13, j*65537), fmt.Sprintf("%x_%x", i*127, j*2_147_483_647)))
}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = matchersKey(sets[i%matcherSets])
}
}

View file

@ -16,8 +16,6 @@ package tsdb
import (
"fmt"
"math"
"strings"
"unicode/utf8"
"github.com/oklog/ulid"
"github.com/pkg/errors"
@ -32,20 +30,6 @@ import (
"github.com/prometheus/prometheus/tsdb/tombstones"
)
// Bitmap used by func isRegexMetaCharacter to check whether a character needs to be escaped.
var regexMetaCharacterBytes [16]byte
// isRegexMetaCharacter reports whether byte b needs to be escaped.
func isRegexMetaCharacter(b byte) bool {
return b < utf8.RuneSelf && regexMetaCharacterBytes[b%16]&(1<<(b/16)) != 0
}
func init() {
for _, b := range []byte(`.+*?()|[]{}^$`) {
regexMetaCharacterBytes[b%16] |= 1 << (b / 16)
}
}
type blockBaseQuerier struct {
blockID ulid.ULID
index IndexReader
@ -128,11 +112,14 @@ func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ..
mint := q.mint
maxt := q.maxt
disableTrimming := false
p, err := PostingsForMatchers(q.index, ms...)
sharded := hints != nil && hints.ShardCount > 0
p, err := q.index.PostingsForMatchers(sharded, ms...)
if err != nil {
return storage.ErrSeriesSet(err)
}
if sharded {
p = q.index.ShardedPostings(p, hints.ShardIndex, hints.ShardCount)
}
if sortSeries {
p = q.index.SortedPostings(p)
}
@ -173,61 +160,23 @@ func (q *blockChunkQuerier) Select(sortSeries bool, hints *storage.SelectHints,
maxt = hints.End
disableTrimming = hints.DisableTrimming
}
p, err := PostingsForMatchers(q.index, ms...)
sharded := hints != nil && hints.ShardCount > 0
p, err := q.index.PostingsForMatchers(sharded, ms...)
if err != nil {
return storage.ErrChunkSeriesSet(err)
}
if sharded {
p = q.index.ShardedPostings(p, hints.ShardIndex, hints.ShardCount)
}
if sortSeries {
p = q.index.SortedPostings(p)
}
return newBlockChunkSeriesSet(q.blockID, q.index, q.chunks, q.tombstones, p, mint, maxt, disableTrimming)
}
func findSetMatches(pattern string) []string {
// Return empty matches if the wrapper from Prometheus is missing.
if len(pattern) < 6 || pattern[:4] != "^(?:" || pattern[len(pattern)-2:] != ")$" {
return nil
}
escaped := false
sets := []*strings.Builder{{}}
for i := 4; i < len(pattern)-2; i++ {
if escaped {
switch {
case isRegexMetaCharacter(pattern[i]):
sets[len(sets)-1].WriteByte(pattern[i])
case pattern[i] == '\\':
sets[len(sets)-1].WriteByte('\\')
default:
return nil
}
escaped = false
} else {
switch {
case isRegexMetaCharacter(pattern[i]):
if pattern[i] == '|' {
sets = append(sets, &strings.Builder{})
} else {
return nil
}
case pattern[i] == '\\':
escaped = true
default:
sets[len(sets)-1].WriteByte(pattern[i])
}
}
}
matches := make([]string, 0, len(sets))
for _, s := range sets {
if s.Len() > 0 {
matches = append(matches, s.String())
}
}
return matches
}
// PostingsForMatchers assembles a single postings iterator against the index reader
// based on the given matchers. The resulting postings are not ordered by series.
func PostingsForMatchers(ix IndexReader, ms ...*labels.Matcher) (index.Postings, error) {
func PostingsForMatchers(ix IndexPostingsReader, ms ...*labels.Matcher) (index.Postings, error) {
var its, notIts []index.Postings
// See which label must be non-empty.
// Optimization for case like {l=~".", l!="1"}.
@ -322,7 +271,7 @@ func PostingsForMatchers(ix IndexReader, ms ...*labels.Matcher) (index.Postings,
return it, nil
}
func postingsForMatcher(ix IndexReader, m *labels.Matcher) (index.Postings, error) {
func postingsForMatcher(ix IndexPostingsReader, m *labels.Matcher) (index.Postings, error) {
// This method will not return postings for missing labels.
// Fast-path for equal matching.
@ -332,7 +281,7 @@ func postingsForMatcher(ix IndexReader, m *labels.Matcher) (index.Postings, erro
// Fast-path for set matching.
if m.Type == labels.MatchRegexp {
setMatches := findSetMatches(m.GetRegexString())
setMatches := m.SetMatches()
if len(setMatches) > 0 {
return ix.Postings(m.Name, setMatches...)
}
@ -358,7 +307,7 @@ func postingsForMatcher(ix IndexReader, m *labels.Matcher) (index.Postings, erro
}
// inversePostingsForMatcher returns the postings for the series with the label name set but not matching the matcher.
func inversePostingsForMatcher(ix IndexReader, m *labels.Matcher) (index.Postings, error) {
func inversePostingsForMatcher(ix IndexPostingsReader, m *labels.Matcher) (index.Postings, error) {
vals, err := ix.LabelValues(m.Name)
if err != nil {
return nil, err
@ -410,7 +359,7 @@ func labelValuesWithMatchers(r IndexReader, name string, matchers ...*labels.Mat
}
func labelNamesWithMatchers(r IndexReader, matchers ...*labels.Matcher) ([]string, error) {
p, err := PostingsForMatchers(r, matchers...)
p, err := r.PostingsForMatchers(false, matchers...)
if err != nil {
return nil, err
}

View file

@ -19,10 +19,12 @@ import (
"strconv"
"testing"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/tsdb/index"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb/hashcache"
"github.com/prometheus/prometheus/tsdb/index"
)
// Make entries ~50B in size, to emulate real-world high cardinality.
@ -240,16 +242,28 @@ func BenchmarkQuerierSelect(b *testing.B) {
}
require.NoError(b, app.Commit())
bench := func(b *testing.B, br BlockReader, sorted bool) {
bench := func(b *testing.B, br BlockReader, sorted, sharding bool) {
matcher := labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")
for s := 1; s <= numSeries; s *= 10 {
b.Run(fmt.Sprintf("%dof%d", s, numSeries), func(b *testing.B) {
q, err := NewBlockQuerier(br, 0, int64(s-1))
mint := int64(0)
maxt := int64(s - 1)
q, err := NewBlockQuerier(br, mint, maxt)
require.NoError(b, err)
b.ResetTimer()
for i := 0; i < b.N; i++ {
ss := q.Select(sorted, nil, matcher)
var hints *storage.SelectHints
if sharding {
hints = &storage.SelectHints{
Start: mint,
End: maxt,
ShardIndex: uint64(i % 16),
ShardCount: 16,
}
}
ss := q.Select(sorted, hints, matcher)
for ss.Next() {
}
require.NoError(b, ss.Err())
@ -260,22 +274,38 @@ func BenchmarkQuerierSelect(b *testing.B) {
}
b.Run("Head", func(b *testing.B) {
bench(b, h, false)
b.Run("without sharding", func(b *testing.B) {
bench(b, h, false, false)
})
b.Run("with sharding", func(b *testing.B) {
bench(b, h, false, true)
})
})
b.Run("SortedHead", func(b *testing.B) {
bench(b, h, true)
b.Run("without sharding", func(b *testing.B) {
bench(b, h, true, false)
})
b.Run("with sharding", func(b *testing.B) {
bench(b, h, true, true)
})
})
tmpdir := b.TempDir()
seriesHashCache := hashcache.NewSeriesHashCache(1024 * 1024 * 1024)
blockdir := createBlockFromHead(b, tmpdir, h)
block, err := OpenBlock(nil, blockdir, nil)
block, err := OpenBlockWithOptions(nil, blockdir, nil, seriesHashCache.GetBlockCacheProvider("test"), defaultPostingsForMatchersCacheTTL, defaultPostingsForMatchersCacheSize, false)
require.NoError(b, err)
defer func() {
require.NoError(b, block.Close())
}()
b.Run("Block", func(b *testing.B) {
bench(b, block, false)
b.Run("without sharding", func(b *testing.B) {
bench(b, block, false, false)
})
b.Run("with sharding", func(b *testing.B) {
bench(b, block, false, true)
})
})
}

View file

@ -381,6 +381,46 @@ func TestBlockQuerier(t *testing.T) {
),
}),
},
{
// This tests query sharding. The label sets involved both hash into this test's result set. The test
// following this is companion to this test (same test but with a different ShardIndex) and should find that
// the label sets involved do not hash to that test's result set.
mint: math.MinInt64,
maxt: math.MaxInt64,
hints: &storage.SelectHints{Start: math.MinInt64, End: math.MaxInt64, ShardIndex: 0, ShardCount: 2},
ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "a", ".*")},
exp: newMockSeriesSet([]storage.Series{
storage.NewListSeries(labels.FromStrings("a", "a"),
[]tsdbutil.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}, sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}},
),
storage.NewListSeries(labels.FromStrings("a", "a", "b", "b"),
[]tsdbutil.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}, sample{5, 3, nil, nil}, sample{6, 6, nil, nil}},
),
storage.NewListSeries(labels.FromStrings("b", "b"),
[]tsdbutil.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}, sample{5, 1, nil, nil}, sample{6, 7, nil, nil}, sample{7, 2, nil, nil}},
),
}),
expChks: newMockChunkSeriesSet([]storage.ChunkSeries{
storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a"),
[]tsdbutil.Sample{sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 4, nil, nil}}, []tsdbutil.Sample{sample{5, 2, nil, nil}, sample{6, 3, nil, nil}, sample{7, 4, nil, nil}},
),
storage.NewListChunkSeriesFromSamples(labels.FromStrings("a", "a", "b", "b"),
[]tsdbutil.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, []tsdbutil.Sample{sample{5, 3, nil, nil}, sample{6, 6, nil, nil}},
),
storage.NewListChunkSeriesFromSamples(labels.FromStrings("b", "b"),
[]tsdbutil.Sample{sample{1, 3, nil, nil}, sample{2, 2, nil, nil}, sample{3, 6, nil, nil}}, []tsdbutil.Sample{sample{5, 1, nil, nil}, sample{6, 7, nil, nil}, sample{7, 2, nil, nil}},
),
}),
},
{
// This is a companion to the test above.
mint: math.MinInt64,
maxt: math.MaxInt64,
hints: &storage.SelectHints{Start: math.MinInt64, End: math.MaxInt64, ShardIndex: 1, ShardCount: 2},
ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "a", ".*")},
exp: newMockSeriesSet([]storage.Series{}),
expChks: newMockChunkSeriesSet([]storage.ChunkSeries{}),
},
} {
t.Run("", func(t *testing.T) {
ir, cr, _, _ := createIdxChkReaders(t, testData)
@ -1282,6 +1322,48 @@ func (m mockIndex) SortedPostings(p index.Postings) index.Postings {
return index.NewListPostings(ep)
}
func (m mockIndex) PostingsForMatchers(concurrent bool, ms ...*labels.Matcher) (index.Postings, error) {
var ps []storage.SeriesRef
for p, s := range m.series {
if matches(ms, s.l) {
ps = append(ps, p)
}
}
sort.Slice(ps, func(i, j int) bool { return ps[i] < ps[j] })
return index.NewListPostings(ps), nil
}
func matches(ms []*labels.Matcher, lbls labels.Labels) bool {
lm := lbls.Map()
for _, m := range ms {
if !m.Matches(lm[m.Name]) {
return false
}
}
return true
}
func (m mockIndex) ShardedPostings(p index.Postings, shardIndex, shardCount uint64) index.Postings {
out := make([]storage.SeriesRef, 0, 128)
for p.Next() {
ref := p.At()
s, ok := m.series[ref]
if !ok {
continue
}
// Check if the series belong to the shard.
if s.l.Hash()%shardCount != shardIndex {
continue
}
out = append(out, ref)
}
return index.NewListPostings(out)
}
func (m mockIndex) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error {
s, ok := m.series[ref]
if !ok {
@ -1593,69 +1675,6 @@ func BenchmarkSetMatcher(b *testing.B) {
}
}
// Refer to https://github.com/prometheus/prometheus/issues/2651.
func TestFindSetMatches(t *testing.T) {
cases := []struct {
pattern string
exp []string
}{
// Single value, coming from a `bar=~"foo"` selector.
{
pattern: "^(?:foo)$",
exp: []string{
"foo",
},
},
// Simple sets.
{
pattern: "^(?:foo|bar|baz)$",
exp: []string{
"foo",
"bar",
"baz",
},
},
// Simple sets containing escaped characters.
{
pattern: "^(?:fo\\.o|bar\\?|\\^baz)$",
exp: []string{
"fo.o",
"bar?",
"^baz",
},
},
// Simple sets containing special characters without escaping.
{
pattern: "^(?:fo.o|bar?|^baz)$",
exp: nil,
},
// Missing wrapper.
{
pattern: "foo|bar|baz",
exp: nil,
},
}
for _, c := range cases {
matches := findSetMatches(c.pattern)
if len(c.exp) == 0 {
if len(matches) != 0 {
t.Errorf("Evaluating %s, unexpected result %v", c.pattern, matches)
}
} else {
if len(matches) != len(c.exp) {
t.Errorf("Evaluating %s, length of result not equal to exp", c.pattern)
} else {
for i := 0; i < len(c.exp); i++ {
if c.exp[i] != matches[i] {
t.Errorf("Evaluating %s, unexpected result %s", c.pattern, matches[i])
}
}
}
}
}
}
func TestPostingsForMatchers(t *testing.T) {
chunkDir := t.TempDir()
opts := DefaultHeadOptions()
@ -2109,10 +2128,18 @@ func (m mockMatcherIndex) Postings(name string, values ...string) (index.Posting
return index.EmptyPostings(), nil
}
func (m mockMatcherIndex) PostingsForMatchers(bool, ...*labels.Matcher) (index.Postings, error) {
return index.EmptyPostings(), nil
}
func (m mockMatcherIndex) SortedPostings(p index.Postings) index.Postings {
return index.EmptyPostings()
}
func (m mockMatcherIndex) ShardedPostings(ps index.Postings, shardIndex, shardCount uint64) index.Postings {
return ps
}
func (m mockMatcherIndex) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error {
return nil
}
@ -2143,7 +2170,7 @@ func TestPostingsForMatcher(t *testing.T) {
{
// Test case for double quoted regex matcher
matcher: labels.MustNewMatcher(labels.MatchRegexp, "test", "^(?:a|b)$"),
hasError: true,
hasError: false,
},
}

380
tsdb/symbols_batch.go Normal file
View file

@ -0,0 +1,380 @@
package tsdb
import (
"container/heap"
"encoding/gob"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"sync"
"github.com/golang/snappy"
"github.com/prometheus/prometheus/tsdb/errors"
)
// symbolFlushers writes symbols to provided files in background goroutines.
type symbolFlushers struct {
jobs chan flusherJob
wg sync.WaitGroup
closed bool
errMu sync.Mutex
err error
pool *sync.Pool
}
func newSymbolFlushers(concurrency int) *symbolFlushers {
f := &symbolFlushers{
jobs: make(chan flusherJob),
pool: &sync.Pool{},
}
for i := 0; i < concurrency; i++ {
f.wg.Add(1)
go f.loop()
}
return f
}
func (f *symbolFlushers) flushSymbols(outputFile string, symbols map[string]struct{}) error {
if len(symbols) == 0 {
return fmt.Errorf("no symbols")
}
f.errMu.Lock()
err := f.err
f.errMu.Unlock()
// If there was any error previously, return it.
if err != nil {
return err
}
f.jobs <- flusherJob{
outputFile: outputFile,
symbols: symbols,
}
return nil
}
func (f *symbolFlushers) loop() {
defer f.wg.Done()
for j := range f.jobs {
var sortedSymbols []string
pooled := f.pool.Get()
if pooled == nil {
sortedSymbols = make([]string, 0, len(j.symbols))
} else {
sortedSymbols = pooled.([]string)
sortedSymbols = sortedSymbols[:0]
}
for s := range j.symbols {
sortedSymbols = append(sortedSymbols, s)
}
sort.Strings(sortedSymbols)
err := writeSymbolsToFile(j.outputFile, sortedSymbols)
sortedSymbols = sortedSymbols[:0]
//nolint:staticcheck // Ignore SA6002 safe to ignore and actually fixing it has some performance penalty.
f.pool.Put(sortedSymbols)
if err != nil {
f.errMu.Lock()
if f.err == nil {
f.err = err
}
f.errMu.Unlock()
break
}
}
for range f.jobs {
// drain the channel, don't do more flushing. only used when error occurs.
}
}
// Stops and waits until all flusher goroutines are finished.
func (f *symbolFlushers) close() error {
if f.closed {
return f.err
}
f.closed = true
close(f.jobs)
f.wg.Wait()
return f.err
}
type flusherJob struct {
outputFile string
symbols map[string]struct{}
}
// symbolsBatcher keeps buffer of symbols in memory. Once the buffer reaches the size limit (number of symbols),
// batcher writes currently buffered symbols to file. At the end remaining symbols must be flushed. After writing
// all batches, symbolsBatcher has list of files that can be used together with newSymbolsIterator to iterate
// through all previously added symbols in sorted order.
type symbolsBatcher struct {
dir string
limit int
symbolsFiles []string // paths of symbol files, which were sent to flushers for flushing
buffer map[string]struct{} // using map to deduplicate
flushers *symbolFlushers
}
func newSymbolsBatcher(limit int, dir string, flushers *symbolFlushers) *symbolsBatcher {
return &symbolsBatcher{
limit: limit,
dir: dir,
buffer: make(map[string]struct{}, limit),
flushers: flushers,
}
}
func (sw *symbolsBatcher) addSymbol(sym string) error {
sw.buffer[sym] = struct{}{}
return sw.flushSymbols(false)
}
func (sw *symbolsBatcher) flushSymbols(force bool) error {
if !force && len(sw.buffer) < sw.limit {
return nil
}
if len(sw.buffer) == 0 {
return nil
}
symbolsFile := filepath.Join(sw.dir, fmt.Sprintf("symbols_%d", len(sw.symbolsFiles)))
sw.symbolsFiles = append(sw.symbolsFiles, symbolsFile)
buf := sw.buffer
sw.buffer = make(map[string]struct{}, sw.limit)
return sw.flushers.flushSymbols(symbolsFile, buf)
}
// getSymbolFiles returns list of symbol files used to flush symbols to. These files are only valid if flushers
// finish successfully.
func (sw *symbolsBatcher) getSymbolFiles() []string {
return sw.symbolsFiles
}
func writeSymbolsToFile(filename string, symbols []string) error {
f, err := os.Create(filename)
if err != nil {
return err
}
// Snappy is used for buffering and to create smaller files.
sn := snappy.NewBufferedWriter(f)
enc := gob.NewEncoder(sn)
errs := errors.NewMulti()
for _, s := range symbols {
err := enc.Encode(s)
if err != nil {
errs.Add(err)
break
}
}
errs.Add(sn.Close())
errs.Add(f.Close())
return errs.Err()
}
// Implements heap.Interface using symbols from files.
type symbolsHeap []*symbolsFile
// Len implements sort.Interface.
func (s *symbolsHeap) Len() int {
return len(*s)
}
// Less implements sort.Interface.
func (s *symbolsHeap) Less(i, j int) bool {
iw, ierr := (*s)[i].Peek()
if ierr != nil {
// Empty string will be sorted first, so error will be returned before any other result.
iw = ""
}
jw, jerr := (*s)[j].Peek()
if jerr != nil {
jw = ""
}
return iw < jw
}
// Swap implements sort.Interface.
func (s *symbolsHeap) Swap(i, j int) {
(*s)[i], (*s)[j] = (*s)[j], (*s)[i]
}
// Push implements heap.Interface. Push should add x as element Len().
func (s *symbolsHeap) Push(x interface{}) {
*s = append(*s, x.(*symbolsFile))
}
// Pop implements heap.Interface. Pop should remove and return element Len() - 1.
func (s *symbolsHeap) Pop() interface{} {
l := len(*s)
res := (*s)[l-1]
*s = (*s)[:l-1]
return res
}
type symbolsIterator struct {
files []*os.File
heap symbolsHeap
// To avoid returning duplicates, we remember last returned symbol. We want to support "" as a valid
// symbol, so we use pointer to a string instead.
lastReturned *string
}
func newSymbolsIterator(filenames []string) (*symbolsIterator, error) {
files, err := openFiles(filenames)
if err != nil {
return nil, err
}
var symFiles []*symbolsFile
for _, f := range files {
symFiles = append(symFiles, newSymbolsFile(f))
}
h := &symbolsIterator{
files: files,
heap: symFiles,
}
heap.Init(&h.heap)
return h, nil
}
// NextSymbol advances iterator forward, and returns next symbol.
// If there is no next element, returns err == io.EOF.
func (sit *symbolsIterator) NextSymbol() (string, error) {
for len(sit.heap) > 0 {
result, err := sit.heap[0].Next()
if err == io.EOF {
// End of file, remove it from heap, and try next file.
heap.Remove(&sit.heap, 0)
continue
}
if err != nil {
return "", err
}
heap.Fix(&sit.heap, 0)
if sit.lastReturned != nil && *sit.lastReturned == result {
// Duplicate symbol, try next one.
continue
}
sit.lastReturned = &result
return result, nil
}
return "", io.EOF
}
// Close all files.
func (sit *symbolsIterator) Close() error {
errs := errors.NewMulti()
for _, f := range sit.files {
errs.Add(f.Close())
}
return errs.Err()
}
type symbolsFile struct {
dec *gob.Decoder
nextValid bool // if true, nextSymbol and nextErr have the next symbol (possibly "")
nextSymbol string
nextErr error
}
func newSymbolsFile(f *os.File) *symbolsFile {
sn := snappy.NewReader(f)
dec := gob.NewDecoder(sn)
return &symbolsFile{
dec: dec,
}
}
// Peek returns next symbol or error, but also preserves them for subsequent Peek or Next calls.
func (sf *symbolsFile) Peek() (string, error) {
if sf.nextValid {
return sf.nextSymbol, sf.nextErr
}
sf.nextValid = true
sf.nextSymbol, sf.nextErr = sf.readNext()
return sf.nextSymbol, sf.nextErr
}
// Next advances iterator and returns the next symbol or error.
func (sf *symbolsFile) Next() (string, error) {
if sf.nextValid {
defer func() {
sf.nextValid = false
sf.nextSymbol = ""
sf.nextErr = nil
}()
return sf.nextSymbol, sf.nextErr
}
return sf.readNext()
}
func (sf *symbolsFile) readNext() (string, error) {
var s string
err := sf.dec.Decode(&s)
// Decode returns io.EOF at the end.
if err != nil {
return "", err
}
return s, nil
}
func openFiles(filenames []string) ([]*os.File, error) {
var result []*os.File
for _, fn := range filenames {
f, err := os.Open(fn)
if err != nil {
// Close files opened so far.
for _, sf := range result {
_ = sf.Close()
}
return nil, err
}
result = append(result, f)
}
return result, nil
}

View file

@ -0,0 +1,71 @@
package tsdb
import (
"fmt"
"io"
"testing"
"github.com/stretchr/testify/require"
)
func TestSymbolsBatchAndIteration1(t *testing.T) {
testSymbolsBatchAndIterationWithFlushersConcurrency(t, 1)
}
func TestSymbolsBatchAndIteration5(t *testing.T) {
testSymbolsBatchAndIterationWithFlushersConcurrency(t, 5)
}
func testSymbolsBatchAndIterationWithFlushersConcurrency(t *testing.T, flushersConcurrency int) {
flushers := newSymbolFlushers(flushersConcurrency)
defer func() { _ = flushers.close() }()
dir := t.TempDir()
b := newSymbolsBatcher(100, dir, flushers)
allWords := map[string]struct{}{}
for i := 0; i < 10*flushersConcurrency; i++ {
require.NoError(t, b.addSymbol(""))
allWords[""] = struct{}{}
for j := 0; j < 123; j++ {
w := fmt.Sprintf("word_%d_%d", i%3, j)
require.NoError(t, b.addSymbol(w))
allWords[w] = struct{}{}
}
}
require.NoError(t, b.flushSymbols(true))
require.NoError(t, b.flushSymbols(true)) // call again, this should do nothing, and not create new empty file.
require.NoError(t, flushers.close())
symbols := b.getSymbolFiles()
it, err := newSymbolsIterator(symbols)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, it.Close())
})
first := true
var w, prev string
for w, err = it.NextSymbol(); err == nil; w, err = it.NextSymbol() {
if !first {
require.True(t, w != "")
require.True(t, prev < w)
}
first = false
_, known := allWords[w]
require.True(t, known)
delete(allWords, w)
prev = w
}
require.Equal(t, io.EOF, err)
require.Equal(t, 0, len(allWords))
}

View file

@ -27,12 +27,12 @@ import (
"strconv"
"strings"
"time"
"unsafe"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/grafana/regexp"
jsoniter "github.com/json-iterator/go"
"github.com/munnerz/goautoneg"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
@ -40,8 +40,6 @@ import (
"golang.org/x/exp/slices"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/textparse"
"github.com/prometheus/prometheus/model/timestamp"
@ -54,7 +52,6 @@ import (
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/tsdb/index"
"github.com/prometheus/prometheus/util/httputil"
"github.com/prometheus/prometheus/util/jsonutil"
"github.com/prometheus/prometheus/util/stats"
)
@ -72,14 +69,15 @@ const (
type errorType string
const (
errorNone errorType = ""
errorTimeout errorType = "timeout"
errorCanceled errorType = "canceled"
errorExec errorType = "execution"
errorBadData errorType = "bad_data"
errorInternal errorType = "internal"
errorUnavailable errorType = "unavailable"
errorNotFound errorType = "not_found"
errorNone errorType = ""
errorTimeout errorType = "timeout"
errorCanceled errorType = "canceled"
errorExec errorType = "execution"
errorBadData errorType = "bad_data"
errorInternal errorType = "internal"
errorUnavailable errorType = "unavailable"
errorNotFound errorType = "not_found"
errorNotAcceptable errorType = "not_acceptable"
)
var LocalhostRepresentations = []string{"127.0.0.1", "localhost", "::1"}
@ -149,7 +147,8 @@ type RuntimeInfo struct {
StorageRetention string `json:"storageRetention"`
}
type response struct {
// Response contains a response to a HTTP API request.
type Response struct {
Status status `json:"status"`
Data interface{} `json:"data,omitempty"`
ErrorType errorType `json:"errorType,omitempty"`
@ -212,13 +211,8 @@ type API struct {
remoteWriteHandler http.Handler
remoteReadHandler http.Handler
}
func init() {
jsoniter.RegisterTypeEncoderFunc("promql.Series", marshalSeriesJSON, marshalSeriesJSONIsEmpty)
jsoniter.RegisterTypeEncoderFunc("promql.Sample", marshalSampleJSON, marshalSampleJSONIsEmpty)
jsoniter.RegisterTypeEncoderFunc("promql.Point", marshalPointJSON, marshalPointJSONIsEmpty)
jsoniter.RegisterTypeEncoderFunc("exemplar.Exemplar", marshalExemplarJSON, marshalExemplarJSONEmpty)
codecs []Codec
}
// NewAPI returns an initialized API type.
@ -279,6 +273,8 @@ func NewAPI(
remoteReadHandler: remote.NewReadHandler(logger, registerer, q, configFunc, remoteReadSampleLimit, remoteReadConcurrencyLimit, remoteReadMaxBytesInFrame),
}
a.InstallCodec(JSONCodec{})
if statsRenderer != nil {
a.statsRenderer = statsRenderer
}
@ -290,6 +286,18 @@ func NewAPI(
return a
}
// InstallCodec adds codec to this API's available codecs.
// Codecs installed first take precedence over codecs installed later when evaluating wildcards in Accept headers.
// The first installed codec is used as a fallback when the Accept header cannot be satisfied or if there is no Accept header.
func (api *API) InstallCodec(codec Codec) {
api.codecs = append(api.codecs, codec)
}
// ClearCodecs removes all available codecs from this API, including the default codec installed by NewAPI.
func (api *API) ClearCodecs() {
api.codecs = nil
}
func setUnavailStatusOnTSDBNotReady(r apiFuncResult) apiFuncResult {
if r.err != nil && errors.Cause(r.err.err) == tsdb.ErrNotReady {
r.err.typ = errorUnavailable
@ -312,7 +320,7 @@ func (api *API) Register(r *route.Router) {
}
if result.data != nil {
api.respond(w, result.data, result.warnings)
api.respond(w, r, result.data, result.warnings)
return
}
w.WriteHeader(http.StatusNoContent)
@ -380,7 +388,7 @@ func (api *API) Register(r *route.Router) {
r.Put("/admin/tsdb/snapshot", wrapAgent(api.snapshot))
}
type queryData struct {
type QueryData struct {
ResultType parser.ValueType `json:"resultType"`
Result parser.Value `json:"result"`
Stats stats.QueryStats `json:"stats,omitempty"`
@ -445,7 +453,7 @@ func (api *API) query(r *http.Request) (result apiFuncResult) {
}
qs := sr(ctx, qry.Stats(), r.FormValue("stats"))
return apiFuncResult{&queryData{
return apiFuncResult{&QueryData{
ResultType: res.Value.Type(),
Result: res.Value,
Stats: qs,
@ -547,7 +555,7 @@ func (api *API) queryRange(r *http.Request) (result apiFuncResult) {
}
qs := sr(ctx, qry.Stats(), r.FormValue("stats"))
return apiFuncResult{&queryData{
return apiFuncResult{&QueryData{
ResultType: res.Value.Type(),
Result: res.Value,
Stats: qs,
@ -1474,7 +1482,7 @@ func (api *API) serveWALReplayStatus(w http.ResponseWriter, r *http.Request) {
if err != nil {
api.respondError(w, &apiError{errorInternal, err}, nil)
}
api.respond(w, walReplayStatus{
api.respond(w, r, walReplayStatus{
Min: status.Min,
Max: status.Max,
Current: status.Current,
@ -1576,34 +1584,59 @@ func (api *API) cleanTombstones(r *http.Request) apiFuncResult {
return apiFuncResult{nil, nil, nil, nil}
}
func (api *API) respond(w http.ResponseWriter, data interface{}, warnings storage.Warnings) {
func (api *API) respond(w http.ResponseWriter, req *http.Request, data interface{}, warnings storage.Warnings) {
statusMessage := statusSuccess
var warningStrings []string
for _, warning := range warnings {
warningStrings = append(warningStrings, warning.Error())
}
json := jsoniter.ConfigCompatibleWithStandardLibrary
b, err := json.Marshal(&response{
resp := &Response{
Status: statusMessage,
Data: data,
Warnings: warningStrings,
})
}
codec, err := api.negotiateCodec(req, resp)
if err != nil {
level.Error(api.logger).Log("msg", "error marshaling json response", "err", err)
api.respondError(w, &apiError{errorNotAcceptable, err}, nil)
return
}
b, err := codec.Encode(resp)
if err != nil {
level.Error(api.logger).Log("msg", "error marshaling response", "err", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Type", codec.ContentType().String())
w.WriteHeader(http.StatusOK)
if n, err := w.Write(b); err != nil {
level.Error(api.logger).Log("msg", "error writing response", "bytesWritten", n, "err", err)
}
}
func (api *API) negotiateCodec(req *http.Request, resp *Response) (Codec, error) {
for _, clause := range goautoneg.ParseAccept(req.Header.Get("Accept")) {
for _, codec := range api.codecs {
if codec.ContentType().Satisfies(clause) && codec.CanEncode(resp) {
return codec, nil
}
}
}
defaultCodec := api.codecs[0]
if !defaultCodec.CanEncode(resp) {
return nil, fmt.Errorf("cannot encode response as %s", defaultCodec.ContentType())
}
return defaultCodec, nil
}
func (api *API) respondError(w http.ResponseWriter, apiErr *apiError, data interface{}) {
json := jsoniter.ConfigCompatibleWithStandardLibrary
b, err := json.Marshal(&response{
b, err := json.Marshal(&Response{
Status: statusError,
ErrorType: apiErr.typ,
Error: apiErr.err.Error(),
@ -1629,6 +1662,8 @@ func (api *API) respondError(w http.ResponseWriter, apiErr *apiError, data inter
code = http.StatusInternalServerError
case errorNotFound:
code = http.StatusNotFound
case errorNotAcceptable:
code = http.StatusNotAcceptable
default:
code = http.StatusInternalServerError
}
@ -1710,247 +1745,3 @@ OUTER:
}
return matcherSets, nil
}
// marshalSeriesJSON writes something like the following:
//
// {
// "metric" : {
// "__name__" : "up",
// "job" : "prometheus",
// "instance" : "localhost:9090"
// },
// "values": [
// [ 1435781451.781, "1" ],
// < more values>
// ],
// "histograms": [
// [ 1435781451.781, { < histogram, see below > } ],
// < more histograms >
// ],
// },
func marshalSeriesJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
s := *((*promql.Series)(ptr))
stream.WriteObjectStart()
stream.WriteObjectField(`metric`)
m, err := s.Metric.MarshalJSON()
if err != nil {
stream.Error = err
return
}
stream.SetBuffer(append(stream.Buffer(), m...))
// We make two passes through the series here: In the first marshaling
// all value points, in the second marshaling all histogram
// points. That's probably cheaper than just one pass in which we copy
// out histogram Points into a newly allocated slice for separate
// marshaling. (Could be benchmarked, though.)
var foundValue, foundHistogram bool
for _, p := range s.Points {
if p.H == nil {
stream.WriteMore()
if !foundValue {
stream.WriteObjectField(`values`)
stream.WriteArrayStart()
}
foundValue = true
marshalPointJSON(unsafe.Pointer(&p), stream)
} else {
foundHistogram = true
}
}
if foundValue {
stream.WriteArrayEnd()
}
if foundHistogram {
firstHistogram := true
for _, p := range s.Points {
if p.H != nil {
stream.WriteMore()
if firstHistogram {
stream.WriteObjectField(`histograms`)
stream.WriteArrayStart()
}
firstHistogram = false
marshalPointJSON(unsafe.Pointer(&p), stream)
}
}
stream.WriteArrayEnd()
}
stream.WriteObjectEnd()
}
func marshalSeriesJSONIsEmpty(ptr unsafe.Pointer) bool {
return false
}
// marshalSampleJSON writes something like the following for normal value samples:
//
// {
// "metric" : {
// "__name__" : "up",
// "job" : "prometheus",
// "instance" : "localhost:9090"
// },
// "value": [ 1435781451.781, "1" ]
// },
//
// For histogram samples, it writes something like this:
//
// {
// "metric" : {
// "__name__" : "up",
// "job" : "prometheus",
// "instance" : "localhost:9090"
// },
// "histogram": [ 1435781451.781, { < histogram, see below > } ]
// },
func marshalSampleJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
s := *((*promql.Sample)(ptr))
stream.WriteObjectStart()
stream.WriteObjectField(`metric`)
m, err := s.Metric.MarshalJSON()
if err != nil {
stream.Error = err
return
}
stream.SetBuffer(append(stream.Buffer(), m...))
stream.WriteMore()
if s.Point.H == nil {
stream.WriteObjectField(`value`)
} else {
stream.WriteObjectField(`histogram`)
}
marshalPointJSON(unsafe.Pointer(&s.Point), stream)
stream.WriteObjectEnd()
}
func marshalSampleJSONIsEmpty(ptr unsafe.Pointer) bool {
return false
}
// marshalPointJSON writes `[ts, "val"]`.
func marshalPointJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
p := *((*promql.Point)(ptr))
stream.WriteArrayStart()
jsonutil.MarshalTimestamp(p.T, stream)
stream.WriteMore()
if p.H == nil {
jsonutil.MarshalValue(p.V, stream)
} else {
marshalHistogram(p.H, stream)
}
stream.WriteArrayEnd()
}
func marshalPointJSONIsEmpty(ptr unsafe.Pointer) bool {
return false
}
// marshalHistogramJSON writes something like:
//
// {
// "count": "42",
// "sum": "34593.34",
// "buckets": [
// [ 3, "-0.25", "0.25", "3"],
// [ 0, "0.25", "0.5", "12"],
// [ 0, "0.5", "1", "21"],
// [ 0, "2", "4", "6"]
// ]
// }
//
// The 1st element in each bucket array determines if the boundaries are
// inclusive (AKA closed) or exclusive (AKA open):
//
// 0: lower exclusive, upper inclusive
// 1: lower inclusive, upper exclusive
// 2: both exclusive
// 3: both inclusive
//
// The 2nd and 3rd elements are the lower and upper boundary. The 4th element is
// the bucket count.
func marshalHistogram(h *histogram.FloatHistogram, stream *jsoniter.Stream) {
stream.WriteObjectStart()
stream.WriteObjectField(`count`)
jsonutil.MarshalValue(h.Count, stream)
stream.WriteMore()
stream.WriteObjectField(`sum`)
jsonutil.MarshalValue(h.Sum, stream)
bucketFound := false
it := h.AllBucketIterator()
for it.Next() {
bucket := it.At()
if bucket.Count == 0 {
continue // No need to expose empty buckets in JSON.
}
stream.WriteMore()
if !bucketFound {
stream.WriteObjectField(`buckets`)
stream.WriteArrayStart()
}
bucketFound = true
boundaries := 2 // Exclusive on both sides AKA open interval.
if bucket.LowerInclusive {
if bucket.UpperInclusive {
boundaries = 3 // Inclusive on both sides AKA closed interval.
} else {
boundaries = 1 // Inclusive only on lower end AKA right open.
}
} else {
if bucket.UpperInclusive {
boundaries = 0 // Inclusive only on upper end AKA left open.
}
}
stream.WriteArrayStart()
stream.WriteInt(boundaries)
stream.WriteMore()
jsonutil.MarshalValue(bucket.Lower, stream)
stream.WriteMore()
jsonutil.MarshalValue(bucket.Upper, stream)
stream.WriteMore()
jsonutil.MarshalValue(bucket.Count, stream)
stream.WriteArrayEnd()
}
if bucketFound {
stream.WriteArrayEnd()
}
stream.WriteObjectEnd()
}
// marshalExemplarJSON writes.
//
// {
// labels: <labels>,
// value: "<string>",
// timestamp: <float>
// }
func marshalExemplarJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
p := *((*exemplar.Exemplar)(ptr))
stream.WriteObjectStart()
// "labels" key.
stream.WriteObjectField(`labels`)
lbls, err := p.Labels.MarshalJSON()
if err != nil {
stream.Error = err
return
}
stream.SetBuffer(append(stream.Buffer(), lbls...))
// "value" key.
stream.WriteMore()
stream.WriteObjectField(`value`)
jsonutil.MarshalValue(p.Value, stream)
// "timestamp" key.
stream.WriteMore()
stream.WriteObjectField(`timestamp`)
jsonutil.MarshalTimestamp(p.Ts, stream)
stream.WriteObjectEnd()
}
func marshalExemplarJSONEmpty(ptr unsafe.Pointer) bool {
return false
}

View file

@ -18,7 +18,6 @@ import (
"encoding/json"
"fmt"
"io"
"math"
"net/http"
"net/http/httptest"
"net/url"
@ -30,7 +29,6 @@ import (
"testing"
"time"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/util/stats"
@ -835,8 +833,8 @@ func TestStats(t *testing.T) {
name: "stats is blank",
param: "",
expected: func(t *testing.T, i interface{}) {
require.IsType(t, i, &queryData{})
qd := i.(*queryData)
require.IsType(t, i, &QueryData{})
qd := i.(*QueryData)
require.Nil(t, qd.Stats)
},
},
@ -844,8 +842,8 @@ func TestStats(t *testing.T) {
name: "stats is true",
param: "true",
expected: func(t *testing.T, i interface{}) {
require.IsType(t, i, &queryData{})
qd := i.(*queryData)
require.IsType(t, i, &QueryData{})
qd := i.(*QueryData)
require.NotNil(t, qd.Stats)
qs := qd.Stats.Builtin()
require.NotNil(t, qs.Timings)
@ -859,8 +857,8 @@ func TestStats(t *testing.T) {
name: "stats is all",
param: "all",
expected: func(t *testing.T, i interface{}) {
require.IsType(t, i, &queryData{})
qd := i.(*queryData)
require.IsType(t, i, &QueryData{})
qd := i.(*QueryData)
require.NotNil(t, qd.Stats)
qs := qd.Stats.Builtin()
require.NotNil(t, qs.Timings)
@ -880,8 +878,8 @@ func TestStats(t *testing.T) {
},
param: "known",
expected: func(t *testing.T, i interface{}) {
require.IsType(t, i, &queryData{})
qd := i.(*queryData)
require.IsType(t, i, &QueryData{})
qd := i.(*QueryData)
require.NotNil(t, qd.Stats)
j, err := json.Marshal(qd.Stats)
require.NoError(t, err)
@ -1041,7 +1039,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
"query": []string{"2"},
"time": []string{"123.4"},
},
response: &queryData{
response: &QueryData{
ResultType: parser.ValueTypeScalar,
Result: promql.Scalar{
V: 2,
@ -1055,7 +1053,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
"query": []string{"0.333"},
"time": []string{"1970-01-01T00:02:03Z"},
},
response: &queryData{
response: &QueryData{
ResultType: parser.ValueTypeScalar,
Result: promql.Scalar{
V: 0.333,
@ -1069,7 +1067,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
"query": []string{"0.333"},
"time": []string{"1970-01-01T01:02:03+01:00"},
},
response: &queryData{
response: &QueryData{
ResultType: parser.ValueTypeScalar,
Result: promql.Scalar{
V: 0.333,
@ -1082,7 +1080,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
query: url.Values{
"query": []string{"0.333"},
},
response: &queryData{
response: &QueryData{
ResultType: parser.ValueTypeScalar,
Result: promql.Scalar{
V: 0.333,
@ -1098,7 +1096,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
"end": []string{"2"},
"step": []string{"1"},
},
response: &queryData{
response: &QueryData{
ResultType: parser.ValueTypeMatrix,
Result: promql.Matrix{
promql.Series{
@ -2767,39 +2765,123 @@ func TestAdminEndpoints(t *testing.T) {
}
func TestRespondSuccess(t *testing.T) {
api := API{
logger: log.NewNopLogger(),
}
api.ClearCodecs()
api.InstallCodec(JSONCodec{})
api.InstallCodec(&testCodec{contentType: MIMEType{"test", "cannot-encode"}, canEncode: false})
api.InstallCodec(&testCodec{contentType: MIMEType{"test", "can-encode"}, canEncode: true})
api.InstallCodec(&testCodec{contentType: MIMEType{"test", "can-encode-2"}, canEncode: true})
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
api := API{}
api.respond(w, "test", nil)
api.respond(w, r, "test", nil)
}))
defer s.Close()
resp, err := http.Get(s.URL)
if err != nil {
t.Fatalf("Error on test request: %s", err)
for _, tc := range []struct {
name string
acceptHeader string
expectedContentType string
expectedBody string
}{
{
name: "no Accept header",
expectedContentType: "application/json",
expectedBody: `{"status":"success","data":"test"}`,
},
{
name: "Accept header with single content type which is suitable",
acceptHeader: "test/can-encode",
expectedContentType: "test/can-encode",
expectedBody: `response from test/can-encode codec`,
},
{
name: "Accept header with single content type which is not available",
acceptHeader: "test/not-registered",
expectedContentType: "application/json",
expectedBody: `{"status":"success","data":"test"}`,
},
{
name: "Accept header with single content type which cannot encode the response payload",
acceptHeader: "test/cannot-encode",
expectedContentType: "application/json",
expectedBody: `{"status":"success","data":"test"}`,
},
{
name: "Accept header with multiple content types, all of which are suitable",
acceptHeader: "test/can-encode, test/can-encode-2",
expectedContentType: "test/can-encode",
expectedBody: `response from test/can-encode codec`,
},
{
name: "Accept header with multiple content types, only one of which is available",
acceptHeader: "test/not-registered, test/can-encode",
expectedContentType: "test/can-encode",
expectedBody: `response from test/can-encode codec`,
},
{
name: "Accept header with multiple content types, only one of which can encode the response payload",
acceptHeader: "test/cannot-encode, test/can-encode",
expectedContentType: "test/can-encode",
expectedBody: `response from test/can-encode codec`,
},
{
name: "Accept header with multiple content types, none of which are available",
acceptHeader: "test/not-registered, test/also-not-registered",
expectedContentType: "application/json",
expectedBody: `{"status":"success","data":"test"}`,
},
} {
t.Run(tc.name, func(t *testing.T) {
req, err := http.NewRequest(http.MethodGet, s.URL, nil)
require.NoError(t, err)
if tc.acceptHeader != "" {
req.Header.Set("Accept", tc.acceptHeader)
}
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
body, err := io.ReadAll(resp.Body)
defer resp.Body.Close()
require.NoError(t, err)
require.Equal(t, http.StatusOK, resp.StatusCode)
require.Equal(t, tc.expectedContentType, resp.Header.Get("Content-Type"))
require.Equal(t, tc.expectedBody, string(body))
})
}
}
func TestRespondSuccess_DefaultCodecCannotEncodeResponse(t *testing.T) {
api := API{
logger: log.NewNopLogger(),
}
api.ClearCodecs()
api.InstallCodec(&testCodec{contentType: MIMEType{"application", "default-format"}, canEncode: false})
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
api.respond(w, r, "test", nil)
}))
defer s.Close()
req, err := http.NewRequest(http.MethodGet, s.URL, nil)
require.NoError(t, err)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
body, err := io.ReadAll(resp.Body)
defer resp.Body.Close()
if err != nil {
t.Fatalf("Error reading response body: %s", err)
}
require.NoError(t, err)
if resp.StatusCode != http.StatusOK {
t.Fatalf("Return code %d expected in success response but got %d", 200, resp.StatusCode)
}
if h := resp.Header.Get("Content-Type"); h != "application/json" {
t.Fatalf("Expected Content-Type %q but got %q", "application/json", h)
}
var res response
if err = json.Unmarshal([]byte(body), &res); err != nil {
t.Fatalf("Error unmarshaling JSON body: %s", err)
}
exp := &response{
Status: statusSuccess,
Data: "test",
}
require.Equal(t, exp, &res)
require.Equal(t, http.StatusNotAcceptable, resp.StatusCode)
require.Equal(t, "application/json", resp.Header.Get("Content-Type"))
require.Equal(t, `{"status":"error","errorType":"not_acceptable","error":"cannot encode response as application/default-format"}`, string(body))
}
func TestRespondError(t *testing.T) {
@ -2826,12 +2908,12 @@ func TestRespondError(t *testing.T) {
t.Fatalf("Expected Content-Type %q but got %q", "application/json", h)
}
var res response
var res Response
if err = json.Unmarshal([]byte(body), &res); err != nil {
t.Fatalf("Error unmarshaling JSON body: %s", err)
}
exp := &response{
exp := &Response{
Status: statusError,
Data: "test",
ErrorType: errorTimeout,
@ -3049,165 +3131,6 @@ func TestOptionsMethod(t *testing.T) {
}
}
func TestRespond(t *testing.T) {
cases := []struct {
response interface{}
expected string
}{
{
response: &queryData{
ResultType: parser.ValueTypeMatrix,
Result: promql.Matrix{
promql.Series{
Points: []promql.Point{{V: 1, T: 1000}},
Metric: labels.FromStrings("__name__", "foo"),
},
},
},
expected: `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"foo"},"values":[[1,"1"]]}]}}`,
},
{
response: &queryData{
ResultType: parser.ValueTypeMatrix,
Result: promql.Matrix{
promql.Series{
Points: []promql.Point{{H: &histogram.FloatHistogram{
Schema: 2,
ZeroThreshold: 0.001,
ZeroCount: 12,
Count: 10,
Sum: 20,
PositiveSpans: []histogram.Span{
{Offset: 3, Length: 2},
{Offset: 1, Length: 3},
},
NegativeSpans: []histogram.Span{
{Offset: 2, Length: 2},
},
PositiveBuckets: []float64{1, 2, 2, 1, 1},
NegativeBuckets: []float64{2, 1},
}, T: 1000}},
Metric: labels.FromStrings("__name__", "foo"),
},
},
},
expected: `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"foo"},"histograms":[[1,{"count":"10","sum":"20","buckets":[[1,"-1.6817928305074288","-1.414213562373095","1"],[1,"-1.414213562373095","-1.189207115002721","2"],[3,"-0.001","0.001","12"],[0,"1.414213562373095","1.6817928305074288","1"],[0,"1.6817928305074288","2","2"],[0,"2.378414230005442","2.82842712474619","2"],[0,"2.82842712474619","3.3635856610148576","1"],[0,"3.3635856610148576","4","1"]]}]]}]}}`,
},
{
response: promql.Point{V: 0, T: 0},
expected: `{"status":"success","data":[0,"0"]}`,
},
{
response: promql.Point{V: 20, T: 1},
expected: `{"status":"success","data":[0.001,"20"]}`,
},
{
response: promql.Point{V: 20, T: 10},
expected: `{"status":"success","data":[0.010,"20"]}`,
},
{
response: promql.Point{V: 20, T: 100},
expected: `{"status":"success","data":[0.100,"20"]}`,
},
{
response: promql.Point{V: 20, T: 1001},
expected: `{"status":"success","data":[1.001,"20"]}`,
},
{
response: promql.Point{V: 20, T: 1010},
expected: `{"status":"success","data":[1.010,"20"]}`,
},
{
response: promql.Point{V: 20, T: 1100},
expected: `{"status":"success","data":[1.100,"20"]}`,
},
{
response: promql.Point{V: 20, T: 12345678123456555},
expected: `{"status":"success","data":[12345678123456.555,"20"]}`,
},
{
response: promql.Point{V: 20, T: -1},
expected: `{"status":"success","data":[-0.001,"20"]}`,
},
{
response: promql.Point{V: math.NaN(), T: 0},
expected: `{"status":"success","data":[0,"NaN"]}`,
},
{
response: promql.Point{V: math.Inf(1), T: 0},
expected: `{"status":"success","data":[0,"+Inf"]}`,
},
{
response: promql.Point{V: math.Inf(-1), T: 0},
expected: `{"status":"success","data":[0,"-Inf"]}`,
},
{
response: promql.Point{V: 1.2345678e6, T: 0},
expected: `{"status":"success","data":[0,"1234567.8"]}`,
},
{
response: promql.Point{V: 1.2345678e-6, T: 0},
expected: `{"status":"success","data":[0,"0.0000012345678"]}`,
},
{
response: promql.Point{V: 1.2345678e-67, T: 0},
expected: `{"status":"success","data":[0,"1.2345678e-67"]}`,
},
{
response: []exemplar.QueryResult{
{
SeriesLabels: labels.FromStrings("foo", "bar"),
Exemplars: []exemplar.Exemplar{
{
Labels: labels.FromStrings("traceID", "abc"),
Value: 100.123,
Ts: 1234,
},
},
},
},
expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"traceID":"abc"},"value":"100.123","timestamp":1.234}]}]}`,
},
{
response: []exemplar.QueryResult{
{
SeriesLabels: labels.FromStrings("foo", "bar"),
Exemplars: []exemplar.Exemplar{
{
Labels: labels.FromStrings("traceID", "abc"),
Value: math.Inf(1),
Ts: 1234,
},
},
},
},
expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"traceID":"abc"},"value":"+Inf","timestamp":1.234}]}]}`,
},
}
for _, c := range cases {
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
api := API{}
api.respond(w, c.response, nil)
}))
defer s.Close()
resp, err := http.Get(s.URL)
if err != nil {
t.Fatalf("Error on test request: %s", err)
}
body, err := io.ReadAll(resp.Body)
defer resp.Body.Close()
if err != nil {
t.Fatalf("Error reading response body: %s", err)
}
if string(body) != c.expected {
t.Fatalf("Expected response \n%v\n but got \n%v\n", c.expected, string(body))
}
}
}
func TestTSDBStatus(t *testing.T) {
tsdb := &fakeDB{}
tsdbStatusAPI := func(api *API) apiFunc { return api.serveTSDBStatus }
@ -3283,11 +3206,13 @@ var testResponseWriter = httptest.ResponseRecorder{}
func BenchmarkRespond(b *testing.B) {
b.ReportAllocs()
request, err := http.NewRequest(http.MethodGet, "/does-not-matter", nil)
require.NoError(b, err)
points := []promql.Point{}
for i := 0; i < 10000; i++ {
points = append(points, promql.Point{V: float64(i * 1000000), T: int64(i)})
}
response := &queryData{
response := &QueryData{
ResultType: parser.ValueTypeMatrix,
Result: promql.Matrix{
promql.Series{
@ -3299,7 +3224,7 @@ func BenchmarkRespond(b *testing.B) {
b.ResetTimer()
api := API{}
for n := 0; n < b.N; n++ {
api.respond(&testResponseWriter, response, nil)
api.respond(&testResponseWriter, request, response, nil)
}
}
@ -3411,6 +3336,23 @@ func TestGetGlobalURL(t *testing.T) {
}
}
type testCodec struct {
contentType MIMEType
canEncode bool
}
func (t *testCodec) ContentType() MIMEType {
return t.contentType
}
func (t *testCodec) CanEncode(_ *Response) bool {
return t.canEncode
}
func (t *testCodec) Encode(_ *Response) ([]byte, error) {
return []byte(fmt.Sprintf("response from %v codec", t.contentType)), nil
}
func TestExtractQueryOpts(t *testing.T) {
tests := []struct {
name string

53
web/api/v1/codec.go Normal file
View file

@ -0,0 +1,53 @@
// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import "github.com/munnerz/goautoneg"
// A Codec performs encoding of API responses.
type Codec interface {
// ContentType returns the MIME time that this Codec emits.
ContentType() MIMEType
// CanEncode determines if this Codec can encode resp.
CanEncode(resp *Response) bool
// Encode encodes resp, ready for transmission to an API consumer.
Encode(resp *Response) ([]byte, error)
}
type MIMEType struct {
Type string
SubType string
}
func (m MIMEType) String() string {
return m.Type + "/" + m.SubType
}
func (m MIMEType) Satisfies(accept goautoneg.Accept) bool {
if accept.Type == "*" && accept.SubType == "*" {
return true
}
if accept.Type == m.Type && accept.SubType == "*" {
return true
}
if accept.Type == m.Type && accept.SubType == m.SubType {
return true
}
return false
}

68
web/api/v1/codec_test.go Normal file
View file

@ -0,0 +1,68 @@
// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"testing"
"github.com/munnerz/goautoneg"
"github.com/stretchr/testify/require"
)
func TestMIMEType_String(t *testing.T) {
m := MIMEType{Type: "application", SubType: "json"}
require.Equal(t, "application/json", m.String())
}
func TestMIMEType_Satisfies(t *testing.T) {
m := MIMEType{Type: "application", SubType: "json"}
scenarios := map[string]struct {
accept goautoneg.Accept
expected bool
}{
"exact match": {
accept: goautoneg.Accept{Type: "application", SubType: "json"},
expected: true,
},
"sub-type wildcard match": {
accept: goautoneg.Accept{Type: "application", SubType: "*"},
expected: true,
},
"full wildcard match": {
accept: goautoneg.Accept{Type: "*", SubType: "*"},
expected: true,
},
"inverted": {
accept: goautoneg.Accept{Type: "json", SubType: "application"},
expected: false,
},
"inverted sub-type wildcard": {
accept: goautoneg.Accept{Type: "json", SubType: "*"},
expected: false,
},
"complete mismatch": {
accept: goautoneg.Accept{Type: "text", SubType: "plain"},
expected: false,
},
}
for name, scenario := range scenarios {
t.Run(name, func(t *testing.T) {
actual := m.Satisfies(scenario.accept)
require.Equal(t, scenario.expected, actual)
})
}
}

292
web/api/v1/json_codec.go Normal file
View file

@ -0,0 +1,292 @@
// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"unsafe"
jsoniter "github.com/json-iterator/go"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/util/jsonutil"
)
func init() {
jsoniter.RegisterTypeEncoderFunc("promql.Series", marshalSeriesJSON, marshalSeriesJSONIsEmpty)
jsoniter.RegisterTypeEncoderFunc("promql.Sample", marshalSampleJSON, marshalSampleJSONIsEmpty)
jsoniter.RegisterTypeEncoderFunc("promql.Point", marshalPointJSON, marshalPointJSONIsEmpty)
jsoniter.RegisterTypeEncoderFunc("exemplar.Exemplar", marshalExemplarJSON, marshalExemplarJSONEmpty)
}
// JSONCodec is a Codec that encodes API responses as JSON.
type JSONCodec struct{}
func (j JSONCodec) ContentType() MIMEType {
return MIMEType{Type: "application", SubType: "json"}
}
func (j JSONCodec) CanEncode(_ *Response) bool {
return true
}
func (j JSONCodec) Encode(resp *Response) ([]byte, error) {
json := jsoniter.ConfigCompatibleWithStandardLibrary
return json.Marshal(resp)
}
// marshalSeriesJSON writes something like the following:
//
// {
// "metric" : {
// "__name__" : "up",
// "job" : "prometheus",
// "instance" : "localhost:9090"
// },
// "values": [
// [ 1435781451.781, "1" ],
// < more values>
// ],
// "histograms": [
// [ 1435781451.781, { < histogram, see below > } ],
// < more histograms >
// ],
// },
func marshalSeriesJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
s := *((*promql.Series)(ptr))
stream.WriteObjectStart()
stream.WriteObjectField(`metric`)
m, err := s.Metric.MarshalJSON()
if err != nil {
stream.Error = err
return
}
stream.SetBuffer(append(stream.Buffer(), m...))
// We make two passes through the series here: In the first marshaling
// all value points, in the second marshaling all histogram
// points. That's probably cheaper than just one pass in which we copy
// out histogram Points into a newly allocated slice for separate
// marshaling. (Could be benchmarked, though.)
var foundValue, foundHistogram bool
for _, p := range s.Points {
if p.H == nil {
stream.WriteMore()
if !foundValue {
stream.WriteObjectField(`values`)
stream.WriteArrayStart()
}
foundValue = true
marshalPointJSON(unsafe.Pointer(&p), stream)
} else {
foundHistogram = true
}
}
if foundValue {
stream.WriteArrayEnd()
}
if foundHistogram {
firstHistogram := true
for _, p := range s.Points {
if p.H != nil {
stream.WriteMore()
if firstHistogram {
stream.WriteObjectField(`histograms`)
stream.WriteArrayStart()
}
firstHistogram = false
marshalPointJSON(unsafe.Pointer(&p), stream)
}
}
stream.WriteArrayEnd()
}
stream.WriteObjectEnd()
}
func marshalSeriesJSONIsEmpty(ptr unsafe.Pointer) bool {
return false
}
// marshalSampleJSON writes something like the following for normal value samples:
//
// {
// "metric" : {
// "__name__" : "up",
// "job" : "prometheus",
// "instance" : "localhost:9090"
// },
// "value": [ 1435781451.781, "1" ]
// },
//
// For histogram samples, it writes something like this:
//
// {
// "metric" : {
// "__name__" : "up",
// "job" : "prometheus",
// "instance" : "localhost:9090"
// },
// "histogram": [ 1435781451.781, { < histogram, see below > } ]
// },
func marshalSampleJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
s := *((*promql.Sample)(ptr))
stream.WriteObjectStart()
stream.WriteObjectField(`metric`)
m, err := s.Metric.MarshalJSON()
if err != nil {
stream.Error = err
return
}
stream.SetBuffer(append(stream.Buffer(), m...))
stream.WriteMore()
if s.Point.H == nil {
stream.WriteObjectField(`value`)
} else {
stream.WriteObjectField(`histogram`)
}
marshalPointJSON(unsafe.Pointer(&s.Point), stream)
stream.WriteObjectEnd()
}
func marshalSampleJSONIsEmpty(ptr unsafe.Pointer) bool {
return false
}
// marshalPointJSON writes `[ts, "val"]`.
func marshalPointJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
p := *((*promql.Point)(ptr))
stream.WriteArrayStart()
jsonutil.MarshalTimestamp(p.T, stream)
stream.WriteMore()
if p.H == nil {
jsonutil.MarshalValue(p.V, stream)
} else {
marshalHistogram(p.H, stream)
}
stream.WriteArrayEnd()
}
func marshalPointJSONIsEmpty(ptr unsafe.Pointer) bool {
return false
}
// marshalHistogramJSON writes something like:
//
// {
// "count": "42",
// "sum": "34593.34",
// "buckets": [
// [ 3, "-0.25", "0.25", "3"],
// [ 0, "0.25", "0.5", "12"],
// [ 0, "0.5", "1", "21"],
// [ 0, "2", "4", "6"]
// ]
// }
//
// The 1st element in each bucket array determines if the boundaries are
// inclusive (AKA closed) or exclusive (AKA open):
//
// 0: lower exclusive, upper inclusive
// 1: lower inclusive, upper exclusive
// 2: both exclusive
// 3: both inclusive
//
// The 2nd and 3rd elements are the lower and upper boundary. The 4th element is
// the bucket count.
func marshalHistogram(h *histogram.FloatHistogram, stream *jsoniter.Stream) {
stream.WriteObjectStart()
stream.WriteObjectField(`count`)
jsonutil.MarshalValue(h.Count, stream)
stream.WriteMore()
stream.WriteObjectField(`sum`)
jsonutil.MarshalValue(h.Sum, stream)
bucketFound := false
it := h.AllBucketIterator()
for it.Next() {
bucket := it.At()
if bucket.Count == 0 {
continue // No need to expose empty buckets in JSON.
}
stream.WriteMore()
if !bucketFound {
stream.WriteObjectField(`buckets`)
stream.WriteArrayStart()
}
bucketFound = true
boundaries := 2 // Exclusive on both sides AKA open interval.
if bucket.LowerInclusive {
if bucket.UpperInclusive {
boundaries = 3 // Inclusive on both sides AKA closed interval.
} else {
boundaries = 1 // Inclusive only on lower end AKA right open.
}
} else {
if bucket.UpperInclusive {
boundaries = 0 // Inclusive only on upper end AKA left open.
}
}
stream.WriteArrayStart()
stream.WriteInt(boundaries)
stream.WriteMore()
jsonutil.MarshalValue(bucket.Lower, stream)
stream.WriteMore()
jsonutil.MarshalValue(bucket.Upper, stream)
stream.WriteMore()
jsonutil.MarshalValue(bucket.Count, stream)
stream.WriteArrayEnd()
}
if bucketFound {
stream.WriteArrayEnd()
}
stream.WriteObjectEnd()
}
// marshalExemplarJSON writes.
//
// {
// labels: <labels>,
// value: "<string>",
// timestamp: <float>
// }
func marshalExemplarJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
p := *((*exemplar.Exemplar)(ptr))
stream.WriteObjectStart()
// "labels" key.
stream.WriteObjectField(`labels`)
lbls, err := p.Labels.MarshalJSON()
if err != nil {
stream.Error = err
return
}
stream.SetBuffer(append(stream.Buffer(), lbls...))
// "value" key.
stream.WriteMore()
stream.WriteObjectField(`value`)
jsonutil.MarshalValue(p.Value, stream)
// "timestamp" key.
stream.WriteMore()
stream.WriteObjectField(`timestamp`)
jsonutil.MarshalTimestamp(p.Ts, stream)
stream.WriteObjectEnd()
}
func marshalExemplarJSONEmpty(ptr unsafe.Pointer) bool {
return false
}

View file

@ -0,0 +1,178 @@
// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"math"
"testing"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/promql/parser"
)
func TestJsonCodec_Encode(t *testing.T) {
cases := []struct {
response interface{}
expected string
}{
{
response: &QueryData{
ResultType: parser.ValueTypeMatrix,
Result: promql.Matrix{
promql.Series{
Points: []promql.Point{{V: 1, T: 1000}},
Metric: labels.FromStrings("__name__", "foo"),
},
},
},
expected: `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"foo"},"values":[[1,"1"]]}]}}`,
},
{
response: &QueryData{
ResultType: parser.ValueTypeMatrix,
Result: promql.Matrix{
promql.Series{
Points: []promql.Point{{H: &histogram.FloatHistogram{
Schema: 2,
ZeroThreshold: 0.001,
ZeroCount: 12,
Count: 10,
Sum: 20,
PositiveSpans: []histogram.Span{
{Offset: 3, Length: 2},
{Offset: 1, Length: 3},
},
NegativeSpans: []histogram.Span{
{Offset: 2, Length: 2},
},
PositiveBuckets: []float64{1, 2, 2, 1, 1},
NegativeBuckets: []float64{2, 1},
}, T: 1000}},
Metric: labels.FromStrings("__name__", "foo"),
},
},
},
expected: `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"foo"},"histograms":[[1,{"count":"10","sum":"20","buckets":[[1,"-1.6817928305074288","-1.414213562373095","1"],[1,"-1.414213562373095","-1.189207115002721","2"],[3,"-0.001","0.001","12"],[0,"1.414213562373095","1.6817928305074288","1"],[0,"1.6817928305074288","2","2"],[0,"2.378414230005442","2.82842712474619","2"],[0,"2.82842712474619","3.3635856610148576","1"],[0,"3.3635856610148576","4","1"]]}]]}]}}`,
},
{
response: promql.Point{V: 0, T: 0},
expected: `{"status":"success","data":[0,"0"]}`,
},
{
response: promql.Point{V: 20, T: 1},
expected: `{"status":"success","data":[0.001,"20"]}`,
},
{
response: promql.Point{V: 20, T: 10},
expected: `{"status":"success","data":[0.010,"20"]}`,
},
{
response: promql.Point{V: 20, T: 100},
expected: `{"status":"success","data":[0.100,"20"]}`,
},
{
response: promql.Point{V: 20, T: 1001},
expected: `{"status":"success","data":[1.001,"20"]}`,
},
{
response: promql.Point{V: 20, T: 1010},
expected: `{"status":"success","data":[1.010,"20"]}`,
},
{
response: promql.Point{V: 20, T: 1100},
expected: `{"status":"success","data":[1.100,"20"]}`,
},
{
response: promql.Point{V: 20, T: 12345678123456555},
expected: `{"status":"success","data":[12345678123456.555,"20"]}`,
},
{
response: promql.Point{V: 20, T: -1},
expected: `{"status":"success","data":[-0.001,"20"]}`,
},
{
response: promql.Point{V: math.NaN(), T: 0},
expected: `{"status":"success","data":[0,"NaN"]}`,
},
{
response: promql.Point{V: math.Inf(1), T: 0},
expected: `{"status":"success","data":[0,"+Inf"]}`,
},
{
response: promql.Point{V: math.Inf(-1), T: 0},
expected: `{"status":"success","data":[0,"-Inf"]}`,
},
{
response: promql.Point{V: 1.2345678e6, T: 0},
expected: `{"status":"success","data":[0,"1234567.8"]}`,
},
{
response: promql.Point{V: 1.2345678e-6, T: 0},
expected: `{"status":"success","data":[0,"0.0000012345678"]}`,
},
{
response: promql.Point{V: 1.2345678e-67, T: 0},
expected: `{"status":"success","data":[0,"1.2345678e-67"]}`,
},
{
response: []exemplar.QueryResult{
{
SeriesLabels: labels.FromStrings("foo", "bar"),
Exemplars: []exemplar.Exemplar{
{
Labels: labels.FromStrings("traceID", "abc"),
Value: 100.123,
Ts: 1234,
},
},
},
},
expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"traceID":"abc"},"value":"100.123","timestamp":1.234}]}]}`,
},
{
response: []exemplar.QueryResult{
{
SeriesLabels: labels.FromStrings("foo", "bar"),
Exemplars: []exemplar.Exemplar{
{
Labels: labels.FromStrings("traceID", "abc"),
Value: math.Inf(1),
Ts: 1234,
},
},
},
},
expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"traceID":"abc"},"value":"+Inf","timestamp":1.234}]}]}`,
},
}
codec := JSONCodec{}
for _, c := range cases {
body, err := codec.Encode(&Response{
Status: statusSuccess,
Data: c.response,
})
if err != nil {
t.Fatalf("Error encoding response body: %s", err)
}
if string(body) != c.expected {
t.Fatalf("Expected response \n%v\n but got \n%v\n", c.expected, string(body))
}
}
}

1187
web/ui/assets_vfsdata.go Normal file

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,19 @@
// This file was generated by lezer-generator. You probably shouldn't edit it.
import {LRParser} from "@lezer/lr"
import {specializeIdentifier, extendIdentifier} from "./tokens"
const spec_Identifier = {__proto__:null,absent_over_time:307, absent:309, abs:311, acos:313, acosh:315, asin:317, asinh:319, atan:321, atanh:323, avg_over_time:325, ceil:327, changes:329, clamp:331, clamp_max:333, clamp_min:335, cos:337, cosh:339, count_over_time:341, days_in_month:343, day_of_month:345, day_of_week:347, deg:349, delta:351, deriv:353, exp:355, floor:357, histogram_quantile:359, holt_winters:361, hour:363, idelta:365, increase:367, irate:369, label_replace:371, label_join:373, last_over_time:375, ln:377, log10:379, log2:381, max_over_time:383, min_over_time:385, minute:387, month:389, pi:391, predict_linear:393, present_over_time:395, quantile_over_time:397, rad:399, rate:401, resets:403, round:405, scalar:407, sgn:409, sin:411, sinh:413, sort:415, sort_desc:417, sqrt:419, stddev_over_time:421, stdvar_over_time:423, sum_over_time:425, tan:427, tanh:429, timestamp:431, time:433, vector:435, year:437}
export const parser = LRParser.deserialize({
version: 13,
states: "6[OYQPOOO&{QPOOOOQO'#C{'#C{O'QQPO'#CzQ']QQOOOOQO'#De'#DeO'WQPO'#DdOOQO'#E}'#E}O(jQPO'#FTOYQPO'#FPOYQPO'#FSOOQO'#FV'#FVO.fQSO'#FWO.nQQO'#FUOOQO'#FU'#FUOOQO'#Cy'#CyOOQO'#Df'#DfOOQO'#Dh'#DhOOQO'#Di'#DiOOQO'#Dj'#DjOOQO'#Dk'#DkOOQO'#Dl'#DlOOQO'#Dm'#DmOOQO'#Dn'#DnOOQO'#Do'#DoOOQO'#Dp'#DpOOQO'#Dq'#DqOOQO'#Dr'#DrOOQO'#Ds'#DsOOQO'#Dt'#DtOOQO'#Du'#DuOOQO'#Dv'#DvOOQO'#Dw'#DwOOQO'#Dx'#DxOOQO'#Dy'#DyOOQO'#Dz'#DzOOQO'#D{'#D{OOQO'#D|'#D|OOQO'#D}'#D}OOQO'#EO'#EOOOQO'#EP'#EPOOQO'#EQ'#EQOOQO'#ER'#EROOQO'#ES'#ESOOQO'#ET'#ETOOQO'#EU'#EUOOQO'#EV'#EVOOQO'#EW'#EWOOQO'#EX'#EXOOQO'#EY'#EYOOQO'#EZ'#EZOOQO'#E['#E[OOQO'#E]'#E]OOQO'#E^'#E^OOQO'#E_'#E_OOQO'#E`'#E`OOQO'#Ea'#EaOOQO'#Eb'#EbOOQO'#Ec'#EcOOQO'#Ed'#EdOOQO'#Ee'#EeOOQO'#Ef'#EfOOQO'#Eg'#EgOOQO'#Eh'#EhOOQO'#Ei'#EiOOQO'#Ej'#EjOOQO'#Ek'#EkOOQO'#El'#ElOOQO'#Em'#EmOOQO'#En'#EnOOQO'#Eo'#EoOOQO'#Ep'#EpOOQO'#Eq'#EqOOQO'#Er'#ErOOQO'#Es'#EsOOQO'#Et'#EtOOQO'#Eu'#EuOOQO'#Ev'#EvOOQO'#Ew'#EwOOQO'#Ex'#ExOOQO'#Ey'#EyOOQO'#Ez'#EzQOQPOOO0XQPO'#C|O0^QPO'#DRO'WQPO,59fO0eQQO,59fO2RQPO,59oO2RQPO,59oO2RQPO,59oO2RQPO,59oO2RQPO,59oO7}QQO,5;gO8SQQO,5;jO8[QPO,5;yOOQO,5:O,5:OOOQO,5;i,5;iO8sQQO,5;kO8zQQO,5;nO:bQPO'#FYO:pQPO,5;rOOQO'#FX'#FXOOQO,5;r,5;rOOQO,5;p,5;pO:xQSO'#C}OOQO,59h,59hO;QQPO,59mO;YQQO'#DSOOQO,59m,59mOOQO1G/Q1G/QO0XQPO'#DWOAVQPO'#DVOAaQPO'#DVOYQPO1G/ZOYQPO1G/ZOYQPO1G/ZOYQPO1G/ZOYQPO1G/ZOAkQSO1G1ROOQO1G1U1G1UOAsQQO1G1UOAxQPO'#E}OOQO'#Fa'#FaOOQO1G1e1G1eOBTQPO1G1eOOQO1G1V1G1VOOQO'#FZ'#FZOBYQPO,5;tOB_QSO1G1^OOQO1G1^1G1^OOQO'#DP'#DPOBgQPO,59iOOQO'#DO'#DOOOQO,59i,59iOYQPO,59nOOQO1G/X1G/XOOQO,59r,59rOH_QPO,59qOHfQPO,59qOI}QQO7+$uOJ_QQO7+$uOKsQQO7+$uOLZQQO7+$uOMrQQO7+$uOOQO7+&m7+&mON]QQO7+&sOOQO7+&p7+&pONeQPO7+'POOQO1G1`1G1`OOQO1G1_1G1_OOQO7+&x7+&xONjQSO1G/TOOQO1G/T1G/TONrQQO1G/YOOQO1G/]1G/]ON|QPO1G/]OOQO<<J_<<J_O!&oQPO<<J_OOQO<<Jk<<JkOOQO1G/U1G/UOOQO7+$o7+$oOOQO7+$w7+$wOOQOAN?yAN?y",
stateData: "!&t~O$ZOSkOS~OWQOXQOYQOZQO[QO]QO^QO_QO`QOaQObQOcQO!ZZO#t_O$WVO$XVO$[XO$_`O$`aO$abO$bcO$cdO$deO$efO$fgO$ghO$hiO$ijO$jkO$klO$lmO$mnO$noO$opO$pqO$qrO$rsO$stO$tuO$uvO$vwO$wxO$xyO$yzO$z{O${|O$|}O$}!OO%O!PO%P!QO%Q!RO%R!SO%S!TO%T!UO%U!VO%V!WO%W!XO%X!YO%Y!ZO%Z![O%[!]O%]!^O%^!_O%_!`O%`!aO%a!bO%b!cO%c!dO%d!eO%e!fO%f!gO%g!hO%h!iO%i!jO%j!kO%k!lO%l!mO%m!nO%n!oO%o!pO%p!qO%q!rO%r!sO%uWO%vWO%wVO%y[O~O!ZZO~Od!uOe!uO$[!vO~OU#POV!yOf!|Og!}Oh!|Ox!yO{!yO|!yO}!yO!O!zO!P!zO!Q!{O!R!{O!S!{O!T!{O!U!{O!V!{O$S#QO%s#OO~O$W#SO$X#SO%w#SOW#wXX#wXY#wXZ#wX[#wX]#wX^#wX_#wX`#wXa#wXb#wXc#wX!Z#wX#t#wX$W#wX$X#wX$[#wX$_#wX$`#wX$a#wX$b#wX$c#wX$d#wX$e#wX$f#wX$g#wX$h#wX$i#wX$j#wX$k#wX$l#wX$m#wX$n#wX$o#wX$p#wX$q#wX$r#wX$s#wX$t#wX$u#wX$v#wX$w#wX$x#wX$y#wX$z#wX${#wX$|#wX$}#wX%O#wX%P#wX%Q#wX%R#wX%S#wX%T#wX%U#wX%V#wX%W#wX%X#wX%Y#wX%Z#wX%[#wX%]#wX%^#wX%_#wX%`#wX%a#wX%b#wX%c#wX%d#wX%e#wX%f#wX%g#wX%h#wX%i#wX%j#wX%k#wX%l#wX%m#wX%n#wX%o#wX%p#wX%q#wX%r#wX%u#wX%v#wX%w#wX%y#wX~Ot#VO%z#YO~O%y[OU#xXV#xXf#xXg#xXh#xXx#xX{#xX|#xX}#xX!O#xX!P#xX!Q#xX!R#xX!S#xX!T#xX!U#xX!V#xX$S#xX$V#xX%s#xX$^#xX$]#xX~O$[#[O~O$^#`O~PYOd!uOe!uOUnaVnafnagnahnaxna{na|na}na!Ona!Pna!Qna!Rna!Sna!Tna!Una!Vna$Sna$Vna%sna$^na$]na~OP#dOQ#bOR#bOWyPXyPYyPZyP[yP]yP^yP_yP`yPayPbyPcyP!ZyP#tyP$WyP$XyP$[yP$_yP$`yP$ayP$byP$cyP$dyP$eyP$fyP$gyP$hyP$iyP$jyP$kyP$lyP$myP$nyP$oyP$pyP$qyP$ryP$syP$tyP$uyP$vyP$wyP$xyP$yyP$zyP${yP$|yP$}yP%OyP%PyP%QyP%RyP%SyP%TyP%UyP%VyP%WyP%XyP%YyP%ZyP%[yP%]yP%^yP%_yP%`yP%ayP%byP%cyP%dyP%eyP%fyP%gyP%hyP%iyP%jyP%kyP%lyP%myP%nyP%oyP%pyP%qyP%ryP%uyP%vyP%wyP%yyP~O#p#jO~O!P#lO#p#kO~Oi#nOj#nO$WVO$XVO%u#mO%v#mO%wVO~O$^#qO~P']Ox!yOU#vaV#vaf#vag#vah#va{#va|#va}#va!O#va!P#va!Q#va!R#va!S#va!T#va!U#va!V#va$S#va$V#va%s#va$^#va$]#va~O!V#rO$O#rO$P#rO$Q#rO~O$]#tO%z#uO~Ot#vO$^#yO~O$]#zO$^#{O~O$]vX$^vX~P']OWyXXyXYyXZyX[yX]yX^yX_yX`yXayXbyXcyX!ZyX#tyX$WyX$XyX$[yX$_yX$`yX$ayX$byX$cyX$dyX$eyX$fyX$gyX$hyX$iyX$jyX$kyX$lyX$myX$nyX$oyX$pyX$qyX$ryX$syX$tyX$uyX$vyX$wyX$xyX$yyX$zyX${yX$|yX$}yX%OyX%PyX%QyX%RyX%SyX%TyX%UyX%VyX%WyX%XyX%YyX%ZyX%[yX%]yX%^yX%_yX%`yX%ayX%byX%cyX%dyX%eyX%fyX%gyX%hyX%iyX%jyX%kyX%lyX%myX%nyX%oyX%pyX%qyX%ryX%uyX%vyX%wyX%yyX~OS#}OT#}O~P;dOQ#bOR#bO~P;dO%t$UO%x$VO~O#p$WO~O$W#SO$X#SO%w#SO~O$[$XO~O#t$YO~Ot#VO%z$[O~O$]$]O$^$^O~OWyaXyaYyaZya[ya]ya^ya_ya`yaayabyacya!Zya#tya$Wya$Xya$_ya$`ya$aya$bya$cya$dya$eya$fya$gya$hya$iya$jya$kya$lya$mya$nya$oya$pya$qya$rya$sya$tya$uya$vya$wya$xya$yya$zya${ya$|ya$}ya%Oya%Pya%Qya%Rya%Sya%Tya%Uya%Vya%Wya%Xya%Yya%Zya%[ya%]ya%^ya%_ya%`ya%aya%bya%cya%dya%eya%fya%gya%hya%iya%jya%kya%lya%mya%nya%oya%pya%qya%rya%uya%vya%wya%yya~O$[#[O~PBoOS$aOT$aO$[ya~PBoOx!yOUwqfwqgwqhwq!Owq!Pwq!Qwq!Rwq!Swq!Twq!Uwq!Vwq$Swq$Vwq%swq$^wq$]wq~OVwq{wq|wq}wq~PHsOV!yO{!yO|!yO}!yO~PHsOV!yOx!yO{!yO|!yO}!yO!O!zO!P!zOUwqfwqgwqhwq$Swq$Vwq%swq$^wq$]wq~O!Qwq!Rwq!Swq!Twq!Uwq!Vwq~PJoO!Q!{O!R!{O!S!{O!T!{O!U!{O!V!{O~PJoOV!yOf!|Oh!|Ox!yO{!yO|!yO}!yO!O!zO!P!zO!Q!{O!R!{O!S!{O!T!{O!U!{O!V!{O~OUwqgwq$Swq$Vwq%swq$^wq$]wq~PLqO#p$cO%t$bO~O$^$dO~Ot#vO$^$fO~O$]vi$^vi~P']O$[#[OWyiXyiYyiZyi[yi]yi^yi_yi`yiayibyicyi!Zyi#tyi$Wyi$Xyi$_yi$`yi$ayi$byi$cyi$dyi$eyi$fyi$gyi$hyi$iyi$jyi$kyi$lyi$myi$nyi$oyi$pyi$qyi$ryi$syi$tyi$uyi$vyi$wyi$xyi$yyi$zyi${yi$|yi$}yi%Oyi%Pyi%Qyi%Ryi%Syi%Tyi%Uyi%Vyi%Wyi%Xyi%Yyi%Zyi%[yi%]yi%^yi%_yi%`yi%ayi%byi%cyi%dyi%eyi%fyi%gyi%hyi%iyi%jyi%kyi%lyi%myi%nyi%oyi%pyi%qyi%ryi%uyi%vyi%wyi%yyi~O%t$hO~O",
goto: "(u$UPPPPPPPPPPPPPPPPPPPPPPPPPPPPP$V$u%R%_%e%q%tP%z&T$uP&W&gPPPPPPPPPPP$u&q&}P&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}&}$uP'Z$u$uP$u$u'j$u'v(V(f(i(oPPP$uP(rQSOQ#TXQ#UYQ#_!vQ$P#eQ$Q#fQ$R#gQ$S#hQ$T#iR$_#ze_OXY!v#e#f#g#h#i#zeROXY!v#e#f#g#h#i#zQ!wRR#a!xQ#]!uQ#|#bQ$`#}R$g$aR#w#[Q#x#[R$e$]Q!xRQ#RUR#a!wR#^!vQ#e!yQ#f!zQ#g!{Q#h!|R#i!}Y#c!y!z!{!|!}R$O#deUOXY!v#e#f#g#h#i#zeTOXY!v#e#f#g#h#i#zd_OXY!v#e#f#g#h#i#zR#o#QeYOXY!v#e#f#g#h#i#zd]OXY!v#e#f#g#h#i#zR!tPd^OXY!v#e#f#g#h#i#zR#Z]R#W[Q#X[R$Z#tR#s#VR#p#Q",
nodeNames: "⚠ Bool Ignoring On GroupLeft GroupRight Offset Atan2 Avg Bottomk Count CountValues Group Max Min Quantile Stddev Stdvar Sum Topk By Without And Or Unless Start End LineComment PromQL Expr AggregateExpr AggregateOp AggregateModifier GroupingLabels GroupingLabelList GroupingLabel LabelName FunctionCallBody FunctionCallArgs BinaryExpr Pow BinModifiers OnOrIgnoring Mul Div Mod Add Sub Eql Gte Gtr Lte Lss Neq FunctionCall FunctionIdentifier AbsentOverTime Identifier Absent Abs Acos Acosh Asin Asinh Atan Atanh AvgOverTime Ceil Changes Clamp ClampMax ClampMin Cos Cosh CountOverTime DaysInMonth DayOfMonth DayOfWeek Deg Delta Deriv Exp Floor HistogramQuantile HoltWinters Hour Idelta Increase Irate LabelReplace LabelJoin LastOverTime Ln Log10 Log2 MaxOverTime MinOverTime Minute Month Pi PredictLinear PresentOverTime QuantileOverTime Rad Rate Resets Round Scalar Sgn Sin Sinh Sort SortDesc Sqrt StddevOverTime StdvarOverTime SumOverTime Tan Tanh Timestamp Time Vector Year MatrixSelector Duration NumberLiteral OffsetExpr ParenExpr StringLiteral SubqueryExpr UnaryExpr UnaryOp VectorSelector MetricIdentifier LabelMatchers LabelMatchList LabelMatcher MatchOp EqlSingle EqlRegex NeqRegex StepInvariantExpr At AtModifierPreprocessors MetricName",
maxTerm: 226,
skippedNodes: [0,27],
repeatNodeCount: 0,
tokenData: "1R~RwX^#lpq#lqr$ars$tst%huv%swx%xxy&gyz&lz{&q{|&v|}&}}!O'S!O!P'Z!P!Q(Z!Q!R(`!R![)W![!]-r!^!_.n!_!`.{!`!a/b!b!c/o!c!}/t!}#O0[#P#Q0a#Q#R0f#R#S/t#S#T0k#T#o/t#o#p0w#q#r0|#y#z#l$f$g#l#BY#BZ#l$IS$I_#l$I|$JO#l$JT$JU#l$KV$KW#l&FU&FV#l~#qY$Z~X^#lpq#l#y#z#l$f$g#l#BY#BZ#l$IS$I_#l$I|$JO#l$JT$JU#l$KV$KW#l&FU&FV#l~$dQ!_!`$j#r#s$o~$oO!V~~$tO$Q~~$yU#t~OY$tZr$trs%]s#O$t#O#P%b#P~$t~%bO#t~~%ePO~$t~%mQk~OY%hZ~%h~%xO}~~%}U#t~OY%xZw%xwx%]x#O%x#O#P&a#P~%x~&dPO~%x~&lO$[~~&qO$^~~&vO{~R&}O%vP!OQ~'SO$]~R'ZO%uP!PQP'^P!Q!['aP'fR%wP!Q!['a!g!h'o#X#Y'oP'rR{|'{}!O'{!Q![(RP(OP!Q![(RP(WP%wP!Q![(R~(`O|~R(eZ%wP!O!P'a!Q![)W!g!h'o#W#X){#X#Y'o#[#]*d#a#b*x#g#h+l#k#l+}#l#m-W#m#n,iR)]Y%wP!O!P'a!Q![)W!g!h'o#W#X){#X#Y'o#[#]*d#a#b*x#g#h+l#k#l+}#m#n,iQ*QP#pQ!Q![*TQ*WS!Q![*T#[#]*d#a#b*x#g#h+lQ*iP#pQ!Q![*lQ*oR!Q![*l#a#b*x#g#h+lQ*}Q#pQ!Q![+T#g#h+gQ+WR!Q![+T#a#b+a#g#h+lQ+dP#g#h+gQ+lO#pQQ+qP#pQ!Q![+tQ+wQ!Q![+t#a#b+aQ,SP#pQ!Q![,VQ,YT!Q![,V#W#X){#[#]*d#a#b*x#g#h+lQ,nP#pQ!Q![,qQ,tU!Q![,q#W#X){#[#]*d#a#b*x#g#h+l#k#l+}P-ZR!Q![-d!c!i-d#T#Z-dP-iR%wP!Q![-d!c!i-d#T#Z-dV-yT%xS!ZR!Q![.Y![!].Y!c!}.Y#R#S.Y#T#o.YR._T!ZR!Q![.Y![!].Y!c!}.Y#R#S.Y#T#o.Y~.sP!U~!_!`.v~.{O!T~~/QQ$OP!_!`/W#r#s/]Q/]O!QQ~/bO$P~~/gP!S~!_!`/j~/oO!R~~/tO$S~V/{T!ZRtS!Q![/t![!].Y!c!}/t#R#S/t#T#o/t~0aO%s~~0fO%t~~0kOx~~0nRO#S0k#S#T%]#T~0k~0|O%y~~1RO%z~",
tokenizers: [0, 1, 2],
topRules: {"PromQL":[0,28],"MetricName":[1,144]},
specialized: [{term: 57, get: (value, stack) => (specializeIdentifier(value, stack) << 1)},{term: 57, get: (value, stack) => (extendIdentifier(value, stack) << 1) | 1},{term: 57, get: value => spec_Identifier[value] || -1}],
tokenPrec: 0
})

View file

@ -0,0 +1,148 @@
// This file was generated by lezer-generator. You probably shouldn't edit it.
export const
inf = 146,
nan = 147,
Bool = 1,
Ignoring = 2,
On = 3,
GroupLeft = 4,
GroupRight = 5,
Offset = 6,
Atan2 = 7,
Avg = 8,
Bottomk = 9,
Count = 10,
CountValues = 11,
Group = 12,
Max = 13,
Min = 14,
Quantile = 15,
Stddev = 16,
Stdvar = 17,
Sum = 18,
Topk = 19,
By = 20,
Without = 21,
And = 22,
Or = 23,
Unless = 24,
Start = 25,
End = 26,
LineComment = 27,
PromQL = 28,
Expr = 29,
AggregateExpr = 30,
AggregateOp = 31,
AggregateModifier = 32,
GroupingLabels = 33,
GroupingLabelList = 34,
GroupingLabel = 35,
LabelName = 36,
FunctionCallBody = 37,
FunctionCallArgs = 38,
BinaryExpr = 39,
Pow = 40,
BinModifiers = 41,
OnOrIgnoring = 42,
Mul = 43,
Div = 44,
Mod = 45,
Add = 46,
Sub = 47,
Eql = 48,
Gte = 49,
Gtr = 50,
Lte = 51,
Lss = 52,
Neq = 53,
FunctionCall = 54,
FunctionIdentifier = 55,
AbsentOverTime = 56,
Identifier = 57,
Absent = 58,
Abs = 59,
Acos = 60,
Acosh = 61,
Asin = 62,
Asinh = 63,
Atan = 64,
Atanh = 65,
AvgOverTime = 66,
Ceil = 67,
Changes = 68,
Clamp = 69,
ClampMax = 70,
ClampMin = 71,
Cos = 72,
Cosh = 73,
CountOverTime = 74,
DaysInMonth = 75,
DayOfMonth = 76,
DayOfWeek = 77,
Deg = 78,
Delta = 79,
Deriv = 80,
Exp = 81,
Floor = 82,
HistogramQuantile = 83,
HoltWinters = 84,
Hour = 85,
Idelta = 86,
Increase = 87,
Irate = 88,
LabelReplace = 89,
LabelJoin = 90,
LastOverTime = 91,
Ln = 92,
Log10 = 93,
Log2 = 94,
MaxOverTime = 95,
MinOverTime = 96,
Minute = 97,
Month = 98,
Pi = 99,
PredictLinear = 100,
PresentOverTime = 101,
QuantileOverTime = 102,
Rad = 103,
Rate = 104,
Resets = 105,
Round = 106,
Scalar = 107,
Sgn = 108,
Sin = 109,
Sinh = 110,
Sort = 111,
SortDesc = 112,
Sqrt = 113,
StddevOverTime = 114,
StdvarOverTime = 115,
SumOverTime = 116,
Tan = 117,
Tanh = 118,
Timestamp = 119,
Time = 120,
Vector = 121,
Year = 122,
MatrixSelector = 123,
Duration = 124,
NumberLiteral = 125,
OffsetExpr = 126,
ParenExpr = 127,
StringLiteral = 128,
SubqueryExpr = 129,
UnaryExpr = 130,
UnaryOp = 131,
VectorSelector = 132,
MetricIdentifier = 133,
LabelMatchers = 134,
LabelMatchList = 135,
LabelMatcher = 136,
MatchOp = 137,
EqlSingle = 138,
EqlRegex = 139,
NeqRegex = 140,
StepInvariantExpr = 141,
At = 142,
AtModifierPreprocessors = 143,
MetricName = 144