mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-11-09 23:24:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main' into syncp

Browse source
This commit is contained in:
Ganesh Vernekar 2021-12-09 16:07:02 +05:30
parent cc9ddf50c9 c965a7555b
commit df36efb98a
No known key found for this signature in database
GPG key ID: 0F8729A5EB59B965
62 changed files with 1616 additions and 699 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
CHANGELOG.md
View file

@ -1,3 +1,15 @@
## 2.32.0-rc.0 / 2021-12-01
This list of changes is relative to [v2.32.0-beta.0](https://github.com/prometheus/prometheus/releases/tag/v2.32.0-beta.0), which was a special 2.32 beta release to pre-publish the Prometheus Agent features.
* [FEATURE] TSDB: Add support for forwarding exemplars in Agent mode. #9664
* [FEATURE] UI: Adapt web UI for Prometheus Agent mode. #9851
* [ENHANCEMENT] Linode SD: Tune API request page sizes. #9779
* [BUGFIX] TSDB: Fix panic when checkpoint directory is empty. #9687
* [BUGFIX] TSDB: Fix panic, out of order chunks, and race warning during WAL replay. #9856
* [BUGFIX] UI: Correctly render links for targets with IPv6 addresses that contain a Zone ID. #9853
* [BUGFIX] Promtool: Fix checking of `authorization.credentials_file` and `bearer_token_file` fields. #9883
## 2.32.0-beta.0 / 2021-11-16
This beta release introduces the Prometheus Agent, a new mode of operation for

5
NOTICE
View file

@ -91,6 +91,11 @@ https://github.com/dgryski/go-tsz
Copyright (c) 2015,2016 Damian Gryski <damian@gryski.com>
See https://github.com/dgryski/go-tsz/blob/master/LICENSE for license details.
The Go programming language
https://go.dev/
Copyright (c) 2009 The Go Authors
See https://go.dev/LICENSE for license details.
The Codicon icon font from Microsoft
https://github.com/microsoft/vscode-codicons
Copyright (c) Microsoft Corporation and other contributors

2
VERSION
View file

@ -1 +1 @@
2.32.0-beta.0
2.32.0-rc.0

76
cmd/promtool/main.go
View file

@ -73,6 +73,7 @@ func main() {
"config-files",
"The config files to check.",
).Required().ExistingFiles()
checkConfigSyntaxOnly := checkConfigCmd.Flag("syntax-only", "Only check the config file syntax, ignoring file and content validation referenced in the config").Bool()
checkWebConfigCmd := checkCmd.Command("web-config", "Check if the web config files are valid or not.")
webConfigFiles := checkWebConfigCmd.Arg(
@ -211,7 +212,7 @@ func main() {
os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout))
case checkConfigCmd.FullCommand():
os.Exit(CheckConfig(*agentMode, *configFiles...))
os.Exit(CheckConfig(*agentMode, *checkConfigSyntaxOnly, *configFiles...))
case checkWebConfigCmd.FullCommand():
os.Exit(CheckWebConfig(*webConfigFiles...))
@ -267,16 +268,19 @@ func main() {
}
// CheckConfig validates configuration files.
func CheckConfig(agentMode bool, files ...string) int {
func CheckConfig(agentMode, checkSyntaxOnly bool, files ...string) int {
failed := false
for _, f := range files {
ruleFiles, err := checkConfig(agentMode, f)
ruleFiles, err := checkConfig(agentMode, f, checkSyntaxOnly)
if err != nil {
fmt.Fprintln(os.Stderr, " FAILED:", err)
failed = true
} else {
fmt.Printf(" SUCCESS: %d rule files found\n", len(ruleFiles))
if len(ruleFiles) > 0 {
fmt.Printf(" SUCCESS: %d rule files found\n", len(ruleFiles))
}
fmt.Printf(" SUCCESS: %s is valid prometheus config file syntax\n", f)
}
fmt.Println()
@ -326,7 +330,7 @@ func checkFileExists(fn string) error {
return err
}
func checkConfig(agentMode bool, filename string) ([]string, error) {
func checkConfig(agentMode bool, filename string, checkSyntaxOnly bool) ([]string, error) {
fmt.Println("Checking", filename)
cfg, err := config.LoadFile(filename, agentMode, false, log.NewNopLogger())
@ -335,39 +339,46 @@ func checkConfig(agentMode bool, filename string) ([]string, error) {
}
var ruleFiles []string
for _, rf := range cfg.RuleFiles {
rfs, err := filepath.Glob(rf)
if err != nil {
return nil, err
}
// If an explicit file was given, error if it is not accessible.
if !strings.Contains(rf, "*") {
if len(rfs) == 0 {
return nil, errors.Errorf("%q does not point to an existing file", rf)
if !checkSyntaxOnly {
for _, rf := range cfg.RuleFiles {
rfs, err := filepath.Glob(rf)
if err != nil {
return nil, err
}
if err := checkFileExists(rfs[0]); err != nil {
return nil, errors.Wrapf(err, "error checking rule file %q", rfs[0])
// If an explicit file was given, error if it is not accessible.
if !strings.Contains(rf, "*") {
if len(rfs) == 0 {
return nil, errors.Errorf("%q does not point to an existing file", rf)
}
if err := checkFileExists(rfs[0]); err != nil {
return nil, errors.Wrapf(err, "error checking rule file %q", rfs[0])
}
}
ruleFiles = append(ruleFiles, rfs...)
}
ruleFiles = append(ruleFiles, rfs...)
}
for _, scfg := range cfg.ScrapeConfigs {
if err := checkFileExists(scfg.HTTPClientConfig.BearerTokenFile); err != nil {
return nil, errors.Wrapf(err, "error checking bearer token file %q", scfg.HTTPClientConfig.BearerTokenFile)
if !checkSyntaxOnly && scfg.HTTPClientConfig.Authorization != nil {
if err := checkFileExists(scfg.HTTPClientConfig.Authorization.CredentialsFile); err != nil {
return nil, errors.Wrapf(err, "error checking authorization credentials or bearer token file %q", scfg.HTTPClientConfig.Authorization.CredentialsFile)
}
}
if err := checkTLSConfig(scfg.HTTPClientConfig.TLSConfig); err != nil {
if err := checkTLSConfig(scfg.HTTPClientConfig.TLSConfig, checkSyntaxOnly); err != nil {
return nil, err
}
for _, c := range scfg.ServiceDiscoveryConfigs {
switch c := c.(type) {
case *kubernetes.SDConfig:
if err := checkTLSConfig(c.HTTPClientConfig.TLSConfig); err != nil {
if err := checkTLSConfig(c.HTTPClientConfig.TLSConfig, checkSyntaxOnly); err != nil {
return nil, err
}
case *file.SDConfig:
if checkSyntaxOnly {
break
}
for _, file := range c.Files {
files, err := filepath.Glob(file)
if err != nil {
@ -401,6 +412,9 @@ func checkConfig(agentMode bool, filename string) ([]string, error) {
for _, c := range amcfg.ServiceDiscoveryConfigs {
switch c := c.(type) {
case *file.SDConfig:
if checkSyntaxOnly {
break
}
for _, file := range c.Files {
files, err := filepath.Glob(file)
if err != nil {
@ -432,14 +446,7 @@ func checkConfig(agentMode bool, filename string) ([]string, error) {
return ruleFiles, nil
}
func checkTLSConfig(tlsConfig config_util.TLSConfig) error {
if err := checkFileExists(tlsConfig.CertFile); err != nil {
return errors.Wrapf(err, "error checking client cert file %q", tlsConfig.CertFile)
}
if err := checkFileExists(tlsConfig.KeyFile); err != nil {
return errors.Wrapf(err, "error checking client key file %q", tlsConfig.KeyFile)
}
func checkTLSConfig(tlsConfig config_util.TLSConfig, checkSyntaxOnly bool) error {
if len(tlsConfig.CertFile) > 0 && len(tlsConfig.KeyFile) == 0 {
return errors.Errorf("client cert file %q specified without client key file", tlsConfig.CertFile)
}
@ -447,6 +454,17 @@ func checkTLSConfig(tlsConfig config_util.TLSConfig) error {
return errors.Errorf("client key file %q specified without client cert file", tlsConfig.KeyFile)
}
if checkSyntaxOnly {
return nil
}
if err := checkFileExists(tlsConfig.CertFile); err != nil {
return errors.Wrapf(err, "error checking client cert file %q", tlsConfig.CertFile)
}
if err := checkFileExists(tlsConfig.KeyFile); err != nil {
return errors.Wrapf(err, "error checking client key file %q", tlsConfig.KeyFile)
}
return nil
}

121
cmd/promtool/main_test.go
View file

@ -18,6 +18,8 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"runtime"
"strings"
"testing"
"time"
@ -194,7 +196,7 @@ func TestCheckTargetConfig(t *testing.T) {
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
_, err := checkConfig(false, "testdata/"+test.file)
_, err := checkConfig(false, "testdata/"+test.file, false)
if test.err != "" {
require.Equalf(t, test.err, err.Error(), "Expected error %q, got %q", test.err, err.Error())
return
@ -203,3 +205,120 @@ func TestCheckTargetConfig(t *testing.T) {
})
}
}
func TestCheckConfigSyntax(t *testing.T) {
cases := []struct {
name string
file string
syntaxOnly bool
err string
errWindows string
}{
{
name: "check with syntax only succeeds with nonexistent rule files",
file: "config_with_rule_files.yml",
syntaxOnly: true,
err: "",
errWindows: "",
},
{
name: "check without syntax only fails with nonexistent rule files",
file: "config_with_rule_files.yml",
syntaxOnly: false,
err: "\"testdata/non-existent-file.yml\" does not point to an existing file",
errWindows: "\"testdata\\\\non-existent-file.yml\" does not point to an existing file",
},
{
name: "check with syntax only succeeds with nonexistent service discovery files",
file: "config_with_service_discovery_files.yml",
syntaxOnly: true,
err: "",
errWindows: "",
},
// The test below doesn't fail because the file verification for ServiceDiscoveryConfigs doesn't fail the check if
// file isn't found; it only outputs a warning message.
{
name: "check without syntax only succeeds with nonexistent service discovery files",
file: "config_with_service_discovery_files.yml",
syntaxOnly: false,
err: "",
errWindows: "",
},
{
name: "check with syntax only succeeds with nonexistent TLS files",
file: "config_with_tls_files.yml",
syntaxOnly: true,
err: "",
errWindows: "",
},
{
name: "check without syntax only fails with nonexistent TLS files",
file: "config_with_tls_files.yml",
syntaxOnly: false,
err: "error checking client cert file \"testdata/nonexistent_cert_file.yml\": " +
"stat testdata/nonexistent_cert_file.yml: no such file or directory",
errWindows: "error checking client cert file \"testdata\\\\nonexistent_cert_file.yml\": " +
"CreateFile testdata\\nonexistent_cert_file.yml: The system cannot find the file specified.",
},
{
name: "check with syntax only succeeds with nonexistent credentials file",
file: "authorization_credentials_file.bad.yml",
syntaxOnly: true,
err: "",
errWindows: "",
},
{
name: "check without syntax only fails with nonexistent credentials file",
file: "authorization_credentials_file.bad.yml",
syntaxOnly: false,
err: "error checking authorization credentials or bearer token file \"/random/file/which/does/not/exist.yml\": " +
"stat /random/file/which/does/not/exist.yml: no such file or directory",
errWindows: "error checking authorization credentials or bearer token file \"testdata\\\\random\\\\file\\\\which\\\\does\\\\not\\\\exist.yml\": " +
"CreateFile testdata\\random\\file\\which\\does\\not\\exist.yml: The system cannot find the path specified.",
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
_, err := checkConfig(false, "testdata/"+test.file, test.syntaxOnly)
expectedErrMsg := test.err
if strings.Contains(runtime.GOOS, "windows") {
expectedErrMsg = test.errWindows
}
if expectedErrMsg != "" {
require.Equalf(t, expectedErrMsg, err.Error(), "Expected error %q, got %q", test.err, err.Error())
return
}
require.NoError(t, err)
})
}
}
func TestAuthorizationConfig(t *testing.T) {
cases := []struct {
name string
file string
err string
}{
{
name: "authorization_credentials_file.bad",
file: "authorization_credentials_file.bad.yml",
err: "error checking authorization credentials or bearer token file",
},
{
name: "authorization_credentials_file.good",
file: "authorization_credentials_file.good.yml",
err: "",
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
_, err := checkConfig(false, "testdata/"+test.file, false)
if test.err != "" {
require.Contains(t, err.Error(), test.err, "Expected error to contain %q, got %q", test.err, err.Error())
return
}
require.NoError(t, err)
})
}
}

4
cmd/promtool/testdata/authorization_credentials_file.bad.yml vendored Normal file
View file

@ -0,0 +1,4 @@
scrape_configs:
- job_name: test
authorization:
credentials_file: "/random/file/which/does/not/exist.yml"

4
cmd/promtool/testdata/authorization_credentials_file.good.yml vendored Normal file
View file

@ -0,0 +1,4 @@
scrape_configs:
- job_name: test
authorization:
credentials_file: "."

3
cmd/promtool/testdata/config_with_rule_files.yml vendored Normal file
View file

@ -0,0 +1,3 @@
rule_files:
- non-existent-file.yml
- /etc/non/existent/file.yml

12
cmd/promtool/testdata/config_with_service_discovery_files.yml vendored Normal file
View file

@ -0,0 +1,12 @@
scrape_configs:
- job_name: prometheus
file_sd_configs:
- files:
- nonexistent_file.yml
alerting:
alertmanagers:
- scheme: http
api_version: v1
file_sd_configs:
- files:
- nonexistent_file.yml

5
cmd/promtool/testdata/config_with_tls_files.yml vendored Normal file
View file

@ -0,0 +1,5 @@
scrape_configs:
- job_name: "some job"
tls_config:
cert_file: nonexistent_cert_file.yml
key_file: nonexistent_key_file.yml

7
config/config_test.go
View file

@ -227,7 +227,6 @@ var expectedConf = &Config{
},
},
{
JobName: "service-x",
HonorTimestamps: true,
@ -954,7 +953,7 @@ var expectedConf = &Config{
Scheme: DefaultScrapeConfig.Scheme,
ServiceDiscoveryConfigs: discovery.Configs{
&uyuni.SDConfig{
Server: kubernetesSDHostURL(),
Server: "https://localhost:1234",
Username: "gopher",
Password: "hole",
Entitlement: "monitoring_entitled",
@ -1434,6 +1433,10 @@ var expectedErrors = []struct {
filename: "empty_scrape_config_action.bad.yml",
errMsg: "relabel action cannot be empty",
},
{
filename: "uyuni_no_server.bad.yml",
errMsg: "Uyuni SD configuration requires server host",
},
}
func TestBadConfigs(t *testing.T) {

4
config/testdata/uyuni_no_server.bad.yml vendored Normal file
View file

@ -0,0 +1,4 @@
scrape_configs:
- job_name: uyuni
uyuni_sd_configs:
- server:

2
discovery/kubernetes/node.go
View file

@ -149,6 +149,7 @@ func nodeSourceFromName(name string) string {
const (
nodeNameLabel = metaLabelPrefix + "node_name"
nodeProviderIDLabel = metaLabelPrefix + "node_provider_id"
nodeLabelPrefix = metaLabelPrefix + "node_label_"
nodeLabelPresentPrefix = metaLabelPrefix + "node_labelpresent_"
nodeAnnotationPrefix = metaLabelPrefix + "node_annotation_"
@ -161,6 +162,7 @@ func nodeLabels(n *apiv1.Node) model.LabelSet {
ls := make(model.LabelSet, 2*(len(n.Labels)+len(n.Annotations))+1)
ls[nodeNameLabel] = lv(n.Name)
ls[nodeProviderIDLabel] = lv(n.Spec.ProviderID)
for k, v := range n.Labels {
ln := strutil.SanitizeLabelName(k)

14
discovery/kubernetes/node_test.go
View file

@ -25,13 +25,16 @@ import (
"github.com/prometheus/prometheus/discovery/targetgroup"
)
func makeNode(name, address string, labels, annotations map[string]string) *v1.Node {
func makeNode(name, address, providerID string, labels, annotations map[string]string) *v1.Node {
return &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: labels,
Annotations: annotations,
},
Spec: v1.NodeSpec{
ProviderID: providerID,
},
Status: v1.NodeStatus{
Addresses: []v1.NodeAddress{
{
@ -49,7 +52,7 @@ func makeNode(name, address string, labels, annotations map[string]string) *v1.N
}
func makeEnumeratedNode(i int) *v1.Node {
return makeNode(fmt.Sprintf("test%d", i), "1.2.3.4", map[string]string{}, map[string]string{})
return makeNode(fmt.Sprintf("test%d", i), "1.2.3.4", fmt.Sprintf("aws:///de-west-3a/i-%d", i), map[string]string{}, map[string]string{})
}
func TestNodeDiscoveryBeforeStart(t *testing.T) {
@ -61,6 +64,7 @@ func TestNodeDiscoveryBeforeStart(t *testing.T) {
obj := makeNode(
"test",
"1.2.3.4",
"aws:///nl-north-7b/i-03149834983492827",
map[string]string{"test-label": "testvalue"},
map[string]string{"test-annotation": "testannotationvalue"},
)
@ -78,6 +82,7 @@ func TestNodeDiscoveryBeforeStart(t *testing.T) {
},
Labels: model.LabelSet{
"__meta_kubernetes_node_name": "test",
"__meta_kubernetes_node_provider_id": "aws:///nl-north-7b/i-03149834983492827",
"__meta_kubernetes_node_label_test_label": "testvalue",
"__meta_kubernetes_node_labelpresent_test_label": "true",
"__meta_kubernetes_node_annotation_test_annotation": "testannotationvalue",
@ -109,7 +114,8 @@ func TestNodeDiscoveryAdd(t *testing.T) {
},
},
Labels: model.LabelSet{
"__meta_kubernetes_node_name": "test1",
"__meta_kubernetes_node_name": "test1",
"__meta_kubernetes_node_provider_id": "aws:///de-west-3a/i-1",
},
Source: "node/test1",
},
@ -146,6 +152,7 @@ func TestNodeDiscoveryUpdate(t *testing.T) {
obj2 := makeNode(
"test0",
"1.2.3.4",
"aws:///fr-south-1c/i-49508290343823952",
map[string]string{"Unschedulable": "true"},
map[string]string{},
)
@ -165,6 +172,7 @@ func TestNodeDiscoveryUpdate(t *testing.T) {
"__meta_kubernetes_node_label_Unschedulable": "true",
"__meta_kubernetes_node_labelpresent_Unschedulable": "true",
"__meta_kubernetes_node_name": "test0",
"__meta_kubernetes_node_provider_id": "aws:///fr-south-1c/i-49508290343823952",
},
Source: "node/test0",
},

12
discovery/uyuni/uyuni.go
View file

@ -62,7 +62,7 @@ func init() {
// SDConfig is the configuration for Uyuni based service discovery.
type SDConfig struct {
Server config.URL `yaml:"server"`
Server string `yaml:"server"`
Username string `yaml:"username"`
Password config.Secret `yaml:"password"`
HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
@ -122,11 +122,11 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
if err != nil {
return err
}
if c.Server.URL == nil {
if c.Server == "" {
return errors.New("Uyuni SD configuration requires server host")
}
_, err = url.Parse(c.Server.String())
_, err = url.Parse(c.Server)
if err != nil {
return errors.Wrap(err, "Uyuni Server URL is not valid")
}
@ -199,8 +199,10 @@ func getEndpointInfoForSystems(
// NewDiscovery returns a uyuni discovery for the given configuration.
func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
var apiURL *url.URL
*apiURL = *conf.Server.URL
apiURL, err := url.Parse(conf.Server)
if err != nil {
return nil, err
}
apiURL.Path = path.Join(apiURL.Path, uyuniXMLRPCAPIPath)
rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "uyuni_sd")

58
discovery/uyuni/uyuni_test.go Normal file
View file

@ -0,0 +1,58 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package uyuni
import (
"context"
"io"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/discovery/targetgroup"
)
func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, error) {
// Create a test server with mock HTTP handler.
ts := httptest.NewServer(respHandler)
defer ts.Close()
conf := SDConfig{
Server: ts.URL,
}
md, err := NewDiscovery(&conf, nil)
if err != nil {
return nil, err
}
return md.refresh(context.Background())
}
func TestUyuniSDHandleError(t *testing.T) {
var (
errTesting = "unable to login to Uyuni API: request error: bad status code - 500"
respHandler = func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, ``)
}
)
tgs, err := testUpdateServices(respHandler)
require.EqualError(t, err, errTesting)
require.Equal(t, len(tgs), 0)
}

15
docs/configuration/configuration.md
View file

@ -95,6 +95,10 @@ remote_write:
# Settings related to the remote read feature.
remote_read:
[ - <remote_read> ... ]
# Storage related settings that are runtime reloadable.
storage:
[ - <exemplars> ... ]
```
### `<scrape_config>`
@ -1509,6 +1513,7 @@ node object in the address type order of `NodeInternalIP`, `NodeExternalIP`,
Available meta labels:
* `__meta_kubernetes_node_name`: The name of the node object.
* `__meta_kubernetes_node_provider_id`: The cloud provider's name for the node object.
* `__meta_kubernetes_node_label_<labelname>`: Each label from the node object.
* `__meta_kubernetes_node_labelpresent_<labelname>`: `true` for each label from the node object.
* `__meta_kubernetes_node_annotation_<annotationname>`: Each annotation from the node object.
@ -1597,6 +1602,7 @@ address referenced in the endpointslice object one target is discovered. If the
additional container ports of the pod, not bound to an endpoint port, are discovered as targets as well.
Available meta labels:
* `__meta_kubernetes_namespace`: The namespace of the endpoints object.
* `__meta_kubernetes_endpointslice_name`: The name of endpointslice object.
* For all targets discovered directly from the endpointslice list (those not additionally inferred
@ -2839,3 +2845,12 @@ tls_config:
There is a list of
[integrations](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage)
with this feature.
### `<exemplars>`
Note that exemplar storage is still considered experimental and must be enabled via `--enable-feature=exemplar-storage`.
```yaml
# Configures the maximum size of the circular buffer used to store exemplars for all series. Resizable during runtime.
[ max_exemplars: <int> | default = 100000 ]
```

2
docs/feature_flags.md
View file

@ -52,7 +52,7 @@ The remote write receiver allows Prometheus to accept remote write requests from
[OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars) introduces the ability for scrape targets to add exemplars to certain metrics. Exemplars are references to data outside of the MetricSet. A common use case are IDs of program traces.
Exemplar storage is implemented as a fixed size circular buffer that stores exemplars in memory for all series. Enabling this feature will enable the storage of exemplars scraped by Prometheus. The flag `storage.exemplars.exemplars-limit` can be used to control the size of circular buffer by # of exemplars. An exemplar with just a `traceID=<jaeger-trace-id>` uses roughly 100 bytes of memory via the in-memory exemplar storage. If the exemplar storage is enabled, we will also append the exemplars to WAL for local persistence (for WAL duration).
Exemplar storage is implemented as a fixed size circular buffer that stores exemplars in memory for all series. Enabling this feature will enable the storage of exemplars scraped by Prometheus. The config file block [storage](configuration/configuration.md#configuration-file)/[exemplars](configuration/configuration.md#exemplars) can be used to control the size of circular buffer by # of exemplars. An exemplar with just a `traceID=<jaeger-trace-id>` uses roughly 100 bytes of memory via the in-memory exemplar storage. If the exemplar storage is enabled, we will also append the exemplars to WAL for local persistence (for WAL duration).
## Memory snapshot on shutdown

24
go.mod
View file

@ -3,18 +3,18 @@ module github.com/prometheus/prometheus
go 1.14
require (
github.com/Azure/azure-sdk-for-go v58.3.0+incompatible
github.com/Azure/azure-sdk-for-go v59.4.0+incompatible
github.com/Azure/go-autorest/autorest v0.11.22
github.com/Azure/go-autorest/autorest/adal v0.9.17
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a
github.com/aws/aws-sdk-go v1.42.10
github.com/aws/aws-sdk-go v1.42.20
github.com/cespare/xxhash/v2 v2.1.2
github.com/containerd/containerd v1.5.7 // indirect
github.com/dennwc/varint v1.0.0
github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245
github.com/digitalocean/godo v1.71.0
github.com/digitalocean/godo v1.73.0
github.com/docker/docker v20.10.11+incompatible
github.com/docker/go-connections v0.4.0 // indirect
github.com/edsrzf/mmap-go v1.0.0
@ -23,11 +23,11 @@ require (
github.com/fsnotify/fsnotify v1.5.1
github.com/go-kit/log v0.2.0
github.com/go-logfmt/logfmt v0.5.1
github.com/go-openapi/strfmt v0.21.0
github.com/go-openapi/strfmt v0.21.1
github.com/go-zookeeper/zk v1.0.2
github.com/gogo/protobuf v1.3.2
github.com/golang/snappy v0.0.4
github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0
github.com/google/pprof v0.0.0-20211122183932-1daafda22083
github.com/gophercloud/gophercloud v0.23.0
github.com/grpc-ecosystem/grpc-gateway v1.16.0
github.com/hashicorp/consul/api v1.11.0
@ -59,14 +59,14 @@ require (
github.com/uber/jaeger-lib v2.4.1+incompatible
go.uber.org/atomic v1.9.0
go.uber.org/goleak v1.1.12
golang.org/x/net v0.0.0-20211020060615-d418f374d309
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
golang.org/x/tools v0.1.7
google.golang.org/api v0.60.0
google.golang.org/genproto v0.0.0-20211021150943-2b146023228c
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
golang.org/x/tools v0.1.8
google.golang.org/api v0.61.0
google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12
google.golang.org/protobuf v1.27.1
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/yaml.v2 v2.4.0

61
go.sum
View file

@ -51,8 +51,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/azure-sdk-for-go v41.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/azure-sdk-for-go v58.3.0+incompatible h1:lb9OWePNuJMiibdxg9XvdbiOldR0Yifge37L4LoOxIs=
github.com/Azure/azure-sdk-for-go v58.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/azure-sdk-for-go v59.4.0+incompatible h1:gDA8odnngdNd3KYHL2NoK1j9vpWBgEnFSjKKLpkC8Aw=
github.com/Azure/azure-sdk-for-go v59.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
@ -187,8 +187,8 @@ github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZve
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go v1.42.10 h1:PW9G/hnsuKttbFtOcgNKD0vQrp4yfNrtACA+X0p9mjM=
github.com/aws/aws-sdk-go v1.42.10/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go v1.42.20 h1:nQkkmTWK5N2Ao1iVzoOx1HTIxwbSWErxyZ1eiwLJWc4=
github.com/aws/aws-sdk-go v1.42.20/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps=
@ -372,8 +372,8 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245 h1:9cOfvEwjQxdwKuNDTQSaMKNRvwKwgZG+U4HrjeRKHso=
github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/digitalocean/godo v1.71.0 h1:a4UZCG1kr8eQ3MmsGoPzcAwkEtJG2Lc7eelzEkfZwtA=
github.com/digitalocean/godo v1.71.0/go.mod h1:GBmu8MkjZmNARE7IXRPmkbbnocNN8+uBm0xbEVw2LCs=
github.com/digitalocean/godo v1.73.0 h1:VEPb2YIgvbG5WP9+2Yin6ty+1s01mTUrSEW4CO6alVc=
github.com/digitalocean/godo v1.73.0/go.mod h1:GBmu8MkjZmNARE7IXRPmkbbnocNN8+uBm0xbEVw2LCs=
github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY=
github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
@ -547,8 +547,8 @@ github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk
github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
github.com/go-openapi/strfmt v0.20.1/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk=
github.com/go-openapi/strfmt v0.21.0 h1:hX2qEZKmYks+t0hKeb4VTJpUm2UYsdL3+DCid5swxIs=
github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg=
github.com/go-openapi/strfmt v0.21.1 h1:G6s2t5V5kGCHLVbSdZ/6lI8Wm4OzoPFkc3/cjAsKQrM=
github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k=
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
@ -718,8 +718,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0 h1:zHs+jv3LO743/zFGcByu2KmpbliCU2AhjcGgrdTwSG4=
github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
github.com/google/pprof v0.0.0-20211122183932-1daafda22083 h1:c8EUapQFi+kjzedr4c6WqbwMdmB95+oDBWZ5XFHFYxY=
github.com/google/pprof v0.0.0-20211122183932-1daafda22083/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -1332,7 +1332,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
@ -1351,8 +1351,8 @@ go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4S
go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw=
go.mongodb.org/mongo-driver v1.7.3 h1:G4l/eYY9VrQAK/AUgkV0koQKzQnyddnWxrd/Etf0jIs=
go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg=
go.mongodb.org/mongo-driver v1.7.5 h1:ny3p0reEpgsR2cfA5cjgwFZg3Cv/ofFh/8jbhGtz9VI=
go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@ -1460,8 +1460,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -1526,10 +1527,10 @@ golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 h1:0qxwC5n+ttVOINCBeRHO0nq9X7uy8SDsPoi5OaCdIEI=
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1545,8 +1546,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ
golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 h1:B333XXssMuKQeBwiNODx4TupZy7bf4sxFZnN2ZOcvUE=
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg=
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -1666,13 +1667,13 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
@ -1696,8 +1697,9 @@ golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M=
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -1775,8 +1777,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1820,8 +1822,8 @@ google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6
google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
google.golang.org/api v0.60.0 h1:eq/zs5WPH4J9undYM9IP1O7dSr7Yh8Y0GtSCpzGzIUk=
google.golang.org/api v0.60.0/go.mod h1:d7rl65NZAkEQ90JFzqBjcRq1TVeG5ZoGV3sSpEnnVb4=
google.golang.org/api v0.61.0 h1:TXXKS1slM3b2bZNJwD5DV/Tp6/M2cLzLOLh9PjDhrw8=
google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -1895,8 +1897,9 @@ google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEc
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211021150943-2b146023228c h1:FqrtZMB5Wr+/RecOM3uPJNPfWR8Upb5hAPnt7PU6i4k=
google.golang.org/genproto v0.0.0-20211021150943-2b146023228c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12 h1:DN5b3HU13J4sMd/QjDx34U6afpaexKTDdop+26pdjdk=
google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=

4
promql/engine.go
View file

@ -1643,7 +1643,7 @@ func (ev *evaluator) vectorSelectorSingle(it *storage.MemoizedSeriesIterator, no
}
if ok {
t, v = it.Values()
t, v = it.At()
}
if !ok || t > refTime {
@ -1763,7 +1763,7 @@ func (ev *evaluator) matrixIterSlice(it *storage.BufferedSeriesIterator, mint, m
}
// The seeked sample might also be in the range.
if ok {
t, v := it.Values()
t, v := it.At()
if t == maxt && !value.IsStaleNaN(v) {
if ev.currentSamples >= ev.maxSamples {
ev.error(ErrTooManySamples(env))

14
promql/engine_test.go
View file

@ -58,10 +58,19 @@ func TestQueryConcurrency(t *testing.T) {
block := make(chan struct{})
processing := make(chan struct{})
done := make(chan int)
defer close(done)
f := func(context.Context) error {
processing <- struct{}{}
<-block
select {
case processing <- struct{}{}:
case <-done:
}
select {
case <-block:
case <-done:
}
return nil
}
@ -370,7 +379,6 @@ func TestSelectHintsSetCorrectly(t *testing.T) {
{Start: -45000, End: 80000, Func: "count_over_time"},
},
}, {
query: "foo", start: 10000, end: 20000,
expected: []*storage.SelectHints{
{Start: 5000, End: 20000, Step: 1000},

8
promql/value.go
View file

@ -88,6 +88,14 @@ func (p Point) String() string {
}
// MarshalJSON implements json.Marshaler.
//
// JSON marshaling is only needed for the HTTP API. Since Point is such a
// frequently marshaled type, it gets an optimized treatment directly in
// web/api/v1/api.go. Therefore, this method is unused within Prometheus. It is
// still provided here as convenience for debugging and for other users of this
// code. Also note that the different marshaling implementations might lead to
// slightly different results in terms of formatting and rounding of the
// timestamp.
func (p Point) MarshalJSON() ([]byte, error) {
v := strconv.FormatFloat(p.V, 'f', -1, 64)
return json.Marshal([...]interface{}{float64(p.T) / 1000, v})

10
storage/buffer.go
View file

@ -82,14 +82,14 @@ func (b *BufferedSeriesIterator) Seek(t int64) bool {
// If the delta would cause us to seek backwards, preserve the buffer
// and just continue regular advancement while filling the buffer on the way.
if t0 > b.lastTime {
if b.ok && t0 > b.lastTime {
b.buf.reset()
b.ok = b.it.Seek(t0)
if !b.ok {
return false
}
b.lastTime, _ = b.Values()
b.lastTime, _ = b.At()
}
if b.lastTime >= t {
@ -115,14 +115,14 @@ func (b *BufferedSeriesIterator) Next() bool {
b.ok = b.it.Next()
if b.ok {
b.lastTime, _ = b.Values()
b.lastTime, _ = b.At()
}
return b.ok
}
// Values returns the current element of the iterator.
func (b *BufferedSeriesIterator) Values() (int64, float64) {
// At returns the current element of the iterator.
func (b *BufferedSeriesIterator) At() (int64, float64) {
return b.it.At()
}

3
storage/buffer_test.go
View file

@ -99,7 +99,7 @@ func TestBufferedSeriesIterator(t *testing.T) {
require.Equal(t, exp, b, "buffer mismatch")
}
sampleEq := func(ets int64, ev float64) {
ts, v := it.Values()
ts, v := it.At()
require.Equal(t, ets, ts, "timestamp mismatch")
require.Equal(t, ev, v, "value mismatch")
}
@ -149,6 +149,7 @@ func TestBufferedSeriesIterator(t *testing.T) {
bufferEq([]sample{{t: 99, v: 8}, {t: 100, v: 9}})
require.False(t, it.Next(), "next succeeded unexpectedly")
require.False(t, it.Seek(1024), "seek succeeded unexpectedly")
}
// At() should not be called once Next() returns false.

6
storage/memoized_iterator.go
View file

@ -71,7 +71,7 @@ func (b *MemoizedSeriesIterator) PeekPrev() (t int64, v float64, ok bool) {
func (b *MemoizedSeriesIterator) Seek(t int64) bool {
t0 := t - b.delta
if t0 > b.lastTime {
if b.ok && t0 > b.lastTime {
// Reset the previously stored element because the seek advanced
// more than the delta.
b.prevTime = math.MinInt64
@ -112,8 +112,8 @@ func (b *MemoizedSeriesIterator) Next() bool {
return b.ok
}
// Values returns the current element of the iterator.
func (b *MemoizedSeriesIterator) Values() (int64, float64) {
// At returns the current element of the iterator.
func (b *MemoizedSeriesIterator) At() (int64, float64) {
return b.it.At()
}

3
storage/memoized_iterator_test.go
View file

@ -23,7 +23,7 @@ func TestMemoizedSeriesIterator(t *testing.T) {
var it *MemoizedSeriesIterator
sampleEq := func(ets int64, ev float64) {
ts, v := it.Values()
ts, v := it.At()
require.Equal(t, ets, ts, "timestamp mismatch")
require.Equal(t, ev, v, "value mismatch")
}
@ -68,6 +68,7 @@ func TestMemoizedSeriesIterator(t *testing.T) {
prevSampleEq(100, 9, true)
require.False(t, it.Next(), "next succeeded unexpectedly")
require.False(t, it.Seek(1024), "seek succeeded unexpectedly")
}
func BenchmarkMemoizedSeriesIterator(b *testing.B) {

5
storage/merge.go
View file

@ -457,6 +457,11 @@ func NewChainSampleIterator(iterators []chunkenc.Iterator) chunkenc.Iterator {
}
func (c *chainSampleIterator) Seek(t int64) bool {
// No-op check
if c.curr != nil && c.lastt >= t {
return true
}
c.h = samplesIteratorHeap{}
for _, iter := range c.iterators {
if iter.Seek(t) {

12
storage/remote/codec.go
View file

@ -357,8 +357,16 @@ func newConcreteSeriersIterator(series *concreteSeries) chunkenc.Iterator {
// Seek implements storage.SeriesIterator.
func (c *concreteSeriesIterator) Seek(t int64) bool {
c.cur = sort.Search(len(c.series.samples), func(n int) bool {
return c.series.samples[n].Timestamp >= t
if c.cur == -1 {
c.cur = 0
}
// No-op check.
if s := c.series.samples[c.cur]; s.Timestamp >= t {
return true
}
// Do binary search between current position and end.
c.cur += sort.Search(len(c.series.samples)-c.cur, func(n int) bool {
return c.series.samples[n+c.cur].Timestamp >= t
})
return c.cur < len(c.series.samples)
}

44
storage/remote/codec_test.go
View file

@ -191,6 +191,50 @@ func TestConcreteSeriesClonesLabels(t *testing.T) {
require.Equal(t, lbls, gotLabels)
}
func TestConcreteSeriesIterator(t *testing.T) {
series := &concreteSeries{
labels: labels.FromStrings("foo", "bar"),
samples: []prompb.Sample{
{Value: 1, Timestamp: 1},
{Value: 1.5, Timestamp: 1},
{Value: 2, Timestamp: 2},
{Value: 3, Timestamp: 3},
{Value: 4, Timestamp: 4},
},
}
it := series.Iterator()
// Seek to the first sample with ts=1.
require.True(t, it.Seek(1))
ts, v := it.At()
require.Equal(t, int64(1), ts)
require.Equal(t, 1., v)
// Seek one further, next sample still has ts=1.
require.True(t, it.Next())
ts, v = it.At()
require.Equal(t, int64(1), ts)
require.Equal(t, 1.5, v)
// Seek again to 1 and make sure we stay where we are.
require.True(t, it.Seek(1))
ts, v = it.At()
require.Equal(t, int64(1), ts)
require.Equal(t, 1.5, v)
// Another seek.
require.True(t, it.Seek(3))
ts, v = it.At()
require.Equal(t, int64(3), ts)
require.Equal(t, 3., v)
// And we don't go back.
require.True(t, it.Seek(2))
ts, v = it.At()
require.Equal(t, int64(3), ts)
require.Equal(t, 3., v)
}
func TestFromQueryResultWithDuplicates(t *testing.T) {
ts1 := prompb.TimeSeries{
Labels: []prompb.Label{

213
storage/remote/queue_manager.go
View file

@ -915,7 +915,7 @@ type shards struct {
mtx sync.RWMutex // With the WAL, this is never actually contended.
qm *QueueManager
queues []chan interface{}
queues []*queue
// So we can accurately track how many of each are lost during shard shutdowns.
enqueuedSamples atomic.Int64
enqueuedExemplars atomic.Int64
@ -943,9 +943,9 @@ func (s *shards) start(n int) {
s.qm.metrics.pendingSamples.Set(0)
s.qm.metrics.numShards.Set(float64(n))
newQueues := make([]chan interface{}, n)
newQueues := make([]*queue, n)
for i := 0; i < n; i++ {
newQueues[i] = make(chan interface{}, s.qm.cfg.Capacity)
newQueues[i] = newQueue(s.qm.cfg.MaxSamplesPerSend, s.qm.cfg.Capacity)
}
s.queues = newQueues
@ -978,7 +978,7 @@ func (s *shards) stop() {
s.mtx.Lock()
defer s.mtx.Unlock()
for _, queue := range s.queues {
close(queue)
go queue.FlushAndShutdown(s.done)
}
select {
case <-s.done:
@ -1013,7 +1013,11 @@ func (s *shards) enqueue(ref chunks.HeadSeriesRef, data interface{}) bool {
select {
case <-s.softShutdown:
return false
case s.queues[shard] <- data:
default:
appended := s.queues[shard].Append(data, s.softShutdown)
if !appended {
return false
}
switch data.(type) {
case writeSample:
s.qm.metrics.pendingSamples.Inc()
@ -1028,7 +1032,93 @@ func (s *shards) enqueue(ref chunks.HeadSeriesRef, data interface{}) bool {
}
}
func (s *shards) runShard(ctx context.Context, shardID int, queue chan interface{}) {
type queue struct {
batch []interface{}
batchQueue chan []interface{}
// Since we know there are a limited number of batches out, using a stack
// is easy and safe so a sync.Pool is not necessary.
batchPool [][]interface{}
// This mutex covers adding and removing batches from the batchPool.
poolMux sync.Mutex
}
func newQueue(batchSize, capacity int) *queue {
batches := capacity / batchSize
return &queue{
batch: make([]interface{}, 0, batchSize),
batchQueue: make(chan []interface{}, batches),
// batchPool should have capacity for everything in the channel + 1 for
// the batch being processed.
batchPool: make([][]interface{}, 0, batches+1),
}
}
func (q *queue) Append(datum interface{}, stop <-chan struct{}) bool {
q.batch = append(q.batch, datum)
if len(q.batch) == cap(q.batch) {
select {
case q.batchQueue <- q.batch:
q.batch = q.newBatch(cap(q.batch))
return true
case <-stop:
// Remove the sample we just appended. It will get retried.
q.batch = q.batch[:len(q.batch)-1]
return false
}
}
return true
}
func (q *queue) Chan() <-chan []interface{} {
return q.batchQueue
}
// Batch returns the current batch and allocates a new batch. Must not be
// called concurrently with Append.
func (q *queue) Batch() []interface{} {
batch := q.batch
q.batch = q.newBatch(cap(batch))
return batch
}
// ReturnForReuse adds the batch buffer back to the internal pool.
func (q *queue) ReturnForReuse(batch []interface{}) {
q.poolMux.Lock()
defer q.poolMux.Unlock()
if len(q.batchPool) < cap(q.batchPool) {
q.batchPool = append(q.batchPool, batch[:0])
}
}
// FlushAndShutdown stops the queue and flushes any samples. No appends can be
// made after this is called.
func (q *queue) FlushAndShutdown(done <-chan struct{}) {
if len(q.batch) > 0 {
select {
case q.batchQueue <- q.batch:
case <-done:
// The shard has been hard shut down, so no more samples can be
// sent. Drop everything left in the queue.
}
}
q.batch = nil
close(q.batchQueue)
}
func (q *queue) newBatch(capacity int) []interface{} {
q.poolMux.Lock()
defer q.poolMux.Unlock()
batches := len(q.batchPool)
if batches > 0 {
batch := q.batchPool[batches-1]
q.batchPool = q.batchPool[:batches-1]
return batch
}
return make([]interface{}, 0, capacity)
}
func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) {
defer func() {
if s.running.Dec() == 0 {
close(s.done)
@ -1040,8 +1130,7 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue chan interface
// Send batches of at most MaxSamplesPerSend samples to the remote storage.
// If we have fewer samples than that, flush them out after a deadline anyways.
var (
max = s.qm.cfg.MaxSamplesPerSend
nPending, nPendingSamples, nPendingExemplars = 0, 0, 0
max = s.qm.cfg.MaxSamplesPerSend
pBuf = proto.NewBuffer(nil)
buf []byte
@ -1050,6 +1139,7 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue chan interface
max += int(float64(max) * 0.1)
}
batchQueue := queue.Chan()
pendingData := make([]prompb.TimeSeries, max)
for i := range pendingData {
pendingData[i].Samples = []prompb.Sample{{}}
@ -1074,8 +1164,8 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue chan interface
case <-ctx.Done():
// In this case we drop all samples in the buffer and the queue.
// Remove them from pending and mark them as failed.
droppedSamples := nPendingSamples + int(s.enqueuedSamples.Load())
droppedExemplars := nPendingExemplars + int(s.enqueuedExemplars.Load())
droppedSamples := int(s.enqueuedSamples.Load())
droppedExemplars := int(s.enqueuedExemplars.Load())
s.qm.metrics.pendingSamples.Sub(float64(droppedSamples))
s.qm.metrics.pendingExemplars.Sub(float64(droppedExemplars))
s.qm.metrics.failedSamplesTotal.Add(float64(droppedSamples))
@ -1084,66 +1174,67 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue chan interface
s.exemplarsDroppedOnHardShutdown.Add(uint32(droppedExemplars))
return
case sample, ok := <-queue:
case batch, ok := <-batchQueue:
if !ok {
if nPendingSamples > 0 || nPendingExemplars > 0 {
level.Debug(s.qm.logger).Log("msg", "Flushing data to remote storage...", "samples", nPendingSamples, "exemplars", nPendingExemplars)
s.sendSamples(ctx, pendingData[:nPending], nPendingSamples, nPendingExemplars, pBuf, &buf)
s.qm.metrics.pendingSamples.Sub(float64(nPendingSamples))
s.qm.metrics.pendingExemplars.Sub(float64(nPendingExemplars))
level.Debug(s.qm.logger).Log("msg", "Done flushing.")
}
return
}
nPendingSamples, nPendingExemplars := s.populateTimeSeries(batch, pendingData)
queue.ReturnForReuse(batch)
n := nPendingSamples + nPendingExemplars
s.sendSamples(ctx, pendingData[:n], nPendingSamples, nPendingExemplars, pBuf, &buf)
pendingData[nPending].Samples = pendingData[nPending].Samples[:0]
if s.qm.sendExemplars {
pendingData[nPending].Exemplars = pendingData[nPending].Exemplars[:0]
}
// Number of pending samples is limited by the fact that sendSamples (via sendSamplesWithBackoff)
// retries endlessly, so once we reach max samples, if we can never send to the endpoint we'll
// stop reading from the queue. This makes it safe to reference pendingSamples by index.
switch d := sample.(type) {
case writeSample:
pendingData[nPending].Labels = labelsToLabelsProto(d.seriesLabels, pendingData[nPending].Labels)
pendingData[nPending].Samples = append(pendingData[nPending].Samples, d.sample)
nPendingSamples++
nPending++
case writeExemplar:
pendingData[nPending].Labels = labelsToLabelsProto(d.seriesLabels, pendingData[nPending].Labels)
pendingData[nPending].Exemplars = append(pendingData[nPending].Exemplars, d.exemplar)
nPendingExemplars++
nPending++
}
if nPending >= max {
s.sendSamples(ctx, pendingData[:nPending], nPendingSamples, nPendingExemplars, pBuf, &buf)
s.qm.metrics.pendingSamples.Sub(float64(nPendingSamples))
s.qm.metrics.pendingExemplars.Sub(float64(nPendingExemplars))
nPendingSamples = 0
nPendingExemplars = 0
nPending = 0
stop()
timer.Reset(time.Duration(s.qm.cfg.BatchSendDeadline))
}
stop()
timer.Reset(time.Duration(s.qm.cfg.BatchSendDeadline))
case <-timer.C:
if nPendingSamples > 0 || nPendingExemplars > 0 {
level.Debug(s.qm.logger).Log("msg", "runShard timer ticked, sending buffered data", "samples", nPendingSamples, "exemplars", nPendingExemplars, "shard", shardNum)
s.sendSamples(ctx, pendingData[:nPending], nPendingSamples, nPendingExemplars, pBuf, &buf)
s.qm.metrics.pendingSamples.Sub(float64(nPendingSamples))
s.qm.metrics.pendingExemplars.Sub(float64(nPendingExemplars))
nPendingSamples = 0
nPendingExemplars = 0
nPending = 0
// We need to take the lock when getting a batch to avoid
// concurrent Appends. Generally this will only happen on low
// traffic instances.
s.mtx.Lock()
// First, we need to see if we can happen to get a batch from the queue if it filled while acquiring the lock.
var batch []interface{}
select {
case batch = <-batchQueue:
default:
batch = queue.Batch()
}
s.mtx.Unlock()
if len(batch) > 0 {
nPendingSamples, nPendingExemplars := s.populateTimeSeries(batch, pendingData)
n := nPendingSamples + nPendingExemplars
level.Debug(s.qm.logger).Log("msg", "runShard timer ticked, sending buffered data", "samples", nPendingSamples, "exemplars", nPendingExemplars, "shard", shardNum)
s.sendSamples(ctx, pendingData[:n], nPendingSamples, nPendingExemplars, pBuf, &buf)
}
queue.ReturnForReuse(batch)
timer.Reset(time.Duration(s.qm.cfg.BatchSendDeadline))
}
}
}
func (s *shards) populateTimeSeries(batch []interface{}, pendingData []prompb.TimeSeries) (int, int) {
var nPendingSamples, nPendingExemplars int
for nPending, sample := range batch {
pendingData[nPending].Samples = pendingData[nPending].Samples[:0]
if s.qm.sendExemplars {
pendingData[nPending].Exemplars = pendingData[nPending].Exemplars[:0]
}
// Number of pending samples is limited by the fact that sendSamples (via sendSamplesWithBackoff)
// retries endlessly, so once we reach max samples, if we can never send to the endpoint we'll
// stop reading from the queue. This makes it safe to reference pendingSamples by index.
switch d := sample.(type) {
case writeSample:
pendingData[nPending].Labels = labelsToLabelsProto(d.seriesLabels, pendingData[nPending].Labels)
pendingData[nPending].Samples = append(pendingData[nPending].Samples, d.sample)
nPendingSamples++
case writeExemplar:
pendingData[nPending].Labels = labelsToLabelsProto(d.seriesLabels, pendingData[nPending].Labels)
pendingData[nPending].Exemplars = append(pendingData[nPending].Exemplars, d.exemplar)
nPendingExemplars++
}
}
return nPendingSamples, nPendingExemplars
}
func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, sampleCount, exemplarCount int, pBuf *proto.Buffer, buf *[]byte) {
begin := time.Now()
err := s.sendSamplesWithBackoff(ctx, samples, sampleCount, exemplarCount, pBuf, buf)
@ -1158,6 +1249,12 @@ func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, s
s.qm.dataOut.incr(int64(len(samples)))
s.qm.dataOutDuration.incr(int64(time.Since(begin)))
s.qm.lastSendTimestamp.Store(time.Now().Unix())
// Pending samples/exemplars also should be subtracted as an error means
// they will not be retried.
s.qm.metrics.pendingSamples.Sub(float64(sampleCount))
s.qm.metrics.pendingExemplars.Sub(float64(exemplarCount))
s.enqueuedSamples.Sub(int64(sampleCount))
s.enqueuedExemplars.Sub(int64(exemplarCount))
}
// sendSamples to the remote storage with backoff for recoverable errors.

5
storage/remote/write_handler.go
View file

@ -54,7 +54,6 @@ func (h *writeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
case nil:
case storage.ErrOutOfOrderSample, storage.ErrOutOfBounds, storage.ErrDuplicateSampleForTimestamp:
// Indicated an out of order sample is a bad request to prevent retries.
level.Error(h.logger).Log("msg", "Out of order sample from remote write", "err", err.Error())
http.Error(w, err.Error(), http.StatusBadRequest)
return
default:
@ -98,6 +97,10 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err
for _, s := range ts.Samples {
_, err = app.Append(0, labels, s.Timestamp, s.Value)
if err != nil {
switch errors.Cause(err) {
case storage.ErrOutOfOrderSample, storage.ErrOutOfBounds, storage.ErrDuplicateSampleForTimestamp:
level.Error(h.logger).Log("msg", "Out of order sample from remote write", "err", err.Error(), "series", labels.String(), "timestamp", s.Timestamp)
}
return err
}

6
storage/series.go
View file

@ -99,8 +99,12 @@ func (it *listSeriesIterator) Seek(t int64) bool {
if it.idx == -1 {
it.idx = 0
}
// No-op check.
if s := it.samples.Get(it.idx); s.T() >= t {
return true
}
// Do binary search between current position and end.
it.idx = sort.Search(it.samples.Len()-it.idx, func(i int) bool {
it.idx += sort.Search(it.samples.Len()-it.idx, func(i int) bool {
s := it.samples.Get(i + it.idx)
return s.T() >= t
})

54
storage/series_test.go Normal file
View file

@ -0,0 +1,54 @@
// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestListSeriesIterator(t *testing.T) {
it := NewListSeriesIterator(samples{sample{0, 0}, sample{1, 1}, sample{1, 1.5}, sample{2, 2}, sample{3, 3}})
// Seek to the first sample with ts=1.
require.True(t, it.Seek(1))
ts, v := it.At()
require.Equal(t, int64(1), ts)
require.Equal(t, 1., v)
// Seek one further, next sample still has ts=1.
require.True(t, it.Next())
ts, v = it.At()
require.Equal(t, int64(1), ts)
require.Equal(t, 1.5, v)
// Seek again to 1 and make sure we stay where we are.
require.True(t, it.Seek(1))
ts, v = it.At()
require.Equal(t, int64(1), ts)
require.Equal(t, 1.5, v)
// Another seek.
require.True(t, it.Seek(3))
ts, v = it.At()
require.Equal(t, int64(3), ts)
require.Equal(t, 3., v)
// And we don't go back.
require.True(t, it.Seek(2))
ts, v = it.At()
require.Equal(t, int64(3), ts)
require.Equal(t, 3., v)
}

203
tsdb/agent/db.go
View file

@ -16,9 +16,11 @@ package agent
import (
"context"
"fmt"
"math"
"path/filepath"
"sync"
"time"
"unicode/utf8"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
@ -93,6 +95,8 @@ type dbMetrics struct {
numActiveSeries prometheus.Gauge
numWALSeriesPendingDeletion prometheus.Gauge
totalAppendedSamples prometheus.Counter
totalAppendedExemplars prometheus.Counter
totalOutOfOrderSamples prometheus.Counter
walTruncateDuration prometheus.Summary
walCorruptionsTotal prometheus.Counter
walTotalReplayDuration prometheus.Gauge
@ -119,6 +123,16 @@ func newDBMetrics(r prometheus.Registerer) *dbMetrics {
Help: "Total number of samples appended to the storage",
})
m.totalAppendedExemplars = prometheus.NewCounter(prometheus.CounterOpts{
Name: "prometheus_agent_exemplars_appended_total",
Help: "Total number of exemplars appended to the storage",
})
m.totalOutOfOrderSamples = prometheus.NewCounter(prometheus.CounterOpts{
Name: "prometheus_agent_out_of_order_samples_total",
Help: "Total number of out of order samples ingestion failed attempts.",
})
m.walTruncateDuration = prometheus.NewSummary(prometheus.SummaryOpts{
Name: "prometheus_agent_truncate_duration_seconds",
Help: "Duration of WAL truncation.",
@ -159,6 +173,8 @@ func newDBMetrics(r prometheus.Registerer) *dbMetrics {
m.numActiveSeries,
m.numWALSeriesPendingDeletion,
m.totalAppendedSamples,
m.totalAppendedExemplars,
m.totalOutOfOrderSamples,
m.walTruncateDuration,
m.walCorruptionsTotal,
m.walTotalReplayDuration,
@ -180,6 +196,15 @@ func (m *dbMetrics) Unregister() {
m.numActiveSeries,
m.numWALSeriesPendingDeletion,
m.totalAppendedSamples,
m.totalAppendedExemplars,
m.totalOutOfOrderSamples,
m.walTruncateDuration,
m.walCorruptionsTotal,
m.walTotalReplayDuration,
m.checkpointDeleteFail,
m.checkpointDeleteTotal,
m.checkpointCreationFail,
m.checkpointCreationTotal,
}
for _, c := range cs {
m.r.Unregister(c)
@ -257,9 +282,10 @@ func Open(l log.Logger, reg prometheus.Registerer, rs *remote.Storage, dir strin
db.appenderPool.New = func() interface{} {
return &appender{
DB: db,
pendingSeries: make([]record.RefSeries, 0, 100),
pendingSamples: make([]record.RefSample, 0, 100),
DB: db,
pendingSeries: make([]record.RefSeries, 0, 100),
pendingSamples: make([]record.RefSample, 0, 100),
pendingExamplars: make([]record.RefExemplar, 0, 10),
}
}
@ -411,11 +437,8 @@ func (db *DB) loadWAL(r *wal.Reader, multiRef map[chunks.HeadSeriesRef]chunks.He
return
}
decoded <- samples
case record.Tombstones:
// We don't care about tombstones
continue
case record.Exemplars:
// We don't care about exemplars
case record.Tombstones, record.Exemplars:
// We don't care about tombstones or exemplars during replay.
continue
default:
errCh <- &wal.CorruptionErr{
@ -665,82 +688,114 @@ func (db *DB) Close() error {
type appender struct {
*DB
pendingSeries []record.RefSeries
pendingSamples []record.RefSample
pendingSeries []record.RefSeries
pendingSamples []record.RefSample
pendingExamplars []record.RefExemplar
// Pointers to the series referenced by each element of pendingSamples.
// Series lock is not held on elements.
sampleSeries []*memSeries
}
func (a *appender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) {
if ref == 0 {
r, err := a.Add(l, t, v)
return storage.SeriesRef(r), err
}
return ref, a.AddFast(chunks.HeadSeriesRef(ref), t, v)
}
// series references and chunk references are identical for agent mode.
headRef := chunks.HeadSeriesRef(ref)
func (a *appender) Add(l labels.Labels, t int64, v float64) (chunks.HeadSeriesRef, error) {
hash := l.Hash()
series := a.series.GetByHash(hash, l)
if series != nil {
return series.ref, a.AddFast(series.ref, t, v)
}
// Ensure no empty or duplicate labels have gotten through. This mirrors the
// equivalent validation code in the TSDB's headAppender.
l = l.WithoutEmpty()
if len(l) == 0 {
return 0, errors.Wrap(tsdb.ErrInvalidSample, "empty labelset")
}
if lbl, dup := l.HasDuplicateLabelNames(); dup {
return 0, errors.Wrap(tsdb.ErrInvalidSample, fmt.Sprintf(`label name "%s" is not unique`, lbl))
}
ref := chunks.HeadSeriesRef(a.nextRef.Inc())
series = &memSeries{ref: ref, lset: l, lastTs: t}
a.pendingSeries = append(a.pendingSeries, record.RefSeries{
Ref: ref,
Labels: l,
})
a.pendingSamples = append(a.pendingSamples, record.RefSample{
Ref: ref,
T: t,
V: v,
})
a.series.Set(hash, series)
a.metrics.numActiveSeries.Inc()
a.metrics.totalAppendedSamples.Inc()
return series.ref, nil
}
func (a *appender) AddFast(ref chunks.HeadSeriesRef, t int64, v float64) error {
series := a.series.GetByID(ref)
series := a.series.GetByID(headRef)
if series == nil {
return storage.ErrNotFound
// Ensure no empty or duplicate labels have gotten through. This mirrors the
// equivalent validation code in the TSDB's headAppender.
l = l.WithoutEmpty()
if len(l) == 0 {
return 0, errors.Wrap(tsdb.ErrInvalidSample, "empty labelset")
}
if lbl, dup := l.HasDuplicateLabelNames(); dup {
return 0, errors.Wrap(tsdb.ErrInvalidSample, fmt.Sprintf(`label name "%s" is not unique`, lbl))
}
var created bool
series, created = a.getOrCreate(l)
if created {
a.pendingSeries = append(a.pendingSeries, record.RefSeries{
Ref: series.ref,
Labels: l,
})
a.metrics.numActiveSeries.Inc()
}
}
series.Lock()
defer series.Unlock()
// Update last recorded timestamp. Used by Storage.gc to determine if a
// series is dead.
series.lastTs = t
if t < series.lastTs {
a.metrics.totalOutOfOrderSamples.Inc()
return 0, storage.ErrOutOfOrderSample
}
// NOTE: always modify pendingSamples and sampleSeries together
a.pendingSamples = append(a.pendingSamples, record.RefSample{
Ref: ref,
Ref: series.ref,
T: t,
V: v,
})
a.sampleSeries = append(a.sampleSeries, series)
a.metrics.totalAppendedSamples.Inc()
return nil
return storage.SeriesRef(series.ref), nil
}
func (a *appender) getOrCreate(l labels.Labels) (series *memSeries, created bool) {
hash := l.Hash()
series = a.series.GetByHash(hash, l)
if series != nil {
return series, false
}
ref := chunks.HeadSeriesRef(a.nextRef.Inc())
series = &memSeries{ref: ref, lset: l, lastTs: math.MinInt64}
a.series.Set(hash, series)
return series, true
}
func (a *appender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) {
// remote_write doesn't support exemplars yet, so do nothing here.
return 0, nil
// series references and chunk references are identical for agent mode.
headRef := chunks.HeadSeriesRef(ref)
s := a.series.GetByID(headRef)
if s == nil {
return 0, fmt.Errorf("unknown series ref when trying to add exemplar: %d", ref)
}
// Ensure no empty labels have gotten through.
e.Labels = e.Labels.WithoutEmpty()
if lbl, dup := e.Labels.HasDuplicateLabelNames(); dup {
return 0, errors.Wrap(tsdb.ErrInvalidExemplar, fmt.Sprintf(`label name "%s" is not unique`, lbl))
}
// Exemplar label length does not include chars involved in text rendering such as quotes
// equals sign, or commas. See definition of const ExemplarMaxLabelLength.
labelSetLen := 0
for _, l := range e.Labels {
labelSetLen += utf8.RuneCountInString(l.Name)
labelSetLen += utf8.RuneCountInString(l.Value)
if labelSetLen > exemplar.ExemplarMaxLabelSetLength {
return 0, storage.ErrExemplarLabelLength
}
}
a.pendingExamplars = append(a.pendingExamplars, record.RefExemplar{
Ref: s.ref,
T: e.Ts,
V: e.Value,
Labels: e.Labels,
})
return storage.SeriesRef(s.ref), nil
}
// Commit submits the collected samples and purges the batch.
@ -767,6 +822,22 @@ func (a *appender) Commit() error {
buf = buf[:0]
}
if len(a.pendingExamplars) > 0 {
buf = encoder.Exemplars(a.pendingExamplars, buf)
if err := a.wal.Log(buf); err != nil {
return err
}
buf = buf[:0]
}
var series *memSeries
for i, s := range a.pendingSamples {
series = a.sampleSeries[i]
if !series.updateTimestamp(s.T) {
a.metrics.totalOutOfOrderSamples.Inc()
}
}
//nolint:staticcheck
a.bufPool.Put(buf)
return a.Rollback()
@ -775,6 +846,8 @@ func (a *appender) Commit() error {
func (a *appender) Rollback() error {
a.pendingSeries = a.pendingSeries[:0]
a.pendingSamples = a.pendingSamples[:0]
a.pendingExamplars = a.pendingExamplars[:0]
a.sampleSeries = a.sampleSeries[:0]
a.appenderPool.Put(a)
return nil
}

366
tsdb/agent/db_test.go
View file

@ -15,8 +15,8 @@ package agent
import (
"context"
"path/filepath"
"strconv"
"sync"
"testing"
"time"
@ -26,25 +26,70 @@ import (
dto "github.com/prometheus/client_model/go"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/storage/remote"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/tsdb/record"
"github.com/prometheus/prometheus/tsdb/tsdbutil"
"github.com/prometheus/prometheus/tsdb/wal"
"github.com/prometheus/prometheus/util/testutil"
)
func TestUnsupported(t *testing.T) {
promAgentDir := t.TempDir()
func TestDB_InvalidSeries(t *testing.T) {
s := createTestAgentDB(t, nil, DefaultOptions())
defer s.Close()
opts := DefaultOptions()
logger := log.NewNopLogger()
app := s.Appender(context.Background())
s, err := Open(logger, prometheus.NewRegistry(), nil, promAgentDir, opts)
t.Run("Samples", func(t *testing.T) {
_, err := app.Append(0, labels.Labels{}, 0, 0)
require.ErrorIs(t, err, tsdb.ErrInvalidSample, "should reject empty labels")
_, err = app.Append(0, labels.Labels{{Name: "a", Value: "1"}, {Name: "a", Value: "2"}}, 0, 0)
require.ErrorIs(t, err, tsdb.ErrInvalidSample, "should reject duplicate labels")
})
t.Run("Exemplars", func(t *testing.T) {
sRef, err := app.Append(0, labels.Labels{{Name: "a", Value: "1"}}, 0, 0)
require.NoError(t, err, "should not reject valid series")
_, err = app.AppendExemplar(0, nil, exemplar.Exemplar{})
require.EqualError(t, err, "unknown series ref when trying to add exemplar: 0")
e := exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}, {Name: "a", Value: "2"}}}
_, err = app.AppendExemplar(sRef, nil, e)
require.ErrorIs(t, err, tsdb.ErrInvalidExemplar, "should reject duplicate labels")
e = exemplar.Exemplar{Labels: labels.Labels{{Name: "a_somewhat_long_trace_id", Value: "nYJSNtFrFTY37VR7mHzEE/LIDt7cdAQcuOzFajgmLDAdBSRHYPDzrxhMA4zz7el8naI/AoXFv9/e/G0vcETcIoNUi3OieeLfaIRQci2oa"}}}
_, err = app.AppendExemplar(sRef, nil, e)
require.ErrorIs(t, err, storage.ErrExemplarLabelLength, "should reject too long label length")
// Inverse check
e = exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}}, Value: 20, Ts: 10, HasTs: true}
_, err = app.AppendExemplar(sRef, nil, e)
require.NoError(t, err, "should not reject valid exemplars")
})
}
func createTestAgentDB(t *testing.T, reg prometheus.Registerer, opts *Options) *DB {
t.Helper()
dbDir := t.TempDir()
rs := remote.NewStorage(log.NewNopLogger(), reg, startTime, dbDir, time.Second*30, nil)
t.Cleanup(func() {
require.NoError(t, rs.Close())
})
db, err := Open(log.NewNopLogger(), reg, rs, dbDir, opts)
require.NoError(t, err)
defer func() {
require.NoError(t, s.Close())
}()
return db
}
func TestUnsupportedFunctions(t *testing.T) {
s := createTestAgentDB(t, nil, DefaultOptions())
defer s.Close()
t.Run("Querier", func(t *testing.T) {
_, err := s.Querier(context.TODO(), 0, 0)
@ -68,93 +113,74 @@ func TestCommit(t *testing.T) {
numSeries = 8
)
promAgentDir := t.TempDir()
s := createTestAgentDB(t, nil, DefaultOptions())
app := s.Appender(context.TODO())
lbls := labelsForTest(t.Name(), numSeries)
opts := DefaultOptions()
logger := log.NewNopLogger()
reg := prometheus.NewRegistry()
remoteStorage := remote.NewStorage(log.With(logger, "component", "remote"), reg, startTime, promAgentDir, time.Second*30, nil)
defer func(rs *remote.Storage) {
require.NoError(t, rs.Close())
}(remoteStorage)
s, err := Open(logger, reg, remoteStorage, promAgentDir, opts)
require.NoError(t, err)
a := s.Appender(context.TODO())
for _, l := range lbls {
lset := labels.New(l...)
for i := 0; i < numDatapoints; i++ {
sample := tsdbutil.GenerateSamples(0, 1)
_, err := a.Append(0, lset, sample[0].T(), sample[0].V())
ref, err := app.Append(0, lset, sample[0].T(), sample[0].V())
require.NoError(t, err)
e := exemplar.Exemplar{
Labels: lset,
Ts: sample[0].T(),
Value: sample[0].V(),
HasTs: true,
}
_, err = app.AppendExemplar(ref, lset, e)
require.NoError(t, err)
}
}
require.NoError(t, a.Commit())
require.NoError(t, app.Commit())
require.NoError(t, s.Close())
// Read records from WAL and check for expected count of series and samples.
walSeriesCount := 0
walSamplesCount := 0
reg = prometheus.NewRegistry()
remoteStorage = remote.NewStorage(log.With(logger, "component", "remote"), reg, startTime, promAgentDir, time.Second*30, nil)
defer func() {
require.NoError(t, remoteStorage.Close())
}()
s1, err := Open(logger, nil, remoteStorage, promAgentDir, opts)
sr, err := wal.NewSegmentsReader(s.wal.Dir())
require.NoError(t, err)
defer func() {
require.NoError(t, s1.Close())
require.NoError(t, sr.Close())
}()
var dec record.Decoder
// Read records from WAL and check for expected count of series, samples, and exemplars.
var (
r = wal.NewReader(sr)
dec record.Decoder
if err == nil {
sr, err := wal.NewSegmentsReader(s1.wal.Dir())
require.NoError(t, err)
defer func() {
require.NoError(t, sr.Close())
}()
walSeriesCount, walSamplesCount, walExemplarsCount int
)
for r.Next() {
rec := r.Record()
switch dec.Type(rec) {
case record.Series:
var series []record.RefSeries
series, err = dec.Series(rec, series)
require.NoError(t, err)
walSeriesCount += len(series)
r := wal.NewReader(sr)
seriesPool := sync.Pool{
New: func() interface{} {
return []record.RefSeries{}
},
}
samplesPool := sync.Pool{
New: func() interface{} {
return []record.RefSample{}
},
}
case record.Samples:
var samples []record.RefSample
samples, err = dec.Samples(rec, samples)
require.NoError(t, err)
walSamplesCount += len(samples)
for r.Next() {
rec := r.Record()
switch dec.Type(rec) {
case record.Series:
series := seriesPool.Get().([]record.RefSeries)[:0]
series, _ = dec.Series(rec, series)
walSeriesCount += len(series)
case record.Samples:
samples := samplesPool.Get().([]record.RefSample)[:0]
samples, _ = dec.Samples(rec, samples)
walSamplesCount += len(samples)
default:
}
case record.Exemplars:
var exemplars []record.RefExemplar
exemplars, err = dec.Exemplars(rec, exemplars)
require.NoError(t, err)
walExemplarsCount += len(exemplars)
default:
}
}
// Retrieved series count from WAL should match the count of series been added to the WAL.
require.Equal(t, walSeriesCount, numSeries)
// Retrieved samples count from WAL should match the count of samples been added to the WAL.
require.Equal(t, walSamplesCount, numSeries*numDatapoints)
// Check that the WAL contained the same number of commited series/samples/exemplars.
require.Equal(t, numSeries, walSeriesCount, "unexpected number of series")
require.Equal(t, numSeries*numDatapoints, walSamplesCount, "unexpected number of samples")
require.Equal(t, numSeries*numDatapoints, walExemplarsCount, "unexpected number of exemplars")
}
func TestRollback(t *testing.T) {
@ -163,93 +189,68 @@ func TestRollback(t *testing.T) {
numSeries = 8
)
promAgentDir := t.TempDir()
s := createTestAgentDB(t, nil, DefaultOptions())
app := s.Appender(context.TODO())
lbls := labelsForTest(t.Name(), numSeries)
opts := DefaultOptions()
logger := log.NewNopLogger()
reg := prometheus.NewRegistry()
remoteStorage := remote.NewStorage(log.With(logger, "component", "remote"), reg, startTime, promAgentDir, time.Second*30, nil)
defer func(rs *remote.Storage) {
require.NoError(t, rs.Close())
}(remoteStorage)
s, err := Open(logger, reg, remoteStorage, promAgentDir, opts)
require.NoError(t, err)
a := s.Appender(context.TODO())
for _, l := range lbls {
lset := labels.New(l...)
for i := 0; i < numDatapoints; i++ {
sample := tsdbutil.GenerateSamples(0, 1)
_, err := a.Append(0, lset, sample[0].T(), sample[0].V())
_, err := app.Append(0, lset, sample[0].T(), sample[0].V())
require.NoError(t, err)
}
}
require.NoError(t, a.Rollback())
// Do a rollback, which should clear uncommitted data. A followup call to
// commit should persist nothing to the WAL.
require.NoError(t, app.Rollback())
require.NoError(t, app.Commit())
require.NoError(t, s.Close())
// Read records from WAL and check for expected count of series and samples.
walSeriesCount := 0
walSamplesCount := 0
reg = prometheus.NewRegistry()
remoteStorage = remote.NewStorage(log.With(logger, "component", "remote"), reg, startTime, promAgentDir, time.Second*30, nil)
defer func() {
require.NoError(t, remoteStorage.Close())
}()
s1, err := Open(logger, nil, remoteStorage, promAgentDir, opts)
sr, err := wal.NewSegmentsReader(s.wal.Dir())
require.NoError(t, err)
defer func() {
require.NoError(t, s1.Close())
require.NoError(t, sr.Close())
}()
var dec record.Decoder
// Read records from WAL and check for expected count of series and samples.
var (
r = wal.NewReader(sr)
dec record.Decoder
if err == nil {
sr, err := wal.NewSegmentsReader(s1.wal.Dir())
require.NoError(t, err)
defer func() {
require.NoError(t, sr.Close())
}()
walSeriesCount, walSamplesCount, walExemplarsCount int
)
for r.Next() {
rec := r.Record()
switch dec.Type(rec) {
case record.Series:
var series []record.RefSeries
series, err = dec.Series(rec, series)
require.NoError(t, err)
walSeriesCount += len(series)
r := wal.NewReader(sr)
seriesPool := sync.Pool{
New: func() interface{} {
return []record.RefSeries{}
},
}
samplesPool := sync.Pool{
New: func() interface{} {
return []record.RefSample{}
},
}
case record.Samples:
var samples []record.RefSample
samples, err = dec.Samples(rec, samples)
require.NoError(t, err)
walSamplesCount += len(samples)
for r.Next() {
rec := r.Record()
switch dec.Type(rec) {
case record.Series:
series := seriesPool.Get().([]record.RefSeries)[:0]
series, _ = dec.Series(rec, series)
walSeriesCount += len(series)
case record.Samples:
samples := samplesPool.Get().([]record.RefSample)[:0]
samples, _ = dec.Samples(rec, samples)
walSamplesCount += len(samples)
default:
}
case record.Exemplars:
var exemplars []record.RefExemplar
exemplars, err = dec.Exemplars(rec, exemplars)
require.NoError(t, err)
walExemplarsCount += len(exemplars)
default:
}
}
// Retrieved series count from WAL should be zero.
require.Equal(t, walSeriesCount, 0)
// Retrieved samples count from WAL should be zero.
require.Equal(t, walSamplesCount, 0)
// Check that the rollback ensured nothing got stored.
require.Equal(t, 0, walSeriesCount, "series should not have been written to WAL")
require.Equal(t, 0, walSamplesCount, "samples should not have been written to WAL")
require.Equal(t, 0, walExemplarsCount, "exemplars should not have been written to WAL")
}
func TestFullTruncateWAL(t *testing.T) {
@ -259,34 +260,25 @@ func TestFullTruncateWAL(t *testing.T) {
lastTs = 500
)
promAgentDir := t.TempDir()
lbls := labelsForTest(t.Name(), numSeries)
reg := prometheus.NewRegistry()
opts := DefaultOptions()
opts.TruncateFrequency = time.Minute * 2
logger := log.NewNopLogger()
reg := prometheus.NewRegistry()
remoteStorage := remote.NewStorage(log.With(logger, "component", "remote"), reg, startTime, promAgentDir, time.Second*30, nil)
defer func() {
require.NoError(t, remoteStorage.Close())
}()
s, err := Open(logger, reg, remoteStorage, promAgentDir, opts)
require.NoError(t, err)
s := createTestAgentDB(t, reg, opts)
defer func() {
require.NoError(t, s.Close())
}()
app := s.Appender(context.TODO())
a := s.Appender(context.TODO())
lbls := labelsForTest(t.Name(), numSeries)
for _, l := range lbls {
lset := labels.New(l...)
for i := 0; i < numDatapoints; i++ {
_, err := a.Append(0, lset, int64(lastTs), 0)
_, err := app.Append(0, lset, int64(lastTs), 0)
require.NoError(t, err)
}
require.NoError(t, a.Commit())
require.NoError(t, app.Commit())
}
// Truncate WAL with mint to GC all the samples.
@ -302,52 +294,40 @@ func TestPartialTruncateWAL(t *testing.T) {
numSeries = 800
)
promAgentDir := t.TempDir()
opts := DefaultOptions()
opts.TruncateFrequency = time.Minute * 2
logger := log.NewNopLogger()
reg := prometheus.NewRegistry()
remoteStorage := remote.NewStorage(log.With(logger, "component", "remote"), reg, startTime, promAgentDir, time.Second*30, nil)
defer func() {
require.NoError(t, remoteStorage.Close())
}()
s, err := Open(logger, reg, remoteStorage, promAgentDir, opts)
require.NoError(t, err)
reg := prometheus.NewRegistry()
s := createTestAgentDB(t, reg, opts)
defer func() {
require.NoError(t, s.Close())
}()
a := s.Appender(context.TODO())
var lastTs int64
app := s.Appender(context.TODO())
// Create first batch of 800 series with 1000 data-points with a fixed lastTs as 500.
lastTs = 500
var lastTs int64 = 500
lbls := labelsForTest(t.Name()+"batch-1", numSeries)
for _, l := range lbls {
lset := labels.New(l...)
for i := 0; i < numDatapoints; i++ {
_, err := a.Append(0, lset, lastTs, 0)
_, err := app.Append(0, lset, lastTs, 0)
require.NoError(t, err)
}
require.NoError(t, a.Commit())
require.NoError(t, app.Commit())
}
// Create second batch of 800 series with 1000 data-points with a fixed lastTs as 600.
lastTs = 600
lbls = labelsForTest(t.Name()+"batch-2", numSeries)
for _, l := range lbls {
lset := labels.New(l...)
for i := 0; i < numDatapoints; i++ {
_, err := a.Append(0, lset, lastTs, 0)
_, err := app.Append(0, lset, lastTs, 0)
require.NoError(t, err)
}
require.NoError(t, a.Commit())
require.NoError(t, app.Commit())
}
// Truncate WAL with mint to GC only the first batch of 800 series and retaining 2nd batch of 800 series.
@ -364,53 +344,41 @@ func TestWALReplay(t *testing.T) {
lastTs = 500
)
promAgentDir := t.TempDir()
s := createTestAgentDB(t, nil, DefaultOptions())
app := s.Appender(context.TODO())
lbls := labelsForTest(t.Name(), numSeries)
opts := DefaultOptions()
logger := log.NewNopLogger()
reg := prometheus.NewRegistry()
remoteStorage := remote.NewStorage(log.With(logger, "component", "remote"), reg, startTime, promAgentDir, time.Second*30, nil)
defer func() {
require.NoError(t, remoteStorage.Close())
}()
s, err := Open(logger, reg, remoteStorage, promAgentDir, opts)
require.NoError(t, err)
a := s.Appender(context.TODO())
for _, l := range lbls {
lset := labels.New(l...)
for i := 0; i < numDatapoints; i++ {
_, err := a.Append(0, lset, lastTs, 0)
_, err := app.Append(0, lset, lastTs, 0)
require.NoError(t, err)
}
}
require.NoError(t, a.Commit())
require.NoError(t, app.Commit())
require.NoError(t, s.Close())
restartOpts := DefaultOptions()
restartLogger := log.NewNopLogger()
restartReg := prometheus.NewRegistry()
// Hack: s.wal.Dir() is the /wal subdirectory of the original storage path.
// We need the original directory so we can recreate the storage for replay.
storageDir := filepath.Dir(s.wal.Dir())
// Open a new DB with the same WAL to check that series from the previous DB
// get replayed.
replayDB, err := Open(restartLogger, restartReg, nil, promAgentDir, restartOpts)
require.NoError(t, err)
reg := prometheus.NewRegistry()
replayStorage, err := Open(s.logger, reg, nil, storageDir, s.opts)
if err != nil {
t.Fatalf("unable to create storage for the agent: %v", err)
}
defer func() {
require.NoError(t, replayDB.Close())
require.NoError(t, replayStorage.Close())
}()
// Check if all the series are retrieved back from the WAL.
m := gatherFamily(t, restartReg, "prometheus_agent_active_series")
m := gatherFamily(t, reg, "prometheus_agent_active_series")
require.Equal(t, float64(numSeries), m.Metric[0].Gauge.GetValue(), "agent wal replay mismatch of active series count")
// Check if lastTs of the samples retrieved from the WAL is retained.
metrics := replayDB.series.series
metrics := replayStorage.series.series
for i := 0; i < len(metrics); i++ {
mp := metrics[i]
for _, v := range mp {

19
tsdb/agent/series.go
View file

@ -24,11 +24,26 @@ import (
type memSeries struct {
sync.Mutex
ref chunks.HeadSeriesRef
lset labels.Labels
ref chunks.HeadSeriesRef
lset labels.Labels
// Last recorded timestamp. Used by Storage.gc to determine if a series is
// stale.
lastTs int64
}
// updateTimestamp obtains the lock on s and will attempt to update lastTs.
// fails if newTs < lastTs.
func (m *memSeries) updateTimestamp(newTs int64) bool {
m.Lock()
defer m.Unlock()
if newTs >= m.lastTs {
m.lastTs = newTs
return true
}
return false
}
// seriesHashmap is a simple hashmap for memSeries by their label set.
// It is built on top of a regular hashmap and holds a slice of series to
// resolve hash collisions. Its methods require the hash to be submitted

13
tsdb/head.go
View file

@ -499,7 +499,9 @@ const cardinalityCacheExpirationTime = time.Duration(30) * time.Second
// limits the ingested samples to the head min valid time.
func (h *Head) Init(minValidTime int64) error {
h.minValidTime.Store(minValidTime)
defer h.postings.EnsureOrder()
defer func() {
h.postings.EnsureOrder()
}()
defer h.gc() // After loading the wal remove the obsolete data from the head.
defer func() {
// Loading of m-mapped chunks and snapshot can make the mint of the Head
@ -712,17 +714,18 @@ func (h *Head) ApplyConfig(cfg *config.Config) error {
}
// Head uses opts.MaxExemplars in combination with opts.EnableExemplarStorage
// to decide if it should pass exemplars along to it's exemplar storage, so we
// to decide if it should pass exemplars along to its exemplar storage, so we
// need to update opts.MaxExemplars here.
prevSize := h.opts.MaxExemplars.Load()
h.opts.MaxExemplars.Store(cfg.StorageConfig.ExemplarsConfig.MaxExemplars)
newSize := h.opts.MaxExemplars.Load()
if prevSize == h.opts.MaxExemplars.Load() {
if prevSize == newSize {
return nil
}
migrated := h.exemplars.(*CircularExemplarStorage).Resize(h.opts.MaxExemplars.Load())
level.Info(h.logger).Log("msg", "Exemplar storage resized", "from", prevSize, "to", h.opts.MaxExemplars, "migrated", migrated)
migrated := h.exemplars.(*CircularExemplarStorage).Resize(newSize)
level.Info(h.logger).Log("msg", "Exemplar storage resized", "from", prevSize, "to", newSize, "migrated", migrated)
return nil
}

104
tsdb/head_test.go
View file

@ -3107,3 +3107,107 @@ func TestSnapshotError(t *testing.T) {
require.NoError(t, err)
require.Equal(t, 0, len(tm))
}
// Tests https://github.com/prometheus/prometheus/issues/9725.
func TestChunkSnapshotReplayBug(t *testing.T) {
dir := t.TempDir()
wlog, err := wal.NewSize(nil, nil, filepath.Join(dir, "wal"), 32768, true)
require.NoError(t, err)
// Write few series records and samples such that the series references are not in order in the WAL
// for status_code="200".
var buf []byte
for i := 1; i <= 1000; i++ {
var ref chunks.HeadSeriesRef
if i <= 500 {
ref = chunks.HeadSeriesRef(i * 100)
} else {
ref = chunks.HeadSeriesRef((i - 500) * 50)
}
seriesRec := record.RefSeries{
Ref: ref,
Labels: labels.Labels{
{Name: "__name__", Value: "request_duration"},
{Name: "status_code", Value: "200"},
{Name: "foo", Value: fmt.Sprintf("baz%d", rand.Int())},
},
}
// Add a sample so that the series is not garbage collected.
samplesRec := record.RefSample{Ref: ref, T: 1000, V: 1000}
var enc record.Encoder
rec := enc.Series([]record.RefSeries{seriesRec}, buf)
buf = rec[:0]
require.NoError(t, wlog.Log(rec))
rec = enc.Samples([]record.RefSample{samplesRec}, buf)
buf = rec[:0]
require.NoError(t, wlog.Log(rec))
}
// Write a corrupt snapshot to fail the replay on startup.
snapshotName := chunkSnapshotDir(0, 100)
cpdir := filepath.Join(dir, snapshotName)
require.NoError(t, os.MkdirAll(cpdir, 0o777))
err = ioutil.WriteFile(filepath.Join(cpdir, "00000000"), []byte{1, 5, 3, 5, 6, 7, 4, 2, 2}, 0o777)
require.NoError(t, err)
opts := DefaultHeadOptions()
opts.ChunkDirRoot = dir
opts.EnableMemorySnapshotOnShutdown = true
head, err := NewHead(nil, nil, wlog, opts, nil)
require.NoError(t, err)
require.NoError(t, head.Init(math.MinInt64))
defer func() {
require.NoError(t, head.Close())
}()
// Snapshot replay should error out.
require.Equal(t, 1.0, prom_testutil.ToFloat64(head.metrics.snapshotReplayErrorTotal))
// Querying `request_duration{status_code!="200"}` should return no series since all of
// them have status_code="200".
q, err := NewBlockQuerier(head, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
series := query(t, q,
labels.MustNewMatcher(labels.MatchEqual, "__name__", "request_duration"),
labels.MustNewMatcher(labels.MatchNotEqual, "status_code", "200"),
)
require.Len(t, series, 0, "there should be no series found")
}
func TestChunkSnapshotTakenAfterIncompleteSnapshot(t *testing.T) {
dir := t.TempDir()
wlog, err := wal.NewSize(nil, nil, filepath.Join(dir, "wal"), 32768, true)
require.NoError(t, err)
// Write a snapshot with .tmp suffix. This used to fail taking any further snapshots or replay of snapshots.
snapshotName := chunkSnapshotDir(0, 100) + ".tmp"
cpdir := filepath.Join(dir, snapshotName)
require.NoError(t, os.MkdirAll(cpdir, 0o777))
opts := DefaultHeadOptions()
opts.ChunkDirRoot = dir
opts.EnableMemorySnapshotOnShutdown = true
head, err := NewHead(nil, nil, wlog, opts, nil)
require.NoError(t, err)
require.NoError(t, head.Init(math.MinInt64))
require.Equal(t, 0.0, prom_testutil.ToFloat64(head.metrics.snapshotReplayErrorTotal))
// Add some samples for the snapshot.
app := head.Appender(context.Background())
_, err = app.Append(0, labels.Labels{{Name: "foo", Value: "bar"}}, 10, 10)
require.NoError(t, err)
require.NoError(t, app.Commit())
// Should not return any error for a successful snapshot.
require.NoError(t, head.Close())
// Verify the snapshot.
name, idx, offset, err := LastChunkSnapshot(dir)
require.NoError(t, err)
require.True(t, name != "")
require.Equal(t, 0, idx)
require.Greater(t, offset, 0)
}

3
tsdb/head_wal.go
View file

@ -777,7 +777,8 @@ func LastChunkSnapshot(dir string) (string, int, int, error) {
splits := strings.Split(fi.Name()[len(chunkSnapshotPrefix):], ".")
if len(splits) != 2 {
return "", 0, 0, errors.Errorf("chunk snapshot %s is not in the right format", fi.Name())
// Chunk snapshots is not in the right format, we do not care about it.
continue
}
idx, err := strconv.Atoi(splits[0])

6
tsdb/tsdbutil/buffer_test.go
View file

@ -159,8 +159,12 @@ func (it *listSeriesIterator) Seek(t int64) bool {
if it.idx == -1 {
it.idx = 0
}
// No-op check.
if s := it.list[it.idx]; s.T() >= t {
return true
}
// Do binary search between current position and end.
it.idx = sort.Search(len(it.list)-it.idx, func(i int) bool {
it.idx += sort.Search(len(it.list)-it.idx, func(i int) bool {
s := it.list[i+it.idx]
return s.t >= t
})

46
util/strutil/quote.go
View file

@ -10,6 +10,45 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// NOTE: The functions in this file (Unquote, unquoteChar, contains, unhex)
// have been adapted from the "strconv" package of the Go standard library
// to work for Prometheus-style strings. Go's special-casing for single
// quotes was removed and single quoted strings are now treated the same as
// double-quoted ones.
//
// The original copyright notice from the Go project for these parts is
// reproduced here:
//
// ========================================================================
// Copyright (c) 2009 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// ========================================================================
package strutil
@ -24,13 +63,6 @@ var ErrSyntax = errors.New("invalid syntax")
// Unquote interprets s as a single-quoted, double-quoted, or backquoted
// Prometheus query language string literal, returning the string value that s
// quotes.
//
// NOTE: This function as well as the necessary helper functions below
// (unquoteChar, contains, unhex) and associated tests have been adapted from
// the corresponding functions in the "strconv" package of the Go standard
// library to work for Prometheus-style strings. Go's special-casing for single
// quotes was removed and single quoted strings are now treated the same as
// double quoted ones.
func Unquote(s string) (t string, err error) {
n := len(s)
if n < 2 {

36
util/strutil/quote_test.go
View file

@ -10,6 +10,42 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// NOTE: The test code in this file has been adapted from the "strconv"
// package of the Go standard library to work for Prometheus-style strings.
//
// The original copyright notice from the Go project for these parts is
// reproduced here:
//
// ========================================================================
// Copyright (c) 2009 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// ========================================================================
package strutil

7
web/api/v1/api_test.go
View file

@ -407,9 +407,7 @@ func TestEndpoints(t *testing.T) {
Format: &af,
}
dbDir, err := ioutil.TempDir("", "tsdb-api-ready")
require.NoError(t, err)
defer os.RemoveAll(dbDir)
dbDir := t.TempDir()
remote := remote.NewStorage(promlog.New(&promlogConfig), prometheus.DefaultRegisterer, func() (int64, error) {
return 0, nil
@ -2344,8 +2342,7 @@ func TestAdminEndpoints(t *testing.T) {
} {
tc := tc
t.Run("", func(t *testing.T) {
dir, _ := ioutil.TempDir("", "fakeDB")
defer func() { require.NoError(t, os.RemoveAll(dir)) }()
dir := t.TempDir()
api := &API{
db: tc.db,

2
web/federate.go
View file

@ -113,7 +113,7 @@ func (h *Handler) federation(w http.ResponseWriter, req *http.Request) {
ok := it.Seek(maxt)
if ok {
t, v = it.Values()
t, v = it.At()
} else {
t, v, ok = it.PeekBack(1)
if !ok {

34
web/ui/module/codemirror-promql/package.json
View file

@ -29,29 +29,29 @@
"bugs": {
"url": "https://github.com/prometheus/codemirror-promql/issues"
},
"homepage": "https://github.com/prometheus/codemirror-promql/blob/master/README.md",
"homepage": "https://github.com/prometheus/codemirror-promql/blob/main/README.md",
"dependencies": {
"lru-cache": "^6.0.0"
},
"devDependencies": {
"@codemirror/autocomplete": "^0.19.8",
"@codemirror/autocomplete": "^0.19.9",
"@codemirror/basic-setup": "^0.19.0",
"@codemirror/highlight": "^0.19.6",
"@codemirror/language": "^0.19.5",
"@codemirror/language": "^0.19.7",
"@codemirror/lint": "^0.19.3",
"@codemirror/state": "^0.19.6",
"@codemirror/view": "^0.19.20",
"@lezer/common": "^0.15.8",
"@codemirror/view": "^0.19.27",
"@lezer/common": "^0.15.10",
"@lezer/generator": "^0.15.2",
"@types/chai": "^4.2.22",
"@types/lru-cache": "^5.1.0",
"@types/lru-cache": "^5.1.1",
"@types/mocha": "^9.0.0",
"@types/node": "^16.11.9",
"@typescript-eslint/eslint-plugin": "^5.3.1",
"@typescript-eslint/parser": "^5.3.1",
"@types/node": "^16.11.12",
"@typescript-eslint/eslint-plugin": "^5.5.0",
"@typescript-eslint/parser": "^5.5.0",
"chai": "^4.2.0",
"codecov": "^3.8.1",
"eslint": "^8.3.0",
"codecov": "^3.8.3",
"eslint": "^8.4.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-flowtype": "^8.0.3",
"eslint-plugin-import": "^2.25.3",
@ -60,20 +60,20 @@
"mocha": "^8.4.0",
"nock": "^13.2.1",
"nyc": "^15.1.0",
"prettier": "^2.4.1",
"ts-loader": "^7.0.4",
"prettier": "^2.5.1",
"ts-loader": "^7.0.5",
"ts-mocha": "^8.0.0",
"ts-node": "^10.4.0",
"typescript": "^4.5.2"
},
"peerDependencies": {
"@codemirror/autocomplete": "^0.19.8",
"@codemirror/autocomplete": "^0.19.9",
"@codemirror/highlight": "^0.19.6",
"@codemirror/language": "^0.19.5",
"@codemirror/language": "^0.19.7",
"@codemirror/lint": "^0.19.3",
"@codemirror/state": "^0.19.6",
"@codemirror/view": "^0.19.20",
"@lezer/common": "^0.15.8"
"@codemirror/view": "^0.19.27",
"@lezer/common": "^0.15.10"
},
"prettier": {
"singleQuote": true,

7
web/ui/module/codemirror-promql/src/client/prometheus.ts
View file

@ -47,7 +47,7 @@ export interface CacheConfig {
}
export interface PrometheusConfig {
url: string;
url?: string;
lookbackInterval?: number;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
httpErrorHandler?: (error: any) => void;
@ -84,7 +84,7 @@ export class HTTPPrometheusClient implements PrometheusClient {
private readonly fetchFn: FetchFn = (input: RequestInfo, init?: RequestInit): Promise<Response> => fetch(input, init);
constructor(config: PrometheusConfig) {
this.url = config.url;
this.url = config.url ? config.url : '';
this.errorHandler = config.httpErrorHandler;
if (config.lookbackInterval) {
this.lookbackInterval = config.lookbackInterval;
@ -242,12 +242,15 @@ export class HTTPPrometheusClient implements PrometheusClient {
private labelsEndpoint(): string {
return `${this.apiPrefix}/labels`;
}
private labelValuesEndpoint(): string {
return `${this.apiPrefix}/label/:name/values`;
}
private seriesEndpoint(): string {
return `${this.apiPrefix}/series`;
}
private metricMetadataEndpoint(): string {
return `${this.apiPrefix}/metadata`;
}

31
web/ui/module/codemirror-promql/src/complete/hybrid.test.ts
View file

@ -1247,6 +1247,37 @@ describe('autocomplete promQL test', () => {
span: /^[a-zA-Z0-9_:]+$/,
},
},
{
title: 'online autocomplete with initial metric list',
expr: 'rat',
pos: 3,
conf: { remote: { cache: { initialMetricList: ['metric1', 'metric2', 'rat'] } } },
expectedResult: {
options: ([] as Completion[]).concat(
[
{
label: 'metric1',
type: 'constant',
},
{
label: 'metric2',
type: 'constant',
},
{
label: 'rat',
type: 'constant',
},
],
functionIdentifierTerms,
aggregateOpTerms,
numberTerms,
snippets
),
from: 0,
to: 3,
span: /^[a-zA-Z0-9_:]+$/,
},
},
];
testCases.forEach((value) => {
it(value.title, async () => {

11
web/ui/module/codemirror-promql/src/complete/index.ts
View file

@ -32,7 +32,16 @@ export interface CompleteConfiguration {
}
function isPrometheusConfig(remoteConfig: PrometheusConfig | PrometheusClient): remoteConfig is PrometheusConfig {
return (remoteConfig as PrometheusConfig).url !== undefined;
const cfg = remoteConfig as PrometheusConfig;
return (
cfg.url !== undefined ||
cfg.lookbackInterval !== undefined ||
cfg.httpErrorHandler !== undefined ||
cfg.fetchFn !== undefined ||
cfg.cache !== undefined ||
cfg.httpMethod !== undefined ||
cfg.apiPrefix !== undefined
);
}
export function newCompleteStrategy(conf?: CompleteConfiguration): CompleteStrategy {

383
web/ui/package-lock.json generated
View file

@ -20,24 +20,24 @@
"lru-cache": "^6.0.0"
},
"devDependencies": {
"@codemirror/autocomplete": "^0.19.8",
"@codemirror/autocomplete": "^0.19.9",
"@codemirror/basic-setup": "^0.19.0",
"@codemirror/highlight": "^0.19.6",
"@codemirror/language": "^0.19.5",
"@codemirror/language": "^0.19.7",
"@codemirror/lint": "^0.19.3",
"@codemirror/state": "^0.19.6",
"@codemirror/view": "^0.19.20",
"@lezer/common": "^0.15.8",
"@codemirror/view": "^0.19.27",
"@lezer/common": "^0.15.10",
"@lezer/generator": "^0.15.2",
"@types/chai": "^4.2.22",
"@types/lru-cache": "^5.1.0",
"@types/lru-cache": "^5.1.1",
"@types/mocha": "^9.0.0",
"@types/node": "^16.11.9",
"@typescript-eslint/eslint-plugin": "^5.3.1",
"@typescript-eslint/parser": "^5.3.1",
"@types/node": "^16.11.12",
"@typescript-eslint/eslint-plugin": "^5.5.0",
"@typescript-eslint/parser": "^5.5.0",
"chai": "^4.2.0",
"codecov": "^3.8.1",
"eslint": "^8.3.0",
"codecov": "^3.8.3",
"eslint": "^8.4.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-flowtype": "^8.0.3",
"eslint-plugin-import": "^2.25.3",
@ -46,8 +46,8 @@
"mocha": "^8.4.0",
"nock": "^13.2.1",
"nyc": "^15.1.0",
"prettier": "^2.4.1",
"ts-loader": "^7.0.4",
"prettier": "^2.5.1",
"ts-loader": "^7.0.5",
"ts-mocha": "^8.0.0",
"ts-node": "^10.4.0",
"typescript": "^4.5.2"
@ -56,23 +56,24 @@
"node": ">=12.0.0"
},
"peerDependencies": {
"@codemirror/autocomplete": "^0.19.8",
"@codemirror/autocomplete": "^0.19.9",
"@codemirror/highlight": "^0.19.6",
"@codemirror/language": "^0.19.5",
"@codemirror/language": "^0.19.7",
"@codemirror/lint": "^0.19.3",
"@codemirror/state": "^0.19.6",
"@codemirror/view": "^0.19.20",
"@lezer/common": "^0.15.8"
"@codemirror/view": "^0.19.27",
"@lezer/common": "^0.15.10"
}
},
"module/codemirror-promql/node_modules/@eslint/eslintrc": {
"version": "1.0.4",
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.0.5.tgz",
"integrity": "sha512-BLxsnmK3KyPunz5wmCCpqy0YelEoxxGmH73Is+Z74oOTMtExcjkr3dDR6quwrjh1YspA8DH9gnX1o069KiS9AQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"ajv": "^6.12.4",
"debug": "^4.3.2",
"espree": "^9.0.0",
"espree": "^9.2.0",
"globals": "^13.9.0",
"ignore": "^4.0.6",
"import-fresh": "^3.2.1",
@ -85,11 +86,12 @@
}
},
"module/codemirror-promql/node_modules/@humanwhocodes/config-array": {
"version": "0.6.0",
"version": "0.9.2",
"resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.2.tgz",
"integrity": "sha512-UXOuFCGcwciWckOpmfKDq/GyhlTf9pN/BzG//x8p8zTOFEcGuA68ANXheFS0AGvy3qgZqLBUkMs7hqzqCKOVwA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@humanwhocodes/object-schema": "^1.2.0",
"@humanwhocodes/object-schema": "^1.2.1",
"debug": "^4.1.1",
"minimatch": "^3.0.4"
},
@ -98,12 +100,13 @@
}
},
"module/codemirror-promql/node_modules/@typescript-eslint/eslint-plugin": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.5.0.tgz",
"integrity": "sha512-4bV6fulqbuaO9UMXU0Ia0o6z6if+kmMRW8rMRyfqXj/eGrZZRGedS4n0adeGNnjr8LKAM495hrQ7Tea52UWmQA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/experimental-utils": "5.4.0",
"@typescript-eslint/scope-manager": "5.4.0",
"@typescript-eslint/experimental-utils": "5.5.0",
"@typescript-eslint/scope-manager": "5.5.0",
"debug": "^4.3.2",
"functional-red-black-tree": "^1.0.1",
"ignore": "^5.1.8",
@ -137,14 +140,15 @@
}
},
"module/codemirror-promql/node_modules/@typescript-eslint/experimental-utils": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-5.5.0.tgz",
"integrity": "sha512-kjWeeVU+4lQ1SLYErRKV5yDXbWDPkpbzTUUlfAUifPYvpX0qZlrcCZ96/6oWxt3QxtK5WVhXz+KsnwW9cIW+3A==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/json-schema": "^7.0.9",
"@typescript-eslint/scope-manager": "5.4.0",
"@typescript-eslint/types": "5.4.0",
"@typescript-eslint/typescript-estree": "5.4.0",
"@typescript-eslint/scope-manager": "5.5.0",
"@typescript-eslint/types": "5.5.0",
"@typescript-eslint/typescript-estree": "5.5.0",
"eslint-scope": "^5.1.1",
"eslint-utils": "^3.0.0"
},
@ -160,13 +164,14 @@
}
},
"module/codemirror-promql/node_modules/@typescript-eslint/parser": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.5.0.tgz",
"integrity": "sha512-JsXBU+kgQOAgzUn2jPrLA+Rd0Y1dswOlX3hp8MuRO1hQDs6xgHtbCXEiAu7bz5hyVURxbXcA2draasMbNqrhmg==",
"dev": true,
"license": "BSD-2-Clause",
"dependencies": {
"@typescript-eslint/scope-manager": "5.4.0",
"@typescript-eslint/types": "5.4.0",
"@typescript-eslint/typescript-estree": "5.4.0",
"@typescript-eslint/scope-manager": "5.5.0",
"@typescript-eslint/types": "5.5.0",
"@typescript-eslint/typescript-estree": "5.5.0",
"debug": "^4.3.2"
},
"engines": {
@ -186,12 +191,13 @@
}
},
"module/codemirror-promql/node_modules/@typescript-eslint/scope-manager": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.5.0.tgz",
"integrity": "sha512-0/r656RmRLo7CbN4Mdd+xZyPJ/fPCKhYdU6mnZx+8msAD8nJSP8EyCFkzbd6vNVZzZvWlMYrSNekqGrCBqFQhg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/types": "5.4.0",
"@typescript-eslint/visitor-keys": "5.4.0"
"@typescript-eslint/types": "5.5.0",
"@typescript-eslint/visitor-keys": "5.5.0"
},
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
@ -202,9 +208,10 @@
}
},
"module/codemirror-promql/node_modules/@typescript-eslint/types": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.5.0.tgz",
"integrity": "sha512-OaYTqkW3GnuHxqsxxJ6KypIKd5Uw7bFiQJZRyNi1jbMJnK3Hc/DR4KwB6KJj6PBRkJJoaNwzMNv9vtTk87JhOg==",
"dev": true,
"license": "MIT",
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
},
@ -214,12 +221,13 @@
}
},
"module/codemirror-promql/node_modules/@typescript-eslint/typescript-estree": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.5.0.tgz",
"integrity": "sha512-pVn8btYUiYrjonhMAO0yG8lm7RApzy2L4RC7Td/mC/qFkyf6vRbGyZozoA94+w6D2Y2GRqpMoCWcwx/EUOzyoQ==",
"dev": true,
"license": "BSD-2-Clause",
"dependencies": {
"@typescript-eslint/types": "5.4.0",
"@typescript-eslint/visitor-keys": "5.4.0",
"@typescript-eslint/types": "5.5.0",
"@typescript-eslint/visitor-keys": "5.5.0",
"debug": "^4.3.2",
"globby": "^11.0.4",
"is-glob": "^4.0.3",
@ -240,11 +248,12 @@
}
},
"module/codemirror-promql/node_modules/@typescript-eslint/visitor-keys": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.5.0.tgz",
"integrity": "sha512-4GzJ1kRtsWzHhdM40tv0ZKHNSbkDhF0Woi/TDwVJX6UICwJItvP7ZTXbjTkCdrors7ww0sYe0t+cIKDAJwZ7Kw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/types": "5.4.0",
"@typescript-eslint/types": "5.5.0",
"eslint-visitor-keys": "^3.0.0"
},
"engines": {
@ -304,13 +313,13 @@
}
},
"module/codemirror-promql/node_modules/eslint": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.3.0.tgz",
"integrity": "sha512-aIay56Ph6RxOTC7xyr59Kt3ewX185SaGnAr8eWukoPLeriCrvGjvAubxuvaXOfsxhtwV5g0uBOsyhAom4qJdww==",
"version": "8.4.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.4.0.tgz",
"integrity": "sha512-kv0XQcAQJL/VD9THQKhTQZVqkJKA+tIj/v2ZKNaIHRAADcJWFb+B/BAewUYuF6UVg1s2xC5qXVoDk0G8sKGeTA==",
"dev": true,
"dependencies": {
"@eslint/eslintrc": "^1.0.4",
"@humanwhocodes/config-array": "^0.6.0",
"@eslint/eslintrc": "^1.0.5",
"@humanwhocodes/config-array": "^0.9.2",
"ajv": "^6.10.0",
"chalk": "^4.0.0",
"cross-spawn": "^7.0.2",
@ -321,7 +330,7 @@
"eslint-scope": "^7.1.0",
"eslint-utils": "^3.0.0",
"eslint-visitor-keys": "^3.1.0",
"espree": "^9.1.0",
"espree": "^9.2.0",
"esquery": "^1.4.0",
"esutils": "^2.0.2",
"fast-deep-equal": "^3.1.3",
@ -397,9 +406,9 @@
}
},
"module/codemirror-promql/node_modules/espree": {
"version": "9.1.0",
"resolved": "https://registry.npmjs.org/espree/-/espree-9.1.0.tgz",
"integrity": "sha512-ZgYLvCS1wxOczBYGcQT9DDWgicXwJ4dbocr9uYN+/eresBAUuBu+O4WzB21ufQ/JqQT8gyp7hJ3z8SHii32mTQ==",
"version": "9.2.0",
"resolved": "https://registry.npmjs.org/espree/-/espree-9.2.0.tgz",
"integrity": "sha512-oP3utRkynpZWF/F2x/HZJ+AGtnIclaR7z1pYPxy7NYM2fSO6LgK/Rkny8anRSPK/VwEA1eqm2squui0T7ZMOBg==",
"dev": true,
"dependencies": {
"acorn": "^8.6.0",
@ -1138,9 +1147,9 @@
}
},
"node_modules/@codemirror/autocomplete": {
"version": "0.19.8",
"resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-0.19.8.tgz",
"integrity": "sha512-o4I1pRlFjhBHOYab+QfpKcW0B8FqAH+2pdmCYrkTz3bm1djVwhlMEhv1s/aTKhdjLtkfZFUbdHBi+8xe22wUCA==",
"version": "0.19.9",
"resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-0.19.9.tgz",
"integrity": "sha512-Ph1LWHtFFqNUIqEVrws6I263ihe5TH+TRBPwxQ78j7st7Q67FDAmgKX6mNbUPh02dxfqQrc9qxlo5JIqKeiVdg==",
"dependencies": {
"@codemirror/language": "^0.19.0",
"@codemirror/state": "^0.19.4",
@ -1247,9 +1256,9 @@
}
},
"node_modules/@codemirror/language": {
"version": "0.19.5",
"resolved": "https://registry.npmjs.org/@codemirror/language/-/language-0.19.5.tgz",
"integrity": "sha512-FnIST07vaM99mv1mJaMMLvxiHSDGgP3wdlcEZzmidndWdbxjrYYYnJzVUOEkeZJNGOfrtPRMF62UCyrTjQMR3g==",
"version": "0.19.7",
"resolved": "https://registry.npmjs.org/@codemirror/language/-/language-0.19.7.tgz",
"integrity": "sha512-pNNUtYWMIMG0lUSKyUXJr8U0rFiCKsKFXbA2Oj17PC+S1FY99hV0z1vcntW67ekAIZw9DMEUQnLsKBuIbAUX7Q==",
"dependencies": {
"@codemirror/state": "^0.19.0",
"@codemirror/text": "^0.19.0",
@ -1343,9 +1352,9 @@
}
},
"node_modules/@codemirror/view": {
"version": "0.19.20",
"resolved": "https://registry.npmjs.org/@codemirror/view/-/view-0.19.20.tgz",
"integrity": "sha512-j4cI/Egdhha77pMfKQQWZmpkcF7vhe21LqdZs8hsG09OtvsPVCHUXmfB0u7nMpcX1JR8aZ3ob9g6FMr+OVtjgA==",
"version": "0.19.27",
"resolved": "https://registry.npmjs.org/@codemirror/view/-/view-0.19.27.tgz",
"integrity": "sha512-Uz/LecEf7CyvMWaQBlKtbJCYn0hRnEZ2yYvuZVy9YMhmvGmES6ec7FaKw7lDFFOMLwLbBThc9kfw4DCHreHN1w==",
"dependencies": {
"@codemirror/rangeset": "^0.19.0",
"@codemirror/state": "^0.19.3",
@ -1441,9 +1450,10 @@
}
},
"node_modules/@humanwhocodes/object-schema": {
"version": "1.2.0",
"dev": true,
"license": "BSD-3-Clause"
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz",
"integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
"dev": true
},
"node_modules/@istanbuljs/load-nyc-config": {
"version": "1.1.0",
@ -1557,8 +1567,9 @@
}
},
"node_modules/@lezer/common": {
"version": "0.15.8",
"license": "MIT"
"version": "0.15.10",
"resolved": "https://registry.npmjs.org/@lezer/common/-/common-0.15.10.tgz",
"integrity": "sha512-vlr+be73zTDoQBIknBVOh/633tmbQcjxUu9PIeVeYESeBK3V6TuBW96RRFg93Y2cyK9lglz241gOgSn452HFvA=="
},
"node_modules/@lezer/generator": {
"version": "0.15.2",
@ -1775,9 +1786,9 @@
"license": "MIT"
},
"node_modules/@types/node": {
"version": "16.11.9",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.9.tgz",
"integrity": "sha512-MKmdASMf3LtPzwLyRrFjtFFZ48cMf8jmX5VRYrDQiJa8Ybu5VAmkqBWqKU8fdCwD8ysw4mQ9nrEHvzg6gunR7A==",
"version": "16.11.12",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.12.tgz",
"integrity": "sha512-+2Iggwg7PxoO5Kyhvsq9VarmPbIelXP070HMImEpbtGCoyWNINQj4wzjbQCXzdHTRXnqufutJb5KAURZANNBAw==",
"dev": true
},
"node_modules/@types/prop-types": {
@ -4141,6 +4152,11 @@
"minimatch": "^3.0.4"
}
},
"node_modules/immutable": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/immutable/-/immutable-4.0.0.tgz",
"integrity": "sha512-zIE9hX70qew5qTUjSS7wi1iwj/l7+m54KWU247nhM3v806UdGj1yDndXj+IOYxxtW9zyLI+xqFNZjTuDaLUqFw=="
},
"node_modules/import-fresh": {
"version": "3.3.0",
"dev": true,
@ -5754,9 +5770,10 @@
}
},
"node_modules/prettier": {
"version": "2.4.1",
"version": "2.5.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.5.1.tgz",
"integrity": "sha512-vBZcPRUR5MZJwoyi3ZoyQlc1rXeEck8KgeC9AwwOn+exuxLxq5toTRDTSaVrXHxelDMHy9zlicw8u66yxoSUFg==",
"dev": true,
"license": "MIT",
"bin": {
"prettier": "bin-prettier.js"
},
@ -6147,9 +6164,9 @@
"license": "MIT"
},
"node_modules/sanitize-html": {
"version": "2.5.3",
"resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.5.3.tgz",
"integrity": "sha512-DGATXd1fs/Rm287/i5FBKVYSBBUL0iAaztOA1/RFhEs4yqo39/X52i/q/CwsfCUG5cilmXSBmnQmyWfnKhBlOg==",
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.6.0.tgz",
"integrity": "sha512-qc+NqTeaOi/QVAVs5gOY9tqIVk4r1pcUbx1Q2N1a609Os3TNlm85zll2d8g8m/RM6RWg9llir0SpAcePQVIC1w==",
"dependencies": {
"deepmerge": "^4.2.2",
"escape-string-regexp": "^4.0.0",
@ -6160,10 +6177,12 @@
}
},
"node_modules/sass": {
"version": "1.43.4",
"license": "MIT",
"version": "1.44.0",
"resolved": "https://registry.npmjs.org/sass/-/sass-1.44.0.tgz",
"integrity": "sha512-0hLREbHFXGQqls/K8X+koeP+ogFRPF4ZqetVB19b7Cst9Er8cOR0rc6RU7MaI4W1JmUShd1BPgPoeqmmgMMYFw==",
"dependencies": {
"chokidar": ">=3.0.0 <4.0.0"
"chokidar": ">=3.0.0 <4.0.0",
"immutable": "^4.0.0"
},
"bin": {
"sass": "sass.js"
@ -7207,21 +7226,21 @@
"name": "graph",
"version": "0.1.0",
"dependencies": {
"@codemirror/autocomplete": "^0.19.8",
"@codemirror/autocomplete": "^0.19.9",
"@codemirror/closebrackets": "^0.19.0",
"@codemirror/commands": "^0.19.5",
"@codemirror/comment": "^0.19.0",
"@codemirror/highlight": "^0.19.6",
"@codemirror/history": "^0.19.0",
"@codemirror/language": "^0.19.5",
"@codemirror/language": "^0.19.7",
"@codemirror/lint": "^0.19.3",
"@codemirror/matchbrackets": "^0.19.3",
"@codemirror/search": "^0.19.3",
"@codemirror/state": "^0.19.6",
"@codemirror/view": "^0.19.20",
"@codemirror/view": "^0.19.27",
"@forevolve/bootstrap-dark": "^1.0.0",
"@fortawesome/fontawesome-svg-core": "^1.2.14",
"@fortawesome/free-solid-svg-icons": "^5.7.1",
"@fortawesome/fontawesome-svg-core": "^1.2.36",
"@fortawesome/free-solid-svg-icons": "^5.7.2",
"@fortawesome/react-fontawesome": "^0.1.16",
"@nexucis/fuzzy": "^0.3.0",
"bootstrap": "^4.6.1",
@ -7241,8 +7260,8 @@
"react-router-dom": "^5.2.1",
"react-test-renderer": "^17.0.2",
"reactstrap": "^8.9.0",
"sanitize-html": "^2.5.3",
"sass": "1.43.4",
"sanitize-html": "^2.6.0",
"sass": "1.44.0",
"tempusdominus-bootstrap-4": "^5.1.2",
"tempusdominus-core": "^5.0.3"
},
@ -7252,7 +7271,7 @@
"@types/flot": "0.0.32",
"@types/jest": "^27.0.3",
"@types/jquery": "^3.5.9",
"@types/node": "^16.11.9",
"@types/node": "^16.11.12",
"@types/react": "^17.0.36",
"@types/react-copy-to-clipboard": "^5.0.2",
"@types/react-dom": "^17.0.11",
@ -7269,7 +7288,7 @@
"jest-canvas-mock": "^2.3.1",
"jest-fetch-mock": "^3.0.3",
"mutationobserver-shim": "^0.3.7",
"prettier": "^2.4.1",
"prettier": "^2.5.1",
"react-scripts": "4.0.3",
"sinon": "^12.0.1",
"typescript": "^4.5.2"
@ -27282,9 +27301,9 @@
}
},
"@codemirror/autocomplete": {
"version": "0.19.8",
"resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-0.19.8.tgz",
"integrity": "sha512-o4I1pRlFjhBHOYab+QfpKcW0B8FqAH+2pdmCYrkTz3bm1djVwhlMEhv1s/aTKhdjLtkfZFUbdHBi+8xe22wUCA==",
"version": "0.19.9",
"resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-0.19.9.tgz",
"integrity": "sha512-Ph1LWHtFFqNUIqEVrws6I263ihe5TH+TRBPwxQ78j7st7Q67FDAmgKX6mNbUPh02dxfqQrc9qxlo5JIqKeiVdg==",
"requires": {
"@codemirror/language": "^0.19.0",
"@codemirror/state": "^0.19.4",
@ -27384,9 +27403,9 @@
}
},
"@codemirror/language": {
"version": "0.19.5",
"resolved": "https://registry.npmjs.org/@codemirror/language/-/language-0.19.5.tgz",
"integrity": "sha512-FnIST07vaM99mv1mJaMMLvxiHSDGgP3wdlcEZzmidndWdbxjrYYYnJzVUOEkeZJNGOfrtPRMF62UCyrTjQMR3g==",
"version": "0.19.7",
"resolved": "https://registry.npmjs.org/@codemirror/language/-/language-0.19.7.tgz",
"integrity": "sha512-pNNUtYWMIMG0lUSKyUXJr8U0rFiCKsKFXbA2Oj17PC+S1FY99hV0z1vcntW67ekAIZw9DMEUQnLsKBuIbAUX7Q==",
"requires": {
"@codemirror/state": "^0.19.0",
"@codemirror/text": "^0.19.0",
@ -27476,9 +27495,9 @@
}
},
"@codemirror/view": {
"version": "0.19.20",
"resolved": "https://registry.npmjs.org/@codemirror/view/-/view-0.19.20.tgz",
"integrity": "sha512-j4cI/Egdhha77pMfKQQWZmpkcF7vhe21LqdZs8hsG09OtvsPVCHUXmfB0u7nMpcX1JR8aZ3ob9g6FMr+OVtjgA==",
"version": "0.19.27",
"resolved": "https://registry.npmjs.org/@codemirror/view/-/view-0.19.27.tgz",
"integrity": "sha512-Uz/LecEf7CyvMWaQBlKtbJCYn0hRnEZ2yYvuZVy9YMhmvGmES6ec7FaKw7lDFFOMLwLbBThc9kfw4DCHreHN1w==",
"requires": {
"@codemirror/rangeset": "^0.19.0",
"@codemirror/state": "^0.19.3",
@ -27548,7 +27567,9 @@
}
},
"@humanwhocodes/object-schema": {
"version": "1.2.0",
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz",
"integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
"dev": true
},
"@istanbuljs/load-nyc-config": {
@ -27623,7 +27644,9 @@
}
},
"@lezer/common": {
"version": "0.15.8"
"version": "0.15.10",
"resolved": "https://registry.npmjs.org/@lezer/common/-/common-0.15.10.tgz",
"integrity": "sha512-vlr+be73zTDoQBIknBVOh/633tmbQcjxUu9PIeVeYESeBK3V6TuBW96RRFg93Y2cyK9lglz241gOgSn452HFvA=="
},
"@lezer/generator": {
"version": "0.15.2",
@ -27808,9 +27831,9 @@
"dev": true
},
"@types/node": {
"version": "16.11.9",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.9.tgz",
"integrity": "sha512-MKmdASMf3LtPzwLyRrFjtFFZ48cMf8jmX5VRYrDQiJa8Ybu5VAmkqBWqKU8fdCwD8ysw4mQ9nrEHvzg6gunR7A==",
"version": "16.11.12",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.12.tgz",
"integrity": "sha512-+2Iggwg7PxoO5Kyhvsq9VarmPbIelXP070HMImEpbtGCoyWNINQj4wzjbQCXzdHTRXnqufutJb5KAURZANNBAw==",
"dev": true
},
"@types/prop-types": {
@ -28401,24 +28424,24 @@
"codemirror-promql": {
"version": "file:module/codemirror-promql",
"requires": {
"@codemirror/autocomplete": "^0.19.8",
"@codemirror/autocomplete": "^0.19.9",
"@codemirror/basic-setup": "^0.19.0",
"@codemirror/highlight": "^0.19.6",
"@codemirror/language": "^0.19.5",
"@codemirror/language": "^0.19.7",
"@codemirror/lint": "^0.19.3",
"@codemirror/state": "^0.19.6",
"@codemirror/view": "^0.19.20",
"@lezer/common": "^0.15.8",
"@codemirror/view": "^0.19.27",
"@lezer/common": "^0.15.10",
"@lezer/generator": "^0.15.2",
"@types/chai": "^4.2.22",
"@types/lru-cache": "^5.1.0",
"@types/lru-cache": "^5.1.1",
"@types/mocha": "^9.0.0",
"@types/node": "^16.11.9",
"@typescript-eslint/eslint-plugin": "^5.3.1",
"@typescript-eslint/parser": "^5.3.1",
"@types/node": "^16.11.12",
"@typescript-eslint/eslint-plugin": "^5.5.0",
"@typescript-eslint/parser": "^5.5.0",
"chai": "^4.2.0",
"codecov": "^3.8.1",
"eslint": "^8.3.0",
"codecov": "^3.8.3",
"eslint": "^8.4.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-flowtype": "^8.0.3",
"eslint-plugin-import": "^2.25.3",
@ -28428,20 +28451,22 @@
"mocha": "^8.4.0",
"nock": "^13.2.1",
"nyc": "^15.1.0",
"prettier": "^2.4.1",
"ts-loader": "^7.0.4",
"prettier": "^2.5.1",
"ts-loader": "^7.0.5",
"ts-mocha": "^8.0.0",
"ts-node": "^10.4.0",
"typescript": "^4.5.2"
},
"dependencies": {
"@eslint/eslintrc": {
"version": "1.0.4",
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.0.5.tgz",
"integrity": "sha512-BLxsnmK3KyPunz5wmCCpqy0YelEoxxGmH73Is+Z74oOTMtExcjkr3dDR6quwrjh1YspA8DH9gnX1o069KiS9AQ==",
"dev": true,
"requires": {
"ajv": "^6.12.4",
"debug": "^4.3.2",
"espree": "^9.0.0",
"espree": "^9.2.0",
"globals": "^13.9.0",
"ignore": "^4.0.6",
"import-fresh": "^3.2.1",
@ -28451,20 +28476,24 @@
}
},
"@humanwhocodes/config-array": {
"version": "0.6.0",
"version": "0.9.2",
"resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.2.tgz",
"integrity": "sha512-UXOuFCGcwciWckOpmfKDq/GyhlTf9pN/BzG//x8p8zTOFEcGuA68ANXheFS0AGvy3qgZqLBUkMs7hqzqCKOVwA==",
"dev": true,
"requires": {
"@humanwhocodes/object-schema": "^1.2.0",
"@humanwhocodes/object-schema": "^1.2.1",
"debug": "^4.1.1",
"minimatch": "^3.0.4"
}
},
"@typescript-eslint/eslint-plugin": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.5.0.tgz",
"integrity": "sha512-4bV6fulqbuaO9UMXU0Ia0o6z6if+kmMRW8rMRyfqXj/eGrZZRGedS4n0adeGNnjr8LKAM495hrQ7Tea52UWmQA==",
"dev": true,
"requires": {
"@typescript-eslint/experimental-utils": "5.4.0",
"@typescript-eslint/scope-manager": "5.4.0",
"@typescript-eslint/experimental-utils": "5.5.0",
"@typescript-eslint/scope-manager": "5.5.0",
"debug": "^4.3.2",
"functional-red-black-tree": "^1.0.1",
"ignore": "^5.1.8",
@ -28480,45 +28509,55 @@
}
},
"@typescript-eslint/experimental-utils": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-5.5.0.tgz",
"integrity": "sha512-kjWeeVU+4lQ1SLYErRKV5yDXbWDPkpbzTUUlfAUifPYvpX0qZlrcCZ96/6oWxt3QxtK5WVhXz+KsnwW9cIW+3A==",
"dev": true,
"requires": {
"@types/json-schema": "^7.0.9",
"@typescript-eslint/scope-manager": "5.4.0",
"@typescript-eslint/types": "5.4.0",
"@typescript-eslint/typescript-estree": "5.4.0",
"@typescript-eslint/scope-manager": "5.5.0",
"@typescript-eslint/types": "5.5.0",
"@typescript-eslint/typescript-estree": "5.5.0",
"eslint-scope": "^5.1.1",
"eslint-utils": "^3.0.0"
}
},
"@typescript-eslint/parser": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.5.0.tgz",
"integrity": "sha512-JsXBU+kgQOAgzUn2jPrLA+Rd0Y1dswOlX3hp8MuRO1hQDs6xgHtbCXEiAu7bz5hyVURxbXcA2draasMbNqrhmg==",
"dev": true,
"requires": {
"@typescript-eslint/scope-manager": "5.4.0",
"@typescript-eslint/types": "5.4.0",
"@typescript-eslint/typescript-estree": "5.4.0",
"@typescript-eslint/scope-manager": "5.5.0",
"@typescript-eslint/types": "5.5.0",
"@typescript-eslint/typescript-estree": "5.5.0",
"debug": "^4.3.2"
}
},
"@typescript-eslint/scope-manager": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.5.0.tgz",
"integrity": "sha512-0/r656RmRLo7CbN4Mdd+xZyPJ/fPCKhYdU6mnZx+8msAD8nJSP8EyCFkzbd6vNVZzZvWlMYrSNekqGrCBqFQhg==",
"dev": true,
"requires": {
"@typescript-eslint/types": "5.4.0",
"@typescript-eslint/visitor-keys": "5.4.0"
"@typescript-eslint/types": "5.5.0",
"@typescript-eslint/visitor-keys": "5.5.0"
}
},
"@typescript-eslint/types": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.5.0.tgz",
"integrity": "sha512-OaYTqkW3GnuHxqsxxJ6KypIKd5Uw7bFiQJZRyNi1jbMJnK3Hc/DR4KwB6KJj6PBRkJJoaNwzMNv9vtTk87JhOg==",
"dev": true
},
"@typescript-eslint/typescript-estree": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.5.0.tgz",
"integrity": "sha512-pVn8btYUiYrjonhMAO0yG8lm7RApzy2L4RC7Td/mC/qFkyf6vRbGyZozoA94+w6D2Y2GRqpMoCWcwx/EUOzyoQ==",
"dev": true,
"requires": {
"@typescript-eslint/types": "5.4.0",
"@typescript-eslint/visitor-keys": "5.4.0",
"@typescript-eslint/types": "5.5.0",
"@typescript-eslint/visitor-keys": "5.5.0",
"debug": "^4.3.2",
"globby": "^11.0.4",
"is-glob": "^4.0.3",
@ -28527,10 +28566,12 @@
}
},
"@typescript-eslint/visitor-keys": {
"version": "5.4.0",
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.5.0.tgz",
"integrity": "sha512-4GzJ1kRtsWzHhdM40tv0ZKHNSbkDhF0Woi/TDwVJX6UICwJItvP7ZTXbjTkCdrors7ww0sYe0t+cIKDAJwZ7Kw==",
"dev": true,
"requires": {
"@typescript-eslint/types": "5.4.0",
"@typescript-eslint/types": "5.5.0",
"eslint-visitor-keys": "^3.0.0"
}
},
@ -28568,13 +28609,13 @@
}
},
"eslint": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.3.0.tgz",
"integrity": "sha512-aIay56Ph6RxOTC7xyr59Kt3ewX185SaGnAr8eWukoPLeriCrvGjvAubxuvaXOfsxhtwV5g0uBOsyhAom4qJdww==",
"version": "8.4.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.4.0.tgz",
"integrity": "sha512-kv0XQcAQJL/VD9THQKhTQZVqkJKA+tIj/v2ZKNaIHRAADcJWFb+B/BAewUYuF6UVg1s2xC5qXVoDk0G8sKGeTA==",
"dev": true,
"requires": {
"@eslint/eslintrc": "^1.0.4",
"@humanwhocodes/config-array": "^0.6.0",
"@eslint/eslintrc": "^1.0.5",
"@humanwhocodes/config-array": "^0.9.2",
"ajv": "^6.10.0",
"chalk": "^4.0.0",
"cross-spawn": "^7.0.2",
@ -28585,7 +28626,7 @@
"eslint-scope": "^7.1.0",
"eslint-utils": "^3.0.0",
"eslint-visitor-keys": "^3.1.0",
"espree": "^9.1.0",
"espree": "^9.2.0",
"esquery": "^1.4.0",
"esutils": "^2.0.2",
"fast-deep-equal": "^3.1.3",
@ -28638,9 +28679,9 @@
"dev": true
},
"espree": {
"version": "9.1.0",
"resolved": "https://registry.npmjs.org/espree/-/espree-9.1.0.tgz",
"integrity": "sha512-ZgYLvCS1wxOczBYGcQT9DDWgicXwJ4dbocr9uYN+/eresBAUuBu+O4WzB21ufQ/JqQT8gyp7hJ3z8SHii32mTQ==",
"version": "9.2.0",
"resolved": "https://registry.npmjs.org/espree/-/espree-9.2.0.tgz",
"integrity": "sha512-oP3utRkynpZWF/F2x/HZJ+AGtnIclaR7z1pYPxy7NYM2fSO6LgK/Rkny8anRSPK/VwEA1eqm2squui0T7ZMOBg==",
"dev": true,
"requires": {
"acorn": "^8.6.0",
@ -29623,21 +29664,21 @@
"graph": {
"version": "file:react-app",
"requires": {
"@codemirror/autocomplete": "^0.19.8",
"@codemirror/autocomplete": "^0.19.9",
"@codemirror/closebrackets": "^0.19.0",
"@codemirror/commands": "^0.19.5",
"@codemirror/comment": "^0.19.0",
"@codemirror/highlight": "^0.19.6",
"@codemirror/history": "^0.19.0",
"@codemirror/language": "^0.19.5",
"@codemirror/language": "^0.19.7",
"@codemirror/lint": "^0.19.3",
"@codemirror/matchbrackets": "^0.19.3",
"@codemirror/search": "^0.19.3",
"@codemirror/state": "^0.19.6",
"@codemirror/view": "^0.19.20",
"@codemirror/view": "^0.19.27",
"@forevolve/bootstrap-dark": "^1.0.0",
"@fortawesome/fontawesome-svg-core": "^1.2.14",
"@fortawesome/free-solid-svg-icons": "^5.7.1",
"@fortawesome/fontawesome-svg-core": "^1.2.36",
"@fortawesome/free-solid-svg-icons": "^5.7.2",
"@fortawesome/react-fontawesome": "^0.1.16",
"@nexucis/fuzzy": "^0.3.0",
"@testing-library/react-hooks": "^7.0.1",
@ -29645,7 +29686,7 @@
"@types/flot": "0.0.32",
"@types/jest": "^27.0.3",
"@types/jquery": "^3.5.9",
"@types/node": "^16.11.9",
"@types/node": "^16.11.12",
"@types/react": "^17.0.36",
"@types/react-copy-to-clipboard": "^5.0.2",
"@types/react-dom": "^17.0.11",
@ -29673,7 +29714,7 @@
"moment-timezone": "^0.5.34",
"mutationobserver-shim": "^0.3.7",
"popper.js": "^1.14.3",
"prettier": "^2.4.1",
"prettier": "^2.5.1",
"react": "^17.0.2",
"react-copy-to-clipboard": "^5.0.4",
"react-dom": "^17.0.2",
@ -29682,8 +29723,8 @@
"react-scripts": "4.0.3",
"react-test-renderer": "^17.0.2",
"reactstrap": "^8.9.0",
"sanitize-html": "^2.5.3",
"sass": "1.43.4",
"sanitize-html": "^2.6.0",
"sass": "1.44.0",
"sinon": "^12.0.1",
"tempusdominus-bootstrap-4": "^5.1.2",
"tempusdominus-core": "^5.0.3",
@ -43260,6 +43301,11 @@
"minimatch": "^3.0.4"
}
},
"immutable": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/immutable/-/immutable-4.0.0.tgz",
"integrity": "sha512-zIE9hX70qew5qTUjSS7wi1iwj/l7+m54KWU247nhM3v806UdGj1yDndXj+IOYxxtW9zyLI+xqFNZjTuDaLUqFw=="
},
"import-fresh": {
"version": "3.3.0",
"dev": true,
@ -44324,7 +44370,9 @@
"dev": true
},
"prettier": {
"version": "2.4.1",
"version": "2.5.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.5.1.tgz",
"integrity": "sha512-vBZcPRUR5MZJwoyi3ZoyQlc1rXeEck8KgeC9AwwOn+exuxLxq5toTRDTSaVrXHxelDMHy9zlicw8u66yxoSUFg==",
"dev": true
},
"prettier-linter-helpers": {
@ -44576,9 +44624,9 @@
"dev": true
},
"sanitize-html": {
"version": "2.5.3",
"resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.5.3.tgz",
"integrity": "sha512-DGATXd1fs/Rm287/i5FBKVYSBBUL0iAaztOA1/RFhEs4yqo39/X52i/q/CwsfCUG5cilmXSBmnQmyWfnKhBlOg==",
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.6.0.tgz",
"integrity": "sha512-qc+NqTeaOi/QVAVs5gOY9tqIVk4r1pcUbx1Q2N1a609Os3TNlm85zll2d8g8m/RM6RWg9llir0SpAcePQVIC1w==",
"requires": {
"deepmerge": "^4.2.2",
"escape-string-regexp": "^4.0.0",
@ -44589,9 +44637,12 @@
}
},
"sass": {
"version": "1.43.4",
"version": "1.44.0",
"resolved": "https://registry.npmjs.org/sass/-/sass-1.44.0.tgz",
"integrity": "sha512-0hLREbHFXGQqls/K8X+koeP+ogFRPF4ZqetVB19b7Cst9Er8cOR0rc6RU7MaI4W1JmUShd1BPgPoeqmmgMMYFw==",
"requires": {
"chokidar": ">=3.0.0 <4.0.0"
"chokidar": ">=3.0.0 <4.0.0",
"immutable": "^4.0.0"
}
},
"scheduler": {

18
web/ui/react-app/package.json
View file

@ -3,21 +3,21 @@
"version": "0.1.0",
"private": true,
"dependencies": {
"@codemirror/autocomplete": "^0.19.8",
"@codemirror/autocomplete": "^0.19.9",
"@codemirror/closebrackets": "^0.19.0",
"@codemirror/commands": "^0.19.5",
"@codemirror/comment": "^0.19.0",
"@codemirror/highlight": "^0.19.6",
"@codemirror/history": "^0.19.0",
"@codemirror/language": "^0.19.5",
"@codemirror/language": "^0.19.7",
"@codemirror/lint": "^0.19.3",
"@codemirror/matchbrackets": "^0.19.3",
"@codemirror/search": "^0.19.3",
"@codemirror/state": "^0.19.6",
"@codemirror/view": "^0.19.20",
"@codemirror/view": "^0.19.27",
"@forevolve/bootstrap-dark": "^1.0.0",
"@fortawesome/fontawesome-svg-core": "^1.2.14",
"@fortawesome/free-solid-svg-icons": "^5.7.1",
"@fortawesome/fontawesome-svg-core": "^1.2.36",
"@fortawesome/free-solid-svg-icons": "^5.7.2",
"@fortawesome/react-fontawesome": "^0.1.16",
"@nexucis/fuzzy": "^0.3.0",
"bootstrap": "^4.6.1",
@ -37,8 +37,8 @@
"react-router-dom": "^5.2.1",
"react-test-renderer": "^17.0.2",
"reactstrap": "^8.9.0",
"sanitize-html": "^2.5.3",
"sass": "1.43.4",
"sanitize-html": "^2.6.0",
"sass": "1.44.0",
"tempusdominus-bootstrap-4": "^5.1.2",
"tempusdominus-core": "^5.0.3"
},
@ -69,7 +69,7 @@
"@types/flot": "0.0.32",
"@types/jest": "^27.0.3",
"@types/jquery": "^3.5.9",
"@types/node": "^16.11.9",
"@types/node": "^16.11.12",
"@types/react": "^17.0.36",
"@types/react-copy-to-clipboard": "^5.0.2",
"@types/react-dom": "^17.0.11",
@ -86,7 +86,7 @@
"jest-canvas-mock": "^2.3.1",
"jest-fetch-mock": "^3.0.3",
"mutationobserver-shim": "^0.3.7",
"prettier": "^2.4.1",
"prettier": "^2.5.1",
"react-scripts": "4.0.3",
"sinon": "^12.0.1",
"typescript": "^4.5.2"

3
web/ui/react-app/public/index.html
View file

@ -15,10 +15,11 @@
It will render a "Consoles" link in the navbar when it is non-empty.
- PROMETHEUS_AGENT_MODE is replaced by a boolean indicating if Prometheus is running in agent mode.
It true, it will disable querying capacities in the UI and generally adapt the UI to the agent mode.
It has to be represented as a string, because booleans can be mangled to !1 in production builds.
-->
<script>
const GLOBAL_CONSOLES_LINK='CONSOLES_LINK_PLACEHOLDER';
const GLOBAL_PROMETHEUS_AGENT_MODE=PROMETHEUS_AGENT_MODE_PLACEHOLDER;
const GLOBAL_AGENT_MODE='AGENT_MODE_PLACEHOLDER';
</script>
<!--

4
web/ui/react-app/src/App.test.tsx
View file

@ -5,6 +5,7 @@ import Navigation from './Navbar';
import { Container } from 'reactstrap';
import { Route } from 'react-router-dom';
import {
AgentPage,
AlertsPage,
ConfigPage,
FlagsPage,
@ -24,6 +25,7 @@ describe('App', () => {
});
it('routes', () => {
[
AgentPage,
AlertsPage,
ConfigPage,
FlagsPage,
@ -37,7 +39,7 @@ describe('App', () => {
const c = app.find(component);
expect(c).toHaveLength(1);
});
expect(app.find(Route)).toHaveLength(9);
expect(app.find(Route)).toHaveLength(10);
expect(app.find(Container)).toHaveLength(1);
});
});

12
web/ui/react-app/src/App.tsx
View file

@ -4,6 +4,7 @@ import { Container } from 'reactstrap';
import { BrowserRouter as Router, Redirect, Switch, Route } from 'react-router-dom';
import {
AgentPage,
AlertsPage,
ConfigPage,
FlagsPage,
@ -22,14 +23,16 @@ import useMedia from './hooks/useMedia';
interface AppProps {
consolesLink: string | null;
agentMode: boolean;
}
const App: FC<AppProps> = ({ consolesLink }) => {
const App: FC<AppProps> = ({ consolesLink, agentMode }) => {
// This dynamically/generically determines the pathPrefix by stripping the first known
// endpoint suffix from the window location path. It works out of the box for both direct
// hosting and reverse proxy deployments with no additional configurations required.
let basePath = window.location.pathname;
const paths = [
'/agent',
'/graph',
'/alerts',
'/status',
@ -70,14 +73,17 @@ const App: FC<AppProps> = ({ consolesLink }) => {
<Theme />
<PathPrefixContext.Provider value={basePath}>
<Router basename={basePath}>
<Navigation consolesLink={consolesLink} />
<Navigation consolesLink={consolesLink} agentMode={agentMode} />
<Container fluid style={{ paddingTop: 70 }}>
<Switch>
<Redirect exact from="/" to={`graph`} />
<Redirect exact from="/" to={agentMode ? '/agent' : '/graph'} />
{/*
NOTE: Any route added here needs to also be added to the list of
React-handled router paths ("reactRouterPaths") in /web/web.go.
*/}
<Route path="/agent">
<AgentPage />
</Route>
<Route path="/graph">
<PanelListPage />
</Route>

55
web/ui/react-app/src/Navbar.tsx
View file

@ -17,17 +17,18 @@ import { ThemeToggle } from './Theme';
interface NavbarProps {
consolesLink: string | null;
agentMode: boolean;
}
const Navigation: FC<NavbarProps> = ({ consolesLink }) => {
const Navigation: FC<NavbarProps> = ({ consolesLink, agentMode }) => {
const [isOpen, setIsOpen] = useState(false);
const toggle = () => setIsOpen(!isOpen);
const pathPrefix = usePathPrefix();
return (
<Navbar className="mb-3" dark color="dark" expand="md" fixed="top">
<NavbarToggler onClick={toggle} className="mr-2" />
<Link className="pt-0 navbar-brand" to="/graph">
Prometheus
<Link className="pt-0 navbar-brand" to={agentMode ? '/agent' : '/graph'}>
Prometheus{agentMode && ' Agent'}
</Link>
<Collapse isOpen={isOpen} navbar style={{ justifyContent: 'space-between' }}>
<Nav className="ml-0" navbar>
@ -36,16 +37,20 @@ const Navigation: FC<NavbarProps> = ({ consolesLink }) => {
<NavLink href={consolesLink}>Consoles</NavLink>
</NavItem>
)}
<NavItem>
<NavLink tag={Link} to="/alerts">
Alerts
</NavLink>
</NavItem>
<NavItem>
<NavLink tag={Link} to="/graph">
Graph
</NavLink>
</NavItem>
{!agentMode && (
<>
<NavItem>
<NavLink tag={Link} to="/alerts">
Alerts
</NavLink>
</NavItem>
<NavItem>
<NavLink tag={Link} to="/graph">
Graph
</NavLink>
</NavItem>
</>
)}
<UncontrolledDropdown nav inNavbar>
<DropdownToggle nav caret>
Status
@ -54,18 +59,22 @@ const Navigation: FC<NavbarProps> = ({ consolesLink }) => {
<DropdownItem tag={Link} to="/status">
Runtime & Build Information
</DropdownItem>
<DropdownItem tag={Link} to="/tsdb-status">
TSDB Status
</DropdownItem>
{!agentMode && (
<DropdownItem tag={Link} to="/tsdb-status">
TSDB Status
</DropdownItem>
)}
<DropdownItem tag={Link} to="/flags">
Command-Line Flags
</DropdownItem>
<DropdownItem tag={Link} to="/config">
Configuration
</DropdownItem>
<DropdownItem tag={Link} to="/rules">
Rules
</DropdownItem>
{!agentMode && (
<DropdownItem tag={Link} to="/rules">
Rules
</DropdownItem>
)}
<DropdownItem tag={Link} to="/targets">
Targets
</DropdownItem>
@ -77,9 +86,11 @@ const Navigation: FC<NavbarProps> = ({ consolesLink }) => {
<NavItem>
<NavLink href="https://prometheus.io/docs/prometheus/latest/getting_started/">Help</NavLink>
</NavItem>
<NavItem>
<NavLink href={`${pathPrefix}/classic/graph${window.location.search}`}>Classic UI</NavLink>
</NavItem>
{!agentMode && (
<NavItem>
<NavLink href={`${pathPrefix}/classic/graph${window.location.search}`}>Classic UI</NavLink>
</NavItem>
)}
</Nav>
</Collapse>
<ThemeToggle />

4
web/ui/react-app/src/index.tsx
View file

@ -10,8 +10,10 @@ import { isPresent } from './utils';
// Declared/defined in public/index.html, value replaced by Prometheus when serving bundle.
declare const GLOBAL_CONSOLES_LINK: string;
declare const GLOBAL_AGENT_MODE: string;
let consolesLink: string | null = GLOBAL_CONSOLES_LINK;
const agentMode: string | null = GLOBAL_AGENT_MODE;
if (
GLOBAL_CONSOLES_LINK === 'CONSOLES_LINK_PLACEHOLDER' ||
@ -21,4 +23,4 @@ if (
consolesLink = null;
}
ReactDOM.render(<App consolesLink={consolesLink} />, document.getElementById('root'));
ReactDOM.render(<App consolesLink={consolesLink} agentMode={agentMode === 'true'} />, document.getElementById('root'));

16
web/ui/react-app/src/pages/agent/Agent.tsx Normal file
View file

@ -0,0 +1,16 @@
import React, { FC } from 'react';
const Agent: FC = () => {
return (
<>
<h2>Prometheus Agent</h2>
<p>
This Prometheus instance is running in <strong>agent mode</strong>. In this mode, Prometheus is only used to scrape
discovered targets and forward them to remote write endpoints.
</p>
<p>Some features are not available in this mode, such as querying and alerting.</p>
</>
);
};
export default Agent;

3
web/ui/react-app/src/pages/index.ts
View file

@ -1,3 +1,4 @@
import Agent from './agent/Agent';
import Alerts from './alerts/Alerts';
import Config from './config/Config';
import Flags from './flags/Flags';
@ -9,6 +10,7 @@ import PanelList from './graph/PanelList';
import TSDBStatus from './tsdbStatus/TSDBStatus';
import { withStartingIndicator } from '../components/withStartingIndicator';
const AgentPage = withStartingIndicator(Agent);
const AlertsPage = withStartingIndicator(Alerts);
const ConfigPage = withStartingIndicator(Config);
const FlagsPage = withStartingIndicator(Flags);
@ -21,6 +23,7 @@ const PanelListPage = withStartingIndicator(PanelList);
// prettier-ignore
export {
AgentPage,
AlertsPage,
ConfigPage,
FlagsPage,

25
web/ui/react-app/src/pages/targets/EndpointLink.test.tsx
View file

@ -1,6 +1,6 @@
import React from 'react';
import { shallow, mount } from 'enzyme';
import { Badge, Alert } from 'reactstrap';
import { Badge } from 'reactstrap';
import EndpointLink from './EndpointLink';
describe('EndpointLink', () => {
@ -29,11 +29,24 @@ describe('EndpointLink', () => {
const targetLabel = badges.filterWhere((badge) => badge.children().text() === 'target="http://some-service"');
expect(targetLabel.length).toEqual(1);
});
it('renders an alert if url is invalid', () => {
const endpointLink = shallow(<EndpointLink endpoint={'afdsacas'} globalUrl={'afdsacas'} />);
const err = endpointLink.find(Alert);
expect(err.render().text()).toEqual('Error: Invalid URL: afdsacas');
// In cases of IPv6 addresses with a Zone ID, URL may not be parseable.
// See https://github.com/prometheus/prometheus/issues/9760
it('renders an anchor for IPv6 link with zone ID including labels for query params', () => {
const endpoint =
'http://[fe80::f1ee:adeb:371d:983%eth1]:9100/stats/prometheus?module=http_2xx&target=http://some-service';
const globalURL =
'http://[fe80::f1ee:adeb:371d:983%eth1]:9100/stats/prometheus?module=http_2xx&target=http://some-service';
const endpointLink = shallow(<EndpointLink endpoint={endpoint} globalUrl={globalURL} />);
const anchor = endpointLink.find('a');
const badges = endpointLink.find(Badge);
expect(anchor.prop('href')).toEqual(globalURL);
expect(anchor.children().text()).toEqual('http://[fe80::f1ee:adeb:371d:983%eth1]:9100/stats/prometheus');
expect(endpointLink.find('br')).toHaveLength(1);
expect(badges).toHaveLength(2);
const moduleLabel = badges.filterWhere((badge) => badge.children().text() === 'module="http_2xx"');
expect(moduleLabel.length).toEqual(1);
const targetLabel = badges.filterWhere((badge) => badge.children().text() === 'target="http://some-service"');
expect(targetLabel.length).toEqual(1);
});
it('handles params with multiple values correctly', () => {

23
web/ui/react-app/src/pages/targets/EndpointLink.tsx
View file

@ -1,5 +1,5 @@
import React, { FC } from 'react';
import { Badge, Alert } from 'reactstrap';
import { Badge } from 'reactstrap';
export interface EndpointLinkProps {
endpoint: string;
@ -8,23 +8,28 @@ export interface EndpointLinkProps {
const EndpointLink: FC<EndpointLinkProps> = ({ endpoint, globalUrl }) => {
let url: URL;
let search = '';
let invalidURL = false;
try {
url = new URL(endpoint);
} catch (err: unknown) {
const error = err as Error;
return (
<Alert color="danger">
<strong>Error:</strong> {error.message}
</Alert>
);
// In cases of IPv6 addresses with a Zone ID, URL may not be parseable.
// See https://github.com/prometheus/prometheus/issues/9760
// In this case, we attempt to prepare a synthetic URL with the
// same query parameters, for rendering purposes.
invalidURL = true;
if (endpoint.indexOf('?') > -1) {
search = endpoint.substring(endpoint.indexOf('?'));
}
url = new URL('http://0.0.0.0' + search);
}
const { host, pathname, protocol, searchParams }: URL = url;
const params = Array.from(searchParams.entries());
const displayLink = invalidURL ? endpoint.replace(search, '') : `${protocol}//${host}${pathname}`;
return (
<>
<a href={globalUrl}>{`${protocol}//${host}${pathname}`}</a>
<a href={globalUrl}>{displayLink}</a>
{params.length > 0 ? <br /> : null}
{params.map(([labelName, labelValue]: [string, string]) => {
return (

36
web/web.go
View file

@ -71,18 +71,27 @@ import (
// Paths that are handled by the React / Reach router that should all be served the main React app's index.html.
var reactRouterPaths = []string{
"/alerts",
"/config",
"/flags",
"/graph",
"/rules",
"/service-discovery",
"/status",
"/targets",
"/tsdb-status",
"/starting",
}
// Paths that are handled by the React router when the Agent mode is set.
var reactRouterAgentPaths = []string{
"/agent",
}
// Paths that are handled by the React router when the Agent mode is not set.
var reactRouterServerPaths = []string{
"/alerts",
"/graph",
"/rules",
"/tsdb-status",
}
// withStackTrace logs the stack trace in case the request panics. The function
// will re-raise the error which will then be handled by the net/http package.
// It is needed because the go-kit log package doesn't manage properly the
@ -346,10 +355,15 @@ func New(logger log.Logger, o *Options) *Handler {
router = router.WithPrefix(o.RoutePrefix)
}
homePage := "/graph"
if o.IsAgent {
homePage = "/agent"
}
readyf := h.testReady
router.Get("/", func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, path.Join(o.ExternalURL.Path, "/graph"), http.StatusFound)
http.Redirect(w, r, path.Join(o.ExternalURL.Path, homePage), http.StatusFound)
})
router.Get("/classic/", func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, path.Join(o.ExternalURL.Path, "/classic/graph"), http.StatusFound)
@ -409,7 +423,7 @@ func New(logger log.Logger, o *Options) *Handler {
}
replacedIdx := bytes.ReplaceAll(idx, []byte("CONSOLES_LINK_PLACEHOLDER"), []byte(h.consolesPath()))
replacedIdx = bytes.ReplaceAll(replacedIdx, []byte("TITLE_PLACEHOLDER"), []byte(h.options.PageTitle))
replacedIdx = bytes.ReplaceAll(replacedIdx, []byte("PROMETHEUS_AGENT_MODE_PLACEHOLDER"), []byte(strconv.FormatBool(h.options.IsAgent)))
replacedIdx = bytes.ReplaceAll(replacedIdx, []byte("AGENT_MODE_PLACEHOLDER"), []byte(strconv.FormatBool(h.options.IsAgent)))
w.Write(replacedIdx)
}
@ -418,6 +432,16 @@ func New(logger log.Logger, o *Options) *Handler {
router.Get(p, serveReactApp)
}
if h.options.IsAgent {
for _, p := range reactRouterAgentPaths {
router.Get(p, serveReactApp)
}
} else {
for _, p := range reactRouterServerPaths {
router.Get(p, serveReactApp)
}
}
// The favicon and manifest are bundled as part of the React app, but we want to serve
// them on the root.
for _, p := range []string{"/favicon.ico", "/manifest.json"} {

15
web/web_test.go
View file

@ -112,9 +112,7 @@ func (a *dbAdapter) WALReplayStatus() (tsdb.WALReplayStatus, error) {
func TestReadyAndHealthy(t *testing.T) {
t.Parallel()
dbDir, err := ioutil.TempDir("", "tsdb-ready")
require.NoError(t, err)
defer func() { require.NoError(t, os.RemoveAll(dbDir)) }()
dbDir := t.TempDir()
db, err := tsdb.Open(dbDir, nil, nil, nil, nil)
require.NoError(t, err)
@ -235,9 +233,7 @@ func TestReadyAndHealthy(t *testing.T) {
func TestRoutePrefix(t *testing.T) {
t.Parallel()
dbDir, err := ioutil.TempDir("", "tsdb-ready")
require.NoError(t, err)
defer func() { require.NoError(t, os.RemoveAll(dbDir)) }()
dbDir := t.TempDir()
db, err := tsdb.Open(dbDir, nil, nil, nil, nil)
require.NoError(t, err)
@ -404,9 +400,7 @@ func TestHTTPMetrics(t *testing.T) {
}
func TestShutdownWithStaleConnection(t *testing.T) {
dbDir, err := ioutil.TempDir("", "tsdb-ready")
require.NoError(t, err)
defer func() { require.NoError(t, os.RemoveAll(dbDir)) }()
dbDir := t.TempDir()
db, err := tsdb.Open(dbDir, nil, nil, nil, nil)
require.NoError(t, err)
@ -585,6 +579,8 @@ func TestAgentAPIEndPoints(t *testing.T) {
"/query",
"/query_range",
"/query_exemplars",
"/graph",
"/rules",
} {
w := httptest.NewRecorder()
req, err := http.NewRequest("GET", baseURL+u, nil)
@ -595,6 +591,7 @@ func TestAgentAPIEndPoints(t *testing.T) {
// Test for available endpoints in the Agent mode.
for _, u := range []string{
"/agent",
"/targets",
"/status",
} {