From e635ca834bf14c6922810ca7cfdc87df4b638fa2 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Thu, 25 Mar 2021 22:28:58 +0100 Subject: [PATCH] Add environment variable expansion in external label values Signed-off-by: Julien Pivotto --- cmd/prometheus/main.go | 18 +++++--- cmd/promtool/main.go | 2 +- config/config.go | 28 ++++++++++-- config/config_test.go | 58 +++++++++++++++++------- config/testdata/external_labels.good.yml | 5 ++ docs/disabled_features.md | 8 ++++ 6 files changed, 92 insertions(+), 27 deletions(-) create mode 100644 config/testdata/external_labels.good.yml diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 37a7f3535..2fa828a43 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -120,6 +120,7 @@ type flagConfig struct { // for ease of use. enablePromQLAtModifier bool enablePromQLNegativeOffset bool + enableExpandExternalLabels bool prometheusURL string corsRegexString string @@ -145,6 +146,9 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error { case "remote-write-receiver": c.web.RemoteWriteReceiver = true level.Info(logger).Log("msg", "Experimental remote-write-receiver enabled") + case "expand-external-labels": + c.enableExpandExternalLabels = true + level.Info(logger).Log("msg", "Experimental expand-external-labels enabled") case "exemplar-storage": c.tsdb.MaxExemplars = maxExemplars level.Info(logger).Log("msg", "Experimental in-memory exemplar storage enabled") @@ -307,7 +311,7 @@ func main() { a.Flag("query.max-samples", "Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return."). Default("50000000").IntVar(&cfg.queryMaxSamples) - a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: promql-at-modifier, promql-negative-offset, remote-write-receiver, exemplar-storage. See https://prometheus.io/docs/prometheus/latest/disabled_features/ for more details."). + a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: promql-at-modifier, promql-negative-offset, remote-write-receiver, exemplar-storage, expand-external-labels. See https://prometheus.io/docs/prometheus/latest/disabled_features/ for more details."). Default("").StringsVar(&cfg.featureList) promlogflag.AddFlags(a, &cfg.promlogConfig) @@ -343,7 +347,7 @@ func main() { } // Throw error for invalid config before starting other components. - if _, err := config.LoadFile(cfg.configFile); err != nil { + if _, err := config.LoadFile(cfg.configFile, false, log.NewNopLogger()); err != nil { level.Error(logger).Log("msg", fmt.Sprintf("Error loading config (--config.file=%s)", cfg.configFile), "err", err) os.Exit(2) } @@ -721,11 +725,11 @@ func main() { for { select { case <-hup: - if err := reloadConfig(cfg.configFile, logger, noStepSubqueryInterval, reloaders...); err != nil { + if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, logger, noStepSubqueryInterval, reloaders...); err != nil { level.Error(logger).Log("msg", "Error reloading config", "err", err) } case rc := <-webHandler.Reload(): - if err := reloadConfig(cfg.configFile, logger, noStepSubqueryInterval, reloaders...); err != nil { + if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, logger, noStepSubqueryInterval, reloaders...); err != nil { level.Error(logger).Log("msg", "Error reloading config", "err", err) rc <- err } else { @@ -757,7 +761,7 @@ func main() { return nil } - if err := reloadConfig(cfg.configFile, logger, noStepSubqueryInterval, reloaders...); err != nil { + if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, logger, noStepSubqueryInterval, reloaders...); err != nil { return errors.Wrapf(err, "error loading config from %q", cfg.configFile) } @@ -938,7 +942,7 @@ type reloader struct { reloader func(*config.Config) error } -func reloadConfig(filename string, logger log.Logger, noStepSuqueryInterval *safePromQLNoStepSubqueryInterval, rls ...reloader) (err error) { +func reloadConfig(filename string, expandExternalLabels bool, logger log.Logger, noStepSuqueryInterval *safePromQLNoStepSubqueryInterval, rls ...reloader) (err error) { start := time.Now() timings := []interface{}{} level.Info(logger).Log("msg", "Loading configuration file", "filename", filename) @@ -952,7 +956,7 @@ func reloadConfig(filename string, logger log.Logger, noStepSuqueryInterval *saf } }() - conf, err := config.LoadFile(filename) + conf, err := config.LoadFile(filename, expandExternalLabels, logger) if err != nil { return errors.Wrapf(err, "couldn't load configuration (--config.file=%q)", filename) } diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go index 15fd4426f..9a2532376 100644 --- a/cmd/promtool/main.go +++ b/cmd/promtool/main.go @@ -291,7 +291,7 @@ func checkFileExists(fn string) error { func checkConfig(filename string) ([]string, error) { fmt.Println("Checking", filename) - cfg, err := config.LoadFile(filename) + cfg, err := config.LoadFile(filename, false, log.NewNopLogger()) if err != nil { return nil, err } diff --git a/config/config.go b/config/config.go index 419c26a94..02d9f763d 100644 --- a/config/config.go +++ b/config/config.go @@ -17,11 +17,14 @@ import ( "fmt" "io/ioutil" "net/url" + "os" "path/filepath" "regexp" "strings" "time" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/config" "github.com/prometheus/common/model" @@ -60,7 +63,7 @@ var ( ) // Load parses the YAML input s into a Config. -func Load(s string) (*Config, error) { +func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, error) { cfg := &Config{} // If the entire config body is empty the UnmarshalYAML method is // never called. We thus have to set the DefaultConfig at the entry @@ -71,16 +74,35 @@ func Load(s string) (*Config, error) { if err != nil { return nil, err } + + if !expandExternalLabels { + return cfg, nil + } + + for i, v := range cfg.GlobalConfig.ExternalLabels { + newV := os.Expand(v.Value, func(s string) string { + if v := os.Getenv(s); v != "" { + return v + } + level.Warn(logger).Log("msg", "Empty environment variable", "name", s) + return "" + }) + if newV != v.Value { + level.Debug(logger).Log("msg", "External label replaced", "label", v.Name, "input", v.Value, "output", newV) + v.Value = newV + cfg.GlobalConfig.ExternalLabels[i] = v + } + } return cfg, nil } // LoadFile parses the given YAML file into a Config. -func LoadFile(filename string) (*Config, error) { +func LoadFile(filename string, expandExternalLabels bool, logger log.Logger) (*Config, error) { content, err := ioutil.ReadFile(filename) if err != nil { return nil, err } - cfg, err := Load(string(content)) + cfg, err := Load(string(content), expandExternalLabels, logger) if err != nil { return nil, errors.Wrapf(err, "parsing YAML file %s", filename) } diff --git a/config/config_test.go b/config/config_test.go index eb8e4a1c0..8684ad72f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -17,11 +17,13 @@ import ( "encoding/json" "io/ioutil" "net/url" + "os" "path/filepath" "regexp" "testing" "time" + "github.com/go-kit/kit/log" "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" @@ -826,7 +828,7 @@ var expectedConf = &Config{ } func TestYAMLRoundtrip(t *testing.T) { - want, err := LoadFile("testdata/roundtrip.good.yml") + want, err := LoadFile("testdata/roundtrip.good.yml", false, log.NewNopLogger()) require.NoError(t, err) out, err := yaml.Marshal(want) @@ -839,7 +841,7 @@ func TestYAMLRoundtrip(t *testing.T) { } func TestRemoteWriteRetryOnRateLimit(t *testing.T) { - want, err := LoadFile("testdata/remote_write_retry_on_rate_limit.good.yml") + want, err := LoadFile("testdata/remote_write_retry_on_rate_limit.good.yml", false, log.NewNopLogger()) require.NoError(t, err) out, err := yaml.Marshal(want) @@ -855,16 +857,16 @@ func TestRemoteWriteRetryOnRateLimit(t *testing.T) { func TestLoadConfig(t *testing.T) { // Parse a valid file that sets a global scrape timeout. This tests whether parsing // an overwritten default field in the global config permanently changes the default. - _, err := LoadFile("testdata/global_timeout.good.yml") + _, err := LoadFile("testdata/global_timeout.good.yml", false, log.NewNopLogger()) require.NoError(t, err) - c, err := LoadFile("testdata/conf.good.yml") + c, err := LoadFile("testdata/conf.good.yml", false, log.NewNopLogger()) require.NoError(t, err) require.Equal(t, expectedConf, c) } func TestScrapeIntervalLarger(t *testing.T) { - c, err := LoadFile("testdata/scrape_interval_larger.good.yml") + c, err := LoadFile("testdata/scrape_interval_larger.good.yml", false, log.NewNopLogger()) require.NoError(t, err) require.Equal(t, 1, len(c.ScrapeConfigs)) for _, sc := range c.ScrapeConfigs { @@ -874,7 +876,7 @@ func TestScrapeIntervalLarger(t *testing.T) { // YAML marshaling must not reveal authentication credentials. func TestElideSecrets(t *testing.T) { - c, err := LoadFile("testdata/conf.good.yml") + c, err := LoadFile("testdata/conf.good.yml", false, log.NewNopLogger()) require.NoError(t, err) secretRe := regexp.MustCompile(`\\u003csecret\\u003e|`) @@ -891,26 +893,26 @@ func TestElideSecrets(t *testing.T) { func TestLoadConfigRuleFilesAbsolutePath(t *testing.T) { // Parse a valid file that sets a rule files with an absolute path - c, err := LoadFile(ruleFilesConfigFile) + c, err := LoadFile(ruleFilesConfigFile, false, log.NewNopLogger()) require.NoError(t, err) require.Equal(t, ruleFilesExpectedConf, c) } func TestKubernetesEmptyAPIServer(t *testing.T) { - _, err := LoadFile("testdata/kubernetes_empty_apiserver.good.yml") + _, err := LoadFile("testdata/kubernetes_empty_apiserver.good.yml", false, log.NewNopLogger()) require.NoError(t, err) } func TestKubernetesSelectors(t *testing.T) { - _, err := LoadFile("testdata/kubernetes_selectors_endpoints.good.yml") + _, err := LoadFile("testdata/kubernetes_selectors_endpoints.good.yml", false, log.NewNopLogger()) require.NoError(t, err) - _, err = LoadFile("testdata/kubernetes_selectors_node.good.yml") + _, err = LoadFile("testdata/kubernetes_selectors_node.good.yml", false, log.NewNopLogger()) require.NoError(t, err) - _, err = LoadFile("testdata/kubernetes_selectors_ingress.good.yml") + _, err = LoadFile("testdata/kubernetes_selectors_ingress.good.yml", false, log.NewNopLogger()) require.NoError(t, err) - _, err = LoadFile("testdata/kubernetes_selectors_pod.good.yml") + _, err = LoadFile("testdata/kubernetes_selectors_pod.good.yml", false, log.NewNopLogger()) require.NoError(t, err) - _, err = LoadFile("testdata/kubernetes_selectors_service.good.yml") + _, err = LoadFile("testdata/kubernetes_selectors_service.good.yml", false, log.NewNopLogger()) require.NoError(t, err) } @@ -1173,7 +1175,7 @@ var expectedErrors = []struct { func TestBadConfigs(t *testing.T) { for _, ee := range expectedErrors { - _, err := LoadFile("testdata/" + ee.filename) + _, err := LoadFile("testdata/"+ee.filename, false, log.NewNopLogger()) require.Error(t, err, "%s", ee.filename) require.Contains(t, err.Error(), ee.errMsg, "Expected error for %s to contain %q but got: %s", ee.filename, ee.errMsg, err) @@ -1197,14 +1199,38 @@ func TestBadStaticConfigsYML(t *testing.T) { } func TestEmptyConfig(t *testing.T) { - c, err := Load("") + c, err := Load("", false, log.NewNopLogger()) require.NoError(t, err) exp := DefaultConfig require.Equal(t, exp, *c) } +func TestExpandExternalLabels(t *testing.T) { + // Cleanup ant TEST env variable that could exist on the system. + os.Setenv("TEST", "") + + c, err := LoadFile("testdata/external_labels.good.yml", false, log.NewNopLogger()) + require.NoError(t, err) + require.Equal(t, labels.Label{Name: "bar", Value: "foo"}, c.GlobalConfig.ExternalLabels[0]) + require.Equal(t, labels.Label{Name: "baz", Value: "foo${TEST}bar"}, c.GlobalConfig.ExternalLabels[1]) + require.Equal(t, labels.Label{Name: "foo", Value: "${TEST}"}, c.GlobalConfig.ExternalLabels[2]) + + c, err = LoadFile("testdata/external_labels.good.yml", true, log.NewNopLogger()) + require.NoError(t, err) + require.Equal(t, labels.Label{Name: "bar", Value: "foo"}, c.GlobalConfig.ExternalLabels[0]) + require.Equal(t, labels.Label{Name: "baz", Value: "foobar"}, c.GlobalConfig.ExternalLabels[1]) + require.Equal(t, labels.Label{Name: "foo", Value: ""}, c.GlobalConfig.ExternalLabels[2]) + + os.Setenv("TEST", "TestValue") + c, err = LoadFile("testdata/external_labels.good.yml", true, log.NewNopLogger()) + require.NoError(t, err) + require.Equal(t, labels.Label{Name: "bar", Value: "foo"}, c.GlobalConfig.ExternalLabels[0]) + require.Equal(t, labels.Label{Name: "baz", Value: "fooTestValuebar"}, c.GlobalConfig.ExternalLabels[1]) + require.Equal(t, labels.Label{Name: "foo", Value: "TestValue"}, c.GlobalConfig.ExternalLabels[2]) +} + func TestEmptyGlobalBlock(t *testing.T) { - c, err := Load("global:\n") + c, err := Load("global:\n", false, log.NewNopLogger()) require.NoError(t, err) exp := DefaultConfig require.Equal(t, exp, *c) diff --git a/config/testdata/external_labels.good.yml b/config/testdata/external_labels.good.yml new file mode 100644 index 000000000..828312e4d --- /dev/null +++ b/config/testdata/external_labels.good.yml @@ -0,0 +1,5 @@ +global: + external_labels: + bar: foo + foo: ${TEST} + baz: foo${TEST}bar diff --git a/docs/disabled_features.md b/docs/disabled_features.md index 38753512a..620038e9c 100644 --- a/docs/disabled_features.md +++ b/docs/disabled_features.md @@ -18,6 +18,14 @@ They may be enabled by default in future versions. The `@` modifier lets you specify the evaluation time for instant vector selectors, range vector selectors, and subqueries. More details can be found [here](querying/basics.md#modifier). +## Expand environment variables in external labels + +`--enable-feature=expand-external-labels` + +Replace `${var}` or `$var` in the [`external_labels`](configuration/configuration.md#configuration-file) +values according to the values of the current environment variables. References +to undefined variables are replaced by the empty string. + ## Negative offset in PromQL This negative offset is disabled by default since it breaks the invariant