From c21e0b713d174aa73f549ac23064c0e01ed5f3af Mon Sep 17 00:00:00 2001 From: Chris Aniszczyk Date: Wed, 2 Dec 2020 15:48:39 -0800 Subject: [PATCH 01/12] Add CodeQL Security Scanning Signed-off-by: Chris Aniszczyk --- .github/workflows/codeql-analysis.yml | 67 +++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000000..1bb0569056 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,67 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master, release-* ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '26 14 * * 1' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'go', 'javascript' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From cd1dafc2fd05deb24181a4eaa7852909749586f3 Mon Sep 17 00:00:00 2001 From: Nguyen Le Vu Long Date: Thu, 7 Jan 2021 13:41:32 +0700 Subject: [PATCH 02/12] tsdb: Expose total number of label pairs in head in TSDB stats page (#8343) * tsdb: Expose total number of label pairs in head Signed-off-by: Nguyen Le Vu Long * fix: add comment for NumLabelPairs Signed-off-by: Nguyen Le Vu Long * fix: remove comment Signed-off-by: Nguyen Le Vu Long --- tsdb/index/postings.go | 4 ++++ web/api/v1/api.go | 18 ++++++++++-------- .../src/pages/tsdbStatus/TSDBStatus.test.tsx | 3 ++- .../src/pages/tsdbStatus/TSDBStatus.tsx | 4 +++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/tsdb/index/postings.go b/tsdb/index/postings.go index 055f74118e..a9048e4c62 100644 --- a/tsdb/index/postings.go +++ b/tsdb/index/postings.go @@ -115,6 +115,7 @@ type PostingsStats struct { CardinalityLabelStats []Stat LabelValueStats []Stat LabelValuePairsStats []Stat + NumLabelPairs int } // Stats calculates the cardinality statistics from postings. @@ -128,6 +129,7 @@ func (p *MemPostings) Stats(label string) *PostingsStats { labels := &maxHeap{} labelValueLength := &maxHeap{} labelValuePairs := &maxHeap{} + numLabelPairs := 0 metrics.init(maxNumOfRecords) labels.init(maxNumOfRecords) @@ -139,6 +141,7 @@ func (p *MemPostings) Stats(label string) *PostingsStats { continue } labels.push(Stat{Name: n, Count: uint64(len(e))}) + numLabelPairs += len(e) size = 0 for name, values := range e { if n == label { @@ -157,6 +160,7 @@ func (p *MemPostings) Stats(label string) *PostingsStats { CardinalityLabelStats: labels.get(), LabelValueStats: labelValueLength.get(), LabelValuePairsStats: labelValuePairs.get(), + NumLabelPairs: numLabelPairs, } } diff --git a/web/api/v1/api.go b/web/api/v1/api.go index df5640d545..9963f52ec4 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -1236,10 +1236,11 @@ type stat struct { // HeadStats has information about the TSDB head. type HeadStats struct { - NumSeries uint64 `json:"numSeries"` - ChunkCount int64 `json:"chunkCount"` - MinTime int64 `json:"minTime"` - MaxTime int64 `json:"maxTime"` + NumSeries uint64 `json:"numSeries"` + NumLabelPairs int `json:"numLabelPairs"` + ChunkCount int64 `json:"chunkCount"` + MinTime int64 `json:"minTime"` + MaxTime int64 `json:"maxTime"` } // tsdbStatus has information of cardinality statistics from postings. @@ -1281,10 +1282,11 @@ func (api *API) serveTSDBStatus(*http.Request) apiFuncResult { } return apiFuncResult{tsdbStatus{ HeadStats: HeadStats{ - NumSeries: s.NumSeries, - ChunkCount: chunkCount, - MinTime: s.MinTime, - MaxTime: s.MaxTime, + NumSeries: s.NumSeries, + ChunkCount: chunkCount, + MinTime: s.MinTime, + MaxTime: s.MaxTime, + NumLabelPairs: s.IndexPostingStats.NumLabelPairs, }, SeriesCountByMetricName: convertStats(s.IndexPostingStats.CardinalityMetricsStats), LabelValueCountByLabelName: convertStats(s.IndexPostingStats.CardinalityLabelStats), diff --git a/web/ui/react-app/src/pages/tsdbStatus/TSDBStatus.test.tsx b/web/ui/react-app/src/pages/tsdbStatus/TSDBStatus.test.tsx index f8e360f4d8..f25c784275 100644 --- a/web/ui/react-app/src/pages/tsdbStatus/TSDBStatus.test.tsx +++ b/web/ui/react-app/src/pages/tsdbStatus/TSDBStatus.test.tsx @@ -15,6 +15,7 @@ const fakeTSDBStatusResponse: { data: { headStats: { numSeries: 508, + numLabelPairs: 1234, chunkCount: 937, minTime: 1591516800000, maxTime: 1598896800143, @@ -85,7 +86,7 @@ describe('TSDB Stats', () => { .at(0) .find('tbody') .find('td'); - ['508', '937', '2020-06-07T08:00:00.000Z (1591516800000)', '2020-08-31T18:00:00.143Z (1598896800143)'].forEach( + ['508', '937', '1234', '2020-06-07T08:00:00.000Z (1591516800000)', '2020-08-31T18:00:00.143Z (1598896800143)'].forEach( (value, i) => { expect(headStats.at(i).text()).toEqual(value); } diff --git a/web/ui/react-app/src/pages/tsdbStatus/TSDBStatus.tsx b/web/ui/react-app/src/pages/tsdbStatus/TSDBStatus.tsx index d5539ccf7b..c124d79213 100644 --- a/web/ui/react-app/src/pages/tsdbStatus/TSDBStatus.tsx +++ b/web/ui/react-app/src/pages/tsdbStatus/TSDBStatus.tsx @@ -14,6 +14,7 @@ interface Stats { interface HeadStats { numSeries: number; + numLabelPairs: number; chunkCount: number; minTime: number; maxTime: number; @@ -35,10 +36,11 @@ export const TSDBStatusContent: FC = ({ seriesCountByLabelValuePair, }) => { const unixToTime = (unix: number): string => new Date(unix).toISOString(); - const { chunkCount, numSeries, minTime, maxTime } = headStats; + const { chunkCount, numSeries, numLabelPairs, minTime, maxTime } = headStats; const stats = [ { header: 'Number of Series', value: numSeries }, { header: 'Number of Chunks', value: chunkCount }, + { header: 'Number of Label Pairs', value: numLabelPairs }, { header: 'Current Min Time', value: `${unixToTime(minTime)} (${minTime})` }, { header: 'Current Max Time', value: `${unixToTime(maxTime)} (${maxTime})` }, ]; From 7e932637e38ef24c4d0641472df4250edb18bdbd Mon Sep 17 00:00:00 2001 From: Arthur Silva Sens Date: Thu, 7 Jan 2021 04:30:08 -0300 Subject: [PATCH 03/12] Reload tsdb blocks every minute (#8340) * Reload tsdb blocks every minute Signed-off-by: ArthurSens * Proteced tsdb with mutex locks Signed-off-by: ArthurSens --- tsdb/db.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tsdb/db.go b/tsdb/db.go index 73b87b0a57..9c15dd4c25 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -732,6 +732,12 @@ func (db *DB) run() { select { case <-time.After(1 * time.Minute): + db.cmtx.Lock() + if err := db.reloadBlocks(); err != nil { + level.Error(db.logger).Log("msg", "reloadBlocks", "err", err) + } + db.cmtx.Unlock() + select { case db.compactc <- struct{}{}: default: From 2df1a482daee9e8fa7433b0392ef81dcc2322f9f Mon Sep 17 00:00:00 2001 From: Guangwen Feng Date: Thu, 7 Jan 2021 18:01:08 +0800 Subject: [PATCH 04/12] Fix misspelled word in comment (#8348) Signed-off-by: Guangwen Feng --- cmd/promtool/testdata/bad-rules-error-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/promtool/testdata/bad-rules-error-test.yml b/cmd/promtool/testdata/bad-rules-error-test.yml index 963994d8b1..329a201817 100644 --- a/cmd/promtool/testdata/bad-rules-error-test.yml +++ b/cmd/promtool/testdata/bad-rules-error-test.yml @@ -10,5 +10,5 @@ tests: - series: 'join_2{a="1",b="4"}' values: 3 - # Just the existance of the data, that can't be joined, makes the recording + # Just the existence of the data, that can't be joined, makes the recording # rules error. From 34656bbf8eb8f45422bf24a9720ea087ab4e6434 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Thu, 7 Jan 2021 11:38:38 +0100 Subject: [PATCH 05/12] PromQL parser: fastpath for non-empty matchers and metric name (#8345) Signed-off-by: Julien Pivotto --- promql/parser/parse.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/promql/parser/parse.go b/promql/parser/parse.go index 99879445d8..a89c9424c2 100644 --- a/promql/parser/parse.go +++ b/promql/parser/parse.go @@ -583,6 +583,21 @@ func (p *parser) checkAST(node Node) (typ ValueType) { p.checkAST(n.VectorSelector) case *VectorSelector: + if n.Name != "" { + // In this case the last LabelMatcher is checking for the metric name + // set outside the braces. This checks if the name has already been set + // previously. + for _, m := range n.LabelMatchers[0 : len(n.LabelMatchers)-1] { + if m != nil && m.Name == labels.MetricName { + p.addParseErrf(n.PositionRange(), "metric name must not be set twice: %q or %q", n.Name, m.Value) + } + } + + // Skip the check for non-empty matchers because an explicit + // metric name is a non-empty matcher. + break + } + // A Vector selector must contain at least one non-empty matcher to prevent // implicit selection of all metrics (e.g. by a typo). notEmpty := false @@ -596,17 +611,6 @@ func (p *parser) checkAST(node Node) (typ ValueType) { p.addParseErrf(n.PositionRange(), "vector selector must contain at least one non-empty matcher") } - if n.Name != "" { - // In this case the last LabelMatcher is checking for the metric name - // set outside the braces. This checks if the name has already been set - // previously - for _, m := range n.LabelMatchers[0 : len(n.LabelMatchers)-1] { - if m != nil && m.Name == labels.MetricName { - p.addParseErrf(n.PositionRange(), "metric name must not be set twice: %q or %q", n.Name, m.Value) - } - } - } - case *NumberLiteral, *StringLiteral: // Nothing to do for terminals. From fd022965c022f1f02858287325fb01f5ba29b936 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Sat, 9 Jan 2021 09:58:53 +0100 Subject: [PATCH 06/12] Add SECURITY.md file (#8292) * Add SECURITY.md file Signed-off-by: Julien Pivotto --- SECURITY.md | 6 ++++++ scripts/sync_repo_files.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..67741f015a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,6 @@ +# Reporting a security issue + +The Prometheus security policy, including how to report vulnerabilities, can be +found here: + +https://prometheus.io/docs/operating/security/ diff --git a/scripts/sync_repo_files.sh b/scripts/sync_repo_files.sh index 560ad812be..0077e58d3f 100755 --- a/scripts/sync_repo_files.sh +++ b/scripts/sync_repo_files.sh @@ -20,7 +20,7 @@ if [ -z "${GITHUB_TOKEN}" ]; then fi # List of files that should be synced. -SYNC_FILES="CODE_OF_CONDUCT.md LICENSE Makefile.common" +SYNC_FILES="CODE_OF_CONDUCT.md LICENSE Makefile.common SECURITY.md" # Go to the root of the repo cd "$(git rev-parse --show-cdup)" || exit 1 From fbe960f2c1ad9d6f5fe2f267d2559bf7ecfab6df Mon Sep 17 00:00:00 2001 From: Nguyen Le Vu Long Date: Sat, 9 Jan 2021 16:02:26 +0700 Subject: [PATCH 07/12] fix: remove pre-2.21 tmp blocks on start (#8353) * fix: remove pre-2.21 tmp blocks on start Signed-off-by: Nguyen Le Vu Long * fix: commenting Signed-off-by: Nguyen Le Vu Long --- tsdb/db.go | 4 +++- tsdb/db_test.go | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tsdb/db.go b/tsdb/db.go index 9c15dd4c25..3ddf73d21a 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -56,6 +56,8 @@ const ( // about removing those too on start to save space. Currently only blocks tmp dirs are removed. tmpForDeletionBlockDirSuffix = ".tmp-for-deletion" tmpForCreationBlockDirSuffix = ".tmp-for-creation" + // Pre-2.21 tmp dir suffix, used in clean-up functions. + tmpLegacy = ".tmp" ) var ( @@ -1570,7 +1572,7 @@ func isTmpBlockDir(fi os.FileInfo) bool { fn := fi.Name() ext := filepath.Ext(fn) - if ext == tmpForDeletionBlockDirSuffix || ext == tmpForCreationBlockDirSuffix { + if ext == tmpForDeletionBlockDirSuffix || ext == tmpForCreationBlockDirSuffix || ext == tmpLegacy { if _, err := ulid.ParseStrict(fn[:len(fn)-len(ext)]); err == nil { return true } diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 6bedae9f35..e00f0a8f85 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -2795,15 +2795,20 @@ func TestOpen_VariousBlockStates(t *testing.T) { require.NoError(t, os.Remove(filepath.Join(dir, metaFilename))) } { - // Tmp blocks during creation & deletion; those should be removed on start. + // Tmp blocks during creation; those should be removed on start. dir := createBlock(t, tmpDir, genSeries(10, 2, 30, 40)) require.NoError(t, fileutil.Replace(dir, dir+tmpForCreationBlockDirSuffix)) expectedRemovedDirs[dir+tmpForCreationBlockDirSuffix] = struct{}{} - // Tmp blocks during creation & deletion; those should be removed on start. + // Tmp blocks during deletion; those should be removed on start. dir = createBlock(t, tmpDir, genSeries(10, 2, 40, 50)) require.NoError(t, fileutil.Replace(dir, dir+tmpForDeletionBlockDirSuffix)) expectedRemovedDirs[dir+tmpForDeletionBlockDirSuffix] = struct{}{} + + // Pre-2.21 tmp blocks; those should be removed on start. + dir = createBlock(t, tmpDir, genSeries(10, 2, 50, 60)) + require.NoError(t, fileutil.Replace(dir, dir+tmpLegacy)) + expectedRemovedDirs[dir+tmpLegacy] = struct{}{} } { // One ok block; but two should be replaced. From ac2626757ce674db9109187df270c42804f9ea6a Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Wed, 13 Jan 2021 21:37:01 +0100 Subject: [PATCH 08/12] Update exporter-toolkit to 0.5.0 Signed-off-by: Julien Pivotto --- cmd/prometheus/main.go | 10 +-- cmd/promtool/main.go | 4 +- go.mod | 2 +- go.sum | 4 +- .../exporter-toolkit/https/README.md | 81 ------------------- .../prometheus/exporter-toolkit/web/README.md | 10 +++ .../{https => web}/kingpinflag/flag.go | 0 .../{https => web}/tls_config.go | 15 +++- .../exporter-toolkit/{https => web}/users.go | 2 +- .../{https => web}/web-config.yml | 0 vendor/modules.txt | 6 +- web/web.go | 6 +- 12 files changed, 38 insertions(+), 102 deletions(-) delete mode 100644 vendor/github.com/prometheus/exporter-toolkit/https/README.md create mode 100644 vendor/github.com/prometheus/exporter-toolkit/web/README.md rename vendor/github.com/prometheus/exporter-toolkit/{https => web}/kingpinflag/flag.go (100%) rename vendor/github.com/prometheus/exporter-toolkit/{https => web}/tls_config.go (96%) rename vendor/github.com/prometheus/exporter-toolkit/{https => web}/users.go (99%) rename vendor/github.com/prometheus/exporter-toolkit/{https => web}/web-config.yml (100%) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 07a9381dbb..c6d30ebb91 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -46,8 +46,8 @@ import ( "github.com/prometheus/common/promlog" promlogflag "github.com/prometheus/common/promlog/flag" "github.com/prometheus/common/version" - "github.com/prometheus/exporter-toolkit/https" - httpsflag "github.com/prometheus/exporter-toolkit/https/kingpinflag" + toolkit_web "github.com/prometheus/exporter-toolkit/web" + toolkit_webflag "github.com/prometheus/exporter-toolkit/web/kingpinflag" jcfg "github.com/uber/jaeger-client-go/config" jprom "github.com/uber/jaeger-lib/metrics/prometheus" "go.uber.org/atomic" @@ -153,7 +153,7 @@ func main() { a.Flag("web.listen-address", "Address to listen on for UI, API, and telemetry."). Default("0.0.0.0:9090").StringVar(&cfg.web.ListenAddress) - httpsConfig := httpsflag.AddFlags(a) + webConfig := toolkit_webflag.AddFlags(a) a.Flag("web.read-timeout", "Maximum duration before timing out read of the request, and closing idle connections."). @@ -568,7 +568,7 @@ func main() { os.Exit(1) } - err = https.Validate(*httpsConfig) + err = toolkit_web.Validate(*webConfig) if err != nil { level.Error(logger).Log("msg", "Unable to validate web configuration file", "err", err) os.Exit(1) @@ -788,7 +788,7 @@ func main() { // Web handler. g.Add( func() error { - if err := webHandler.Run(ctxWeb, listener, *httpsConfig); err != nil { + if err := webHandler.Run(ctxWeb, listener, *webConfig); err != nil { return errors.Wrapf(err, "error starting web server") } return nil diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go index 1a13c537ed..631b8d40a0 100644 --- a/cmd/promtool/main.go +++ b/cmd/promtool/main.go @@ -37,7 +37,7 @@ import ( config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" - "github.com/prometheus/exporter-toolkit/https" + "github.com/prometheus/exporter-toolkit/web" "gopkg.in/alecthomas/kingpin.v2" "github.com/prometheus/prometheus/config" @@ -249,7 +249,7 @@ func CheckWebConfig(files ...string) int { failed := false for _, f := range files { - if err := https.Validate(f); err != nil { + if err := web.Validate(f); err != nil { fmt.Fprintln(os.Stderr, f, "FAILED:", err) failed = true continue diff --git a/go.mod b/go.mod index b6f41c170e..dafadc3cc6 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/prometheus/client_golang v1.9.0 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.15.0 - github.com/prometheus/exporter-toolkit v0.4.0 + github.com/prometheus/exporter-toolkit v0.5.0 github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 diff --git a/go.sum b/go.sum index 238d8cdf83..c0c555dbed 100644 --- a/go.sum +++ b/go.sum @@ -711,8 +711,8 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/exporter-toolkit v0.4.0 h1:O7Bw+ZKEMzW7vD10IuVF70b8EE4JIG7BvHFj9UKz49g= -github.com/prometheus/exporter-toolkit v0.4.0/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg= +github.com/prometheus/exporter-toolkit v0.5.0 h1:GwrxhCviqOl8Mm0vKqkh7Xy54m+FPlHEJacFs48M3gY= +github.com/prometheus/exporter-toolkit v0.5.0/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= diff --git a/vendor/github.com/prometheus/exporter-toolkit/https/README.md b/vendor/github.com/prometheus/exporter-toolkit/https/README.md deleted file mode 100644 index e6c9896f14..0000000000 --- a/vendor/github.com/prometheus/exporter-toolkit/https/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# HTTPS Package for Prometheus - -The `https` directory contains a Go package and a sample configuration file for -running `node_exporter` with HTTPS instead of HTTP. We currently support TLS 1.3 -and TLS 1.2. - -To run a server with TLS, use the flag `--web.config`. - -e.g. `./node_exporter --web.config="web-config.yml"` -If the config is kept within the https directory. - -The config file should be written in YAML format, and is reloaded on each connection to check for new certificates and/or authentication policy. - -## Sample Config - -``` -tls_server_config: - # Certificate and key files for server to use to authenticate to client. - cert_file: - key_file: - - # Server policy for client authentication. Maps to ClientAuth Policies. - # For more detail on clientAuth options: [ClientAuthType](https://golang.org/pkg/crypto/tls/#ClientAuthType) - [ client_auth_type: | default = "NoClientCert" ] - - # CA certificate for client certificate authentication to the server. - [ client_ca_file: ] - - # Minimum TLS version that is acceptable. - [ min_version: | default = "TLS12" ] - - # Maximum TLS version that is acceptable. - [ max_version: | default = "TLS13" ] - - # List of supported cipher suites for TLS versions up to TLS 1.2. If empty, - # Go default cipher suites are used. Available cipher suites are documented - # in the go documentation: - # https://golang.org/pkg/crypto/tls/#pkg-constants - [ cipher_suites: - [ - ] ] - - # prefer_server_cipher_suites controls whether the server selects the - # client's most preferred ciphersuite, or the server's most preferred - # ciphersuite. If true then the server's preference, as expressed in - # the order of elements in cipher_suites, is used. - [ prefer_server_cipher_suites: | default = true ] - - # Elliptic curves that will be used in an ECDHE handshake, in preference - # order. Available curves are documented in the go documentation: - # https://golang.org/pkg/crypto/tls/#CurveID - [ curve_preferences: - [ - ] ] - -http_server_config: - # Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. - # This can not be changed on the fly. - [ http2: | default = true ] - -# Usernames and hashed passwords that have full access to the web -# server via basic authentication. If empty, no basic authentication is -# required. Passwords are hashed with bcrypt. -basic_auth_users: - [ : ... ] -``` - -## About bcrypt - -There are several tools out there to generate bcrypt passwords, e.g. -[htpasswd](https://httpd.apache.org/docs/2.4/programs/htpasswd.html): - -`htpasswd -nBC 10 "" | tr -d ':\n'` - -That command will prompt you for a password and output the hashed password, -which will look something like: -`$2y$10$X0h1gDsPszWURQaxFh.zoubFi6DXncSjhoQNJgRrnGs7EsimhC7zG` - -The cost (10 in the example) influences the time it takes for computing the -hash. A higher cost will en up slowing down the authentication process. -Depending on the machine, a cost of 10 will take about ~70ms where a cost of -18 can take up to a few seconds. That hash will be computed on every -password-protected request. diff --git a/vendor/github.com/prometheus/exporter-toolkit/web/README.md b/vendor/github.com/prometheus/exporter-toolkit/web/README.md new file mode 100644 index 0000000000..1e16644bed --- /dev/null +++ b/vendor/github.com/prometheus/exporter-toolkit/web/README.md @@ -0,0 +1,10 @@ +# web package + +This package can be used by Prometheus exporters to enable TLS and +authentication. + +We actively encourage the community to use this repository, to provide a +consistent experience across the ecosystem. + +Developers documentation can be found on +[pkg.go.dev](https://pkg.go.dev/github.com/prometheus/exporter-toolkit/). diff --git a/vendor/github.com/prometheus/exporter-toolkit/https/kingpinflag/flag.go b/vendor/github.com/prometheus/exporter-toolkit/web/kingpinflag/flag.go similarity index 100% rename from vendor/github.com/prometheus/exporter-toolkit/https/kingpinflag/flag.go rename to vendor/github.com/prometheus/exporter-toolkit/web/kingpinflag/flag.go diff --git a/vendor/github.com/prometheus/exporter-toolkit/https/tls_config.go b/vendor/github.com/prometheus/exporter-toolkit/web/tls_config.go similarity index 96% rename from vendor/github.com/prometheus/exporter-toolkit/https/tls_config.go rename to vendor/github.com/prometheus/exporter-toolkit/web/tls_config.go index 192e5338af..2f244a5120 100644 --- a/vendor/github.com/prometheus/exporter-toolkit/https/tls_config.go +++ b/vendor/github.com/prometheus/exporter-toolkit/web/tls_config.go @@ -11,8 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package https allows the implementation of TLS. -package https +package web import ( "crypto/tls" @@ -174,9 +173,9 @@ func ConfigToTLSConfig(c *TLSStruct) (*tls.Config, error) { return cfg, nil } -// Listen starts the server on the given address. Based on the file +// ListenAndServe starts the server on the given address. Based on the file // tlsConfigPath, TLS or basic auth could be enabled. -func Listen(server *http.Server, tlsConfigPath string, logger log.Logger) error { +func ListenAndServe(server *http.Server, tlsConfigPath string, logger log.Logger) error { listener, err := net.Listen("tcp", server.Addr) if err != nil { return err @@ -341,3 +340,11 @@ func (tv *tlsVersion) MarshalYAML() (interface{}, error) { } return fmt.Sprintf("%v", tv), nil } + +// Listen starts the server on the given address. Based on the file +// tlsConfigPath, TLS or basic auth could be enabled. +// +// Deprecated: Use ListenAndServe instead. +func Listen(server *http.Server, tlsConfigPath string, logger log.Logger) error { + return ListenAndServe(server, tlsConfigPath, logger) +} diff --git a/vendor/github.com/prometheus/exporter-toolkit/https/users.go b/vendor/github.com/prometheus/exporter-toolkit/web/users.go similarity index 99% rename from vendor/github.com/prometheus/exporter-toolkit/https/users.go rename to vendor/github.com/prometheus/exporter-toolkit/web/users.go index 317b0e4f8a..7b9cd6a2ae 100644 --- a/vendor/github.com/prometheus/exporter-toolkit/https/users.go +++ b/vendor/github.com/prometheus/exporter-toolkit/web/users.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package https +package web import ( "net/http" diff --git a/vendor/github.com/prometheus/exporter-toolkit/https/web-config.yml b/vendor/github.com/prometheus/exporter-toolkit/web/web-config.yml similarity index 100% rename from vendor/github.com/prometheus/exporter-toolkit/https/web-config.yml rename to vendor/github.com/prometheus/exporter-toolkit/web/web-config.yml diff --git a/vendor/modules.txt b/vendor/modules.txt index fbcb0543df..ff31f22fd0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -395,10 +395,10 @@ github.com/prometheus/common/promlog/flag github.com/prometheus/common/route github.com/prometheus/common/server github.com/prometheus/common/version -# github.com/prometheus/exporter-toolkit v0.4.0 +# github.com/prometheus/exporter-toolkit v0.5.0 ## explicit -github.com/prometheus/exporter-toolkit/https -github.com/prometheus/exporter-toolkit/https/kingpinflag +github.com/prometheus/exporter-toolkit/web +github.com/prometheus/exporter-toolkit/web/kingpinflag # github.com/prometheus/procfs v0.2.0 github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs diff --git a/web/web.go b/web/web.go index 2160e5b4d3..c780612f09 100644 --- a/web/web.go +++ b/web/web.go @@ -50,7 +50,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/common/route" "github.com/prometheus/common/server" - "github.com/prometheus/exporter-toolkit/https" + toolkit_web "github.com/prometheus/exporter-toolkit/web" "go.uber.org/atomic" "golang.org/x/net/netutil" @@ -544,7 +544,7 @@ func (h *Handler) Listener() (net.Listener, error) { } // Run serves the HTTP endpoints. -func (h *Handler) Run(ctx context.Context, listener net.Listener, httpsConfig string) error { +func (h *Handler) Run(ctx context.Context, listener net.Listener, webConfig string) error { if listener == nil { var err error listener, err = h.Listener() @@ -580,7 +580,7 @@ func (h *Handler) Run(ctx context.Context, listener net.Listener, httpsConfig st errCh := make(chan error) go func() { - errCh <- https.Serve(listener, httpSrv, httpsConfig, h.logger) + errCh <- toolkit_web.Serve(listener, httpSrv, webConfig, h.logger) }() select { From 08c259cda6f7da460d944947811a5677fc8a0053 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Tue, 29 Dec 2020 21:14:45 +0100 Subject: [PATCH 09/12] https: Add example configuration file Signed-off-by: Julien Pivotto --- docs/configuration/https.md | 2 ++ documentation/examples/web-config.yml | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 documentation/examples/web-config.yml diff --git a/docs/configuration/https.md b/docs/configuration/https.md index 157df84d83..380f9d6426 100644 --- a/docs/configuration/https.md +++ b/docs/configuration/https.md @@ -20,6 +20,8 @@ Generic placeholders are defined as follows: * ``: a regular string that is a secret, such as a password * ``: a regular string +A valid example file can be found [here](/documentation/examples/web-config.yml). + ``` tls_server_config: # Certificate and key files for server to use to authenticate to client. diff --git a/documentation/examples/web-config.yml b/documentation/examples/web-config.yml new file mode 100644 index 0000000000..84e5070612 --- /dev/null +++ b/documentation/examples/web-config.yml @@ -0,0 +1,12 @@ +# TLS and basic authentication configuration example. +# +# Additionally, a certificate and a key file are needed. +tls_server_config: + cert_file: server.crt + key_file: server.key + +# Usernames and passwords required to connect to Prometheus. +# Passwords are hashed with bcrypt: https://github.com/prometheus/exporter-toolkit/blob/master/https/README.md#about-bcrypt +basic_auth_users: + alice: $2y$10$mDwo.lAisC94iLAyP81MCesa29IzH37oigHC/42V2pdJlUprsJPze + bob: $2y$10$hLqFl9jSjoAAy95Z/zw8Ye8wkdMBM8c5Bn1ptYqP/AXyV0.oy0S8m From 9602933db265ef71676b2ab7ddcadf7f8985e6f1 Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Fri, 15 Jan 2021 16:19:57 +0800 Subject: [PATCH 10/12] Cleanup tempfiles in tests (#8179) * test: cleanup tempdir for TestBlockWriter Signed-off-by: Li Zhijian * test: cleanup tempdir for TestLogPartialWrite Signed-off-by: Li Zhijian --- tsdb/blockwriter_test.go | 2 ++ tsdb/wal/wal_test.go | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tsdb/blockwriter_test.go b/tsdb/blockwriter_test.go index bc038e7ac8..8f1dab73dc 100644 --- a/tsdb/blockwriter_test.go +++ b/tsdb/blockwriter_test.go @@ -32,6 +32,7 @@ func TestBlockWriter(t *testing.T) { ctx := context.Background() outputDir, err := ioutil.TempDir(os.TempDir(), "output") require.NoError(t, err) + defer func() { require.NoError(t, os.RemoveAll(outputDir)) }() w, err := NewBlockWriter(log.NewNopLogger(), outputDir, DefaultBlockDuration) require.NoError(t, err) @@ -55,6 +56,7 @@ func TestBlockWriter(t *testing.T) { blockpath := filepath.Join(outputDir, id.String()) b, err := OpenBlock(nil, blockpath, nil) require.NoError(t, err) + defer func() { require.NoError(t, b.Close()) }() q, err := NewBlockQuerier(b, math.MinInt64, math.MaxInt64) require.NoError(t, err) series := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "", ".*")) diff --git a/tsdb/wal/wal_test.go b/tsdb/wal/wal_test.go index 24d909801e..b12c0d60c8 100644 --- a/tsdb/wal/wal_test.go +++ b/tsdb/wal/wal_test.go @@ -454,8 +454,9 @@ func TestLogPartialWrite(t *testing.T) { for testName, testData := range tests { t.Run(testName, func(t *testing.T) { - dirPath, err := ioutil.TempDir("", "") + dirPath, err := ioutil.TempDir("", "logpartialwrite") require.NoError(t, err) + defer func() { require.NoError(t, os.RemoveAll(dirPath)) }() w, err := NewSize(nil, nil, dirPath, segmentSize, false) require.NoError(t, err) @@ -480,6 +481,7 @@ func TestLogPartialWrite(t *testing.T) { // Read it back. We expect no corruption. s, err := OpenReadSegment(SegmentName(dirPath, 0)) require.NoError(t, err) + defer func() { require.NoError(t, s.Close()) }() r := NewReader(NewSegmentBufReader(s)) for i := 0; i < testData.numRecords; i++ { From 2fd1243b0e5db450fc5dc4218d516ffab96b8e28 Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Sat, 16 Jan 2021 20:07:55 +0100 Subject: [PATCH 11/12] Enforce SECURITY.md updates Add SECURITY.md to the enforced repo sync updates. Signed-off-by: Ben Kochie --- scripts/sync_repo_files.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/sync_repo_files.sh b/scripts/sync_repo_files.sh index 0077e58d3f..1380ae139b 100755 --- a/scripts/sync_repo_files.sh +++ b/scripts/sync_repo_files.sh @@ -73,10 +73,12 @@ process_repo() { fi if [[ -z "${target_file}" ]]; then echo "${source_file} doesn't exist in ${org_repo}" - if [[ "${source_file}" == 'CODE_OF_CONDUCT.md' ]] ; then - echo "CODE_OF_CONDUCT.md missing in ${org_repo}, force updating." - needs_update+=('CODE_OF_CONDUCT.md') - fi + case "${source_file}" in + CODE_OF_CONDUCT.md | SECURITY.md) + echo "${source_file} missing in ${org_repo}, force updating." + needs_update+=("${source_file}") + ;; + esac continue fi target_checksum="$(echo "${target_file}" | sha256sum | cut -d' ' -f1)" From 2d8738c9476f9550c97bdba34824a84ee489ecf6 Mon Sep 17 00:00:00 2001 From: Hu Shuai Date: Mon, 18 Jan 2021 10:14:47 +0800 Subject: [PATCH 12/12] Fix a typo in docs/configuration/configuration.md Signed-off-by: Hu Shuai --- docs/configuration/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 02aa1b2486..0df6874516 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -662,7 +662,7 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_ec2_instance_lifecycle`: the lifecycle of the EC2 instance, set only for 'spot' or 'scheduled' instances, absent otherwise * `__meta_ec2_instance_state`: the state of the EC2 instance * `__meta_ec2_instance_type`: the type of the EC2 instance -* `__meta_ec2_ipv6_addresses`: comma seperated list of IPv6 addresses assigned to the instance's network interfaces, if present +* `__meta_ec2_ipv6_addresses`: comma separated list of IPv6 addresses assigned to the instance's network interfaces, if present * `__meta_ec2_owner_id`: the ID of the AWS account that owns the EC2 instance * `__meta_ec2_platform`: the Operating System platform, set to 'windows' on Windows servers, absent otherwise * `__meta_ec2_primary_subnet_id`: the subnet ID of the primary network interface, if available