diff --git a/CHANGELOG.md b/CHANGELOG.md index c5e858b542..bb99ef410e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 2.39.1 / 2022-10-07 + +* [BUGFIX] Rules: Fix notifier relabel changing the labels on active alerts. #11427 + ## 2.39.0 / 2022-10-05 * [FEATURE] **experimental** TSDB: Add support for ingesting out-of-order samples. This is configured via `out_of_order_time_window` field in the config file; check config file docs for more info. #11075 diff --git a/README.md b/README.md index 6ca98143cb..6b3f6cf01b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,11 @@ -# Prometheus +

+ Prometheus
Prometheus +

+ +

Visit prometheus.io for the full documentation, +examples and guides.

+ +
[![CircleCI](https://circleci.com/gh/prometheus/prometheus/tree/main.svg?style=shield)][circleci] [![Docker Repository on Quay](https://quay.io/repository/prometheus/prometheus/status)][quay] @@ -8,8 +15,7 @@ [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/prometheus/prometheus) [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/prometheus.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:prometheus) -Visit [prometheus.io](https://prometheus.io) for the full documentation, -examples and guides. +
Prometheus, a [Cloud Native Computing Foundation](https://cncf.io/) project, is a systems and service monitoring system. It collects metrics from configured targets at given intervals, evaluates rule expressions, diff --git a/VERSION b/VERSION index cde8adf34d..ec12822552 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.39.0 +2.39.1 diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 65fcf15238..fe533069eb 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -72,7 +72,6 @@ import ( "github.com/prometheus/prometheus/tsdb/agent" "github.com/prometheus/prometheus/util/logging" prom_runtime "github.com/prometheus/prometheus/util/runtime" - "github.com/prometheus/prometheus/util/strutil" "github.com/prometheus/prometheus/web" ) @@ -619,7 +618,7 @@ func main() { Appendable: fanoutStorage, Queryable: localStorage, QueryFunc: rules.EngineQueryFunc(queryEngine, fanoutStorage), - NotifyFunc: sendAlerts(notifierManager, cfg.web.ExternalURL.String()), + NotifyFunc: rules.SendAlerts(notifierManager, cfg.web.ExternalURL.String()), Context: ctxRule, ExternalURL: cfg.web.ExternalURL, Registerer: prometheus.DefaultRegisterer, @@ -1270,36 +1269,6 @@ func computeExternalURL(u, listenAddr string) (*url.URL, error) { return eu, nil } -type sender interface { - Send(alerts ...*notifier.Alert) -} - -// sendAlerts implements the rules.NotifyFunc for a Notifier. -func sendAlerts(s sender, externalURL string) rules.NotifyFunc { - return func(ctx context.Context, expr string, alerts ...*rules.Alert) { - var res []*notifier.Alert - - for _, alert := range alerts { - a := ¬ifier.Alert{ - StartsAt: alert.FiredAt, - Labels: alert.Labels, - Annotations: alert.Annotations, - GeneratorURL: externalURL + strutil.TableLinkForExpression(expr), - } - if !alert.ResolvedAt.IsZero() { - a.EndsAt = alert.ResolvedAt - } else { - a.EndsAt = alert.ValidUntil - } - res = append(res, a) - } - - if len(alerts) > 0 { - s.Send(res...) - } - } -} - // readyStorage implements the Storage interface while allowing to set the actual // storage at a later point in time. type readyStorage struct { diff --git a/cmd/prometheus/main_test.go b/cmd/prometheus/main_test.go index 7dec5b9a59..9fbca5c336 100644 --- a/cmd/prometheus/main_test.go +++ b/cmd/prometheus/main_test.go @@ -198,7 +198,7 @@ func TestSendAlerts(t *testing.T) { } require.Equal(t, tc.exp, alerts) }) - sendAlerts(senderFunc, "http://localhost:9090")(context.TODO(), "up", tc.in...) + rules.SendAlerts(senderFunc, "http://localhost:9090")(context.TODO(), "up", tc.in...) }) } } diff --git a/documentation/images/prometheus-logo.svg b/documentation/images/prometheus-logo.svg new file mode 100644 index 0000000000..026f9e5bcc --- /dev/null +++ b/documentation/images/prometheus-logo.svg @@ -0,0 +1,50 @@ + + + +image/svg+xml diff --git a/go.mod b/go.mod index 1b92bb401b..071bef831b 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/golang/snappy v0.0.4 github.com/google/pprof v0.0.0-20220829040838-70bd9ae97f40 github.com/gophercloud/gophercloud v1.0.0 - github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2 + github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/consul/api v1.15.2 github.com/hashicorp/nomad/api v0.0.0-20220921012004-ddeeb1040edf diff --git a/go.sum b/go.sum index ebb005596b..17b06cc4b7 100644 --- a/go.sum +++ b/go.sum @@ -438,8 +438,8 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2 h1:uirlL/j72L93RhV4+mkWhjv0cov2I0MIgPOG9rMDr1k= -github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= +github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6 h1:A3dhViTeFDSQcGOXuUi6ukCQSMyDtDISBp2z6OOo2YM= +github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= diff --git a/rules/alerting.go b/rules/alerting.go index 46222e830a..40358154f8 100644 --- a/rules/alerting.go +++ b/rules/alerting.go @@ -507,6 +507,8 @@ func (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay } alert.ValidUntil = ts.Add(4 * delta) anew := *alert + // The notifier re-uses the labels slice, hence make a copy. + anew.Labels = alert.Labels.Copy() alerts = append(alerts, &anew) } }) diff --git a/rules/alerting_test.go b/rules/alerting_test.go index 947a85dc7a..ca55995aee 100644 --- a/rules/alerting_test.go +++ b/rules/alerting_test.go @@ -20,10 +20,13 @@ import ( "time" "github.com/go-kit/log" + "github.com/prometheus/common/model" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/relabel" "github.com/prometheus/prometheus/model/timestamp" + "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" @@ -659,3 +662,53 @@ func TestQueryForStateSeries(t *testing.T) { testFunc(tst) } } + +// TestSendAlertsDontAffectActiveAlerts tests a fix for https://github.com/prometheus/prometheus/issues/11424. +func TestSendAlertsDontAffectActiveAlerts(t *testing.T) { + rule := NewAlertingRule( + "TestRule", + nil, + time.Minute, + labels.FromStrings("severity", "critical"), + labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil, + ) + + // Set an active alert. + lbls := labels.FromStrings("a1", "1") + h := lbls.Hash() + al := &Alert{State: StateFiring, Labels: lbls, ActiveAt: time.Now()} + rule.active[h] = al + + expr, err := parser.ParseExpr("foo") + require.NoError(t, err) + rule.vector = expr + + // The relabel rule reproduced the bug here. + opts := notifier.Options{ + QueueCapacity: 1, + RelabelConfigs: []*relabel.Config{ + { + SourceLabels: model.LabelNames{"a1"}, + Regex: relabel.MustNewRegexp("(.+)"), + TargetLabel: "a1", + Replacement: "bug", + Action: "replace", + }, + }, + } + nm := notifier.NewManager(&opts, log.NewNopLogger()) + + f := SendAlerts(nm, "") + notifyFunc := func(ctx context.Context, expr string, alerts ...*Alert) { + require.Len(t, alerts, 1) + require.Equal(t, al, alerts[0]) + f(ctx, expr, alerts...) + } + + rule.sendAlerts(context.Background(), time.Now(), 0, 0, notifyFunc) + nm.Stop() + + // The relabel rule changes a1=1 to a1=bug. + // But the labels with the AlertingRule should not be changed. + require.Equal(t, labels.FromStrings("a1", "1"), rule.active[h].Labels) +} diff --git a/rules/manager.go b/rules/manager.go index 2fe65eb21c..5cdf50f527 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -35,9 +35,11 @@ import ( "github.com/prometheus/prometheus/model/rulefmt" "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/model/value" + "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/strutil" ) // RuleHealth describes the health state of a rule. @@ -1226,3 +1228,33 @@ func (m *Manager) AlertingRules() []*AlertingRule { return alerts } + +type Sender interface { + Send(alerts ...*notifier.Alert) +} + +// SendAlerts implements the rules.NotifyFunc for a Notifier. +func SendAlerts(s Sender, externalURL string) NotifyFunc { + return func(ctx context.Context, expr string, alerts ...*Alert) { + var res []*notifier.Alert + + for _, alert := range alerts { + a := ¬ifier.Alert{ + StartsAt: alert.FiredAt, + Labels: alert.Labels, + Annotations: alert.Annotations, + GeneratorURL: externalURL + strutil.TableLinkForExpression(expr), + } + if !alert.ResolvedAt.IsZero() { + a.EndsAt = alert.ResolvedAt + } else { + a.EndsAt = alert.ValidUntil + } + res = append(res, a) + } + + if len(alerts) > 0 { + s.Send(res...) + } + } +} diff --git a/tsdb/README.md b/tsdb/README.md index ad9354586c..80770e8dd4 100644 --- a/tsdb/README.md +++ b/tsdb/README.md @@ -13,7 +13,7 @@ which handles storage and querying of all Prometheus v2 data. ## External resources -* A writeup of the original design can be found [here](https://fabxc.org/blog/2017-04-10-writing-a-tsdb/). +* A writeup of the original design can be found [here](https://web.archive.org/web/20210803115658/https://fabxc.org/tsdb/). * Video: [Storing 16 Bytes at Scale](https://youtu.be/b_pEevMAC3I) from [PromCon 2017](https://promcon.io/2017-munich/). * Compression is based on the Gorilla TSDB [white paper](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). diff --git a/tsdb/head_wal.go b/tsdb/head_wal.go index 1daf0bb860..aa02451846 100644 --- a/tsdb/head_wal.go +++ b/tsdb/head_wal.go @@ -770,188 +770,6 @@ func (wp *wblSubsetProcessor) waitUntilIdle() { } } -func (h *Head) loadWbl(r *wal.Reader, multiRef map[chunks.HeadSeriesRef]chunks.HeadSeriesRef, lastMmapRef chunks.ChunkDiskMapperRef) (err error) { - // Track number of samples, m-map markers, that referenced a series we don't know about - // for error reporting. - var unknownRefs, mmapMarkerUnknownRefs atomic.Uint64 - - lastSeq, lastOff := lastMmapRef.Unpack() - // Start workers that each process samples for a partition of the series ID space. - var ( - wg sync.WaitGroup - n = runtime.GOMAXPROCS(0) - processors = make([]wblSubsetProcessor, n) - - dec record.Decoder - shards = make([][]record.RefSample, n) - - decoded = make(chan interface{}, 10) - decodeErr error - samplesPool = sync.Pool{ - New: func() interface{} { - return []record.RefSample{} - }, - } - markersPool = sync.Pool{ - New: func() interface{} { - return []record.RefMmapMarker{} - }, - } - ) - - defer func() { - // For CorruptionErr ensure to terminate all workers before exiting. - // We also wrap it to identify OOO WBL corruption. - _, ok := err.(*wal.CorruptionErr) - if ok { - err = &errLoadWbl{err: err} - for i := 0; i < n; i++ { - processors[i].closeAndDrain() - } - wg.Wait() - } - }() - - wg.Add(n) - for i := 0; i < n; i++ { - processors[i].setup() - - go func(wp *wblSubsetProcessor) { - unknown := wp.processWBLSamples(h) - unknownRefs.Add(unknown) - wg.Done() - }(&processors[i]) - } - - go func() { - defer close(decoded) - for r.Next() { - rec := r.Record() - switch dec.Type(rec) { - case record.Samples: - samples := samplesPool.Get().([]record.RefSample)[:0] - samples, err = dec.Samples(rec, samples) - if err != nil { - decodeErr = &wal.CorruptionErr{ - Err: errors.Wrap(err, "decode samples"), - Segment: r.Segment(), - Offset: r.Offset(), - } - return - } - decoded <- samples - case record.MmapMarkers: - markers := markersPool.Get().([]record.RefMmapMarker)[:0] - markers, err = dec.MmapMarkers(rec, markers) - if err != nil { - decodeErr = &wal.CorruptionErr{ - Err: errors.Wrap(err, "decode mmap markers"), - Segment: r.Segment(), - Offset: r.Offset(), - } - return - } - decoded <- markers - default: - // Noop. - } - } - }() - - // The records are always replayed from the oldest to the newest. - for d := range decoded { - switch v := d.(type) { - case []record.RefSample: - samples := v - // We split up the samples into parts of 5000 samples or less. - // With O(300 * #cores) in-flight sample batches, large scrapes could otherwise - // cause thousands of very large in flight buffers occupying large amounts - // of unused memory. - for len(samples) > 0 { - m := 5000 - if len(samples) < m { - m = len(samples) - } - for i := 0; i < n; i++ { - shards[i] = processors[i].reuseBuf() - } - for _, sam := range samples[:m] { - if r, ok := multiRef[sam.Ref]; ok { - sam.Ref = r - } - mod := uint64(sam.Ref) % uint64(n) - shards[mod] = append(shards[mod], sam) - } - for i := 0; i < n; i++ { - processors[i].input <- shards[i] - } - samples = samples[m:] - } - //nolint:staticcheck // Ignore SA6002 relax staticcheck verification. - samplesPool.Put(d) - case []record.RefMmapMarker: - markers := v - for _, rm := range markers { - seq, off := rm.MmapRef.Unpack() - if seq > lastSeq || (seq == lastSeq && off > lastOff) { - // This m-map chunk from markers was not present during - // the load of mmapped chunks that happened in the head - // initialization. - continue - } - - if r, ok := multiRef[rm.Ref]; ok { - rm.Ref = r - } - - ms := h.series.getByID(rm.Ref) - if ms == nil { - mmapMarkerUnknownRefs.Inc() - continue - } - - idx := uint64(ms.ref) % uint64(n) - // It is possible that some old sample is being processed in processWALSamples that - // could cause race below. So we wait for the goroutine to empty input the buffer and finish - // processing all old samples after emptying the buffer. - processors[idx].waitUntilIdle() - // Lock the subset so we can modify the series object - processors[idx].mx.Lock() - - // All samples till now have been m-mapped. Hence clear out the headChunk. - // In case some samples slipped through and went into m-map chunks because of changed - // chunk size parameters, we are not taking care of that here. - // TODO(codesome): see if there is a way to avoid duplicate m-map chunks if - // the size of ooo chunk was reduced between restart. - ms.oooHeadChunk = nil - - processors[idx].mx.Unlock() - } - default: - panic(fmt.Errorf("unexpected decoded type: %T", d)) - } - } - - if decodeErr != nil { - return decodeErr - } - - // Signal termination to each worker and wait for it to close its output channel. - for i := 0; i < n; i++ { - processors[i].closeAndDrain() - } - wg.Wait() - - if r.Err() != nil { - return errors.Wrap(r.Err(), "read records") - } - - if unknownRefs.Load() > 0 || mmapMarkerUnknownRefs.Load() > 0 { - level.Warn(h.logger).Log("msg", "Unknown series references for ooo WAL replay", "samples", unknownRefs.Load(), "mmap_markers", mmapMarkerUnknownRefs.Load()) - } - return nil -} - const ( chunkSnapshotRecordTypeSeries uint8 = 1 chunkSnapshotRecordTypeTombstones uint8 = 2 diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index e5e11fc218..544a10698b 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.39.0", + "version": "0.39.1", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "^0.39.0", + "@prometheus-io/lezer-promql": "^0.39.1", "lru-cache": "^6.0.0" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index 5df04782ef..8d0bdea453 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.39.0", + "version": "0.39.1", "description": "lezer-based PromQL grammar", "main": "index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index d58ca378d4..e75ca42cfa 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -28,10 +28,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.39.0", + "version": "0.39.1", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "^0.39.0", + "@prometheus-io/lezer-promql": "^0.39.1", "lru-cache": "^6.0.0" }, "devDependencies": { @@ -61,7 +61,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.39.0", + "version": "0.39.1", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.1.1", @@ -17625,7 +17625,7 @@ }, "react-app": { "name": "@prometheus-io/app", - "version": "0.39.0", + "version": "0.39.1", "dependencies": { "@codemirror/autocomplete": "^6.2.0", "@codemirror/commands": "^6.1.0", @@ -17643,7 +17643,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.39.0", + "@prometheus-io/codemirror-promql": "^0.39.1", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11", @@ -19883,7 +19883,7 @@ "@lezer/lr": "^1.2.3", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.39.0", + "@prometheus-io/codemirror-promql": "^0.39.1", "@testing-library/react-hooks": "^7.0.2", "@types/enzyme": "^3.10.12", "@types/flot": "0.0.32", @@ -19935,7 +19935,7 @@ "@lezer/common": "^1.0.1", "@lezer/highlight": "^1.1.0", "@lezer/lr": "^1.2.3", - "@prometheus-io/lezer-promql": "^0.39.0", + "@prometheus-io/lezer-promql": "^0.39.1", "@types/lru-cache": "^5.1.1", "isomorphic-fetch": "^3.0.0", "lru-cache": "^6.0.0", diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index 8e80ca253f..8e7b103a08 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/app", - "version": "0.39.0", + "version": "0.39.1", "private": true, "dependencies": { "@codemirror/autocomplete": "^6.2.0", @@ -19,7 +19,7 @@ "@lezer/common": "^1.0.1", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "^0.39.0", + "@prometheus-io/codemirror-promql": "^0.39.1", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^6.1.11",