diff --git a/rules/group.go b/rules/group.go index 05c3a7f508..ba52bbb91d 100644 --- a/rules/group.go +++ b/rules/group.go @@ -131,7 +131,6 @@ func NewGroup(o GroupOptions) *Group { } alertStoreFunc := o.AlertStoreFunc - //var alertStore *AlertStore if alertStoreFunc == nil { alertStoreFunc = DefaultAlertStoreFunc } @@ -553,13 +552,13 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) { // Restore alerts when feature is enabled and it is the first evaluation for the group if ar, ok := rule.(*AlertingRule); ok { restoredAlerts, _ := g.alertStore.GetAlerts(ar.GetFingerprint(GroupKey(g.File(), g.Name()))) - if restoredAlerts != nil && len(restoredAlerts) > 0 { + if len(restoredAlerts) > 0 { ar.SetActiveAlerts(restoredAlerts) logger.Info("Restored alerts from store", "rule", ar.name, "alerts", len(restoredAlerts)) } } } - + vector, err := rule.Eval(ctx, ruleQueryOffset, ts, g.opts.QueryFunc, g.opts.ExternalURL, g.Limit()) if err != nil { rule.SetHealth(HealthBad) diff --git a/rules/manager.go b/rules/manager.go index 9d178551fb..8c459ad168 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -26,6 +26,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/promslog" + "golang.org/x/sync/semaphore" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" @@ -34,7 +35,6 @@ import ( "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/strutil" - "golang.org/x/sync/semaphore" ) // QueryFunc processes PromQL queries. @@ -88,7 +88,7 @@ func DefaultEvalIterationFunc(ctx context.Context, g *Group, evalTimestamp time. g.setLastEvalTimestamp(evalTimestamp) if g.alertStore != nil { - //feature enabled + // feature enabled. go func() { g.alertStoreFunc(g) }() diff --git a/rules/manager_test.go b/rules/manager_test.go index 2f8cc497af..8a3cf49740 100644 --- a/rules/manager_test.go +++ b/rules/manager_test.go @@ -2310,7 +2310,7 @@ func TestKeepFiringForStateRestore(t *testing.T) { 0, keepFiringForDuration, labels.FromStrings("severity", "critical"), - labels.FromStrings("annotation1", "rule1"), labels.EmptyLabels(), "", true, nil, + labels.FromStrings("annotation_test", "rule1"), labels.EmptyLabels(), "", true, nil, ) keepFiringForDuration2 := 60 * time.Minute rule2 := NewAlertingRule( @@ -2319,7 +2319,7 @@ func TestKeepFiringForStateRestore(t *testing.T) { 0, keepFiringForDuration2, labels.FromStrings("severity", "critical"), - labels.FromStrings("annotation2", "rule2"), labels.EmptyLabels(), "", true, nil, + labels.FromStrings("annotation_test", "rule2"), labels.EmptyLabels(), "", true, nil, ) group := NewGroup(GroupOptions{ @@ -2387,7 +2387,7 @@ func TestKeepFiringForStateRestore(t *testing.T) { 0, keepFiringForDuration, labels.FromStrings("severity", "critical"), - labels.FromStrings("annotation1", "rule1"), labels.EmptyLabels(), "", false, nil, + labels.FromStrings("annotation_test", "rule1"), labels.EmptyLabels(), "", false, nil, ) newRule2 := NewAlertingRule( "HTTPRequestRateLow", @@ -2395,9 +2395,9 @@ func TestKeepFiringForStateRestore(t *testing.T) { 0, keepFiringForDuration2, labels.FromStrings("severity", "critical"), - labels.FromStrings("annotation2", "rule2"), labels.EmptyLabels(), "", true, nil, + labels.FromStrings("annotation_test", "rule2"), labels.EmptyLabels(), "", true, nil, ) - // Restart alert store + // Restart alert store. newAlertStore := NewFileStore(promslog.NewNopLogger(), testStoreFile) newGroup := NewGroup(GroupOptions{ @@ -2425,23 +2425,33 @@ func TestKeepFiringForStateRestore(t *testing.T) { require.Equal(t, tt.alertsExpected, len(got)+len(got2)) results := [][]*Alert{got, got2} - - for i, result := range results { + for _, result := range results { sort.Slice(result, func(i, j int) bool { - return labels.Compare(got[i].Labels, got[j].Labels) < 0 + return labels.Compare(result[i].Labels, result[j].Labels) < 0 }) sortAlerts(result) - sortAlerts(expectedAlerts[i]) + } + for _, alerts := range expectedAlerts { + sort.Slice(alerts, func(i, j int) bool { + return labels.Compare(alerts[i].Labels, alerts[j].Labels) < 0 + }) + sortAlerts(alerts) } for i, expected := range expectedAlerts { got = results[i] require.Equal(t, len(expected), len(got)) for j, alert := range expected { - require.Equal(t, alert.Labels, got[j].Labels) - require.Equal(t, alert.Annotations, got[j].Annotations) - require.Equal(t, alert.ActiveAt, got[j].ActiveAt) - require.Equal(t, alert.KeepFiringSince, got[j].KeepFiringSince) + diff := float64(alert.KeepFiringSince.Unix() - got[j].KeepFiringSince.Unix()) + require.Equal(t, 0.0, math.Abs(diff), "'keep_firing_for' restored time is wrong") + diff = float64(alert.ActiveAt.Unix() - got[j].ActiveAt.Unix()) + require.Equal(t, 0.0, math.Abs(diff), "'keep_firing_for' state restored activeAt time is wrong") + + require.Equal(t, alert.Value, got[j].Value) + require.NotEmpty(t, alert.Labels) + require.Equal(t, alert.Labels.Get("instance"), got[j].Labels.Get("instance")) + require.NotEmpty(t, alert.Annotations) + require.Equal(t, alert.Annotations.Get("annotation_test"), got[j].Annotations.Get("annotation_test")) } } }) diff --git a/rules/store.go b/rules/store.go index 721ece982d..b23afb6c4f 100644 --- a/rules/store.go +++ b/rules/store.go @@ -20,7 +20,7 @@ type AlertStore interface { type FileStore struct { logger *slog.Logger alertsByRule map[uint64][]*Alert - //protects the `alertsByRule` map + // protects the `alertsByRule` map. stateMtx sync.RWMutex path string } @@ -35,7 +35,7 @@ func NewFileStore(l *slog.Logger, storagePath string) *FileStore { return s } -// initState reads the state from file storage into the alertsByRule map +// initState reads the state from file storage into the alertsByRule map. func (s *FileStore) initState() { file, err := os.OpenFile(s.path, os.O_RDWR|os.O_CREATE, 0o666) if err != nil { @@ -56,7 +56,7 @@ func (s *FileStore) initState() { } // GetAlerts returns the stored alerts for an alerting rule -// Alert state is read from the in memory map which is populated during initialization +// Alert state is read from the in memory map which is populated during initialization. func (s *FileStore) GetAlerts(key uint64) (map[uint64]*Alert, error) { s.stateMtx.RLock() defer s.stateMtx.RUnlock() @@ -76,7 +76,7 @@ func (s *FileStore) GetAlerts(key uint64) (map[uint64]*Alert, error) { return alerts, nil } -// SetAlerts updates the stateByRule map and writes state to file storage +// SetAlerts updates the stateByRule map and writes state to file storage. func (s *FileStore) SetAlerts(key uint64, alerts []*Alert) error { s.stateMtx.Lock() defer s.stateMtx.Unlock() diff --git a/rules/store_test.go b/rules/store_test.go index 96e95efa88..ba5cf7f018 100644 --- a/rules/store_test.go +++ b/rules/store_test.go @@ -5,9 +5,10 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/require" ) func TestAlertStore(t *testing.T) { @@ -27,14 +28,22 @@ func TestAlertStore(t *testing.T) { alertsByRule[4] = []*Alert{} for key, alerts := range alertsByRule { + sortAlerts(alerts) err := alertStore.SetAlerts(key, alerts) require.NoError(t, err) got, err := alertStore.GetAlerts(key) require.NoError(t, err) require.Equal(t, len(alerts), len(got)) + + result := make([]*Alert, 0, len(got)) + for _, value := range got { + result = append(result, value) + } + sortAlerts(result) + j := 0 - for _, al := range got { + for _, al := range result { require.Equal(t, alerts[j].State, al.State) require.Equal(t, alerts[j].Labels, al.Labels) require.Equal(t, alerts[j].Annotations, al.Annotations)