mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-25 05:34:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #12999 from rajagopalanand/filter-alerts

Browse source 
Adding a query parameter to filter out active alerts
This commit is contained in:
Julien Pivotto 2023-10-30 13:10:24 +01:00 committed by GitHub
parent e9b94515ca 2ade8adf9e
commit 156a23d06d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 231 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
docs/querying/api.md
View file

@ -679,6 +679,7 @@ URL query parameters:
- `rule_name[]=<string>`: only return rules with the given rule name. If the parameter is repeated, rules with any of the provided names are returned. If we've filtered out all the rules of a group, the group is not returned. When the parameter is absent or empty, no filtering is done.
- `rule_group[]=<string>`: only return rules with the given rule group name. If the parameter is repeated, rules with any of the provided rule group names are returned. When the parameter is absent or empty, no filtering is done.
- `file[]=<string>`: only return rules with the given filepath. If the parameter is repeated, rules with any of the provided filepaths are returned. When the parameter is absent or empty, no filtering is done.
- `exclude_alerts`: only return rules without active alerts. When this parameter is absent or empty, no filtering is done.
```json
$ curl http://localhost:9090/api/v1/rules
@ -1307,4 +1308,4 @@ Enable the OTLP receiver by the feature flag
`--enable-feature=otlp-write-receiver`. When enabled, the OTLP receiver
endpoint is `/api/v1/otlp/v1/metrics`.
*New in v2.47*
*New in v2.47*

25
web/api/v1/api.go
View file

@ -1373,6 +1373,11 @@ func (api *API) rules(r *http.Request) apiFuncResult {
returnAlerts := typ == "" || typ == "alert"
returnRecording := typ == "" || typ == "record"
excludeAlerts, err := parseExcludeAlerts(r)
if err != nil {
return invalidParamError(err, "exclude_alerts")
}
rgs := make([]*RuleGroup, 0, len(ruleGroups))
for _, grp := range ruleGroups {
if len(rgSet) > 0 {
@ -1414,6 +1419,10 @@ func (api *API) rules(r *http.Request) apiFuncResult {
if !returnAlerts {
break
}
var activeAlerts []*Alert
if !excludeAlerts {
activeAlerts = rulesAlertsToAPIAlerts(rule.ActiveAlerts())
}
enrichedRule = AlertingRule{
State: rule.State().String(),
Name: rule.Name(),
@ -1422,7 +1431,7 @@ func (api *API) rules(r *http.Request) apiFuncResult {
KeepFiringFor: rule.KeepFiringFor().Seconds(),
Labels: rule.Labels(),
Annotations: rule.Annotations(),
Alerts: rulesAlertsToAPIAlerts(rule.ActiveAlerts()),
Alerts: activeAlerts,
Health: rule.Health(),
LastError: lastError,
EvaluationTime: rule.GetEvaluationDuration().Seconds(),
@ -1462,6 +1471,20 @@ func (api *API) rules(r *http.Request) apiFuncResult {
return apiFuncResult{res, nil, nil, nil}
}
func parseExcludeAlerts(r *http.Request) (bool, error) {
excludeAlertsParam := strings.ToLower(r.URL.Query().Get("exclude_alerts"))
if excludeAlertsParam == "" {
return false, nil
}
excludeAlerts, err := strconv.ParseBool(excludeAlertsParam)
if err != nil {
return false, fmt.Errorf("error converting exclude_alerts: %w", err)
}
return excludeAlerts, nil
}
type prometheusConfig struct {
YAML string `json:"yaml"`
}

228
web/api/v1/api_test.go
View file

@ -209,10 +209,12 @@ func (t testAlertmanagerRetriever) toFactory() func(context.Context) Alertmanage
}
type rulesRetrieverMock struct {
testing *testing.T
alertingRules []*rules.AlertingRule
ruleGroups []*rules.Group
testing *testing.T
}
func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
func (m *rulesRetrieverMock) CreateAlertingRules() {
expr1, err := parser.ParseExpr(`absent(test_metric3) != 1`)
if err != nil {
m.testing.Fatalf("unable to parse alert expression: %s", err)
@ -222,6 +224,11 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
m.testing.Fatalf("Unable to parse alert expression: %s", err)
}
expr3, err := parser.ParseExpr(`vector(1)`)
if err != nil {
m.testing.Fatalf("Unable to parse alert expression: %s", err)
}
rule1 := rules.NewAlertingRule(
"test_metric3",
expr1,
@ -246,15 +253,29 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
true,
log.NewNopLogger(),
)
rule3 := rules.NewAlertingRule(
"test_metric5",
expr3,
time.Second,
0,
labels.FromStrings("name", "tm5"),
labels.Labels{},
labels.FromStrings("name", "tm5"),
"",
false,
log.NewNopLogger(),
)
var r []*rules.AlertingRule
r = append(r, rule1)
r = append(r, rule2)
return r
r = append(r, rule3)
m.alertingRules = r
}
func (m rulesRetrieverMock) RuleGroups() []*rules.Group {
var ar rulesRetrieverMock
arules := ar.AlertingRules()
func (m *rulesRetrieverMock) CreateRuleGroups() {
m.CreateAlertingRules()
arules := m.AlertingRules()
storage := teststorage.New(m.testing)
defer storage.Close()
@ -271,6 +292,7 @@ func (m rulesRetrieverMock) RuleGroups() []*rules.Group {
Appendable: storage,
Context: context.Background(),
Logger: log.NewNopLogger(),
NotifyFunc: func(ctx context.Context, expr string, alerts ...*rules.Alert) {},
}
var r []rules.Rule
@ -294,10 +316,18 @@ func (m rulesRetrieverMock) RuleGroups() []*rules.Group {
ShouldRestore: false,
Opts: opts,
})
return []*rules.Group{group}
m.ruleGroups = []*rules.Group{group}
}
func (m rulesRetrieverMock) toFactory() func(context.Context) RulesRetriever {
func (m *rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
return m.alertingRules
}
func (m *rulesRetrieverMock) RuleGroups() []*rules.Group {
return m.ruleGroups
}
func (m *rulesRetrieverMock) toFactory() func(context.Context) RulesRetriever {
return func(context.Context) RulesRetriever { return m }
}
@ -380,12 +410,14 @@ func TestEndpoints(t *testing.T) {
now := time.Now()
t.Run("local", func(t *testing.T) {
var algr rulesRetrieverMock
algr := rulesRetrieverMock{}
algr.testing = t
algr.AlertingRules()
algr.CreateAlertingRules()
algr.CreateRuleGroups()
algr.RuleGroups()
g := algr.RuleGroups()
g[0].Eval(context.Background(), time.Now())
testTargetRetriever := setupTestTargetRetriever(t)
@ -442,12 +474,14 @@ func TestEndpoints(t *testing.T) {
})
require.NoError(t, err)
var algr rulesRetrieverMock
algr := rulesRetrieverMock{}
algr.testing = t
algr.AlertingRules()
algr.CreateAlertingRules()
algr.CreateRuleGroups()
algr.RuleGroups()
g := algr.RuleGroups()
g[0].Eval(context.Background(), time.Now())
testTargetRetriever := setupTestTargetRetriever(t)
@ -1036,6 +1070,36 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
sorter func(interface{})
metadata []targetMetadata
exemplars []exemplar.QueryResult
zeroFunc func(interface{})
}
rulesZeroFunc := func(i interface{}) {
if i != nil {
v := i.(*RuleDiscovery)
for _, ruleGroup := range v.RuleGroups {
ruleGroup.EvaluationTime = float64(0)
ruleGroup.LastEvaluation = time.Time{}
for k, rule := range ruleGroup.Rules {
switch r := rule.(type) {
case AlertingRule:
r.LastEvaluation = time.Time{}
r.EvaluationTime = float64(0)
r.LastError = ""
r.Health = "ok"
for _, alert := range r.Alerts {
alert.ActiveAt = nil
}
ruleGroup.Rules[k] = r
case RecordingRule:
r.LastEvaluation = time.Time{}
r.EvaluationTime = float64(0)
r.LastError = ""
r.Health = "ok"
ruleGroup.Rules[k] = r
}
}
}
}
}
tests := []test{
@ -1988,7 +2052,22 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
{
endpoint: api.alerts,
response: &AlertDiscovery{
Alerts: []*Alert{},
Alerts: []*Alert{
{
Labels: labels.FromStrings("alertname", "test_metric5", "name", "tm5"),
Annotations: labels.Labels{},
State: "pending",
Value: "1e+00",
},
},
},
zeroFunc: func(i interface{}) {
if i != nil {
v := i.(*AlertDiscovery)
for _, alert := range v.Alerts {
alert.ActiveAt = nil
}
}
},
},
{
@ -2009,7 +2088,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
Labels: labels.Labels{},
Annotations: labels.Labels{},
Alerts: []*Alert{},
Health: "unknown",
Health: "ok",
Type: "alerting",
},
AlertingRule{
@ -2020,20 +2099,98 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
Labels: labels.Labels{},
Annotations: labels.Labels{},
Alerts: []*Alert{},
Health: "unknown",
Health: "ok",
Type: "alerting",
},
AlertingRule{
State: "pending",
Name: "test_metric5",
Query: "vector(1)",
Duration: 1,
Labels: labels.FromStrings("name", "tm5"),
Annotations: labels.Labels{},
Alerts: []*Alert{
{
Labels: labels.FromStrings("alertname", "test_metric5", "name", "tm5"),
Annotations: labels.Labels{},
State: "pending",
Value: "1e+00",
},
},
Health: "ok",
Type: "alerting",
},
RecordingRule{
Name: "recording-rule-1",
Query: "vector(1)",
Labels: labels.Labels{},
Health: "unknown",
Health: "ok",
Type: "recording",
},
},
},
},
},
zeroFunc: rulesZeroFunc,
},
{
endpoint: api.rules,
query: url.Values{
"exclude_alerts": []string{"true"},
},
response: &RuleDiscovery{
RuleGroups: []*RuleGroup{
{
Name: "grp",
File: "/path/to/file",
Interval: 1,
Limit: 0,
Rules: []Rule{
AlertingRule{
State: "inactive",
Name: "test_metric3",
Query: "absent(test_metric3) != 1",
Duration: 1,
Labels: labels.Labels{},
Annotations: labels.Labels{},
Alerts: nil,
Health: "ok",
Type: "alerting",
},
AlertingRule{
State: "inactive",
Name: "test_metric4",
Query: "up == 1",
Duration: 1,
Labels: labels.Labels{},
Annotations: labels.Labels{},
Alerts: nil,
Health: "ok",
Type: "alerting",
},
AlertingRule{
State: "pending",
Name: "test_metric5",
Query: "vector(1)",
Duration: 1,
Labels: labels.FromStrings("name", "tm5"),
Annotations: labels.Labels{},
Alerts: nil,
Health: "ok",
Type: "alerting",
},
RecordingRule{
Name: "recording-rule-1",
Query: "vector(1)",
Labels: labels.Labels{},
Health: "ok",
Type: "recording",
},
},
},
},
},
zeroFunc: rulesZeroFunc,
},
{
endpoint: api.rules,
@ -2056,7 +2213,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
Labels: labels.Labels{},
Annotations: labels.Labels{},
Alerts: []*Alert{},
Health: "unknown",
Health: "ok",
Type: "alerting",
},
AlertingRule{
@ -2067,13 +2224,32 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
Labels: labels.Labels{},
Annotations: labels.Labels{},
Alerts: []*Alert{},
Health: "unknown",
Health: "ok",
Type: "alerting",
},
AlertingRule{
State: "pending",
Name: "test_metric5",
Query: "vector(1)",
Duration: 1,
Labels: labels.FromStrings("name", "tm5"),
Annotations: labels.Labels{},
Alerts: []*Alert{
{
Labels: labels.FromStrings("alertname", "test_metric5", "name", "tm5"),
Annotations: labels.Labels{},
State: "pending",
Value: "1e+00",
},
},
Health: "ok",
Type: "alerting",
},
},
},
},
},
zeroFunc: rulesZeroFunc,
},
{
endpoint: api.rules,
@ -2092,13 +2268,14 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
Name: "recording-rule-1",
Query: "vector(1)",
Labels: labels.Labels{},
Health: "unknown",
Health: "ok",
Type: "recording",
},
},
},
},
},
zeroFunc: rulesZeroFunc,
},
{
endpoint: api.rules,
@ -2119,13 +2296,14 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
Labels: labels.Labels{},
Annotations: labels.Labels{},
Alerts: []*Alert{},
Health: "unknown",
Health: "ok",
Type: "alerting",
},
},
},
},
},
zeroFunc: rulesZeroFunc,
},
{
endpoint: api.rules,
@ -2151,13 +2329,14 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
Labels: labels.Labels{},
Annotations: labels.Labels{},
Alerts: []*Alert{},
Health: "unknown",
Health: "ok",
Type: "alerting",
},
},
},
},
},
zeroFunc: rulesZeroFunc,
},
{
endpoint: api.queryExemplars,
@ -2696,6 +2875,9 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
assertAPIResponseMetadataLen(t, res.data, test.responseMetadataTotal)
}
} else {
if test.zeroFunc != nil {
test.zeroFunc(res.data)
}
assertAPIResponse(t, res.data, test.response)
}
})