// Copyright 2013 Prometheus Team // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package notification import ( "bytes" "encoding/json" "flag" "io" "io/ioutil" "net/http" "text/template" "time" "github.com/golang/glog" clientmodel "github.com/prometheus/client_golang/model" "github.com/prometheus/prometheus/utility" ) const ( alertmanagerApiEventsPath = "/api/events" contentTypeJson = "application/json" ) var ( deadline = flag.Duration("alertmanager.httpDeadline", 10*time.Second, "Alert manager HTTP API timeout.") ) // A request for sending a notification to the alert manager for a single alert // vector element. type NotificationReq struct { // Short-form alert summary. May contain text/template-style interpolations. Summary string // Longer alert description. May contain text/template-style interpolations. Description string // Labels associated with this alert notification, including alert name. Labels clientmodel.LabelSet // Current value of alert Value clientmodel.SampleValue // Since when this alert has been active (pending or firing). ActiveSince time.Time // A textual representation of the rule that triggered the alert. RuleString string } type NotificationReqs []*NotificationReq type httpPoster interface { Post(url string, bodyType string, body io.Reader) (*http.Response, error) } // NotificationHandler is responsible for dispatching alert notifications to an // alert manager service. type NotificationHandler struct { // The URL of the alert manager to send notifications to. alertmanagerUrl string // The URL of this Prometheus instance to include in notifications. prometheusUrl string // Buffer of notifications that have not yet been sent. pendingNotifications <-chan NotificationReqs // HTTP client with custom timeout settings. httpClient httpPoster } // Construct a new NotificationHandler. func NewNotificationHandler(alertmanagerUrl string, prometheusUrl string, notificationReqs <-chan NotificationReqs) *NotificationHandler { return &NotificationHandler{ alertmanagerUrl: alertmanagerUrl, pendingNotifications: notificationReqs, httpClient: utility.NewDeadlineClient(*deadline), prometheusUrl: prometheusUrl, } } // Interpolate alert information into summary/description templates. func interpolateMessage(msg string, labels clientmodel.LabelSet, value clientmodel.SampleValue) string { t := template.New("message") // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. defs := "{{$labels := .Labels}}" + "{{$value := .Value}}" if _, err := t.Parse(defs + msg); err != nil { glog.Warning("Error parsing template: ", err) return msg } l := map[string]string{} for k, v := range labels { l[string(k)] = string(v) } tmplData := struct { Labels map[string]string Value clientmodel.SampleValue }{ Labels: l, Value: value, } var buf bytes.Buffer if err := t.Execute(&buf, &tmplData); err != nil { glog.Warning("Error executing template: ", err) return msg } return buf.String() } // Send a list of notifications to the configured alert manager. func (n *NotificationHandler) sendNotifications(reqs NotificationReqs) error { alerts := make([]map[string]interface{}, 0, len(reqs)) for _, req := range reqs { alerts = append(alerts, map[string]interface{}{ "Summary": interpolateMessage(req.Summary, req.Labels, req.Value), "Description": interpolateMessage(req.Description, req.Labels, req.Value), "Labels": req.Labels, "Payload": map[string]interface{}{ "Value": req.Value, "ActiveSince": req.ActiveSince, "GeneratorUrl": n.prometheusUrl, "AlertingRule": req.RuleString, }, }) } buf, err := json.Marshal(alerts) if err != nil { return err } resp, err := n.httpClient.Post( n.alertmanagerUrl+alertmanagerApiEventsPath, contentTypeJson, bytes.NewBuffer(buf), ) if err != nil { return err } defer resp.Body.Close() _, err = ioutil.ReadAll(resp.Body) if err != nil { return err } // BUG: Do we need to check the response code? return nil } // Report notification queue occupancy and capacity. func (n *NotificationHandler) reportQueues() { notificationsQueueSize.Set(map[string]string{facet: occupancy}, float64(len(n.pendingNotifications))) notificationsQueueSize.Set(map[string]string{facet: capacity}, float64(cap(n.pendingNotifications))) } // Continuously dispatch notifications. func (n *NotificationHandler) Run() { queueReportTicker := time.NewTicker(time.Second) go func() { for _ = range queueReportTicker.C { n.reportQueues() } }() defer queueReportTicker.Stop() for reqs := range n.pendingNotifications { if n.alertmanagerUrl == "" { glog.Warning("No alert manager configured, not dispatching notification") notificationsCount.Increment(map[string]string{result: dropped}) continue } begin := time.Now() err := n.sendNotifications(reqs) recordOutcome(time.Since(begin), err) if err != nil { glog.Error("Error sending notification: ", err) } } }