func (c *QueryCondition) Eval(context *alerting.EvalContext) {
timerange := tsdb.NewTimerange(c.Query.From, c.Query.To)
seriesList, err := c.executeQuery(context, timerange)
if err != nil {
context.Error = err
return
}
emptySerieCount := 0
for _, series := range seriesList {
reducedValue := c.Reducer.Reduce(series)
evalMatch := c.Evaluator.Eval(reducedValue)
if reducedValue == nil {
emptySerieCount++
continue
}
if context.IsTestRun {
context.Logs = append(context.Logs, &alerting.ResultLogEntry{
Message: fmt.Sprintf("Condition[%d]: Eval: %v, Metric: %s, Value: %1.3f", c.Index, evalMatch, series.Name, *reducedValue),
})
}
if evalMatch {
context.EvalMatches = append(context.EvalMatches, &alerting.EvalMatch{
Metric: series.Name,
Value: *reducedValue,
})
}
}
context.NoDataFound = emptySerieCount == len(seriesList)
context.Firing = len(context.EvalMatches) > 0
}
func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.ConditionResult, error) {
timeRange := tsdb.NewTimeRange(c.Query.From, c.Query.To)
seriesList, err := c.executeQuery(context, timeRange)
if err != nil {
return nil, err
}
emptySerieCount := 0
evalMatchCount := 0
var matches []*alerting.EvalMatch
for _, series := range seriesList {
reducedValue := c.Reducer.Reduce(series)
evalMatch := c.Evaluator.Eval(reducedValue)
if reducedValue.Valid == false {
emptySerieCount++
}
if context.IsTestRun {
context.Logs = append(context.Logs, &alerting.ResultLogEntry{
Message: fmt.Sprintf("Condition[%d]: Eval: %v, Metric: %s, Value: %s", c.Index, evalMatch, series.Name, reducedValue),
})
}
if evalMatch {
evalMatchCount++
matches = append(matches, &alerting.EvalMatch{
Metric: series.Name,
Value: reducedValue,
})
}
}
// handle no series special case
if len(seriesList) == 0 {
// eval condition for null value
evalMatch := c.Evaluator.Eval(null.FloatFromPtr(nil))
if context.IsTestRun {
context.Logs = append(context.Logs, &alerting.ResultLogEntry{
Message: fmt.Sprintf("Condition[%d]: Eval: %v, Query Returned No Series (reduced to null/no value)", evalMatch),
})
}
if evalMatch {
evalMatchCount++
matches = append(matches, &alerting.EvalMatch{Metric: "NoData", Value: null.FloatFromPtr(nil)})
}
}
return &alerting.ConditionResult{
Firing: evalMatchCount > 0,
NoDataFound: emptySerieCount == len(seriesList),
Operator: c.Operator,
EvalMatches: matches,
}, nil
}
func (this *LineNotifier) createAlert(evalContext *alerting.EvalContext) error {
this.log.Info("Creating Line notify", "ruleId", evalContext.Rule.Id, "notification", this.Name)
ruleUrl, err := evalContext.GetRuleUrl()
if err != nil {
this.log.Error("Failed get rule link", "error", err)
return err
}
form := url.Values{}
body := fmt.Sprintf("%s - %s\n%s", evalContext.Rule.Name, ruleUrl, evalContext.Rule.Message)
form.Add("message", body)
cmd := &m.SendWebhookSync{
Url: lineNotifyUrl,
HttpMethod: "POST",
HttpHeader: map[string]string{
"Authorization": fmt.Sprintf("Bearer %s", this.Token),
"Content-Type": "application/x-www-form-urlencoded",
},
Body: form.Encode(),
}
if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
this.log.Error("Failed to send notification to LINE", "error", err, "body", string(body))
return err
}
return nil
}
func (this *TelegramNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Sending alert notification to", "bot_token", this.BotToken)
this.log.Info("Sending alert notification to", "chat_id", this.ChatID)
metrics.M_Alerting_Notification_Sent_Telegram.Inc(1)
bodyJSON := simplejson.New()
bodyJSON.Set("chat_id", this.ChatID)
bodyJSON.Set("parse_mode", "html")
message := fmt.Sprintf("%s\nState: %s\nMessage: %s\n", evalContext.GetNotificationTitle(), evalContext.Rule.Name, evalContext.Rule.Message)
ruleUrl, err := evalContext.GetRuleUrl()
if err == nil {
message = message + fmt.Sprintf("URL: %s\n", ruleUrl)
}
bodyJSON.Set("text", message)
url := fmt.Sprintf(telegeramApiUrl, this.BotToken, "sendMessage")
body, _ := bodyJSON.MarshalJSON()
cmd := &m.SendWebhookSync{
Url: url,
Body: string(body),
HttpMethod: "POST",
}
if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
this.log.Error("Failed to send webhook", "error", err, "webhook", this.Name)
return err
}
return nil
}
func (this *WebhookNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Sending webhook")
metrics.M_Alerting_Notification_Sent_Webhook.Inc(1)
bodyJSON := simplejson.New()
bodyJSON.Set("title", evalContext.GetNotificationTitle())
bodyJSON.Set("ruleId", evalContext.Rule.Id)
bodyJSON.Set("ruleName", evalContext.Rule.Name)
bodyJSON.Set("state", evalContext.Rule.State)
bodyJSON.Set("evalMatches", evalContext.EvalMatches)
ruleUrl, err := evalContext.GetRuleUrl()
if err == nil {
bodyJSON.Set("rule_url", ruleUrl)
}
if evalContext.ImagePublicUrl != "" {
bodyJSON.Set("image_url", evalContext.ImagePublicUrl)
}
body, _ := bodyJSON.MarshalJSON()
cmd := &m.SendWebhookSync{
Url: this.Url,
User: this.User,
Password: this.Password,
Body: string(body),
}
if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
this.log.Error("Failed to send webhook", "error", err, "webhook", this.Name)
}
return nil
}
// Notify sends notification to Victorops via POST to URL endpoint
func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
metrics.M_Alerting_Notification_Sent_Victorops.Inc(1)
ruleUrl, err := evalContext.GetRuleUrl()
if err != nil {
this.log.Error("Failed get rule link", "error", err)
return err
}
fields := make([]map[string]interface{}, 0)
fieldLimitCount := 4
for index, evt := range evalContext.EvalMatches {
fields = append(fields, map[string]interface{}{
"title": evt.Metric,
"value": evt.Value,
"short": true,
})
if index > fieldLimitCount {
break
}
}
if evalContext.Error != nil {
fields = append(fields, map[string]interface{}{
"title": "Error message",
"value": evalContext.Error.Error(),
"short": false,
})
}
messageType := evalContext.Rule.State
if evalContext.Rule.State == models.AlertStateAlerting { // translate 'Alerting' to 'CRITICAL' (Victorops analog)
messageType = AlertStateCritical
}
body := map[string]interface{}{
"message_type": messageType,
"entity_id": evalContext.Rule.Name,
"timestamp": time.Now().Unix(),
"state_start_time": evalContext.StartTime.Unix(),
"state_message": evalContext.Rule.Message + "\n" + ruleUrl,
"monitoring_tool": "Grafana v" + setting.BuildVersion,
}
data, _ := json.Marshal(&body)
cmd := &models.SendWebhookSync{Url: this.URL, Body: string(data)}
if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
this.log.Error("Failed to send victorops notification", "error", err, "webhook", this.Name)
return err
}
return nil
}
func (this *SensuNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Sending sensu result")
metrics.M_Alerting_Notification_Sent_Sensu.Inc(1)
bodyJSON := simplejson.New()
bodyJSON.Set("ruleId", evalContext.Rule.Id)
// Sensu alerts cannot have spaces in them
bodyJSON.Set("name", strings.Replace(evalContext.Rule.Name, " ", "_", -1))
// Sensu alerts require a command
// We set it to the grafana ruleID
bodyJSON.Set("source", "grafana_rule_"+strconv.FormatInt(evalContext.Rule.Id, 10))
// Finally, sensu expects an output
// We set it to a default output
bodyJSON.Set("output", "Grafana Metric Condition Met")
bodyJSON.Set("evalMatches", evalContext.EvalMatches)
if evalContext.Rule.State == "alerting" {
bodyJSON.Set("status", 2)
} else if evalContext.Rule.State == "no_data" {
bodyJSON.Set("status", 1)
} else {
bodyJSON.Set("status", 0)
}
ruleUrl, err := evalContext.GetRuleUrl()
if err == nil {
bodyJSON.Set("ruleUrl", ruleUrl)
}
if evalContext.ImagePublicUrl != "" {
bodyJSON.Set("imageUrl", evalContext.ImagePublicUrl)
}
if evalContext.Rule.Message != "" {
bodyJSON.Set("message", evalContext.Rule.Message)
}
body, _ := bodyJSON.MarshalJSON()
cmd := &m.SendWebhookSync{
Url: this.Url,
User: this.User,
Password: this.Password,
Body: string(body),
HttpMethod: "POST",
}
if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
this.log.Error("Failed to send sensu event", "error", err, "sensu", this.Name)
return err
}
return nil
}
func (this *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
metrics.M_Alerting_Notification_Sent_PagerDuty.Inc(1)
if evalContext.Rule.State == m.AlertStateOK && !this.AutoResolve {
this.log.Info("Not sending a trigger to Pagerduty", "state", evalContext.Rule.State, "auto resolve", this.AutoResolve)
return nil
}
eventType := "trigger"
if evalContext.Rule.State == m.AlertStateOK {
eventType = "resolve"
}
this.log.Info("Notifying Pagerduty", "event_type", eventType)
bodyJSON := simplejson.New()
bodyJSON.Set("service_key", this.Key)
bodyJSON.Set("description", evalContext.Rule.Name+" - "+evalContext.Rule.Message)
bodyJSON.Set("client", "Grafana")
bodyJSON.Set("event_type", eventType)
bodyJSON.Set("incident_key", "alertId-"+strconv.FormatInt(evalContext.Rule.Id, 10))
ruleUrl, err := evalContext.GetRuleUrl()
if err != nil {
this.log.Error("Failed get rule link", "error", err)
return err
}
bodyJSON.Set("client_url", ruleUrl)
if evalContext.ImagePublicUrl != "" {
contexts := make([]interface{}, 1)
imageJSON := simplejson.New()
imageJSON.Set("type", "image")
imageJSON.Set("src", evalContext.ImagePublicUrl)
contexts[0] = imageJSON
bodyJSON.Set("contexts", contexts)
}
body, _ := bodyJSON.MarshalJSON()
cmd := &m.SendWebhookSync{
Url: pagerdutyEventApiUrl,
Body: string(body),
HttpMethod: "POST",
}
if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
this.log.Error("Failed to send notification to Pagerduty", "error", err, "body", string(body))
return err
}
return nil
}
func (c *QueryCondition) executeQuery(context *alerting.EvalContext, timeRange *tsdb.TimeRange) (tsdb.TimeSeriesSlice, error) {
getDsInfo := &m.GetDataSourceByIdQuery{
Id: c.Query.DatasourceId,
OrgId: context.Rule.OrgId,
}
if err := bus.Dispatch(getDsInfo); err != nil {
return nil, fmt.Errorf("Could not find datasource")
}
req := c.getRequestForAlertRule(getDsInfo.Result, timeRange)
result := make(tsdb.TimeSeriesSlice, 0)
resp, err := c.HandleRequest(context.Ctx, req)
if err != nil {
return nil, fmt.Errorf("tsdb.HandleRequest() error %v", err)
}
for _, v := range resp.Results {
if v.Error != nil {
return nil, fmt.Errorf("tsdb.HandleRequest() response error %v", v)
}
result = append(result, v.Series...)
if context.IsTestRun {
context.Logs = append(context.Logs, &alerting.ResultLogEntry{
Message: fmt.Sprintf("Condition[%d]: Query Result", c.Index),
Data: v.Series,
})
}
}
return result, nil
}
func (this *EmailNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Sending alert notification to", "addresses", this.Addresses)
metrics.M_Alerting_Notification_Sent_Email.Inc(1)
ruleUrl, err := evalContext.GetRuleUrl()
if err != nil {
this.log.Error("Failed get rule link", "error", err)
return err
}
cmd := &m.SendEmailCommandSync{
SendEmailCommand: m.SendEmailCommand{
Subject: evalContext.GetNotificationTitle(),
Data: map[string]interface{}{
"Title": evalContext.GetNotificationTitle(),
"State": evalContext.Rule.State,
"Name": evalContext.Rule.Name,
"StateModel": evalContext.GetStateModel(),
"Message": evalContext.Rule.Message,
"RuleUrl": ruleUrl,
"ImageLink": "",
"EmbededImage": "",
"AlertPageUrl": setting.AppUrl + "alerting",
"EvalMatches": evalContext.EvalMatches,
},
To: this.Addresses,
Template: "alert_notification.html",
EmbededFiles: []string{},
},
}
if evalContext.ImagePublicUrl != "" {
cmd.Data["ImageLink"] = evalContext.ImagePublicUrl
} else {
file, err := os.Stat(evalContext.ImageOnDiskPath)
if err == nil {
cmd.EmbededFiles = []string{evalContext.ImageOnDiskPath}
cmd.Data["EmbededImage"] = file.Name()
}
}
err = bus.DispatchCtx(evalContext.Ctx, cmd)
if err != nil {
this.log.Error("Failed to send alert notification email", "error", err)
return err
}
return nil
}
func (this *EmailNotifier) Notify(context *alerting.EvalContext) {
this.log.Info("Sending alert notification to", "addresses", this.Addresses)
metrics.M_Alerting_Notification_Sent_Email.Inc(1)
ruleUrl, err := context.GetRuleUrl()
if err != nil {
this.log.Error("Failed get rule link", "error", err)
return
}
cmd := &m.SendEmailCommand{
Data: map[string]interface{}{
"Title": context.GetNotificationTitle(),
"State": context.Rule.State,
"Name": context.Rule.Name,
"StateModel": context.GetStateModel(),
"Message": context.Rule.Message,
"RuleUrl": ruleUrl,
"ImageLink": context.ImagePublicUrl,
"AlertPageUrl": setting.AppUrl + "alerting",
"EvalMatches": context.EvalMatches,
},
To: this.Addresses,
Template: "alert_notification.html",
}
if err := bus.Dispatch(cmd); err != nil {
this.log.Error("Failed to send alert notification email", "error", err)
}
}
func (this *OpsGenieNotifier) createAlert(evalContext *alerting.EvalContext) error {
this.log.Info("Creating OpsGenie alert", "ruleId", evalContext.Rule.Id, "notification", this.Name)
ruleUrl, err := evalContext.GetRuleUrl()
if err != nil {
this.log.Error("Failed get rule link", "error", err)
return err
}
bodyJSON := simplejson.New()
bodyJSON.Set("apiKey", this.ApiKey)
bodyJSON.Set("message", evalContext.Rule.Name)
bodyJSON.Set("source", "Grafana")
bodyJSON.Set("alias", "alertId-"+strconv.FormatInt(evalContext.Rule.Id, 10))
bodyJSON.Set("description", fmt.Sprintf("%s - %s\n%s", evalContext.Rule.Name, ruleUrl, evalContext.Rule.Message))
details := simplejson.New()
details.Set("url", ruleUrl)
if evalContext.ImagePublicUrl != "" {
details.Set("image", evalContext.ImagePublicUrl)
}
bodyJSON.Set("details", details)
body, _ := bodyJSON.MarshalJSON()
cmd := &m.SendWebhookSync{
Url: opsgenieCreateAlertURL,
Body: string(body),
HttpMethod: "POST",
}
if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
this.log.Error("Failed to send notification to OpsGenie", "error", err, "body", string(body))
}
return nil
}
func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.ConditionResult, error) {
timeRange := tsdb.NewTimeRange(c.Query.From, c.Query.To)
seriesList, err := c.executeQuery(context, timeRange)
if err != nil {
return nil, err
}
emptySerieCount := 0
evalMatchCount := 0
var matches []*alerting.EvalMatch
for _, series := range seriesList {
reducedValue := c.Reducer.Reduce(series)
evalMatch := c.Evaluator.Eval(reducedValue)
if reducedValue.Valid == false {
emptySerieCount++
continue
}
if context.IsTestRun {
context.Logs = append(context.Logs, &alerting.ResultLogEntry{
Message: fmt.Sprintf("Condition[%d]: Eval: %v, Metric: %s, Value: %1.3f", c.Index, evalMatch, series.Name, reducedValue.Float64),
})
}
if evalMatch {
evalMatchCount++
matches = append(matches, &alerting.EvalMatch{
Metric: series.Name,
Value: reducedValue.Float64,
})
}
}
return &alerting.ConditionResult{
Firing: evalMatchCount > 0,
NoDataFound: emptySerieCount == len(seriesList),
Operator: c.Operator,
EvalMatches: matches,
}, nil
}
func (this *SlackNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Executing slack notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
metrics.M_Alerting_Notification_Sent_Slack.Inc(1)
ruleUrl, err := evalContext.GetRuleUrl()
if err != nil {
this.log.Error("Failed get rule link", "error", err)
return err
}
fields := make([]map[string]interface{}, 0)
fieldLimitCount := 4
for index, evt := range evalContext.EvalMatches {
fields = append(fields, map[string]interface{}{
"title": evt.Metric,
"value": evt.Value,
"short": true,
})
if index > fieldLimitCount {
break
}
}
if evalContext.Error != nil {
fields = append(fields, map[string]interface{}{
"title": "Error message",
"value": evalContext.Error.Error(),
"short": false,
})
}
message := ""
if evalContext.Rule.State != m.AlertStateOK { //dont add message when going back to alert state ok.
message = evalContext.Rule.Message
}
body := map[string]interface{}{
"attachments": []map[string]interface{}{
{
"color": evalContext.GetStateModel().Color,
"title": evalContext.GetNotificationTitle(),
"title_link": ruleUrl,
"text": message,
"fields": fields,
"image_url": evalContext.ImagePublicUrl,
"footer": "Grafana v" + setting.BuildVersion,
"footer_icon": "http://grafana.org/assets/img/fav32.png",
"ts": time.Now().Unix(),
},
},
}
data, _ := json.Marshal(&body)
cmd := &m.SendWebhookSync{Url: this.Url, Body: string(data)}
if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
this.log.Error("Failed to send slack notification", "error", err, "webhook", this.Name)
}
return nil
}
func (this *SlackNotifier) Notify(context *alerting.EvalContext) {
this.log.Info("Executing slack notification", "ruleId", context.Rule.Id, "notification", this.Name)
metrics.M_Alerting_Notification_Sent_Slack.Inc(1)
ruleUrl, err := context.GetRuleUrl()
if err != nil {
this.log.Error("Failed get rule link", "error", err)
return
}
fields := make([]map[string]interface{}, 0)
fieldLimitCount := 4
for index, evt := range context.EvalMatches {
fields = append(fields, map[string]interface{}{
"title": evt.Metric,
"value": evt.Value,
"short": true,
})
if index > fieldLimitCount {
break
}
}
if context.Error != nil {
fields = append(fields, map[string]interface{}{
"title": "Error message",
"value": context.Error.Error(),
"short": false,
})
}
message := ""
if context.Rule.State != m.AlertStateOK { //dont add message when going back to alert state ok.
message = context.Rule.Message
}
body := map[string]interface{}{
"attachments": []map[string]interface{}{
{
"color": context.GetStateModel().Color,
"title": context.GetNotificationTitle(),
"title_link": ruleUrl,
"text": message,
"fields": fields,
"image_url": context.ImagePublicUrl,
"footer": "Grafana v" + setting.BuildVersion,
"footer_icon": "http://grafana.org/assets/img/fav32.png",
"ts": time.Now().Unix(),
//"pretext": "Optional text that appears above the attachment block",
// "author_name": "Bobby Tables",
// "author_link": "http://flickr.com/bobby/",
// "author_icon": "http://flickr.com/icons/bobby.jpg",
// "thumb_url": "http://example.com/path/to/thumb.png",
},
},
}
data, _ := json.Marshal(&body)
cmd := &m.SendWebhook{Url: this.Url, Body: string(data)}
if err := bus.Dispatch(cmd); err != nil {
this.log.Error("Failed to send slack notification", "error", err, "webhook", this.Name)
}
}