func (s *Schedule) notify(st *models.IncidentState, n *conf.Notification) {
if len(st.EmailSubject) == 0 {
st.EmailSubject = []byte(st.Subject)
}
if len(st.EmailBody) == 0 {
st.EmailBody = []byte(st.Body)
}
n.Notify(st.Subject, st.Body, st.EmailSubject, st.EmailBody, s.SystemConf, string(st.AlertKey), st.Attachments...)
}
// utnotify is single notification for N unknown groups into a single notification
func (s *Schedule) utnotify(groups map[string]models.AlertKeys, n *conf.Notification) {
var total int
now := utcNow()
for _, group := range groups {
// Don't know what the following line does, just copied from unotify
s.Group[now] = group
total += len(group)
}
subject := fmt.Sprintf("%v unknown alert instances suppressed", total)
body := new(bytes.Buffer)
if err := unknownMultiGroup.Execute(body, struct {
Groups map[string]models.AlertKeys
Threshold int
}{
groups,
s.SystemConf.GetUnknownThreshold(),
}); err != nil {
slog.Errorln(err)
}
n.Notify(subject, body.String(), []byte(subject), body.Bytes(), s.SystemConf, "unknown_treshold")
}
func (s *Schedule) unotify(name string, group models.AlertKeys, n *conf.Notification) {
subject := new(bytes.Buffer)
body := new(bytes.Buffer)
now := utcNow()
s.Group[now] = group
t := s.RuleConf.GetUnknownTemplate()
if t == nil {
t = defaultUnknownTemplate
}
data := s.unknownData(now, name, group)
if t.Body != nil {
if err := t.Body.Execute(body, &data); err != nil {
slog.Infoln("unknown template error:", err)
}
}
if t.Subject != nil {
if err := t.Subject.Execute(subject, &data); err != nil {
slog.Infoln("unknown template error:", err)
}
}
n.Notify(subject.String(), body.String(), subject.Bytes(), body.Bytes(), s.SystemConf, name)
}
func (s *Schedule) unotify(name string, group expr.AlertKeys, n *conf.Notification) {
subject := new(bytes.Buffer)
body := new(bytes.Buffer)
now := time.Now().UTC()
s.Group[now] = group
t := s.Conf.UnknownTemplate
if t == nil {
t = defaultUnknownTemplate
}
data := s.unknownData(now, name, group)
if t.Body != nil {
if err := t.Body.Execute(body, &data); err != nil {
log.Println("unknown template error:", err)
}
}
if t.Subject != nil {
if err := t.Subject.Execute(subject, &data); err != nil {
log.Println("unknown template error:", err)
}
}
n.Notify(subject.String(), body.String(), subject.Bytes(), body.Bytes(), s.Conf, name)
}
func procRule(t miniprofiler.Timer, c *conf.Conf, a *conf.Alert, now time.Time, summary bool, email string, template_group string) (*ruleResult, error) {
s := &sched.Schedule{}
s.DataAccess = schedule.DataAccess
s.Search = schedule.Search
if err := s.Init(c); err != nil {
return nil, err
}
rh := s.NewRunHistory(now, cacheObj)
if _, err := s.CheckExpr(t, rh, a, a.Warn, models.StWarning, nil); err != nil {
return nil, err
}
if _, err := s.CheckExpr(t, rh, a, a.Crit, models.StCritical, nil); err != nil {
return nil, err
}
keys := make(models.AlertKeys, len(rh.Events))
criticals, warnings, normals := make([]models.AlertKey, 0), make([]models.AlertKey, 0), make([]models.AlertKey, 0)
i := 0
for k, v := range rh.Events {
v.Time = now
keys[i] = k
i++
switch v.Status {
case models.StNormal:
normals = append(normals, k)
case models.StWarning:
warnings = append(warnings, k)
case models.StCritical:
criticals = append(criticals, k)
default:
return nil, fmt.Errorf("unknown state type %v", v.Status)
}
}
sort.Sort(keys)
var subject, body []byte
var data interface{}
warning := make([]string, 0)
if !summary && len(keys) > 0 {
var primaryIncident *models.IncidentState
if template_group != "" {
ts, err := opentsdb.ParseTags(template_group)
if err != nil {
return nil, err
}
for _, ak := range keys {
if ak.Group().Subset(ts) {
primaryIncident = sched.NewIncident(ak)
primaryIncident.Events = []models.Event{*rh.Events[ak]}
break
}
}
}
if primaryIncident == nil {
primaryIncident = sched.NewIncident(keys[0])
primaryIncident.Events = []models.Event{*rh.Events[keys[0]]}
if template_group != "" {
warning = append(warning, fmt.Sprintf("template group %s was not a subset of any result", template_group))
}
}
if e := primaryIncident.Events[0]; e.Crit != nil {
primaryIncident.Result = e.Crit
} else if e.Warn != nil {
primaryIncident.Result = e.Warn
}
var b_err, s_err error
func() {
defer func() {
if err := recover(); err != nil {
s := fmt.Sprint(err)
warning = append(warning, s)
b_err = fmt.Errorf(s)
}
}()
if body, _, b_err = s.ExecuteBody(rh, a, primaryIncident, false); b_err != nil {
warning = append(warning, b_err.Error())
}
}()
func() {
defer func() {
if err := recover(); err != nil {
s := fmt.Sprint(err)
warning = append(warning, s)
s_err = fmt.Errorf(s)
}
}()
subject, s_err = s.ExecuteSubject(rh, a, primaryIncident, false)
if s_err != nil {
warning = append(warning, s_err.Error())
}
}()
if s_err != nil || b_err != nil {
var err error
subject, body, err = s.ExecuteBadTemplate([]error{s_err, b_err}, rh, a, primaryIncident)
if err != nil {
subject = []byte(fmt.Sprintf("unable to create tempalate error notification: %v", err))
}
} else if email != "" {
m, err := mail.ParseAddress(email)
if err != nil {
return nil, err
}
n := conf.Notification{
Email: []*mail.Address{m},
}
email, attachments, b_err := s.ExecuteBody(rh, a, primaryIncident, true)
email_subject, s_err := s.ExecuteSubject(rh, a, primaryIncident, true)
if b_err != nil {
warning = append(warning, b_err.Error())
} else if s_err != nil {
warning = append(warning, s_err.Error())
} else {
n.DoEmail(email_subject, email, schedule.Conf, string(primaryIncident.AlertKey), attachments...)
}
}
data = s.Data(rh, primaryIncident, a, false)
}
return &ruleResult{
criticals,
warnings,
normals,
now,
string(body),
string(subject),
data,
rh.Events,
warning,
}, nil
}
func (s *Schedule) notify(st *State, n *conf.Notification) {
n.Notify(st.Subject, st.Body, st.EmailSubject, st.EmailBody, s.Conf, string(st.AlertKey()), st.Attachments...)
}
func (c *Conf) loadNotification(s *parse.SectionNode) {
name := s.Name.Text
if _, ok := c.Notifications[name]; ok {
c.errorf("duplicate notification name: %s", name)
}
n := conf.Notification{
Vars: make(map[string]string),
ContentType: "application/x-www-form-urlencoded",
Name: name,
RunOnActions: true,
}
n.Text = s.RawText
n.Locator = newSectionLocator(s)
funcs := ttemplate.FuncMap{
"V": func(v string) string {
return c.Expand(v, n.Vars, false)
},
"json": func(v interface{}) string {
b, err := json.Marshal(v)
if err != nil {
slog.Errorln(err)
}
return string(b)
},
}
c.Notifications[name] = &n
pairs := c.getPairs(s, n.Vars, sNormal)
for _, p := range pairs {
c.at(p.node)
v := p.val
switch k := p.key; k {
case "email":
n.RawEmail = v
email, err := mail.ParseAddressList(n.RawEmail)
if err != nil {
c.error(err)
}
n.Email = email
case "post":
n.RawPost = v
post, err := url.Parse(n.RawPost)
if err != nil {
c.error(err)
}
n.Post = post
case "get":
n.RawGet = v
get, err := url.Parse(n.RawGet)
if err != nil {
c.error(err)
}
n.Get = get
case "print":
n.Print = true
case "contentType":
n.ContentType = v
case "next":
n.NextName = v
next, ok := c.Notifications[n.NextName]
if !ok {
c.errorf("unknown notification %s", n.NextName)
}
n.Next = next
case "timeout":
d, err := opentsdb.ParseDuration(v)
if err != nil {
c.error(err)
}
n.Timeout = time.Duration(d)
case "body":
n.RawBody = v
tmpl := ttemplate.New(name).Funcs(funcs)
_, err := tmpl.Parse(n.RawBody)
if err != nil {
c.error(err)
}
n.Body = tmpl
case "runOnActions":
n.RunOnActions = v == "true"
case "useBody":
n.UseBody = v == "true"
default:
c.errorf("unknown key %s", k)
}
}
c.at(s)
if n.Timeout > 0 && n.Next == nil {
c.errorf("timeout specified without next")
}
}