func (p *Pipeline) ProcessIncident(in *event.Incident) {
// start tracking this incident in memory so we can call back to it
p.tracker.TrackIncident(in)
// dedup the incident
if p.Dedupe(in) {
// update the incident in the index
if in.Status != event.OK {
p.index.PutIncident(in)
} else {
p.index.DeleteIncidentById(in.IndexName())
}
// fetch the escalation to take
esc, ok := p.escalations.Collection()[in.Escalation]
if ok {
// send to every alarm in the escalation
for _, a := range esc {
a.Send(in)
}
} else {
logrus.Error("unknown escalation", in.Escalation)
}
}
}
func (t *Tracker) GetIncidentResolver(i *event.Incident) chan *event.Incident {
var res chan *event.Incident
t.Query(func(r *Tracker) {
res, _ = r.incidentResolvers[string(i.IndexName())]
})
return res
}
// Send an email via smtp
func (e *Email) Send(i *event.Incident) error {
//For now set the description as both the subject and body
headers := make(map[string]string)
headers["From"] = e.conf.Sender
headers["To"] = strings.Join(e.conf.Recipients, ",")
headers["Subject"] = i.FormatDescription()
headers["MIME-Version"] = "1.0"
headers["Content-Type"] = "text/plain; charset=\"utf-8\""
headers["Content-Transfer-Encoding"] = "base64"
// make the body a json encoded representation of the incidnent
body, err := json.MarshalIndent(i, "", " ")
if err != nil {
logrus.Errorf("Unable to encode incidnet for email %s", err.Error())
}
log.Println("sending email")
err = smtp.SendMail(e.conf.Host+":"+strconv.Itoa(e.conf.Port), *e.Auth,
e.conf.Sender, e.conf.Recipients, []byte(writeEmailBuffer(headers, string(body))))
if err != nil {
logrus.Errorf("Unable to send email via smtp %s", err)
}
log.Println("done sending mail")
return err
}
// returns true if this is a new incident, false if it is a duplicate
func (p *Pipeline) Dedupe(i *event.Incident) bool {
old := p.index.GetIncident(i.IndexName())
if old == nil {
return i.Status != event.OK
}
return old.Status != i.Status
}
// PassIncident takes an incident into the escalation for processing
func (e *EscalationPolicy) PassIncident(i *event.Incident) {
// only process incidents that this policy subscribes to
if e.isSubscribed(i) {
// send if off to every escalation known about
for _, ep := range e.Escalations {
err := ep.Send(i)
if err != nil {
logrus.Errorf("Unable to forward incident %s to escalation %+v", i.FormatDescription(), ep)
}
}
}
}
func (p *Pipeline) PutIncident(in *event.Incident) {
if in.Id == 0 {
in.Id = p.index.GetIncidentCounter()
p.index.UpdateIncidentCounter(in.Id + 1)
}
p.index.PutIncident(in)
}
// TrackIncident will allow the tracker to keep state about an incident
func (t *Tracker) TrackIncident(i *event.Incident) {
if i.GetResolve() != nil {
t.Query(func(r *Tracker) {
// Don't keep track of "OK" incident resolvers, as ok's can't be resolved
if i.Status != event.OK {
r.incidentResolvers[string(i.IndexName())] = i.GetResolve()
}
r.totalIncidents.inc()
})
}
}
func (c *Console) Send(i *event.Incident) error {
switch i.Status {
case event.OK:
logrus.Info(i.FormatDescription())
case event.WARNING:
logrus.Warn(i.FormatDescription())
case event.CRITICAL:
logrus.Error(i.FormatDescription())
}
return nil
}
func (p *PagerDuty) Send(i *event.Incident) error {
var pdPevent *pagerduty.Event
switch i.Status {
case event.CRITICAL, event.WARNING:
pdPevent = pagerduty.NewTriggerEvent(p.conf.Key, i.FormatDescription())
case event.OK:
pdPevent = pagerduty.NewResolveEvent(p.conf.Key, i.FormatDescription())
}
pdPevent.IncidentKey = string(string(i.IndexName()))
_, _, err := pagerduty.Submit(pdPevent)
return err
}
// processIncident forwards a deduped incident on to every escalation
func (p *Pipeline) processIncident(in *event.Incident) {
in.GetEvent().SetState(event.StateIncident)
// start tracking this incident in memory so we can call back to it
p.tracker.TrackIncident(in)
// dedup the incident
if p.Dedupe(in) {
// update the incident in the index
if in.Status != event.OK {
p.index.PutIncident(in)
} else {
p.index.DeleteIncidentById(in.IndexName())
}
// send it on to every escalation
for _, esc := range p.escalations {
esc.PassIncident(in)
}
}
in.GetEvent().SetState(event.StateComplete)
}
// bangarang.annotation.{status}.{host}.{service}
func formatName(i *event.Incident) string {
return strings.Replace(fmt.Sprintf("%s.%s.%s.%s", ANNOTATION_PREFIX, event.Status(i.Status), i.Tags.Get("host"), strings.Replace(i.FormatDescription(), ".", "_", -1)), " ", "_", -1)
}
func (t *testingPasser) PassIncident(i *event.Incident) {
if t.incidents == nil {
t.incidents = map[string]*event.Incident{}
}
t.incidents[string(i.IndexName())] = i
}