func (p *Pipeline) ProcessIncident(in *event.Incident) {
// start tracking this incident in memory so we can call back to it
p.tracker.TrackIncident(in)
// dedup the incident
if p.Dedupe(in) {
// update the incident in the index
if in.Status != event.OK {
p.index.PutIncident(in)
} else {
p.index.DeleteIncidentById(in.IndexName())
}
// fetch the escalation to take
esc, ok := p.escalations.Collection()[in.Escalation]
if ok {
// send to every alarm in the escalation
for _, a := range esc {
a.Send(in)
}
} else {
logrus.Error("unknown escalation", in.Escalation)
}
}
}
func (t *Tracker) GetIncidentResolver(i *event.Incident) chan *event.Incident {
var res chan *event.Incident
t.Query(func(r *Tracker) {
res, _ = r.incidentResolvers[string(i.IndexName())]
})
return res
}
// returns true if this is a new incident, false if it is a duplicate
func (p *Pipeline) Dedupe(i *event.Incident) bool {
old := p.index.GetIncident(i.IndexName())
if old == nil {
return i.Status != event.OK
}
return old.Status != i.Status
}
// TrackIncident will allow the tracker to keep state about an incident
func (t *Tracker) TrackIncident(i *event.Incident) {
if i.GetResolve() != nil {
t.Query(func(r *Tracker) {
// Don't keep track of "OK" incident resolvers, as ok's can't be resolved
if i.Status != event.OK {
r.incidentResolvers[string(i.IndexName())] = i.GetResolve()
}
r.totalIncidents.inc()
})
}
}
func (p *PagerDuty) Send(i *event.Incident) error {
var pdPevent *pagerduty.Event
switch i.Status {
case event.CRITICAL, event.WARNING:
pdPevent = pagerduty.NewTriggerEvent(p.conf.Key, i.FormatDescription())
case event.OK:
pdPevent = pagerduty.NewResolveEvent(p.conf.Key, i.FormatDescription())
}
pdPevent.IncidentKey = string(string(i.IndexName()))
_, _, err := pagerduty.Submit(pdPevent)
return err
}
// processIncident forwards a deduped incident on to every escalation
func (p *Pipeline) processIncident(in *event.Incident) {
in.GetEvent().SetState(event.StateIncident)
// start tracking this incident in memory so we can call back to it
p.tracker.TrackIncident(in)
// dedup the incident
if p.Dedupe(in) {
// update the incident in the index
if in.Status != event.OK {
p.index.PutIncident(in)
} else {
p.index.DeleteIncidentById(in.IndexName())
}
// send it on to every escalation
for _, esc := range p.escalations {
esc.PassIncident(in)
}
}
in.GetEvent().SetState(event.StateComplete)
}
func (t *testingPasser) PassIncident(i *event.Incident) {
if t.incidents == nil {
t.incidents = map[string]*event.Incident{}
}
t.incidents[string(i.IndexName())] = i
}