func TestPutIncident(t *testing.T) {
x := runningTestContext()
x.runTest(func(p *Pipeline) {
in := event.NewIncident("test", event.OK, event.NewEvent())
p.PutIncident(in)
})
}
func TestDedupe(t *testing.T) {
x := runningTestContext()
x.runTest(func(p *Pipeline) {
in := event.NewIncident("test", event.OK, event.NewEvent())
p.PutIncident(in)
if p.Dedupe(in) {
t.Fatal("Incident was not deduped")
}
})
}
func TestListIncidents(t *testing.T) {
x := runningTestContext()
x.runTest(func(p *Pipeline) {
in := event.NewIncident("test", event.OK, event.NewEvent())
p.PutIncident(in)
if l := p.ListIncidents(); l[0].Policy != in.Policy {
t.Fatal("Incident was not added to the index")
}
})
}
// start the policy listening for events
func (p *Policy) start() {
go func() {
var e *event.Event
for {
select {
case toResolve := <-p.resolve:
var c *Condition
// fetch the condition that created this incident
if toResolve.Status == event.CRITICAL {
c = p.Crit
} else if toResolve.Status == event.WARNING {
c = p.Warn
}
if c != nil {
t := c.getTracker(&toResolve.Event)
t.refresh()
}
case <-p.stop:
logrus.Info("Stopping policy", p.Name)
// cleanup the policy. Sometimes they hangaround.
p.clean()
// stop this policy from being stopped again
p.stop = nil
// catch resolve's that could be sent to this policy
res := p.resolve
p.resolve = nil
go func() {
for {
select {
case <-res:
logrus.Info("Attempted to resolve an incident on a policy that no longer exists")
case <-time.After(1 * time.Minute):
return
}
}
}()
return
case e = <-p.in:
e.SetState(event.StatePolicy)
// process the event if it matches the policy
if p.Matches(e) {
// check critical
if shouldAlert, status := p.ActionCrit(e); shouldAlert {
incident := event.NewIncident(p.Name, status, e)
incident.SetResolve(p.resolve)
// send send it off to the next hop
p.next.PassIncident(incident)
// check warning
} else if shouldAlert, status := p.ActionWarn(e); shouldAlert {
incident := event.NewIncident(p.Name, status, e)
incident.SetResolve(p.resolve)
// send it off to the next hop
p.next.PassIncident(incident)
} else {
e.SetState(event.StateComplete)
}
}
}
}
}()
}
// start the policy listening for events
func (p *Policy) start() {
go func() {
var in *pack
for {
select {
case toResolve := <-p.resolve:
var c *Condition
// fetch the condition that created this incident
if toResolve.Status == event.CRITICAL {
c = p.Crit
} else if toResolve.Status == event.WARNING {
c = p.Warn
}
if c != nil {
e := &event.Event{}
e.Host = toResolve.Host
e.Service = toResolve.Service
e.SubService = toResolve.SubService
t := c.getTracker(e)
t.refresh()
}
case <-p.stop:
logrus.Info("Stopping policy", p.Name)
// cleanup the policy. Sometimes they hangaround.
p.clean()
// stop this policy from being stopped again
p.stop = nil
// catch resolve's that could be sent to this policy
res := p.resolve
p.resolve = nil
go func() {
for {
select {
case <-res:
logrus.Info("Attempted to resolve an incident on a policy that no longer exists")
case <-time.After(1 * time.Minute):
return
}
}
}()
return
case in = <-p.in:
// process the event if it matches the policy
if p.Matches(in.e) {
// process the request
// check critical
if shouldAlert, status := p.ActionCrit(in.e); shouldAlert {
incident := event.NewIncident(p.Name, p.Crit.Escalation, status, in.e)
incident.SetResolve(p.resolve)
in.n(incident)
logrus.Info(incident.FormatDescription())
// check warning
} else if shouldAlert, status := p.ActionWarn(in.e); shouldAlert {
incident := event.NewIncident(p.Name, p.Warn.Escalation, status, in.e)
incident.SetResolve(p.resolve)
in.n(incident)
logrus.Info(incident.FormatDescription())
}
}
in.e.WaitDec()
}
}
}()
}