func TestEventRuleRecovers(t *testing.T) {
t.Parallel()
act := mockAction()
svc := &Service{&Entity{"me", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(os.Getpid(), services.Up), services.MockInit()}
rule := &Rule{svc, "memory", "rss", LT, "100m", 100 * 1024 * 1024, 0, false, 1, 0, Ok, []Action{act}}
svc.rules = []*Rule{rule}
svc.Collect(false, func(_ Checkable) {})
events := svc.Verify()
assert.Equal(t, 1, len(events))
assert.Equal(t, 1, act.Size())
assert.Equal(t, RuleFailed, act.Latest().Type)
// recovery takes 2 cycles so we don't flap unnecessarily
rule.Threshold = 1
svc.Collect(false, func(_ Checkable) {})
events = svc.Verify()
assert.Equal(t, 0, len(events))
svc.Collect(false, func(_ Checkable) {})
events = svc.Verify()
assert.Equal(t, 1, len(events))
assert.Equal(t, 2, act.Size())
assert.Equal(t, RuleRecovered, act.Latest().Type)
}
func TestEventProcessAppearsDuringDeploy(t *testing.T) {
t.Parallel()
init := services.MockInit()
init.CurrentStatus = services.WithStatus(os.Getpid(), services.Up)
act := mockAction()
assert.Equal(t, 0, act.Size())
svc := &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Down), init}
svc.Collect(true, func(_ Checkable) {})
assert.Equal(t, services.Up, svc.Process.Status)
assert.Equal(t, os.Getpid(), svc.Process.Pid)
assert.Equal(t, 0, act.Size())
assert.Nil(t, act.Latest())
}
func TestEventProcessDisappears(t *testing.T) {
t.Parallel()
init := services.MockInit()
init.CurrentStatus = services.WithStatus(0, services.Down)
act := mockAction()
assert.Equal(t, 0, act.Size())
svc := &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(findDownPid(), services.Up), init}
svc.Collect(false, func(_ Checkable) {})
assert.Equal(t, services.Down, svc.Process.Status)
assert.Equal(t, 0, svc.Process.Pid)
assert.Equal(t, 1, act.Size())
assert.Equal(t, ProcessDoesNotExist, act.Latest().Type)
}
func TestEventProcessExistsAtStartup(t *testing.T) {
t.Parallel()
init := services.MockInit()
init.CurrentStatus = services.WithStatus(100, services.Up)
act := mockAction()
assert.Equal(t, 0, act.Size())
svc := &Service{&Entity{"exists", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Unknown), init}
svc.Resolve([]services.InitSystem{init})
assert.Equal(t, services.Up, svc.Process.Status)
assert.Equal(t, 100, svc.Process.Pid)
assert.Equal(t, 0, act.Size())
}
func TestEventProcessDneAtStartup(t *testing.T) {
t.Parallel()
init := services.MockInit()
init.CurrentStatus = services.WithStatus(0, services.Down)
act := mockAction()
assert.Equal(t, 0, act.Size())
svc := &Service{&Entity{"dne", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Unknown), nil}
svc.Resolve([]services.InitSystem{init})
assert.Equal(t, services.Down, svc.Process.Status)
assert.Equal(t, 0, svc.Process.Pid)
assert.Equal(t, 1, act.Size())
assert.Equal(t, ProcessDoesNotExist, act.Latest().Type)
}
func TestExport(t *testing.T) {
t.Parallel()
i, err := New("_", "")
i.Services = []Checkable{
&Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(99, services.Up), nil},
}
var resp bytes.Buffer
assert.Nil(t, err)
proc := CommandHandlers["export"]
proc(i, []string{}, &resp)
line, err := resp.ReadString('\n')
assert.Nil(t, err)
assert.True(t, strings.Contains(line, "\"pid\":99"))
assert.True(t, strings.Contains(line, "\"name\":\"foo\""))
assert.True(t, strings.Contains(line, "\"memory\":{\"rss\":-1}"))
}
func TestStatus(t *testing.T) {
t.Parallel()
i, err := New("_", "")
i.Services = []Checkable{
&Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(99, services.Up), nil},
}
var resp bytes.Buffer
assert.Nil(t, err)
proc := CommandHandlers["status"]
proc(i, []string{}, &resp)
line, err := resp.ReadString('\n')
assert.Nil(t, err)
idxs := regexp.MustCompile(fmt.Sprintf("\\AInspeqtor %s, uptime: ", VERSION)).FindStringIndex(line)
assert.NotNil(t, idxs)
assert.Equal(t, 0, idxs[0])
}
func TestEventRuleFails(t *testing.T) {
t.Parallel()
act := mockAction()
svc := &Service{&Entity{"me", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(os.Getpid(), services.Up), services.MockInit()}
rule := &Rule{svc, "memory", "rss", LT, "100m", 100 * 1024 * 1024, 0, false, 2, 0, Ok, []Action{act}}
svc.rules = []*Rule{rule}
// first collection should trip but not trigger since rule requires 2 cycles
svc.Collect(false, func(_ Checkable) {})
events := svc.Verify()
assert.Equal(t, 0, len(events))
assert.Equal(t, 0, act.Size())
svc.Collect(false, func(_ Checkable) {})
events = svc.Verify()
assert.Equal(t, 1, len(events))
assert.Equal(t, 1, act.Size())
assert.Equal(t, RuleFailed, act.Latest().Type)
}
/*
Called for each service each cycle, in parallel. This
method must be thread-safe. Since this method executes
in a goroutine, errors must be handled/logged here and
not just returned.
Each cycle we need to:
1. verify service is Up and running.
2. capture process metrics
3. run rules
4. trigger any necessary actions
*/
func (svc *Service) Collect(silenced bool, completeCallback func(Checkable)) {
defer completeCallback(svc)
if svc.Manager == nil {
// Couldn't resolve it when we started up so we can't collect it.
return
}
if svc.Process.Status != services.Up {
status, err := svc.Manager.LookupService(svc.Name())
if err != nil {
util.Warn("%s", err)
} else {
svc.Transition(status, func(et EventType) {
if !silenced {
counters.Add("events", 1)
err = svc.EventHandler.Trigger(&Event{et, svc, nil})
if err != nil {
util.Warn("Error firing event: %s", err.Error())
}
}
})
}
}
if svc.Process.Status == services.Up {
merr := svc.Metrics().Collect(svc.Process.Pid)
if merr != nil {
err := syscall.Kill(svc.Process.Pid, syscall.Signal(0))
if err != nil {
// Process disappeared in the last cycle, mark it as Down.
util.Info("Service %s with process %d does not exist: %s", svc.Name(), svc.Process.Pid, err)
svc.Transition(services.WithStatus(0, services.Down), func(et EventType) {
if !silenced {
counters.Add("events", 1)
err = svc.EventHandler.Trigger(&Event{et, svc, nil})
if err != nil {
util.Warn("Error firing event: %s", err.Error())
}
}
})
// Immediately try to find the replacement PID so we don't have
// to wait for another cycle to mark it as Up.
status, err := svc.Manager.LookupService(svc.Name())
if err != nil {
util.Warn("%s", err)
} else {
svc.Transition(status, func(et EventType) {
if !silenced {
counters.Add("events", 1)
err = svc.EventHandler.Trigger(&Event{et, svc, nil})
if err != nil {
util.Warn("Error firing event: %s", err.Error())
}
}
})
}
} else {
util.Warn("Error capturing metrics for process %d: %s", svc.Process.Pid, merr)
}
}
}
}
func validProcessEvent(etype EventType) *Event {
svc := &Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(100, services.Up), nil}
return &Event{etype, svc, nil}
}
func mockService(name string) *Service {
return &Service{&Entity{name, nil, nil, nil}, nil, services.WithStatus(999, services.Up), services.MockInit()}
}
func validRuleEvent(etype EventType) *Event {
svc := &Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(100, services.Up), nil}
return &Event{
etype, svc, &Rule{svc, "memory", "rss", GT, "64m", 64 * 1024 * 1024, 0, false, 1, 0, Ok, []Action{mockAction()}},
}
}