func TestEventRuleRecovers(t *testing.T) { t.Parallel() act := mockAction() svc := &Service{&Entity{"me", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(os.Getpid(), services.Up), services.MockInit()} rule := &Rule{svc, "memory", "rss", LT, "100m", 100 * 1024 * 1024, 0, false, 1, 0, Ok, []Action{act}} svc.rules = []*Rule{rule} svc.Collect(false, func(_ Checkable) {}) events := svc.Verify() assert.Equal(t, 1, len(events)) assert.Equal(t, 1, act.Size()) assert.Equal(t, RuleFailed, act.Latest().Type) // recovery takes 2 cycles so we don't flap unnecessarily rule.Threshold = 1 svc.Collect(false, func(_ Checkable) {}) events = svc.Verify() assert.Equal(t, 0, len(events)) svc.Collect(false, func(_ Checkable) {}) events = svc.Verify() assert.Equal(t, 1, len(events)) assert.Equal(t, 2, act.Size()) assert.Equal(t, RuleRecovered, act.Latest().Type) }
func TestEventProcessAppearsDuringDeploy(t *testing.T) { t.Parallel() init := services.MockInit() init.CurrentStatus = services.WithStatus(os.Getpid(), services.Up) act := mockAction() assert.Equal(t, 0, act.Size()) svc := &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Down), init} svc.Collect(true, func(_ Checkable) {}) assert.Equal(t, services.Up, svc.Process.Status) assert.Equal(t, os.Getpid(), svc.Process.Pid) assert.Equal(t, 0, act.Size()) assert.Nil(t, act.Latest()) }
func TestEventProcessDisappears(t *testing.T) { t.Parallel() init := services.MockInit() init.CurrentStatus = services.WithStatus(0, services.Down) act := mockAction() assert.Equal(t, 0, act.Size()) svc := &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(findDownPid(), services.Up), init} svc.Collect(false, func(_ Checkable) {}) assert.Equal(t, services.Down, svc.Process.Status) assert.Equal(t, 0, svc.Process.Pid) assert.Equal(t, 1, act.Size()) assert.Equal(t, ProcessDoesNotExist, act.Latest().Type) }
func TestEventProcessExistsAtStartup(t *testing.T) { t.Parallel() init := services.MockInit() init.CurrentStatus = services.WithStatus(100, services.Up) act := mockAction() assert.Equal(t, 0, act.Size()) svc := &Service{&Entity{"exists", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Unknown), init} svc.Resolve([]services.InitSystem{init}) assert.Equal(t, services.Up, svc.Process.Status) assert.Equal(t, 100, svc.Process.Pid) assert.Equal(t, 0, act.Size()) }
func TestEventProcessDneAtStartup(t *testing.T) { t.Parallel() init := services.MockInit() init.CurrentStatus = services.WithStatus(0, services.Down) act := mockAction() assert.Equal(t, 0, act.Size()) svc := &Service{&Entity{"dne", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Unknown), nil} svc.Resolve([]services.InitSystem{init}) assert.Equal(t, services.Down, svc.Process.Status) assert.Equal(t, 0, svc.Process.Pid) assert.Equal(t, 1, act.Size()) assert.Equal(t, ProcessDoesNotExist, act.Latest().Type) }
func TestExport(t *testing.T) { t.Parallel() i, err := New("_", "") i.Services = []Checkable{ &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(99, services.Up), nil}, } var resp bytes.Buffer assert.Nil(t, err) proc := CommandHandlers["export"] proc(i, []string{}, &resp) line, err := resp.ReadString('\n') assert.Nil(t, err) assert.True(t, strings.Contains(line, "\"pid\":99")) assert.True(t, strings.Contains(line, "\"name\":\"foo\"")) assert.True(t, strings.Contains(line, "\"memory\":{\"rss\":-1}")) }
func TestStatus(t *testing.T) { t.Parallel() i, err := New("_", "") i.Services = []Checkable{ &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(99, services.Up), nil}, } var resp bytes.Buffer assert.Nil(t, err) proc := CommandHandlers["status"] proc(i, []string{}, &resp) line, err := resp.ReadString('\n') assert.Nil(t, err) idxs := regexp.MustCompile(fmt.Sprintf("\\AInspeqtor %s, uptime: ", VERSION)).FindStringIndex(line) assert.NotNil(t, idxs) assert.Equal(t, 0, idxs[0]) }
func TestEventRuleFails(t *testing.T) { t.Parallel() act := mockAction() svc := &Service{&Entity{"me", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(os.Getpid(), services.Up), services.MockInit()} rule := &Rule{svc, "memory", "rss", LT, "100m", 100 * 1024 * 1024, 0, false, 2, 0, Ok, []Action{act}} svc.rules = []*Rule{rule} // first collection should trip but not trigger since rule requires 2 cycles svc.Collect(false, func(_ Checkable) {}) events := svc.Verify() assert.Equal(t, 0, len(events)) assert.Equal(t, 0, act.Size()) svc.Collect(false, func(_ Checkable) {}) events = svc.Verify() assert.Equal(t, 1, len(events)) assert.Equal(t, 1, act.Size()) assert.Equal(t, RuleFailed, act.Latest().Type) }
/* Called for each service each cycle, in parallel. This method must be thread-safe. Since this method executes in a goroutine, errors must be handled/logged here and not just returned. Each cycle we need to: 1. verify service is Up and running. 2. capture process metrics 3. run rules 4. trigger any necessary actions */ func (svc *Service) Collect(silenced bool, completeCallback func(Checkable)) { defer completeCallback(svc) if svc.Manager == nil { // Couldn't resolve it when we started up so we can't collect it. return } if svc.Process.Status != services.Up { status, err := svc.Manager.LookupService(svc.Name()) if err != nil { util.Warn("%s", err) } else { svc.Transition(status, func(et EventType) { if !silenced { counters.Add("events", 1) err = svc.EventHandler.Trigger(&Event{et, svc, nil}) if err != nil { util.Warn("Error firing event: %s", err.Error()) } } }) } } if svc.Process.Status == services.Up { merr := svc.Metrics().Collect(svc.Process.Pid) if merr != nil { err := syscall.Kill(svc.Process.Pid, syscall.Signal(0)) if err != nil { // Process disappeared in the last cycle, mark it as Down. util.Info("Service %s with process %d does not exist: %s", svc.Name(), svc.Process.Pid, err) svc.Transition(services.WithStatus(0, services.Down), func(et EventType) { if !silenced { counters.Add("events", 1) err = svc.EventHandler.Trigger(&Event{et, svc, nil}) if err != nil { util.Warn("Error firing event: %s", err.Error()) } } }) // Immediately try to find the replacement PID so we don't have // to wait for another cycle to mark it as Up. status, err := svc.Manager.LookupService(svc.Name()) if err != nil { util.Warn("%s", err) } else { svc.Transition(status, func(et EventType) { if !silenced { counters.Add("events", 1) err = svc.EventHandler.Trigger(&Event{et, svc, nil}) if err != nil { util.Warn("Error firing event: %s", err.Error()) } } }) } } else { util.Warn("Error capturing metrics for process %d: %s", svc.Process.Pid, merr) } } } }
func validProcessEvent(etype EventType) *Event { svc := &Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(100, services.Up), nil} return &Event{etype, svc, nil} }
func mockService(name string) *Service { return &Service{&Entity{name, nil, nil, nil}, nil, services.WithStatus(999, services.Up), services.MockInit()} }
func validRuleEvent(etype EventType) *Event { svc := &Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(100, services.Up), nil} return &Event{ etype, svc, &Rule{svc, "memory", "rss", GT, "64m", 64 * 1024 * 1024, 0, false, 1, 0, Ok, []Action{mockAction()}}, } }