func TestRulesCheck(t *testing.T) { t.Parallel() svc := Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, nil, nil} rule := &Rule{&svc, "memory", "rss", LT, "64m", 64 * MB, 0, false, 2, 0, Ok, nil} // no data in the buffer result := rule.Check(15) assert.Equal(t, Ok, rule.State) assert.Nil(t, result) // Walk thru a series of cycles to verify state transitions svc.metrics = metrics.NewProcessStore("/proc", 15) loadValue(svc.metrics, "memory", "rss", 65*MB) result = rule.Check(15) assert.Nil(t, result) assert.Equal(t, float64(65*MB), rule.CurrentValue) assert.Equal(t, Ok, rule.State) loadValue(svc.metrics, "memory", "rss", 63*MB) result = rule.Check(15) assert.Nil(t, result) assert.Equal(t, 1, rule.TrippedCount) assert.Equal(t, float64(63*MB), rule.CurrentValue) assert.Equal(t, Ok, rule.State) loadValue(svc.metrics, "memory", "rss", 62*MB) result = rule.Check(15) assert.NotNil(t, result) assert.Equal(t, result.Type, RuleFailed) assert.Equal(t, 2, rule.TrippedCount) assert.Equal(t, float64(62*MB), rule.CurrentValue) assert.Equal(t, Triggered, rule.State) loadValue(svc.metrics, "memory", "rss", 62*MB) result = rule.Check(15) assert.Nil(t, result) assert.Equal(t, 3, rule.TrippedCount) assert.Equal(t, float64(62*MB), rule.CurrentValue) assert.Equal(t, Triggered, rule.State) loadValue(svc.metrics, "memory", "rss", 65*MB) result = rule.Check(15) assert.Nil(t, result) assert.Equal(t, float64(65*MB), rule.CurrentValue) assert.Equal(t, Recovered, rule.State) loadValue(svc.metrics, "memory", "rss", 66*MB) result = rule.Check(15) assert.NotNil(t, result) assert.Equal(t, result.Type, RuleRecovered) assert.Equal(t, float64(66*MB), rule.CurrentValue) assert.Equal(t, Ok, rule.State) }
func TestEventRuleRecovers(t *testing.T) { t.Parallel() act := mockAction() svc := &Service{&Entity{"me", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(os.Getpid(), services.Up), services.MockInit()} rule := &Rule{svc, "memory", "rss", LT, "100m", 100 * 1024 * 1024, 0, false, 1, 0, Ok, []Action{act}} svc.rules = []*Rule{rule} svc.Collect(false, func(_ Checkable) {}) events := svc.Verify() assert.Equal(t, 1, len(events)) assert.Equal(t, 1, act.Size()) assert.Equal(t, RuleFailed, act.Latest().Type) // recovery takes 2 cycles so we don't flap unnecessarily rule.Threshold = 1 svc.Collect(false, func(_ Checkable) {}) events = svc.Verify() assert.Equal(t, 0, len(events)) svc.Collect(false, func(_ Checkable) {}) events = svc.Verify() assert.Equal(t, 1, len(events)) assert.Equal(t, 2, act.Size()) assert.Equal(t, RuleRecovered, act.Latest().Type) }
func TestEventProcessAppearsDuringDeploy(t *testing.T) { t.Parallel() init := services.MockInit() init.CurrentStatus = services.WithStatus(os.Getpid(), services.Up) act := mockAction() assert.Equal(t, 0, act.Size()) svc := &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Down), init} svc.Collect(true, func(_ Checkable) {}) assert.Equal(t, services.Up, svc.Process.Status) assert.Equal(t, os.Getpid(), svc.Process.Pid) assert.Equal(t, 0, act.Size()) assert.Nil(t, act.Latest()) }
func TestEventProcessDisappears(t *testing.T) { t.Parallel() init := services.MockInit() init.CurrentStatus = services.WithStatus(0, services.Down) act := mockAction() assert.Equal(t, 0, act.Size()) svc := &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(findDownPid(), services.Up), init} svc.Collect(false, func(_ Checkable) {}) assert.Equal(t, services.Down, svc.Process.Status) assert.Equal(t, 0, svc.Process.Pid) assert.Equal(t, 1, act.Size()) assert.Equal(t, ProcessDoesNotExist, act.Latest().Type) }
func TestEventProcessExistsAtStartup(t *testing.T) { t.Parallel() init := services.MockInit() init.CurrentStatus = services.WithStatus(100, services.Up) act := mockAction() assert.Equal(t, 0, act.Size()) svc := &Service{&Entity{"exists", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Unknown), init} svc.Resolve([]services.InitSystem{init}) assert.Equal(t, services.Up, svc.Process.Status) assert.Equal(t, 100, svc.Process.Pid) assert.Equal(t, 0, act.Size()) }
func convertService(global *ConfigFile, inqsvc *ast.ProcessCheck) (*Service, error) { rules := make([]*Rule, len(inqsvc.Rules)) storage := metrics.NewProcessStore("/proc", global.CycleTime) svc := &Service{&Entity{inqsvc.Name, nil, storage, inqsvc.Parameters}, nil, services.NewStatus(), nil} action, err := BuildAction(global, svc, &ast.SimpleAction{ActionName: "alert"}) if err != nil { return nil, err } svc.EventHandler = action for idx, rule := range inqsvc.Rules { rule, err := convertRule(global, svc, rule) if err != nil { return nil, err } util.DebugDebug("Rule: %+v", *rule) rules[idx] = rule } svc.rules = rules for _, r := range rules { _, err := storage.AddSource(r.MetricFamily, svc.Parameters()) if err != nil { return nil, err } err = storage.Watch(r.MetricFamily, r.MetricName) if err != nil { return nil, err } util.Debug("Watching %s:%s", r.MetricFamily, r.MetricName) } if len(inqsvc.Exposed) > 0 { err := BuildExpose(global, svc, inqsvc.Exposed, inqsvc.Parameters) if err != nil { return nil, err } } err = storage.Prepare() if err != nil { return nil, err } return svc, nil }
func TestEventProcessDneAtStartup(t *testing.T) { t.Parallel() init := services.MockInit() init.CurrentStatus = services.WithStatus(0, services.Down) act := mockAction() assert.Equal(t, 0, act.Size()) svc := &Service{&Entity{"dne", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Unknown), nil} svc.Resolve([]services.InitSystem{init}) assert.Equal(t, services.Down, svc.Process.Status) assert.Equal(t, 0, svc.Process.Pid) assert.Equal(t, 1, act.Size()) assert.Equal(t, ProcessDoesNotExist, act.Latest().Type) }
func TestExport(t *testing.T) { t.Parallel() i, err := New("_", "") i.Services = []Checkable{ &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(99, services.Up), nil}, } var resp bytes.Buffer assert.Nil(t, err) proc := CommandHandlers["export"] proc(i, []string{}, &resp) line, err := resp.ReadString('\n') assert.Nil(t, err) assert.True(t, strings.Contains(line, "\"pid\":99")) assert.True(t, strings.Contains(line, "\"name\":\"foo\"")) assert.True(t, strings.Contains(line, "\"memory\":{\"rss\":-1}")) }
func TestStatus(t *testing.T) { t.Parallel() i, err := New("_", "") i.Services = []Checkable{ &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(99, services.Up), nil}, } var resp bytes.Buffer assert.Nil(t, err) proc := CommandHandlers["status"] proc(i, []string{}, &resp) line, err := resp.ReadString('\n') assert.Nil(t, err) idxs := regexp.MustCompile(fmt.Sprintf("\\AInspeqtor %s, uptime: ", VERSION)).FindStringIndex(line) assert.NotNil(t, idxs) assert.Equal(t, 0, idxs[0]) }
func TestEventRuleFails(t *testing.T) { t.Parallel() act := mockAction() svc := &Service{&Entity{"me", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(os.Getpid(), services.Up), services.MockInit()} rule := &Rule{svc, "memory", "rss", LT, "100m", 100 * 1024 * 1024, 0, false, 2, 0, Ok, []Action{act}} svc.rules = []*Rule{rule} // first collection should trip but not trigger since rule requires 2 cycles svc.Collect(false, func(_ Checkable) {}) events := svc.Verify() assert.Equal(t, 0, len(events)) assert.Equal(t, 0, act.Size()) svc.Collect(false, func(_ Checkable) {}) events = svc.Verify() assert.Equal(t, 1, len(events)) assert.Equal(t, 1, act.Size()) assert.Equal(t, RuleFailed, act.Latest().Type) }
func TestPerSecRulesCheck(t *testing.T) { t.Parallel() basic := metrics.NewProcessStore("/proc", 15) fmt.Printf("%v\n", metrics.Sources) source, err := basic.AddSource("mysql", map[string]string{}) assert.NotNil(t, source) assert.Nil(t, err) basic.Watch("mysql", "Queries") basic.Watch("mysql", "Queries") svc := Service{&Entity{"mysql", nil, basic, nil}, nil, nil, nil} rule := &Rule{&svc, "mysql", "Queries", GT, "1k/sec", 1024, 0, true, 2, 0, Ok, nil} // no data in the buffer result := rule.Check(15) assert.Equal(t, Ok, rule.State) assert.Nil(t, result) // Walk thru a series of cycles to verify state transitions loadValue(basic, "mysql", "Queries", 1000) result = rule.Check(15) assert.Nil(t, result) assert.Equal(t, 0, rule.CurrentValue) assert.Equal(t, Ok, rule.State) loadValue(basic, "mysql", "Queries", 4000) result = rule.Check(15) assert.Nil(t, result) assert.Equal(t, 3000, rule.CurrentValue) assert.Equal(t, Ok, rule.State) loadValue(basic, "mysql", "Queries", 20000) result = rule.Check(15) assert.Nil(t, result) assert.Equal(t, 1, rule.TrippedCount) assert.Equal(t, 16000, rule.CurrentValue) assert.Equal(t, Ok, rule.State) }
func MockCheckable(name string) Checkable { return &mockCheckable{name, metrics.NewProcessStore("/", 15)} }
func NewService(name string) *Service { return &Service{&Entity{name, nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.NewStatus(), nil} }
func validProcessEvent(etype EventType) *Event { svc := &Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(100, services.Up), nil} return &Event{etype, svc, nil} }
func validRuleEvent(etype EventType) *Event { svc := &Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(100, services.Up), nil} return &Event{ etype, svc, &Rule{svc, "memory", "rss", GT, "64m", 64 * 1024 * 1024, 0, false, 1, 0, Ok, []Action{mockAction()}}, } }