func TestRulesCheck(t *testing.T) {
t.Parallel()
svc := Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, nil, nil}
rule := &Rule{&svc, "memory", "rss", LT, "64m", 64 * MB, 0, false, 2, 0, Ok, nil}
// no data in the buffer
result := rule.Check(15)
assert.Equal(t, Ok, rule.State)
assert.Nil(t, result)
// Walk thru a series of cycles to verify state transitions
svc.metrics = metrics.NewProcessStore("/proc", 15)
loadValue(svc.metrics, "memory", "rss", 65*MB)
result = rule.Check(15)
assert.Nil(t, result)
assert.Equal(t, float64(65*MB), rule.CurrentValue)
assert.Equal(t, Ok, rule.State)
loadValue(svc.metrics, "memory", "rss", 63*MB)
result = rule.Check(15)
assert.Nil(t, result)
assert.Equal(t, 1, rule.TrippedCount)
assert.Equal(t, float64(63*MB), rule.CurrentValue)
assert.Equal(t, Ok, rule.State)
loadValue(svc.metrics, "memory", "rss", 62*MB)
result = rule.Check(15)
assert.NotNil(t, result)
assert.Equal(t, result.Type, RuleFailed)
assert.Equal(t, 2, rule.TrippedCount)
assert.Equal(t, float64(62*MB), rule.CurrentValue)
assert.Equal(t, Triggered, rule.State)
loadValue(svc.metrics, "memory", "rss", 62*MB)
result = rule.Check(15)
assert.Nil(t, result)
assert.Equal(t, 3, rule.TrippedCount)
assert.Equal(t, float64(62*MB), rule.CurrentValue)
assert.Equal(t, Triggered, rule.State)
loadValue(svc.metrics, "memory", "rss", 65*MB)
result = rule.Check(15)
assert.Nil(t, result)
assert.Equal(t, float64(65*MB), rule.CurrentValue)
assert.Equal(t, Recovered, rule.State)
loadValue(svc.metrics, "memory", "rss", 66*MB)
result = rule.Check(15)
assert.NotNil(t, result)
assert.Equal(t, result.Type, RuleRecovered)
assert.Equal(t, float64(66*MB), rule.CurrentValue)
assert.Equal(t, Ok, rule.State)
}
func TestEventRuleRecovers(t *testing.T) {
t.Parallel()
act := mockAction()
svc := &Service{&Entity{"me", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(os.Getpid(), services.Up), services.MockInit()}
rule := &Rule{svc, "memory", "rss", LT, "100m", 100 * 1024 * 1024, 0, false, 1, 0, Ok, []Action{act}}
svc.rules = []*Rule{rule}
svc.Collect(false, func(_ Checkable) {})
events := svc.Verify()
assert.Equal(t, 1, len(events))
assert.Equal(t, 1, act.Size())
assert.Equal(t, RuleFailed, act.Latest().Type)
// recovery takes 2 cycles so we don't flap unnecessarily
rule.Threshold = 1
svc.Collect(false, func(_ Checkable) {})
events = svc.Verify()
assert.Equal(t, 0, len(events))
svc.Collect(false, func(_ Checkable) {})
events = svc.Verify()
assert.Equal(t, 1, len(events))
assert.Equal(t, 2, act.Size())
assert.Equal(t, RuleRecovered, act.Latest().Type)
}
func TestEventProcessAppearsDuringDeploy(t *testing.T) {
t.Parallel()
init := services.MockInit()
init.CurrentStatus = services.WithStatus(os.Getpid(), services.Up)
act := mockAction()
assert.Equal(t, 0, act.Size())
svc := &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Down), init}
svc.Collect(true, func(_ Checkable) {})
assert.Equal(t, services.Up, svc.Process.Status)
assert.Equal(t, os.Getpid(), svc.Process.Pid)
assert.Equal(t, 0, act.Size())
assert.Nil(t, act.Latest())
}
func TestEventProcessDisappears(t *testing.T) {
t.Parallel()
init := services.MockInit()
init.CurrentStatus = services.WithStatus(0, services.Down)
act := mockAction()
assert.Equal(t, 0, act.Size())
svc := &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(findDownPid(), services.Up), init}
svc.Collect(false, func(_ Checkable) {})
assert.Equal(t, services.Down, svc.Process.Status)
assert.Equal(t, 0, svc.Process.Pid)
assert.Equal(t, 1, act.Size())
assert.Equal(t, ProcessDoesNotExist, act.Latest().Type)
}
func TestEventProcessExistsAtStartup(t *testing.T) {
t.Parallel()
init := services.MockInit()
init.CurrentStatus = services.WithStatus(100, services.Up)
act := mockAction()
assert.Equal(t, 0, act.Size())
svc := &Service{&Entity{"exists", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Unknown), init}
svc.Resolve([]services.InitSystem{init})
assert.Equal(t, services.Up, svc.Process.Status)
assert.Equal(t, 100, svc.Process.Pid)
assert.Equal(t, 0, act.Size())
}
func convertService(global *ConfigFile, inqsvc *ast.ProcessCheck) (*Service, error) {
rules := make([]*Rule, len(inqsvc.Rules))
storage := metrics.NewProcessStore("/proc", global.CycleTime)
svc := &Service{&Entity{inqsvc.Name, nil, storage, inqsvc.Parameters}, nil, services.NewStatus(), nil}
action, err := BuildAction(global, svc, &ast.SimpleAction{ActionName: "alert"})
if err != nil {
return nil, err
}
svc.EventHandler = action
for idx, rule := range inqsvc.Rules {
rule, err := convertRule(global, svc, rule)
if err != nil {
return nil, err
}
util.DebugDebug("Rule: %+v", *rule)
rules[idx] = rule
}
svc.rules = rules
for _, r := range rules {
_, err := storage.AddSource(r.MetricFamily, svc.Parameters())
if err != nil {
return nil, err
}
err = storage.Watch(r.MetricFamily, r.MetricName)
if err != nil {
return nil, err
}
util.Debug("Watching %s:%s", r.MetricFamily, r.MetricName)
}
if len(inqsvc.Exposed) > 0 {
err := BuildExpose(global, svc, inqsvc.Exposed, inqsvc.Parameters)
if err != nil {
return nil, err
}
}
err = storage.Prepare()
if err != nil {
return nil, err
}
return svc, nil
}
func TestEventProcessDneAtStartup(t *testing.T) {
t.Parallel()
init := services.MockInit()
init.CurrentStatus = services.WithStatus(0, services.Down)
act := mockAction()
assert.Equal(t, 0, act.Size())
svc := &Service{&Entity{"dne", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Unknown), nil}
svc.Resolve([]services.InitSystem{init})
assert.Equal(t, services.Down, svc.Process.Status)
assert.Equal(t, 0, svc.Process.Pid)
assert.Equal(t, 1, act.Size())
assert.Equal(t, ProcessDoesNotExist, act.Latest().Type)
}
func TestExport(t *testing.T) {
t.Parallel()
i, err := New("_", "")
i.Services = []Checkable{
&Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(99, services.Up), nil},
}
var resp bytes.Buffer
assert.Nil(t, err)
proc := CommandHandlers["export"]
proc(i, []string{}, &resp)
line, err := resp.ReadString('\n')
assert.Nil(t, err)
assert.True(t, strings.Contains(line, "\"pid\":99"))
assert.True(t, strings.Contains(line, "\"name\":\"foo\""))
assert.True(t, strings.Contains(line, "\"memory\":{\"rss\":-1}"))
}
func TestStatus(t *testing.T) {
t.Parallel()
i, err := New("_", "")
i.Services = []Checkable{
&Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(99, services.Up), nil},
}
var resp bytes.Buffer
assert.Nil(t, err)
proc := CommandHandlers["status"]
proc(i, []string{}, &resp)
line, err := resp.ReadString('\n')
assert.Nil(t, err)
idxs := regexp.MustCompile(fmt.Sprintf("\\AInspeqtor %s, uptime: ", VERSION)).FindStringIndex(line)
assert.NotNil(t, idxs)
assert.Equal(t, 0, idxs[0])
}
func TestEventRuleFails(t *testing.T) {
t.Parallel()
act := mockAction()
svc := &Service{&Entity{"me", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(os.Getpid(), services.Up), services.MockInit()}
rule := &Rule{svc, "memory", "rss", LT, "100m", 100 * 1024 * 1024, 0, false, 2, 0, Ok, []Action{act}}
svc.rules = []*Rule{rule}
// first collection should trip but not trigger since rule requires 2 cycles
svc.Collect(false, func(_ Checkable) {})
events := svc.Verify()
assert.Equal(t, 0, len(events))
assert.Equal(t, 0, act.Size())
svc.Collect(false, func(_ Checkable) {})
events = svc.Verify()
assert.Equal(t, 1, len(events))
assert.Equal(t, 1, act.Size())
assert.Equal(t, RuleFailed, act.Latest().Type)
}
func TestPerSecRulesCheck(t *testing.T) {
t.Parallel()
basic := metrics.NewProcessStore("/proc", 15)
fmt.Printf("%v\n", metrics.Sources)
source, err := basic.AddSource("mysql", map[string]string{})
assert.NotNil(t, source)
assert.Nil(t, err)
basic.Watch("mysql", "Queries")
basic.Watch("mysql", "Queries")
svc := Service{&Entity{"mysql", nil, basic, nil}, nil, nil, nil}
rule := &Rule{&svc, "mysql", "Queries", GT, "1k/sec", 1024, 0, true, 2, 0, Ok, nil}
// no data in the buffer
result := rule.Check(15)
assert.Equal(t, Ok, rule.State)
assert.Nil(t, result)
// Walk thru a series of cycles to verify state transitions
loadValue(basic, "mysql", "Queries", 1000)
result = rule.Check(15)
assert.Nil(t, result)
assert.Equal(t, 0, rule.CurrentValue)
assert.Equal(t, Ok, rule.State)
loadValue(basic, "mysql", "Queries", 4000)
result = rule.Check(15)
assert.Nil(t, result)
assert.Equal(t, 3000, rule.CurrentValue)
assert.Equal(t, Ok, rule.State)
loadValue(basic, "mysql", "Queries", 20000)
result = rule.Check(15)
assert.Nil(t, result)
assert.Equal(t, 1, rule.TrippedCount)
assert.Equal(t, 16000, rule.CurrentValue)
assert.Equal(t, Ok, rule.State)
}
func MockCheckable(name string) Checkable {
return &mockCheckable{name, metrics.NewProcessStore("/", 15)}
}
func NewService(name string) *Service {
return &Service{&Entity{name, nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.NewStatus(), nil}
}
func validProcessEvent(etype EventType) *Event {
svc := &Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(100, services.Up), nil}
return &Event{etype, svc, nil}
}
func validRuleEvent(etype EventType) *Event {
svc := &Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(100, services.Up), nil}
return &Event{
etype, svc, &Rule{svc, "memory", "rss", GT, "64m", 64 * 1024 * 1024, 0, false, 1, 0, Ok, []Action{mockAction()}},
}
}