Example #1
0
func TestRulesCheck(t *testing.T) {
	t.Parallel()
	svc := Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, nil, nil}
	rule := &Rule{&svc, "memory", "rss", LT, "64m", 64 * MB, 0, false, 2, 0, Ok, nil}

	// no data in the buffer
	result := rule.Check(15)
	assert.Equal(t, Ok, rule.State)
	assert.Nil(t, result)

	// Walk thru a series of cycles to verify state transitions
	svc.metrics = metrics.NewProcessStore("/proc", 15)
	loadValue(svc.metrics, "memory", "rss", 65*MB)
	result = rule.Check(15)
	assert.Nil(t, result)
	assert.Equal(t, float64(65*MB), rule.CurrentValue)
	assert.Equal(t, Ok, rule.State)

	loadValue(svc.metrics, "memory", "rss", 63*MB)
	result = rule.Check(15)
	assert.Nil(t, result)
	assert.Equal(t, 1, rule.TrippedCount)
	assert.Equal(t, float64(63*MB), rule.CurrentValue)
	assert.Equal(t, Ok, rule.State)

	loadValue(svc.metrics, "memory", "rss", 62*MB)
	result = rule.Check(15)
	assert.NotNil(t, result)
	assert.Equal(t, result.Type, RuleFailed)
	assert.Equal(t, 2, rule.TrippedCount)
	assert.Equal(t, float64(62*MB), rule.CurrentValue)
	assert.Equal(t, Triggered, rule.State)

	loadValue(svc.metrics, "memory", "rss", 62*MB)
	result = rule.Check(15)
	assert.Nil(t, result)
	assert.Equal(t, 3, rule.TrippedCount)
	assert.Equal(t, float64(62*MB), rule.CurrentValue)
	assert.Equal(t, Triggered, rule.State)

	loadValue(svc.metrics, "memory", "rss", 65*MB)
	result = rule.Check(15)
	assert.Nil(t, result)
	assert.Equal(t, float64(65*MB), rule.CurrentValue)
	assert.Equal(t, Recovered, rule.State)

	loadValue(svc.metrics, "memory", "rss", 66*MB)
	result = rule.Check(15)
	assert.NotNil(t, result)
	assert.Equal(t, result.Type, RuleRecovered)
	assert.Equal(t, float64(66*MB), rule.CurrentValue)
	assert.Equal(t, Ok, rule.State)
}
Example #2
0
func TestEventRuleRecovers(t *testing.T) {
	t.Parallel()

	act := mockAction()

	svc := &Service{&Entity{"me", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(os.Getpid(), services.Up), services.MockInit()}
	rule := &Rule{svc, "memory", "rss", LT, "100m", 100 * 1024 * 1024, 0, false, 1, 0, Ok, []Action{act}}
	svc.rules = []*Rule{rule}

	svc.Collect(false, func(_ Checkable) {})
	events := svc.Verify()
	assert.Equal(t, 1, len(events))
	assert.Equal(t, 1, act.Size())
	assert.Equal(t, RuleFailed, act.Latest().Type)

	// recovery takes 2 cycles so we don't flap unnecessarily
	rule.Threshold = 1
	svc.Collect(false, func(_ Checkable) {})
	events = svc.Verify()
	assert.Equal(t, 0, len(events))

	svc.Collect(false, func(_ Checkable) {})
	events = svc.Verify()
	assert.Equal(t, 1, len(events))
	assert.Equal(t, 2, act.Size())
	assert.Equal(t, RuleRecovered, act.Latest().Type)
}
Example #3
0
func TestEventProcessAppearsDuringDeploy(t *testing.T) {
	t.Parallel()

	init := services.MockInit()
	init.CurrentStatus = services.WithStatus(os.Getpid(), services.Up)
	act := mockAction()

	assert.Equal(t, 0, act.Size())
	svc := &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Down), init}
	svc.Collect(true, func(_ Checkable) {})
	assert.Equal(t, services.Up, svc.Process.Status)
	assert.Equal(t, os.Getpid(), svc.Process.Pid)
	assert.Equal(t, 0, act.Size())
	assert.Nil(t, act.Latest())
}
Example #4
0
func TestEventProcessDisappears(t *testing.T) {
	t.Parallel()

	init := services.MockInit()
	init.CurrentStatus = services.WithStatus(0, services.Down)
	act := mockAction()

	assert.Equal(t, 0, act.Size())
	svc := &Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(findDownPid(), services.Up), init}
	svc.Collect(false, func(_ Checkable) {})
	assert.Equal(t, services.Down, svc.Process.Status)
	assert.Equal(t, 0, svc.Process.Pid)
	assert.Equal(t, 1, act.Size())
	assert.Equal(t, ProcessDoesNotExist, act.Latest().Type)
}
Example #5
0
func TestEventProcessExistsAtStartup(t *testing.T) {
	t.Parallel()

	init := services.MockInit()
	init.CurrentStatus = services.WithStatus(100, services.Up)

	act := mockAction()

	assert.Equal(t, 0, act.Size())
	svc := &Service{&Entity{"exists", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Unknown), init}
	svc.Resolve([]services.InitSystem{init})
	assert.Equal(t, services.Up, svc.Process.Status)
	assert.Equal(t, 100, svc.Process.Pid)
	assert.Equal(t, 0, act.Size())
}
Example #6
0
func convertService(global *ConfigFile, inqsvc *ast.ProcessCheck) (*Service, error) {
	rules := make([]*Rule, len(inqsvc.Rules))
	storage := metrics.NewProcessStore("/proc", global.CycleTime)

	svc := &Service{&Entity{inqsvc.Name, nil, storage, inqsvc.Parameters}, nil, services.NewStatus(), nil}

	action, err := BuildAction(global, svc, &ast.SimpleAction{ActionName: "alert"})
	if err != nil {
		return nil, err
	}
	svc.EventHandler = action

	for idx, rule := range inqsvc.Rules {
		rule, err := convertRule(global, svc, rule)
		if err != nil {
			return nil, err
		}
		util.DebugDebug("Rule: %+v", *rule)
		rules[idx] = rule
	}
	svc.rules = rules

	for _, r := range rules {
		_, err := storage.AddSource(r.MetricFamily, svc.Parameters())
		if err != nil {
			return nil, err
		}

		err = storage.Watch(r.MetricFamily, r.MetricName)
		if err != nil {
			return nil, err
		}
		util.Debug("Watching %s:%s", r.MetricFamily, r.MetricName)
	}

	if len(inqsvc.Exposed) > 0 {
		err := BuildExpose(global, svc, inqsvc.Exposed, inqsvc.Parameters)
		if err != nil {
			return nil, err
		}
	}

	err = storage.Prepare()
	if err != nil {
		return nil, err
	}
	return svc, nil
}
Example #7
0
func TestEventProcessDneAtStartup(t *testing.T) {
	t.Parallel()

	init := services.MockInit()
	init.CurrentStatus = services.WithStatus(0, services.Down)

	act := mockAction()

	assert.Equal(t, 0, act.Size())
	svc := &Service{&Entity{"dne", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(0, services.Unknown), nil}
	svc.Resolve([]services.InitSystem{init})
	assert.Equal(t, services.Down, svc.Process.Status)
	assert.Equal(t, 0, svc.Process.Pid)
	assert.Equal(t, 1, act.Size())
	assert.Equal(t, ProcessDoesNotExist, act.Latest().Type)
}
Example #8
0
func TestExport(t *testing.T) {
	t.Parallel()
	i, err := New("_", "")
	i.Services = []Checkable{
		&Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(99, services.Up), nil},
	}

	var resp bytes.Buffer

	assert.Nil(t, err)
	proc := CommandHandlers["export"]
	proc(i, []string{}, &resp)

	line, err := resp.ReadString('\n')
	assert.Nil(t, err)
	assert.True(t, strings.Contains(line, "\"pid\":99"))
	assert.True(t, strings.Contains(line, "\"name\":\"foo\""))
	assert.True(t, strings.Contains(line, "\"memory\":{\"rss\":-1}"))
}
Example #9
0
func TestStatus(t *testing.T) {
	t.Parallel()
	i, err := New("_", "")
	i.Services = []Checkable{
		&Service{&Entity{"foo", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(99, services.Up), nil},
	}

	var resp bytes.Buffer

	assert.Nil(t, err)
	proc := CommandHandlers["status"]
	proc(i, []string{}, &resp)

	line, err := resp.ReadString('\n')
	assert.Nil(t, err)

	idxs := regexp.MustCompile(fmt.Sprintf("\\AInspeqtor %s, uptime: ", VERSION)).FindStringIndex(line)
	assert.NotNil(t, idxs)
	assert.Equal(t, 0, idxs[0])
}
Example #10
0
func TestEventRuleFails(t *testing.T) {
	t.Parallel()

	act := mockAction()

	svc := &Service{&Entity{"me", nil, metrics.NewProcessStore("/proc", 15), nil}, act, services.WithStatus(os.Getpid(), services.Up), services.MockInit()}
	rule := &Rule{svc, "memory", "rss", LT, "100m", 100 * 1024 * 1024, 0, false, 2, 0, Ok, []Action{act}}
	svc.rules = []*Rule{rule}

	// first collection should trip but not trigger since rule requires 2 cycles
	svc.Collect(false, func(_ Checkable) {})
	events := svc.Verify()
	assert.Equal(t, 0, len(events))
	assert.Equal(t, 0, act.Size())

	svc.Collect(false, func(_ Checkable) {})
	events = svc.Verify()
	assert.Equal(t, 1, len(events))
	assert.Equal(t, 1, act.Size())
	assert.Equal(t, RuleFailed, act.Latest().Type)
}
Example #11
0
func TestPerSecRulesCheck(t *testing.T) {
	t.Parallel()

	basic := metrics.NewProcessStore("/proc", 15)

	fmt.Printf("%v\n", metrics.Sources)
	source, err := basic.AddSource("mysql", map[string]string{})
	assert.NotNil(t, source)
	assert.Nil(t, err)
	basic.Watch("mysql", "Queries")
	basic.Watch("mysql", "Queries")

	svc := Service{&Entity{"mysql", nil, basic, nil}, nil, nil, nil}
	rule := &Rule{&svc, "mysql", "Queries", GT, "1k/sec", 1024, 0, true, 2, 0, Ok, nil}

	// no data in the buffer
	result := rule.Check(15)
	assert.Equal(t, Ok, rule.State)
	assert.Nil(t, result)

	// Walk thru a series of cycles to verify state transitions
	loadValue(basic, "mysql", "Queries", 1000)
	result = rule.Check(15)
	assert.Nil(t, result)
	assert.Equal(t, 0, rule.CurrentValue)
	assert.Equal(t, Ok, rule.State)

	loadValue(basic, "mysql", "Queries", 4000)
	result = rule.Check(15)
	assert.Nil(t, result)
	assert.Equal(t, 3000, rule.CurrentValue)
	assert.Equal(t, Ok, rule.State)

	loadValue(basic, "mysql", "Queries", 20000)
	result = rule.Check(15)
	assert.Nil(t, result)
	assert.Equal(t, 1, rule.TrippedCount)
	assert.Equal(t, 16000, rule.CurrentValue)
	assert.Equal(t, Ok, rule.State)
}
Example #12
0
func MockCheckable(name string) Checkable {
	return &mockCheckable{name, metrics.NewProcessStore("/", 15)}
}
Example #13
0
func NewService(name string) *Service {
	return &Service{&Entity{name, nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.NewStatus(), nil}
}
Example #14
0
func validProcessEvent(etype EventType) *Event {
	svc := &Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(100, services.Up), nil}
	return &Event{etype, svc, nil}
}
Example #15
0
func validRuleEvent(etype EventType) *Event {
	svc := &Service{&Entity{"mysql", nil, metrics.NewProcessStore("/proc", 15), nil}, nil, services.WithStatus(100, services.Up), nil}
	return &Event{
		etype, svc, &Rule{svc, "memory", "rss", GT, "64m", 64 * 1024 * 1024, 0, false, 1, 0, Ok, []Action{mockAction()}},
	}
}