Exemple #1
0
// TestCheckNotifyUnknownDefault tests the default unknownTemplate.
func TestCheckNotifyUnknownDefault(t *testing.T) {
	defer setup()()
	nc := make(chan string, 1)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		b, _ := ioutil.ReadAll(r.Body)
		nc <- string(b)
	}))
	defer ts.Close()
	u, err := url.Parse(ts.URL)
	if err != nil {
		t.Fatal(err)
	}
	c, err := rule.NewConf("", conf.EnabledBackends{}, fmt.Sprintf(`
		template t {
			subject = template
		}
		notification n {
			post = http://%s/
		}
		alert a {
			template = t
			critNotification = n
			crit = 1
		}
	`, u.Host))
	if err != nil {
		t.Fatal(err)
	}
	s, err := initSched(&conf.SystemConf{MinGroupSize: 2}, c)
	if err != nil {
		t.Fatal(err)
	}
	r := &RunHistory{
		Events: map[models.AlertKey]*models.Event{
			models.NewAlertKey("a", opentsdb.TagSet{"h": "x"}): {Status: models.StUnknown},
			models.NewAlertKey("a", opentsdb.TagSet{"h": "y"}): {Status: models.StUnknown},
		},
	}
	s.RunHistory(r)
	s.CheckNotifications()
	s.sendUnknownNotifications()
	gotExpected := false
Loop:
	for {
		select {
		case r := <-nc:
			if r == "a: 2 unknown alerts" {
				gotExpected = true
			} else {
				t.Fatalf("unexpected: %v", r)
			}
		// TODO: remove this silly timeout-based test
		case <-time.After(time.Second):
			break Loop
		}
	}
	if !gotExpected {
		t.Errorf("didn't get expected result")
	}
}
Exemple #2
0
func TestIncidentIds(t *testing.T) {
	defer setup()()
	c, err := rule.NewConf("", conf.EnabledBackends{}, `
		alert a {
			crit = 1
		}
	`)
	if err != nil {
		t.Fatal(err)
	}
	s, _ := initSched(&conf.SystemConf{}, c)
	ak := models.NewAlertKey("a", nil)
	r := &RunHistory{
		Events: map[models.AlertKey]*models.Event{
			ak: {Status: models.StWarning},
		},
	}
	expect := func(id int64) {
		incident, err := s.DataAccess.State().GetLatestIncident(ak)
		if err != nil {
			t.Fatal(err)
		}
		if incident.Id != id {
			t.Fatalf("Expeted incident id %d. Got %d.", id, incident.Id)
		}
	}
	s.RunHistory(r)
	expect(1)

	r.Events[ak].Status = models.StNormal
	s.RunHistory(r)
	expect(1)

	r.Events[ak].Status = models.StWarning
	s.RunHistory(r)
	expect(1)

	r.Events[ak].Status = models.StNormal
	s.RunHistory(r)
	err = s.ActionByAlertKey("", "", models.ActionClose, ak)
	if err != nil {
		t.Fatal(err)
	}
	r.Events[ak].Status = models.StWarning
	s.RunHistory(r)
	expect(2)
}
Exemple #3
0
func TestIncidentIds(t *testing.T) {
	c, err := conf.New("", `
		alert a {
			crit = 1
		}
	`)
	if err != nil {
		t.Fatal(err)
	}
	s, _ := initSched(c)
	ak := models.NewAlertKey("a", nil)
	r := &RunHistory{
		Events: map[models.AlertKey]*Event{
			ak: {Status: StWarning},
		},
	}
	expect := func(id uint64) {
		if s.status[ak].Last().IncidentId != id {
			t.Fatalf("Expeted incident id %d. Got %d.", id, s.status[ak].Last().IncidentId)
		}
	}
	s.RunHistory(r)
	expect(1)

	r.Events[ak].Status = StNormal
	r.Events[ak].IncidentId = 0
	s.RunHistory(r)
	expect(1)

	r.Events[ak].Status = StWarning
	r.Events[ak].IncidentId = 0
	s.RunHistory(r)
	expect(1)

	r.Events[ak].Status = StNormal
	r.Events[ak].IncidentId = 0
	s.RunHistory(r)
	err = s.Action("", "", ActionClose, ak)
	if err != nil {
		t.Fatal(err)
	}
	r.Events[ak].Status = StWarning
	r.Events[ak].IncidentId = 0
	s.RunHistory(r)
	expect(2)
}
Exemple #4
0
// TestCheckCritUnknownEmpty checks that if an alert goes normal -> crit ->
// unknown, it's body and subject are empty. This is because we should not
// keep around the crit template renders if we are unknown.
func TestCheckCritUnknownEmpty(t *testing.T) {
	c, err := conf.New("", `
		template t {
			subject = 1
			body = 2
		}
		alert a {
			crit = 1
			template = t
		}
	`)
	if err != nil {
		t.Fatal(err)
	}
	s, _ := initSched(c)
	ak := models.NewAlertKey("a", nil)
	r := &RunHistory{
		Events: map[models.AlertKey]*Event{
			ak: {Status: StNormal},
		},
	}
	verify := func(empty bool) {
		st := s.GetStatus(ak)
		if empty {
			if st.Body != "" || st.Subject != "" {
				t.Fatalf("expected empty body and subject")
			}
		} else {
			if st.Body != "<html><head></head><body>2</body></html>" || st.Subject != "1" {
				t.Fatalf("expected body and subject")
			}
		}
	}
	s.RunHistory(r)
	verify(true)
	r.Events[ak].Status = StCritical
	s.RunHistory(r)
	verify(false)
	r.Events[ak].Status = StUnknown
	s.RunHistory(r)
	verify(true)
	r.Events[ak].Status = StNormal
	s.RunHistory(r)
	verify(true)
}
Exemple #5
0
func readDps(r io.Reader, data map[models.AlertKey]int) {
	gr, err := gzip.NewReader(r)
	if err != nil {
		fatal(err)
	}
	jr := json.NewDecoder(gr)
	mdp := []*opentsdb.DataPoint{}
	err = jr.Decode(&mdp)
	if err != nil {
		fatal(err)
	}
	for _, dp := range mdp {
		ak := models.NewAlertKey(dp.Metric, dp.Tags)
		n, ok := data[ak]
		if ok {
			data[ak] = n + 1
		} else {
			data[ak] = 1
		}
	}
}
Exemple #6
0
func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus models.Status, ignore models.AlertKeys) (alerts models.AlertKeys, err error) {
	if e == nil {
		return
	}
	defer func() {
		if err == nil {
			return
		}
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		slog.Errorln(err)
	}()
	results, err := s.executeExpr(T, rh, a, e)
	if err != nil {
		return nil, err
	}
Loop:
	for _, r := range results.Results {
		if s.Conf.Squelched(a, r.Group) {
			continue
		}
		ak := models.NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		var n float64
		n, err = valueToFloat(r.Value)
		if err != nil {
			return
		}
		event := rh.Events[ak]
		if event == nil {
			event = new(models.Event)
			rh.Events[ak] = event
		}
		result := &models.Result{
			Computations: r.Computations,
			Value:        models.Float(n),
			Expr:         e.String(),
		}
		switch checkStatus {
		case models.StWarning:
			event.Warn = result
		case models.StCritical:
			event.Crit = result
		}
		status := checkStatus
		if math.IsNaN(n) {
			status = checkStatus
		} else if n == 0 {
			status = models.StNormal
		}
		if status != models.StNormal {
			alerts = append(alerts, ak)
		}
		if status > rh.Events[ak].Status {
			event.Status = status
		}
	}
	return
}
Exemple #7
0
func (c *Conf) loadLookup(s *parse.SectionNode) {
	name := s.Name.Text
	if _, ok := c.Lookups[name]; ok {
		c.errorf("duplicate lookup name: %s", name)
	}
	l := Lookup{
		Name: name,
	}
	l.Text = s.RawText
	var lookupTags opentsdb.TagSet
	saw := make(map[string]bool)
	for _, n := range s.Nodes.Nodes {
		c.at(n)
		switch n := n.(type) {
		case *parse.SectionNode:
			if n.SectionType.Text != "entry" {
				c.errorf("unexpected subsection type")
			}
			tags, err := opentsdb.ParseTags(n.Name.Text)
			if tags == nil && err != nil {
				c.error(err)
			}
			if _, ok := saw[tags.String()]; ok {
				c.errorf("duplicate entry")
			}
			saw[tags.String()] = true
			if len(tags) == 0 {
				c.errorf("lookup entries require tags")
			}
			empty := make(opentsdb.TagSet)
			for k := range tags {
				empty[k] = ""
			}
			if len(lookupTags) == 0 {
				lookupTags = empty
				for k := range empty {
					l.Tags = append(l.Tags, k)
				}
			} else if !lookupTags.Equal(empty) {
				c.errorf("lookup tags mismatch, expected %v", lookupTags)
			}
			e := Entry{
				Def:  n.RawText,
				Name: n.Name.Text,
				ExprEntry: &ExprEntry{
					AlertKey: models.NewAlertKey("", tags),
					Values:   make(map[string]string),
				},
			}
			for _, en := range n.Nodes.Nodes {
				c.at(en)
				switch en := en.(type) {
				case *parse.PairNode:
					e.Values[en.Key.Text] = en.Val.Text
				default:
					c.errorf("unexpected node")
				}
			}
			l.Entries = append(l.Entries, &e)
		default:
			c.errorf("unexpected node")
		}
	}
	c.at(s)
	c.Lookups[name] = &l
}
Exemple #8
0
func TestCheckFlapping(t *testing.T) {
	defer setup()()
	c, err := rule.NewConf("", conf.EnabledBackends{}, `
		template t {
			subject = 1
		}
		notification n {
			print = true
		}
		alert a {
			warnNotification = n
			warn = 1
			critNotification = n
			crit = 1
			template = t
		}
	`)
	if err != nil {
		t.Fatal(err)
	}
	s, _ := initSched(&conf.SystemConf{}, c)
	ak := models.NewAlertKey("a", nil)
	r := &RunHistory{
		Events: map[models.AlertKey]*models.Event{
			ak: {Status: models.StWarning},
		},
	}
	hasNots := func() bool {
		defer func() {
			s.pendingNotifications = nil
		}()
		if len(s.pendingNotifications) != 1 {
			return false
		}
		for k, v := range s.pendingNotifications {
			if k.Name != "n" || len(v) != 1 || v[0].Alert != "a" {
				return false
			}
			return true
		}
		return false
	}

	type stateTransition struct {
		S          models.Status
		ExpectNots bool
	}
	transitions := []stateTransition{
		{models.StWarning, true},
		{models.StNormal, false},
		{models.StWarning, false},
		{models.StNormal, false},
		{models.StCritical, true},
		{models.StWarning, false},
		{models.StCritical, false},
	}

	for i, trans := range transitions {
		r.Events[ak].Status = trans.S
		s.RunHistory(r)
		has := hasNots()
		if has && !trans.ExpectNots {
			t.Fatalf("unexpected notifications for transition %d.", i)
		} else if !has && trans.ExpectNots {
			t.Fatalf("expected notifications for transition %d.", i)
		}
	}
	r.Events[ak].Status = models.StNormal
	s.RunHistory(r)
	// Close the alert, so it should notify next time.
	if err := s.ActionByAlertKey("", "", models.ActionClose, ak); err != nil {
		t.Fatal(err)
	}
	r.Events[ak].Status = models.StWarning
	s.RunHistory(r)
	if !hasNots() {
		t.Fatal("expected notification")
	}
}
Exemple #9
0
func (s *State) AlertKey() models.AlertKey {
	return models.NewAlertKey(s.Alert, s.Group)
}
Exemple #10
0
func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus models.Status, ignore models.AlertKeys) (alerts models.AlertKeys, err error, cancelled bool) {
	if e == nil {
		return
	}
	defer func() {
		if err == nil {
			return
		}
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		slog.Errorln(err)
	}()
	type res struct {
		results *expr.Results
		error   error
	}
	// See s.CheckAlert for an explanation of execution and cancellation with this channel
	rc := make(chan res, 1)
	var results *expr.Results
	go func() {
		results, err := s.executeExpr(T, rh, a, e)
		rc <- res{results, err}
	}()
	select {
	case res := <-rc:
		results = res.results
		err = res.error
	case <-s.runnerContext.Done():
		return nil, nil, true
	}
	if err != nil {
		return
	}
Loop:
	for _, r := range results.Results {
		if s.RuleConf.Squelched(a, r.Group) {
			continue
		}
		ak := models.NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		var n float64
		n, err = valueToFloat(r.Value)
		if err != nil {
			return
		}
		event := rh.Events[ak]
		if event == nil {
			event = new(models.Event)
			rh.Events[ak] = event
		}
		result := &models.Result{
			Computations: r.Computations,
			Value:        models.Float(n),
			Expr:         e.String(),
		}
		switch checkStatus {
		case models.StWarning:
			event.Warn = result
		case models.StCritical:
			event.Crit = result
		}
		status := checkStatus
		if math.IsNaN(n) {
			status = checkStatus
		} else if n == 0 {
			status = models.StNormal
		}
		if status != models.StNormal {
			alerts = append(alerts, ak)
		}
		if status > rh.Events[ak].Status {
			event.Status = status
		}
	}
	return
}
Exemple #11
0
func TestCheckFlapping(t *testing.T) {
	defer setup()()
	c, err := conf.New("", `
		template t {
			subject = 1
		}
		notification n {
			print = true
		}
		alert a {
			warnNotification = n
			warn = 1
			critNotification = n
			crit = 1
			normNotification = n
			template = t
		}
	`)
	if err != nil {
		t.Fatal(err)
	}
	s, _ := initSched(c)
	ak := models.NewAlertKey("a", nil)
	r := &RunHistory{
		Events: map[models.AlertKey]*models.Event{
			ak: {Status: models.StWarning},
		},
	}
	hasNots := func() bool {
		defer func() {
			s.pendingNotifications = nil
		}()
		if len(s.pendingNotifications) != 1 {
			return false
		}
		for k, v := range s.pendingNotifications {
			if k.Name != "n" || len(v) != 1 || v[0].Alert != "a" {
				return false
			}
			return true
		}
		return false
	}

	type stateTransition struct {
		S          models.Status
		ExpectNots bool
	}

	/**
	transitions := []stateTransition{
		{models.StWarning, true},
		{models.StNormal, false},
		{models.StWarning, false},
		{models.StNormal, false},
		{models.StCritical, true},
		{models.StWarning, false},
		{models.StCritical, false},
	}
	 VICTOROPS INTEGRATION: These state Transitions have been commented out and replaced with a test case which matches the behaviour we want to integrate with victorops
	*/

	transitions := []stateTransition{
		{models.StWarning, true},
		{models.StNormal, true},
		{models.StWarning, true},
		{models.StNormal, true},
		{models.StCritical, true},
		{models.StWarning, false},
		{models.StCritical, false},
		{models.StNormal, true},
	}

	for i, trans := range transitions {
		r.Events[ak].Status = trans.S
		s.RunHistory(r)
		has := hasNots()
		if has && !trans.ExpectNots {
			t.Fatalf("unexpected notifications for transition %d.", i)
		} else if !has && trans.ExpectNots {
			t.Fatalf("expected notifications for transition %d.", i)
		}
	}
	r.Events[ak].Status = models.StNormal
	s.RunHistory(r)
	// Close the alert, so it should notify next time.
	if err := s.Action("", "", models.ActionClose, ak); err != nil {
		t.Fatal(err)
	}
	r.Events[ak].Status = models.StWarning
	s.RunHistory(r)
	if !hasNots() {
		t.Fatal("expected notification")
	}
}
Exemple #12
0
func TestCheckFlapping(t *testing.T) {

	c, err := conf.New("", `
		template t {
			subject = 1
		}
		notification n {
			print = true
		}
		alert a {
			warnNotification = n
			warn = 1
			critNotification = n
			crit = 1
			template = t
		}
	`)
	if err != nil {
		t.Fatal(err)
	}
	s, _ := initSched(c)
	ak := models.NewAlertKey("a", nil)
	r := &RunHistory{
		Events: map[models.AlertKey]*Event{
			ak: {Status: StWarning},
		},
	}
	hasNots := func() bool {
		defer func() {
			s.pendingNotifications = nil
		}()
		if len(s.pendingNotifications) != 1 {
			return false
		}
		for k, v := range s.pendingNotifications {
			if k.Name != "n" || len(v) != 1 || v[0].Alert != "a" {
				return false
			}
			return true
		}
		return false
	}
	s.RunHistory(r)
	if !hasNots() {
		t.Fatalf("expected notification: %v", s.pendingNotifications)
	}
	r.Events[ak].Status = StNormal
	s.RunHistory(r)
	if hasNots() {
		t.Fatal("unexpected notification")
	}
	r.Events[ak].Status = StWarning
	s.RunHistory(r)
	if hasNots() {
		t.Fatal("unexpected notification")
	}
	r.Events[ak].Status = StNormal
	s.RunHistory(r)
	if hasNots() {
		t.Fatal("unexpected notification")
	}
	r.Events[ak].Status = StCritical
	s.RunHistory(r)
	if !hasNots() {
		t.Fatal("expected notification")
	}
	r.Events[ak].Status = StNormal
	s.RunHistory(r)
	if hasNots() {
		t.Fatal("unexpected notification")
	}
	s.RunHistory(r)
	// Close the alert, so it should notify next time.
	if err := s.Action("", "", ActionClose, ak); err != nil {
		t.Fatal(err)
	}
	r.Events[ak].Status = StWarning
	s.RunHistory(r)
	if !hasNots() {
		t.Fatal("expected notification")
	}
}