// TestCheckNotifyUnknownDefault tests the default unknownTemplate. func TestCheckNotifyUnknownDefault(t *testing.T) { defer setup()() nc := make(chan string, 1) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { b, _ := ioutil.ReadAll(r.Body) nc <- string(b) })) defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatal(err) } c, err := rule.NewConf("", conf.EnabledBackends{}, fmt.Sprintf(` template t { subject = template } notification n { post = http://%s/ } alert a { template = t critNotification = n crit = 1 } `, u.Host)) if err != nil { t.Fatal(err) } s, err := initSched(&conf.SystemConf{MinGroupSize: 2}, c) if err != nil { t.Fatal(err) } r := &RunHistory{ Events: map[models.AlertKey]*models.Event{ models.NewAlertKey("a", opentsdb.TagSet{"h": "x"}): {Status: models.StUnknown}, models.NewAlertKey("a", opentsdb.TagSet{"h": "y"}): {Status: models.StUnknown}, }, } s.RunHistory(r) s.CheckNotifications() s.sendUnknownNotifications() gotExpected := false Loop: for { select { case r := <-nc: if r == "a: 2 unknown alerts" { gotExpected = true } else { t.Fatalf("unexpected: %v", r) } // TODO: remove this silly timeout-based test case <-time.After(time.Second): break Loop } } if !gotExpected { t.Errorf("didn't get expected result") } }
func TestIncidentIds(t *testing.T) { defer setup()() c, err := rule.NewConf("", conf.EnabledBackends{}, ` alert a { crit = 1 } `) if err != nil { t.Fatal(err) } s, _ := initSched(&conf.SystemConf{}, c) ak := models.NewAlertKey("a", nil) r := &RunHistory{ Events: map[models.AlertKey]*models.Event{ ak: {Status: models.StWarning}, }, } expect := func(id int64) { incident, err := s.DataAccess.State().GetLatestIncident(ak) if err != nil { t.Fatal(err) } if incident.Id != id { t.Fatalf("Expeted incident id %d. Got %d.", id, incident.Id) } } s.RunHistory(r) expect(1) r.Events[ak].Status = models.StNormal s.RunHistory(r) expect(1) r.Events[ak].Status = models.StWarning s.RunHistory(r) expect(1) r.Events[ak].Status = models.StNormal s.RunHistory(r) err = s.ActionByAlertKey("", "", models.ActionClose, ak) if err != nil { t.Fatal(err) } r.Events[ak].Status = models.StWarning s.RunHistory(r) expect(2) }
func TestIncidentIds(t *testing.T) { c, err := conf.New("", ` alert a { crit = 1 } `) if err != nil { t.Fatal(err) } s, _ := initSched(c) ak := models.NewAlertKey("a", nil) r := &RunHistory{ Events: map[models.AlertKey]*Event{ ak: {Status: StWarning}, }, } expect := func(id uint64) { if s.status[ak].Last().IncidentId != id { t.Fatalf("Expeted incident id %d. Got %d.", id, s.status[ak].Last().IncidentId) } } s.RunHistory(r) expect(1) r.Events[ak].Status = StNormal r.Events[ak].IncidentId = 0 s.RunHistory(r) expect(1) r.Events[ak].Status = StWarning r.Events[ak].IncidentId = 0 s.RunHistory(r) expect(1) r.Events[ak].Status = StNormal r.Events[ak].IncidentId = 0 s.RunHistory(r) err = s.Action("", "", ActionClose, ak) if err != nil { t.Fatal(err) } r.Events[ak].Status = StWarning r.Events[ak].IncidentId = 0 s.RunHistory(r) expect(2) }
// TestCheckCritUnknownEmpty checks that if an alert goes normal -> crit -> // unknown, it's body and subject are empty. This is because we should not // keep around the crit template renders if we are unknown. func TestCheckCritUnknownEmpty(t *testing.T) { c, err := conf.New("", ` template t { subject = 1 body = 2 } alert a { crit = 1 template = t } `) if err != nil { t.Fatal(err) } s, _ := initSched(c) ak := models.NewAlertKey("a", nil) r := &RunHistory{ Events: map[models.AlertKey]*Event{ ak: {Status: StNormal}, }, } verify := func(empty bool) { st := s.GetStatus(ak) if empty { if st.Body != "" || st.Subject != "" { t.Fatalf("expected empty body and subject") } } else { if st.Body != "<html><head></head><body>2</body></html>" || st.Subject != "1" { t.Fatalf("expected body and subject") } } } s.RunHistory(r) verify(true) r.Events[ak].Status = StCritical s.RunHistory(r) verify(false) r.Events[ak].Status = StUnknown s.RunHistory(r) verify(true) r.Events[ak].Status = StNormal s.RunHistory(r) verify(true) }
func readDps(r io.Reader, data map[models.AlertKey]int) { gr, err := gzip.NewReader(r) if err != nil { fatal(err) } jr := json.NewDecoder(gr) mdp := []*opentsdb.DataPoint{} err = jr.Decode(&mdp) if err != nil { fatal(err) } for _, dp := range mdp { ak := models.NewAlertKey(dp.Metric, dp.Tags) n, ok := data[ak] if ok { data[ak] = n + 1 } else { data[ak] = 1 } } }
func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus models.Status, ignore models.AlertKeys) (alerts models.AlertKeys, err error) { if e == nil { return } defer func() { if err == nil { return } collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1) slog.Errorln(err) }() results, err := s.executeExpr(T, rh, a, e) if err != nil { return nil, err } Loop: for _, r := range results.Results { if s.Conf.Squelched(a, r.Group) { continue } ak := models.NewAlertKey(a.Name, r.Group) for _, v := range ignore { if ak == v { continue Loop } } var n float64 n, err = valueToFloat(r.Value) if err != nil { return } event := rh.Events[ak] if event == nil { event = new(models.Event) rh.Events[ak] = event } result := &models.Result{ Computations: r.Computations, Value: models.Float(n), Expr: e.String(), } switch checkStatus { case models.StWarning: event.Warn = result case models.StCritical: event.Crit = result } status := checkStatus if math.IsNaN(n) { status = checkStatus } else if n == 0 { status = models.StNormal } if status != models.StNormal { alerts = append(alerts, ak) } if status > rh.Events[ak].Status { event.Status = status } } return }
func (c *Conf) loadLookup(s *parse.SectionNode) { name := s.Name.Text if _, ok := c.Lookups[name]; ok { c.errorf("duplicate lookup name: %s", name) } l := Lookup{ Name: name, } l.Text = s.RawText var lookupTags opentsdb.TagSet saw := make(map[string]bool) for _, n := range s.Nodes.Nodes { c.at(n) switch n := n.(type) { case *parse.SectionNode: if n.SectionType.Text != "entry" { c.errorf("unexpected subsection type") } tags, err := opentsdb.ParseTags(n.Name.Text) if tags == nil && err != nil { c.error(err) } if _, ok := saw[tags.String()]; ok { c.errorf("duplicate entry") } saw[tags.String()] = true if len(tags) == 0 { c.errorf("lookup entries require tags") } empty := make(opentsdb.TagSet) for k := range tags { empty[k] = "" } if len(lookupTags) == 0 { lookupTags = empty for k := range empty { l.Tags = append(l.Tags, k) } } else if !lookupTags.Equal(empty) { c.errorf("lookup tags mismatch, expected %v", lookupTags) } e := Entry{ Def: n.RawText, Name: n.Name.Text, ExprEntry: &ExprEntry{ AlertKey: models.NewAlertKey("", tags), Values: make(map[string]string), }, } for _, en := range n.Nodes.Nodes { c.at(en) switch en := en.(type) { case *parse.PairNode: e.Values[en.Key.Text] = en.Val.Text default: c.errorf("unexpected node") } } l.Entries = append(l.Entries, &e) default: c.errorf("unexpected node") } } c.at(s) c.Lookups[name] = &l }
func TestCheckFlapping(t *testing.T) { defer setup()() c, err := rule.NewConf("", conf.EnabledBackends{}, ` template t { subject = 1 } notification n { print = true } alert a { warnNotification = n warn = 1 critNotification = n crit = 1 template = t } `) if err != nil { t.Fatal(err) } s, _ := initSched(&conf.SystemConf{}, c) ak := models.NewAlertKey("a", nil) r := &RunHistory{ Events: map[models.AlertKey]*models.Event{ ak: {Status: models.StWarning}, }, } hasNots := func() bool { defer func() { s.pendingNotifications = nil }() if len(s.pendingNotifications) != 1 { return false } for k, v := range s.pendingNotifications { if k.Name != "n" || len(v) != 1 || v[0].Alert != "a" { return false } return true } return false } type stateTransition struct { S models.Status ExpectNots bool } transitions := []stateTransition{ {models.StWarning, true}, {models.StNormal, false}, {models.StWarning, false}, {models.StNormal, false}, {models.StCritical, true}, {models.StWarning, false}, {models.StCritical, false}, } for i, trans := range transitions { r.Events[ak].Status = trans.S s.RunHistory(r) has := hasNots() if has && !trans.ExpectNots { t.Fatalf("unexpected notifications for transition %d.", i) } else if !has && trans.ExpectNots { t.Fatalf("expected notifications for transition %d.", i) } } r.Events[ak].Status = models.StNormal s.RunHistory(r) // Close the alert, so it should notify next time. if err := s.ActionByAlertKey("", "", models.ActionClose, ak); err != nil { t.Fatal(err) } r.Events[ak].Status = models.StWarning s.RunHistory(r) if !hasNots() { t.Fatal("expected notification") } }
func (s *State) AlertKey() models.AlertKey { return models.NewAlertKey(s.Alert, s.Group) }
func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus models.Status, ignore models.AlertKeys) (alerts models.AlertKeys, err error, cancelled bool) { if e == nil { return } defer func() { if err == nil { return } collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1) slog.Errorln(err) }() type res struct { results *expr.Results error error } // See s.CheckAlert for an explanation of execution and cancellation with this channel rc := make(chan res, 1) var results *expr.Results go func() { results, err := s.executeExpr(T, rh, a, e) rc <- res{results, err} }() select { case res := <-rc: results = res.results err = res.error case <-s.runnerContext.Done(): return nil, nil, true } if err != nil { return } Loop: for _, r := range results.Results { if s.RuleConf.Squelched(a, r.Group) { continue } ak := models.NewAlertKey(a.Name, r.Group) for _, v := range ignore { if ak == v { continue Loop } } var n float64 n, err = valueToFloat(r.Value) if err != nil { return } event := rh.Events[ak] if event == nil { event = new(models.Event) rh.Events[ak] = event } result := &models.Result{ Computations: r.Computations, Value: models.Float(n), Expr: e.String(), } switch checkStatus { case models.StWarning: event.Warn = result case models.StCritical: event.Crit = result } status := checkStatus if math.IsNaN(n) { status = checkStatus } else if n == 0 { status = models.StNormal } if status != models.StNormal { alerts = append(alerts, ak) } if status > rh.Events[ak].Status { event.Status = status } } return }
func TestCheckFlapping(t *testing.T) { defer setup()() c, err := conf.New("", ` template t { subject = 1 } notification n { print = true } alert a { warnNotification = n warn = 1 critNotification = n crit = 1 normNotification = n template = t } `) if err != nil { t.Fatal(err) } s, _ := initSched(c) ak := models.NewAlertKey("a", nil) r := &RunHistory{ Events: map[models.AlertKey]*models.Event{ ak: {Status: models.StWarning}, }, } hasNots := func() bool { defer func() { s.pendingNotifications = nil }() if len(s.pendingNotifications) != 1 { return false } for k, v := range s.pendingNotifications { if k.Name != "n" || len(v) != 1 || v[0].Alert != "a" { return false } return true } return false } type stateTransition struct { S models.Status ExpectNots bool } /** transitions := []stateTransition{ {models.StWarning, true}, {models.StNormal, false}, {models.StWarning, false}, {models.StNormal, false}, {models.StCritical, true}, {models.StWarning, false}, {models.StCritical, false}, } VICTOROPS INTEGRATION: These state Transitions have been commented out and replaced with a test case which matches the behaviour we want to integrate with victorops */ transitions := []stateTransition{ {models.StWarning, true}, {models.StNormal, true}, {models.StWarning, true}, {models.StNormal, true}, {models.StCritical, true}, {models.StWarning, false}, {models.StCritical, false}, {models.StNormal, true}, } for i, trans := range transitions { r.Events[ak].Status = trans.S s.RunHistory(r) has := hasNots() if has && !trans.ExpectNots { t.Fatalf("unexpected notifications for transition %d.", i) } else if !has && trans.ExpectNots { t.Fatalf("expected notifications for transition %d.", i) } } r.Events[ak].Status = models.StNormal s.RunHistory(r) // Close the alert, so it should notify next time. if err := s.Action("", "", models.ActionClose, ak); err != nil { t.Fatal(err) } r.Events[ak].Status = models.StWarning s.RunHistory(r) if !hasNots() { t.Fatal("expected notification") } }
func TestCheckFlapping(t *testing.T) { c, err := conf.New("", ` template t { subject = 1 } notification n { print = true } alert a { warnNotification = n warn = 1 critNotification = n crit = 1 template = t } `) if err != nil { t.Fatal(err) } s, _ := initSched(c) ak := models.NewAlertKey("a", nil) r := &RunHistory{ Events: map[models.AlertKey]*Event{ ak: {Status: StWarning}, }, } hasNots := func() bool { defer func() { s.pendingNotifications = nil }() if len(s.pendingNotifications) != 1 { return false } for k, v := range s.pendingNotifications { if k.Name != "n" || len(v) != 1 || v[0].Alert != "a" { return false } return true } return false } s.RunHistory(r) if !hasNots() { t.Fatalf("expected notification: %v", s.pendingNotifications) } r.Events[ak].Status = StNormal s.RunHistory(r) if hasNots() { t.Fatal("unexpected notification") } r.Events[ak].Status = StWarning s.RunHistory(r) if hasNots() { t.Fatal("unexpected notification") } r.Events[ak].Status = StNormal s.RunHistory(r) if hasNots() { t.Fatal("unexpected notification") } r.Events[ak].Status = StCritical s.RunHistory(r) if !hasNots() { t.Fatal("expected notification") } r.Events[ak].Status = StNormal s.RunHistory(r) if hasNots() { t.Fatal("unexpected notification") } s.RunHistory(r) // Close the alert, so it should notify next time. if err := s.Action("", "", ActionClose, ak); err != nil { t.Fatal(err) } r.Events[ak].Status = StWarning s.RunHistory(r) if !hasNots() { t.Fatal("expected notification") } }