// TestCheckNotifyUnknownDefault tests the default unknownTemplate.
func TestCheckNotifyUnknownDefault(t *testing.T) {
defer setup()()
nc := make(chan string, 1)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, _ := ioutil.ReadAll(r.Body)
nc <- string(b)
}))
defer ts.Close()
u, err := url.Parse(ts.URL)
if err != nil {
t.Fatal(err)
}
c, err := rule.NewConf("", conf.EnabledBackends{}, fmt.Sprintf(`
template t {
subject = template
}
notification n {
post = http://%s/
}
alert a {
template = t
critNotification = n
crit = 1
}
`, u.Host))
if err != nil {
t.Fatal(err)
}
s, err := initSched(&conf.SystemConf{MinGroupSize: 2}, c)
if err != nil {
t.Fatal(err)
}
r := &RunHistory{
Events: map[models.AlertKey]*models.Event{
models.NewAlertKey("a", opentsdb.TagSet{"h": "x"}): {Status: models.StUnknown},
models.NewAlertKey("a", opentsdb.TagSet{"h": "y"}): {Status: models.StUnknown},
},
}
s.RunHistory(r)
s.CheckNotifications()
s.sendUnknownNotifications()
gotExpected := false
Loop:
for {
select {
case r := <-nc:
if r == "a: 2 unknown alerts" {
gotExpected = true
} else {
t.Fatalf("unexpected: %v", r)
}
// TODO: remove this silly timeout-based test
case <-time.After(time.Second):
break Loop
}
}
if !gotExpected {
t.Errorf("didn't get expected result")
}
}
func TestIncidentIds(t *testing.T) {
defer setup()()
c, err := rule.NewConf("", conf.EnabledBackends{}, `
alert a {
crit = 1
}
`)
if err != nil {
t.Fatal(err)
}
s, _ := initSched(&conf.SystemConf{}, c)
ak := models.NewAlertKey("a", nil)
r := &RunHistory{
Events: map[models.AlertKey]*models.Event{
ak: {Status: models.StWarning},
},
}
expect := func(id int64) {
incident, err := s.DataAccess.State().GetLatestIncident(ak)
if err != nil {
t.Fatal(err)
}
if incident.Id != id {
t.Fatalf("Expeted incident id %d. Got %d.", id, incident.Id)
}
}
s.RunHistory(r)
expect(1)
r.Events[ak].Status = models.StNormal
s.RunHistory(r)
expect(1)
r.Events[ak].Status = models.StWarning
s.RunHistory(r)
expect(1)
r.Events[ak].Status = models.StNormal
s.RunHistory(r)
err = s.ActionByAlertKey("", "", models.ActionClose, ak)
if err != nil {
t.Fatal(err)
}
r.Events[ak].Status = models.StWarning
s.RunHistory(r)
expect(2)
}
func TestIncidentIds(t *testing.T) {
c, err := conf.New("", `
alert a {
crit = 1
}
`)
if err != nil {
t.Fatal(err)
}
s, _ := initSched(c)
ak := models.NewAlertKey("a", nil)
r := &RunHistory{
Events: map[models.AlertKey]*Event{
ak: {Status: StWarning},
},
}
expect := func(id uint64) {
if s.status[ak].Last().IncidentId != id {
t.Fatalf("Expeted incident id %d. Got %d.", id, s.status[ak].Last().IncidentId)
}
}
s.RunHistory(r)
expect(1)
r.Events[ak].Status = StNormal
r.Events[ak].IncidentId = 0
s.RunHistory(r)
expect(1)
r.Events[ak].Status = StWarning
r.Events[ak].IncidentId = 0
s.RunHistory(r)
expect(1)
r.Events[ak].Status = StNormal
r.Events[ak].IncidentId = 0
s.RunHistory(r)
err = s.Action("", "", ActionClose, ak)
if err != nil {
t.Fatal(err)
}
r.Events[ak].Status = StWarning
r.Events[ak].IncidentId = 0
s.RunHistory(r)
expect(2)
}
// TestCheckCritUnknownEmpty checks that if an alert goes normal -> crit ->
// unknown, it's body and subject are empty. This is because we should not
// keep around the crit template renders if we are unknown.
func TestCheckCritUnknownEmpty(t *testing.T) {
c, err := conf.New("", `
template t {
subject = 1
body = 2
}
alert a {
crit = 1
template = t
}
`)
if err != nil {
t.Fatal(err)
}
s, _ := initSched(c)
ak := models.NewAlertKey("a", nil)
r := &RunHistory{
Events: map[models.AlertKey]*Event{
ak: {Status: StNormal},
},
}
verify := func(empty bool) {
st := s.GetStatus(ak)
if empty {
if st.Body != "" || st.Subject != "" {
t.Fatalf("expected empty body and subject")
}
} else {
if st.Body != "<html><head></head><body>2</body></html>" || st.Subject != "1" {
t.Fatalf("expected body and subject")
}
}
}
s.RunHistory(r)
verify(true)
r.Events[ak].Status = StCritical
s.RunHistory(r)
verify(false)
r.Events[ak].Status = StUnknown
s.RunHistory(r)
verify(true)
r.Events[ak].Status = StNormal
s.RunHistory(r)
verify(true)
}
func readDps(r io.Reader, data map[models.AlertKey]int) {
gr, err := gzip.NewReader(r)
if err != nil {
fatal(err)
}
jr := json.NewDecoder(gr)
mdp := []*opentsdb.DataPoint{}
err = jr.Decode(&mdp)
if err != nil {
fatal(err)
}
for _, dp := range mdp {
ak := models.NewAlertKey(dp.Metric, dp.Tags)
n, ok := data[ak]
if ok {
data[ak] = n + 1
} else {
data[ak] = 1
}
}
}
func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus models.Status, ignore models.AlertKeys) (alerts models.AlertKeys, err error) {
if e == nil {
return
}
defer func() {
if err == nil {
return
}
collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
slog.Errorln(err)
}()
results, err := s.executeExpr(T, rh, a, e)
if err != nil {
return nil, err
}
Loop:
for _, r := range results.Results {
if s.Conf.Squelched(a, r.Group) {
continue
}
ak := models.NewAlertKey(a.Name, r.Group)
for _, v := range ignore {
if ak == v {
continue Loop
}
}
var n float64
n, err = valueToFloat(r.Value)
if err != nil {
return
}
event := rh.Events[ak]
if event == nil {
event = new(models.Event)
rh.Events[ak] = event
}
result := &models.Result{
Computations: r.Computations,
Value: models.Float(n),
Expr: e.String(),
}
switch checkStatus {
case models.StWarning:
event.Warn = result
case models.StCritical:
event.Crit = result
}
status := checkStatus
if math.IsNaN(n) {
status = checkStatus
} else if n == 0 {
status = models.StNormal
}
if status != models.StNormal {
alerts = append(alerts, ak)
}
if status > rh.Events[ak].Status {
event.Status = status
}
}
return
}
func (c *Conf) loadLookup(s *parse.SectionNode) {
name := s.Name.Text
if _, ok := c.Lookups[name]; ok {
c.errorf("duplicate lookup name: %s", name)
}
l := Lookup{
Name: name,
}
l.Text = s.RawText
var lookupTags opentsdb.TagSet
saw := make(map[string]bool)
for _, n := range s.Nodes.Nodes {
c.at(n)
switch n := n.(type) {
case *parse.SectionNode:
if n.SectionType.Text != "entry" {
c.errorf("unexpected subsection type")
}
tags, err := opentsdb.ParseTags(n.Name.Text)
if tags == nil && err != nil {
c.error(err)
}
if _, ok := saw[tags.String()]; ok {
c.errorf("duplicate entry")
}
saw[tags.String()] = true
if len(tags) == 0 {
c.errorf("lookup entries require tags")
}
empty := make(opentsdb.TagSet)
for k := range tags {
empty[k] = ""
}
if len(lookupTags) == 0 {
lookupTags = empty
for k := range empty {
l.Tags = append(l.Tags, k)
}
} else if !lookupTags.Equal(empty) {
c.errorf("lookup tags mismatch, expected %v", lookupTags)
}
e := Entry{
Def: n.RawText,
Name: n.Name.Text,
ExprEntry: &ExprEntry{
AlertKey: models.NewAlertKey("", tags),
Values: make(map[string]string),
},
}
for _, en := range n.Nodes.Nodes {
c.at(en)
switch en := en.(type) {
case *parse.PairNode:
e.Values[en.Key.Text] = en.Val.Text
default:
c.errorf("unexpected node")
}
}
l.Entries = append(l.Entries, &e)
default:
c.errorf("unexpected node")
}
}
c.at(s)
c.Lookups[name] = &l
}
func TestCheckFlapping(t *testing.T) {
defer setup()()
c, err := rule.NewConf("", conf.EnabledBackends{}, `
template t {
subject = 1
}
notification n {
print = true
}
alert a {
warnNotification = n
warn = 1
critNotification = n
crit = 1
template = t
}
`)
if err != nil {
t.Fatal(err)
}
s, _ := initSched(&conf.SystemConf{}, c)
ak := models.NewAlertKey("a", nil)
r := &RunHistory{
Events: map[models.AlertKey]*models.Event{
ak: {Status: models.StWarning},
},
}
hasNots := func() bool {
defer func() {
s.pendingNotifications = nil
}()
if len(s.pendingNotifications) != 1 {
return false
}
for k, v := range s.pendingNotifications {
if k.Name != "n" || len(v) != 1 || v[0].Alert != "a" {
return false
}
return true
}
return false
}
type stateTransition struct {
S models.Status
ExpectNots bool
}
transitions := []stateTransition{
{models.StWarning, true},
{models.StNormal, false},
{models.StWarning, false},
{models.StNormal, false},
{models.StCritical, true},
{models.StWarning, false},
{models.StCritical, false},
}
for i, trans := range transitions {
r.Events[ak].Status = trans.S
s.RunHistory(r)
has := hasNots()
if has && !trans.ExpectNots {
t.Fatalf("unexpected notifications for transition %d.", i)
} else if !has && trans.ExpectNots {
t.Fatalf("expected notifications for transition %d.", i)
}
}
r.Events[ak].Status = models.StNormal
s.RunHistory(r)
// Close the alert, so it should notify next time.
if err := s.ActionByAlertKey("", "", models.ActionClose, ak); err != nil {
t.Fatal(err)
}
r.Events[ak].Status = models.StWarning
s.RunHistory(r)
if !hasNots() {
t.Fatal("expected notification")
}
}
func (s *State) AlertKey() models.AlertKey {
return models.NewAlertKey(s.Alert, s.Group)
}
func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus models.Status, ignore models.AlertKeys) (alerts models.AlertKeys, err error, cancelled bool) {
if e == nil {
return
}
defer func() {
if err == nil {
return
}
collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
slog.Errorln(err)
}()
type res struct {
results *expr.Results
error error
}
// See s.CheckAlert for an explanation of execution and cancellation with this channel
rc := make(chan res, 1)
var results *expr.Results
go func() {
results, err := s.executeExpr(T, rh, a, e)
rc <- res{results, err}
}()
select {
case res := <-rc:
results = res.results
err = res.error
case <-s.runnerContext.Done():
return nil, nil, true
}
if err != nil {
return
}
Loop:
for _, r := range results.Results {
if s.RuleConf.Squelched(a, r.Group) {
continue
}
ak := models.NewAlertKey(a.Name, r.Group)
for _, v := range ignore {
if ak == v {
continue Loop
}
}
var n float64
n, err = valueToFloat(r.Value)
if err != nil {
return
}
event := rh.Events[ak]
if event == nil {
event = new(models.Event)
rh.Events[ak] = event
}
result := &models.Result{
Computations: r.Computations,
Value: models.Float(n),
Expr: e.String(),
}
switch checkStatus {
case models.StWarning:
event.Warn = result
case models.StCritical:
event.Crit = result
}
status := checkStatus
if math.IsNaN(n) {
status = checkStatus
} else if n == 0 {
status = models.StNormal
}
if status != models.StNormal {
alerts = append(alerts, ak)
}
if status > rh.Events[ak].Status {
event.Status = status
}
}
return
}
func TestCheckFlapping(t *testing.T) {
defer setup()()
c, err := conf.New("", `
template t {
subject = 1
}
notification n {
print = true
}
alert a {
warnNotification = n
warn = 1
critNotification = n
crit = 1
normNotification = n
template = t
}
`)
if err != nil {
t.Fatal(err)
}
s, _ := initSched(c)
ak := models.NewAlertKey("a", nil)
r := &RunHistory{
Events: map[models.AlertKey]*models.Event{
ak: {Status: models.StWarning},
},
}
hasNots := func() bool {
defer func() {
s.pendingNotifications = nil
}()
if len(s.pendingNotifications) != 1 {
return false
}
for k, v := range s.pendingNotifications {
if k.Name != "n" || len(v) != 1 || v[0].Alert != "a" {
return false
}
return true
}
return false
}
type stateTransition struct {
S models.Status
ExpectNots bool
}
/**
transitions := []stateTransition{
{models.StWarning, true},
{models.StNormal, false},
{models.StWarning, false},
{models.StNormal, false},
{models.StCritical, true},
{models.StWarning, false},
{models.StCritical, false},
}
VICTOROPS INTEGRATION: These state Transitions have been commented out and replaced with a test case which matches the behaviour we want to integrate with victorops
*/
transitions := []stateTransition{
{models.StWarning, true},
{models.StNormal, true},
{models.StWarning, true},
{models.StNormal, true},
{models.StCritical, true},
{models.StWarning, false},
{models.StCritical, false},
{models.StNormal, true},
}
for i, trans := range transitions {
r.Events[ak].Status = trans.S
s.RunHistory(r)
has := hasNots()
if has && !trans.ExpectNots {
t.Fatalf("unexpected notifications for transition %d.", i)
} else if !has && trans.ExpectNots {
t.Fatalf("expected notifications for transition %d.", i)
}
}
r.Events[ak].Status = models.StNormal
s.RunHistory(r)
// Close the alert, so it should notify next time.
if err := s.Action("", "", models.ActionClose, ak); err != nil {
t.Fatal(err)
}
r.Events[ak].Status = models.StWarning
s.RunHistory(r)
if !hasNots() {
t.Fatal("expected notification")
}
}
func TestCheckFlapping(t *testing.T) {
c, err := conf.New("", `
template t {
subject = 1
}
notification n {
print = true
}
alert a {
warnNotification = n
warn = 1
critNotification = n
crit = 1
template = t
}
`)
if err != nil {
t.Fatal(err)
}
s, _ := initSched(c)
ak := models.NewAlertKey("a", nil)
r := &RunHistory{
Events: map[models.AlertKey]*Event{
ak: {Status: StWarning},
},
}
hasNots := func() bool {
defer func() {
s.pendingNotifications = nil
}()
if len(s.pendingNotifications) != 1 {
return false
}
for k, v := range s.pendingNotifications {
if k.Name != "n" || len(v) != 1 || v[0].Alert != "a" {
return false
}
return true
}
return false
}
s.RunHistory(r)
if !hasNots() {
t.Fatalf("expected notification: %v", s.pendingNotifications)
}
r.Events[ak].Status = StNormal
s.RunHistory(r)
if hasNots() {
t.Fatal("unexpected notification")
}
r.Events[ak].Status = StWarning
s.RunHistory(r)
if hasNots() {
t.Fatal("unexpected notification")
}
r.Events[ak].Status = StNormal
s.RunHistory(r)
if hasNots() {
t.Fatal("unexpected notification")
}
r.Events[ak].Status = StCritical
s.RunHistory(r)
if !hasNots() {
t.Fatal("expected notification")
}
r.Events[ak].Status = StNormal
s.RunHistory(r)
if hasNots() {
t.Fatal("unexpected notification")
}
s.RunHistory(r)
// Close the alert, so it should notify next time.
if err := s.Action("", "", ActionClose, ak); err != nil {
t.Fatal(err)
}
r.Events[ak].Status = StWarning
s.RunHistory(r)
if !hasNots() {
t.Fatal("expected notification")
}
}