// work waits for events to alert. func (al *Alerter) work() { for { ev := <-al.In ew := models.NewWrapperOfEvent(ev) // Avoid locks if al.checkAlertAt(ew.Metric) { // Check alert interval continue } if al.checkOneDayAlerts(ew.Metric) { // Check one day limit continue } // Avoid noises by issuing alerts only when same alert has occurred // predefined times. if al.checkAlertCount(ew.Metric) { al.setAlertRecord(ew.Metric) log.Warnf("Not enough alerts with in `AlertCheckInterval` time skipping..: %v", ew.Metric.Name) continue } al.setAlertRecord(ew.Metric) al.incrAlertNum(ew.Metric) // Store event if err := al.storeEvent(ev); err != nil { log.Warnf("failed to store event:%v, skipping..", err) continue } // Do alert. var err error if ew.Project, err = al.getProjByRule(ew.Rule); err != nil { continue } if al.shoudProjBeSilent(ew.Project) { continue } var users []models.User if users, err = al.getUsersByProj(ew.Project); err != nil { continue } for _, user := range users { ew.User = &user if ew.Rule.Level < user.RuleLevel { continue } if len(al.cfg.Alerter.Command) == 0 { log.Warnf("alert command not configured") continue } if err = al.execCommand(ew); err != nil { // Execute command log.Errorf("exec %s: %v", al.cfg.Alerter.Command, err) continue } log.Infof("send to %s with %s ok", user.Name, ew.Metric.Name) } if len(users) != 0 { al.setAlertAt(ew.Metric) health.IncrNumAlertingEvents(1) } } }
// work waits for detected metrics, then check each metric with all the // rules, the configured shell command will be executed once a rule is hit. func (al *Alerter) work() { for { ev := <-al.In // Check interval. v, ok := al.m.Get(ev.Metric.Name) if ok && ev.Metric.Stamp-v.(uint32) < al.cfg.Alerter.Interval { continue } // Check alert times in one day v, ok = al.c.Get(ev.Metric.Name) if ok && atomic.LoadUint32(v.(*uint32)) > al.cfg.Alerter.OneDayLimit { log.Warn("%s hit alerting one day limit, skipping..", ev.Metric.Name) continue } if !ok { var newCounter uint32 newCounter = 1 al.c.Set(ev.Metric.Name, &newCounter) } else { atomic.AddUint32(v.(*uint32), 1) } // Universals var univs []models.User if err := al.db.Admin.DB().Where("universal = ?", true).Find(&univs).Error; err != nil { log.Error("get universal users: %v, skiping..", err) continue } for _, rule := range ev.Metric.TestedRules { ev.Rule = rule ev.TranslateRuleComment() // Project proj := &models.Project{} if err := al.db.Admin.DB().Model(rule).Related(proj).Error; err != nil { log.Error("project, %v, skiping..", err) continue } ev.Project = proj // Silent if al.shouldSilent(proj) { continue } // Users var users []models.User if err := al.db.Admin.DB().Model(proj).Related(&users, "Users").Error; err != nil { log.Error("get users: %v, skiping..", err) continue } users = append(users, univs...) // Send for _, user := range users { ev.User = &user if rule.Level < user.RuleLevel { continue } // Exec if len(al.cfg.Alerter.Command) == 0 { log.Warn("alert command not configured") continue } if err := al.execCommand(ev); err != nil { log.Error("exec %s: %v", al.cfg.Alerter.Command, err) continue } log.Info("send message to %s with %s ok", user.Name, ev.Metric.Name) } if len(users) != 0 { al.m.Set(ev.Metric.Name, ev.Metric.Stamp) health.IncrNumAlertingEvents(1) } } } }
// work waits for detected metrics, then check each metric with all the // rules, the configured shell command will be executed once a rule is hit. func (al *Alerter) work() { for { metric := <-al.In // Check interval. v, ok := al.m.Get(metric.Name) if ok && metric.Stamp-v.(uint32) < al.cfg.Alerter.Interval { continue } // Check alert times in one day v, ok = al.c.Get(metric.Name) if ok && atomic.LoadUint32(v.(*uint32)) > al.cfg.Alerter.OneDayLimit { log.Warn("%s hit alerting one day limit, skipping..", metric.Name) continue } if !ok { var newCounter uint32 newCounter = 1 al.c.Set(metric.Name, &newCounter) } else { atomic.AddUint32(v.(*uint32), 1) } // Universals var univs []models.User if err := al.db.Admin.DB().Where("universal = ?", true).Find(&univs).Error; err != nil { log.Error("get universal users: %v, skiping..", err) continue } for _, rule := range metric.TestedRules { // Project proj := &models.Project{} if err := al.db.Admin.DB().Model(rule).Related(proj).Error; err != nil { log.Error("project, %v, skiping..", err) continue } // Users var users []models.User if err := al.db.Admin.DB().Model(proj).Related(&users, "Users").Error; err != nil { log.Error("get users: %v, skiping..", err) continue } users = append(users, univs...) // Send for _, user := range users { d := &msg{ Project: proj, Metric: metric, User: &user, Rule: rule, } // Exec if len(al.cfg.Alerter.Command) == 0 { log.Warn("alert command not configured") continue } b, _ := json.Marshal(d) cmd := exec.Command(al.cfg.Alerter.Command, string(b)) if err := cmd.Run(); err != nil { log.Error("exec %s: %v", al.cfg.Alerter.Command, err) continue } log.Info("send message to %s with %s ok", user.Name, metric.Name) } if len(users) != 0 { al.m.Set(metric.Name, metric.Stamp) health.IncrNumAlertingEvents(1) } } } }