// work waits for events to alert.
func (al *Alerter) work() {
for {
ev := <-al.In
ew := models.NewWrapperOfEvent(ev) // Avoid locks
if al.checkAlertAt(ew.Metric) { // Check alert interval
continue
}
if al.checkOneDayAlerts(ew.Metric) { // Check one day limit
continue
}
// Avoid noises by issuing alerts only when same alert has occurred
// predefined times.
if al.checkAlertCount(ew.Metric) {
al.setAlertRecord(ew.Metric)
log.Warnf("Not enough alerts with in `AlertCheckInterval` time skipping..: %v", ew.Metric.Name)
continue
}
al.setAlertRecord(ew.Metric)
al.incrAlertNum(ew.Metric)
// Store event
if err := al.storeEvent(ev); err != nil {
log.Warnf("failed to store event:%v, skipping..", err)
continue
}
// Do alert.
var err error
if ew.Project, err = al.getProjByRule(ew.Rule); err != nil {
continue
}
if al.shoudProjBeSilent(ew.Project) {
continue
}
var users []models.User
if users, err = al.getUsersByProj(ew.Project); err != nil {
continue
}
for _, user := range users {
ew.User = &user
if ew.Rule.Level < user.RuleLevel {
continue
}
if len(al.cfg.Alerter.Command) == 0 {
log.Warnf("alert command not configured")
continue
}
if err = al.execCommand(ew); err != nil { // Execute command
log.Errorf("exec %s: %v", al.cfg.Alerter.Command, err)
continue
}
log.Infof("send to %s with %s ok", user.Name, ew.Metric.Name)
}
if len(users) != 0 {
al.setAlertAt(ew.Metric)
health.IncrNumAlertingEvents(1)
}
}
}
// process the input metric.
// Steps:
// 1. Match metric with all rules.
// 2. Detect the metric with matched rules.
// 3. Output detection results to receivers.
func (d *Detector) process(m *models.Metric, shouldAdjustIdle bool) {
health.IncrNumMetricIncomed(1)
timer := util.NewTimer() // Detection cost timer
// Match
ok, rules := d.match(m)
if !ok {
return
}
if shouldAdjustIdle {
d.adjustIdleM(m, rules)
}
// Detect
evs, err := d.detect(m, rules)
if err != nil {
log.Errorf("detect: %v, skipping..", err)
return
}
health.IncrNumMetricDetected(1)
// Output
for _, ev := range evs {
d.output(ev)
}
// Time end.
elapsed := timer.Elapsed()
if elapsed > float64(d.cfg.Detector.WarningTimeout) {
log.Warnf("detection is slow: %.2fms", elapsed)
}
health.AddDetectionCost(elapsed)
}
func initConfig() {
// Config parsing.
if flag.NFlag() == 0 || (flag.NFlag() == 1 && *debug) { // Case ./program [-d]
log.Warnf("no config specified, using default..")
} else { // Case ./program -c filename
err := cfg.UpdateWithYamlFile(*fileName)
if err != nil {
log.Fatalf("failed to load %s, %s", *fileName, err)
}
}
// Config validation.
err := cfg.Validate()
if err != nil {
log.Fatalf("config: %s", err)
}
}