// 处理事件(去掉了优先级区分) func consumeEvents(event *model.Event, action *api.Action) { if action.Uic == "" { return } //获取联系方式 phones, mails, ims := api.ParseTeams(action.Uic) smsContent := GenerateSmsContent(event) mailContent := GenerateMailContent(event) imContent := GenerateIMSmsContent(event) phoneContent := GeneratePhoneContent(event) // level 0 will have phone,im,mail // level 1-2 will have im,mail // level 3-6 will only have im if event.Priority() == 0 { redis.WritePhone(phones, phoneContent) } if event.Priority() < 3 { redis.WriteMail(mails, smsContent, mailContent) } redis.WriteIMSms(ims, imContent) }
func ParseUserMail(event *model.Event, action *api.Action) { userMap := api.GetUsers(action.Uic) metric := event.Metric() subject := GenerateSmsContent(event) content := GenerateMailContent(event) status := event.Status priority := event.Priority() queue := g.Config().Redis.UserMailQueue rc := g.RedisConnPool.Get() defer rc.Close() for _, user := range userMap { dto := MailDto{ Priority: priority, Metric: metric, Subject: subject, Content: content, Email: user.Email, Status: status, } bs, err := json.Marshal(dto) if err != nil { log.Println("json marshal MailDto fail:", err) continue } _, err = rc.Do("LPUSH", queue, string(bs)) if err != nil { log.Println("LPUSH redis", queue, "fail:", err, "dto:", string(bs)) } } }
// 低优先级的做报警合并 func consumeLowEvents(event *model.Event, action *api.Action) { if action.Uic == "" { return } if event.Priority() < 3 { ParseUserSms(event, action) } ParseUserMail(event, action) }
func Link(event *model.Event) string { tplId := event.TplId() if tplId != 0 { return fmt.Sprintf("%s/template/view/%d", Config().Api.Portal, tplId) } eid := event.ExpressionId() if eid != 0 { return fmt.Sprintf("%s/expression/view/%d", Config().Api.Portal, eid) } return "" }
// 高优先级的不做报警合并 func consumeHighEvents(event *model.Event, action *api.Action) { if action.Uic == "" { return } phones, mails := api.ParseTeams(action.Uic) smsContent := GenerateSmsContent(event) mailContent := GenerateMailContent(event) if event.Priority() < 3 { redis.WriteSms(phones, smsContent) } redis.WriteMail(mails, smsContent, mailContent) }
func sendEvent(event *model.Event) { // update last event g.LastEvents.Set(event.Id, event) bs, err := json.Marshal(event) if err != nil { log.Printf("json marshal event %v fail: %v", event, err) return } // send to redis redisKey := fmt.Sprintf(g.Config().Alarm.QueuePattern, event.Priority()) rc := g.RedisConnPool.Get() defer rc.Close() rc.Do("LPUSH", redisKey, string(bs)) }
func sendEventIfNeed(historyData []*model.HistoryData, isTriggered bool, now int64, event *model.Event, maxStep int) { lastEvent, exists := g.LastEvents.Get(event.Id) if isTriggered { event.Status = "PROBLEM" if !exists || lastEvent.Status[0] == 'O' { // 本次触发了阈值,之前又没报过警,得产生一个报警Event event.CurrentStep = 1 // 但是有些用户把最大报警次数配置成了0,相当于屏蔽了,要检查一下 if maxStep == 0 { return } sendEvent(event) return } // 逻辑走到这里,说明之前Event是PROBLEM状态 if lastEvent.CurrentStep >= maxStep { // 报警次数已经足够多,到达了最多报警次数了,不再报警 return } if historyData[len(historyData)-1].Timestamp <= lastEvent.EventTime { // 产生过报警的点,就不能再使用来判断了,否则容易出现一分钟报一次的情况 // 只需要拿最后一个historyData来做判断即可,因为它的时间最老 return } if now-lastEvent.EventTime < g.Config().Alarm.MinInterval { // 报警不能太频繁,两次报警之间至少要间隔MinInterval秒,否则就不能报警 return } event.CurrentStep = lastEvent.CurrentStep + 1 sendEvent(event) } else { // 如果LastEvent是Problem,报OK,否则啥都不做 if exists && lastEvent.Status[0] == 'P' { event.Status = "OK" event.CurrentStep = 1 sendEvent(event) } } }
func consume(event *model.Event, isHigh bool) { actionId := event.ActionId() if actionId <= 0 { return } action := api.GetAction(actionId) if action == nil { return } if action.Callback == 1 { HandleCallback(event, action) return } consumeEvents(event, action) }
func popEvent(queues []string) (*model.Event, error) { count := len(queues) params := make([]interface{}, count+1) for i := 0; i < count; i++ { params[i] = queues[i] } // set timeout 0 params[count] = 0 rc := g.RedisConnPool.Get() defer rc.Close() reply, err := redis.Strings(rc.Do("BRPOP", params...)) if err != nil { log.Printf("get alarm event from redis fail: %v", err) return nil, err } var event model.Event err = json.Unmarshal([]byte(reply[1]), &event) if err != nil { log.Printf("parse alarm event fail: %v", err) return nil, err } if g.Config().Debug { log.Println("======>>>>") log.Println(event.String()) } // save in memory. display in dashboard g.Events.Put(&event) return &event, nil }
func Callback(event *model.Event, action *api.Action) string { if action.Url == "" { return "callback url is blank" } L := make([]string, 0) if len(event.PushedTags) > 0 { for k, v := range event.PushedTags { L = append(L, fmt.Sprintf("%s:%s", k, v)) } } tags := "" if len(L) > 0 { tags = strings.Join(L, ",") } req := httplib.Get(action.Url).SetTimeout(3*time.Second, 20*time.Second) req.Param("endpoint", event.Endpoint) req.Param("metric", event.Metric()) req.Param("status", event.Status) req.Param("step", fmt.Sprintf("%d", event.CurrentStep)) req.Param("priority", fmt.Sprintf("%d", event.Priority())) req.Param("time", event.FormattedTime()) req.Param("tpl_id", fmt.Sprintf("%d", event.TplId())) req.Param("exp_id", fmt.Sprintf("%d", event.ExpressionId())) req.Param("stra_id", fmt.Sprintf("%d", event.StrategyId())) req.Param("tags", tags) resp, e := req.String() success := "success" if e != nil { success = fmt.Sprintf("fail:%s", e.Error()) } message := fmt.Sprintf("curl %s %s. resp: %s", action.Url, success, resp) return message }
func (this *SafeEvents) Put(event *model.Event) { if event.Status == "OK" { this.Delete(event.Id) return } dto := &EventDto{} dto.Id = event.Id dto.Endpoint = event.Endpoint dto.Metric = event.Metric() dto.Counter = event.Counter() dto.Func = event.Func() dto.LeftValue = utils.ReadableFloat(event.LeftValue) dto.Operator = event.Operator() dto.RightValue = utils.ReadableFloat(event.RightValue()) dto.Note = event.Note() dto.MaxStep = event.MaxStep() dto.CurrentStep = event.CurrentStep dto.Priority = event.Priority() dto.Status = event.Status dto.Timestamp = event.EventTime dto.ExpressionId = event.ExpressionId() dto.StrategyId = event.StrategyId() dto.TemplateId = event.TplId() dto.Link = Link(event) this.Lock() defer this.Unlock() this.M[dto.Id] = dto }
func BuildCommonSMSContent(event *model.Event) string { return fmt.Sprintf( "[P%d][%s][%s][][%s %s %s %s %s%s%s][O%d %s]", event.Priority(), event.Status, event.Endpoint, event.Note(), event.Func(), event.Metric(), utils.SortedTags(event.PushedTags), utils.ReadableFloat(event.LeftValue), event.Operator(), utils.ReadableFloat(event.RightValue()), event.CurrentStep, event.FormattedTime(), ) }
func BuildCommonMailContent(event *model.Event) string { link := g.Link(event) return fmt.Sprintf( "%s\r\nP%d\r\nEndpoint:%s\r\nMetric:%s\r\nTags:%s\r\n%s: %s%s%s\r\nNote:%s\r\nMax:%d, Current:%d\r\nTimestamp:%s\r\n%s\r\n", event.Status, event.Priority(), event.Endpoint, event.Metric(), utils.SortedTags(event.PushedTags), event.Func(), utils.ReadableFloat(event.LeftValue), event.Operator(), utils.ReadableFloat(event.RightValue()), event.Note(), event.MaxStep(), event.CurrentStep, event.FormattedTime(), link, ) }