// 处理事件（去掉了优先级区分）
func consumeEvents(event *model.Event, action *api.Action) {
if action.Uic == "" {
return
}
//获取联系方式
phones, mails, ims := api.ParseTeams(action.Uic)
smsContent := GenerateSmsContent(event)
mailContent := GenerateMailContent(event)
imContent := GenerateIMSmsContent(event)
phoneContent := GeneratePhoneContent(event)
// level 0 will have phone,im,mail
// level 1-2 will have im,mail
// level 3-6 will only have im
if event.Priority() == 0 {
redis.WritePhone(phones, phoneContent)
}
if event.Priority() < 3 {
redis.WriteMail(mails, smsContent, mailContent)
}
redis.WriteIMSms(ims, imContent)
}
func ParseUserMail(event *model.Event, action *api.Action) {
userMap := api.GetUsers(action.Uic)
metric := event.Metric()
subject := GenerateSmsContent(event)
content := GenerateMailContent(event)
status := event.Status
priority := event.Priority()
queue := g.Config().Redis.UserMailQueue
rc := g.RedisConnPool.Get()
defer rc.Close()
for _, user := range userMap {
dto := MailDto{
Priority: priority,
Metric: metric,
Subject: subject,
Content: content,
Email: user.Email,
Status: status,
}
bs, err := json.Marshal(dto)
if err != nil {
log.Println("json marshal MailDto fail:", err)
continue
}
_, err = rc.Do("LPUSH", queue, string(bs))
if err != nil {
log.Println("LPUSH redis", queue, "fail:", err, "dto:", string(bs))
}
}
}
// 低优先级的做报警合并
func consumeLowEvents(event *model.Event, action *api.Action) {
if action.Uic == "" {
return
}
if event.Priority() < 3 {
ParseUserSms(event, action)
}
ParseUserMail(event, action)
}
func Link(event *model.Event) string {
tplId := event.TplId()
if tplId != 0 {
return fmt.Sprintf("%s/template/view/%d", Config().Api.Portal, tplId)
}
eid := event.ExpressionId()
if eid != 0 {
return fmt.Sprintf("%s/expression/view/%d", Config().Api.Portal, eid)
}
return ""
}
// 高优先级的不做报警合并
func consumeHighEvents(event *model.Event, action *api.Action) {
if action.Uic == "" {
return
}
phones, mails := api.ParseTeams(action.Uic)
smsContent := GenerateSmsContent(event)
mailContent := GenerateMailContent(event)
if event.Priority() < 3 {
redis.WriteSms(phones, smsContent)
}
redis.WriteMail(mails, smsContent, mailContent)
}
func sendEvent(event *model.Event) {
// update last event
g.LastEvents.Set(event.Id, event)
bs, err := json.Marshal(event)
if err != nil {
log.Printf("json marshal event %v fail: %v", event, err)
return
}
// send to redis
redisKey := fmt.Sprintf(g.Config().Alarm.QueuePattern, event.Priority())
rc := g.RedisConnPool.Get()
defer rc.Close()
rc.Do("LPUSH", redisKey, string(bs))
}
func sendEventIfNeed(historyData []*model.HistoryData, isTriggered bool, now int64, event *model.Event, maxStep int) {
lastEvent, exists := g.LastEvents.Get(event.Id)
if isTriggered {
event.Status = "PROBLEM"
if !exists || lastEvent.Status[0] == 'O' {
// 本次触发了阈值，之前又没报过警，得产生一个报警Event
event.CurrentStep = 1
// 但是有些用户把最大报警次数配置成了0，相当于屏蔽了，要检查一下
if maxStep == 0 {
return
}
sendEvent(event)
return
}
// 逻辑走到这里，说明之前Event是PROBLEM状态
if lastEvent.CurrentStep >= maxStep {
// 报警次数已经足够多，到达了最多报警次数了，不再报警
return
}
if historyData[len(historyData)-1].Timestamp <= lastEvent.EventTime {
// 产生过报警的点，就不能再使用来判断了，否则容易出现一分钟报一次的情况
// 只需要拿最后一个historyData来做判断即可，因为它的时间最老
return
}
if now-lastEvent.EventTime < g.Config().Alarm.MinInterval {
// 报警不能太频繁，两次报警之间至少要间隔MinInterval秒，否则就不能报警
return
}
event.CurrentStep = lastEvent.CurrentStep + 1
sendEvent(event)
} else {
// 如果LastEvent是Problem，报OK，否则啥都不做
if exists && lastEvent.Status[0] == 'P' {
event.Status = "OK"
event.CurrentStep = 1
sendEvent(event)
}
}
}
func consume(event *model.Event, isHigh bool) {
actionId := event.ActionId()
if actionId <= 0 {
return
}
action := api.GetAction(actionId)
if action == nil {
return
}
if action.Callback == 1 {
HandleCallback(event, action)
return
}
consumeEvents(event, action)
}
func popEvent(queues []string) (*model.Event, error) {
count := len(queues)
params := make([]interface{}, count+1)
for i := 0; i < count; i++ {
params[i] = queues[i]
}
// set timeout 0
params[count] = 0
rc := g.RedisConnPool.Get()
defer rc.Close()
reply, err := redis.Strings(rc.Do("BRPOP", params...))
if err != nil {
log.Printf("get alarm event from redis fail: %v", err)
return nil, err
}
var event model.Event
err = json.Unmarshal([]byte(reply[1]), &event)
if err != nil {
log.Printf("parse alarm event fail: %v", err)
return nil, err
}
if g.Config().Debug {
log.Println("======>>>>")
log.Println(event.String())
}
// save in memory. display in dashboard
g.Events.Put(&event)
return &event, nil
}
func Callback(event *model.Event, action *api.Action) string {
if action.Url == "" {
return "callback url is blank"
}
L := make([]string, 0)
if len(event.PushedTags) > 0 {
for k, v := range event.PushedTags {
L = append(L, fmt.Sprintf("%s:%s", k, v))
}
}
tags := ""
if len(L) > 0 {
tags = strings.Join(L, ",")
}
req := httplib.Get(action.Url).SetTimeout(3*time.Second, 20*time.Second)
req.Param("endpoint", event.Endpoint)
req.Param("metric", event.Metric())
req.Param("status", event.Status)
req.Param("step", fmt.Sprintf("%d", event.CurrentStep))
req.Param("priority", fmt.Sprintf("%d", event.Priority()))
req.Param("time", event.FormattedTime())
req.Param("tpl_id", fmt.Sprintf("%d", event.TplId()))
req.Param("exp_id", fmt.Sprintf("%d", event.ExpressionId()))
req.Param("stra_id", fmt.Sprintf("%d", event.StrategyId()))
req.Param("tags", tags)
resp, e := req.String()
success := "success"
if e != nil {
success = fmt.Sprintf("fail:%s", e.Error())
}
message := fmt.Sprintf("curl %s %s. resp: %s", action.Url, success, resp)
return message
}
func (this *SafeEvents) Put(event *model.Event) {
if event.Status == "OK" {
this.Delete(event.Id)
return
}
dto := &EventDto{}
dto.Id = event.Id
dto.Endpoint = event.Endpoint
dto.Metric = event.Metric()
dto.Counter = event.Counter()
dto.Func = event.Func()
dto.LeftValue = utils.ReadableFloat(event.LeftValue)
dto.Operator = event.Operator()
dto.RightValue = utils.ReadableFloat(event.RightValue())
dto.Note = event.Note()
dto.MaxStep = event.MaxStep()
dto.CurrentStep = event.CurrentStep
dto.Priority = event.Priority()
dto.Status = event.Status
dto.Timestamp = event.EventTime
dto.ExpressionId = event.ExpressionId()
dto.StrategyId = event.StrategyId()
dto.TemplateId = event.TplId()
dto.Link = Link(event)
this.Lock()
defer this.Unlock()
this.M[dto.Id] = dto
}
func BuildCommonSMSContent(event *model.Event) string {
return fmt.Sprintf(
"[P%d][%s][%s][][%s %s %s %s %s%s%s][O%d %s]",
event.Priority(),
event.Status,
event.Endpoint,
event.Note(),
event.Func(),
event.Metric(),
utils.SortedTags(event.PushedTags),
utils.ReadableFloat(event.LeftValue),
event.Operator(),
utils.ReadableFloat(event.RightValue()),
event.CurrentStep,
event.FormattedTime(),
)
}
func BuildCommonMailContent(event *model.Event) string {
link := g.Link(event)
return fmt.Sprintf(
"%s\r\nP%d\r\nEndpoint:%s\r\nMetric:%s\r\nTags:%s\r\n%s: %s%s%s\r\nNote:%s\r\nMax:%d, Current:%d\r\nTimestamp:%s\r\n%s\r\n",
event.Status,
event.Priority(),
event.Endpoint,
event.Metric(),
utils.SortedTags(event.PushedTags),
event.Func(),
utils.ReadableFloat(event.LeftValue),
event.Operator(),
utils.ReadableFloat(event.RightValue()),
event.Note(),
event.MaxStep(),
event.CurrentStep,
event.FormattedTime(),
link,
)
}