func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus models.Status, ignore models.AlertKeys) (alerts models.AlertKeys, err error, cancelled bool) {
if e == nil {
return
}
defer func() {
if err == nil {
return
}
collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
slog.Errorln(err)
}()
type res struct {
results *expr.Results
error error
}
// See s.CheckAlert for an explanation of execution and cancellation with this channel
rc := make(chan res, 1)
var results *expr.Results
go func() {
results, err := s.executeExpr(T, rh, a, e)
rc <- res{results, err}
}()
select {
case res := <-rc:
results = res.results
err = res.error
case <-s.runnerContext.Done():
return nil, nil, true
}
if err != nil {
return
}
Loop:
for _, r := range results.Results {
if s.RuleConf.Squelched(a, r.Group) {
continue
}
ak := models.NewAlertKey(a.Name, r.Group)
for _, v := range ignore {
if ak == v {
continue Loop
}
}
var n float64
n, err = valueToFloat(r.Value)
if err != nil {
return
}
event := rh.Events[ak]
if event == nil {
event = new(models.Event)
rh.Events[ak] = event
}
result := &models.Result{
Computations: r.Computations,
Value: models.Float(n),
Expr: e.String(),
}
switch checkStatus {
case models.StWarning:
event.Warn = result
case models.StCritical:
event.Crit = result
}
status := checkStatus
if math.IsNaN(n) {
status = checkStatus
} else if n == 0 {
status = models.StNormal
}
if status != models.StNormal {
alerts = append(alerts, ak)
}
if status > rh.Events[ak].Status {
event.Status = status
}
}
return
}
func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus models.Status, ignore models.AlertKeys) (alerts models.AlertKeys, err error) {
if e == nil {
return
}
defer func() {
if err == nil {
return
}
collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
slog.Errorln(err)
}()
results, err := s.executeExpr(T, rh, a, e)
if err != nil {
return nil, err
}
Loop:
for _, r := range results.Results {
if s.Conf.Squelched(a, r.Group) {
continue
}
ak := models.NewAlertKey(a.Name, r.Group)
for _, v := range ignore {
if ak == v {
continue Loop
}
}
var n float64
n, err = valueToFloat(r.Value)
if err != nil {
return
}
event := rh.Events[ak]
if event == nil {
event = new(models.Event)
rh.Events[ak] = event
}
result := &models.Result{
Computations: r.Computations,
Value: models.Float(n),
Expr: e.String(),
}
switch checkStatus {
case models.StWarning:
event.Warn = result
case models.StCritical:
event.Crit = result
}
status := checkStatus
if math.IsNaN(n) {
status = checkStatus
} else if n == 0 {
status = models.StNormal
}
if status != models.StNormal {
alerts = append(alerts, ak)
}
if status > rh.Events[ak].Status {
event.Status = status
}
}
return
}
func migrateState(db *bolt.DB, data database.DataAccess) error {
migrated, err := isMigrated(db, "state")
if err != nil {
return err
}
if migrated {
return nil
}
//redefine the structs as they were when we gob encoded them
type Result struct {
*expr.Result
Expr string
}
mResult := func(r *Result) *models.Result {
if r == nil || r.Result == nil {
return &models.Result{}
}
v, _ := valueToFloat(r.Result.Value)
return &models.Result{
Computations: r.Result.Computations,
Value: models.Float(v),
Expr: r.Expr,
}
}
type Event struct {
Warn, Crit *Result
Status models.Status
Time time.Time
Unevaluated bool
IncidentId uint64
}
type State struct {
*Result
History []Event
Actions []models.Action
Touched time.Time
Alert string
Tags string
Group opentsdb.TagSet
Subject string
Body string
EmailBody []byte
EmailSubject []byte
Attachments []*models.Attachment
NeedAck bool
Open bool
Forgotten bool
Unevaluated bool
LastLogTime time.Time
}
type OldStates map[models.AlertKey]*State
slog.Info("migrating state")
states := OldStates{}
if err := decode(db, "status", &states); err != nil {
return err
}
for ak, state := range states {
if len(state.History) == 0 {
continue
}
var thisId uint64
events := []Event{}
addIncident := func(saveBody bool) error {
if thisId == 0 || len(events) == 0 || state == nil {
return nil
}
incident := NewIncident(ak)
incident.Expr = state.Expr
incident.NeedAck = state.NeedAck
incident.Open = state.Open
incident.Result = mResult(state.Result)
incident.Unevaluated = state.Unevaluated
incident.Start = events[0].Time
incident.Id = int64(thisId)
incident.Subject = state.Subject
if saveBody {
incident.Body = state.Body
}
for _, ev := range events {
incident.CurrentStatus = ev.Status
mEvent := models.Event{
Crit: mResult(ev.Crit),
Status: ev.Status,
Time: ev.Time,
Unevaluated: ev.Unevaluated,
Warn: mResult(ev.Warn),
}
incident.Events = append(incident.Events, mEvent)
if ev.Status > incident.WorstStatus {
incident.WorstStatus = ev.Status
}
if ev.Status > models.StNormal {
incident.LastAbnormalStatus = ev.Status
incident.LastAbnormalTime = ev.Time.UTC().Unix()
}
}
for _, ac := range state.Actions {
if ac.Time.Before(incident.Start) {
continue
}
incident.Actions = append(incident.Actions, ac)
if ac.Time.After(incident.Events[len(incident.Events)-1].Time) && ac.Type == models.ActionClose {
incident.End = &ac.Time
break
}
}
if err := data.State().ImportIncidentState(incident); err != nil {
return err
}
return nil
}
//essentially a rle algorithm to assign events to incidents
for _, e := range state.History {
if e.Status > models.StUnknown {
continue
}
if e.IncidentId == 0 {
//include all non-assigned incidents up to the next non-match
events = append(events, e)
continue
}
if thisId == 0 {
thisId = e.IncidentId
events = append(events, e)
}
if e.IncidentId != thisId {
if err := addIncident(false); err != nil {
return err
}
thisId = e.IncidentId
events = []Event{e}
} else {
events = append(events, e)
}
}
if err := addIncident(true); err != nil {
return err
}
}
if err = setMigrated(db, "state"); err != nil {
return err
}
return nil
}