Exemple #1
0
func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus models.Status, ignore models.AlertKeys) (alerts models.AlertKeys, err error, cancelled bool) {
	if e == nil {
		return
	}
	defer func() {
		if err == nil {
			return
		}
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		slog.Errorln(err)
	}()
	type res struct {
		results *expr.Results
		error   error
	}
	// See s.CheckAlert for an explanation of execution and cancellation with this channel
	rc := make(chan res, 1)
	var results *expr.Results
	go func() {
		results, err := s.executeExpr(T, rh, a, e)
		rc <- res{results, err}
	}()
	select {
	case res := <-rc:
		results = res.results
		err = res.error
	case <-s.runnerContext.Done():
		return nil, nil, true
	}
	if err != nil {
		return
	}
Loop:
	for _, r := range results.Results {
		if s.RuleConf.Squelched(a, r.Group) {
			continue
		}
		ak := models.NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		var n float64
		n, err = valueToFloat(r.Value)
		if err != nil {
			return
		}
		event := rh.Events[ak]
		if event == nil {
			event = new(models.Event)
			rh.Events[ak] = event
		}
		result := &models.Result{
			Computations: r.Computations,
			Value:        models.Float(n),
			Expr:         e.String(),
		}
		switch checkStatus {
		case models.StWarning:
			event.Warn = result
		case models.StCritical:
			event.Crit = result
		}
		status := checkStatus
		if math.IsNaN(n) {
			status = checkStatus
		} else if n == 0 {
			status = models.StNormal
		}
		if status != models.StNormal {
			alerts = append(alerts, ak)
		}
		if status > rh.Events[ak].Status {
			event.Status = status
		}
	}
	return
}
Exemple #2
0
func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus models.Status, ignore models.AlertKeys) (alerts models.AlertKeys, err error) {
	if e == nil {
		return
	}
	defer func() {
		if err == nil {
			return
		}
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		slog.Errorln(err)
	}()
	results, err := s.executeExpr(T, rh, a, e)
	if err != nil {
		return nil, err
	}
Loop:
	for _, r := range results.Results {
		if s.Conf.Squelched(a, r.Group) {
			continue
		}
		ak := models.NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		var n float64
		n, err = valueToFloat(r.Value)
		if err != nil {
			return
		}
		event := rh.Events[ak]
		if event == nil {
			event = new(models.Event)
			rh.Events[ak] = event
		}
		result := &models.Result{
			Computations: r.Computations,
			Value:        models.Float(n),
			Expr:         e.String(),
		}
		switch checkStatus {
		case models.StWarning:
			event.Warn = result
		case models.StCritical:
			event.Crit = result
		}
		status := checkStatus
		if math.IsNaN(n) {
			status = checkStatus
		} else if n == 0 {
			status = models.StNormal
		}
		if status != models.StNormal {
			alerts = append(alerts, ak)
		}
		if status > rh.Events[ak].Status {
			event.Status = status
		}
	}
	return
}
Exemple #3
0
func migrateState(db *bolt.DB, data database.DataAccess) error {
	migrated, err := isMigrated(db, "state")
	if err != nil {
		return err
	}
	if migrated {
		return nil
	}
	//redefine the structs as they were when we gob encoded them
	type Result struct {
		*expr.Result
		Expr string
	}
	mResult := func(r *Result) *models.Result {
		if r == nil || r.Result == nil {
			return &models.Result{}
		}
		v, _ := valueToFloat(r.Result.Value)
		return &models.Result{
			Computations: r.Result.Computations,
			Value:        models.Float(v),
			Expr:         r.Expr,
		}
	}
	type Event struct {
		Warn, Crit  *Result
		Status      models.Status
		Time        time.Time
		Unevaluated bool
		IncidentId  uint64
	}
	type State struct {
		*Result
		History      []Event
		Actions      []models.Action
		Touched      time.Time
		Alert        string
		Tags         string
		Group        opentsdb.TagSet
		Subject      string
		Body         string
		EmailBody    []byte
		EmailSubject []byte
		Attachments  []*models.Attachment
		NeedAck      bool
		Open         bool
		Forgotten    bool
		Unevaluated  bool
		LastLogTime  time.Time
	}
	type OldStates map[models.AlertKey]*State
	slog.Info("migrating state")
	states := OldStates{}
	if err := decode(db, "status", &states); err != nil {
		return err
	}
	for ak, state := range states {
		if len(state.History) == 0 {
			continue
		}
		var thisId uint64
		events := []Event{}
		addIncident := func(saveBody bool) error {
			if thisId == 0 || len(events) == 0 || state == nil {
				return nil
			}
			incident := NewIncident(ak)
			incident.Expr = state.Expr

			incident.NeedAck = state.NeedAck
			incident.Open = state.Open
			incident.Result = mResult(state.Result)
			incident.Unevaluated = state.Unevaluated
			incident.Start = events[0].Time
			incident.Id = int64(thisId)
			incident.Subject = state.Subject
			if saveBody {
				incident.Body = state.Body
			}
			for _, ev := range events {
				incident.CurrentStatus = ev.Status
				mEvent := models.Event{
					Crit:        mResult(ev.Crit),
					Status:      ev.Status,
					Time:        ev.Time,
					Unevaluated: ev.Unevaluated,
					Warn:        mResult(ev.Warn),
				}
				incident.Events = append(incident.Events, mEvent)
				if ev.Status > incident.WorstStatus {
					incident.WorstStatus = ev.Status
				}
				if ev.Status > models.StNormal {
					incident.LastAbnormalStatus = ev.Status
					incident.LastAbnormalTime = ev.Time.UTC().Unix()
				}
			}
			for _, ac := range state.Actions {
				if ac.Time.Before(incident.Start) {
					continue
				}
				incident.Actions = append(incident.Actions, ac)
				if ac.Time.After(incident.Events[len(incident.Events)-1].Time) && ac.Type == models.ActionClose {
					incident.End = &ac.Time
					break
				}
			}
			if err := data.State().ImportIncidentState(incident); err != nil {
				return err
			}
			return nil
		}
		//essentially a rle algorithm to assign events to incidents
		for _, e := range state.History {
			if e.Status > models.StUnknown {
				continue
			}
			if e.IncidentId == 0 {
				//include all non-assigned incidents up to the next non-match
				events = append(events, e)
				continue
			}
			if thisId == 0 {
				thisId = e.IncidentId
				events = append(events, e)
			}
			if e.IncidentId != thisId {
				if err := addIncident(false); err != nil {
					return err
				}
				thisId = e.IncidentId
				events = []Event{e}

			} else {
				events = append(events, e)
			}
		}
		if err := addIncident(true); err != nil {
			return err
		}
	}
	if err = setMigrated(db, "state"); err != nil {
		return err
	}
	return nil
}