func (e *DefaultEvalHandler) eval(context *EvalContext) {
defer func() {
if err := recover(); err != nil {
e.log.Error("Alerting rule eval panic", "error", err, "stack", log.Stack(1))
if panicErr, ok := err.(error); ok {
context.Error = panicErr
}
}
}()
for _, condition := range context.Rule.Conditions {
condition.Eval(context)
// break if condition could not be evaluated
if context.Error != nil {
break
}
// break if result has not triggered yet
if context.Firing == false {
break
}
}
context.EndTime = time.Now()
elapsedTime := context.EndTime.Sub(context.StartTime) / time.Millisecond
metrics.M_Alerting_Exeuction_Time.Update(elapsedTime)
context.DoneChan <- true
}
func (e *Engine) processJob(grafanaCtx context.Context, job *Job) error {
defer func() {
if err := recover(); err != nil {
e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1))
}
}()
alertCtx, cancelFn := context.WithTimeout(context.TODO(), alertTimeout)
job.Running = true
evalContext := NewEvalContext(alertCtx, job.Rule)
done := make(chan struct{})
go func() {
defer func() {
if err := recover(); err != nil {
e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1))
close(done)
}
}()
e.evalHandler.Eval(evalContext)
e.resultHandler.Handle(evalContext)
close(done)
}()
var err error = nil
select {
case <-grafanaCtx.Done():
select {
case <-time.After(unfinishedWorkTimeout):
cancelFn()
err = grafanaCtx.Err()
case <-done:
}
case <-done:
}
e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.Id, "name", evalContext.Rule.Name, "firing", evalContext.Firing)
job.Running = false
cancelFn()
return err
}
func (e *Engine) resultDispatcher() {
defer func() {
if err := recover(); err != nil {
e.log.Error("Panic in resultDispatcher", "error", err, "stack", log.Stack(1))
}
}()
for result := range e.resultQueue {
e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "firing", result.Firing)
e.resultHandler.Handle(result)
}
}
func (e *Engine) executeJob(job *Job) {
defer func() {
if err := recover(); err != nil {
e.log.Error("Execute Alert Panic", "error", err, "stack", log.Stack(1))
}
}()
job.Running = true
context := NewEvalContext(job.Rule)
e.evalHandler.Eval(context)
job.Running = false
e.resultQueue <- context
}
func (e *Engine) alertingTicker() {
defer func() {
if err := recover(); err != nil {
e.log.Error("Scheduler Panic: stopping alertingTicker", "error", err, "stack", log.Stack(1))
}
}()
tickIndex := 0
for {
select {
case tick := <-e.ticker.C:
// TEMP SOLUTION update rules ever tenth tick
if tickIndex%10 == 0 {
e.scheduler.Update(e.ruleReader.Fetch())
}
e.scheduler.Tick(tick, e.execQueue)
tickIndex++
}
}
}