func catchTimeout(rs loop.Recoverings) (loop.Recoverings, error) {
if len(rs) > 0 {
if err, ok := rs.Last().Reason.(error); ok {
if err.Error() == "timed out" {
return nil, err
}
}
}
return nil, nil
}
// checkRecovering checks if the cell may recover after a panic. It will
// signal an error and let the cell stop working if there have been 12 recoverings
// during the last minute or the behaviors Recover() signals, that it cannot
// handle the error.
func (c *cell) checkRecovering(rs loop.Recoverings) (loop.Recoverings, error) {
logger.Errorf("recovering cell %q after error: %v", c.id, rs.Last().Reason)
// Check frequency.
if rs.Frequency(c.recoveringNumber, c.recoveringDuration) {
return nil, errors.New(ErrRecoveredTooOften, errorMessages, rs.Last().Reason)
}
// Try to recover.
if err := c.behavior.Recover(rs.Last().Reason); err != nil {
return nil, errors.Annotate(err, ErrEventRecovering, errorMessages, rs.Last().Reason)
}
return rs.Trim(c.recoveringNumber), nil
}
// checkRecovering checks if the backend can be recovered.
func (c *Crontab) checkRecovering(rs loop.Recoverings) (loop.Recoverings, error) {
if rs.Frequency(12, time.Minute) {
logger.Errorf("crontab cannot be recovered: %v", rs.Last().Reason)
return nil, errors.New(ErrCrontabCannotBeRecovered, errorMessages, rs.Last().Reason)
}
logger.Warningf("crontab recovered: %v", rs.Last().Reason)
return rs.Trim(12), nil
}
// checkRecovering checks if the backend can be recovered.
func (b *stdBackend) checkRecovering(rs loop.Recoverings) (loop.Recoverings, error) {
if rs.Frequency(12, time.Minute) {
logger.Errorf("standard monitor cannot be recovered: %v", rs.Last().Reason)
return nil, errors.New(ErrMonitoringCannotBeRecovered, errorMessages, rs.Last().Reason)
}
logger.Warningf("standard monitor recovered: %v", rs.Last().Reason)
return rs.Trim(12), nil
}
// checkRecovering checks if the cell may recover after a panic. It will
// signal an error and let the cell stop working if there have been 12 recoverings
// during the last minute or the behaviors Recover() signals, that it cannot
// handle the error.
func (c *cell) checkRecovering(rs loop.Recoverings) (loop.Recoverings, error) {
logger.Warningf("recovering cell %q after error: %v", c.id, rs.Last().Reason)
// Check frequency.
if rs.Frequency(c.recoveringNumber, c.recoveringDuration) {
err := errors.New(ErrRecoveredTooOften, errorMessages, rs.Last().Reason)
logger.Errorf("recovering frequency of cell %q too high", c.id)
return nil, err
}
// Try to recover.
if err := c.behavior.Recover(rs.Last().Reason); err != nil {
err := errors.Annotate(err, ErrEventRecovering, errorMessages, rs.Last().Reason)
logger.Errorf("recovering of cell %q failed: %v", c.id, err)
return nil, err
}
logger.Infof("successfully recovered cell %q", c.id)
return rs.Trim(c.recoveringNumber), nil
}