func (sr *Runner) flapIncr(flapID string) {
rconn := sr.rpool.Get()
defer rconn.Close()
rconn.Send("MULTI")
rconn.Send("INCR", flapID)
rconn.Send("EXPIRE", flapID, sr.conf.flapWindow)
_, err := rconn.Do("EXEC")
stalker.OnlyLogIf(err)
}
// randomly reshuffle all checks that need to be done right now and schedule
// them for a future time. i.e. if the stalker-manager was offline
// for an extended period of time.
// TODO: y u no actually shuffle!
// TODO: optomize Get & Update
func (sm *Manager) startupShuffle() {
log.Debugln("Reshuffling checks")
var err error
rquery := r.Db(STALKERDB).Table("checks").Between(0, time.Now().Unix(), r.BetweenOpts{Index: "next", RightBound: "closed"})
cursor, err := rquery.Run(sm.rsess)
defer cursor.Close()
if err != nil {
log.Panic(err)
}
result := stalker.Check{}
for cursor.Next(&result) {
_, err := r.Db(STALKERDB).Table("checks").Get(result.ID).Update(map[string]int{"next": int(time.Now().Unix()) + stalker.RandIntInRange(1, sm.shuffleT)}).RunWrite(sm.rsess)
stalker.OnlyLogIf(err)
}
stalker.OnlyLogIf(cursor.Err())
}
// place a check on the queue
func (sm *Manager) enqueueCheck(check stalker.Check) {
cb, err := json.Marshal(check)
stalker.OnlyLogIf(err)
rconn := sm.rpool.Get()
defer rconn.Close()
res, err := rconn.Do("RPUSH", "worker1", cb)
log.Debugln("Checks now on queue:", res)
if err != nil {
log.Warningln("error pushing check on queue:", err)
}
}
// scan the checks db for checks that need to run
// mark them as pending and then drop'em on the q for the runner.
func (sm *Manager) scanChecks() {
sm.pauseIfAsked()
log.Debugln("Scanning for checks past due")
qcount := 0
rquery := r.Db(STALKERDB).Table("checks").Between(0, time.Now().Unix(), r.BetweenOpts{Index: "next", RightBound: "closed"})
rquery = rquery.Filter(r.Row.Field("pending").Eq(false).And(r.Row.Field("suspended").Eq(false)))
cursor, err := rquery.Run(sm.rsess)
if err != nil {
log.Errorln("Error scanning check db!")
log.Errorln(err)
return
}
defer cursor.Close()
result := stalker.Check{}
for cursor.Next(&result) {
_, err := r.Db(STALKERDB).Table("checks").Get(result.ID).Update(map[string]bool{"pending": true}).RunWrite(sm.rsess)
stalker.OnlyLogIf(err)
sm.enqueueCheck(result)
qcount++
}
stalker.OnlyLogIf(cursor.Err())
log.Debugln("Queued:", qcount)
}
// Sanitize scan the checks db for checks marked pending but not actually
// in progress. i.e. redis died, or services where kill -9'd.
func (sm *Manager) Sanitize(flushQueued bool) {
if flushQueued {
rconn := sm.rpool.Get()
defer rconn.Close()
_, err := rconn.Do("DEL", "worker1")
stalker.OnlyLogIf(err)
}
log.Debugln("Sanatizing DB")
rquery := r.Db(STALKERDB).Table("checks").Filter(r.Row.Field("pending").Eq(true))
rquery = rquery.Update(map[string]bool{"pending": false})
res, err := rquery.RunWrite(sm.rsess)
if err != nil {
log.Panic(err)
}
log.Debugln(res)
}