func (this *WatchActord) dueJobsWithin(topic string, timeSpan int64,
now time.Time) (backlog int64, archive int64) {
jobTable := jm.JobTable(topic)
appid := manager.Default.TopicAppid(topic)
aid := jm.App_id(appid)
sql := fmt.Sprintf("SELECT count(job_id) FROM %s WHERE due_time<=?", jobTable)
rows, err := this.mc.Query(jm.AppPool, jobTable, aid, sql, now.Unix()+timeSpan)
if err != nil {
log.Error("%s: %s", this.ident(), err)
return
}
var n int
for rows.Next() {
rows.Scan(&n)
}
rows.Close()
backlog += int64(n)
archiveTable := jm.HistoryTable(topic)
sql = fmt.Sprintf("SELECT count(job_id) FROM %s WHERE due_time>=?", archiveTable)
rows, err = this.mc.Query(jm.AppPool, archiveTable, aid, sql, now.Unix()-timeSpan)
if err != nil {
log.Error("%s: %s", this.ident(), err)
return
}
for rows.Next() {
rows.Scan(&n)
}
rows.Close()
archive += int64(n)
return
}
func (this *Job) connectMysqlCluster(appid string) int {
b, err := this.zkzone.KatewayJobClusterConfig()
if err != nil {
panic(err)
}
var mcc = &config.ConfigMysql{}
if err = mcc.From(b); err != nil {
panic(err)
}
this.mc = mysql.New(mcc)
return jm.App_id(appid)
}
// poll mysql for due jobs and send to kafka.
func (this *JobExecutor) Run() {
this.appid = manager.Default.TopicAppid(this.topic)
if this.appid == "" {
log.Warn("invalid topic: %s", this.topic)
return
}
this.aid = jm.App_id(this.appid)
this.table = jm.JobTable(this.topic)
this.ident = this.topic
log.Trace("starting %s", this.Ident())
var (
wg sync.WaitGroup
item job.JobItem
tick = time.NewTicker(time.Second)
sql = fmt.Sprintf("SELECT job_id,payload,ctime,due_time FROM %s WHERE due_time<=?", this.table)
)
for i := 0; i < HandlerConcurrentN; i++ {
wg.Add(1)
go this.handleDueJobs(&wg)
}
for {
select {
case <-this.stopper:
log.Debug("%s stopping", this.ident)
wg.Wait()
return
case now := <-tick.C:
rows, err := this.mc.Query(jm.AppPool, this.topic, this.aid, sql, now.Unix())
if err != nil {
log.Error("%s: %v", this.ident, err)
continue
}
for rows.Next() {
err = rows.Scan(&item.JobId, &item.Payload, &item.Ctime, &item.DueTime)
if err == nil {
log.Debug("%s due %s", this.ident, item)
if lag := now.Unix() - item.DueTime; lag > LagWarnThreshold {
log.Warn("%s lag %ds %s", this.ident, lag, item)
}
this.dueJobs <- item
} else {
log.Error("%s: %s", this.ident, err)
}
}
if err = rows.Err(); err != nil {
log.Error("%s: %s", this.ident, err)
}
rows.Close()
}
}
}