コード例 #1
0
ファイル: actord.go プロジェクト: funkygao/gafka
func (this *WatchActord) dueJobsWithin(topic string, timeSpan int64,
	now time.Time) (backlog int64, archive int64) {
	jobTable := jm.JobTable(topic)
	appid := manager.Default.TopicAppid(topic)
	aid := jm.App_id(appid)
	sql := fmt.Sprintf("SELECT count(job_id) FROM %s WHERE due_time<=?", jobTable)
	rows, err := this.mc.Query(jm.AppPool, jobTable, aid, sql, now.Unix()+timeSpan)
	if err != nil {
		log.Error("%s: %s", this.ident(), err)
		return
	}
	var n int
	for rows.Next() {
		rows.Scan(&n)
	}
	rows.Close()
	backlog += int64(n)

	archiveTable := jm.HistoryTable(topic)
	sql = fmt.Sprintf("SELECT count(job_id) FROM %s WHERE due_time>=?", archiveTable)
	rows, err = this.mc.Query(jm.AppPool, archiveTable, aid, sql, now.Unix()-timeSpan)
	if err != nil {
		log.Error("%s: %s", this.ident(), err)
		return
	}
	for rows.Next() {
		rows.Scan(&n)
	}
	rows.Close()
	archive += int64(n)

	return

}
コード例 #2
0
ファイル: job.go プロジェクト: funkygao/gafka
func (this *Job) connectMysqlCluster(appid string) int {
	b, err := this.zkzone.KatewayJobClusterConfig()
	if err != nil {
		panic(err)
	}
	var mcc = &config.ConfigMysql{}
	if err = mcc.From(b); err != nil {
		panic(err)
	}

	this.mc = mysql.New(mcc)
	return jm.App_id(appid)
}
コード例 #3
0
ファイル: exe_job.go プロジェクト: funkygao/gafka
// poll mysql for due jobs and send to kafka.
func (this *JobExecutor) Run() {
	this.appid = manager.Default.TopicAppid(this.topic)
	if this.appid == "" {
		log.Warn("invalid topic: %s", this.topic)
		return
	}
	this.aid = jm.App_id(this.appid)
	this.table = jm.JobTable(this.topic)
	this.ident = this.topic

	log.Trace("starting %s", this.Ident())

	var (
		wg   sync.WaitGroup
		item job.JobItem
		tick = time.NewTicker(time.Second)
		sql  = fmt.Sprintf("SELECT job_id,payload,ctime,due_time FROM %s WHERE due_time<=?", this.table)
	)

	for i := 0; i < HandlerConcurrentN; i++ {
		wg.Add(1)
		go this.handleDueJobs(&wg)
	}

	for {
		select {
		case <-this.stopper:
			log.Debug("%s stopping", this.ident)
			wg.Wait()
			return

		case now := <-tick.C:
			rows, err := this.mc.Query(jm.AppPool, this.topic, this.aid, sql, now.Unix())
			if err != nil {
				log.Error("%s: %v", this.ident, err)
				continue
			}

			for rows.Next() {
				err = rows.Scan(&item.JobId, &item.Payload, &item.Ctime, &item.DueTime)
				if err == nil {
					log.Debug("%s due %s", this.ident, item)
					if lag := now.Unix() - item.DueTime; lag > LagWarnThreshold {
						log.Warn("%s lag %ds %s", this.ident, lag, item)
					}

					this.dueJobs <- item
				} else {
					log.Error("%s: %s", this.ident, err)
				}
			}

			if err = rows.Err(); err != nil {
				log.Error("%s: %s", this.ident, err)
			}

			rows.Close()
		}
	}

}