예제 #1
0
파일: actord.go 프로젝트: funkygao/gafka
func (this *WatchActord) dueJobsWithin(topic string, timeSpan int64,
	now time.Time) (backlog int64, archive int64) {
	jobTable := jm.JobTable(topic)
	appid := manager.Default.TopicAppid(topic)
	aid := jm.App_id(appid)
	sql := fmt.Sprintf("SELECT count(job_id) FROM %s WHERE due_time<=?", jobTable)
	rows, err := this.mc.Query(jm.AppPool, jobTable, aid, sql, now.Unix()+timeSpan)
	if err != nil {
		log.Error("%s: %s", this.ident(), err)
		return
	}
	var n int
	for rows.Next() {
		rows.Scan(&n)
	}
	rows.Close()
	backlog += int64(n)

	archiveTable := jm.HistoryTable(topic)
	sql = fmt.Sprintf("SELECT count(job_id) FROM %s WHERE due_time>=?", archiveTable)
	rows, err = this.mc.Query(jm.AppPool, archiveTable, aid, sql, now.Unix()-timeSpan)
	if err != nil {
		log.Error("%s: %s", this.ident(), err)
		return
	}
	for rows.Next() {
		rows.Scan(&n)
	}
	rows.Close()
	archive += int64(n)

	return

}
예제 #2
0
파일: job.go 프로젝트: funkygao/gafka
// TODO diagnose all app's jobs status
func (this *Job) displayAppJobs(appid string) {
	aid := this.connectMysqlCluster(appid)
	lines := make([]string, 0)
	header := "Topic|Type|JobId|Due|Payload"
	lines = append(lines, header)

	// FIXME does not respect appid, show all now
	this.forSortedJobQueues(func(topic string) {
		table := jm.JobTable(topic)
		archiveTable := jm.HistoryTable(topic)

		sqlRealTime := fmt.Sprintf("SELECT job_id,payload,due_time FROM %s ORDER BY due_time DESC", table)
		if this.due > 0 {
			sqlRealTime = fmt.Sprintf("SELECT job_id,payload,due_time FROM %s WHERE due_time<=? ORDER BY due_time DESC",
				table, time.Now().Unix()+int64(this.due))
		}
		rows, err := this.mc.Query(jm.AppPool, table, aid, sqlRealTime)
		swallow(err)

		var item job.JobItem
		for rows.Next() {
			err = rows.Scan(&item.JobId, &item.Payload, &item.DueTime)
			swallow(err)
			lines = append(lines, fmt.Sprintf("%s|RT|%d|%d|%s", topic, item.JobId, item.DueTime, item.PayloadString(50)))
		}
		rows.Close()

		sqlArchive := fmt.Sprintf("SELECT job_id,payload,due_time FROM %s ORDER BY due_time ASC LIMIT 100", archiveTable)
		rows, err = this.mc.Query(jm.AppPool, table, aid, sqlArchive)
		swallow(err)

		for rows.Next() {
			err = rows.Scan(&item.JobId, &item.Payload, &item.DueTime)
			swallow(err)
			lines = append(lines, fmt.Sprintf("%s|AR|%d|%d|%s", topic, item.JobId, item.DueTime, item.PayloadString(50)))
		}
		rows.Close()
	})

	if len(lines) > 1 {
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}
예제 #3
0
파일: exe_job.go 프로젝트: funkygao/gafka
// poll mysql for due jobs and send to kafka.
func (this *JobExecutor) Run() {
	this.appid = manager.Default.TopicAppid(this.topic)
	if this.appid == "" {
		log.Warn("invalid topic: %s", this.topic)
		return
	}
	this.aid = jm.App_id(this.appid)
	this.table = jm.JobTable(this.topic)
	this.ident = this.topic

	log.Trace("starting %s", this.Ident())

	var (
		wg   sync.WaitGroup
		item job.JobItem
		tick = time.NewTicker(time.Second)
		sql  = fmt.Sprintf("SELECT job_id,payload,ctime,due_time FROM %s WHERE due_time<=?", this.table)
	)

	for i := 0; i < HandlerConcurrentN; i++ {
		wg.Add(1)
		go this.handleDueJobs(&wg)
	}

	for {
		select {
		case <-this.stopper:
			log.Debug("%s stopping", this.ident)
			wg.Wait()
			return

		case now := <-tick.C:
			rows, err := this.mc.Query(jm.AppPool, this.topic, this.aid, sql, now.Unix())
			if err != nil {
				log.Error("%s: %v", this.ident, err)
				continue
			}

			for rows.Next() {
				err = rows.Scan(&item.JobId, &item.Payload, &item.Ctime, &item.DueTime)
				if err == nil {
					log.Debug("%s due %s", this.ident, item)
					if lag := now.Unix() - item.DueTime; lag > LagWarnThreshold {
						log.Warn("%s lag %ds %s", this.ident, lag, item)
					}

					this.dueJobs <- item
				} else {
					log.Error("%s: %s", this.ident, err)
				}
			}

			if err = rows.Err(); err != nil {
				log.Error("%s: %s", this.ident, err)
			}

			rows.Close()
		}
	}

}