func (d *ddl) handleJobQueue() error {
for {
if d.isClosed() {
return nil
}
waitTime := 2 * d.lease
var job *model.Job
err := kv.RunInNewTxn(d.store, false, func(txn kv.Transaction) error {
t := meta.NewMeta(txn)
owner, err := d.checkOwner(t)
if terror.ErrorEqual(err, ErrNotOwner) {
// we are not owner, return and retry checking later.
return nil
} else if err != nil {
return errors.Trace(err)
}
// become the owner
// get the first job and run
job, err = d.getFirstJob(t)
if job == nil || err != nil {
return errors.Trace(err)
}
if job.IsRunning() {
// if we enter a new state, crash when waiting 2 * lease time, and restart quickly,
// we may run the job immediately again, but we don't wait enough 2 * lease time to
// let other servers update the schema.
// so here we must check the elapsed time from last update, if < 2 * lease, we must
// wait again.
elapsed := time.Duration(time.Now().UnixNano() - job.LastUpdateTS)
if elapsed > 0 && elapsed < waitTime {
log.Warnf("the elapsed time from last update is %s < %s, wait again", elapsed, waitTime)
waitTime -= elapsed
return nil
}
}
log.Warnf("run DDL job %v", job)
d.hook.OnJobRunBefore(job)
// if run job meets error, we will save this error in job Error
// and retry later if the job is not cancelled.
d.runJob(t, job)
if job.IsFinished() {
err = d.finishJob(t, job)
} else {
err = d.updateJob(t, job)
}
if err != nil {
return errors.Trace(err)
}
// running job may cost some time, so here we must update owner status to
// prevent other become the owner.
owner.LastUpdateTS = time.Now().UnixNano()
if err = t.SetDDLOwner(owner); err != nil {
return errors.Trace(err)
}
return errors.Trace(err)
})
if err != nil {
return errors.Trace(err)
} else if job == nil {
// no job now, return and retry get later.
return nil
}
d.hook.OnJobUpdated(job)
// here means the job enters another state (delete only, write only, public, etc...) or is cancelled.
// if the job is done or still running, we will wait 2 * lease time to guarantee other servers to update
// the newest schema.
if job.State == model.JobRunning || job.State == model.JobDone {
d.waitSchemaChanged(waitTime)
}
if job.IsFinished() {
asyncNotify(d.jobDoneCh)
}
}
}