// How to add index in reorganization state?
// 1. Generate a snapshot with special version.
// 2. Traverse the snapshot, get every row in the table.
// 3. For one row, if the row has been already deleted, skip to next row.
// 4. If not deleted, check whether index has existed, if existed, skip to next row.
// 5. If index doesn't exist, create the index and then continue to handle next row.
func (d *ddl) addTableIndex(t table.Table, indexInfo *model.IndexInfo, reorgInfo *reorgInfo, job *model.Job) error {
seekHandle := reorgInfo.Handle
version := reorgInfo.SnapshotVer
count := job.GetRowCount()
for {
startTS := time.Now()
handles, err := d.getSnapshotRows(t, version, seekHandle)
if err != nil {
return errors.Trace(err)
} else if len(handles) == 0 {
return nil
}
count += int64(len(handles))
seekHandle = handles[len(handles)-1] + 1
err = d.backfillTableIndex(t, indexInfo, handles, reorgInfo)
sub := time.Since(startTS).Seconds()
if err != nil {
log.Warnf("[ddl] added index for %v rows failed, take time %v", count, sub)
return errors.Trace(err)
}
job.SetRowCount(count)
batchHandleDataHistogram.WithLabelValues(batchAddIdx).Observe(sub)
log.Infof("[ddl] added index for %v rows, take time %v", count, sub)
}
}
// delKeysWithStartKey deletes keys with start key in a limited number. If limit < 0, deletes all keys.
// It returns the number of rows deleted, next start key and the error.
func (d *ddl) delKeysWithStartKey(prefix, startKey kv.Key, jobType JobType, job *model.Job, limit int) (int, kv.Key, error) {
limitedDel := limit >= 0
var count int
total := job.GetRowCount()
keys := make([]kv.Key, 0, defaultBatchSize)
for {
if limitedDel && count >= limit {
break
}
batch := defaultBatchSize
if limitedDel && count+batch > limit {
batch = limit - count
}
startTS := time.Now()
err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err1 := d.isReorgRunnable(txn, jobType); err1 != nil {
return errors.Trace(err1)
}
iter, err := txn.Seek(startKey)
if err != nil {
return errors.Trace(err)
}
defer iter.Close()
for i := 0; i < batch; i++ {
if iter.Valid() && iter.Key().HasPrefix(prefix) {
keys = append(keys, iter.Key().Clone())
err = iter.Next()
if err != nil {
return errors.Trace(err)
}
} else {
break
}
}
for _, key := range keys {
err := txn.Delete(key)
// must skip ErrNotExist
// if key doesn't exist, skip this error.
if err != nil && !terror.ErrorEqual(err, kv.ErrNotExist) {
return errors.Trace(err)
}
}
count += len(keys)
total += int64(len(keys))
return nil
})
sub := time.Since(startTS).Seconds()
if err != nil {
log.Warnf("[ddl] deleted %d keys failed, take time %v, deleted %d keys in total", len(keys), sub, total)
return 0, startKey, errors.Trace(err)
}
job.SetRowCount(total)
batchHandleDataHistogram.WithLabelValues(batchDelData).Observe(sub)
log.Infof("[ddl] deleted %d keys take time %v, deleted %d keys in total", len(keys), sub, total)
if len(keys) > 0 {
startKey = keys[len(keys)-1]
}
if noMoreKeysToDelete := len(keys) < batch; noMoreKeysToDelete {
break
}
keys = keys[:0]
}
return count, startKey, nil
}