func fetchRowColVals(txn kv.Transaction, t table.Table, handle int64, indexInfo *model.IndexInfo) (
kv.Key, []types.Datum, error) {
// fetch datas
cols := t.Cols()
colMap := make(map[int64]*types.FieldType)
for _, v := range indexInfo.Columns {
col := cols[v.Offset]
colMap[col.ID] = &col.FieldType
}
rowKey := tablecodec.EncodeRecordKey(t.RecordPrefix(), handle)
rowVal, err := txn.Get(rowKey)
if err != nil {
return nil, nil, errors.Trace(err)
}
row, err := tablecodec.DecodeRow(rowVal, colMap)
if err != nil {
return nil, nil, errors.Trace(err)
}
vals := make([]types.Datum, 0, len(indexInfo.Columns))
for _, v := range indexInfo.Columns {
col := cols[v.Offset]
vals = append(vals, row[col.ID])
}
return rowKey, vals, nil
}
// GetTableRecordsCount returns the total number of table records from startHandle.
// If startHandle = 0, returns the total number of table records.
func GetTableRecordsCount(txn kv.Transaction, t table.Table, startHandle int64) (int64, error) {
startKey := t.RecordKey(startHandle, nil)
it, err := txn.Seek(startKey)
if err != nil {
return 0, errors.Trace(err)
}
var cnt int64
prefix := t.RecordPrefix()
for it.Valid() && it.Key().HasPrefix(prefix) {
handle, err := tables.DecodeRecordKeyHandle(it.Key())
if err != nil {
return 0, errors.Trace(err)
}
it.Close()
rk := t.RecordKey(handle+1, nil)
it, err = txn.Seek(rk)
if err != nil {
return 0, errors.Trace(err)
}
cnt++
}
it.Close()
return cnt, nil
}
func iterRecords(retriever kv.Retriever, t table.Table, startKey kv.Key, cols []*table.Column,
fn table.RecordIterFunc) error {
it, err := retriever.Seek(startKey)
if err != nil {
return errors.Trace(err)
}
defer it.Close()
if !it.Valid() {
return nil
}
log.Debugf("startKey:%q, key:%q, value:%q", startKey, it.Key(), it.Value())
colMap := make(map[int64]*types.FieldType, len(cols))
for _, col := range cols {
colMap[col.ID] = &col.FieldType
}
prefix := t.RecordPrefix()
for it.Valid() && it.Key().HasPrefix(prefix) {
// first kv pair is row lock information.
// TODO: check valid lock
// get row handle
handle, err := tablecodec.DecodeRowKey(it.Key())
if err != nil {
return errors.Trace(err)
}
rowMap, err := tablecodec.DecodeRow(it.Value(), colMap)
if err != nil {
return errors.Trace(err)
}
data := make([]types.Datum, 0, len(cols))
for _, col := range cols {
if col.IsPKHandleColumn(t.Meta()) {
data = append(data, types.NewIntDatum(handle))
} else {
data = append(data, rowMap[col.ID])
}
}
more, err := fn(handle, data, cols)
if !more || err != nil {
return errors.Trace(err)
}
rk := t.RecordKey(handle)
err = kv.NextUntil(it, util.RowKeyPrefixFilter(rk))
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (d *ddl) dropTableData(t table.Table) error {
// delete table data
err := d.delKeysWithPrefix(t.RecordPrefix())
if err != nil {
return errors.Trace(err)
}
// delete table index
err = d.delKeysWithPrefix(t.IndexPrefix())
return errors.Trace(err)
}
func (d *ddl) backfillTableIndex(t table.Table, indexInfo *model.IndexInfo, handles []int64, reorgInfo *reorgInfo) error {
kvX := tables.NewIndex(t.Meta(), indexInfo)
for _, handle := range handles {
log.Debug("[ddl] building index...", handle)
err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err := d.isReorgRunnable(txn); err != nil {
return errors.Trace(err)
}
vals, err1 := fetchRowColVals(txn, t, handle, indexInfo)
if terror.ErrorEqual(err1, kv.ErrNotExist) {
// row doesn't exist, skip it.
return nil
}
if err1 != nil {
return errors.Trace(err1)
}
exist, _, err1 := kvX.Exist(txn, vals, handle)
if err1 != nil {
return errors.Trace(err1)
} else if exist {
// index already exists, skip it.
return nil
}
rowKey := tablecodec.EncodeRecordKey(t.RecordPrefix(), handle)
err1 = txn.LockKeys(rowKey)
if err1 != nil {
return errors.Trace(err1)
}
// create the index.
err1 = kvX.Create(txn, vals, handle)
if err1 != nil {
return errors.Trace(err1)
}
// update reorg next handle
return errors.Trace(reorgInfo.UpdateHandle(txn, handle))
})
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (d *ddl) getSnapshotRows(t table.Table, version uint64, seekHandle int64) ([]int64, error) {
ver := kv.Version{Ver: version}
snap, err := d.store.GetSnapshot(ver)
if err != nil {
return nil, errors.Trace(err)
}
defer snap.Release()
firstKey := t.RecordKey(seekHandle, nil)
it, err := snap.Seek(firstKey)
if err != nil {
return nil, errors.Trace(err)
}
defer it.Close()
handles := make([]int64, 0, maxBatchSize)
for it.Valid() {
if !it.Key().HasPrefix(t.RecordPrefix()) {
break
}
var handle int64
handle, err = tables.DecodeRecordKeyHandle(it.Key())
if err != nil {
return nil, errors.Trace(err)
}
rk := t.RecordKey(handle, nil)
handles = append(handles, handle)
if len(handles) == maxBatchSize {
seekHandle = handle + 1
break
}
err = kv.NextUntil(it, util.RowKeyPrefixFilter(rk))
if terror.ErrorEqual(err, kv.ErrNotExist) {
break
} else if err != nil {
return nil, errors.Trace(err)
}
}
return handles, nil
}
func (d *ddl) fetchRowColVals(txn kv.Transaction, t table.Table, handles []int64, indexInfo *model.IndexInfo) (
[]*indexRecord, error) {
// Through handles access to get all row keys.
handlesLen := len(handles)
rowKeys := make([]kv.Key, 0, handlesLen)
for _, h := range handles {
rowKey := tablecodec.EncodeRecordKey(t.RecordPrefix(), h)
rowKeys = append(rowKeys, rowKey)
}
// Get corresponding raw values for rowKeys.
ver := kv.Version{Ver: txn.StartTS()}
snap, err := d.store.GetSnapshot(ver)
if err != nil {
return nil, errors.Trace(err)
}
pairMap, err := snap.BatchGet(rowKeys)
if err != nil {
return nil, errors.Trace(err)
}
// Get corresponding values for pairMap.
cols := t.Cols()
colMap := make(map[int64]*types.FieldType)
for _, v := range indexInfo.Columns {
col := cols[v.Offset]
colMap[col.ID] = &col.FieldType
}
idxRecords := make([]*indexRecord, 0, handlesLen)
for i, rowKey := range rowKeys {
rawVal, ok := pairMap[string(rowKey)]
if !ok {
// Row doesn't exist, skip it.
continue
}
row, err := tablecodec.DecodeRow(rawVal, colMap)
if err != nil {
return nil, errors.Trace(err)
}
rowVal := make([]types.Datum, 0, len(indexInfo.Columns))
for _, v := range indexInfo.Columns {
col := cols[v.Offset]
rowVal = append(rowVal, row[col.ID])
}
idxRecord := &indexRecord{handle: handles[i], key: rowKey, vals: rowVal}
idxRecords = append(idxRecords, idxRecord)
}
return idxRecords, nil
}
func iterRecords(retriever kv.Retriever, t table.Table, startKey kv.Key, cols []*table.Column,
fn table.RecordIterFunc) error {
it, err := retriever.Seek(startKey)
if err != nil {
return errors.Trace(err)
}
defer it.Close()
if !it.Valid() {
return nil
}
log.Debugf("startKey:%q, key:%q, value:%q", startKey, it.Key(), it.Value())
prefix := t.RecordPrefix()
for it.Valid() && it.Key().HasPrefix(prefix) {
// first kv pair is row lock information.
// TODO: check valid lock
// get row handle
handle, err := tables.DecodeRecordKeyHandle(it.Key())
if err != nil {
return errors.Trace(err)
}
data, err := rowWithCols(retriever, t, handle, cols)
if err != nil {
return errors.Trace(err)
}
more, err := fn(handle, data, cols)
if !more || err != nil {
return errors.Trace(err)
}
rk := t.RecordKey(handle, nil)
err = kv.NextUntil(it, util.RowKeyPrefixFilter(rk))
if err != nil {
return errors.Trace(err)
}
}
return nil
}