func (s *testColumnSuite) checkColumnKVExist(ctx context.Context, t table.Table, handle int64, col *table.Column, columnValue interface{}, isExist bool) error {
txn, err := ctx.GetTxn(true)
if err != nil {
return errors.Trace(err)
}
defer ctx.CommitTxn()
key := t.RecordKey(handle)
data, err := txn.Get(key)
if !isExist {
if terror.ErrorEqual(err, kv.ErrNotExist) {
return nil
}
}
if err != nil {
return errors.Trace(err)
}
colMap := make(map[int64]*types.FieldType)
colMap[col.ID] = &col.FieldType
rowMap, err := tablecodec.DecodeRow(data, colMap)
if err != nil {
return errors.Trace(err)
}
val, ok := rowMap[col.ID]
if isExist {
if !ok || val.GetValue() != columnValue {
return errors.Errorf("%v is not equal to %v", val.GetValue(), columnValue)
}
} else {
if ok {
return errors.Errorf("column value should not exists")
}
}
return nil
}
func fetchRowColVals(txn kv.Transaction, t table.Table, handle int64, indexInfo *model.IndexInfo) (
kv.Key, []types.Datum, error) {
// fetch datas
cols := t.Cols()
colMap := make(map[int64]*types.FieldType)
for _, v := range indexInfo.Columns {
col := cols[v.Offset]
colMap[col.ID] = &col.FieldType
}
rowKey := tablecodec.EncodeRecordKey(t.RecordPrefix(), handle)
rowVal, err := txn.Get(rowKey)
if err != nil {
return nil, nil, errors.Trace(err)
}
row, err := tablecodec.DecodeRow(rowVal, colMap)
if err != nil {
return nil, nil, errors.Trace(err)
}
vals := make([]types.Datum, 0, len(indexInfo.Columns))
for _, v := range indexInfo.Columns {
col := cols[v.Offset]
vals = append(vals, row[col.ID])
}
return rowKey, vals, nil
}
func (d *ddl) backfillColumnData(t table.Table, columnInfo *model.ColumnInfo, handles []int64, reorgInfo *reorgInfo) error {
defaultVal, _, err := table.GetColDefaultValue(nil, columnInfo)
if err != nil {
return errors.Trace(err)
}
colMap := make(map[int64]*types.FieldType)
for _, col := range t.Meta().Columns {
colMap[col.ID] = &col.FieldType
}
for _, handle := range handles {
log.Info("[ddl] backfill column...", handle)
err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err := d.isReorgRunnable(txn); err != nil {
return errors.Trace(err)
}
rowKey := t.RecordKey(handle)
rowVal, err := txn.Get(rowKey)
if terror.ErrorEqual(err, kv.ErrNotExist) {
// If row doesn't exist, skip it.
return nil
}
if err != nil {
return errors.Trace(err)
}
rowColumns, err := tablecodec.DecodeRow(rowVal, colMap)
if err != nil {
return errors.Trace(err)
}
if _, ok := rowColumns[columnInfo.ID]; ok {
// The column is already added by update or insert statement, skip it.
return nil
}
newColumnIDs := make([]int64, 0, len(rowColumns)+1)
newRow := make([]types.Datum, 0, len(rowColumns)+1)
for colID, val := range rowColumns {
newColumnIDs = append(newColumnIDs, colID)
newRow = append(newRow, val)
}
newColumnIDs = append(newColumnIDs, columnInfo.ID)
newRow = append(newRow, defaultVal)
newRowVal, err := tablecodec.EncodeRow(newRow, newColumnIDs)
if err != nil {
return errors.Trace(err)
}
err = txn.Set(rowKey, newRowVal)
if err != nil {
return errors.Trace(err)
}
return errors.Trace(reorgInfo.UpdateHandle(txn, handle))
})
if err != nil {
return errors.Trace(err)
}
}
return nil
}
// IterRecords implements table.Table IterRecords interface.
func (t *Table) IterRecords(ctx context.Context, startKey kv.Key, cols []*table.Column,
fn table.RecordIterFunc) error {
txn, err := ctx.GetTxn(false)
if err != nil {
return errors.Trace(err)
}
it, err := txn.Seek(startKey)
if err != nil {
return errors.Trace(err)
}
defer it.Close()
if !it.Valid() {
return nil
}
log.Debugf("startKey:%q, key:%q, value:%q", startKey, it.Key(), it.Value())
colMap := make(map[int64]*types.FieldType)
for _, col := range cols {
colMap[col.ID] = &col.FieldType
}
prefix := t.RecordPrefix()
for it.Valid() && it.Key().HasPrefix(prefix) {
// first kv pair is row lock information.
// TODO: check valid lock
// get row handle
handle, err := tablecodec.DecodeRowKey(it.Key())
if err != nil {
return errors.Trace(err)
}
rowMap, err := tablecodec.DecodeRow(it.Value(), colMap)
if err != nil {
return errors.Trace(err)
}
data := make([]types.Datum, 0, len(cols))
for _, col := range cols {
if col.IsPKHandleColumn(t.Meta()) {
data = append(data, types.NewIntDatum(handle))
} else {
data = append(data, rowMap[col.ID])
}
}
more, err := fn(handle, data, cols)
if !more || err != nil {
return errors.Trace(err)
}
rk := t.RecordKey(handle)
err = kv.NextUntil(it, util.RowKeyPrefixFilter(rk))
if err != nil {
return errors.Trace(err)
}
}
return nil
}
// RowWithCols implements table.Table RowWithCols interface.
func (t *Table) RowWithCols(ctx context.Context, h int64, cols []*table.Column) ([]types.Datum, error) {
txn, err := ctx.GetTxn(false)
if err != nil {
return nil, errors.Trace(err)
}
// Get raw row data from kv.
key := t.RecordKey(h)
value, err := txn.Get(key)
if err != nil {
return nil, errors.Trace(err)
}
// Decode raw row data.
v := make([]types.Datum, len(cols))
colTps := make(map[int64]*types.FieldType, len(cols))
for i, col := range cols {
if col == nil {
continue
}
if col.State != model.StatePublic {
return nil, table.ErrColumnStateNonPublic.Gen("Cannot use none public column - %v", cols)
}
if col.IsPKHandleColumn(t.meta) {
if mysql.HasUnsignedFlag(col.Flag) {
v[i].SetUint64(uint64(h))
} else {
v[i].SetInt64(h)
}
continue
}
colTps[col.ID] = &col.FieldType
}
row, err := tablecodec.DecodeRow(value, colTps)
if err != nil {
return nil, errors.Trace(err)
}
for i, col := range cols {
if col == nil {
continue
}
if col.State != model.StatePublic {
// TODO: check this
return nil, table.ErrColumnStateNonPublic.Gen("Cannot use none public column - %v", cols)
}
if col.IsPKHandleColumn(t.meta) {
continue
}
ri, ok := row[col.ID]
if !ok && mysql.HasNotNullFlag(col.Flag) {
return nil, errors.New("Miss column")
}
v[i] = ri
}
return v, nil
}
func (d *ddl) fetchRowColVals(txn kv.Transaction, t table.Table, handles []int64, indexInfo *model.IndexInfo) (
[]*indexRecord, error) {
// Through handles access to get all row keys.
handlesLen := len(handles)
rowKeys := make([]kv.Key, 0, handlesLen)
for _, h := range handles {
rowKey := tablecodec.EncodeRecordKey(t.RecordPrefix(), h)
rowKeys = append(rowKeys, rowKey)
}
// Get corresponding raw values for rowKeys.
ver := kv.Version{Ver: txn.StartTS()}
snap, err := d.store.GetSnapshot(ver)
if err != nil {
return nil, errors.Trace(err)
}
pairMap, err := snap.BatchGet(rowKeys)
if err != nil {
return nil, errors.Trace(err)
}
// Get corresponding values for pairMap.
cols := t.Cols()
colMap := make(map[int64]*types.FieldType)
for _, v := range indexInfo.Columns {
col := cols[v.Offset]
colMap[col.ID] = &col.FieldType
}
idxRecords := make([]*indexRecord, 0, handlesLen)
for i, rowKey := range rowKeys {
rawVal, ok := pairMap[string(rowKey)]
if !ok {
// Row doesn't exist, skip it.
continue
}
row, err := tablecodec.DecodeRow(rawVal, colMap)
if err != nil {
return nil, errors.Trace(err)
}
rowVal := make([]types.Datum, 0, len(indexInfo.Columns))
for _, v := range indexInfo.Columns {
col := cols[v.Offset]
rowVal = append(rowVal, row[col.ID])
}
idxRecord := &indexRecord{handle: handles[i], key: rowKey, vals: rowVal}
idxRecords = append(idxRecords, idxRecord)
}
return idxRecords, nil
}
func (h *rpcHandler) getRowData(value []byte, colTps map[int64]*types.FieldType) (map[int64][]byte, error) {
values, err := tablecodec.DecodeRow(value, colTps)
if err != nil {
return nil, errors.Trace(err)
}
ret := make(map[int64][]byte, len(colTps))
for id, value := range values {
v, err1 := tablecodec.EncodeValue(value)
if err1 != nil {
return nil, errors.Trace(err1)
}
ret[id] = v
}
return ret, nil
}
func rowWithCols(txn kv.Retriever, t table.Table, h int64, cols []*table.Column) ([]types.Datum, error) {
key := t.RecordKey(h)
value, err := txn.Get(key)
if err != nil {
return nil, errors.Trace(err)
}
v := make([]types.Datum, len(cols))
colTps := make(map[int64]*types.FieldType, len(cols))
for i, col := range cols {
if col == nil {
continue
}
if col.State != model.StatePublic {
return nil, errInvalidColumnState.Gen("Cannot use none public column - %v", cols)
}
if col.IsPKHandleColumn(t.Meta()) {
if mysql.HasUnsignedFlag(col.Flag) {
v[i].SetUint64(uint64(h))
} else {
v[i].SetInt64(h)
}
continue
}
colTps[col.ID] = &col.FieldType
}
row, err := tablecodec.DecodeRow(value, colTps)
if err != nil {
return nil, errors.Trace(err)
}
for i, col := range cols {
if col == nil {
continue
}
if col.State != model.StatePublic {
// TODO: check this
return nil, errInvalidColumnState.Gen("Cannot use none public column - %v", cols)
}
if col.IsPKHandleColumn(t.Meta()) {
continue
}
ri, ok := row[col.ID]
if !ok && mysql.HasNotNullFlag(col.Flag) {
return nil, errors.New("Miss")
}
v[i] = ri
}
return v, nil
}
// backfillColumnInTxn deals with a part of backfilling column data in a Transaction.
// This part of the column data rows is defaultSmallBatchCnt.
func (d *ddl) backfillColumnInTxn(t table.Table, colID int64, handles []int64, colMap map[int64]*types.FieldType,
defaultVal types.Datum, txn kv.Transaction) (int64, error) {
nextHandle := handles[0]
for _, handle := range handles {
log.Debug("[ddl] backfill column...", handle)
rowKey := t.RecordKey(handle)
rowVal, err := txn.Get(rowKey)
if terror.ErrorEqual(err, kv.ErrNotExist) {
// If row doesn't exist, skip it.
continue
}
if err != nil {
return 0, errors.Trace(err)
}
rowColumns, err := tablecodec.DecodeRow(rowVal, colMap)
if err != nil {
return 0, errors.Trace(err)
}
if _, ok := rowColumns[colID]; ok {
// The column is already added by update or insert statement, skip it.
continue
}
newColumnIDs := make([]int64, 0, len(rowColumns)+1)
newRow := make([]types.Datum, 0, len(rowColumns)+1)
for colID, val := range rowColumns {
newColumnIDs = append(newColumnIDs, colID)
newRow = append(newRow, val)
}
newColumnIDs = append(newColumnIDs, colID)
newRow = append(newRow, defaultVal)
newRowVal, err := tablecodec.EncodeRow(newRow, newColumnIDs)
if err != nil {
return 0, errors.Trace(err)
}
err = txn.Set(rowKey, newRowVal)
if err != nil {
return 0, errors.Trace(err)
}
}
return nextHandle, nil
}