func (d *ddl) dropTableColumn(t table.Table, colInfo *model.ColumnInfo, reorgInfo *reorgInfo) error {
version := reorgInfo.SnapshotVer
seekHandle := reorgInfo.Handle
col := &column.Col{ColumnInfo: *colInfo}
for {
handles, err := d.getSnapshotRows(t, version, seekHandle)
if err != nil {
return errors.Trace(err)
} else if len(handles) == 0 {
return nil
}
seekHandle = handles[len(handles)-1] + 1
err = kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err1 := d.isReorgRunnable(txn); err1 != nil {
return errors.Trace(err1)
}
var h int64
for _, h = range handles {
key := t.RecordKey(h, col)
err1 := txn.Delete(key)
if err1 != nil && !terror.ErrorEqual(err1, kv.ErrNotExist) {
return errors.Trace(err1)
}
}
return errors.Trace(reorgInfo.UpdateHandle(txn, h))
})
if err != nil {
return errors.Trace(err)
}
}
}
func checkRecordAndIndex(txn kv.Transaction, t table.Table, idx table.Index) error {
cols := make([]*table.Column, len(idx.Meta().Columns))
for i, col := range idx.Meta().Columns {
cols[i] = t.Cols()[col.Offset]
}
startKey := t.RecordKey(0, nil)
filterFunc := func(h1 int64, vals1 []types.Datum, cols []*table.Column) (bool, error) {
isExist, h2, err := idx.Exist(txn, vals1, h1)
if terror.ErrorEqual(err, kv.ErrKeyExists) {
record1 := &RecordData{Handle: h1, Values: vals1}
record2 := &RecordData{Handle: h2, Values: vals1}
return false, errDateNotEqual.Gen("index:%v != record:%v", record2, record1)
}
if err != nil {
return false, errors.Trace(err)
}
if !isExist {
record := &RecordData{Handle: h1, Values: vals1}
return false, errDateNotEqual.Gen("index:%v != record:%v", nil, record)
}
return true, nil
}
err := iterRecords(txn, t, startKey, cols, filterFunc)
if err != nil {
return errors.Trace(err)
}
return nil
}
func replaceRow(ctx context.Context, t table.Table, handle int64, replaceRow []interface{}) error {
row, err := t.Row(ctx, handle)
if err != nil {
return errors.Trace(err)
}
result := 0
isReplace := false
touched := make(map[int]bool, len(row))
for i, val := range row {
result, err = types.Compare(val, replaceRow[i])
if err != nil {
return errors.Trace(err)
}
if result != 0 {
touched[i] = true
isReplace = true
}
}
if isReplace {
variable.GetSessionVars(ctx).AddAffectedRows(1)
if err = t.UpdateRecord(ctx, handle, row, replaceRow, touched); err != nil {
return errors.Trace(err)
}
}
return nil
}
func (s *InsertValues) getRows(ctx context.Context, t table.Table, cols []*column.Col) (rows [][]interface{}, err error) {
// process `insert|replace ... set x=y...`
if err = s.fillValueList(); err != nil {
return nil, errors.Trace(err)
}
evalMap, err := s.getColumnDefaultValues(ctx, t.Cols())
if err != nil {
return nil, errors.Trace(err)
}
rows = make([][]interface{}, len(s.Lists))
for i, list := range s.Lists {
if err = s.checkValueCount(len(s.Lists[0]), len(list), i, cols); err != nil {
return nil, errors.Trace(err)
}
vals := make([]interface{}, len(list))
for j, expr := range list {
// For "insert into t values (default)" Default Eval.
evalMap[expression.ExprEvalDefaultName] = cols[j].Name.O
vals[j], err = expr.Eval(ctx, evalMap)
if err != nil {
return nil, errors.Trace(err)
}
}
rows[i], err = s.fillRowData(ctx, t, cols, vals)
if err != nil {
return nil, errors.Trace(err)
}
}
return
}
func (s *InsertValues) getRow(ctx context.Context, t table.Table, cols []*column.Col, list []expression.Expression, m map[interface{}]interface{}) ([]interface{}, error) {
r := make([]interface{}, len(t.Cols()))
marked := make(map[int]struct{}, len(list))
for i, expr := range list {
// For "insert into t values (default)" Default Eval.
m[expression.ExprEvalDefaultName] = cols[i].Name.O
val, err := expr.Eval(ctx, m)
if err != nil {
return nil, errors.Trace(err)
}
r[cols[i].Offset] = val
marked[cols[i].Offset] = struct{}{}
}
// Clear last insert id.
variable.GetSessionVars(ctx).SetLastInsertID(0)
err := s.initDefaultValues(ctx, t, r, marked)
if err != nil {
return nil, errors.Trace(err)
}
if err = column.CastValues(ctx, r, cols); err != nil {
return nil, errors.Trace(err)
}
if err = column.CheckNotNull(t.Cols(), r); err != nil {
return nil, errors.Trace(err)
}
return r, nil
}
func rowWithCols(txn kv.Retriever, t table.Table, h int64, cols []*table.Column) ([]types.Datum, error) {
v := make([]types.Datum, len(cols))
for i, col := range cols {
if col.State != model.StatePublic {
return nil, errInvalidColumnState.Gen("Cannot use none public column - %v", cols)
}
if col.IsPKHandleColumn(t.Meta()) {
v[i].SetInt64(h)
continue
}
k := t.RecordKey(h, col)
data, err := txn.Get(k)
if terror.ErrorEqual(err, kv.ErrNotExist) && !mysql.HasNotNullFlag(col.Flag) {
continue
} else if err != nil {
return nil, errors.Trace(err)
}
val, err := tables.DecodeValue(data, &col.FieldType)
if err != nil {
return nil, errors.Trace(err)
}
v[i] = val
}
return v, nil
}
func rowWithCols(txn kv.Retriever, t table.Table, h int64, cols []*column.Col) ([]types.Datum, error) {
v := make([]types.Datum, len(cols))
for i, col := range cols {
if col.State != model.StatePublic {
return nil, errors.Errorf("Cannot use none public column - %v", cols)
}
if col.IsPKHandleColumn(t.Meta()) {
v[i].SetInt64(h)
continue
}
k := t.RecordKey(h, col)
data, err := txn.Get(k)
if err != nil {
return nil, errors.Trace(err)
}
val, err := tables.DecodeValue(data, &col.FieldType)
if err != nil {
return nil, errors.Trace(err)
}
v[i] = val
}
return v, nil
}
func (s *testColumnSuite) checkColumnKVExist(ctx context.Context, t table.Table, handle int64, col *table.Column, columnValue interface{}, isExist bool) error {
txn, err := ctx.GetTxn(true)
if err != nil {
return errors.Trace(err)
}
defer ctx.CommitTxn()
key := t.RecordKey(handle)
data, err := txn.Get(key)
if !isExist {
if terror.ErrorEqual(err, kv.ErrNotExist) {
return nil
}
}
if err != nil {
return errors.Trace(err)
}
colMap := make(map[int64]*types.FieldType)
colMap[col.ID] = &col.FieldType
rowMap, err := tablecodec.DecodeRow(data, colMap)
if err != nil {
return errors.Trace(err)
}
val, ok := rowMap[col.ID]
if isExist {
if !ok || val.GetValue() != columnValue {
return errors.Errorf("%v is not equal to %v", val.GetValue(), columnValue)
}
} else {
if ok {
return errors.Errorf("column value should not exists")
}
}
return nil
}
func lockRow(txn kv.Transaction, t table.Table, h int64) error {
// Get row lock key
lockKey := t.RecordKey(h, nil)
// set row lock key to current txn
err := txn.Set(lockKey, []byte(txn.String()))
return errors.Trace(err)
}
func refillTable(tbl table.Table, rows [][]types.Datum) error {
err := tbl.Truncate(nil)
if err != nil {
return errors.Trace(err)
}
return insertData(tbl, rows)
}
func touchedMap(t table.Table) map[int]bool {
touched := make(map[int]bool)
for _, col := range t.Cols() {
touched[col.Offset] = true
}
return touched
}
func fetchRowColVals(txn kv.Transaction, t table.Table, handle int64, indexInfo *model.IndexInfo) (
kv.Key, []types.Datum, error) {
// fetch datas
cols := t.Cols()
colMap := make(map[int64]*types.FieldType)
for _, v := range indexInfo.Columns {
col := cols[v.Offset]
colMap[col.ID] = &col.FieldType
}
rowKey := tablecodec.EncodeRecordKey(t.RecordPrefix(), handle)
rowVal, err := txn.Get(rowKey)
if err != nil {
return nil, nil, errors.Trace(err)
}
row, err := tablecodec.DecodeRow(rowVal, colMap)
if err != nil {
return nil, nil, errors.Trace(err)
}
vals := make([]types.Datum, 0, len(indexInfo.Columns))
for _, v := range indexInfo.Columns {
col := cols[v.Offset]
vals = append(vals, row[col.ID])
}
return rowKey, vals, nil
}
func scanTableData(retriever kv.Retriever, t table.Table, cols []*table.Column, startHandle, limit int64) (
[]*RecordData, int64, error) {
var records []*RecordData
startKey := t.RecordKey(startHandle, nil)
filterFunc := func(h int64, d []types.Datum, cols []*table.Column) (bool, error) {
if limit != 0 {
r := &RecordData{
Handle: h,
Values: d,
}
records = append(records, r)
limit--
return true, nil
}
return false, nil
}
err := iterRecords(retriever, t, startKey, cols, filterFunc)
if err != nil {
return nil, 0, errors.Trace(err)
}
if len(records) == 0 {
return records, startHandle, nil
}
nextHandle := records[len(records)-1].Handle + 1
return records, nextHandle, nil
}
func getUpdateColumns(t table.Table, assignList []expression.Assignment, isMultipleTable bool, tblAliasMap map[string]string) ([]*column.Col, []expression.Assignment, error) {
// TODO: We should check the validate if assignList in somewhere else. Maybe in building plan.
// TODO: We should use field.GetFieldIndex to replace this function.
tcols := make([]*column.Col, 0, len(assignList))
tAsgns := make([]expression.Assignment, 0, len(assignList))
tname := t.TableName()
for _, asgn := range assignList {
if isMultipleTable {
if tblAliasMap != nil {
if alias, ok := tblAliasMap[asgn.TableName]; ok {
if !strings.EqualFold(tname.O, alias) {
continue
}
}
} else if !strings.EqualFold(tname.O, asgn.TableName) {
continue
}
}
col := column.FindCol(t.Cols(), asgn.ColName)
if col == nil {
if isMultipleTable {
continue
}
return nil, nil, errors.Errorf("UPDATE: unknown column %s", asgn.ColName)
}
tcols = append(tcols, col)
tAsgns = append(tAsgns, asgn)
}
return tcols, tAsgns, nil
}
func (s *testIndexSuite) testGetIndex(c *C, t table.Table, name string, isExist bool) {
index := t.FindIndexByColName(name)
if isExist {
c.Assert(index, NotNil)
} else {
c.Assert(index, IsNil)
}
}
func (s *testColumnSuite) testGetColumn(c *C, t table.Table, name string, isExist bool) {
col := column.FindCol(t.Cols(), name)
if isExist {
c.Assert(col, NotNil)
} else {
c.Assert(col, IsNil)
}
}
func (e *DeleteExec) removeRow(ctx context.Context, t table.Table, h int64, data []types.Datum) error {
err := t.RemoveRecord(ctx, h, data)
if err != nil {
return errors.Trace(err)
}
variable.GetSessionVars(ctx).AddAffectedRows(1)
return nil
}
func (d *ddl) backfillColumnData(t table.Table, columnInfo *model.ColumnInfo, handles []int64, reorgInfo *reorgInfo) error {
for _, handle := range handles {
log.Info("[ddl] backfill column...", handle)
err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err := d.isReorgRunnable(txn); err != nil {
return errors.Trace(err)
}
// First check if row exists.
exist, err := checkRowExist(txn, t, handle)
if err != nil {
return errors.Trace(err)
} else if !exist {
// If row doesn't exist, skip it.
return nil
}
backfillKey := t.RecordKey(handle, &column.Col{ColumnInfo: *columnInfo})
backfillValue, err := txn.Get(backfillKey)
if err != nil && !kv.IsErrNotFound(err) {
return errors.Trace(err)
}
if backfillValue != nil {
return nil
}
value, _, err := tables.GetColDefaultValue(nil, columnInfo)
if err != nil {
return errors.Trace(err)
}
// must convert to the column field type.
v, err := types.Convert(value, &columnInfo.FieldType)
if err != nil {
return errors.Trace(err)
}
err = lockRow(txn, t, handle)
if err != nil {
return errors.Trace(err)
}
err = t.SetColValue(txn, backfillKey, v)
if err != nil {
return errors.Trace(err)
}
return errors.Trace(reorgInfo.UpdateHandle(txn, handle))
})
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (d *ddl) backfillColumnData(t table.Table, columnInfo *model.ColumnInfo, handles []int64, reorgInfo *reorgInfo) error {
defaultVal, _, err := table.GetColDefaultValue(nil, columnInfo)
if err != nil {
return errors.Trace(err)
}
colMap := make(map[int64]*types.FieldType)
for _, col := range t.Meta().Columns {
colMap[col.ID] = &col.FieldType
}
for _, handle := range handles {
log.Info("[ddl] backfill column...", handle)
err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err := d.isReorgRunnable(txn); err != nil {
return errors.Trace(err)
}
rowKey := t.RecordKey(handle)
rowVal, err := txn.Get(rowKey)
if terror.ErrorEqual(err, kv.ErrNotExist) {
// If row doesn't exist, skip it.
return nil
}
if err != nil {
return errors.Trace(err)
}
rowColumns, err := tablecodec.DecodeRow(rowVal, colMap)
if err != nil {
return errors.Trace(err)
}
if _, ok := rowColumns[columnInfo.ID]; ok {
// The column is already added by update or insert statement, skip it.
return nil
}
newColumnIDs := make([]int64, 0, len(rowColumns)+1)
newRow := make([]types.Datum, 0, len(rowColumns)+1)
for colID, val := range rowColumns {
newColumnIDs = append(newColumnIDs, colID)
newRow = append(newRow, val)
}
newColumnIDs = append(newColumnIDs, columnInfo.ID)
newRow = append(newRow, defaultVal)
newRowVal, err := tablecodec.EncodeRow(newRow, newColumnIDs)
if err != nil {
return errors.Trace(err)
}
err = txn.Set(rowKey, newRowVal)
if err != nil {
return errors.Trace(err)
}
return errors.Trace(reorgInfo.UpdateHandle(txn, handle))
})
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func removeRow(ctx context.Context, t table.Table, h int64, data []interface{}) error {
err := t.RemoveRecord(ctx, h, data)
if err != nil {
return errors.Trace(err)
}
variable.GetSessionVars(ctx).AddAffectedRows(1)
return nil
}
func insertData(tbl table.Table, rows [][]types.Datum) error {
for _, r := range rows {
_, err := tbl.AddRecord(nil, r)
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (d *ddl) dropTableIndex(t table.Table, indexInfo *model.IndexInfo) error {
prefix := kv.GenIndexPrefix(t.IndexPrefix(), indexInfo.Name.L)
prefixBytes := []byte(prefix)
keys := make([]string, maxBatchSize)
for {
keys := keys[0:0]
err := kv.RunInNewTxn(d.store, false, func(txn kv.Transaction) error {
iter, err := txn.Seek(prefixBytes)
if err != nil {
return errors.Trace(err)
}
defer iter.Close()
for i := 0; i < maxBatchSize; i++ {
if iter.Valid() && strings.HasPrefix(iter.Key(), prefix) {
keys = append(keys, iter.Key())
err = iter.Next()
if err != nil {
return errors.Trace(err)
}
} else {
break
}
}
return nil
})
// if err or delete no keys, return.
if err != nil || len(keys) == 0 {
return errors.Trace(err)
}
// delete index key one by one
for _, key := range keys {
err = kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err := d.isReorgRunnable(txn); err != nil {
return errors.Trace(err)
}
err1 := txn.Delete([]byte(key))
// if key doesn't exist, skip this error.
if err1 != nil && !terror.ErrorEqual(err1, kv.ErrNotExist) {
return errors.Trace(err1)
}
return nil
})
if err != nil {
return errors.Trace(err)
}
}
}
}
func (e *DeleteExec) removeRow(ctx context.Context, t table.Table, h int64, data []types.Datum) error {
err := t.RemoveRecord(ctx, h, data)
if err != nil {
return errors.Trace(err)
}
getDirtyDB(ctx).deleteRow(t.Meta().ID, h)
ctx.GetSessionVars().StmtCtx.AddAffectedRows(1)
return nil
}
// Add a column into table
func (d *ddl) addColumn(ctx context.Context, schema model.CIStr, tbl table.Table, spec *AlterSpecification) error {
// Find position
cols := tbl.Cols()
position := len(cols)
name := spec.Column.Name
// Check column name duplicate
dc := column.FindCol(cols, name)
if dc != nil {
return errors.Errorf("Try to add a column with the same name of an already exists column.")
}
if spec.Position.Type == ColumnPositionFirst {
position = 0
} else if spec.Position.Type == ColumnPositionAfter {
// Find the mentioned column
c := column.FindCol(cols, spec.Position.RelativeColumn)
if c == nil {
return errors.Errorf("No such column: %v", name)
}
// insert position is after the mentioned column
position = c.Offset + 1
}
// TODO: Set constraint
col, _, err := d.buildColumnAndConstraint(position, spec.Column)
if err != nil {
return errors.Trace(err)
}
// insert col into the right place of the column list
newCols := make([]*column.Col, 0, len(cols)+1)
newCols = append(newCols, cols[:position]...)
newCols = append(newCols, col)
newCols = append(newCols, cols[position:]...)
// adjust position
if position != len(cols) {
offsetChange := make(map[int]int)
for i := position + 1; i < len(newCols); i++ {
offsetChange[newCols[i].Offset] = i
newCols[i].Offset = i
}
// Update index offset info
for _, idx := range tbl.Indices() {
for _, c := range idx.Columns {
newOffset, ok := offsetChange[c.Offset]
if ok {
c.Offset = newOffset
}
}
}
}
tb := tbl.(*tables.Table)
tb.Columns = newCols
// TODO: update index
// TODO: update default value
// update infomation schema
err = d.updateInfoSchema(ctx, schema, tb.Meta())
return errors.Trace(err)
}
func (d *ddl) backfillTableIndex(t table.Table, indexInfo *model.IndexInfo, handles []int64, reorgInfo *reorgInfo) error {
kvX := kv.NewKVIndex(t.IndexPrefix(), indexInfo.Name.L, indexInfo.ID, indexInfo.Unique)
for _, handle := range handles {
log.Debug("[ddl] building index...", handle)
err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err := d.isReorgRunnable(txn); err != nil {
return errors.Trace(err)
}
// first check row exists
exist, err := checkRowExist(txn, t, handle)
if err != nil {
return errors.Trace(err)
} else if !exist {
// row doesn't exist, skip it.
return nil
}
var vals []interface{}
vals, err = fetchRowColVals(txn, t, handle, indexInfo)
if err != nil {
return errors.Trace(err)
}
exist, _, err = kvX.Exist(txn, vals, handle)
if err != nil {
return errors.Trace(err)
} else if exist {
// index already exists, skip it.
return nil
}
err = lockRow(txn, t, handle)
if err != nil {
return errors.Trace(err)
}
// create the index.
err = kvX.Create(txn, vals, handle)
if err != nil {
return errors.Trace(err)
}
// update reorg next handle
return errors.Trace(reorgInfo.UpdateHandle(txn, handle))
})
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func getIndex(t table.Table, name string) *column.IndexedCol {
for _, idx := range t.Indices() {
// only public index can be read.
if len(idx.Columns) == 1 && strings.EqualFold(idx.Columns[0].Name.L, name) {
return idx
}
}
return nil
}
func findColumnByName(t table.Table, tableName, colName string) (*table.Column, error) {
if len(tableName) > 0 && tableName != t.Meta().Name.O {
return nil, errors.Errorf("unknown field %s.%s", tableName, colName)
}
c := table.FindCol(t.Cols(), colName)
if c == nil {
return nil, errors.Errorf("unknown field %s", colName)
}
return c, nil
}
func (s *DeleteStmt) removeRow(ctx context.Context, t table.Table, h int64, data []interface{}) error {
// remove row's all indexies
if err := t.RemoveRowAllIndex(ctx, h, data); err != nil {
return err
}
// remove row
if err := t.RemoveRow(ctx, h); err != nil {
return err
}
variable.GetSessionVars(ctx).AddAffectedRows(1)
return nil
}
func (d *ddl) dropTableData(t table.Table) error {
// delete table data
err := d.delKeysWithPrefix(t.RecordPrefix())
if err != nil {
return errors.Trace(err)
}
// delete table index
err = d.delKeysWithPrefix(t.IndexPrefix())
return errors.Trace(err)
}
func findColumnByName(t table.Table, name string) (*column.Col, error) {
_, tableName, colName := field.SplitQualifiedName(name)
if len(tableName) > 0 && tableName != t.TableName().O {
return nil, errors.Errorf("unknown field %s.%s", tableName, colName)
}
c := column.FindCol(t.Cols(), colName)
if c == nil {
return nil, errors.Errorf("unknown field %s", colName)
}
return c, nil
}