func (d *ddl) backfillColumnData(t table.Table, columnInfo *model.ColumnInfo, handles []int64, reorgInfo *reorgInfo) error {
for _, handle := range handles {
log.Info("[ddl] backfill column...", handle)
err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err := d.isReorgRunnable(txn); err != nil {
return errors.Trace(err)
}
// First check if row exists.
exist, err := checkRowExist(txn, t, handle)
if err != nil {
return errors.Trace(err)
} else if !exist {
// If row doesn't exist, skip it.
return nil
}
backfillKey := t.RecordKey(handle, &table.Column{ColumnInfo: *columnInfo})
backfillValue, err := txn.Get(backfillKey)
if err != nil && !kv.IsErrNotFound(err) {
return errors.Trace(err)
}
if backfillValue != nil {
return nil
}
value, _, err := table.GetColDefaultValue(nil, columnInfo)
if err != nil {
return errors.Trace(err)
}
// must convert to the column field type.
v, err := value.ConvertTo(&columnInfo.FieldType)
if err != nil {
return errors.Trace(err)
}
err = lockRow(txn, t, handle)
if err != nil {
return errors.Trace(err)
}
err = tables.SetColValue(txn, backfillKey, v)
if err != nil {
return errors.Trace(err)
}
return errors.Trace(reorgInfo.UpdateHandle(txn, handle))
})
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (d *ddl) backfillColumnData(t table.Table, columnInfo *model.ColumnInfo, handles []int64, reorgInfo *reorgInfo) error {
defaultVal, _, err := table.GetColDefaultValue(nil, columnInfo)
if err != nil {
return errors.Trace(err)
}
colMap := make(map[int64]*types.FieldType)
for _, col := range t.Meta().Columns {
colMap[col.ID] = &col.FieldType
}
for _, handle := range handles {
log.Info("[ddl] backfill column...", handle)
err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err := d.isReorgRunnable(txn); err != nil {
return errors.Trace(err)
}
rowKey := t.RecordKey(handle)
rowVal, err := txn.Get(rowKey)
if terror.ErrorEqual(err, kv.ErrNotExist) {
// If row doesn't exist, skip it.
return nil
}
if err != nil {
return errors.Trace(err)
}
rowColumns, err := tablecodec.DecodeRow(rowVal, colMap)
if err != nil {
return errors.Trace(err)
}
if _, ok := rowColumns[columnInfo.ID]; ok {
// The column is already added by update or insert statement, skip it.
return nil
}
newColumnIDs := make([]int64, 0, len(rowColumns)+1)
newRow := make([]types.Datum, 0, len(rowColumns)+1)
for colID, val := range rowColumns {
newColumnIDs = append(newColumnIDs, colID)
newRow = append(newRow, val)
}
newColumnIDs = append(newColumnIDs, columnInfo.ID)
newRow = append(newRow, defaultVal)
newRowVal, err := tablecodec.EncodeRow(newRow, newColumnIDs)
if err != nil {
return errors.Trace(err)
}
err = txn.Set(rowKey, newRowVal)
if err != nil {
return errors.Trace(err)
}
return errors.Trace(reorgInfo.UpdateHandle(txn, handle))
})
if err != nil {
return errors.Trace(err)
}
}
return nil
}
// UpdateRecord implements table.Table UpdateRecord interface.
func (t *Table) UpdateRecord(ctx context.Context, h int64, oldData []types.Datum, newData []types.Datum, touched map[int]bool) error {
// We should check whether this table has on update column which state is write only.
currentData := make([]types.Datum, len(t.WritableCols()))
copy(currentData, newData)
// If they are not set, and other data are changed, they will be updated by current timestamp too.
err := t.setOnUpdateData(ctx, touched, currentData)
if err != nil {
return errors.Trace(err)
}
txn, err := ctx.GetTxn(false)
if err != nil {
return errors.Trace(err)
}
bs := kv.NewBufferStore(txn)
// Compose new row
t.composeNewData(touched, currentData, oldData)
colIDs := make([]int64, 0, len(t.WritableCols()))
for i, col := range t.WritableCols() {
if col.State != model.StatePublic && currentData[i].IsNull() {
defaultVal, _, err1 := table.GetColDefaultValue(ctx, col.ToInfo())
if err1 != nil {
return errors.Trace(err1)
}
currentData[i] = defaultVal
}
colIDs = append(colIDs, col.ID)
}
// Set new row data into KV.
key := t.RecordKey(h)
value, err := tablecodec.EncodeRow(currentData, colIDs)
if err = txn.Set(key, value); err != nil {
return errors.Trace(err)
}
if err = bs.SaveTo(txn); err != nil {
return errors.Trace(err)
}
// rebuild index
if err = t.rebuildIndices(bs, h, touched, oldData, currentData); err != nil {
return errors.Trace(err)
}
err = bs.SaveTo(txn)
if err != nil {
return errors.Trace(err)
}
if shouldWriteBinlog(ctx) {
t.addUpdateBinlog(ctx, h, oldData, value, colIDs)
}
return nil
}
func (e *InsertValues) getColumnDefaultValues(cols []*table.Column) (map[string]types.Datum, error) {
defaultValMap := map[string]types.Datum{}
for _, col := range cols {
if value, ok, err := table.GetColDefaultValue(e.ctx, col.ToInfo()); ok {
if err != nil {
return nil, errors.Trace(err)
}
defaultValMap[col.Name.L] = value
}
}
return defaultValMap, nil
}
func (d *ddl) backfillColumn(ctx context.Context, t table.Table, columnInfo *model.ColumnInfo, handles []int64, reorgInfo *reorgInfo) error {
var defaultVal types.Datum
var err error
if columnInfo.DefaultValue != nil {
defaultVal, _, err = table.GetColDefaultValue(ctx, columnInfo)
if err != nil {
return errors.Trace(err)
}
} else if mysql.HasNotNullFlag(columnInfo.Flag) {
defaultVal = table.GetZeroValue(columnInfo)
}
colMap := make(map[int64]*types.FieldType)
for _, col := range t.Meta().Columns {
colMap[col.ID] = &col.FieldType
}
var endIdx int
for len(handles) > 0 {
if len(handles) >= defaultSmallBatchCnt {
endIdx = defaultSmallBatchCnt
} else {
endIdx = len(handles)
}
err = kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err := d.isReorgRunnable(txn, ddlJobFlag); err != nil {
return errors.Trace(err)
}
nextHandle, err1 := d.backfillColumnInTxn(t, columnInfo.ID, handles[:endIdx], colMap, defaultVal, txn)
if err1 != nil {
return errors.Trace(err1)
}
return errors.Trace(reorgInfo.UpdateHandle(txn, nextHandle))
})
if err != nil {
return errors.Trace(err)
}
handles = handles[endIdx:]
}
return nil
}
func (e *InsertValues) initDefaultValues(row []types.Datum, marked map[int]struct{}, ignoreErr bool) error {
var defaultValueCols []*table.Column
sc := e.ctx.GetSessionVars().StmtCtx
for i, c := range e.Table.Cols() {
// It's used for retry.
if mysql.HasAutoIncrementFlag(c.Flag) && row[i].IsNull() &&
e.ctx.GetSessionVars().RetryInfo.Retrying {
id, err := e.ctx.GetSessionVars().RetryInfo.GetCurrAutoIncrementID()
if err != nil {
return errors.Trace(err)
}
row[i].SetInt64(id)
}
if !row[i].IsNull() {
// Column value isn't nil and column isn't auto-increment, continue.
if !mysql.HasAutoIncrementFlag(c.Flag) {
continue
}
val, err := row[i].ToInt64(sc)
if filterErr(errors.Trace(err), ignoreErr) != nil {
return errors.Trace(err)
}
row[i].SetInt64(val)
if val != 0 {
e.Table.RebaseAutoID(val, true)
continue
}
}
// If the nil value is evaluated in insert list, we will use nil except auto increment column.
if _, ok := marked[i]; ok && !mysql.HasAutoIncrementFlag(c.Flag) && !mysql.HasTimestampFlag(c.Flag) {
continue
}
if mysql.HasAutoIncrementFlag(c.Flag) {
recordID, err := e.Table.AllocAutoID()
if err != nil {
return errors.Trace(err)
}
row[i].SetInt64(recordID)
// It's compatible with mysql. So it sets last insert id to the first row.
if e.currRow == 0 {
e.lastInsertID = uint64(recordID)
}
// It's used for retry.
if !e.ctx.GetSessionVars().RetryInfo.Retrying {
e.ctx.GetSessionVars().RetryInfo.AddAutoIncrementID(recordID)
}
} else {
var err error
row[i], _, err = table.GetColDefaultValue(e.ctx, c.ToInfo())
if filterErr(err, ignoreErr) != nil {
return errors.Trace(err)
}
}
defaultValueCols = append(defaultValueCols, c)
}
if err := table.CastValues(e.ctx, row, defaultValueCols, ignoreErr); err != nil {
return errors.Trace(err)
}
return nil
}
// AddRecord implements table.Table AddRecord interface.
func (t *Table) AddRecord(ctx context.Context, r []types.Datum) (recordID int64, err error) {
var hasRecordID bool
for _, col := range t.Cols() {
if col.IsPKHandleColumn(t.meta) {
recordID = r[col.Offset].GetInt64()
hasRecordID = true
break
}
}
if !hasRecordID {
recordID, err = t.alloc.Alloc(t.ID)
if err != nil {
return 0, errors.Trace(err)
}
}
txn, err := ctx.GetTxn(false)
if err != nil {
return 0, errors.Trace(err)
}
bs := kv.NewBufferStore(txn)
// Insert new entries into indices.
h, err := t.addIndices(ctx, recordID, r, bs)
if err != nil {
return h, errors.Trace(err)
}
colIDs := make([]int64, 0, len(r))
row := make([]types.Datum, 0, len(r))
// Set public and write only column value.
for _, col := range t.WritableCols() {
if col.IsPKHandleColumn(t.meta) {
continue
}
var value types.Datum
if col.State == model.StateWriteOnly || col.State == model.StateWriteReorganization {
// if col is in write only or write reorganization state, we must add it with its default value.
value, _, err = table.GetColDefaultValue(ctx, col.ToInfo())
if err != nil {
return 0, errors.Trace(err)
}
} else {
value = r[col.Offset]
if col.DefaultValue == nil && r[col.Offset].IsNull() {
// Save storage space by not storing null value.
continue
}
}
colIDs = append(colIDs, col.ID)
row = append(row, value)
}
key := t.RecordKey(recordID)
value, err := tablecodec.EncodeRow(row, colIDs)
if err != nil {
return 0, errors.Trace(err)
}
if err = txn.Set(key, value); err != nil {
return 0, errors.Trace(err)
}
if err = bs.SaveTo(txn); err != nil {
return 0, errors.Trace(err)
}
if shouldWriteBinlog(ctx) {
mutation := t.getMutation(ctx)
// prepend handle to the row value
handleVal, _ := codec.EncodeValue(nil, types.NewIntDatum(recordID))
bin := append(handleVal, value...)
mutation.InsertedRows = append(mutation.InsertedRows, bin)
mutation.Sequence = append(mutation.Sequence, binlog.MutationType_Insert)
}
ctx.GetSessionVars().StmtCtx.AddAffectedRows(1)
return recordID, nil
}
// AddRecord implements table.Table AddRecord interface.
func (t *Table) AddRecord(ctx context.Context, r []types.Datum) (recordID int64, err error) {
var hasRecordID bool
for _, col := range t.Cols() {
if col.IsPKHandleColumn(t.meta) {
recordID = r[col.Offset].GetInt64()
hasRecordID = true
break
}
}
if !hasRecordID {
recordID, err = t.alloc.Alloc(t.ID)
if err != nil {
return 0, errors.Trace(err)
}
}
txn, err := ctx.GetTxn(false)
if err != nil {
return 0, errors.Trace(err)
}
bs := kv.NewBufferStore(txn)
defer bs.Release()
// Insert new entries into indices.
h, err := t.addIndices(ctx, recordID, r, bs)
if err != nil {
return h, errors.Trace(err)
}
if err = t.LockRow(ctx, recordID, false); err != nil {
return 0, errors.Trace(err)
}
// Set public and write only column value.
for _, col := range t.writableCols() {
if col.IsPKHandleColumn(t.meta) {
continue
}
if col.DefaultValue == nil && r[col.Offset].IsNull() {
// Save storage space by not storing null value.
continue
}
var value types.Datum
if col.State == model.StateWriteOnly || col.State == model.StateWriteReorganization {
// if col is in write only or write reorganization state, we must add it with its default value.
value, _, err = table.GetColDefaultValue(ctx, &col.ColumnInfo)
if err != nil {
return 0, errors.Trace(err)
}
value, err = table.CastValue(ctx, value, col)
if err != nil {
return 0, errors.Trace(err)
}
} else {
value = r[col.Offset]
}
key := t.RecordKey(recordID, col)
err = SetColValue(txn, key, value)
if err != nil {
return 0, errors.Trace(err)
}
}
if err = bs.SaveTo(txn); err != nil {
return 0, errors.Trace(err)
}
variable.GetSessionVars(ctx).AddAffectedRows(1)
return recordID, nil
}
func (e *InsertValues) initDefaultValues(row []types.Datum, marked map[int]struct{}) error {
var defaultValueCols []*column.Col
for i, c := range e.Table.Cols() {
// It's used for retry.
if mysql.HasAutoIncrementFlag(c.Flag) && row[i].Kind() == types.KindNull &&
variable.GetSessionVars(e.ctx).RetryInfo.Retrying {
id, err := variable.GetSessionVars(e.ctx).RetryInfo.GetCurrAutoIncrementID()
if err != nil {
return errors.Trace(err)
}
row[i].SetInt64(id)
}
if row[i].Kind() != types.KindNull {
// Column value isn't nil and column isn't auto-increment, continue.
if !mysql.HasAutoIncrementFlag(c.Flag) {
continue
}
val, err := row[i].ToInt64()
if err != nil {
return errors.Trace(err)
}
if val != 0 {
e.Table.RebaseAutoID(val, true)
continue
}
}
// If the nil value is evaluated in insert list, we will use nil except auto increment column.
if _, ok := marked[i]; ok && !mysql.HasAutoIncrementFlag(c.Flag) && !mysql.HasTimestampFlag(c.Flag) {
continue
}
if mysql.HasAutoIncrementFlag(c.Flag) {
recordID, err := e.Table.AllocAutoID()
if err != nil {
return errors.Trace(err)
}
row[i].SetInt64(recordID)
// Notes: incompatible with mysql
// MySQL will set last insert id to the first row, as follows:
// `t(id int AUTO_INCREMENT, c1 int, PRIMARY KEY (id))`
// `insert t (c1) values(1),(2),(3);`
// Last insert id will be 1, not 3.
variable.GetSessionVars(e.ctx).SetLastInsertID(uint64(recordID))
// It's used for retry.
if !variable.GetSessionVars(e.ctx).RetryInfo.Retrying {
variable.GetSessionVars(e.ctx).RetryInfo.AddAutoIncrementID(recordID)
}
} else {
var err error
row[i], _, err = table.GetColDefaultValue(e.ctx, &c.ColumnInfo)
if err != nil {
return errors.Trace(err)
}
}
defaultValueCols = append(defaultValueCols, c)
}
if err := column.CastValues(e.ctx, row, defaultValueCols); err != nil {
return errors.Trace(err)
}
return nil
}