// Next implements plan.Plan Next interface.
func (r *TableDefaultPlan) Next(ctx context.Context) (row *plan.Row, err error) {
if r.iter == nil {
var txn kv.Transaction
txn, err = ctx.GetTxn(false)
if err != nil {
return nil, errors.Trace(err)
}
r.iter, err = txn.Seek([]byte(r.T.FirstKey()))
if err != nil {
return nil, errors.Trace(err)
}
}
if !r.iter.Valid() || !strings.HasPrefix(r.iter.Key(), r.T.KeyPrefix()) {
return
}
// TODO: check if lock valid
// the record layout in storage (key -> value):
// r1 -> lock-version
// r1_col1 -> r1 col1 value
// r1_col2 -> r1 col2 value
// r2 -> lock-version
// r2_col1 -> r2 col1 value
// r2_col2 -> r2 col2 value
// ...
rowKey := r.iter.Key()
handle, err := util.DecodeHandleFromRowKey(rowKey)
if err != nil {
return nil, errors.Trace(err)
}
txn, err := ctx.GetTxn(false)
if err != nil {
return nil, errors.Trace(err)
}
// It is very likely that we will fetch rows after current row later, enable the RangePrefetchOnCacheMiss
// option may help reducing RPC calls.
// TODO: choose a wiser option value.
txn.SetOption(kv.RangePrefetchOnCacheMiss, nil)
defer txn.DelOption(kv.RangePrefetchOnCacheMiss)
// TODO: we could just fetch mentioned columns' values
row = &plan.Row{}
row.Data, err = r.T.Row(ctx, handle)
if err != nil {
return nil, errors.Trace(err)
}
// Put rowKey to the tail of record row
rke := &plan.RowKeyEntry{
Tbl: r.T,
Key: rowKey,
}
row.RowKeys = append(row.RowKeys, rke)
rk := r.T.RecordKey(handle, nil)
err = kv.NextUntil(r.iter, util.RowKeyPrefixFilter(rk))
if err != nil {
return nil, errors.Trace(err)
}
return
}
// Next implements plan.Plan Next interface.
func (r *TableNilPlan) Next(ctx context.Context) (row *plan.Row, err error) {
if r.iter == nil {
var txn kv.Transaction
txn, err = ctx.GetTxn(false)
if err != nil {
return nil, errors.Trace(err)
}
r.iter, err = txn.Seek([]byte(r.T.FirstKey()), nil)
if err != nil {
return nil, errors.Trace(err)
}
}
if !r.iter.Valid() || !strings.HasPrefix(r.iter.Key(), r.T.KeyPrefix()) {
return
}
id, err := util.DecodeHandleFromRowKey(r.iter.Key())
if err != nil {
return nil, errors.Trace(err)
}
rk := r.T.RecordKey(id, nil)
// Even though the data is nil, we should return not nil row,
// or the iteration will stop.
row = &plan.Row{}
r.iter, err = kv.NextUntil(r.iter, util.RowKeyPrefixFilter(rk))
return
}
// Do implements the plan.Plan interface, iterates rows but does nothing.
func (r *TableNilPlan) Do(ctx context.Context, f plan.RowIterFunc) error {
h := r.T.FirstKey()
prefix := r.T.KeyPrefix()
txn, err := ctx.GetTxn(false)
if err != nil {
return err
}
it, err := txn.Seek([]byte(h), nil)
if err != nil {
return err
}
defer it.Close()
for it.Valid() && strings.HasPrefix(it.Key(), prefix) {
var err error
id, err := util.DecodeHandleFromRowKey(it.Key())
if err != nil {
return err
}
// do nothing
if m, err := f(id, nil); !m || err != nil {
return err
}
rk := r.T.RecordKey(id, nil)
if it, err = kv.NextUntil(it, util.RowKeyPrefixFilter(rk)); err != nil {
return err
}
}
return nil
}
func updateOldRows(ctx context.Context, t *tables.Table, col *column.Col) error {
txn, err := ctx.GetTxn(false)
if err != nil {
return errors.Trace(err)
}
it, err := txn.Seek([]byte(t.FirstKey()))
if err != nil {
return errors.Trace(err)
}
defer it.Close()
prefix := t.KeyPrefix()
for it.Valid() && strings.HasPrefix(it.Key(), prefix) {
handle, err0 := util.DecodeHandleFromRowKey(it.Key())
if err0 != nil {
return errors.Trace(err0)
}
k := t.RecordKey(handle, col)
// TODO: check and get timestamp/datetime default value.
// refer to getDefaultValue in stmt/stmts/stmt_helper.go.
if err0 = t.SetColValue(txn, k, col.DefaultValue); err0 != nil {
return errors.Trace(err0)
}
rk := t.RecordKey(handle, nil)
if it, err0 = kv.NextUntil(it, util.RowKeyPrefixFilter(rk)); err0 != nil {
return errors.Trace(err0)
}
}
return nil
}
// Do scans over rows' kv pair in the table, and constructs them into row data.
func (r *TableDefaultPlan) Do(ctx context.Context, f plan.RowIterFunc) error {
t := r.T
txn, err := ctx.GetTxn(false)
if err != nil {
return err
}
head := t.FirstKey()
prefix := t.KeyPrefix()
it, err := txn.Seek([]byte(head), nil)
if err != nil {
return err
}
defer it.Close()
for it.Valid() && strings.HasPrefix(it.Key(), prefix) {
// TODO: check if lock valid
// the record layout in storage (key -> value):
// r1 -> lock-version
// r1_col1 -> r1 col1 value
// r1_col2 -> r1 col2 value
// r2 -> lock-version
// r2_col1 -> r2 col1 value
// r2_col2 -> r2 col2 value
// ...
var err error
rowKey := it.Key()
h, err := util.DecodeHandleFromRowKey(rowKey)
if err != nil {
return err
}
// TODO: we could just fetch mentioned columns' values
rec, err := t.Row(ctx, h)
if err != nil {
return err
}
// Put rowKey to the tail of record row
rks := &RowKeyList{}
rke := &RowKeyEntry{
Tbl: t,
Key: rowKey,
}
rks.appendKeys(rke)
rec = append(rec, rks)
m, err := f(int64(0), rec)
if !m || err != nil {
return err
}
rk := t.RecordKey(h, nil)
it, err = kv.NextUntil(it, util.RowKeyPrefixFilter(rk))
if err != nil {
return err
}
}
return nil
}
func (d *ddl) buildIndex(ctx context.Context, t table.Table, idxInfo *model.IndexInfo, unique bool) error {
firstKey := t.FirstKey()
prefix := t.KeyPrefix()
txn, err := ctx.GetTxn(false)
if err != nil {
return errors.Trace(err)
}
it, err := txn.Seek([]byte(firstKey))
if err != nil {
return errors.Trace(err)
}
defer it.Close()
for it.Valid() && strings.HasPrefix(it.Key(), prefix) {
var err error
handle, err := util.DecodeHandleFromRowKey(it.Key())
log.Info("building index...", handle)
if err != nil {
return errors.Trace(err)
}
// TODO: v is timestamp ?
// fetch datas
cols := t.Cols()
var vals []interface{}
for _, v := range idxInfo.Columns {
var (
data []byte
val interface{}
)
col := cols[v.Offset]
k := t.RecordKey(handle, col)
data, err = txn.Get([]byte(k))
if err != nil {
return errors.Trace(err)
}
val, err = t.DecodeValue(data, col)
if err != nil {
return errors.Trace(err)
}
vals = append(vals, val)
}
// build index
kvX := kv.NewKVIndex(t.IndexPrefix(), idxInfo.Name.L, unique)
err = kvX.Create(txn, vals, handle)
if err != nil {
return errors.Trace(err)
}
rk := []byte(t.RecordKey(handle, nil))
it, err = kv.NextUntil(it, util.RowKeyPrefixFilter(rk))
if err != nil {
return errors.Trace(err)
}
}
return nil
}
// Next implements plan.Plan Next interface.
func (r *TableDefaultPlan) Next(ctx context.Context) (row *plan.Row, err error) {
if r.iter == nil {
var txn kv.Transaction
txn, err = ctx.GetTxn(false)
if err != nil {
return nil, errors.Trace(err)
}
r.iter, err = txn.Seek([]byte(r.T.FirstKey()), nil)
if err != nil {
return nil, errors.Trace(err)
}
}
if !r.iter.Valid() || !strings.HasPrefix(r.iter.Key(), r.T.KeyPrefix()) {
return
}
// TODO: check if lock valid
// the record layout in storage (key -> value):
// r1 -> lock-version
// r1_col1 -> r1 col1 value
// r1_col2 -> r1 col2 value
// r2 -> lock-version
// r2_col1 -> r2 col1 value
// r2_col2 -> r2 col2 value
// ...
rowKey := r.iter.Key()
h, err := util.DecodeHandleFromRowKey(rowKey)
if err != nil {
return nil, errors.Trace(err)
}
// TODO: we could just fetch mentioned columns' values
row = &plan.Row{}
row.Data, err = r.T.Row(ctx, h)
if err != nil {
return nil, errors.Trace(err)
}
// Put rowKey to the tail of record row
rke := &plan.RowKeyEntry{
Tbl: r.T,
Key: rowKey,
}
row.RowKeys = append(row.RowKeys, rke)
rk := r.T.RecordKey(h, nil)
r.iter, err = kv.NextUntil(r.iter, util.RowKeyPrefixFilter(rk))
if err != nil {
return nil, errors.Trace(err)
}
return
}
func (d *ddl) getSnapshotRows(t table.Table, version uint64, seekHandle int64) ([]int64, error) {
ver := kv.Version{Ver: version}
snap, err := d.store.GetSnapshot(ver)
if err != nil {
return nil, errors.Trace(err)
}
defer snap.MvccRelease()
firstKey := t.RecordKey(seekHandle, nil)
prefix := []byte(t.KeyPrefix())
it := snap.NewMvccIterator([]byte(firstKey), ver)
defer it.Close()
handles := make([]int64, 0, maxBatchSize)
for it.Valid() {
key := []byte(it.Key())
if !bytes.HasPrefix(key, prefix) {
break
}
var handle int64
handle, err = util.DecodeHandleFromRowKey(string(key))
if err != nil {
return nil, errors.Trace(err)
}
rk := t.RecordKey(handle, nil)
handles = append(handles, handle)
if len(handles) == maxBatchSize {
seekHandle = handle + 1
break
}
err = kv.NextUntil(it, util.RowKeyPrefixFilter(rk))
if terror.ErrorEqual(err, kv.ErrNotExist) {
break
} else if err != nil {
return nil, errors.Trace(err)
}
}
return handles, nil
}
// IterRecords implements table.Table IterRecords interface.
func (t *Table) IterRecords(ctx context.Context, startKey string, cols []*column.Col, fn table.RecordIterFunc) error {
txn, err := ctx.GetTxn(false)
if err != nil {
return errors.Trace(err)
}
it, err := txn.Seek([]byte(startKey))
if err != nil {
return errors.Trace(err)
}
defer it.Close()
if !it.Valid() {
return nil
}
log.Debugf("startKey %q, key:%q,value:%q", startKey, it.Key(), it.Value())
prefix := t.KeyPrefix()
for it.Valid() && strings.HasPrefix(it.Key(), prefix) {
// first kv pair is row lock information.
// TODO: check valid lock
// get row handle
var err error
handle, err := util.DecodeHandleFromRowKey(it.Key())
if err != nil {
return errors.Trace(err)
}
data, err := t.RowWithCols(ctx, handle, cols)
if err != nil {
return errors.Trace(err)
}
more, err := fn(handle, data, cols)
if !more || err != nil {
return errors.Trace(err)
}
rk := t.RecordKey(handle, nil)
err = kv.NextUntil(it, util.RowKeyPrefixFilter(rk))
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (s *DeleteStmt) tryDeleteUsingIndex(ctx context.Context, t table.Table) (bool, error) {
log.Info("try delete with index", ctx)
p, err := s.indexPlan(ctx)
if p == nil || err != nil {
return false, errors.Trace(err)
}
var rowKeys []*plan.RowKeyEntry
for {
row, err1 := p.Next(ctx)
if err1 != nil {
return false, errors.Trace(err1)
}
if row == nil {
break
}
rowKeys = append(rowKeys, row.RowKeys...)
}
var cnt uint64
for _, rowKey := range rowKeys {
if s.Limit != nil && cnt >= s.Limit.Count {
break
}
handle, err := util.DecodeHandleFromRowKey(rowKey.Key)
if err != nil {
return false, errors.Trace(err)
}
data, err := t.Row(ctx, handle)
if err != nil {
return false, err
}
log.Infof("try delete with index id:%d, ctx:%s", handle, ctx)
ok, err := s.hitWhere(ctx, t, data)
if err != nil {
return false, errors.Trace(err)
}
if !ok {
return true, nil
}
err = s.removeRow(ctx, t, handle, data)
if err != nil {
return false, err
}
cnt++
}
return true, nil
}
// Exec implements the stmt.Statement Exec interface.
func (s *DeleteStmt) Exec(ctx context.Context) (_ rset.Recordset, err error) {
if s.MultiTable && len(s.TableIdents) == 0 {
return nil, nil
}
p, err := s.plan(ctx)
if err != nil {
return nil, errors.Trace(err)
}
if p == nil {
return nil, nil
}
defer p.Close()
tblIDMap := make(map[int64]bool, len(s.TableIdents))
// Get table alias map.
tblNames := make(map[string]string)
if s.MultiTable {
// Delete from multiple tables should consider table ident list.
fs := p.GetFields()
for _, f := range fs {
if f.TableName != f.OrgTableName {
tblNames[f.TableName] = f.OrgTableName
} else {
tblNames[f.TableName] = f.TableName
}
}
for _, t := range s.TableIdents {
// Consider DBName.
oname, ok := tblNames[t.Name.O]
if !ok {
return nil, errors.Errorf("Unknown table '%s' in MULTI DELETE", t.Name.O)
}
t.Name.O = oname
t.Name.L = strings.ToLower(oname)
var tbl table.Table
tbl, err = getTable(ctx, t)
if err != nil {
return nil, errors.Trace(err)
}
tblIDMap[tbl.TableID()] = true
}
}
rowKeyMap := make(map[string]table.Table)
for {
row, err1 := p.Next(ctx)
if err1 != nil {
return nil, errors.Trace(err1)
}
if row == nil {
break
}
for _, entry := range row.RowKeys {
if s.MultiTable {
tid := entry.Tbl.TableID()
if _, ok := tblIDMap[tid]; !ok {
continue
}
}
rowKeyMap[entry.Key] = entry.Tbl
}
}
for k, t := range rowKeyMap {
handle, err := util.DecodeHandleFromRowKey(k)
if err != nil {
return nil, errors.Trace(err)
}
data, err := t.Row(ctx, handle)
if err != nil {
return nil, errors.Trace(err)
}
err = s.removeRow(ctx, t, handle, data)
if err != nil {
return nil, errors.Trace(err)
}
}
return nil, nil
}
func (s *DeleteStmt) execMultiTable(ctx context.Context) (_ rset.Recordset, err error) {
log.Info("Delete from multi-table")
if len(s.TableIdents) == 0 {
return nil, nil
}
p, err := s.indexPlan(ctx)
if err != nil {
return nil, errors.Trace(err)
}
if p == nil {
return nil, nil
}
tblIDMap := make(map[int64]bool, len(s.TableIdents))
// Get table alias map.
fs := p.GetFields()
tblAliasMap := make(map[string]string)
for _, f := range fs {
if f.TableName != f.OrgTableName {
tblAliasMap[f.TableName] = f.OrgTableName
}
}
for _, t := range s.TableIdents {
// Consider DBName.
oname, ok := tblAliasMap[t.Name.O]
if ok {
t.Name.O = oname
t.Name.L = strings.ToLower(oname)
}
var tbl table.Table
tbl, err = getTable(ctx, t)
if err != nil {
return nil, errors.Trace(err)
}
tblIDMap[tbl.TableID()] = true
}
rowKeyMap := make(map[string]table.Table)
err = p.Do(ctx, func(_ interface{}, in []interface{}) (bool, error) {
// Generate ids for coming deletion.
var rowKeys *plans.RowKeyList
if in != nil && len(in) > 0 {
t := in[len(in)-1]
switch vt := t.(type) {
case *plans.RowKeyList:
rowKeys = vt
}
}
if rowKeys != nil {
for _, entry := range rowKeys.Keys {
tid := entry.Tbl.TableID()
if _, ok := tblIDMap[tid]; !ok {
continue
}
rowKeyMap[entry.Key] = entry.Tbl
}
}
return true, nil
})
if err != nil {
return nil, errors.Trace(err)
}
for k, t := range rowKeyMap {
id, err := util.DecodeHandleFromRowKey(k)
if err != nil {
return nil, errors.Trace(err)
}
data, err := t.Row(ctx, id)
if err != nil {
return nil, errors.Trace(err)
}
err = s.removeRow(ctx, t, id, data)
if err != nil {
return nil, errors.Trace(err)
}
}
return nil, nil
}
// Exec implements the stmt.Statement Exec interface.
func (s *UpdateStmt) Exec(ctx context.Context) (_ rset.Recordset, err error) {
p, err := s.plan(ctx)
if err != nil {
return nil, errors.Trace(err)
}
defer p.Close()
updatedRowKeys := make(map[string]bool)
// Get table alias map.
fs := p.GetFields()
columns, err0 := getUpdateColumns(s.List, fs)
if err0 != nil {
return nil, errors.Trace(err0)
}
m := map[interface{}]interface{}{}
var records []*plan.Row
for {
row, err1 := p.Next(ctx)
if err1 != nil {
return nil, errors.Trace(err1)
}
if row == nil {
break
}
if len(row.RowKeys) == 0 {
// Nothing to update
continue
}
records = append(records, row)
}
for _, row := range records {
rowData := row.Data
// Set EvalIdentFunc
m[expression.ExprEvalIdentFunc] = func(name string) (interface{}, error) {
return plans.GetIdentValue(name, p.GetFields(), rowData, field.DefaultFieldFlag)
}
// Update rows
offset := 0
for _, entry := range row.RowKeys {
tbl := entry.Tbl
k := entry.Key
lastOffset := offset
offset += len(tbl.Cols())
data := rowData[lastOffset:offset]
_, ok := updatedRowKeys[k]
if ok {
// Each matching row is updated once, even if it matches the conditions multiple times.
continue
}
// Update row
handle, err2 := util.DecodeHandleFromRowKey(k)
if err2 != nil {
return nil, errors.Trace(err2)
}
err2 = updateRecord(ctx, handle, data, tbl, columns, m, lastOffset, false)
if err2 != nil {
return nil, errors.Trace(err2)
}
updatedRowKeys[k] = true
}
}
return nil, nil
}
// Exec implements the stmt.Statement Exec interface.
func (s *UpdateStmt) Exec(ctx context.Context) (_ rset.Recordset, err error) {
p, err := s.plan(ctx)
if err != nil {
return nil, errors.Trace(err)
}
defer p.Close()
updatedRowKeys := make(map[string]bool)
for {
row, err1 := p.Next(ctx)
if err1 != nil {
return nil, errors.Trace(err1)
}
if row == nil {
break
}
rowData := row.Data
if len(row.RowKeys) == 0 {
// Nothing to update
return nil, nil
}
// Set EvalIdentFunc
m := make(map[interface{}]interface{})
m[expressions.ExprEvalIdentFunc] = func(name string) (interface{}, error) {
return plans.GetIdentValue(name, p.GetFields(), rowData, field.DefaultFieldFlag)
}
// Update rows
start := 0
for _, entry := range row.RowKeys {
tbl := entry.Tbl
k := entry.Key
_, ok := updatedRowKeys[k]
if ok {
// Each matching row is updated once, even if it matches the conditions multiple times.
continue
}
// Update row
handle, err2 := util.DecodeHandleFromRowKey(k)
if err2 != nil {
return nil, errors.Trace(err2)
}
end := start + len(tbl.Cols())
data := rowData[start:end]
start = end
tcols, err2 := getUpdateColumns(tbl, s.List, s.MultipleTable)
if err2 != nil {
return nil, errors.Trace(err2)
}
if len(tcols) == 0 {
// Nothing to update for this table.
continue
}
// Get data in the table
err2 = updateRecord(ctx, handle, data, tbl, tcols, s.List, nil, m)
if err2 != nil {
return nil, errors.Trace(err2)
}
updatedRowKeys[k] = true
}
}
return nil, nil
}
func (s *DeleteStmt) execMultiTable(ctx context.Context) (_ rset.Recordset, err error) {
log.Info("Delete from multi-table")
if len(s.TableIdents) == 0 {
return nil, nil
}
p, err := s.indexPlan(ctx)
if err != nil {
return nil, errors.Trace(err)
}
if p == nil {
return nil, nil
}
defer p.Close()
tblIDMap := make(map[int64]bool, len(s.TableIdents))
// Get table alias map.
fs := p.GetFields()
tblAliasMap := make(map[string]string)
for _, f := range fs {
if f.TableName != f.OrgTableName {
tblAliasMap[f.TableName] = f.OrgTableName
}
}
for _, t := range s.TableIdents {
// Consider DBName.
oname, ok := tblAliasMap[t.Name.O]
if ok {
t.Name.O = oname
t.Name.L = strings.ToLower(oname)
}
var tbl table.Table
tbl, err = getTable(ctx, t)
if err != nil {
return nil, errors.Trace(err)
}
tblIDMap[tbl.TableID()] = true
}
rowKeyMap := make(map[string]table.Table)
for {
row, err1 := p.Next(ctx)
if err1 != nil {
return nil, errors.Trace(err1)
}
if row == nil {
break
}
for _, entry := range row.RowKeys {
tid := entry.Tbl.TableID()
if _, ok := tblIDMap[tid]; !ok {
continue
}
rowKeyMap[entry.Key] = entry.Tbl
}
}
for k, t := range rowKeyMap {
id, err := util.DecodeHandleFromRowKey(k)
if err != nil {
return nil, errors.Trace(err)
}
data, err := t.Row(ctx, id)
if err != nil {
return nil, errors.Trace(err)
}
err = s.removeRow(ctx, t, id, data)
if err != nil {
return nil, errors.Trace(err)
}
}
return nil, nil
}
// Exec implements the stmt.Statement Exec interface.
func (s *UpdateStmt) Exec(ctx context.Context) (_ rset.Recordset, err error) {
p, err := s.plan(ctx)
if err != nil {
return nil, errors.Trace(err)
}
defer p.Close()
fs := p.GetFields()
columns, err := getUpdateColumns(s.List, fs)
if err != nil {
return nil, errors.Trace(err)
}
var records []*plan.Row
for {
row, err1 := p.Next(ctx)
if err1 != nil {
return nil, errors.Trace(err1)
}
if row == nil {
break
}
if len(row.RowKeys) == 0 {
// Nothing to update
continue
}
records = append(records, row)
}
evalMap := map[interface{}]interface{}{}
updatedRowKeys := make(map[string]bool)
for _, row := range records {
rowData := row.Data
// Set ExprEvalIdentReferFunc.
evalMap[expression.ExprEvalIdentReferFunc] = func(name string, scope int, index int) (interface{}, error) {
return rowData[index], nil
}
// Update rows.
offset := 0
for _, entry := range row.RowKeys {
tbl := entry.Tbl
k := entry.Key
lastOffset := offset
offset += len(tbl.Cols())
data := rowData[lastOffset:offset]
_, ok := updatedRowKeys[k]
if ok {
// Each matching row is updated once, even if it matches the conditions multiple times.
continue
}
// Update row
handle, err1 := util.DecodeHandleFromRowKey(k)
if err1 != nil {
return nil, errors.Trace(err1)
}
err1 = updateRecord(ctx, handle, data, tbl, columns, evalMap, lastOffset, false)
if err1 != nil {
return nil, errors.Trace(err1)
}
updatedRowKeys[k] = true
}
}
return nil, nil
}