// DecodeRow decodes a byte slice into datums.
// Row layout: colID1, value1, colID2, value2, .....
func DecodeRow(b []byte, cols map[int64]*types.FieldType) (map[int64]types.Datum, error) {
if b == nil {
return nil, nil
}
if len(b) == 1 && b[0] == codec.NilFlag {
return nil, nil
}
row := make(map[int64]types.Datum, len(cols))
cnt := 0
var (
data []byte
err error
)
for len(b) > 0 {
// Get col id.
data, b, err = codec.CutOne(b)
if err != nil {
return nil, errors.Trace(err)
}
_, cid, err := codec.DecodeOne(data)
if err != nil {
return nil, errors.Trace(err)
}
// Get col value.
data, b, err = codec.CutOne(b)
if err != nil {
return nil, errors.Trace(err)
}
id := cid.GetInt64()
ft, ok := cols[id]
if ok {
_, v, err := codec.DecodeOne(data)
if err != nil {
return nil, errors.Trace(err)
}
v, err = Unflatten(v, ft, false)
if err != nil {
return nil, errors.Trace(err)
}
row[id] = v
cnt++
if cnt == len(cols) {
// Get enough data.
break
}
}
}
return row, nil
}
// Update aggregate functions with rows.
func (rs *localRegion) aggregate(ctx *selectContext, row [][]byte) error {
// Put row data into evaluate context for later evaluation.
cols := ctx.sel.TableInfo.Columns
for i, col := range cols {
_, datum, err := codec.DecodeOne(row[i])
if err != nil {
return errors.Trace(err)
}
ctx.eval.Row[col.GetColumnId()] = datum
}
// Get group key.
gk, err := rs.getGroupKey(ctx)
if err != nil {
return errors.Trace(err)
}
if _, ok := ctx.groups[string(gk)]; !ok {
ctx.groups[string(gk)] = true
ctx.groupKeys = append(ctx.groupKeys, gk)
}
// Update aggregate funcs.
for _, agg := range ctx.aggregates {
agg.currentGroup = gk
args := make([]types.Datum, 0, len(agg.expr.Children))
// Evaluate arguments.
for _, x := range agg.expr.Children {
cv, err := ctx.eval.Eval(x)
if err != nil {
return errors.Trace(err)
}
args = append(args, cv)
}
agg.update(ctx, args)
}
return nil
}
// Next returns the next row of the sub result.
// If no more row to return, data would be nil.
func (pr *partialResult) Next() (handle int64, data []types.Datum, err error) {
if !pr.fetched {
err = <-pr.done
pr.fetched = true
if err != nil {
return 0, nil, err
}
}
if len(pr.resp.Chunks) > 0 {
// For new resp rows structure.
chunk := pr.getChunk()
if chunk == nil {
return 0, nil, nil
}
rowMeta := chunk.RowsMeta[pr.cursor]
if !pr.ignoreData {
rowData := chunk.RowsData[pr.dataOffset : pr.dataOffset+rowMeta.Length]
data, err = tablecodec.DecodeValues(rowData, pr.fields, pr.index)
if err != nil {
return 0, nil, errors.Trace(err)
}
pr.dataOffset += rowMeta.Length
}
if data == nil {
data = dummyData
}
if !pr.aggregate {
handle = rowMeta.Handle
}
pr.cursor++
return
}
if pr.cursor >= len(pr.resp.Rows) {
return 0, nil, nil
}
row := pr.resp.Rows[pr.cursor]
if !pr.ignoreData {
data, err = tablecodec.DecodeValues(row.Data, pr.fields, pr.index)
if err != nil {
return 0, nil, errors.Trace(err)
}
}
if data == nil {
// When no column is referenced, the data may be nil, like 'select count(*) from t'.
// In this case, we need to create a zero length datum slice,
// as caller will check if data is nil to finish iteration.
// data = make([]types.Datum, 0)
data = dummyData
}
if !pr.aggregate {
handleBytes := row.GetHandle()
_, datum, err := codec.DecodeOne(handleBytes)
if err != nil {
return 0, nil, errors.Trace(err)
}
handle = datum.GetInt64()
}
pr.cursor++
return
}
// DecodeColumnValue decodes data to a Datum according to the column info.
func DecodeColumnValue(data []byte, ft *types.FieldType) (types.Datum, error) {
_, d, err := codec.DecodeOne(data)
if err != nil {
return types.Datum{}, errors.Trace(err)
}
colDatum, err := Unflatten(d, ft, false)
if err != nil {
return types.Datum{}, errors.Trace(err)
}
return colDatum, nil
}
func (s *testTableCodecSuite) TestCutKey(c *C) {
colIDs := []int64{1, 2, 3}
values := []types.Datum{types.NewIntDatum(1), types.NewBytesDatum([]byte("abc")), types.NewFloat64Datum(5.5)}
handle := types.NewIntDatum(100)
values = append(values, handle)
encodedValue, err := codec.EncodeKey(nil, values...)
c.Assert(err, IsNil)
tableID := int64(4)
indexID := int64(5)
indexKey := EncodeIndexSeekKey(tableID, indexID, encodedValue)
valuesMap, handleBytes, err := CutIndexKey(indexKey, colIDs)
c.Assert(err, IsNil)
for i, colID := range colIDs {
valueBytes := valuesMap[colID]
var val types.Datum
_, val, _ = codec.DecodeOne(valueBytes)
c.Assert(val, DeepEquals, values[i])
}
_, handleVal, _ := codec.DecodeOne(handleBytes)
c.Assert(handleVal, DeepEquals, types.NewIntDatum(100))
}
// IndexRowData extracts the row data and handle in an index key.
// If the index is unique, handle would be nil.
func IndexRowData(key kv.Key, columnCount int) (data, handle []byte, err error) {
b := key[prefixLen:]
// The index key may have primary key appended to the end, we decode the column values
// only to get the column values length.
for columnCount > 0 {
b, _, err = codec.DecodeOne(b)
if err != nil {
return nil, nil, errors.Trace(err)
}
columnCount--
}
handle = b
return key[prefixLen : len(key)-len(handle)], handle, nil
}
// DecodeColumnValue decodes data to a Datum according to the column info.
func DecodeColumnValue(data []byte, col *tipb.ColumnInfo) (types.Datum, error) {
_, d, err := codec.DecodeOne(data)
if err != nil {
return types.Datum{}, errors.Trace(err)
}
ft := &types.FieldType{
Tp: byte(col.GetTp()),
Flen: int(col.GetColumnLen()),
Decimal: int(col.GetDecimal()),
Elems: col.Elems,
Collate: mysql.Collations[uint8(col.GetCollation())],
}
colDatum, err := unflatten(d, ft)
if err != nil {
return types.Datum{}, errors.Trace(err)
}
return colDatum, nil
}
// CutRow cut encoded row into byte slices and return interested columns' byte slice.
// Row layout: colID1, value1, colID2, value2, .....
func CutRow(data []byte, cols map[int64]*types.FieldType) (map[int64][]byte, error) {
if data == nil {
return nil, nil
}
if len(data) == 1 && data[0] == codec.NilFlag {
return nil, nil
}
row := make(map[int64][]byte, len(cols))
cnt := 0
var (
b []byte
err error
)
for len(data) > 0 {
// Get col id.
b, data, err = codec.CutOne(data)
if err != nil {
return nil, errors.Trace(err)
}
_, cid, err := codec.DecodeOne(b)
if err != nil {
return nil, errors.Trace(err)
}
// Get col value.
b, data, err = codec.CutOne(data)
if err != nil {
return nil, errors.Trace(err)
}
id := cid.GetInt64()
_, ok := cols[id]
if ok {
row[id] = b
cnt++
if cnt == len(cols) {
break
}
}
}
return row, nil
}
func (rs *localRegion) getIndexRowFromRange(ctx *selectContext, ran kv.KeyRange, desc bool, limit int64) (count int64, err error) {
idxInfo := ctx.sel.IndexInfo
txn := ctx.txn
var seekKey kv.Key
if desc {
seekKey = ran.EndKey
} else {
seekKey = ran.StartKey
}
ids := make([]int64, len(idxInfo.Columns))
for i, col := range idxInfo.Columns {
ids[i] = col.GetColumnId()
}
for {
if limit == 0 {
break
}
var it kv.Iterator
if desc {
it, err = txn.SeekReverse(seekKey)
if err != nil {
return 0, errors.Trace(err)
}
seekKey = it.Key()
} else {
it, err = txn.Seek(seekKey)
if err != nil {
return 0, errors.Trace(err)
}
seekKey = it.Key().PrefixNext()
}
if !it.Valid() {
break
}
if desc {
if it.Key().Cmp(ran.StartKey) < 0 {
break
}
} else {
if it.Key().Cmp(ran.EndKey) >= 0 {
break
}
}
values, b, err1 := tablecodec.CutIndexKey(it.Key(), ids)
if err1 != nil {
return 0, errors.Trace(err1)
}
var handle int64
if len(b) > 0 {
var handleDatum types.Datum
_, handleDatum, err = codec.DecodeOne(b)
if err != nil {
return 0, errors.Trace(err)
}
handle = handleDatum.GetInt64()
} else {
handle, err = decodeHandle(it.Value())
if err != nil {
return 0, errors.Trace(err)
}
}
gotRow, err := rs.valuesToRow(ctx, handle, values)
if err != nil {
return 0, errors.Trace(err)
}
if gotRow {
limit--
count++
}
}
return count, nil
}
func (h *rpcHandler) getIndexRowFromRange(ctx *selectContext, ran kv.KeyRange, desc bool, limit int64) ([]*tipb.Row, error) {
idxInfo := ctx.sel.IndexInfo
startKey := maxStartKey(ran.StartKey, h.startKey)
endKey := minEndKey(ran.EndKey, h.endKey)
if limit == 0 || bytes.Compare(startKey, endKey) >= 0 {
return nil, nil
}
var rows []*tipb.Row
var seekKey kv.Key
if desc {
seekKey = endKey
} else {
seekKey = startKey
}
ids := make([]int64, len(idxInfo.Columns))
for i, col := range idxInfo.Columns {
ids[i] = col.GetColumnId()
}
for {
if limit == 0 {
break
}
var (
pairs []Pair
pair Pair
err error
)
if desc {
pairs = h.mvccStore.ReverseScan(startKey, seekKey, 1, ctx.sel.GetStartTs())
} else {
pairs = h.mvccStore.Scan(seekKey, endKey, 1, ctx.sel.GetStartTs())
}
if len(pairs) > 0 {
pair = pairs[0]
}
if pair.Err != nil {
// TODO: handle lock error.
return nil, errors.Trace(pair.Err)
}
if pair.Key == nil {
break
}
if desc {
if bytes.Compare(pair.Key, startKey) < 0 {
break
}
seekKey = pair.Key
} else {
if bytes.Compare(pair.Key, endKey) >= 0 {
break
}
seekKey = []byte(kv.Key(pair.Key).PrefixNext())
}
values, b, err := tablecodec.CutIndexKey(pair.Key, ids)
var handle int64
if len(b) > 0 {
var handleDatum types.Datum
_, handleDatum, err = codec.DecodeOne(b)
if err != nil {
return nil, errors.Trace(err)
}
handle = handleDatum.GetInt64()
} else {
handle, err = decodeHandle(pair.Value)
if err != nil {
return nil, errors.Trace(err)
}
}
row, err := h.valuesToRow(ctx, handle, values)
if err != nil {
return nil, errors.Trace(err)
}
if row != nil {
rows = append(rows, row)
limit--
}
}
return rows, nil
}
func (s *testBinlogSuite) TestBinlog(c *C) {
tk := s.tk
pump := s.pump
tk.MustExec("drop table if exists local_binlog")
ddlQuery := "create table local_binlog (id int primary key, name varchar(10))"
tk.MustExec(ddlQuery)
var matched bool // got matched pre DDL and commit DDL
for i := 0; i < 10; i++ {
preDDL, commitDDL := getLatestDDLBinlog(c, pump, ddlQuery)
if preDDL.DdlJobId == commitDDL.DdlJobId {
c.Assert(commitDDL.StartTs, Equals, preDDL.StartTs)
c.Assert(commitDDL.CommitTs, Greater, commitDDL.StartTs)
matched = true
break
}
time.Sleep(time.Millisecond * 10)
}
c.Assert(matched, IsTrue)
tk.MustExec("insert local_binlog values (1, 'abc'), (2, 'cde')")
prewriteVal := getLatestBinlogPrewriteValue(c, pump)
c.Assert(prewriteVal.SchemaVersion, Greater, int64(0))
c.Assert(prewriteVal.Mutations[0].TableId, Greater, int64(0))
expected := [][]types.Datum{
{types.NewIntDatum(1), types.NewStringDatum("abc")},
{types.NewIntDatum(2), types.NewStringDatum("cde")},
}
gotRows := mutationRowsToRows(c, prewriteVal.Mutations[0].InsertedRows, 0, 2)
c.Assert(gotRows, DeepEquals, expected)
tk.MustExec("update local_binlog set name = 'xyz' where id = 2")
prewriteVal = getLatestBinlogPrewriteValue(c, pump)
expected = [][]types.Datum{
{types.NewIntDatum(2), types.NewStringDatum("xyz")},
}
gotRows = mutationRowsToRows(c, prewriteVal.Mutations[0].UpdatedRows, 2, 4)
c.Assert(gotRows, DeepEquals, expected)
tk.MustExec("delete from local_binlog where id = 1")
prewriteVal = getLatestBinlogPrewriteValue(c, pump)
c.Assert(prewriteVal.Mutations[0].DeletedIds, DeepEquals, []int64{1})
// Test table primary key is not integer.
tk.MustExec("create table local_binlog2 (name varchar(64) primary key, age int)")
tk.MustExec("insert local_binlog2 values ('abc', 16), ('def', 18)")
tk.MustExec("delete from local_binlog2 where name = 'def'")
prewriteVal = getLatestBinlogPrewriteValue(c, pump)
c.Assert(prewriteVal.Mutations[0].Sequence[0], Equals, binlog.MutationType_DeletePK)
_, deletedPK, _ := codec.DecodeOne(prewriteVal.Mutations[0].DeletedPks[0])
c.Assert(deletedPK.GetString(), Equals, "def")
// Test Table don't have primary key.
tk.MustExec("create table local_binlog3 (c1 int, c2 int)")
tk.MustExec("insert local_binlog3 values (1, 2), (1, 3), (2, 3)")
tk.MustExec("update local_binlog3 set c1 = 3 where c1 = 2")
prewriteVal = getLatestBinlogPrewriteValue(c, pump)
gotRows = mutationRowsToRows(c, prewriteVal.Mutations[0].UpdatedRows, 5, 7)
expected = [][]types.Datum{
{types.NewIntDatum(3), types.NewIntDatum(3)},
}
c.Assert(gotRows, DeepEquals, expected)
tk.MustExec("delete from local_binlog3 where c1 = 3 and c2 = 3")
prewriteVal = getLatestBinlogPrewriteValue(c, pump)
c.Assert(prewriteVal.Mutations[0].Sequence[0], Equals, binlog.MutationType_DeleteRow)
gotRows = mutationRowsToRows(c, prewriteVal.Mutations[0].DeletedRows, 1, 3)
expected = [][]types.Datum{
{types.NewIntDatum(3), types.NewIntDatum(3)},
}
c.Assert(gotRows, DeepEquals, expected)
// Test Mutation Sequence.
tk.MustExec("create table local_binlog4 (c1 int primary key, c2 int)")
tk.MustExec("insert local_binlog4 values (1, 1), (2, 2), (3, 2)")
tk.MustExec("begin")
tk.MustExec("delete from local_binlog4 where c1 = 1")
tk.MustExec("insert local_binlog4 values (1, 1)")
tk.MustExec("update local_binlog4 set c2 = 3 where c1 = 3")
tk.MustExec("commit")
prewriteVal = getLatestBinlogPrewriteValue(c, pump)
c.Assert(prewriteVal.Mutations[0].Sequence, DeepEquals, []binlog.MutationType{
binlog.MutationType_DeleteID,
binlog.MutationType_Insert,
binlog.MutationType_Update,
})
checkBinlogCount(c, pump)
pump.mu.Lock()
originBinlogLen := len(pump.mu.payloads)
pump.mu.Unlock()
tk.MustExec("set @@global.autocommit = 0")
tk.MustExec("set @@global.autocommit = 1")
pump.mu.Lock()
newBinlogLen := len(pump.mu.payloads)
pump.mu.Unlock()
c.Assert(newBinlogLen, Equals, originBinlogLen)
}
func (s *testBinlogSuite) TestBinlog(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
pump := s.pump
tk.MustExec("drop table if exists local_binlog")
ddlQuery := "create table local_binlog (id int primary key, name varchar(10))"
tk.MustExec(ddlQuery)
time.Sleep(time.Millisecond)
checkLatestBinlogDDL(c, pump, ddlQuery)
tk.MustExec("insert local_binlog values (1, 'abc'), (2, 'cde')")
prewriteVal := getLatestBinlogPrewriteValue(c, pump)
c.Assert(prewriteVal.SchemaVersion, Greater, int64(0))
c.Assert(prewriteVal.Mutations[0].TableId, Greater, int64(0))
expected := [][]types.Datum{
{types.NewIntDatum(1), types.NewStringDatum("abc")},
{types.NewIntDatum(2), types.NewStringDatum("cde")},
}
gotRows := mutationRowsToRows(c, prewriteVal.Mutations[0].InsertedRows, 0, 2)
c.Assert(gotRows, DeepEquals, expected)
tk.MustExec("update local_binlog set name = 'xyz' where id = 2")
prewriteVal = getLatestBinlogPrewriteValue(c, pump)
expected = [][]types.Datum{
{types.NewIntDatum(2), types.NewStringDatum("xyz")},
}
gotRows = mutationRowsToRows(c, prewriteVal.Mutations[0].UpdatedRows, 2, 4)
c.Assert(gotRows, DeepEquals, expected)
tk.MustExec("delete from local_binlog where id = 1")
prewriteVal = getLatestBinlogPrewriteValue(c, pump)
c.Assert(prewriteVal.Mutations[0].DeletedIds, DeepEquals, []int64{1})
// Test table primary key is not integer.
tk.MustExec("create table local_binlog2 (name varchar(64) primary key, age int)")
tk.MustExec("insert local_binlog2 values ('abc', 16), ('def', 18)")
tk.MustExec("delete from local_binlog2 where name = 'def'")
prewriteVal = getLatestBinlogPrewriteValue(c, pump)
_, deletedPK, _ := codec.DecodeOne(prewriteVal.Mutations[0].DeletedPks[0])
c.Assert(deletedPK.GetString(), Equals, "def")
// Test Table don't have primary key.
tk.MustExec("create table local_binlog3 (c1 int, c2 int)")
tk.MustExec("insert local_binlog3 values (1, 2), (1, 3), (2, 3)")
tk.MustExec("update local_binlog3 set c1 = 3 where c1 = 2")
prewriteVal = getLatestBinlogPrewriteValue(c, pump)
gotRows = mutationRowsToRows(c, prewriteVal.Mutations[0].UpdatedRows, 5, 7)
expected = [][]types.Datum{
{types.NewIntDatum(3), types.NewIntDatum(3)},
}
c.Assert(gotRows, DeepEquals, expected)
tk.MustExec("delete from local_binlog3 where c1 = 3 and c2 = 3")
prewriteVal = getLatestBinlogPrewriteValue(c, pump)
gotRows = mutationRowsToRows(c, prewriteVal.Mutations[0].DeletedRows, 1, 3)
expected = [][]types.Datum{
{types.NewIntDatum(3), types.NewIntDatum(3)},
}
c.Assert(gotRows, DeepEquals, expected)
checkBinlogCount(c, pump)
pump.mu.Lock()
originBinlogLen := len(pump.mu.payloads)
pump.mu.Unlock()
tk.MustExec("set @@global.autocommit = 0")
tk.MustExec("set @@global.autocommit = 1")
pump.mu.Lock()
newBinlogLen := len(pump.mu.payloads)
pump.mu.Unlock()
c.Assert(newBinlogLen, Equals, originBinlogLen)
}