// EncodeColumnKey encodes the table id, row handle and columnID into a kv.Key
func EncodeColumnKey(tableID int64, handle int64, columnID int64) kv.Key {
buf := make([]byte, 0, recordRowKeyLen+idLen)
buf = appendTableRecordPrefix(buf, tableID)
buf = codec.EncodeInt(buf, handle)
buf = codec.EncodeInt(buf, columnID)
return buf
}
func (s *testCoprocessorSuite) TestBuildHugeTasks(c *C) {
cluster := mocktikv.NewCluster()
var splitKeys [][]byte
for ch := byte('a'); ch <= byte('z'); ch++ {
splitKeys = append(splitKeys, []byte{ch})
}
mocktikv.BootstrapWithMultiRegions(cluster, splitKeys...)
bo := NewBackoffer(3000)
cache := NewRegionCache(mocktikv.NewPDClient(cluster))
const rangesPerRegion = 1e6
ranges := make([]kv.KeyRange, 0, 26*rangesPerRegion)
for ch := byte('a'); ch <= byte('z'); ch++ {
for i := 0; i < rangesPerRegion; i++ {
start := make([]byte, 0, 9)
end := make([]byte, 0, 9)
ranges = append(ranges, kv.KeyRange{
StartKey: codec.EncodeInt(append(start, ch), int64(i*2)),
EndKey: codec.EncodeInt(append(end, ch), int64(i*2+1)),
})
}
}
_, err := buildCopTasks(bo, cache, &copRanges{mid: ranges}, false)
c.Assert(err, IsNil)
}
func datumExpr(d types.Datum) *tipb.Expr {
expr := new(tipb.Expr)
switch d.Kind() {
case types.KindInt64:
expr.Tp = tipb.ExprType_Int64
expr.Val = codec.EncodeInt(nil, d.GetInt64())
case types.KindUint64:
expr.Tp = tipb.ExprType_Uint64
expr.Val = codec.EncodeUint(nil, d.GetUint64())
case types.KindString:
expr.Tp = tipb.ExprType_String
expr.Val = d.GetBytes()
case types.KindBytes:
expr.Tp = tipb.ExprType_Bytes
expr.Val = d.GetBytes()
case types.KindFloat32:
expr.Tp = tipb.ExprType_Float32
expr.Val = codec.EncodeFloat(nil, d.GetFloat64())
case types.KindFloat64:
expr.Tp = tipb.ExprType_Float64
expr.Val = codec.EncodeFloat(nil, d.GetFloat64())
case types.KindMysqlDuration:
expr.Tp = tipb.ExprType_MysqlDuration
expr.Val = codec.EncodeInt(nil, int64(d.GetMysqlDuration().Duration))
case types.KindMysqlDecimal:
expr.Tp = tipb.ExprType_MysqlDecimal
expr.Val = codec.EncodeDecimal(nil, d)
default:
expr.Tp = tipb.ExprType_Null
}
return expr
}
// EncodeRecordKey encodes the recordPrefix, row handle and columnID into a kv.Key.
// TODO: Remove this function
func EncodeRecordKey(recordPrefix kv.Key, h int64, columnID int64) kv.Key {
buf := make([]byte, 0, len(recordPrefix)+16)
buf = append(buf, recordPrefix...)
buf = codec.EncodeInt(buf, h)
if columnID != 0 {
buf = codec.EncodeInt(buf, columnID)
}
return buf
}
// TODO: add more tests.
func (s *tableCodecSuite) TestTableCodec(c *C) {
key := EncodeRowKey(1, codec.EncodeInt(nil, 2))
h, err := DecodeRowKey(key)
c.Assert(err, IsNil)
c.Assert(h, Equals, int64(2))
key = EncodeColumnKey(1, codec.EncodeInt(nil, 2), 3)
h, err = DecodeRowKey(key)
c.Assert(err, IsNil)
c.Assert(h, Equals, int64(2))
}
func tableRangesToPBRanges(tableRanges []plan.TableRange) []*tipb.KeyRange {
hrs := make([]*tipb.KeyRange, 0, len(tableRanges))
for _, tableRange := range tableRanges {
pbRange := new(tipb.KeyRange)
pbRange.Low = codec.EncodeInt(nil, tableRange.LowVal)
hi := tableRange.HighVal
if hi != math.MaxInt64 {
hi++
}
pbRange.High = codec.EncodeInt(nil, hi)
hrs = append(hrs, pbRange)
}
return hrs
}
func (b *executorBuilder) columnToPBExpr(client kv.Client, column *expression.Column, tbl *model.TableInfo) *tipb.Expr {
if !client.SupportRequestType(kv.ReqTypeSelect, int64(tipb.ExprType_ColumnRef)) {
return nil
}
switch column.GetType().Tp {
case mysql.TypeBit, mysql.TypeSet, mysql.TypeEnum, mysql.TypeDecimal, mysql.TypeGeometry,
mysql.TypeDate, mysql.TypeNewDate, mysql.TypeDatetime, mysql.TypeTimestamp, mysql.TypeYear:
return nil
}
id := int64(-1)
for _, col := range tbl.Columns {
if tbl.Name == column.TblName && col.Name == column.ColName {
id = col.ID
break
}
}
// Zero Column ID is not a column from table, can not support for now.
if id == 0 {
return nil
}
// TODO：If the column ID isn't in fields, it means the column is from an outer table,
// its value is available to use.
if id == -1 {
return nil
}
return &tipb.Expr{
Tp: tipb.ExprType_ColumnRef.Enum(),
Val: codec.EncodeInt(nil, id)}
}
func (t *TxStructure) encodeListDataKey(key []byte, index int64) kv.Key {
ek := make([]byte, 0, len(t.prefix)+len(key)+36)
ek = append(ek, t.prefix...)
ek = codec.EncodeBytes(ek, key)
ek = codec.EncodeUint(ek, uint64(ListData))
return codec.EncodeInt(ek, index)
}
func (e *NewXSelectIndexExec) doTableRequest(handles []int64) (*xapi.SelectResult, error) {
txn, err := e.ctx.GetTxn(false)
if err != nil {
return nil, errors.Trace(err)
}
// The handles are not in original index order, so we can't push limit here.
selTableReq := new(tipb.SelectRequest)
startTs := txn.StartTS()
selTableReq.StartTs = &startTs
selTableReq.TableInfo = &tipb.TableInfo{
TableId: proto.Int64(e.table.Meta().ID),
}
selTableReq.TableInfo.Columns = xapi.ColumnsToProto(e.indexPlan.Columns, e.table.Meta().PKIsHandle)
for _, h := range handles {
if h == math.MaxInt64 {
// We can't convert MaxInt64 into an left closed, right open range.
continue
}
pbRange := new(tipb.KeyRange)
pbRange.Low = codec.EncodeInt(nil, h)
pbRange.High = kv.Key(pbRange.Low).PrefixNext()
selTableReq.Ranges = append(selTableReq.Ranges, pbRange)
}
selTableReq.Where = e.where
// Aggregate Info
resp, err := xapi.Select(txn.GetClient(), selTableReq, defaultConcurrency)
if err != nil {
return nil, errors.Trace(err)
}
return resp, nil
}
func setRow(txn kv.Transaction, handle int64, tbl *simpleTableInfo, gen genValueFunc) error {
rowKey := tablecodec.EncodeRowKey(tbl.tID, codec.EncodeInt(nil, handle))
columnValues := gen(handle, tbl)
value, err := tablecodec.EncodeRow(columnValues, tbl.cIDs)
if err != nil {
return errors.Trace(err)
}
err = txn.Set(rowKey, value)
if err != nil {
return errors.Trace(err)
}
for i, idxCol := range tbl.indices {
idxVal := columnValues[idxCol]
encoded, err := codec.EncodeKey(nil, idxVal, types.NewDatum(handle))
if err != nil {
return errors.Trace(err)
}
idxKey := tablecodec.EncodeIndexSeekKey(tbl.tID, tbl.iIDs[i], encoded)
err = txn.Set(idxKey, []byte{0})
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (e *NewXSelectIndexExec) doTableRequest(handles []int64) (xapi.SelectResult, error) {
// The handles are not in original index order, so we can't push limit here.
selTableReq := new(tipb.SelectRequest)
selTableReq.StartTs = e.txn.StartTS()
selTableReq.TableInfo = &tipb.TableInfo{
TableId: e.table.Meta().ID,
}
selTableReq.TableInfo.Columns = xapi.ColumnsToProto(e.indexPlan.Columns, e.table.Meta().PKIsHandle)
for _, h := range handles {
if h == math.MaxInt64 {
// We can't convert MaxInt64 into an left closed, right open range.
continue
}
pbRange := new(tipb.KeyRange)
pbRange.Low = codec.EncodeInt(nil, h)
pbRange.High = kv.Key(pbRange.Low).PrefixNext()
selTableReq.Ranges = append(selTableReq.Ranges, pbRange)
}
selTableReq.Where = e.where
// Aggregate Info
selTableReq.Aggregates = e.aggFuncs
selTableReq.GroupBy = e.byItems
// Aggregate Info
resp, err := xapi.Select(e.txn.GetClient(), selTableReq, defaultConcurrency, false)
if err != nil {
return nil, errors.Trace(err)
}
if e.aggregate {
// The returned rows should be aggregate partial result.
resp.SetFields(e.aggFields)
}
resp.Fetch()
return resp, nil
}
// EncodeIndexSeekKey encodes an index value to kv.Key.
func EncodeIndexSeekKey(tableID int64, idxID int64, encodedValue []byte) kv.Key {
key := make([]byte, 0, prefixLen+len(encodedValue))
key = appendTableIndexPrefix(key, tableID)
key = codec.EncodeInt(key, idxID)
key = append(key, encodedValue...)
return key
}
func (b *executorBuilder) columnNameToPBExpr(client kv.Client, column *ast.ColumnNameExpr, tn *ast.TableName) *tipb.Expr {
if !client.SupportRequestType(kv.ReqTypeSelect, int64(tipb.ExprType_ColumnRef)) {
return nil
}
// Zero Column ID is not a column from table, can not support for now.
if column.Refer.Column.ID == 0 {
return nil
}
switch column.Refer.Expr.GetType().Tp {
case mysql.TypeBit, mysql.TypeSet, mysql.TypeEnum, mysql.TypeDecimal, mysql.TypeGeometry,
mysql.TypeDate, mysql.TypeNewDate, mysql.TypeDatetime, mysql.TypeTimestamp, mysql.TypeYear:
return nil
}
matched := false
for _, f := range tn.GetResultFields() {
if f.TableName == column.Refer.TableName && f.Column.ID == column.Refer.Column.ID {
matched = true
break
}
}
if matched {
pbExpr := new(tipb.Expr)
pbExpr.Tp = tipb.ExprType_ColumnRef.Enum()
pbExpr.Val = codec.EncodeInt(nil, column.Refer.Column.ID)
return pbExpr
}
// If the column ID isn't in fields, it means the column is from an outer table,
// its value is available to use.
return b.datumToPBExpr(client, *column.Refer.Expr.GetDatum())
}
// index prefix is "t[tableID]_i"
func genTableIndexPrefix(tableID int64) kv.Key {
buf := make([]byte, 0, len(TablePrefix)+8+len(indexPrefixSep))
buf = append(buf, TablePrefix...)
buf = codec.EncodeInt(buf, tableID)
buf = append(buf, indexPrefixSep...)
return buf
}
func (e *XSelectIndexExec) doTableRequest(txn kv.Transaction, handles []int64) (*xapi.SelectResult, error) {
selTableReq := new(tipb.SelectRequest)
startTs := txn.StartTS()
selTableReq.StartTs = &startTs
selTableReq.TableInfo = tablecodec.TableToProto(e.indexPlan.Table)
selTableReq.Fields = resultFieldsToPBExpression(e.indexPlan.Fields())
for _, h := range handles {
if h == math.MaxInt64 {
// We can't convert MaxInt64 into an left closed, right open range.
continue
}
pbRange := new(tipb.KeyRange)
pbRange.Low = codec.EncodeInt(nil, h)
pbRange.High = codec.EncodeInt(nil, h)
selTableReq.Ranges = append(selTableReq.Ranges, pbRange)
}
selTableReq.Where = conditionsToPBExpression(e.indexPlan.FilterConditions...)
return xapi.Select(txn.GetClient(), selTableReq, 10)
}
// TODO: add more tests.
func (s *testTableCodecSuite) TestTableCodec(c *C) {
defer testleak.AfterTest(c)()
key := EncodeRowKey(1, codec.EncodeInt(nil, 2))
h, err := DecodeRowKey(key)
c.Assert(err, IsNil)
c.Assert(h, Equals, int64(2))
key = EncodeRowKeyWithHandle(1, 2)
h, err = DecodeRowKey(key)
c.Assert(err, IsNil)
c.Assert(h, Equals, int64(2))
}
func (b *executorBuilder) datumToPBExpr(client kv.Client, d types.Datum) *tipb.Expr {
var tp tipb.ExprType
var val []byte
switch d.Kind() {
case types.KindNull:
tp = tipb.ExprType_Null
case types.KindInt64:
tp = tipb.ExprType_Int64
val = codec.EncodeInt(nil, d.GetInt64())
case types.KindUint64:
tp = tipb.ExprType_Uint64
val = codec.EncodeUint(nil, d.GetUint64())
case types.KindString:
tp = tipb.ExprType_String
val = d.GetBytes()
case types.KindBytes:
tp = tipb.ExprType_Bytes
val = d.GetBytes()
case types.KindFloat32:
tp = tipb.ExprType_Float32
val = codec.EncodeFloat(nil, d.GetFloat64())
case types.KindFloat64:
tp = tipb.ExprType_Float64
val = codec.EncodeFloat(nil, d.GetFloat64())
case types.KindMysqlDuration:
tp = tipb.ExprType_MysqlDuration
val = codec.EncodeInt(nil, int64(d.GetMysqlDuration().Duration))
case types.KindMysqlDecimal:
tp = tipb.ExprType_MysqlDecimal
val = codec.EncodeDecimal(nil, d.GetMysqlDecimal())
default:
return nil
}
if !client.SupportRequestType(kv.ReqTypeSelect, int64(tp)) {
return nil
}
return &tipb.Expr{Tp: tp.Enum(), Val: val}
}
func (e *XSelectIndexExec) doTableRequest(handles []int64) (*xapi.SelectResult, error) {
txn, err := e.ctx.GetTxn(false)
if err != nil {
return nil, errors.Trace(err)
}
// The handles are not in original index order, so we can't push limit here.
selTableReq := new(tipb.SelectRequest)
startTs := txn.StartTS()
selTableReq.StartTs = &startTs
columns := make([]*model.ColumnInfo, 0, len(e.indexPlan.Fields()))
for _, v := range e.indexPlan.Fields() {
if v.Referenced {
columns = append(columns, v.Column)
}
}
selTableReq.TableInfo = &tipb.TableInfo{
TableId: proto.Int64(e.table.Meta().ID),
}
selTableReq.TableInfo.Columns = xapi.ColumnsToProto(columns, e.table.Meta().PKIsHandle)
selTableReq.Fields = resultFieldsToPBExpression(e.indexPlan.Fields())
for _, h := range handles {
if h == math.MaxInt64 {
// We can't convert MaxInt64 into an left closed, right open range.
continue
}
pbRange := new(tipb.KeyRange)
pbRange.Low = codec.EncodeInt(nil, h)
pbRange.High = kv.Key(pbRange.Low).PrefixNext()
selTableReq.Ranges = append(selTableReq.Ranges, pbRange)
}
selTableReq.Where = e.where
// Aggregate Info
selTableReq.Aggregates = e.aggFuncs
selTableReq.GroupBy = e.byItems
resp, err := xapi.Select(txn.GetClient(), selTableReq, defaultConcurrency)
if err != nil {
return nil, errors.Trace(err)
}
if e.aggregate {
// The returned rows should be aggregate partial result.
resp.SetFields(e.aggFields)
}
return resp, nil
}
func (pc pbConverter) columnToPBExpr(column *expression.Column) *tipb.Expr {
if !pc.client.SupportRequestType(kv.ReqTypeSelect, int64(tipb.ExprType_ColumnRef)) {
return nil
}
switch column.GetType().Tp {
case mysql.TypeBit, mysql.TypeSet, mysql.TypeEnum, mysql.TypeGeometry, mysql.TypeDecimal:
return nil
}
id := column.ID
// Zero Column ID is not a column from table, can not support for now.
if id == 0 || id == -1 {
return nil
}
return &tipb.Expr{
Tp: tipb.ExprType_ColumnRef,
Val: codec.EncodeInt(nil, id)}
}
// GenIndexPrefix generates the index prefix.
func GenIndexPrefix(indexPrefix Key, indexID int64) Key {
buf := make([]byte, 0, len(indexPrefix)+8)
buf = append(buf, indexPrefix...)
buf = codec.EncodeInt(buf, indexID)
return buf
}
func (s *testSuite) testIndex(c *C, tb table.Table, idx table.Index) {
txn, err := s.store.Begin()
c.Assert(err, IsNil)
err = CompareIndexData(txn, tb, idx)
c.Assert(err, IsNil)
cnt, err := GetIndexRecordsCount(txn, idx, nil)
c.Assert(err, IsNil)
c.Assert(cnt, Equals, int64(2))
// set data to:
// index data (handle, data): (1, 10), (2, 20), (3, 30)
// table data (handle, data): (1, 10), (2, 20), (4, 40)
err = idx.Create(txn, types.MakeDatums(int64(30)), 3)
c.Assert(err, IsNil)
key := tablecodec.EncodeRowKey(tb.Meta().ID, codec.EncodeInt(nil, 4))
setColValue(c, txn, key, types.NewDatum(int64(40)))
err = txn.Commit()
c.Assert(err, IsNil)
txn, err = s.store.Begin()
c.Assert(err, IsNil)
err = CompareIndexData(txn, tb, idx)
c.Assert(err, NotNil)
record1 := &RecordData{Handle: int64(3), Values: types.MakeDatums(int64(30))}
diffMsg := newDiffRetError("index", record1, nil)
c.Assert(err.Error(), DeepEquals, diffMsg)
// set data to:
// index data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40)
// table data (handle, data): (1, 10), (2, 20), (4, 40), (3, 31)
err = idx.Create(txn, types.MakeDatums(int64(40)), 4)
c.Assert(err, IsNil)
key = tablecodec.EncodeRowKey(tb.Meta().ID, codec.EncodeInt(nil, 3))
setColValue(c, txn, key, types.NewDatum(int64(31)))
err = txn.Commit()
c.Assert(err, IsNil)
txn, err = s.store.Begin()
c.Assert(err, IsNil)
err = CompareIndexData(txn, tb, idx)
c.Assert(err, NotNil)
record2 := &RecordData{Handle: int64(3), Values: types.MakeDatums(int64(31))}
diffMsg = newDiffRetError("index", record1, record2)
c.Assert(err.Error(), DeepEquals, diffMsg)
// set data to:
// index data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40)
// table data (handle, data): (1, 10), (2, 20), (4, 40), (5, 30)
key = tablecodec.EncodeRowKey(tb.Meta().ID, codec.EncodeInt(nil, 3))
txn.Delete(key)
key = tablecodec.EncodeRowKey(tb.Meta().ID, codec.EncodeInt(nil, 5))
setColValue(c, txn, key, types.NewDatum(int64(30)))
err = txn.Commit()
c.Assert(err, IsNil)
txn, err = s.store.Begin()
c.Assert(err, IsNil)
err = checkRecordAndIndex(txn, tb, idx)
c.Assert(err, NotNil)
record2 = &RecordData{Handle: int64(5), Values: types.MakeDatums(int64(30))}
diffMsg = newDiffRetError("index", record1, record2)
c.Assert(err.Error(), DeepEquals, diffMsg)
// set data to:
// index data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40)
// table data (handle, data): (1, 10), (2, 20), (3, 30)
key = tablecodec.EncodeRowKey(tb.Meta().ID, codec.EncodeInt(nil, 4))
txn.Delete(key)
key = tablecodec.EncodeRowKey(tb.Meta().ID, codec.EncodeInt(nil, 3))
setColValue(c, txn, key, types.NewDatum(int64(30)))
err = txn.Commit()
c.Assert(err, IsNil)
txn, err = s.store.Begin()
c.Assert(err, IsNil)
err = CompareIndexData(txn, tb, idx)
c.Assert(err, NotNil)
record1 = &RecordData{Handle: int64(4), Values: types.MakeDatums(int64(40))}
diffMsg = newDiffRetError("index", record1, nil)
c.Assert(err.Error(), DeepEquals, diffMsg)
// set data to:
// index data (handle, data): (1, 10), (2, 20), (3, 30)
// table data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40)
err = idx.Delete(txn, types.MakeDatums(int64(40)), 4)
c.Assert(err, IsNil)
key = tablecodec.EncodeRowKey(tb.Meta().ID, codec.EncodeInt(nil, 4))
setColValue(c, txn, key, types.NewDatum(int64(40)))
err = txn.Commit()
c.Assert(err, IsNil)
txn, err = s.store.Begin()
c.Assert(err, IsNil)
err = CompareIndexData(txn, tb, idx)
c.Assert(err, NotNil)
diffMsg = newDiffRetError("index", nil, record1)
c.Assert(err.Error(), DeepEquals, diffMsg)
}
// Index prefix is "t[tableID]_i".
func appendTableIndexPrefix(buf []byte, tableID int64) []byte {
buf = append(buf, tablePrefix...)
buf = codec.EncodeInt(buf, tableID)
buf = append(buf, indexPrefixSep...)
return buf
}
// EncodeTablePrefix encodes table prefix with table ID.
func EncodeTablePrefix(tableID int64) kv.Key {
var key kv.Key
key = append(key, tablePrefix...)
key = codec.EncodeInt(key, tableID)
return key
}
// EncodeTableIndexPrefix encodes index prefix with tableID and idxID.
func EncodeTableIndexPrefix(tableID, idxID int64) kv.Key {
key := make([]byte, 0, prefixLen)
key = appendTableIndexPrefix(key, tableID)
key = codec.EncodeInt(key, idxID)
return key
}
func fullTableRange(tid int64) kv.KeyRange {
return kv.KeyRange{
StartKey: tablecodec.EncodeRowKey(tid, codec.EncodeInt(nil, math.MinInt64)),
EndKey: tablecodec.EncodeRowKey(tid, codec.EncodeInt(nil, math.MaxInt64)),
}
}
req.Tp = kv.ReqTypeSelect
req.Concurrency = 1
req.KeyRanges = []kv.KeyRange{fullTableRange(simpleInfo.tID)}
req.Data = data
return req, nil
}
func fullTableRange(tid int64) kv.KeyRange {
return kv.KeyRange{
StartKey: tablecodec.EncodeRowKey(tid, codec.EncodeInt(nil, math.MinInt64)),
EndKey: tablecodec.EncodeRowKey(tid, codec.EncodeInt(nil, math.MaxInt64)),
}
}
var fullPBTableRange = &tipb.KeyRange{
Low: codec.EncodeInt(nil, math.MinInt64),
High: codec.EncodeInt(nil, math.MaxInt64),
}
var fullPBIndexRange = &tipb.KeyRange{
Low: []byte{0},
High: []byte{255},
}
func fullIndexRange(tid int64, idxID int64) kv.KeyRange {
return kv.KeyRange{
StartKey: tablecodec.EncodeIndexSeekKey(tid, idxID, []byte{0}),
EndKey: tablecodec.EncodeIndexSeekKey(tid, idxID, []byte{255}),
}
}
func prepareIndexRequest(simpleInfo *simpleTableInfo, startTs uint64) (*kv.Request, error) {
func columnExpr(columnID int64) *tipb.Expr {
expr := new(tipb.Expr)
expr.Tp = tipb.ExprType_ColumnRef
expr.Val = codec.EncodeInt(nil, columnID)
return expr
}