func (qs *QuerySplitter) parseInt(pkMinMax *mproto.QueryResult) ([]sqltypes.Value, error) {
boundaries := []sqltypes.Value{}
minNumeric := sqltypes.MakeNumeric(pkMinMax.Rows[0][0].Raw())
maxNumeric := sqltypes.MakeNumeric(pkMinMax.Rows[0][1].Raw())
if pkMinMax.Rows[0][0].Raw()[0] == '-' {
// signed values, use int64
min, err := minNumeric.ParseInt64()
if err != nil {
return nil, err
}
max, err := maxNumeric.ParseInt64()
if err != nil {
return nil, err
}
interval := (max - min) / int64(qs.splitCount)
if interval == 0 {
return nil, err
}
qs.rowCount = interval
for i := int64(1); i < int64(qs.splitCount); i++ {
v, err := sqltypes.BuildValue(min + interval*i)
if err != nil {
return nil, err
}
boundaries = append(boundaries, v)
}
return boundaries, nil
}
// unsigned values, use uint64
min, err := minNumeric.ParseUint64()
if err != nil {
return nil, err
}
max, err := maxNumeric.ParseUint64()
if err != nil {
return nil, err
}
interval := (max - min) / uint64(qs.splitCount)
if interval == 0 {
return nil, err
}
qs.rowCount = int64(interval)
for i := uint64(1); i < uint64(qs.splitCount); i++ {
v, err := sqltypes.BuildValue(min + interval*i)
if err != nil {
return nil, err
}
boundaries = append(boundaries, v)
}
return boundaries, nil
}
// Split will split the rows into subset for each distribution
func (rs *RowSplitter) Split(result [][][]sqltypes.Value, rows [][]sqltypes.Value) error {
if rs.Type == key.KIT_UINT64 {
for _, row := range rows {
v := sqltypes.MakeNumeric(row[rs.ValueIndex].Raw())
i, err := v.ParseUint64()
if err != nil {
return fmt.Errorf("Non numerical value: %v", err)
}
k := key.Uint64Key(i).KeyspaceId()
for i, kr := range rs.KeyRanges {
if kr.Contains(k) {
result[i] = append(result[i], row)
break
}
}
}
} else {
for _, row := range rows {
k := key.KeyspaceId(row[rs.ValueIndex].Raw())
for i, kr := range rs.KeyRanges {
if kr.Contains(k) {
result[i] = append(result[i], row)
break
}
}
}
}
return nil
}
// makeValueString returns a string that contains all the passed-in rows
// as an insert SQL command's parameters.
func makeValueString(fields []mproto.Field, rows [][]sqltypes.Value) string {
buf := bytes.Buffer{}
for i, row := range rows {
if i > 0 {
buf.Write([]byte(",("))
} else {
buf.WriteByte('(')
}
for j, value := range row {
if j > 0 {
buf.WriteByte(',')
}
// convert value back to its original type
if !value.IsNull() {
switch fields[j].Type {
case mproto.VT_TINY, mproto.VT_SHORT, mproto.VT_LONG, mproto.VT_LONGLONG, mproto.VT_INT24:
value = sqltypes.MakeNumeric(value.Raw())
case mproto.VT_FLOAT, mproto.VT_DOUBLE:
value = sqltypes.MakeFractional(value.Raw())
}
}
value.EncodeSql(&buf)
}
buf.WriteByte(')')
}
return buf.String()
}
func BuildValue(bytes []byte, fieldType uint32) sqltypes.Value {
if bytes == nil {
return sqltypes.NULL
}
switch fieldType {
case C.MYSQL_TYPE_DECIMAL, C.MYSQL_TYPE_FLOAT, C.MYSQL_TYPE_DOUBLE, C.MYSQL_TYPE_NEWDECIMAL:
return sqltypes.MakeFractional(bytes)
case C.MYSQL_TYPE_TIMESTAMP:
return sqltypes.MakeString(bytes)
}
// The below condition represents the following list of values:
// C.MYSQL_TYPE_TINY, C.MYSQL_TYPE_SHORT, C.MYSQL_TYPE_LONG, C.MYSQL_TYPE_LONGLONG, C.MYSQL_TYPE_INT24, C.MYSQL_TYPE_YEAR:
if fieldType <= C.MYSQL_TYPE_INT24 || fieldType == C.MYSQL_TYPE_YEAR {
return sqltypes.MakeNumeric(bytes)
}
return sqltypes.MakeString(bytes)
}
func (rc *RowCache) decodeRow(b []byte) (row []sqltypes.Value) {
rowlen := pack.Uint32(b)
data := b[4+rowlen*4:]
row = make([]sqltypes.Value, rowlen)
for i := range row {
length := pack.Uint32(b[4+i*4:])
if length == 0xFFFFFFFF {
continue
}
if length > uint32(len(data)) {
// Corrupt data
return nil
}
if rc.tableInfo.Columns[i].Category == schema.CAT_NUMBER {
row[i] = sqltypes.MakeNumeric(data[:length])
} else {
row[i] = sqltypes.MakeString(data[:length])
}
data = data[length:]
}
return row
}
func (vc *vcursor) Execute(query *tproto.BoundQuery) (*mproto.QueryResult, error) {
vc.query = query
switch {
case strings.HasPrefix(query.Sql, "select"):
return &mproto.QueryResult{
Fields: []mproto.Field{{
Type: mproto.VT_LONG,
}},
Rows: [][]sqltypes.Value{
[]sqltypes.Value{
sqltypes.MakeNumeric([]byte("1")),
},
},
RowsAffected: 1,
}, nil
case strings.HasPrefix(query.Sql, "insert"):
return &mproto.QueryResult{InsertId: 1}, nil
case strings.HasPrefix(query.Sql, "delete"):
return &mproto.QueryResult{}, nil
}
panic("unexpected")
}
func (ta *Table) AddColumn(name string, columnType string, defval sqltypes.Value, extra string) {
index := len(ta.Columns)
ta.Columns = append(ta.Columns, TableColumn{Name: name})
if strings.Contains(columnType, "int") {
ta.Columns[index].Category = CAT_NUMBER
} else if strings.HasPrefix(columnType, "varbinary") {
ta.Columns[index].Category = CAT_VARBINARY
} else {
ta.Columns[index].Category = CAT_OTHER
}
if extra == "auto_increment" {
ta.Columns[index].IsAuto = true
// Ignore default value, if any
return
}
if defval.IsNull() {
return
}
if ta.Columns[index].Category == CAT_NUMBER {
ta.Columns[index].Default = sqltypes.MakeNumeric(defval.Raw())
} else {
ta.Columns[index].Default = sqltypes.MakeString(defval.Raw())
}
}
func TestParsedQuery(t *testing.T) {
tcases := []struct {
desc string
query string
bindVars map[string]interface{}
output string
}{
{
"no subs",
"select * from a where id = 2",
map[string]interface{}{
"id": 1,
},
"select * from a where id = 2",
}, {
"simple bindvar sub",
"select * from a where id1 = :id1 and id2 = :id2",
map[string]interface{}{
"id1": 1,
"id2": nil,
},
"select * from a where id1 = 1 and id2 = null",
}, {
"missing bind var",
"select * from a where id1 = :id1 and id2 = :id2",
map[string]interface{}{
"id1": 1,
},
"missing bind var id2",
}, {
"unencodable bind var",
"select * from a where id1 = :id",
map[string]interface{}{
"id": make([]int, 1),
},
"unsupported bind variable type []int: [0]",
}, {
"list inside bind vars",
"select * from a where id in (:vals)",
map[string]interface{}{
"vals": []sqltypes.Value{
sqltypes.MakeNumeric([]byte("1")),
sqltypes.MakeString([]byte("aa")),
},
},
"select * from a where id in (1, 'aa')",
}, {
"two lists inside bind vars",
"select * from a where id in (:vals)",
map[string]interface{}{
"vals": [][]sqltypes.Value{
[]sqltypes.Value{
sqltypes.MakeNumeric([]byte("1")),
sqltypes.MakeString([]byte("aa")),
},
[]sqltypes.Value{
sqltypes.Value{},
sqltypes.MakeString([]byte("bb")),
},
},
},
"select * from a where id in ((1, 'aa'), (null, 'bb'))",
}, {
"list bind vars",
"select * from a where id in ::vals",
map[string]interface{}{
"vals": []interface{}{
1,
"aa",
},
},
"select * from a where id in (1, 'aa')",
}, {
"list bind vars single argument",
"select * from a where id in ::vals",
map[string]interface{}{
"vals": []interface{}{
1,
},
},
"select * from a where id in (1)",
}, {
"list bind vars 0 arguments",
"select * from a where id in ::vals",
map[string]interface{}{
"vals": []interface{}{},
},
"empty list supplied for vals",
}, {
"non-list bind var supplied",
"select * from a where id in ::vals",
map[string]interface{}{
"vals": 1,
},
"unexpected list arg type int for key vals",
}, {
"list bind var for non-list",
"select * from a where id = :vals",
map[string]interface{}{
"vals": []interface{}{1},
},
"unexpected arg type []interface {} for key vals",
}, {
"single column tuple equality",
// We have to use an incorrect construct to get around the parser.
"select * from a where b = :equality",
map[string]interface{}{
"equality": TupleEqualityList{
Columns: []string{"pk"},
Rows: [][]sqltypes.Value{
[]sqltypes.Value{sqltypes.MakeNumeric([]byte("1"))},
[]sqltypes.Value{sqltypes.MakeString([]byte("aa"))},
},
},
},
"select * from a where b = pk in (1, 'aa')",
}, {
"multi column tuple equality",
"select * from a where b = :equality",
map[string]interface{}{
"equality": TupleEqualityList{
Columns: []string{"pk1", "pk2"},
Rows: [][]sqltypes.Value{
[]sqltypes.Value{
sqltypes.MakeNumeric([]byte("1")),
sqltypes.MakeString([]byte("aa")),
},
[]sqltypes.Value{
sqltypes.MakeNumeric([]byte("2")),
sqltypes.MakeString([]byte("bb")),
},
},
},
},
"select * from a where b = (pk1 = 1 and pk2 = 'aa') or (pk1 = 2 and pk2 = 'bb')",
}, {
"0 rows",
"select * from a where b = :equality",
map[string]interface{}{
"equality": TupleEqualityList{
Columns: []string{"pk"},
Rows: [][]sqltypes.Value{},
},
},
"cannot encode with 0 rows",
}, {
"values don't match column count",
"select * from a where b = :equality",
map[string]interface{}{
"equality": TupleEqualityList{
Columns: []string{"pk"},
Rows: [][]sqltypes.Value{
[]sqltypes.Value{
sqltypes.MakeNumeric([]byte("1")),
sqltypes.MakeString([]byte("aa")),
},
},
},
},
"values don't match column count",
},
}
for _, tcase := range tcases {
tree, err := Parse(tcase.query)
if err != nil {
t.Errorf("parse failed for %s: %v", tcase.desc, err)
continue
}
buf := NewTrackedBuffer(nil)
buf.Myprintf("%v", tree)
pq := buf.ParsedQuery()
bytes, err := pq.GenerateQuery(tcase.bindVars)
var got string
if err != nil {
got = err.Error()
} else {
got = string(bytes)
}
if got != tcase.output {
t.Errorf("for test case: %s, got: '%s', want '%s'", tcase.desc, got, tcase.output)
}
}
}
encoded: "E\x00\x00\x00\x04Fields\x00\x05\x00\x00\x00\x00?RowsAffected\x00\x00\x00\x00\x00\x00\x00\x00\x00?InsertId\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04Rows\x00\x05\x00\x00\x00\x00\x00",
},
// Only fields set
{
qr: QueryResult{
Fields: []Field{
{Name: "foo", Type: 1},
},
},
encoded: "i\x00\x00\x00\x04Fields\x00)\x00\x00\x00\x030\x00!\x00\x00\x00\x05Name\x00\x03\x00\x00\x00\x00foo\x12Type\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00?RowsAffected\x00\x00\x00\x00\x00\x00\x00\x00\x00?InsertId\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04Rows\x00\x05\x00\x00\x00\x00\x00",
},
// Only rows, no fields
{
qr: QueryResult{
Rows: [][]sqltypes.Value{
{sqltypes.MakeString([]byte("abcd")), sqltypes.MakeNumeric([]byte("1234")), sqltypes.MakeFractional([]byte("1.234"))},
},
},
encoded: "r\x00\x00\x00\x04Fields\x00\x05\x00\x00\x00\x00?RowsAffected\x00\x00\x00\x00\x00\x00\x00\x00\x00?InsertId\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04Rows\x002\x00\x00\x00\x040\x00*\x00\x00\x00\x050\x00\x04\x00\x00\x00\x00abcd\x051\x00\x04\x00\x00\x00\x001234\x052\x00\x05\x00\x00\x00\x001.234\x00\x00\x00",
},
// one row and one field
{
qr: QueryResult{
Fields: []Field{
{Name: "foo", Type: 1},
},
Rows: [][]sqltypes.Value{
{sqltypes.MakeString([]byte("abcd")), sqltypes.MakeNumeric([]byte("1234")), sqltypes.MakeFractional([]byte("1.234")), sqltypes.Value{}},
},
},
encoded: "",
// findChunks returns an array of chunks to use for splitting up a table
// into multiple data chunks. It only works for tables with a primary key
// (and the primary key first column is an integer type).
// The array will always look like:
// "", "value1", "value2", ""
// A non-split tablet will just return:
// "", ""
func findChunks(wr *wrangler.Wrangler, ti *topo.TabletInfo, td *myproto.TableDefinition, minTableSizeForSplit uint64, sourceReaderCount int) ([]string, error) {
result := []string{"", ""}
// eliminate a few cases we don't split tables for
if len(td.PrimaryKeyColumns) == 0 {
// no primary key, what can we do?
return result, nil
}
if td.DataLength < minTableSizeForSplit {
// table is too small to split up
return result, nil
}
// get the min and max of the leading column of the primary key
query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", td.PrimaryKeyColumns[0], td.PrimaryKeyColumns[0], ti.DbName(), td.Name)
ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
qr, err := wr.TabletManagerClient().ExecuteFetch(ctx, ti, query, 1, true, false)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks: %v", td.Name, err)
return result, nil
}
cancel()
if len(qr.Rows) != 1 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot get min and max", td.Name)
return result, nil
}
if qr.Rows[0][0].IsNull() || qr.Rows[0][1].IsNull() {
wr.Logger().Infof("Not splitting table %v into multiple chunks, min or max is NULL: %v %v", td.Name, qr.Rows[0][0], qr.Rows[0][1])
return result, nil
}
switch qr.Fields[0].Type {
case mproto.VT_TINY, mproto.VT_SHORT, mproto.VT_LONG, mproto.VT_LONGLONG, mproto.VT_INT24:
minNumeric := sqltypes.MakeNumeric(qr.Rows[0][0].Raw())
maxNumeric := sqltypes.MakeNumeric(qr.Rows[0][1].Raw())
if qr.Rows[0][0].Raw()[0] == '-' {
// signed values, use int64
min, err := minNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / int64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := int64(1); i < int64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
}
// unsigned values, use uint64
min, err := minNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / uint64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := uint64(1); i < uint64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
case mproto.VT_FLOAT, mproto.VT_DOUBLE:
min, err := strconv.ParseFloat(qr.Rows[0][0].String(), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, qr.Rows[0][0], err)
return result, nil
}
max, err := strconv.ParseFloat(qr.Rows[0][1].String(), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, qr.Rows[0][1].String(), err)
return result, nil
}
interval := (max - min) / float64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := 1; i < sourceReaderCount; i++ {
result[i] = fmt.Sprintf("%v", min+interval*float64(i))
}
return result, nil
}
wr.Logger().Infof("Not splitting table %v into multiple chunks, primary key not numeric", td.Name)
return result, nil
}