func (krval bvcKeyRange) eval(bv interface{}, op Operator, onMismatch bool) bool {
switch op {
case QR_IN:
switch num, status := getuint64(bv); status {
case QR_OK:
k := key.Uint64Key(num).KeyspaceId()
return key.KeyRange(krval).Contains(k)
case QR_OUT_OF_RANGE:
return false
}
// Not a number. Check string.
switch str, status := getstring(bv); status {
case QR_OK:
return key.KeyRange(krval).Contains(key.KeyspaceId(str))
}
case QR_NOTIN:
switch num, status := getuint64(bv); status {
case QR_OK:
k := key.Uint64Key(num).KeyspaceId()
return !key.KeyRange(krval).Contains(k)
case QR_OUT_OF_RANGE:
return true
}
// Not a number. Check string.
switch str, status := getstring(bv); status {
case QR_OK:
return !key.KeyRange(krval).Contains(key.KeyspaceId(str))
}
default:
panic("unexpected:")
}
return onMismatch
}
func strKeyRange(start, end string) bvcKeyRange {
kr := key.KeyRange{
Start: key.KeyspaceId(start),
End: key.KeyspaceId(end),
}
return bvcKeyRange(kr)
}
func siBytes(start, end string) *topo.ShardInfo {
return topo.NewShardInfo("keyspace", start+"-"+end, &topo.Shard{
KeyRange: key.KeyRange{
Start: key.KeyspaceId(start),
End: key.KeyspaceId(end),
},
}, 0)
}
func TestKeyspaceIdQuery(t *testing.T) {
reflected, err := bson.Marshal(&reflectKeyspaceIdQuery{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
Keyspace: "keyspace",
KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("18")},
TabletType: "replica",
Session: &commonSession,
})
if err != nil {
t.Error(err)
}
want := string(reflected)
custom := KeyspaceIdQuery{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
Keyspace: "keyspace",
KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("18")},
TabletType: "replica",
Session: &commonSession,
}
encoded, err := bson.Marshal(&custom)
if err != nil {
t.Error(err)
}
got := string(encoded)
if want != got {
t.Errorf("want\n%+v, got\n%+v", want, got)
}
var unmarshalled KeyspaceIdQuery
err = bson.Unmarshal(encoded, &unmarshalled)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(custom, unmarshalled) {
t.Errorf("want \n%+v, got \n%+v", custom, unmarshalled)
}
extra, err := bson.Marshal(&extraKeyspaceIdQuery{})
if err != nil {
t.Error(err)
}
err = bson.Unmarshal(extra, &unmarshalled)
if err != nil {
t.Error(err)
}
}
// Split will split the rows into subset for each distribution
func (rs *RowSplitter) Split(result [][][]sqltypes.Value, rows [][]sqltypes.Value) error {
if rs.Type == key.KIT_UINT64 {
for _, row := range rows {
v := sqltypes.MakeNumeric(row[rs.ValueIndex].Raw())
i, err := v.ParseUint64()
if err != nil {
return fmt.Errorf("Non numerical value: %v", err)
}
k := key.Uint64Key(i).KeyspaceId()
for i, kr := range rs.KeyRanges {
if kr.Contains(k) {
result[i] = append(result[i], row)
break
}
}
}
} else {
for _, row := range rows {
k := key.KeyspaceId(row[rs.ValueIndex].Raw())
for i, kr := range rs.KeyRanges {
if kr.Contains(k) {
result[i] = append(result[i], row)
break
}
}
}
}
return nil
}
func TestHashConvert(t *testing.T) {
cases := []struct {
in uint64
out string
}{
{1, "\x16k@\xb4J\xbaK\xd6"},
{0, "\x8c\xa6M\xe9\xc1\xb1#\xa7"},
{11, "\xae\xfcDI\x1c\xfeGL"},
{0x100000000000000, "\r\x9f'\x9b\xa5\xd8r`"},
{0x800000000000000, " \xb9\xe7g\xb2\xfb\x14V"},
{11, "\xae\xfcDI\x1c\xfeGL"},
{0, "\x8c\xa6M\xe9\xc1\xb1#\xa7"},
}
for _, c := range cases {
got := string(vhash(c.in))
want := c.out
if got != want {
t.Errorf("vhash(%d): %#v, want %q", c.in, got, want)
}
back := vunhash(key.KeyspaceId(got))
if back != c.in {
t.Errorf("vunhash(%q): %d, want %d", got, back, c.in)
}
}
}
func buildBindVarCondition(bvc interface{}) (name string, onAbsent, onMismatch bool, op Operator, value interface{}, err error) {
bvcinfo, ok := bvc.(map[string]interface{})
if !ok {
err = NewTabletError(FAIL, "want json object for bind var conditions")
return
}
var v interface{}
v, ok = bvcinfo["Name"]
if !ok {
err = NewTabletError(FAIL, "Name missing in BindVarConds")
return
}
name, ok = v.(string)
if !ok {
err = NewTabletError(FAIL, "want string for Name in BindVarConds")
return
}
v, ok = bvcinfo["OnAbsent"]
if !ok {
err = NewTabletError(FAIL, "OnAbsent missing in BindVarConds")
return
}
onAbsent, ok = v.(bool)
if !ok {
err = NewTabletError(FAIL, "want bool for OnAbsent")
return
}
v, ok = bvcinfo["Operator"]
if !ok {
err = NewTabletError(FAIL, "Operator missing in BindVarConds")
return
}
strop, ok := v.(string)
if !ok {
err = NewTabletError(FAIL, "want string for Operator")
return
}
op, ok = opmap[strop]
if !ok {
err = NewTabletError(FAIL, "invalid Operator %s", strop)
return
}
if op == QR_NOOP {
return
}
v, ok = bvcinfo["Value"]
if !ok {
err = NewTabletError(FAIL, "Value missing in BindVarConds")
return
}
if op >= QR_EQ && op <= QR_LE {
strvalue, ok := v.(string)
if !ok {
err = NewTabletError(FAIL, "want string: %v", v)
return
}
if strop[0] == 'U' {
value, err = strconv.ParseUint(strvalue, 0, 64)
if err != nil {
err = NewTabletError(FAIL, "want uint64: %s", strvalue)
return
}
} else if strop[0] == 'I' {
value, err = strconv.ParseInt(strvalue, 0, 64)
if err != nil {
err = NewTabletError(FAIL, "want int64: %s", strvalue)
return
}
} else if strop[0] == 'S' {
value = strvalue
} else {
panic("unexpected")
}
} else if op == QR_MATCH || op == QR_NOMATCH {
strvalue, ok := v.(string)
if !ok {
err = NewTabletError(FAIL, "want string: %v", v)
return
}
value = strvalue
} else if op == QR_IN || op == QR_NOTIN {
kr, ok := v.(map[string]interface{})
if !ok {
err = NewTabletError(FAIL, "want keyrange for Value")
return
}
var keyrange key.KeyRange
strstart, ok := kr["Start"]
if !ok {
err = NewTabletError(FAIL, "Start missing in KeyRange")
return
}
start, ok := strstart.(string)
if !ok {
err = NewTabletError(FAIL, "want string for Start")
return
}
keyrange.Start = key.KeyspaceId(start)
strend, ok := kr["End"]
if !ok {
err = NewTabletError(FAIL, "End missing in KeyRange")
return
}
end, ok := strend.(string)
if !ok {
err = NewTabletError(FAIL, "want string for End")
return
}
keyrange.End = key.KeyspaceId(end)
value = keyrange
}
v, ok = bvcinfo["OnMismatch"]
if !ok {
err = NewTabletError(FAIL, "OnMismatch missing in BindVarConds")
return
}
onMismatch, ok = v.(bool)
if !ok {
err = NewTabletError(FAIL, "want bool for OnAbsent")
return
}
return
}
func TestKeyspaceIdBatchQuery(t *testing.T) {
reflected, err := bson.Marshal(&reflectKeyspaceIdBatchQuery{
Queries: []reflectBoundQuery{{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
}},
Keyspace: "keyspace",
KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("20")},
Session: &Session{InTransaction: true,
ShardSessions: []*ShardSession{{
Keyspace: "a",
Shard: "0",
TabletType: topo.TabletType("replica"),
TransactionId: 1,
}, {
Keyspace: "b",
Shard: "1",
TabletType: topo.TabletType("master"),
TransactionId: 2,
}},
},
})
if err != nil {
t.Error(err)
}
want := string(reflected)
custom := KeyspaceIdBatchQuery{
Queries: []tproto.BoundQuery{{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
}},
Keyspace: "keyspace",
KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("20")},
Session: &commonSession,
}
encoded, err := bson.Marshal(&custom)
if err != nil {
t.Error(err)
}
got := string(encoded)
if want != got {
t.Errorf("want\n%+v, got\n%+v", want, got)
}
var unmarshalled KeyspaceIdBatchQuery
err = bson.Unmarshal(encoded, &unmarshalled)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(custom, unmarshalled) {
t.Errorf("want \n%+v, got \n%+v", custom, unmarshalled)
}
extra, err := bson.Marshal(&extraKeyspaceIdBatchQuery{})
if err != nil {
t.Error(err)
}
err = bson.Unmarshal(extra, &unmarshalled)
if err != nil {
t.Error(err)
}
}
func vhash(shardKey uint64) key.KeyspaceId {
var keybytes, hashed [8]byte
binary.BigEndian.PutUint64(keybytes[:], shardKey)
block3DES.Encrypt(hashed[:], keybytes[:])
return key.KeyspaceId(hashed[:])
}
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package binlog
import (
"fmt"
"testing"
"github.com/henryanand/vitess/go/vt/binlog/proto"
"github.com/henryanand/vitess/go/vt/key"
myproto "github.com/henryanand/vitess/go/vt/mysqlctl/proto"
)
var testKeyRange = key.KeyRange{
Start: key.KeyspaceId(key.Uint64Key(0).String()),
End: key.KeyspaceId(key.Uint64Key(10).String()),
}
func TestKeyRangeFilterPass(t *testing.T) {
input := proto.BinlogTransaction{
Statements: []proto.Statement{
{
Category: proto.BL_SET,
Sql: []byte("set1"),
}, {
Category: proto.BL_DML,
Sql: []byte("dml1 /* EMD keyspace_id:20 */"),
}, {
Category: proto.BL_DML,
Sql: []byte("dml2 /* EMD keyspace_id:2 */"),
// KeyRangeFilterFunc returns a function that calls sendReply only if statements
// in the transaction match the specified keyrange. The resulting function can be
// passed into the BinlogStreamer: bls.Stream(file, pos, sendTransaction) ->
// bls.Stream(file, pos, KeyRangeFilterFunc(sendTransaction))
func KeyRangeFilterFunc(kit key.KeyspaceIdType, keyrange key.KeyRange, sendReply sendTransactionFunc) sendTransactionFunc {
isInteger := true
if kit == key.KIT_BYTES {
isInteger = false
}
return func(reply *proto.BinlogTransaction) error {
matched := false
filtered := make([]proto.Statement, 0, len(reply.Statements))
for _, statement := range reply.Statements {
switch statement.Category {
case proto.BL_SET:
filtered = append(filtered, statement)
case proto.BL_DDL:
log.Warningf("Not forwarding DDL: %s", string(statement.Sql))
continue
case proto.BL_DML:
keyspaceIndex := bytes.LastIndex(statement.Sql, KEYSPACE_ID_COMMENT)
if keyspaceIndex == -1 {
updateStreamErrors.Add("KeyRangeStream", 1)
log.Errorf("Error parsing keyspace id: %s", string(statement.Sql))
continue
}
idstart := keyspaceIndex + len(KEYSPACE_ID_COMMENT)
idend := bytes.Index(statement.Sql[idstart:], SPACE)
if idend == -1 {
updateStreamErrors.Add("KeyRangeStream", 1)
log.Errorf("Error parsing keyspace id: %s", string(statement.Sql))
continue
}
textId := string(statement.Sql[idstart : idstart+idend])
if isInteger {
id, err := strconv.ParseUint(textId, 10, 64)
if err != nil {
updateStreamErrors.Add("KeyRangeStream", 1)
log.Errorf("Error parsing keyspace id: %s", string(statement.Sql))
continue
}
if !keyrange.Contains(key.Uint64Key(id).KeyspaceId()) {
continue
}
} else {
data, err := base64.StdEncoding.DecodeString(textId)
if err != nil {
updateStreamErrors.Add("KeyRangeStream", 1)
log.Errorf("Error parsing keyspace id: %s", string(statement.Sql))
continue
}
if !keyrange.Contains(key.KeyspaceId(data)) {
continue
}
}
filtered = append(filtered, statement)
matched = true
case proto.BL_UNRECOGNIZED:
updateStreamErrors.Add("KeyRangeStream", 1)
log.Errorf("Error parsing keyspace id: %s", string(statement.Sql))
continue
}
}
if matched {
reply.Statements = filtered
} else {
reply.Statements = nil
}
return sendReply(reply)
}
}
