func (client *client) StreamKeyRange(req *proto.KeyRangeRequest, responseChan chan *proto.BinlogTransaction) binlogplayer.BinlogPlayerResponse {
query := &pb.StreamKeyRangeRequest{
Position: myproto.ReplicationPositionToProto(req.Position),
KeyspaceIdType: key.KeyspaceIdTypeToProto(req.KeyspaceIdType),
KeyRange: key.KeyRangeToProto(req.KeyRange),
Charset: mproto.CharsetToProto(req.Charset),
}
response := &response{}
stream, err := client.c.StreamKeyRange(client.ctx, query)
if err != nil {
response.err = err
close(responseChan)
return response
}
go func() {
for {
r, err := stream.Recv()
if err != nil {
if err != io.EOF {
response.err = err
}
close(responseChan)
return
}
responseChan <- proto.ProtoToBinlogTransaction(r.BinlogTransaction)
}
}()
return response
}
// SrvKeyspaceToProto turns a Tablet into a proto
func SrvKeyspaceToProto(s *SrvKeyspace) *pb.SrvKeyspace {
result := &pb.SrvKeyspace{
ShardingColumnName: s.ShardingColumnName,
ShardingColumnType: key.KeyspaceIdTypeToProto(s.ShardingColumnType),
SplitShardCount: s.SplitShardCount,
}
for tt, p := range s.Partitions {
partition := &pb.SrvKeyspace_KeyspacePartition{
ServedType: TabletTypeToProto(tt),
}
for _, sr := range p.ShardReferences {
partition.ShardReferences = append(partition.ShardReferences, &pb.ShardReference{
Name: sr.Name,
KeyRange: key.KeyRangeToProto(sr.KeyRange),
})
}
result.Partitions = append(result.Partitions, partition)
}
for tt, k := range s.ServedFrom {
result.ServedFrom = append(result.ServedFrom, &pb.SrvKeyspace_ServedFrom{
TabletType: TabletTypeToProto(tt),
Keyspace: k,
})
}
return result
}
func (client *client) StreamKeyRange(ctx context.Context, position string, keyspaceIdType key.KeyspaceIdType, keyRange key.KeyRange, charset *mproto.Charset) (chan *proto.BinlogTransaction, binlogplayer.ErrFunc, error) {
response := make(chan *proto.BinlogTransaction, 10)
query := &pb.StreamKeyRangeRequest{
Position: position,
KeyspaceIdType: key.KeyspaceIdTypeToProto(keyspaceIdType),
KeyRange: key.KeyRangeToProto(keyRange),
Charset: mproto.CharsetToProto(charset),
}
stream, err := client.c.StreamKeyRange(ctx, query)
if err != nil {
return nil, nil, err
}
var finalErr error
go func() {
for {
r, err := stream.Recv()
if err != nil {
if err != io.EOF {
finalErr = err
}
close(response)
return
}
response <- proto.ProtoToBinlogTransaction(r.BinlogTransaction)
}
}()
return response, func() error {
return finalErr
}, nil
}
// RefreshMap reads the right data from topo.Server and makes sure
// we're playing the right logs.
func (blm *BinlogPlayerMap) RefreshMap(ctx context.Context, tablet *topo.Tablet, keyspaceInfo *topo.KeyspaceInfo, shardInfo *topo.ShardInfo) {
log.Infof("Refreshing map of binlog players")
if shardInfo == nil {
log.Warningf("Could not read shardInfo, not changing anything")
return
}
if len(shardInfo.SourceShards) > 0 && keyspaceInfo == nil {
log.Warningf("Could not read keyspaceInfo, not changing anything")
return
}
blm.mu.Lock()
if blm.dbConfig.DbName == "" {
blm.dbConfig.DbName = tablet.DbName()
}
// get the existing sources and build a map of sources to remove
toRemove := make(map[uint32]bool)
hadPlayers := false
for source := range blm.players {
toRemove[source] = true
hadPlayers = true
}
// for each source, add it if not there, and delete from toRemove
for _, sourceShard := range shardInfo.SourceShards {
blm.addPlayer(ctx, tablet.Alias.Cell, keyspaceInfo.ShardingColumnType, key.KeyRangeToProto(tablet.KeyRange), sourceShard, tablet.DbName())
delete(toRemove, sourceShard.Uid)
}
hasPlayers := len(shardInfo.SourceShards) > 0
// remove all entries from toRemove
for source := range toRemove {
blm.players[source].Stop()
delete(blm.players, source)
}
blm.mu.Unlock()
if hadPlayers && !hasPlayers {
// We're done streaming, so turn off special playback settings.
blm.mysqld.DisableBinlogPlayback()
}
}
func TestKeyRangeToShardMap(t *testing.T) {
ts := new(sandboxTopo)
var testCases = []struct {
keyspace string
keyRange string
shards []string
}{
{keyspace: KsTestSharded, keyRange: "20-40", shards: []string{"20-40"}},
// check for partial keyrange, spanning one shard
{keyspace: KsTestSharded, keyRange: "10-18", shards: []string{"-20"}},
// check for keyrange intersecting with multiple shards
{keyspace: KsTestSharded, keyRange: "10-40", shards: []string{"-20", "20-40"}},
// check for keyrange intersecting with multiple shards
{keyspace: KsTestSharded, keyRange: "1c-2a", shards: []string{"-20", "20-40"}},
// check for keyrange where kr.End is Max Key ""
{keyspace: KsTestSharded, keyRange: "80-", shards: []string{"80-a0", "a0-c0", "c0-e0", "e0-"}},
// test for sharded, non-partial keyrange spanning the entire space.
{keyspace: KsTestSharded, keyRange: "", shards: []string{"-20", "20-40", "40-60", "60-80", "80-a0", "a0-c0", "c0-e0", "e0-"}},
// test for unsharded, non-partial keyrange spanning the entire space.
{keyspace: KsTestUnsharded, keyRange: "", shards: []string{"0"}},
}
for _, testCase := range testCases {
var keyRange *pb.KeyRange
var err error
if testCase.keyRange == "" {
keyRange = &pb.KeyRange{}
} else {
krArray, err := key.ParseShardingSpec(testCase.keyRange)
if err != nil {
t.Errorf("Got error while parsing sharding spec %v", err)
}
keyRange = key.KeyRangeToProto(krArray[0])
}
_, _, allShards, err := getKeyspaceShards(context.Background(), ts, "", testCase.keyspace, pb.TabletType_MASTER)
gotShards, err := resolveKeyRangeToShards(allShards, keyRange)
if err != nil {
t.Errorf("want nil, got %v", err)
}
if !reflect.DeepEqual(testCase.shards, gotShards) {
t.Errorf("want \n%#v, got \n%#v", testCase.shards, gotShards)
}
}
}
// TabletToProto turns a Tablet into a proto
func TabletToProto(t *Tablet) *pb.Tablet {
result := &pb.Tablet{
Alias: TabletAliasToProto(t.Alias),
Hostname: t.Hostname,
Ip: t.IPAddr,
PortMap: make(map[string]int32),
Keyspace: t.Keyspace,
Shard: t.Shard,
KeyRange: key.KeyRangeToProto(t.KeyRange),
Type: TabletTypeToProto(t.Type),
DbNameOverride: t.DbNameOverride,
Tags: t.Tags,
HealthMap: t.Health,
}
for k, v := range t.Portmap {
result.PortMap[k] = int32(v)
}
return result
}
// SetSourceShards is a utility function to override the SourceShards fields
// on a Shard.
func (wr *Wrangler) SetSourceShards(ctx context.Context, keyspace, shard string, sources []topo.TabletAlias, tables []string) error {
// read the shard
shardInfo, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return err
}
// If the shard already has sources, maybe it's already been restored,
// so let's be safe and abort right here.
if len(shardInfo.SourceShards) > 0 {
return fmt.Errorf("Shard %v/%v already has SourceShards, not overwriting them", keyspace, shard)
}
// read the source tablets
sourceTablets, err := topo.GetTabletMap(ctx, wr.TopoServer(), sources)
if err != nil {
return err
}
// Insert their KeyRange in the SourceShards array.
// We use a linear 0-based id, that matches what mysqlctld/split.go
// inserts into _vt.blp_checkpoint.
shardInfo.SourceShards = make([]*pb.Shard_SourceShard, len(sourceTablets))
i := 0
for _, ti := range sourceTablets {
shardInfo.SourceShards[i] = &pb.Shard_SourceShard{
Uid: uint32(i),
Keyspace: ti.Keyspace,
Shard: ti.Shard,
KeyRange: key.KeyRangeToProto(ti.KeyRange),
Tables: tables,
}
i++
}
// and write the shard
if err = topo.UpdateShard(ctx, wr.ts, shardInfo); err != nil {
return err
}
return nil
}
// StreamKeyRange is part of the gorpc UpdateStream service
func (server *UpdateStream) StreamKeyRange(req *proto.KeyRangeRequest, sendReply func(reply interface{}) error) (err error) {
defer server.updateStream.HandlePanic(&err)
return server.updateStream.StreamKeyRange(req.Position, req.KeyspaceIdType, key.KeyRangeToProto(req.KeyRange), req.Charset, func(reply *proto.BinlogTransaction) error {
return sendReply(reply)
})
}
