func (wr *Wrangler) migrateServedFrom(ki *topo.KeyspaceInfo, si *topo.ShardInfo, servedType topo.TabletType, reverse bool) (err error) {
// re-read and update keyspace info record
ki, err = wr.ts.GetKeyspace(ki.KeyspaceName())
if err != nil {
return err
}
if reverse {
if _, ok := ki.ServedFrom[servedType]; ok {
return fmt.Errorf("Destination Keyspace %s is not serving type %v", ki.KeyspaceName(), servedType)
}
ki.ServedFrom[servedType] = si.SourceShards[0].Keyspace
} else {
if _, ok := ki.ServedFrom[servedType]; !ok {
return fmt.Errorf("Destination Keyspace %s is already serving type %v", ki.KeyspaceName(), servedType)
}
delete(ki.ServedFrom, servedType)
}
// re-read and check the destination shard
si, err = wr.ts.GetShard(si.Keyspace(), si.ShardName())
if err != nil {
return err
}
if len(si.SourceShards) != 1 {
return fmt.Errorf("Destination shard %v/%v is not a vertical split target", si.Keyspace(), si.ShardName())
}
tables := si.SourceShards[0].Tables
// read the source shard, we'll need its master
sourceShard, err := wr.ts.GetShard(si.SourceShards[0].Keyspace, si.SourceShards[0].Shard)
if err != nil {
return err
}
ev := &events.MigrateServedFrom{
Keyspace: *ki,
SourceShard: *sourceShard,
DestinationShard: *si,
ServedType: servedType,
Reverse: reverse,
}
event.DispatchUpdate(ev, "start")
defer func() {
if err != nil {
event.DispatchUpdate(ev, "failed: "+err.Error())
}
}()
// For master type migration, need to:
// - switch the source shard to read-only
// - gather the replication point
// - wait for filtered replication to catch up before we continue
// - disable filtered replication after the fact
var sourceMasterTabletInfo *topo.TabletInfo
if servedType == topo.TYPE_MASTER {
// set master to read-only
event.DispatchUpdate(ev, "setting source shard master to read-only")
actionPath, err := wr.ai.SetReadOnly(sourceShard.MasterAlias)
if err != nil {
return err
}
if err := wr.WaitForCompletion(actionPath); err != nil {
return err
}
// get the position
event.DispatchUpdate(ev, "getting master position")
sourceMasterTabletInfo, err = wr.ts.GetTablet(sourceShard.MasterAlias)
if err != nil {
return err
}
masterPosition, err := wr.ai.MasterPosition(sourceMasterTabletInfo, wr.ActionTimeout())
if err != nil {
return err
}
// wait for it
event.DispatchUpdate(ev, "waiting for destination master to catch up to source master")
if err := wr.ai.WaitBlpPosition(si.MasterAlias, blproto.BlpPosition{
Uid: 0,
Position: masterPosition,
}, wr.ActionTimeout()); err != nil {
return err
}
// and clear the shard record
si.SourceShards = nil
}
// All is good, we can save the keyspace and shard (if needed) now
event.DispatchUpdate(ev, "updating keyspace")
if err = topo.UpdateKeyspace(wr.ts, ki); err != nil {
return err
}
event.DispatchUpdate(ev, "updating destination shard")
if servedType == topo.TYPE_MASTER {
if err := topo.UpdateShard(wr.ts, si); err != nil {
return err
}
}
// Tell the new shards masters they can now be read-write.
// Invoking a remote action will also make the tablet stop filtered
// replication.
event.DispatchUpdate(ev, "setting destination shard masters read-write")
if servedType == topo.TYPE_MASTER {
if err := wr.makeMastersReadWrite([]*topo.ShardInfo{si}); err != nil {
return err
}
}
// Now blacklist the table list on the right servers
event.DispatchUpdate(ev, "setting blacklisted tables on source shard")
if servedType == topo.TYPE_MASTER {
if err := wr.ai.SetBlacklistedTables(sourceMasterTabletInfo, tables, wr.ActionTimeout()); err != nil {
return err
}
} else {
// We use the list of tables that are replicating
// for the blacklist. In case of a reverse move, we clear the
// blacklist.
if reverse {
tables = nil
}
if err := wr.SetBlacklistedTablesByShard(sourceShard.Keyspace(), sourceShard.ShardName(), servedType, tables); err != nil {
return err
}
}
event.DispatchUpdate(ev, "finished")
return nil
}
// masterMigrateServedFrom handles the master migration. The ordering is
// a bit different than for rdonly / replica to guarantee a smooth transition.
//
// The order is as follows:
// - Add BlacklistedTables on the source shard map for master
// - Refresh the source master, so it stops writing on the tables
// - Get the source master position, wait until destination master reaches it
// - Clear SourceShard on the destination Shard
// - Refresh the destination master, so its stops its filtered
// replication and starts accepting writes
func (wr *Wrangler) masterMigrateServedFrom(ctx context.Context, ki *topo.KeyspaceInfo, sourceShard *topo.ShardInfo, destinationShard *topo.ShardInfo, tables []string, ev *events.MigrateServedFrom, filteredReplicationWaitTime time.Duration) error {
// Read the data we need
sourceMasterTabletInfo, err := wr.ts.GetTablet(ctx, sourceShard.MasterAlias)
if err != nil {
return err
}
destinationMasterTabletInfo, err := wr.ts.GetTablet(ctx, destinationShard.MasterAlias)
if err != nil {
return err
}
// Update source shard (more blacklisted tables)
event.DispatchUpdate(ev, "updating source shard")
if err := sourceShard.UpdateSourceBlacklistedTables(topodatapb.TabletType_MASTER, nil, false, tables); err != nil {
return fmt.Errorf("UpdateSourceBlacklistedTables(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err)
}
if err := wr.ts.UpdateShard(ctx, sourceShard); err != nil {
return fmt.Errorf("UpdateShard(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err)
}
// Now refresh the blacklisted table list on the source master
event.DispatchUpdate(ev, "refreshing source master so it updates its blacklisted tables")
if err := wr.tmc.RefreshState(ctx, sourceMasterTabletInfo); err != nil {
return err
}
// get the position
event.DispatchUpdate(ev, "getting master position")
masterPosition, err := wr.tmc.MasterPosition(ctx, sourceMasterTabletInfo)
if err != nil {
return err
}
// wait for it
event.DispatchUpdate(ev, "waiting for destination master to catch up to source master")
if err := wr.tmc.WaitBlpPosition(ctx, destinationMasterTabletInfo, &tabletmanagerdatapb.BlpPosition{
Uid: 0,
Position: masterPosition,
}, filteredReplicationWaitTime); err != nil {
return err
}
// Update the destination keyspace (its ServedFrom has changed)
event.DispatchUpdate(ev, "updating keyspace")
if err = wr.ts.UpdateKeyspace(ctx, ki); err != nil {
return err
}
// Update the destination shard (no more source shard)
event.DispatchUpdate(ev, "updating destination shard")
destinationShard.SourceShards = nil
if err := wr.ts.UpdateShard(ctx, destinationShard); err != nil {
return err
}
// Tell the new shards masters they can now be read-write.
// Invoking a remote action will also make the tablet stop filtered
// replication.
event.DispatchUpdate(ev, "setting destination shard masters read-write")
if err := wr.refreshMasters(ctx, []*topo.ShardInfo{destinationShard}); err != nil {
return err
}
return nil
}
func (wr *Wrangler) migrateServedFrom(ki *topo.KeyspaceInfo, si *topo.ShardInfo, servedType topo.TabletType, reverse bool) error {
// re-read and update keyspace info record
var err error
ki, err = wr.ts.GetKeyspace(ki.KeyspaceName())
if err != nil {
return err
}
if reverse {
if _, ok := ki.ServedFrom[servedType]; ok {
return fmt.Errorf("Destination Keyspace %s is not serving type %v", ki.KeyspaceName(), servedType)
}
ki.ServedFrom[servedType] = si.SourceShards[0].Keyspace
} else {
if _, ok := ki.ServedFrom[servedType]; !ok {
return fmt.Errorf("Destination Keyspace %s is already serving type %v", ki.KeyspaceName(), servedType)
}
delete(ki.ServedFrom, servedType)
}
// re-read and check the destination shard
si, err = wr.ts.GetShard(si.Keyspace(), si.ShardName())
if err != nil {
return err
}
if len(si.SourceShards) != 1 {
return fmt.Errorf("Destination shard %v/%v is not a vertical split target", si.Keyspace(), si.ShardName())
}
tables := si.SourceShards[0].Tables
// read the source shard, we'll need its master
sourceShard, err := wr.ts.GetShard(si.SourceShards[0].Keyspace, si.SourceShards[0].Shard)
if err != nil {
return err
}
// For master type migration, need to:
// - switch the source shard to read-only
// - gather the replication point
// - wait for filtered replication to catch up before we continue
// - disable filtered replication after the fact
var sourceMasterTabletInfo *topo.TabletInfo
if servedType == topo.TYPE_MASTER {
// set master to read-only
actionPath, err := wr.ai.SetReadOnly(sourceShard.MasterAlias)
if err != nil {
return err
}
if err := wr.ai.WaitForCompletion(actionPath, wr.actionTimeout()); err != nil {
return err
}
// get the position
sourceMasterTabletInfo, err = wr.ts.GetTablet(sourceShard.MasterAlias)
if err != nil {
return err
}
masterPosition, err := wr.ai.MasterPosition(sourceMasterTabletInfo, wr.actionTimeout())
if err != nil {
return err
}
// wait for it
if err := wr.ai.WaitBlpPosition(si.MasterAlias, myproto.BlpPosition{
Uid: 0,
GroupId: masterPosition.MasterLogGroupId,
}, wr.actionTimeout()); err != nil {
return err
}
// and clear the shard record
si.SourceShards = nil
}
// All is good, we can save the keyspace and shard (if needed) now
if err = wr.ts.UpdateKeyspace(ki); err != nil {
return err
}
if servedType == topo.TYPE_MASTER {
if err := wr.ts.UpdateShard(si); err != nil {
return err
}
}
// Tell the new shards masters they can now be read-write.
// Invoking a remote action will also make the tablet stop filtered
// replication.
if servedType == topo.TYPE_MASTER {
if err := wr.makeMastersReadWrite([]*topo.ShardInfo{si}); err != nil {
return err
}
}
// Now blacklist the table list on the right servers
if servedType == topo.TYPE_MASTER {
if err := wr.ai.SetBlacklistedTables(sourceMasterTabletInfo, tables, wr.actionTimeout()); err != nil {
return err
}
} else {
// We use the list of tables that are replicating
// for the blacklist. In case of a reverse move, we clear the
// blacklist.
if reverse {
tables = nil
}
if err := wr.SetBlacklistedTablesByShard(sourceShard.Keyspace(), sourceShard.ShardName(), servedType, tables); err != nil {
return err
}
}
return nil
}
// masterMigrateServedFrom handles the master migration. The ordering is
// a bit different than for rdonly / replica to guarantee a smooth transition.
//
// The order is as follows:
// - Add BlacklistedTables on the source shard map for master
// - Refresh the source master, so it stops writing on the tables
// - Get the source master position, wait until destination master reaches it
// - Clear SourceShard on the destination Shard
// - Refresh the destination master, so its stops its filtered
// replication and starts accepting writes
func (wr *Wrangler) masterMigrateServedFrom(ki *topo.KeyspaceInfo, sourceShard *topo.ShardInfo, destinationShard *topo.ShardInfo, servedType topo.TabletType, tables []string, ev *events.MigrateServedFrom) error {
// Read the data we need
sourceMasterTabletInfo, err := wr.ts.GetTablet(sourceShard.MasterAlias)
if err != nil {
return err
}
destinationMasterTabletInfo, err := wr.ts.GetTablet(destinationShard.MasterAlias)
if err != nil {
return err
}
// Update source shard (more blacklisted tables)
event.DispatchUpdate(ev, "updating source shard")
if sourceShard.BlacklistedTablesMap == nil {
sourceShard.BlacklistedTablesMap = make(map[topo.TabletType][]string)
}
sourceShard.BlacklistedTablesMap[servedType] = tables
if err := topo.UpdateShard(wr.ts, sourceShard); err != nil {
return err
}
// Now refresh the blacklisted table list on the source master
event.DispatchUpdate(ev, "refreshing source master so it updates its blacklisted tables")
if err := wr.tmc.RefreshState(sourceMasterTabletInfo, wr.ActionTimeout()); err != nil {
return err
}
// get the position
event.DispatchUpdate(ev, "getting master position")
masterPosition, err := wr.tmc.MasterPosition(sourceMasterTabletInfo, wr.ActionTimeout())
if err != nil {
return err
}
// wait for it
event.DispatchUpdate(ev, "waiting for destination master to catch up to source master")
if err := wr.tmc.WaitBlpPosition(destinationMasterTabletInfo, blproto.BlpPosition{
Uid: 0,
Position: masterPosition,
}, wr.ActionTimeout()); err != nil {
return err
}
// Update the destination keyspace (its ServedFrom has changed)
event.DispatchUpdate(ev, "updating keyspace")
if err = topo.UpdateKeyspace(wr.ts, ki); err != nil {
return err
}
// Update the destination shard (no more source shard)
event.DispatchUpdate(ev, "updating destination shard")
destinationShard.SourceShards = nil
if err := topo.UpdateShard(wr.ts, destinationShard); err != nil {
return err
}
// Tell the new shards masters they can now be read-write.
// Invoking a remote action will also make the tablet stop filtered
// replication.
event.DispatchUpdate(ev, "setting destination shard masters read-write")
if err := wr.refreshMasters([]*topo.ShardInfo{destinationShard}); err != nil {
return err
}
return nil
}