// replicaMigrateServedFrom handles the slave (replica, rdonly) migration.
func (wr *Wrangler) replicaMigrateServedFrom(ki *topo.KeyspaceInfo, sourceShard *topo.ShardInfo, destinationShard *topo.ShardInfo, servedType topo.TabletType, cells []string, reverse bool, tables []string, ev *events.MigrateServedFrom) error {
// Save the destination keyspace (its ServedFrom has been changed)
event.DispatchUpdate(ev, "updating keyspace")
if err := topo.UpdateKeyspace(wr.ts, ki); err != nil {
return err
}
// Save the source shard (its blacklisted tables field has changed)
event.DispatchUpdate(ev, "updating source shard")
if err := sourceShard.UpdateSourceBlacklistedTables(servedType, cells, reverse, tables); err != nil {
return fmt.Errorf("UpdateSourceBlacklistedTables(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err)
}
if err := topo.UpdateShard(context.TODO(), wr.ts, sourceShard); err != nil {
return fmt.Errorf("UpdateShard(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err)
}
// Now refresh the source servers so they reload their
// blacklisted table list
event.DispatchUpdate(ev, "refreshing sources tablets state so they update their blacklisted tables")
if err := wr.RefreshTablesByShard(sourceShard, servedType, cells); err != nil {
return err
}
return nil
}
func (wr *Wrangler) migrateServedFrom(ki *topo.KeyspaceInfo, destinationShard *topo.ShardInfo, servedType topo.TabletType, cells []string, reverse bool) (err error) {
// re-read and update keyspace info record
ki, err = wr.ts.GetKeyspace(ki.KeyspaceName())
if err != nil {
return err
}
if reverse {
ki.UpdateServedFromMap(servedType, cells, destinationShard.SourceShards[0].Keyspace, false, nil)
} else {
ki.UpdateServedFromMap(servedType, cells, destinationShard.SourceShards[0].Keyspace, true, destinationShard.Cells)
}
// re-read and check the destination shard
destinationShard, err = wr.ts.GetShard(destinationShard.Keyspace(), destinationShard.ShardName())
if err != nil {
return err
}
if len(destinationShard.SourceShards) != 1 {
return fmt.Errorf("Destination shard %v/%v is not a vertical split target", destinationShard.Keyspace(), destinationShard.ShardName())
}
tables := destinationShard.SourceShards[0].Tables
// read the source shard, we'll need its master, and we'll need to
// update the blacklisted tables.
var sourceShard *topo.ShardInfo
sourceShard, err = wr.ts.GetShard(destinationShard.SourceShards[0].Keyspace, destinationShard.SourceShards[0].Shard)
if err != nil {
return err
}
ev := &events.MigrateServedFrom{
Keyspace: *ki,
SourceShard: *sourceShard,
DestinationShard: *destinationShard,
ServedType: servedType,
Reverse: reverse,
}
event.DispatchUpdate(ev, "start")
defer func() {
if err != nil {
event.DispatchUpdate(ev, "failed: "+err.Error())
}
}()
if servedType == topo.TYPE_MASTER {
err = wr.masterMigrateServedFrom(ki, sourceShard, destinationShard, tables, ev)
} else {
err = wr.replicaMigrateServedFrom(ki, sourceShard, destinationShard, servedType, cells, reverse, tables, ev)
}
event.DispatchUpdate(ev, "finished")
return
}
func (scw *SplitCloneWorker) setState(state string) {
scw.mu.Lock()
scw.state = state
scw.mu.Unlock()
event.DispatchUpdate(scw.ev, state)
}
func (vscw *VerticalSplitCloneWorker) setState(state string) {
vscw.mu.Lock()
vscw.state = state
vscw.mu.Unlock()
event.DispatchUpdate(vscw.ev, state)
}
func (scw *SplitCloneWorker) recordError(err error) {
scw.mu.Lock()
scw.state = stateSCError
scw.err = err
scw.mu.Unlock()
event.DispatchUpdate(scw.ev, "error: "+err.Error())
}
func (vscw *VerticalSplitCloneWorker) recordError(err error) {
vscw.mu.Lock()
vscw.state = stateVSCError
vscw.err = err
vscw.mu.Unlock()
event.DispatchUpdate(vscw.ev, "error: "+err.Error())
}
func TestUpdateDispatch(t *testing.T) {
triggered := false
event.AddListener(func(ev *testEvent) {
triggered = true
})
want := "status"
ev := &testEvent{}
event.DispatchUpdate(ev, "status")
if ev.Status != want {
t.Errorf("ev.Status = %#v, want %#v", ev.Status, want)
}
if !triggered {
t.Errorf("listener wasn't triggered on Dispatch()")
}
}
// reparentShardGraceful executes a graceful reparent.
// The ev parameter is an event struct prefilled with information that the
// caller has on hand, which would be expensive for us to re-query.
func (wr *Wrangler) reparentShardGraceful(ev *events.Reparent, si *topo.ShardInfo, slaveTabletMap, masterTabletMap map[topo.TabletAlias]*topo.TabletInfo, masterElectTablet *topo.TabletInfo, leaveMasterReadOnly bool) (err error) {
event.DispatchUpdate(ev, "starting graceful")
defer func() {
if err != nil {
event.DispatchUpdate(ev, "failed: "+err.Error())
}
}()
// Validate a bunch of assumptions we make about the replication graph.
if len(masterTabletMap) != 1 {
aliases := make([]string, 0, len(masterTabletMap))
for _, v := range masterTabletMap {
aliases = append(aliases, v.String())
}
return fmt.Errorf("I have 0 or multiple masters / scrapped tablets in this shard replication graph, please scrap the non-master ones: %v", strings.Join(aliases, " "))
}
var masterTablet *topo.TabletInfo
for _, v := range masterTabletMap {
masterTablet = v
}
if masterTablet.Parent.Uid != topo.NO_TABLET {
return fmt.Errorf("master tablet should not have a ParentUid: %v %v", masterTablet.Parent.Uid, masterTablet.Alias)
}
if masterTablet.Type != topo.TYPE_MASTER {
return fmt.Errorf("master tablet should not be type: %v %v", masterTablet.Type, masterTablet.Alias)
}
if masterTablet.Alias.Uid == masterElectTablet.Alias.Uid {
return fmt.Errorf("master tablet should not match master elect - this must be forced: %v", masterTablet.Alias)
}
if _, ok := slaveTabletMap[masterElectTablet.Alias]; !ok {
return fmt.Errorf("master elect tablet not in replication graph %v %v/%v %v", masterElectTablet.Alias, masterTablet.Keyspace, masterTablet.Shard, topotools.MapKeys(slaveTabletMap))
}
if err := wr.ValidateShard(masterTablet.Keyspace, masterTablet.Shard, true); err != nil {
return fmt.Errorf("ValidateShard verification failed: %v, if the master is dead, run: vtctl ScrapTablet -force %v", err, masterTablet.Alias)
}
// Make sure all tablets have the right parent and reasonable positions.
event.DispatchUpdate(ev, "checking slave replication positions")
err = wr.checkSlaveReplication(slaveTabletMap, masterTablet.Alias.Uid)
if err != nil {
return err
}
// Check the master-elect is fit for duty - call out for hardware checks.
event.DispatchUpdate(ev, "checking that new master is ready to serve")
err = wr.checkMasterElect(masterElectTablet)
if err != nil {
return err
}
event.DispatchUpdate(ev, "demoting old master")
masterPosition, err := wr.demoteMaster(masterTablet)
if err != nil {
// FIXME(msolomon) This suggests that the master is dead and we
// need to take steps. We could either pop a prompt, or make
// retrying the action painless.
return fmt.Errorf("demote master failed: %v, if the master is dead, run: vtctl -force ScrapTablet %v", err, masterTablet.Alias)
}
event.DispatchUpdate(ev, "checking slave consistency")
wr.logger.Infof("check slaves %v/%v", masterTablet.Keyspace, masterTablet.Shard)
restartableSlaveTabletMap := wr.restartableTabletMap(slaveTabletMap)
err = wr.checkSlaveConsistency(restartableSlaveTabletMap, masterPosition)
if err != nil {
return fmt.Errorf("check slave consistency failed %v, demoted master is still read only, run: vtctl SetReadWrite %v", err, masterTablet.Alias)
}
event.DispatchUpdate(ev, "promoting new master")
rsd, err := wr.promoteSlave(masterElectTablet)
if err != nil {
// FIXME(msolomon) This suggests that the master-elect is dead.
// We need to classify certain errors as temporary and retry.
return fmt.Errorf("promote slave failed: %v, demoted master is still read only: vtctl SetReadWrite %v", err, masterTablet.Alias)
}
// Once the slave is promoted, remove it from our map
delete(slaveTabletMap, masterElectTablet.Alias)
event.DispatchUpdate(ev, "restarting slaves")
majorityRestart, restartSlaveErr := wr.restartSlaves(slaveTabletMap, rsd)
// For now, scrap the old master regardless of how many
// slaves restarted.
//
// FIXME(msolomon) We could reintroduce it and reparent it and use
// it as new replica.
event.DispatchUpdate(ev, "scrapping old master")
wr.logger.Infof("scrap demoted master %v", masterTablet.Alias)
if scrapErr := wr.tmc.Scrap(wr.ctx, masterTablet); scrapErr != nil {
// The sub action is non-critical, so just warn.
wr.logger.Warningf("scrap demoted master failed: %v", scrapErr)
}
event.DispatchUpdate(ev, "rebuilding shard serving graph")
err = wr.finishReparent(si, masterElectTablet, majorityRestart, leaveMasterReadOnly)
if err != nil {
return err
}
event.DispatchUpdate(ev, "finished")
if restartSlaveErr != nil {
// This is more of a warning at this point.
return restartSlaveErr
}
return nil
}
// tabletExternallyReparentedLocked is called with the shard lock.
// It returns if agent.refreshTablet should be called, and the error.
// Note both are set independently (can have both true and an error).
func (agent *ActionAgent) tabletExternallyReparentedLocked(ctx context.Context, externalID string, interrupted chan struct{}) (bool, error) {
// re-read the tablet record to be sure we have the latest version
tablet, err := agent.TopoServer.GetTablet(agent.TabletAlias)
if err != nil {
return false, err
}
// read the shard, make sure again the master is not already good.
shardInfo, err := agent.TopoServer.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return false, err
}
if shardInfo.MasterAlias == tablet.Alias {
log.Infof("TabletExternallyReparented: tablet became the master before we get the lock?")
return false, nil
}
log.Infof("TabletExternallyReparented called and we're not the master, doing the work")
// Read the tablets, make sure the master elect is known to the shard
// (it's this tablet, so it better be!).
// Note we will keep going with a partial tablet map, which usually
// happens when a cell is not reachable. After these checks, the
// guarantees we'll have are:
// - global cell is reachable (we just locked and read the shard)
// - the local cell that contains the new master is reachable
// (as we're going to check the new master is in the list)
// That should be enough.
tabletMap, err := topo.GetTabletMapForShard(ctx, agent.TopoServer, tablet.Keyspace, tablet.Shard)
switch err {
case nil:
// keep going
case topo.ErrPartialResult:
log.Warningf("Got topo.ErrPartialResult from GetTabletMapForShard, may need to re-init some tablets")
default:
return false, err
}
masterElectTablet, ok := tabletMap[tablet.Alias]
if !ok {
return false, fmt.Errorf("this master-elect tablet %v not found in replication graph %v/%v %v", tablet.Alias, tablet.Keyspace, tablet.Shard, topotools.MapKeys(tabletMap))
}
// Create reusable Reparent event with available info
ev := &events.Reparent{
ShardInfo: *shardInfo,
NewMaster: *tablet.Tablet,
ExternalID: externalID,
}
if oldMasterTablet, ok := tabletMap[shardInfo.MasterAlias]; ok {
ev.OldMaster = *oldMasterTablet.Tablet
}
defer func() {
if err != nil {
event.DispatchUpdate(ev, "failed: "+err.Error())
}
}()
// sort the tablets, and handle them
slaveTabletMap, masterTabletMap := topotools.SortedTabletMap(tabletMap)
event.DispatchUpdate(ev, "starting external from tablet")
// We fix the new master in the replication graph.
// Note after this call, we may have changed the tablet record,
// so we will always return true, so the tablet record is re-read
// by the agent.
event.DispatchUpdate(ev, "mark ourself as new master")
err = agent.updateReplicationGraphForPromotedSlave(ctx, tablet)
if err != nil {
// This suggests we can't talk to topo server. This is bad.
return true, fmt.Errorf("updateReplicationGraphForPromotedSlave failed: %v", err)
}
// Once this tablet is promoted, remove it from our maps
delete(slaveTabletMap, tablet.Alias)
delete(masterTabletMap, tablet.Alias)
// Then fix all the slaves, including the old master. This
// last step is very likely to time out for some tablets (one
// random guy is dead, the old master is dead, ...). We
// execute them all in parallel until we get to
// wr.ActionTimeout(). After this, no other action with a
// timeout is executed, so even if we got to the timeout,
// we're still good.
event.DispatchUpdate(ev, "restarting slaves")
logger := logutil.NewConsoleLogger()
tmc := tmclient.NewTabletManagerClient()
topotools.RestartSlavesExternal(agent.TopoServer, logger, slaveTabletMap, masterTabletMap, masterElectTablet.Alias, func(ti *topo.TabletInfo, swrd *actionnode.SlaveWasRestartedArgs) error {
return tmc.SlaveWasRestarted(ctx, ti, swrd)
})
// Compute the list of Cells we need to rebuild: old master and
// all other cells if reparenting to another cell.
cells := []string{shardInfo.MasterAlias.Cell}
if shardInfo.MasterAlias.Cell != tablet.Alias.Cell {
cells = nil
}
// now update the master record in the shard object
event.DispatchUpdate(ev, "updating shard record")
log.Infof("Updating Shard's MasterAlias record")
shardInfo.MasterAlias = tablet.Alias
if err = topo.UpdateShard(ctx, agent.TopoServer, shardInfo); err != nil {
return true, err
}
// and rebuild the shard serving graph
event.DispatchUpdate(ev, "rebuilding shard serving graph")
log.Infof("Rebuilding shard serving graph data")
if _, err = topotools.RebuildShard(ctx, logger, agent.TopoServer, tablet.Keyspace, tablet.Shard, cells, agent.LockTimeout, interrupted); err != nil {
return true, err
}
event.DispatchUpdate(ev, "finished")
return true, nil
}
// masterMigrateServedFrom handles the master migration. The ordering is
// a bit different than for rdonly / replica to guarantee a smooth transition.
//
// The order is as follows:
// - Add BlacklistedTables on the source shard map for master
// - Refresh the source master, so it stops writing on the tables
// - Get the source master position, wait until destination master reaches it
// - Clear SourceShard on the destination Shard
// - Refresh the destination master, so its stops its filtered
// replication and starts accepting writes
func (wr *Wrangler) masterMigrateServedFrom(ki *topo.KeyspaceInfo, sourceShard *topo.ShardInfo, destinationShard *topo.ShardInfo, tables []string, ev *events.MigrateServedFrom) error {
// Read the data we need
sourceMasterTabletInfo, err := wr.ts.GetTablet(sourceShard.MasterAlias)
if err != nil {
return err
}
destinationMasterTabletInfo, err := wr.ts.GetTablet(destinationShard.MasterAlias)
if err != nil {
return err
}
// Update source shard (more blacklisted tables)
event.DispatchUpdate(ev, "updating source shard")
if err := sourceShard.UpdateSourceBlacklistedTables(topo.TYPE_MASTER, nil, false, tables); err != nil {
return fmt.Errorf("UpdateSourceBlacklistedTables(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err)
}
if err := topo.UpdateShard(context.TODO(), wr.ts, sourceShard); err != nil {
return fmt.Errorf("UpdateShard(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err)
}
// Now refresh the blacklisted table list on the source master
event.DispatchUpdate(ev, "refreshing source master so it updates its blacklisted tables")
if err := wr.tmc.RefreshState(wr.ctx, sourceMasterTabletInfo); err != nil {
return err
}
// get the position
event.DispatchUpdate(ev, "getting master position")
masterPosition, err := wr.tmc.MasterPosition(wr.ctx, sourceMasterTabletInfo)
if err != nil {
return err
}
// wait for it
event.DispatchUpdate(ev, "waiting for destination master to catch up to source master")
if err := wr.tmc.WaitBlpPosition(context.TODO(), destinationMasterTabletInfo, blproto.BlpPosition{
Uid: 0,
Position: masterPosition,
}, wr.ActionTimeout()); err != nil {
return err
}
// Update the destination keyspace (its ServedFrom has changed)
event.DispatchUpdate(ev, "updating keyspace")
if err = topo.UpdateKeyspace(wr.ts, ki); err != nil {
return err
}
// Update the destination shard (no more source shard)
event.DispatchUpdate(ev, "updating destination shard")
destinationShard.SourceShards = nil
if err := topo.UpdateShard(context.TODO(), wr.ts, destinationShard); err != nil {
return err
}
// Tell the new shards masters they can now be read-write.
// Invoking a remote action will also make the tablet stop filtered
// replication.
event.DispatchUpdate(ev, "setting destination shard masters read-write")
if err := wr.refreshMasters([]*topo.ShardInfo{destinationShard}); err != nil {
return err
}
return nil
}
// migrateServedTypes operates with all concerned shards locked.
func (wr *Wrangler) migrateServedTypes(keyspace string, sourceShards, destinationShards []*topo.ShardInfo, cells []string, servedType topo.TabletType, reverse bool) (err error) {
// re-read all the shards so we are up to date
wr.Logger().Infof("Re-reading all shards")
for i, si := range sourceShards {
if sourceShards[i], err = wr.ts.GetShard(si.Keyspace(), si.ShardName()); err != nil {
return err
}
}
for i, si := range destinationShards {
if destinationShards[i], err = wr.ts.GetShard(si.Keyspace(), si.ShardName()); err != nil {
return err
}
}
ev := &events.MigrateServedTypes{
Keyspace: *topo.NewKeyspaceInfo(keyspace, nil, -1),
SourceShards: sourceShards,
DestinationShards: destinationShards,
ServedType: servedType,
Reverse: reverse,
}
event.DispatchUpdate(ev, "start")
defer func() {
if err != nil {
event.DispatchUpdate(ev, "failed: "+err.Error())
}
}()
// For master type migration, need to:
// - switch the source shards to read-only by disabling query service
// - gather all replication points
// - wait for filtered replication to catch up before we continue
// - disable filtered replication after the fact
if servedType == topo.TYPE_MASTER {
event.DispatchUpdate(ev, "disabling query service on all source masters")
for _, si := range sourceShards {
if err := si.UpdateDisableQueryService(topo.TYPE_MASTER, nil, true); err != nil {
return err
}
if err := topo.UpdateShard(context.TODO(), wr.ts, si); err != nil {
return err
}
}
if err := wr.refreshMasters(sourceShards); err != nil {
return err
}
event.DispatchUpdate(ev, "getting positions of source masters")
masterPositions, err := wr.getMastersPosition(sourceShards)
if err != nil {
return err
}
event.DispatchUpdate(ev, "waiting for destination masters to catch up")
if err := wr.waitForFilteredReplication(masterPositions, destinationShards); err != nil {
return err
}
for _, si := range destinationShards {
si.SourceShards = nil
}
}
// Check and update all shard records, in memory only.
// We remember if we need to refresh the state of the source tablets
// so their query service is enabled again, for reverse migration.
needToRefreshSourceTablets := false
for _, si := range sourceShards {
if err := si.UpdateServedTypesMap(servedType, cells, !reverse); err != nil {
return err
}
if tc, ok := si.TabletControlMap[servedType]; reverse && ok && tc.DisableQueryService {
// this is a backward migration, where the
// source tablets were disabled previously, so
// we need to refresh them
if err := si.UpdateDisableQueryService(servedType, cells, false); err != nil {
return err
}
needToRefreshSourceTablets = true
}
if !reverse && servedType != topo.TYPE_MASTER {
// this is a forward migration, we need to disable
// query service on the source shards.
// (this was already done for masters earlier)
if err := si.UpdateDisableQueryService(servedType, cells, true); err != nil {
return err
}
}
}
for _, si := range destinationShards {
if err := si.UpdateServedTypesMap(servedType, cells, reverse); err != nil {
return err
}
}
// All is good, we can save the shards now
event.DispatchUpdate(ev, "updating source shards")
for _, si := range sourceShards {
if err := topo.UpdateShard(context.TODO(), wr.ts, si); err != nil {
return err
}
}
if needToRefreshSourceTablets {
event.DispatchUpdate(ev, "refreshing source shard tablets so they restart their query service")
for _, si := range sourceShards {
wr.RefreshTablesByShard(si, servedType, cells)
}
}
event.DispatchUpdate(ev, "updating destination shards")
for _, si := range destinationShards {
if err := topo.UpdateShard(context.TODO(), wr.ts, si); err != nil {
return err
}
}
// And tell the new shards masters they can now be read-write.
// Invoking a remote action will also make the tablet stop filtered
// replication.
if servedType == topo.TYPE_MASTER {
event.DispatchUpdate(ev, "setting destination masters read-write")
if err := wr.refreshMasters(destinationShards); err != nil {
return err
}
}
event.DispatchUpdate(ev, "finished")
return nil
}
// reparentShardBrutal executes a brutal reparent.
//
// Assume the master is dead and not coming back. Just push your way
// forward. Force means we are reparenting to the same master
// (assuming the data has been externally synched).
//
// The ev parameter is an event struct prefilled with information that the
// caller has on hand, which would be expensive for us to re-query.
func (wr *Wrangler) reparentShardBrutal(ev *events.Reparent, si *topo.ShardInfo, slaveTabletMap, masterTabletMap map[topo.TabletAlias]*topo.TabletInfo, masterElectTablet *topo.TabletInfo, leaveMasterReadOnly, force bool) (err error) {
event.DispatchUpdate(ev, "starting brutal")
defer func() {
if err != nil {
event.DispatchUpdate(ev, "failed: "+err.Error())
}
}()
wr.logger.Infof("Skipping ValidateShard - not a graceful situation")
if _, ok := slaveTabletMap[masterElectTablet.Alias]; !ok && !force {
return fmt.Errorf("master elect tablet not in replication graph %v %v/%v %v", masterElectTablet.Alias, si.Keyspace(), si.ShardName(), topotools.MapKeys(slaveTabletMap))
}
// Check the master-elect and slaves are in good shape when the action
// has not been forced.
if !force {
// Make sure all tablets have the right parent and reasonable positions.
event.DispatchUpdate(ev, "checking slave replication positions")
if err := wr.checkSlaveReplication(slaveTabletMap, topo.NO_TABLET); err != nil {
return err
}
// Check the master-elect is fit for duty - call out for hardware checks.
event.DispatchUpdate(ev, "checking that new master is ready to serve")
if err := wr.checkMasterElect(masterElectTablet); err != nil {
return err
}
event.DispatchUpdate(ev, "checking slave consistency")
wr.logger.Infof("check slaves %v/%v", masterElectTablet.Keyspace, masterElectTablet.Shard)
restartableSlaveTabletMap := wr.restartableTabletMap(slaveTabletMap)
err = wr.checkSlaveConsistency(restartableSlaveTabletMap, myproto.ReplicationPosition{})
if err != nil {
return err
}
} else {
event.DispatchUpdate(ev, "stopping slave replication")
wr.logger.Infof("forcing reparent to same master %v", masterElectTablet.Alias)
err := wr.breakReplication(slaveTabletMap, masterElectTablet)
if err != nil {
return err
}
}
event.DispatchUpdate(ev, "promoting new master")
rsd, err := wr.promoteSlave(masterElectTablet)
if err != nil {
// FIXME(msolomon) This suggests that the master-elect is dead.
// We need to classify certain errors as temporary and retry.
return fmt.Errorf("promote slave failed: %v %v", err, masterElectTablet.Alias)
}
// Once the slave is promoted, remove it from our maps
delete(slaveTabletMap, masterElectTablet.Alias)
delete(masterTabletMap, masterElectTablet.Alias)
event.DispatchUpdate(ev, "restarting slaves")
majorityRestart, restartSlaveErr := wr.restartSlaves(slaveTabletMap, rsd)
if !force {
for _, failedMaster := range masterTabletMap {
event.DispatchUpdate(ev, "scrapping old master")
wr.logger.Infof("scrap dead master %v", failedMaster.Alias)
// The master is dead so execute the action locally instead of
// enqueing the scrap action for an arbitrary amount of time.
if scrapErr := topotools.Scrap(wr.ts, failedMaster.Alias, false); scrapErr != nil {
wr.logger.Warningf("scrapping failed master failed: %v", scrapErr)
}
}
}
event.DispatchUpdate(ev, "rebuilding shard serving graph")
err = wr.finishReparent(si, masterElectTablet, majorityRestart, leaveMasterReadOnly)
if err != nil {
return err
}
event.DispatchUpdate(ev, "finished")
if restartSlaveErr != nil {
// This is more of a warning at this point.
return restartSlaveErr
}
return nil
}