// DeleteTablet removes a tablet from a shard.
// - if allowMaster is set, we can Delete a master tablet (and clear
// its record from the Shard record if it was the master).
// - if skipRebuild is set, we do not rebuild the serving graph.
func (wr *Wrangler) DeleteTablet(ctx context.Context, tabletAlias *topodatapb.TabletAlias, allowMaster, skipRebuild bool) error {
// load the tablet, see if we'll need to rebuild
ti, err := wr.ts.GetTablet(ctx, tabletAlias)
if err != nil {
return err
}
rebuildRequired := ti.IsInServingGraph()
wasMaster := ti.Type == topodatapb.TabletType_MASTER
if wasMaster && !allowMaster {
return fmt.Errorf("cannot delete tablet %v as it is a master, use allow_master flag", topoproto.TabletAliasString(tabletAlias))
}
// remove the record and its replication graph entry
if err := topotools.DeleteTablet(ctx, wr.ts, ti.Tablet); err != nil {
return err
}
// update the Shard object if the master was scrapped.
// we lock the shard to not conflict with reparent operations.
if wasMaster {
actionNode := actionnode.UpdateShard()
lockPath, err := wr.lockShard(ctx, ti.Keyspace, ti.Shard, actionNode)
if err != nil {
return err
}
// update the shard record's master
if _, err := wr.ts.UpdateShardFields(ctx, ti.Keyspace, ti.Shard, func(s *topodatapb.Shard) error {
if !topoproto.TabletAliasEqual(s.MasterAlias, tabletAlias) {
wr.Logger().Warningf("Deleting master %v from shard %v/%v but master in Shard object was %v", topoproto.TabletAliasString(tabletAlias), ti.Keyspace, ti.Shard, topoproto.TabletAliasString(s.MasterAlias))
return topo.ErrNoUpdateNeeded
}
s.MasterAlias = nil
return nil
}); err != nil {
return wr.unlockShard(ctx, ti.Keyspace, ti.Shard, actionNode, lockPath, err)
}
// and unlock
if err := wr.unlockShard(ctx, ti.Keyspace, ti.Shard, actionNode, lockPath, err); err != nil {
return err
}
}
// and rebuild the original shard if needed
if !rebuildRequired {
wr.Logger().Infof("Rebuild not required")
return nil
}
if skipRebuild {
wr.Logger().Warningf("Rebuild required, but skipping it")
return nil
}
_, err = wr.RebuildShardGraph(ctx, ti.Keyspace, ti.Shard, []string{ti.Alias.Cell})
return err
}
// DeleteTablet removes a tablet from a shard.
// - if allowMaster is set, we can Delete a master tablet (and clear
// its record from the Shard record if it was the master).
func (wr *Wrangler) DeleteTablet(ctx context.Context, tabletAlias *topodatapb.TabletAlias, allowMaster bool) (err error) {
// load the tablet, see if we'll need to rebuild
ti, err := wr.ts.GetTablet(ctx, tabletAlias)
if err != nil {
return err
}
wasMaster := ti.Type == topodatapb.TabletType_MASTER
if wasMaster && !allowMaster {
return fmt.Errorf("cannot delete tablet %v as it is a master, use allow_master flag", topoproto.TabletAliasString(tabletAlias))
}
// remove the record and its replication graph entry
if err := topotools.DeleteTablet(ctx, wr.ts, ti.Tablet); err != nil {
return err
}
// update the Shard object if the master was scrapped.
if wasMaster {
// We lock the shard to not conflict with reparent operations.
ctx, unlock, lockErr := wr.ts.LockShard(ctx, ti.Keyspace, ti.Shard, fmt.Sprintf("DeleteTablet(%v)", topoproto.TabletAliasString(tabletAlias)))
if lockErr != nil {
return lockErr
}
defer unlock(&err)
// update the shard record's master
_, err = wr.ts.UpdateShardFields(ctx, ti.Keyspace, ti.Shard, func(si *topo.ShardInfo) error {
if !topoproto.TabletAliasEqual(si.MasterAlias, tabletAlias) {
wr.Logger().Warningf("Deleting master %v from shard %v/%v but master in Shard object was %v", topoproto.TabletAliasString(tabletAlias), ti.Keyspace, ti.Shard, topoproto.TabletAliasString(si.MasterAlias))
return topo.ErrNoUpdateNeeded
}
si.MasterAlias = nil
return nil
})
return err
}
return nil
}
func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, masterElectTabletAlias *topodatapb.TabletAlias, waitSlaveTimeout time.Duration) error {
shardInfo, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return err
}
ev.ShardInfo = *shardInfo
event.DispatchUpdate(ev, "reading all tablets")
tabletMap, err := wr.ts.GetTabletMapForShard(ctx, keyspace, shard)
if err != nil {
return err
}
// Check corner cases we're going to depend on
masterElectTabletInfo, ok := tabletMap[*masterElectTabletAlias]
if !ok {
return fmt.Errorf("master-elect tablet %v is not in the shard", topoproto.TabletAliasString(masterElectTabletAlias))
}
ev.NewMaster = *masterElectTabletInfo.Tablet
if topoproto.TabletAliasEqual(shardInfo.MasterAlias, masterElectTabletAlias) {
return fmt.Errorf("master-elect tablet %v is already the master", topoproto.TabletAliasString(masterElectTabletAlias))
}
// Deal with the old master: try to remote-scrap it, if it's
// truely dead we force-scrap it. Remove it from our map in any case.
if shardInfo.HasMaster() {
deleteOldMaster := true
oldMasterTabletInfo, ok := tabletMap[*shardInfo.MasterAlias]
if ok {
delete(tabletMap, *shardInfo.MasterAlias)
} else {
oldMasterTabletInfo, err = wr.ts.GetTablet(ctx, shardInfo.MasterAlias)
if err != nil {
wr.logger.Warningf("cannot read old master tablet %v, won't touch it: %v", topoproto.TabletAliasString(shardInfo.MasterAlias), err)
deleteOldMaster = false
}
}
if deleteOldMaster {
ev.OldMaster = *oldMasterTabletInfo.Tablet
wr.logger.Infof("deleting old master %v", topoproto.TabletAliasString(shardInfo.MasterAlias))
ctx, cancel := context.WithTimeout(ctx, waitSlaveTimeout)
defer cancel()
if err := topotools.DeleteTablet(ctx, wr.ts, oldMasterTabletInfo.Tablet); err != nil {
wr.logger.Warningf("failed to delete old master tablet %v: %v", topoproto.TabletAliasString(shardInfo.MasterAlias), err)
}
}
}
// Stop replication on all slaves, get their current
// replication position
event.DispatchUpdate(ev, "stop replication on all slaves")
wg := sync.WaitGroup{}
mu := sync.Mutex{}
statusMap := make(map[topodatapb.TabletAlias]*replicationdatapb.Status)
for alias, tabletInfo := range tabletMap {
wg.Add(1)
go func(alias topodatapb.TabletAlias, tabletInfo *topo.TabletInfo) {
defer wg.Done()
wr.logger.Infof("getting replication position from %v", topoproto.TabletAliasString(&alias))
ctx, cancel := context.WithTimeout(ctx, waitSlaveTimeout)
defer cancel()
rp, err := wr.TabletManagerClient().StopReplicationAndGetStatus(ctx, tabletInfo)
if err != nil {
wr.logger.Warningf("failed to get replication status from %v, ignoring tablet: %v", topoproto.TabletAliasString(&alias), err)
return
}
mu.Lock()
statusMap[alias] = rp
mu.Unlock()
}(alias, tabletInfo)
}
wg.Wait()
// Verify masterElect is alive and has the most advanced position
masterElectStatus, ok := statusMap[*masterElectTabletAlias]
if !ok {
return fmt.Errorf("couldn't get master elect %v replication position", topoproto.TabletAliasString(masterElectTabletAlias))
}
masterElectPos, err := replication.DecodePosition(masterElectStatus.Position)
if err != nil {
return fmt.Errorf("cannot decode master elect position %v: %v", masterElectStatus.Position, err)
}
for alias, status := range statusMap {
if topoproto.TabletAliasEqual(&alias, masterElectTabletAlias) {
continue
}
pos, err := replication.DecodePosition(status.Position)
if err != nil {
return fmt.Errorf("cannot decode slave %v position %v: %v", topoproto.TabletAliasString(&alias), status.Position, err)
}
if !masterElectPos.AtLeast(pos) {
return fmt.Errorf("tablet %v is more advanced than master elect tablet %v: %v > %v", topoproto.TabletAliasString(&alias), topoproto.TabletAliasString(masterElectTabletAlias), status.Position, masterElectStatus)
}
}
// Promote the masterElect
wr.logger.Infof("promote slave %v", topoproto.TabletAliasString(masterElectTabletAlias))
event.DispatchUpdate(ev, "promoting slave")
rp, err := wr.tmc.PromoteSlave(ctx, masterElectTabletInfo)
if err != nil {
return fmt.Errorf("master-elect tablet %v failed to be upgraded to master: %v", topoproto.TabletAliasString(masterElectTabletAlias), err)
}
// Reset replication on all slaves to point to the new master, and
// insert test row in the new master.
// Go through all the tablets:
// - new master: populate the reparent journal
// - everybody else: reparent to new master, wait for row
event.DispatchUpdate(ev, "reparenting all tablets")
now := time.Now().UnixNano()
wgMaster := sync.WaitGroup{}
wgSlaves := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
var masterErr error
for alias, tabletInfo := range tabletMap {
if topoproto.TabletAliasEqual(&alias, masterElectTabletAlias) {
wgMaster.Add(1)
go func(alias topodatapb.TabletAlias, tabletInfo *topo.TabletInfo) {
defer wgMaster.Done()
wr.logger.Infof("populating reparent journal on new master %v", topoproto.TabletAliasString(&alias))
masterErr = wr.TabletManagerClient().PopulateReparentJournal(ctx, tabletInfo, now, emergencyReparentShardOperation, &alias, rp)
}(alias, tabletInfo)
} else {
wgSlaves.Add(1)
go func(alias topodatapb.TabletAlias, tabletInfo *topo.TabletInfo) {
defer wgSlaves.Done()
wr.logger.Infof("setting new master on slave %v", topoproto.TabletAliasString(&alias))
forceStartSlave := false
if status, ok := statusMap[alias]; ok {
forceStartSlave = status.SlaveIoRunning || status.SlaveSqlRunning
}
if err := wr.TabletManagerClient().SetMaster(ctx, tabletInfo, masterElectTabletAlias, now, forceStartSlave); err != nil {
rec.RecordError(fmt.Errorf("Tablet %v SetMaster failed: %v", topoproto.TabletAliasString(&alias), err))
}
}(alias, tabletInfo)
}
}
// After the master is done, we can update the shard record
// (note with semi-sync, it also means at least one slave is done)
wgMaster.Wait()
if masterErr != nil {
wgSlaves.Wait()
return fmt.Errorf("failed to PopulateReparentJournal on master: %v", masterErr)
}
wr.logger.Infof("updating shard record with new master %v", topoproto.TabletAliasString(masterElectTabletAlias))
if _, err := wr.ts.UpdateShardFields(ctx, keyspace, shard, func(s *topodatapb.Shard) error {
s.MasterAlias = masterElectTabletAlias
return nil
}); err != nil {
wgSlaves.Wait()
return fmt.Errorf("failed to update shard master record: %v", err)
}
// Wait for the slaves to complete. If some of them fail, we
// will rebuild the shard serving graph anyway
wgSlaves.Wait()
if err := rec.Error(); err != nil {
wr.Logger().Errorf("Some slaves failed to reparent: %v", err)
return err
}
// Then we rebuild the entire serving graph for the shard,
// to account for all changes.
wr.logger.Infof("rebuilding shard graph")
event.DispatchUpdate(ev, "rebuilding shard serving graph")
_, err = wr.RebuildShardGraph(ctx, keyspace, shard, nil)
return err
}