// ReparentTablet tells a tablet to reparent this tablet to the current
// master, based on the current replication position. If there is no
// match, it will fail.
func (wr *Wrangler) ReparentTablet(ctx context.Context, tabletAlias *pb.TabletAlias) error {
// Get specified tablet.
// Get current shard master tablet.
// Sanity check they are in the same keyspace/shard.
// Issue a SetMaster to the tablet.
ti, err := wr.ts.GetTablet(ctx, tabletAlias)
if err != nil {
return err
}
shardInfo, err := wr.ts.GetShard(ctx, ti.Keyspace, ti.Shard)
if err != nil {
return err
}
if topo.TabletAliasIsZero(shardInfo.MasterAlias) {
return fmt.Errorf("no master tablet for shard %v/%v", ti.Keyspace, ti.Shard)
}
masterTi, err := wr.ts.GetTablet(ctx, shardInfo.MasterAlias)
if err != nil {
return err
}
// Basic sanity checking.
if masterTi.Type != pb.TabletType_MASTER {
return fmt.Errorf("TopologyServer has inconsistent state for shard master %v", shardInfo.MasterAlias)
}
if masterTi.Keyspace != ti.Keyspace || masterTi.Shard != ti.Shard {
return fmt.Errorf("master %v and potential slave not in same keyspace/shard", shardInfo.MasterAlias)
}
// and do the remote command
return wr.TabletManagerClient().SetMaster(ctx, ti, shardInfo.MasterAlias, 0, false)
}
// init phase:
// - read the shard info, make sure it has sources
func (vsdw *VerticalSplitDiffWorker) init(ctx context.Context) error {
vsdw.SetState(WorkerStateInit)
var err error
// read the keyspace and validate it
vsdw.keyspaceInfo, err = vsdw.wr.TopoServer().GetKeyspace(ctx, vsdw.keyspace)
if err != nil {
return fmt.Errorf("cannot read keyspace %v: %v", vsdw.keyspace, err)
}
if len(vsdw.keyspaceInfo.ServedFroms) == 0 {
return fmt.Errorf("keyspace %v has no KeyspaceServedFrom", vsdw.keyspace)
}
// read the shardinfo and validate it
vsdw.shardInfo, err = vsdw.wr.TopoServer().GetShard(ctx, vsdw.keyspace, vsdw.shard)
if err != nil {
return fmt.Errorf("cannot read shard %v/%v: %v", vsdw.keyspace, vsdw.shard, err)
}
if len(vsdw.shardInfo.SourceShards) != 1 {
return fmt.Errorf("shard %v/%v has bad number of source shards", vsdw.keyspace, vsdw.shard)
}
if len(vsdw.shardInfo.SourceShards[0].Tables) == 0 {
return fmt.Errorf("shard %v/%v has no tables in source shard[0]", vsdw.keyspace, vsdw.shard)
}
if topo.TabletAliasIsZero(vsdw.shardInfo.MasterAlias) {
return fmt.Errorf("shard %v/%v has no master", vsdw.keyspace, vsdw.shard)
}
return nil
}
// ValidateVersionKeyspace validates all versions are the same in all
// tablets in a keyspace
func (wr *Wrangler) ValidateVersionKeyspace(ctx context.Context, keyspace string) error {
// find all the shards
shards, err := wr.ts.GetShardNames(ctx, keyspace)
if err != nil {
return err
}
// corner cases
if len(shards) == 0 {
return fmt.Errorf("No shards in keyspace %v", keyspace)
}
sort.Strings(shards)
if len(shards) == 1 {
return wr.ValidateVersionShard(ctx, keyspace, shards[0])
}
// find the reference version using the first shard's master
si, err := wr.ts.GetShard(ctx, keyspace, shards[0])
if err != nil {
return err
}
if topo.TabletAliasIsZero(si.MasterAlias) {
return fmt.Errorf("No master in shard %v/%v", keyspace, shards[0])
}
referenceAlias := si.MasterAlias
log.Infof("Gathering version for reference master %v", topo.TabletAliasString(referenceAlias))
referenceVersion, err := wr.GetVersion(ctx, referenceAlias)
if err != nil {
return err
}
// then diff with all tablets but master 0
er := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for _, shard := range shards {
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard)
if err != nil {
er.RecordError(err)
continue
}
for _, alias := range aliases {
if topo.TabletAliasEqual(alias, si.MasterAlias) {
continue
}
wg.Add(1)
go wr.diffVersion(ctx, referenceVersion, referenceAlias, alias, &wg, &er)
}
}
wg.Wait()
if er.HasErrors() {
return fmt.Errorf("Version diffs:\n%v", er.Error().Error())
}
return nil
}
// updateShardCellsAndMaster will update the 'Cells' and possibly
// MasterAlias records for the shard, if needed.
func (wr *Wrangler) updateShardCellsAndMaster(ctx context.Context, si *topo.ShardInfo, tabletAlias *pb.TabletAlias, tabletType pb.TabletType, force bool) error {
// See if we need to update the Shard:
// - add the tablet's cell to the shard's Cells if needed
// - change the master if needed
shardUpdateRequired := false
if !si.HasCell(tabletAlias.Cell) {
shardUpdateRequired = true
}
if tabletType == pb.TabletType_MASTER && !topo.TabletAliasEqual(si.MasterAlias, tabletAlias) {
shardUpdateRequired = true
}
if !shardUpdateRequired {
return nil
}
actionNode := actionnode.UpdateShard()
keyspace := si.Keyspace()
shard := si.ShardName()
lockPath, err := wr.lockShard(ctx, keyspace, shard, actionNode)
if err != nil {
return err
}
// re-read the shard with the lock
si, err = wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err)
}
// update it
wasUpdated := false
if !si.HasCell(tabletAlias.Cell) {
si.Cells = append(si.Cells, tabletAlias.Cell)
wasUpdated = true
}
if tabletType == pb.TabletType_MASTER && !topo.TabletAliasEqual(si.MasterAlias, tabletAlias) {
if !topo.TabletAliasIsZero(si.MasterAlias) && !force {
return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", topo.TabletAliasString(si.MasterAlias), keyspace, shard))
}
si.MasterAlias = tabletAlias
wasUpdated = true
}
if wasUpdated {
// write it back
if err := topo.UpdateShard(ctx, wr.ts, si); err != nil {
return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err)
}
}
// and unlock
return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err)
}
// ValidateVersionShard validates all versions are the same in all
// tablets in a shard
func (wr *Wrangler) ValidateVersionShard(ctx context.Context, keyspace, shard string) error {
si, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return err
}
// get version from the master, or error
if topo.TabletAliasIsZero(si.MasterAlias) {
return fmt.Errorf("No master in shard %v/%v", keyspace, shard)
}
log.Infof("Gathering version for master %v", topo.TabletAliasString(si.MasterAlias))
masterVersion, err := wr.GetVersion(ctx, si.MasterAlias)
if err != nil {
return err
}
// read all the aliases in the shard, that is all tablets that are
// replicating from the master
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard)
if err != nil {
return err
}
// then diff with all slaves
er := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for _, alias := range aliases {
if topo.TabletAliasEqual(alias, si.MasterAlias) {
continue
}
wg.Add(1)
go wr.diffVersion(ctx, masterVersion, si.MasterAlias, alias, &wg, &er)
}
wg.Wait()
if er.HasErrors() {
return fmt.Errorf("Version diffs:\n%v", er.Error().Error())
}
return nil
}
// ValidateSchemaShard will diff the schema from all the tablets in the shard.
func (wr *Wrangler) ValidateSchemaShard(ctx context.Context, keyspace, shard string, excludeTables []string, includeViews bool) error {
si, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return err
}
// get schema from the master, or error
if topo.TabletAliasIsZero(si.MasterAlias) {
return fmt.Errorf("No master in shard %v/%v", keyspace, shard)
}
log.Infof("Gathering schema for master %v", si.MasterAlias)
masterSchema, err := wr.GetSchema(ctx, topo.ProtoToTabletAlias(si.MasterAlias), nil, excludeTables, includeViews)
if err != nil {
return err
}
// read all the aliases in the shard, that is all tablets that are
// replicating from the master
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard)
if err != nil {
return err
}
// then diff with all slaves
er := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for _, alias := range aliases {
if alias == topo.ProtoToTabletAlias(si.MasterAlias) {
continue
}
wg.Add(1)
go wr.diffSchema(ctx, masterSchema, topo.ProtoToTabletAlias(si.MasterAlias), alias, excludeTables, includeViews, &wg, &er)
}
wg.Wait()
if er.HasErrors() {
return fmt.Errorf("Schema diffs:\n%v", er.Error().Error())
}
return nil
}
// init phase:
// - read the shard info, make sure it has sources
func (sdw *SplitDiffWorker) init(ctx context.Context) error {
sdw.SetState(WorkerStateInit)
var err error
sdw.keyspaceInfo, err = sdw.wr.TopoServer().GetKeyspace(ctx, sdw.keyspace)
if err != nil {
return fmt.Errorf("cannot read keyspace %v: %v", sdw.keyspace, err)
}
sdw.shardInfo, err = sdw.wr.TopoServer().GetShard(ctx, sdw.keyspace, sdw.shard)
if err != nil {
return fmt.Errorf("cannot read shard %v/%v: %v", sdw.keyspace, sdw.shard, err)
}
if len(sdw.shardInfo.SourceShards) == 0 {
return fmt.Errorf("shard %v/%v has no source shard", sdw.keyspace, sdw.shard)
}
if topo.TabletAliasIsZero(sdw.shardInfo.MasterAlias) {
return fmt.Errorf("shard %v/%v has no master", sdw.keyspace, sdw.shard)
}
return nil
}
// ValidatePermissionsShard validates all the permissions are the same
// in a shard
func (wr *Wrangler) ValidatePermissionsShard(ctx context.Context, keyspace, shard string) error {
si, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return err
}
// get permissions from the master, or error
if topo.TabletAliasIsZero(si.MasterAlias) {
return fmt.Errorf("No master in shard %v/%v", keyspace, shard)
}
log.Infof("Gathering permissions for master %v", si.MasterAlias)
masterPermissions, err := wr.GetPermissions(ctx, topo.ProtoToTabletAlias(si.MasterAlias))
if err != nil {
return err
}
// read all the aliases in the shard, that is all tablets that are
// replicating from the master
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard)
if err != nil {
return err
}
// then diff all of them, except master
er := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for _, alias := range aliases {
if alias == topo.ProtoToTabletAlias(si.MasterAlias) {
continue
}
wg.Add(1)
go wr.diffPermissions(ctx, masterPermissions, topo.ProtoToTabletAlias(si.MasterAlias), alias, &wg, &er)
}
wg.Wait()
if er.HasErrors() {
return fmt.Errorf("Permissions diffs:\n%v", er.Error().Error())
}
return nil
}
// Does a topo lookup for a single shard, and returns the tablet record of the master tablet.
func resolveDestinationShardMaster(ctx context.Context, keyspace, shard string, wr *wrangler.Wrangler) (*topo.TabletInfo, error) {
var ti *topo.TabletInfo
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
si, err := topo.GetShard(shortCtx, wr.TopoServer(), keyspace, shard)
cancel()
if err != nil {
return ti, fmt.Errorf("unable to resolve destination shard %v/%v", keyspace, shard)
}
if topo.TabletAliasIsZero(si.MasterAlias) {
return ti, fmt.Errorf("no master in destination shard %v/%v", keyspace, shard)
}
wr.Logger().Infof("Found target master alias %v in shard %v/%v", si.MasterAlias, keyspace, shard)
shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout)
ti, err = topo.GetTablet(shortCtx, wr.TopoServer(), topo.ProtoToTabletAlias(si.MasterAlias))
cancel()
if err != nil {
return ti, fmt.Errorf("unable to get master tablet from alias %v in shard %v/%v",
si.MasterAlias, keyspace, shard)
}
return ti, nil
}
// InitTablet creates or updates a tablet. If no parent is specified
// in the tablet, and the tablet has a slave type, we will find the
// appropriate parent. If createShardAndKeyspace is true and the
// parent keyspace or shard don't exist, they will be created. If
// update is true, and a tablet with the same ID exists, update it.
// If Force is true, and a tablet with the same ID already exists, it
// will be scrapped and deleted, and then recreated.
func (wr *Wrangler) InitTablet(ctx context.Context, tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error {
if err := topo.TabletComplete(tablet); err != nil {
return err
}
if topo.IsInReplicationGraph(tablet.Type) {
// get the shard, possibly creating it
var err error
var si *topo.ShardInfo
if createShardAndKeyspace {
// create the parent keyspace and shard if needed
si, err = topotools.GetOrCreateShard(ctx, wr.ts, tablet.Keyspace, tablet.Shard)
} else {
si, err = wr.ts.GetShard(ctx, tablet.Keyspace, tablet.Shard)
if err == topo.ErrNoNode {
return fmt.Errorf("missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard")
}
}
// get the shard, checks a couple things
if err != nil {
return fmt.Errorf("cannot get (or create) shard %v/%v: %v", tablet.Keyspace, tablet.Shard, err)
}
if key.ProtoToKeyRange(si.KeyRange) != tablet.KeyRange {
return fmt.Errorf("shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange)
}
if tablet.Type == topo.TYPE_MASTER && !topo.TabletAliasIsZero(si.MasterAlias) && topo.ProtoToTabletAlias(si.MasterAlias) != tablet.Alias && !force {
return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, tablet.Keyspace, tablet.Shard)
}
// update the shard record if needed
if err := wr.updateShardCellsAndMaster(ctx, si, topo.TabletAliasToProto(tablet.Alias), topo.TabletTypeToProto(tablet.Type), force); err != nil {
return err
}
}
err := topo.CreateTablet(ctx, wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update || force {
oldTablet, err := wr.ts.GetTablet(ctx, tablet.Alias)
if err != nil {
wr.Logger().Warningf("failed reading tablet %v: %v", tablet.Alias, err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
if err := topo.UpdateTablet(ctx, wr.ts, oldTablet); err != nil {
wr.Logger().Warningf("failed updating tablet %v: %v", tablet.Alias, err)
// now fall through the Scrap case
} else {
if !topo.IsInReplicationGraph(tablet.Type) {
return nil
}
if err := topo.UpdateTabletReplicationData(ctx, wr.ts, tablet); err != nil {
wr.Logger().Warningf("failed updating tablet replication data for %v: %v", tablet.Alias, err)
// now fall through the Scrap case
} else {
return nil
}
}
}
}
}
if force {
if err = wr.Scrap(ctx, tablet.Alias, force, false); err != nil {
wr.Logger().Errorf("failed scrapping tablet %v: %v", tablet.Alias, err)
return err
}
if err := wr.ts.DeleteTablet(ctx, tablet.Alias); err != nil {
// we ignore this
wr.Logger().Errorf("failed deleting tablet %v: %v", tablet.Alias, err)
}
return topo.CreateTablet(ctx, wr.ts, tablet)
}
}
return err
}
func (wr *Wrangler) applySchemaShardComplex(ctx context.Context, statusArray []*tabletStatus, shardInfo *topo.ShardInfo, preflight *myproto.SchemaChangeResult, masterTabletAlias *pb.TabletAlias, change string, newParentTabletAlias *pb.TabletAlias, force bool, waitSlaveTimeout time.Duration) (*myproto.SchemaChangeResult, error) {
// apply the schema change to all replica / slave tablets
for _, status := range statusArray {
// if already applied, we skip this guy
diffs := myproto.DiffSchemaToArray("after", preflight.AfterSchema, topo.TabletAliasString(status.ti.Alias), status.beforeSchema)
if len(diffs) == 0 {
log.Infof("Tablet %v already has the AfterSchema, skipping", status.ti.Alias)
continue
}
// make sure the before schema matches
diffs = myproto.DiffSchemaToArray("master", preflight.BeforeSchema, topo.TabletAliasString(status.ti.Alias), status.beforeSchema)
if len(diffs) > 0 {
if force {
log.Warningf("Tablet %v has inconsistent schema, ignoring: %v", status.ti.Alias, strings.Join(diffs, "\n"))
} else {
return nil, fmt.Errorf("Tablet %v has inconsistent schema: %v", status.ti.Alias, strings.Join(diffs, "\n"))
}
}
// take this guy out of the serving graph if necessary
ti, err := wr.ts.GetTablet(ctx, status.ti.Alias)
if err != nil {
return nil, err
}
typeChangeRequired := ti.IsInServingGraph()
if typeChangeRequired {
// note we want to update the serving graph there
err = wr.changeTypeInternal(ctx, ti.Alias, pb.TabletType_SCHEMA_UPGRADE)
if err != nil {
return nil, err
}
}
// apply the schema change
log.Infof("Applying schema change to slave %v in complex mode", status.ti.Alias)
sc := &myproto.SchemaChange{Sql: change, Force: force, AllowReplication: false, BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}
_, err = wr.ApplySchema(ctx, status.ti.Alias, sc)
if err != nil {
return nil, err
}
// put this guy back into the serving graph
if typeChangeRequired {
err = wr.changeTypeInternal(ctx, ti.Alias, ti.Tablet.Type)
if err != nil {
return nil, err
}
}
}
// if newParentTabletAlias is passed in, use that as the new master
if !topo.TabletAliasIsZero(newParentTabletAlias) {
log.Infof("Reparenting with new master set to %v", newParentTabletAlias)
oldMasterAlias := shardInfo.MasterAlias
// Create reusable Reparent event with available info
ev := &events.Reparent{}
if err := wr.plannedReparentShardLocked(ctx, ev, shardInfo.Keyspace(), shardInfo.ShardName(), newParentTabletAlias, waitSlaveTimeout); err != nil {
return nil, err
}
// Here we would apply the schema change to the old
// master, but we just scrap it, to be consistent
// with the previous implementation of the reparent.
// (this code will be refactored at some point anyway)
if err := wr.Scrap(ctx, oldMasterAlias, false, false); err != nil {
wr.Logger().Warningf("Scrapping old master %v from shard %v/%v failed: %v", oldMasterAlias, shardInfo.Keyspace(), shardInfo.ShardName(), err)
}
}
return &myproto.SchemaChangeResult{BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}, nil
}
// ValidateSchemaKeyspace will diff the schema from all the tablets in
// the keyspace.
func (wr *Wrangler) ValidateSchemaKeyspace(ctx context.Context, keyspace string, excludeTables []string, includeViews bool) error {
// find all the shards
shards, err := wr.ts.GetShardNames(ctx, keyspace)
if err != nil {
return err
}
// corner cases
if len(shards) == 0 {
return fmt.Errorf("No shards in keyspace %v", keyspace)
}
sort.Strings(shards)
if len(shards) == 1 {
return wr.ValidateSchemaShard(ctx, keyspace, shards[0], excludeTables, includeViews)
}
// find the reference schema using the first shard's master
si, err := wr.ts.GetShard(ctx, keyspace, shards[0])
if err != nil {
return err
}
if topo.TabletAliasIsZero(si.MasterAlias) {
return fmt.Errorf("No master in shard %v/%v", keyspace, shards[0])
}
referenceAlias := si.MasterAlias
log.Infof("Gathering schema for reference master %v", referenceAlias)
referenceSchema, err := wr.GetSchema(ctx, referenceAlias, nil, excludeTables, includeViews)
if err != nil {
return err
}
// then diff with all other tablets everywhere
er := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
// first diff the slaves in the reference shard 0
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shards[0])
if err != nil {
return err
}
for _, alias := range aliases {
if topo.TabletAliasEqual(alias, si.MasterAlias) {
continue
}
wg.Add(1)
go wr.diffSchema(ctx, referenceSchema, referenceAlias, alias, excludeTables, includeViews, &wg, &er)
}
// then diffs all tablets in the other shards
for _, shard := range shards[1:] {
si, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
er.RecordError(err)
continue
}
if topo.TabletAliasIsZero(si.MasterAlias) {
er.RecordError(fmt.Errorf("No master in shard %v/%v", keyspace, shard))
continue
}
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard)
if err != nil {
er.RecordError(err)
continue
}
for _, alias := range aliases {
wg.Add(1)
go wr.diffSchema(ctx, referenceSchema, referenceAlias, alias, excludeTables, includeViews, &wg, &er)
}
}
wg.Wait()
if er.HasErrors() {
return fmt.Errorf("Schema diffs:\n%v", er.Error().Error())
}
return nil
}
// finalizeTabletExternallyReparented performs slow, synchronized reconciliation
// tasks that ensure topology is self-consistent, and then marks the reparent as
// finished by updating the global shard record.
func (agent *ActionAgent) finalizeTabletExternallyReparented(ctx context.Context, si *topo.ShardInfo, ev *events.Reparent) (err error) {
var wg sync.WaitGroup
var errs concurrency.AllErrorRecorder
oldMasterAlias := si.MasterAlias
// Update the tablet records and serving graph for the old and new master concurrently.
event.DispatchUpdate(ev, "updating old and new master tablet records")
log.Infof("finalizeTabletExternallyReparented: updating tablet records")
wg.Add(1)
go func() {
defer wg.Done()
// Update our own record to master.
var updatedTablet *topo.Tablet
err := topo.UpdateTabletFields(ctx, agent.TopoServer, agent.TabletAlias,
func(tablet *topo.Tablet) error {
tablet.Type = topo.TYPE_MASTER
tablet.Health = nil
updatedTablet = tablet
return nil
})
if err != nil {
errs.RecordError(err)
return
}
// Update the serving graph for the tablet.
if updatedTablet != nil {
errs.RecordError(
topotools.UpdateTabletEndpoints(ctx, agent.TopoServer, updatedTablet))
}
}()
if !topo.TabletAliasIsZero(oldMasterAlias) {
wg.Add(1)
go func() {
// Force the old master to spare.
var oldMasterTablet *topo.Tablet
err := topo.UpdateTabletFields(ctx, agent.TopoServer, topo.ProtoToTabletAlias(oldMasterAlias),
func(tablet *topo.Tablet) error {
tablet.Type = topo.TYPE_SPARE
oldMasterTablet = tablet
return nil
})
if err != nil {
errs.RecordError(err)
wg.Done()
return
}
if oldMasterTablet != nil {
// We now know more about the old master, so add it to event data.
ev.OldMaster = *oldMasterTablet
// Update the serving graph.
errs.RecordError(
topotools.UpdateTabletEndpoints(ctx, agent.TopoServer, oldMasterTablet))
wg.Done()
// Tell the old master to refresh its state. We don't need to wait for it.
tmc := tmclient.NewTabletManagerClient()
tmc.RefreshState(ctx, topo.NewTabletInfo(oldMasterTablet, -1))
}
}()
}
tablet := agent.Tablet()
// Wait for the tablet records to be updated. At that point, any rebuild will
// see the new master, so we're ready to mark the reparent as done in the
// global shard record.
wg.Wait()
if errs.HasErrors() {
return errs.Error()
}
// Update the master field in the global shard record. We don't use a lock
// here anymore. The lock was only to ensure that the global shard record
// didn't get modified between the time when we read it and the time when we
// write it back. Now we use an update loop pattern to do that instead.
event.DispatchUpdate(ev, "updating global shard record")
log.Infof("finalizeTabletExternallyReparented: updating global shard record")
si, err = topo.UpdateShardFields(ctx, agent.TopoServer, tablet.Keyspace, tablet.Shard, func(shard *pb.Shard) error {
shard.MasterAlias = topo.TabletAliasToProto(tablet.Alias)
return nil
})
if err != nil {
return err
}
// We already took care of updating the serving graph for the old and new masters.
// All that's left now is in case of a cross-cell reparent, we need to update the
// master cell setting in the SrvShard records of all cells.
if oldMasterAlias == nil || oldMasterAlias.Cell != tablet.Alias.Cell {
event.DispatchUpdate(ev, "rebuilding shard serving graph")
log.Infof("finalizeTabletExternallyReparented: updating SrvShard in all cells for cross-cell reparent")
if err := topotools.UpdateAllSrvShards(ctx, agent.TopoServer, si); err != nil {
return err
}
}
event.DispatchUpdate(ev, "finished")
return nil
}
func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, masterElectTabletAlias *pb.TabletAlias, waitSlaveTimeout time.Duration) error {
shardInfo, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return err
}
ev.ShardInfo = *shardInfo
event.DispatchUpdate(ev, "reading all tablets")
tabletMap, err := topo.GetTabletMapForShard(ctx, wr.ts, keyspace, shard)
if err != nil {
return err
}
// Check corner cases we're going to depend on
masterElectTabletInfo, ok := tabletMap[*masterElectTabletAlias]
if !ok {
return fmt.Errorf("master-elect tablet %v is not in the shard", masterElectTabletAlias)
}
ev.NewMaster = *masterElectTabletInfo.Tablet
if topo.TabletAliasEqual(shardInfo.MasterAlias, masterElectTabletAlias) {
return fmt.Errorf("master-elect tablet %v is already the master", masterElectTabletAlias)
}
// Deal with the old master: try to remote-scrap it, if it's
// truely dead we force-scrap it. Remove it from our map in any case.
if !topo.TabletAliasIsZero(shardInfo.MasterAlias) {
scrapOldMaster := true
oldMasterTabletInfo, ok := tabletMap[*shardInfo.MasterAlias]
if ok {
delete(tabletMap, *shardInfo.MasterAlias)
} else {
oldMasterTabletInfo, err = wr.ts.GetTablet(ctx, shardInfo.MasterAlias)
if err != nil {
wr.logger.Warningf("cannot read old master tablet %v, won't touch it: %v", shardInfo.MasterAlias, err)
scrapOldMaster = false
}
}
if scrapOldMaster {
ev.OldMaster = *oldMasterTabletInfo.Tablet
wr.logger.Infof("scrapping old master %v", shardInfo.MasterAlias)
ctx, cancel := context.WithTimeout(ctx, waitSlaveTimeout)
defer cancel()
if err := wr.tmc.Scrap(ctx, oldMasterTabletInfo); err != nil {
wr.logger.Warningf("remote scrapping failed master failed, will force the scrap: %v", err)
if err := topotools.Scrap(ctx, wr.ts, shardInfo.MasterAlias, true); err != nil {
wr.logger.Warningf("old master topo scrapping failed, continuing anyway: %v", err)
}
}
}
}
// Stop replication on all slaves, get their current
// replication position
event.DispatchUpdate(ev, "stop replication on all slaves")
wg := sync.WaitGroup{}
mu := sync.Mutex{}
statusMap := make(map[pb.TabletAlias]myproto.ReplicationStatus)
for alias, tabletInfo := range tabletMap {
wg.Add(1)
go func(alias pb.TabletAlias, tabletInfo *topo.TabletInfo) {
defer wg.Done()
wr.logger.Infof("getting replication position from %v", alias)
ctx, cancel := context.WithTimeout(ctx, waitSlaveTimeout)
defer cancel()
rp, err := wr.TabletManagerClient().StopReplicationAndGetStatus(ctx, tabletInfo)
if err != nil {
wr.logger.Warningf("failed to get replication status from %v, ignoring tablet: %v", alias, err)
return
}
mu.Lock()
statusMap[alias] = rp
mu.Unlock()
}(alias, tabletInfo)
}
wg.Wait()
// Verify masterElect is alive and has the most advanced position
masterElectStatus, ok := statusMap[*masterElectTabletAlias]
if !ok {
return fmt.Errorf("couldn't get master elect %v replication position", masterElectTabletAlias)
}
for alias, status := range statusMap {
if topo.TabletAliasEqual(&alias, masterElectTabletAlias) {
continue
}
if !masterElectStatus.Position.AtLeast(status.Position) {
return fmt.Errorf("tablet %v is more advanced than master elect tablet %v: %v > %v", alias, masterElectTabletAlias, status.Position, masterElectStatus)
}
}
// Promote the masterElect
wr.logger.Infof("promote slave %v", masterElectTabletAlias)
event.DispatchUpdate(ev, "promoting slave")
rp, err := wr.tmc.PromoteSlave(ctx, masterElectTabletInfo)
if err != nil {
return fmt.Errorf("master-elect tablet %v failed to be upgraded to master: %v", masterElectTabletAlias, err)
}
// Reset replication on all slaves to point to the new master, and
// insert test row in the new master.
// Go through all the tablets:
// - new master: populate the reparent journal
// - everybody else: reparent to new master, wait for row
event.DispatchUpdate(ev, "reparenting all tablets")
now := time.Now().UnixNano()
wgMaster := sync.WaitGroup{}
wgSlaves := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
var masterErr error
for alias, tabletInfo := range tabletMap {
if topo.TabletAliasEqual(&alias, masterElectTabletAlias) {
wgMaster.Add(1)
go func(alias pb.TabletAlias, tabletInfo *topo.TabletInfo) {
defer wgMaster.Done()
wr.logger.Infof("populating reparent journal on new master %v", alias)
masterErr = wr.TabletManagerClient().PopulateReparentJournal(ctx, tabletInfo, now, emergencyReparentShardOperation, &alias, rp)
}(alias, tabletInfo)
} else {
wgSlaves.Add(1)
go func(alias pb.TabletAlias, tabletInfo *topo.TabletInfo) {
defer wgSlaves.Done()
wr.logger.Infof("setting new master on slave %v", alias)
forceStartSlave := false
if status, ok := statusMap[alias]; ok {
forceStartSlave = status.SlaveIORunning || status.SlaveSQLRunning
}
if err := wr.TabletManagerClient().SetMaster(ctx, tabletInfo, masterElectTabletAlias, now, forceStartSlave); err != nil {
rec.RecordError(fmt.Errorf("Tablet %v SetMaster failed: %v", alias, err))
}
}(alias, tabletInfo)
}
}
// After the master is done, we can update the shard record
// (note with semi-sync, it also means at least one slave is done)
wgMaster.Wait()
if masterErr != nil {
wgSlaves.Wait()
return fmt.Errorf("failed to PopulateReparentJournal on master: %v", masterErr)
}
wr.logger.Infof("updating shard record with new master %v", masterElectTabletAlias)
shardInfo.MasterAlias = masterElectTabletAlias
if err := topo.UpdateShard(ctx, wr.ts, shardInfo); err != nil {
wgSlaves.Wait()
return fmt.Errorf("failed to update shard master record: %v", err)
}
// Wait for the slaves to complete. If some of them fail, we
// will rebuild the shard serving graph anyway
wgSlaves.Wait()
if err := rec.Error(); err != nil {
wr.Logger().Errorf("Some slaves failed to reparent: %v", err)
return err
}
// Then we rebuild the entire serving graph for the shard,
// to account for all changes.
wr.logger.Infof("rebuilding shard graph")
event.DispatchUpdate(ev, "rebuilding shard serving graph")
_, err = wr.RebuildShardGraph(ctx, keyspace, shard, nil)
return err
}