func (scw *SplitCloneWorker) initShardsForHorizontalResharding(ctx context.Context) error { // find the OverlappingShards in the keyspace shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) osList, err := topotools.FindOverlappingShards(shortCtx, scw.wr.TopoServer(), scw.destinationKeyspace) cancel() if err != nil { return fmt.Errorf("cannot FindOverlappingShards in %v: %v", scw.destinationKeyspace, err) } // find the shard we mentioned in there, if any os := topotools.OverlappingShardsForShard(osList, scw.shard) if os == nil { return fmt.Errorf("the specified shard %v/%v is not in any overlapping shard", scw.destinationKeyspace, scw.shard) } scw.wr.Logger().Infof("Found overlapping shards: %+v\n", os) // one side should have served types, the other one none, // figure out wich is which, then double check them all if len(os.Left[0].ServedTypes) > 0 { scw.sourceShards = os.Left scw.destinationShards = os.Right } else { scw.sourceShards = os.Right scw.destinationShards = os.Left } return nil }
// init phase: // - read the destination keyspace, make sure it has 'servedFrom' values func (scw *SplitCloneWorker) init(ctx context.Context) error { scw.setState(WorkerStateInit) var err error // read the keyspace and validate it shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) scw.keyspaceInfo, err = scw.wr.TopoServer().GetKeyspace(shortCtx, scw.keyspace) cancel() if err != nil { return fmt.Errorf("cannot read keyspace %v: %v", scw.keyspace, err) } // find the OverlappingShards in the keyspace shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout) osList, err := topotools.FindOverlappingShards(shortCtx, scw.wr.TopoServer(), scw.keyspace) cancel() if err != nil { return fmt.Errorf("cannot FindOverlappingShards in %v: %v", scw.keyspace, err) } // find the shard we mentioned in there, if any os := topotools.OverlappingShardsForShard(osList, scw.shard) if os == nil { return fmt.Errorf("the specified shard %v/%v is not in any overlapping shard", scw.keyspace, scw.shard) } scw.wr.Logger().Infof("Found overlapping shards: %+v\n", os) // one side should have served types, the other one none, // figure out wich is which, then double check them all if len(os.Left[0].ServedTypes) > 0 { scw.sourceShards = os.Left scw.destinationShards = os.Right } else { scw.sourceShards = os.Right scw.destinationShards = os.Left } // validate all serving types servingTypes := []pb.TabletType{pb.TabletType_MASTER, pb.TabletType_REPLICA, pb.TabletType_RDONLY} for _, st := range servingTypes { for _, si := range scw.sourceShards { if si.GetServedType(st) == nil { return fmt.Errorf("source shard %v/%v is not serving type %v", si.Keyspace(), si.ShardName(), st) } } } for _, si := range scw.destinationShards { if len(si.ServedTypes) > 0 { return fmt.Errorf("destination shard %v/%v is serving some types", si.Keyspace(), si.ShardName()) } } return nil }
// init phase: // - read the destination keyspace, make sure it has 'servedFrom' values func (scw *SplitCloneWorker) init() error { scw.setState(stateSCInit) var err error // read the keyspace and validate it scw.keyspaceInfo, err = scw.wr.TopoServer().GetKeyspace(scw.keyspace) if err != nil { return fmt.Errorf("cannot read keyspace %v: %v", scw.keyspace, err) } // find the OverlappingShards in the keyspace osList, err := topotools.FindOverlappingShards(scw.wr.TopoServer(), scw.keyspace) if err != nil { return fmt.Errorf("cannot FindOverlappingShards in %v: %v", scw.keyspace, err) } // find the shard we mentioned in there, if any os := topotools.OverlappingShardsForShard(osList, scw.shard) if os == nil { return fmt.Errorf("the specified shard %v/%v is not in any overlapping shard", scw.keyspace, scw.shard) } // one side should have served types, the other one none, // figure out wich is which, then double check them all if len(os.Left[0].ServedTypesMap) > 0 { scw.sourceShards = os.Left scw.destinationShards = os.Right } else { scw.sourceShards = os.Right scw.destinationShards = os.Left } // validate all serving types servingTypes := []topo.TabletType{topo.TYPE_MASTER, topo.TYPE_REPLICA, topo.TYPE_RDONLY} for _, st := range servingTypes { for _, si := range scw.sourceShards { if _, ok := si.ServedTypesMap[st]; !ok { return fmt.Errorf("source shard %v/%v is not serving type %v", si.Keyspace(), si.ShardName(), st) } } } for _, si := range scw.destinationShards { if len(si.ServedTypesMap) > 0 { return fmt.Errorf("destination shard %v/%v is serving some types", si.Keyspace(), si.ShardName()) } } return nil }
func keyspacesWithOverlappingShards(ctx context.Context, wr *wrangler.Wrangler) ([]map[string]string, error) { shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) keyspaces, err := wr.TopoServer().GetKeyspaces(shortCtx) cancel() if err != nil { return nil, fmt.Errorf("failed to get list of keyspaces: %v", err) } wg := sync.WaitGroup{} mu := sync.Mutex{} // protects result result := make([]map[string]string, 0, len(keyspaces)) rec := concurrency.AllErrorRecorder{} for _, keyspace := range keyspaces { wg.Add(1) go func(keyspace string) { defer wg.Done() shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout) osList, err := topotools.FindOverlappingShards(shortCtx, wr.TopoServer(), keyspace) cancel() if err != nil { rec.RecordError(err) return } mu.Lock() for _, os := range osList { result = append(result, map[string]string{ "Keyspace": os.Left[0].Keyspace(), "Shard": os.Left[0].ShardName(), }) } mu.Unlock() }(keyspace) } wg.Wait() if rec.HasErrors() { return nil, rec.Error() } if len(result) == 0 { return nil, fmt.Errorf("There are no keyspaces with overlapping shards") } return result, nil }
// MigrateServedTypes is used during horizontal splits to migrate a // served type from a list of shards to another. func (wr *Wrangler) MigrateServedTypes(ctx context.Context, keyspace, shard string, cells []string, servedType topodatapb.TabletType, reverse, skipReFreshState bool, filteredReplicationWaitTime time.Duration) error { if servedType == topodatapb.TabletType_MASTER { // we cannot migrate a master back, since when master migration // is done, the source shards are dead if reverse { return fmt.Errorf("Cannot migrate master back to %v/%v", keyspace, shard) } // we cannot skip refresh state for a master if skipReFreshState { return fmt.Errorf("Cannot skip refresh state for master migration on %v/%v", keyspace, shard) } } // find overlapping shards in this keyspace wr.Logger().Infof("Finding the overlapping shards in keyspace %v", keyspace) osList, err := topotools.FindOverlappingShards(ctx, wr.ts, keyspace) if err != nil { return fmt.Errorf("FindOverlappingShards failed: %v", err) } // find our shard in there os := topotools.OverlappingShardsForShard(osList, shard) if os == nil { return fmt.Errorf("Shard %v is not involved in any overlapping shards", shard) } // find which list is which: the sources have no source // shards, the destination have source shards. We check the // first entry in the lists, then just check they're // consistent var sourceShards []*topo.ShardInfo var destinationShards []*topo.ShardInfo if len(os.Left[0].SourceShards) == 0 { sourceShards = os.Left destinationShards = os.Right } else { sourceShards = os.Right destinationShards = os.Left } // Verify the sources has the type we're migrating (or not if reverse) for _, si := range sourceShards { if err := si.CheckServedTypesMigration(servedType, cells, !reverse); err != nil { return err } } // Verify the destinations do not have the type we're // migrating (or do if reverse) for _, si := range destinationShards { if err := si.CheckServedTypesMigration(servedType, cells, reverse); err != nil { return err } } // lock the shards: sources, then destinations // (note they're all ordered by shard name) actionNode := actionnode.MigrateServedTypes(servedType) sourceLockPath := make([]string, len(sourceShards)) for i, si := range sourceShards { sourceLockPath[i], err = wr.lockShard(ctx, si.Keyspace(), si.ShardName(), actionNode) if err != nil { wr.Logger().Errorf("Failed to lock source shard %v/%v, may need to unlock other shards manually", si.Keyspace(), si.ShardName()) return err } } destinationLockPath := make([]string, len(destinationShards)) for i, si := range destinationShards { destinationLockPath[i], err = wr.lockShard(ctx, si.Keyspace(), si.ShardName(), actionNode) if err != nil { wr.Logger().Errorf("Failed to lock destination shard %v/%v, may need to unlock other shards manually", si.Keyspace(), si.ShardName()) return err } } // record the action error and all unlock errors rec := concurrency.AllErrorRecorder{} // execute the migration rec.RecordError(wr.migrateServedTypes(ctx, keyspace, sourceShards, destinationShards, cells, servedType, reverse, filteredReplicationWaitTime)) // unlock the shards, we're done for i := len(destinationShards) - 1; i >= 0; i-- { rec.RecordError(wr.unlockShard(ctx, destinationShards[i].Keyspace(), destinationShards[i].ShardName(), actionNode, destinationLockPath[i], nil)) } for i := len(sourceShards) - 1; i >= 0; i-- { rec.RecordError(wr.unlockShard(ctx, sourceShards[i].Keyspace(), sourceShards[i].ShardName(), actionNode, sourceLockPath[i], nil)) } // rebuild the keyspace serving graph if there was no error if !rec.HasErrors() { rec.RecordError(wr.RebuildKeyspaceGraph(ctx, keyspace, cells, false)) } // Send a refresh to the tablets we just disabled, iff: // - we're not migrating a master // - we don't have any errors // - we're not told to skip the refresh if servedType != topodatapb.TabletType_MASTER && !rec.HasErrors() && !skipReFreshState { var refreshShards []*topo.ShardInfo if reverse { // For a backwards migration, we just disabled query service on the destination shards refreshShards = destinationShards } else { // For a forwards migration, we just disabled query service on the source shards refreshShards = sourceShards } for _, si := range refreshShards { rec.RecordError(wr.RefreshTablesByShard(ctx, si, servedType, cells)) } } return rec.Error() }
// MigrateServedTypes is used during horizontal splits to migrate a // served type from a list of shards to another. func (wr *Wrangler) MigrateServedTypes(ctx context.Context, keyspace, shard string, cells []string, servedType topodatapb.TabletType, reverse, skipReFreshState bool, filteredReplicationWaitTime time.Duration) (err error) { // check input parameters if servedType == topodatapb.TabletType_MASTER { // we cannot migrate a master back, since when master migration // is done, the source shards are dead if reverse { return fmt.Errorf("Cannot migrate master back to %v/%v", keyspace, shard) } // we cannot skip refresh state for a master if skipReFreshState { return fmt.Errorf("Cannot skip refresh state for master migration on %v/%v", keyspace, shard) } } // lock the keyspace ctx, unlock, lockErr := wr.ts.LockKeyspace(ctx, keyspace, fmt.Sprintf("MigrateServedTypes(%v)", servedType)) if lockErr != nil { return lockErr } defer unlock(&err) // find overlapping shards in this keyspace wr.Logger().Infof("Finding the overlapping shards in keyspace %v", keyspace) osList, err := topotools.FindOverlappingShards(ctx, wr.ts, keyspace) if err != nil { return fmt.Errorf("FindOverlappingShards failed: %v", err) } // find our shard in there os := topotools.OverlappingShardsForShard(osList, shard) if os == nil { return fmt.Errorf("Shard %v is not involved in any overlapping shards", shard) } // find which list is which: the sources have no source // shards, the destination have source shards. We check the // first entry in the lists, then just check they're // consistent var sourceShards []*topo.ShardInfo var destinationShards []*topo.ShardInfo if len(os.Left[0].SourceShards) == 0 { sourceShards = os.Left destinationShards = os.Right } else { sourceShards = os.Right destinationShards = os.Left } // Verify the sources has the type we're migrating (or not if reverse) for _, si := range sourceShards { if err := si.CheckServedTypesMigration(servedType, cells, !reverse); err != nil { return err } } // Verify the destinations do not have the type we're // migrating (or do if reverse) for _, si := range destinationShards { if err := si.CheckServedTypesMigration(servedType, cells, reverse); err != nil { return err } } // execute the migration if err = wr.migrateServedTypesLocked(ctx, keyspace, sourceShards, destinationShards, cells, servedType, reverse, filteredReplicationWaitTime); err != nil { return err } // rebuild the keyspace serving graph now that there is no error if err = topotools.RebuildKeyspaceLocked(ctx, wr.logger, wr.ts, keyspace, cells); err != nil { return err } // Send a refresh to the tablets we just disabled, iff: // - we're not migrating a master // - we don't have any errors // - we're not told to skip the refresh if servedType != topodatapb.TabletType_MASTER && !skipReFreshState { rec := concurrency.AllErrorRecorder{} var refreshShards []*topo.ShardInfo if reverse { // For a backwards migration, we just disabled query service on the destination shards refreshShards = destinationShards } else { // For a forwards migration, we just disabled query service on the source shards refreshShards = sourceShards } for _, si := range refreshShards { rec.RecordError(wr.RefreshTabletsByShard(ctx, si, servedType, cells)) } return rec.Error() } return nil }
// init phase: // - read the destination keyspace, make sure it has 'servedFrom' values func (scw *SplitCloneWorker) init(ctx context.Context) error { scw.setState(WorkerStateInit) var err error // read the keyspace and validate it shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) scw.keyspaceInfo, err = scw.wr.TopoServer().GetKeyspace(shortCtx, scw.keyspace) cancel() if err != nil { return fmt.Errorf("cannot read keyspace %v: %v", scw.keyspace, err) } // find the OverlappingShards in the keyspace shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout) osList, err := topotools.FindOverlappingShards(shortCtx, scw.wr.TopoServer(), scw.keyspace) cancel() if err != nil { return fmt.Errorf("cannot FindOverlappingShards in %v: %v", scw.keyspace, err) } // find the shard we mentioned in there, if any os := topotools.OverlappingShardsForShard(osList, scw.shard) if os == nil { return fmt.Errorf("the specified shard %v/%v is not in any overlapping shard", scw.keyspace, scw.shard) } scw.wr.Logger().Infof("Found overlapping shards: %+v\n", os) // one side should have served types, the other one none, // figure out wich is which, then double check them all if len(os.Left[0].ServedTypes) > 0 { scw.sourceShards = os.Left scw.destinationShards = os.Right } else { scw.sourceShards = os.Right scw.destinationShards = os.Left } // Verify that filtered replication is not already enabled. for _, si := range scw.destinationShards { if len(si.SourceShards) > 0 { return fmt.Errorf("destination shard %v/%v has filtered replication already enabled from a previous resharding (ShardInfo is set)."+ " This requires manual intervention e.g. use vtctl SourceShardDelete to remove it", si.Keyspace(), si.ShardName()) } } // validate all serving types servingTypes := []topodatapb.TabletType{topodatapb.TabletType_MASTER, topodatapb.TabletType_REPLICA, topodatapb.TabletType_RDONLY} for _, st := range servingTypes { for _, si := range scw.sourceShards { if si.GetServedType(st) == nil { return fmt.Errorf("source shard %v/%v is not serving type %v", si.Keyspace(), si.ShardName(), st) } } } for _, si := range scw.destinationShards { if len(si.ServedTypes) > 0 { return fmt.Errorf("destination shard %v/%v is serving some types", si.Keyspace(), si.ShardName()) } } // read the vschema if needed var keyspaceSchema *vindexes.KeyspaceSchema if *useV3ReshardingMode { kschema, err := scw.wr.TopoServer().GetVSchema(ctx, scw.keyspace) if err != nil { return fmt.Errorf("cannot load VSchema for keyspace %v: %v", scw.keyspace, err) } if kschema == nil { return fmt.Errorf("no VSchema for keyspace %v", scw.keyspace) } keyspaceSchema, err = vindexes.BuildKeyspaceSchema(kschema, scw.keyspace) if err != nil { return fmt.Errorf("cannot build vschema for keyspace %v: %v", scw.keyspace, err) } scw.keyspaceSchema = keyspaceSchema } // Initialize healthcheck and add destination shards to it. scw.healthCheck = discovery.NewHealthCheck(*remoteActionsTimeout, *healthcheckRetryDelay, *healthCheckTimeout) allShards := append(scw.sourceShards, scw.destinationShards...) for _, si := range allShards { watcher := discovery.NewShardReplicationWatcher(scw.wr.TopoServer(), scw.healthCheck, scw.cell, si.Keyspace(), si.ShardName(), *healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency) scw.shardWatchers = append(scw.shardWatchers, watcher) } return nil }
// MigrateServedTypes is used during horizontal splits to migrate a // served type from a list of shards to another. func (wr *Wrangler) MigrateServedTypes(keyspace, shard string, servedType topo.TabletType, reverse, skipRebuild bool) error { if servedType == topo.TYPE_MASTER { // we cannot migrate a master back, since when master migration // is done, the source shards are dead if reverse { return fmt.Errorf("Cannot migrate master back to %v/%v", keyspace, shard) } // we cannot skip rebuild for a master if skipRebuild { return fmt.Errorf("Cannot skip rebuild for master migration on %v/%v", keyspace, shard) } } // find overlapping shards in this keyspace wr.Logger().Infof("Finding the overlapping shards in keyspace %v", keyspace) osList, err := topotools.FindOverlappingShards(wr.ts, keyspace) if err != nil { return fmt.Errorf("FindOverlappingShards failed: %v", err) } // find our shard in there os := topotools.OverlappingShardsForShard(osList, shard) if os == nil { return fmt.Errorf("Shard %v is not involved in any overlapping shards", shard) } // find which list is which: the sources have no source // shards, the destination have source shards. We check the // first entry in the lists, then just check they're // consistent var sourceShards []*topo.ShardInfo var destinationShards []*topo.ShardInfo if len(os.Left[0].SourceShards) == 0 { sourceShards = os.Left destinationShards = os.Right } else { sourceShards = os.Right destinationShards = os.Left } // Verify the sources has the type we're migrating (or not if reverse) for _, si := range sourceShards { foundType := topo.IsTypeInList(servedType, si.ServedTypes) if reverse { if foundType { return fmt.Errorf("Source shard %v/%v is already serving type %v", si.Keyspace(), si.ShardName(), servedType) } } else { if !foundType { return fmt.Errorf("Source shard %v/%v is not serving type %v", si.Keyspace, si.ShardName(), servedType) } } if servedType == topo.TYPE_MASTER && len(si.ServedTypes) > 1 { return fmt.Errorf("Cannot migrate master out of %v/%v until everything else is migrated out", si.Keyspace(), si.ShardName()) } } // Verify the destinations do not have the type we're // migrating (or do if reverse) for _, si := range destinationShards { foundType := topo.IsTypeInList(servedType, si.ServedTypes) if reverse { if !foundType { return fmt.Errorf("Destination shard %v/%v is not serving type %v", si.Keyspace, si.ShardName(), servedType) } } else { if foundType { return fmt.Errorf("Destination shard %v/%v is already serving type %v", si.Keyspace(), si.ShardName(), servedType) } } } // lock the shards: sources, then destinations // (note they're all ordered by shard name) actionNode := actionnode.MigrateServedTypes(servedType) sourceLockPath := make([]string, len(sourceShards)) for i, si := range sourceShards { sourceLockPath[i], err = wr.lockShard(si.Keyspace(), si.ShardName(), actionNode) if err != nil { wr.Logger().Errorf("Failed to lock source shard %v/%v, may need to unlock other shards manually", si.Keyspace(), si.ShardName()) return err } } destinationLockPath := make([]string, len(destinationShards)) for i, si := range destinationShards { destinationLockPath[i], err = wr.lockShard(si.Keyspace(), si.ShardName(), actionNode) if err != nil { wr.Logger().Errorf("Failed to lock destination shard %v/%v, may need to unlock other shards manually", si.Keyspace(), si.ShardName()) return err } } // record the action error and all unlock errors rec := concurrency.AllErrorRecorder{} // execute the migration shardCache := make(map[string]*topo.ShardInfo) rec.RecordError(wr.migrateServedTypes(keyspace, sourceShards, destinationShards, servedType, reverse, shardCache)) // unlock the shards, we're done for i := len(destinationShards) - 1; i >= 0; i-- { rec.RecordError(wr.unlockShard(destinationShards[i].Keyspace(), destinationShards[i].ShardName(), actionNode, destinationLockPath[i], nil)) } for i := len(sourceShards) - 1; i >= 0; i-- { rec.RecordError(wr.unlockShard(sourceShards[i].Keyspace(), sourceShards[i].ShardName(), actionNode, sourceLockPath[i], nil)) } // rebuild the keyspace serving graph if there was no error if rec.Error() == nil { if skipRebuild { wr.Logger().Infof("Skipping keyspace rebuild, please run it at earliest convenience") } else { rec.RecordError(wr.RebuildKeyspaceGraph(keyspace, nil, shardCache)) } } return rec.Error() }
// MigrateServedTypes is used during horizontal splits to migrate a // served type from a list of shards to another. func (wr *Wrangler) MigrateServedTypes(ctx context.Context, keyspace, shard string, cells []string, servedType topodatapb.TabletType, reverse, skipReFreshState bool, filteredReplicationWaitTime time.Duration) (err error) { // check input parameters if servedType == topodatapb.TabletType_MASTER { // we cannot migrate a master back, since when master migration // is done, the source shards are dead if reverse { return fmt.Errorf("Cannot migrate master back to %v/%v", keyspace, shard) } // we cannot skip refresh state for a master if skipReFreshState { return fmt.Errorf("Cannot skip refresh state for master migration on %v/%v", keyspace, shard) } } // lock the keyspace ctx, unlock, lockErr := wr.ts.LockKeyspace(ctx, keyspace, fmt.Sprintf("MigrateServedTypes(%v)", servedType)) if lockErr != nil { return lockErr } defer unlock(&err) // find overlapping shards in this keyspace wr.Logger().Infof("Finding the overlapping shards in keyspace %v", keyspace) osList, err := topotools.FindOverlappingShards(ctx, wr.ts, keyspace) if err != nil { return fmt.Errorf("FindOverlappingShards failed: %v", err) } // find our shard in there os := topotools.OverlappingShardsForShard(osList, shard) if os == nil { return fmt.Errorf("Shard %v is not involved in any overlapping shards", shard) } // find which list is which: the sources have no source // shards, the destination have source shards. We check the // first entry in the lists, then just check they're // consistent var sourceShards []*topo.ShardInfo var destinationShards []*topo.ShardInfo if len(os.Left[0].SourceShards) == 0 { if len(os.Right[0].SourceShards) == 0 { return fmt.Errorf("neither Shard '%v' nor Shard '%v' have a 'SourceShards' entry. Did you successfully run vtworker SplitClone before? Or did you already migrate the MASTER type?", os.Left[0].ShardName(), os.Right[0].ShardName()) } sourceShards = os.Left destinationShards = os.Right } else { sourceShards = os.Right destinationShards = os.Left } // Verify the sources has the type we're migrating (or not if reverse) for _, si := range sourceShards { if err := si.CheckServedTypesMigration(servedType, cells, !reverse); err != nil { return err } } // Verify the destinations do not have the type we're // migrating (or do if reverse) for _, si := range destinationShards { if err := si.CheckServedTypesMigration(servedType, cells, reverse); err != nil { return err } } // execute the migration if err = wr.migrateServedTypesLocked(ctx, keyspace, sourceShards, destinationShards, cells, servedType, reverse, filteredReplicationWaitTime); err != nil { return err } // rebuild the keyspace serving graph now that there is no error if err = topotools.RebuildKeyspaceLocked(ctx, wr.logger, wr.ts, keyspace, cells); err != nil { return err } // Send a refresh to the tablets we just disabled, iff: // - we're not migrating a master // - we don't have any errors // - we're not told to skip the refresh if servedType != topodatapb.TabletType_MASTER && !skipReFreshState { rec := concurrency.AllErrorRecorder{} var refreshShards []*topo.ShardInfo if reverse { // For a backwards migration, we just disabled query service on the destination shards refreshShards = destinationShards } else { // For a forwards migration, we just disabled query service on the source shards refreshShards = sourceShards } // TODO(b/26388813): Integrate vtctl WaitForDrain here instead of just sleeping. var waitForDrainSleep time.Duration switch servedType { case topodatapb.TabletType_RDONLY: waitForDrainSleep = *waitForDrainSleepRdonly case topodatapb.TabletType_REPLICA: waitForDrainSleep = *waitForDrainSleepReplica default: wr.Logger().Warningf("invalid TabletType: %v for MigrateServedTypes command", servedType) } wr.Logger().Infof("WaitForDrain: Sleeping for %.0f seconds before shutting down query service on old tablets...", waitForDrainSleep.Seconds()) time.Sleep(waitForDrainSleep) wr.Logger().Infof("WaitForDrain: Sleeping finished. Shutting down queryservice on old tablets now.") for _, si := range refreshShards { rec.RecordError(wr.RefreshTabletsByShard(ctx, si, []topodatapb.TabletType{servedType}, cells)) } return rec.Error() } return nil }