// init phase:
// - read the destination keyspace, make sure it has 'servedFrom' values
func (scw *SplitCloneWorker) init() error {
scw.setState(stateSCInit)
var err error
// read the keyspace and validate it
scw.keyspaceInfo, err = scw.wr.TopoServer().GetKeyspace(scw.keyspace)
if err != nil {
return fmt.Errorf("cannot read keyspace %v: %v", scw.keyspace, err)
}
// find the OverlappingShards in the keyspace
osList, err := topotools.FindOverlappingShards(scw.wr.TopoServer(), scw.keyspace)
if err != nil {
return fmt.Errorf("cannot FindOverlappingShards in %v: %v", scw.keyspace, err)
}
// find the shard we mentioned in there, if any
os := topotools.OverlappingShardsForShard(osList, scw.shard)
if os == nil {
return fmt.Errorf("the specified shard %v/%v is not in any overlapping shard", scw.keyspace, scw.shard)
}
// one side should have served types, the other one none,
// figure out wich is which, then double check them all
if len(os.Left[0].ServedTypesMap) > 0 {
scw.sourceShards = os.Left
scw.destinationShards = os.Right
} else {
scw.sourceShards = os.Right
scw.destinationShards = os.Left
}
// validate all serving types
servingTypes := []topo.TabletType{topo.TYPE_MASTER, topo.TYPE_REPLICA, topo.TYPE_RDONLY}
for _, st := range servingTypes {
for _, si := range scw.sourceShards {
if _, ok := si.ServedTypesMap[st]; !ok {
return fmt.Errorf("source shard %v/%v is not serving type %v", si.Keyspace(), si.ShardName(), st)
}
}
}
for _, si := range scw.destinationShards {
if len(si.ServedTypesMap) > 0 {
return fmt.Errorf("destination shard %v/%v is serving some types", si.Keyspace(), si.ShardName())
}
}
return nil
}
func keyspacesWithOverlappingShards(wr *wrangler.Wrangler) ([]map[string]string, error) {
keyspaces, err := wr.TopoServer().GetKeyspaces()
if err != nil {
return nil, err
}
wg := sync.WaitGroup{}
mu := sync.Mutex{} // protects result
result := make([]map[string]string, 0, len(keyspaces))
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
osList, err := topotools.FindOverlappingShards(wr.TopoServer(), keyspace)
if err != nil {
rec.RecordError(err)
return
}
mu.Lock()
for _, os := range osList {
result = append(result, map[string]string{
"Keyspace": os.Left[0].Keyspace(),
"Shard": os.Left[0].ShardName(),
})
}
mu.Unlock()
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
return nil, rec.Error()
}
if len(result) == 0 {
return nil, fmt.Errorf("There are no keyspaces with overlapping shards")
}
return result, nil
}
// MigrateServedTypes is used during horizontal splits to migrate a
// served type from a list of shards to another.
func (wr *Wrangler) MigrateServedTypes(keyspace, shard string, cells []string, servedType topo.TabletType, reverse, skipReFreshState bool) error {
if servedType == topo.TYPE_MASTER {
// we cannot migrate a master back, since when master migration
// is done, the source shards are dead
if reverse {
return fmt.Errorf("Cannot migrate master back to %v/%v", keyspace, shard)
}
// we cannot skip refresh state for a master
if skipReFreshState {
return fmt.Errorf("Cannot skip refresh state for master migration on %v/%v", keyspace, shard)
}
}
// find overlapping shards in this keyspace
wr.Logger().Infof("Finding the overlapping shards in keyspace %v", keyspace)
osList, err := topotools.FindOverlappingShards(wr.ts, keyspace)
if err != nil {
return fmt.Errorf("FindOverlappingShards failed: %v", err)
}
// find our shard in there
os := topotools.OverlappingShardsForShard(osList, shard)
if os == nil {
return fmt.Errorf("Shard %v is not involved in any overlapping shards", shard)
}
// find which list is which: the sources have no source
// shards, the destination have source shards. We check the
// first entry in the lists, then just check they're
// consistent
var sourceShards []*topo.ShardInfo
var destinationShards []*topo.ShardInfo
if len(os.Left[0].SourceShards) == 0 {
sourceShards = os.Left
destinationShards = os.Right
} else {
sourceShards = os.Right
destinationShards = os.Left
}
// Verify the sources has the type we're migrating (or not if reverse)
for _, si := range sourceShards {
if err := si.CheckServedTypesMigration(servedType, cells, !reverse); err != nil {
return err
}
}
// Verify the destinations do not have the type we're
// migrating (or do if reverse)
for _, si := range destinationShards {
if err := si.CheckServedTypesMigration(servedType, cells, reverse); err != nil {
return err
}
}
// lock the shards: sources, then destinations
// (note they're all ordered by shard name)
actionNode := actionnode.MigrateServedTypes(servedType)
sourceLockPath := make([]string, len(sourceShards))
for i, si := range sourceShards {
sourceLockPath[i], err = wr.lockShard(si.Keyspace(), si.ShardName(), actionNode)
if err != nil {
wr.Logger().Errorf("Failed to lock source shard %v/%v, may need to unlock other shards manually", si.Keyspace(), si.ShardName())
return err
}
}
destinationLockPath := make([]string, len(destinationShards))
for i, si := range destinationShards {
destinationLockPath[i], err = wr.lockShard(si.Keyspace(), si.ShardName(), actionNode)
if err != nil {
wr.Logger().Errorf("Failed to lock destination shard %v/%v, may need to unlock other shards manually", si.Keyspace(), si.ShardName())
return err
}
}
// record the action error and all unlock errors
rec := concurrency.AllErrorRecorder{}
// execute the migration
rec.RecordError(wr.migrateServedTypes(keyspace, sourceShards, destinationShards, cells, servedType, reverse))
// unlock the shards, we're done
for i := len(destinationShards) - 1; i >= 0; i-- {
rec.RecordError(wr.unlockShard(destinationShards[i].Keyspace(), destinationShards[i].ShardName(), actionNode, destinationLockPath[i], nil))
}
for i := len(sourceShards) - 1; i >= 0; i-- {
rec.RecordError(wr.unlockShard(sourceShards[i].Keyspace(), sourceShards[i].ShardName(), actionNode, sourceLockPath[i], nil))
}
// rebuild the keyspace serving graph if there was no error
if !rec.HasErrors() {
rec.RecordError(wr.RebuildKeyspaceGraph(keyspace, nil))
}
// Send a refresh to the source tablets we just disabled, iff:
// - we're not migrating a master
// - it is not a reverse migration
// - we don't have any errors
// - we're not told to skip the refresh
if servedType != topo.TYPE_MASTER && !reverse && !rec.HasErrors() && !skipReFreshState {
for _, si := range sourceShards {
rec.RecordError(wr.RefreshTablesByShard(si, servedType, cells))
}
}
return rec.Error()
}
