// RefreshTablesByShard calls RefreshState on all the tables of a
// given type in a shard. It would work for the master, but the
// discovery wouldn't be very efficient.
func (wr *Wrangler) RefreshTablesByShard(si *topo.ShardInfo, tabletType topo.TabletType, cells []string) error {
tabletMap, err := topo.GetTabletMapForShardByCell(wr.ts, si.Keyspace(), si.ShardName(), cells)
switch err {
case nil:
// keep going
case topo.ErrPartialResult:
wr.Logger().Warningf("RefreshTablesByShard: got partial result for shard %v/%v, may not refresh all tablets everywhere", si.Keyspace(), si.ShardName())
default:
return err
}
// ignore errors in this phase
wg := sync.WaitGroup{}
for _, ti := range tabletMap {
if ti.Type != tabletType {
continue
}
wg.Add(1)
go func(ti *topo.TabletInfo) {
if err := wr.tmc.RefreshState(ti, wr.ActionTimeout()); err != nil {
wr.Logger().Warningf("RefreshTablesByShard: failed to refresh %v: %v", ti.Alias, err)
}
wg.Done()
}(ti)
}
wg.Wait()
return nil
}
// replicaMigrateServedFrom handles the slave (replica, rdonly) migration.
func (wr *Wrangler) replicaMigrateServedFrom(ki *topo.KeyspaceInfo, sourceShard *topo.ShardInfo, destinationShard *topo.ShardInfo, servedType topo.TabletType, reverse bool, tables []string, ev *events.MigrateServedFrom) error {
// Save the destination keyspace (its ServedFrom has been changed)
event.DispatchUpdate(ev, "updating keyspace")
if err := topo.UpdateKeyspace(wr.ts, ki); err != nil {
return err
}
// Save the source shard (its blacklisted tables field has changed)
event.DispatchUpdate(ev, "updating source shard")
if sourceShard.BlacklistedTablesMap == nil {
sourceShard.BlacklistedTablesMap = make(map[topo.TabletType][]string)
}
if reverse {
delete(sourceShard.BlacklistedTablesMap, servedType)
} else {
sourceShard.BlacklistedTablesMap[servedType] = tables
}
if err := topo.UpdateShard(wr.ts, sourceShard); err != nil {
return err
}
// Now refresh the source servers so they reload their
// blacklisted table list
event.DispatchUpdate(ev, "refreshing sources tablets state so they update their blacklisted tables")
if err := wr.RefreshTablesByShard(sourceShard.Keyspace(), sourceShard.ShardName(), servedType); err != nil {
return err
}
return nil
}
// replicaMigrateServedFrom handles the slave (replica, rdonly) migration.
func (wr *Wrangler) replicaMigrateServedFrom(ctx context.Context, ki *topo.KeyspaceInfo, sourceShard *topo.ShardInfo, destinationShard *topo.ShardInfo, servedType topodatapb.TabletType, cells []string, reverse bool, tables []string, ev *events.MigrateServedFrom) error {
// Save the destination keyspace (its ServedFrom has been changed)
event.DispatchUpdate(ev, "updating keyspace")
if err := wr.ts.UpdateKeyspace(ctx, ki); err != nil {
return err
}
// Save the source shard (its blacklisted tables field has changed)
event.DispatchUpdate(ev, "updating source shard")
if err := sourceShard.UpdateSourceBlacklistedTables(servedType, cells, reverse, tables); err != nil {
return fmt.Errorf("UpdateSourceBlacklistedTables(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err)
}
if err := wr.ts.UpdateShard(ctx, sourceShard); err != nil {
return fmt.Errorf("UpdateShard(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err)
}
// Now refresh the source servers so they reload their
// blacklisted table list
event.DispatchUpdate(ev, "refreshing sources tablets state so they update their blacklisted tables")
if err := wr.RefreshTablesByShard(ctx, sourceShard, servedType, cells); err != nil {
return err
}
return nil
}
func (wr *Wrangler) validateReplication(shardInfo *topo.ShardInfo, tabletMap map[topo.TabletAlias]*topo.TabletInfo, results chan<- vresult) {
_, ok := tabletMap[shardInfo.MasterAlias]
if !ok {
results <- vresult{shardInfo.MasterAlias.String(), fmt.Errorf("master not in tablet map")}
return
}
actionPath, err := wr.ai.GetSlaves(shardInfo.MasterAlias)
if err != nil {
results <- vresult{shardInfo.MasterAlias.String(), err}
return
}
sa, err := wr.ai.WaitForCompletionReply(actionPath, wr.actionTimeout())
if err != nil {
results <- vresult{shardInfo.MasterAlias.String(), err}
return
}
slaveAddrs := sa.(*tm.SlaveList).Addrs
if len(slaveAddrs) == 0 {
results <- vresult{shardInfo.MasterAlias.String(), fmt.Errorf("no slaves found")}
return
}
// Some addresses don't resolve in all locations, just use IP address
if err != nil {
results <- vresult{shardInfo.MasterAlias.String(), fmt.Errorf("resolve slaves failed: %v", err)}
return
}
tabletIpMap := make(map[string]*topo.Tablet)
for _, tablet := range tabletMap {
ipAddr, _, err := net.SplitHostPort(tablet.MysqlIpAddr)
if err != nil {
results <- vresult{tablet.Alias().String(), fmt.Errorf("bad mysql addr: %v %v", tablet.MysqlIpAddr, err)}
continue
}
tabletIpMap[ipAddr] = tablet.Tablet
}
// See if every slave is in the replication graph.
for _, slaveAddr := range slaveAddrs {
if tabletIpMap[slaveAddr] == nil {
results <- vresult{shardInfo.Keyspace() + "/" + shardInfo.ShardName(), fmt.Errorf("slave not in replication graph: %v (mysql instance without vttablet?)", slaveAddr)}
}
}
// See if every entry in the replication graph is connected to the master.
for _, tablet := range tabletMap {
if !tablet.IsSlaveType() {
continue
}
ipAddr, _, err := net.SplitHostPort(tablet.MysqlIpAddr)
if err != nil {
results <- vresult{tablet.Alias().String(), fmt.Errorf("bad mysql addr: %v", err)}
} else if !strInList(slaveAddrs, ipAddr) {
results <- vresult{tablet.Alias().String(), fmt.Errorf("slave not replicating: %v %q", ipAddr, slaveAddrs)}
}
}
}
func (wr *Wrangler) migrateServedFrom(ki *topo.KeyspaceInfo, destinationShard *topo.ShardInfo, servedType topo.TabletType, reverse bool) (err error) {
// re-read and update keyspace info record
ki, err = wr.ts.GetKeyspace(ki.KeyspaceName())
if err != nil {
return err
}
if reverse {
if _, ok := ki.ServedFrom[servedType]; ok {
return fmt.Errorf("Destination Keyspace %s is not serving type %v", ki.KeyspaceName(), servedType)
}
ki.ServedFrom[servedType] = destinationShard.SourceShards[0].Keyspace
} else {
if _, ok := ki.ServedFrom[servedType]; !ok {
return fmt.Errorf("Destination Keyspace %s is already serving type %v", ki.KeyspaceName(), servedType)
}
delete(ki.ServedFrom, servedType)
}
// re-read and check the destination shard
destinationShard, err = wr.ts.GetShard(destinationShard.Keyspace(), destinationShard.ShardName())
if err != nil {
return err
}
if len(destinationShard.SourceShards) != 1 {
return fmt.Errorf("Destination shard %v/%v is not a vertical split target", destinationShard.Keyspace(), destinationShard.ShardName())
}
tables := destinationShard.SourceShards[0].Tables
// read the source shard, we'll need its master, and we'll need to
// update the blacklisted tables.
var sourceShard *topo.ShardInfo
sourceShard, err = wr.ts.GetShard(destinationShard.SourceShards[0].Keyspace, destinationShard.SourceShards[0].Shard)
if err != nil {
return err
}
ev := &events.MigrateServedFrom{
Keyspace: *ki,
SourceShard: *sourceShard,
DestinationShard: *destinationShard,
ServedType: servedType,
Reverse: reverse,
}
event.DispatchUpdate(ev, "start")
defer func() {
if err != nil {
event.DispatchUpdate(ev, "failed: "+err.Error())
}
}()
if servedType == topo.TYPE_MASTER {
err = wr.masterMigrateServedFrom(ki, sourceShard, destinationShard, servedType, tables, ev)
} else {
err = wr.replicaMigrateServedFrom(ki, sourceShard, destinationShard, servedType, reverse, tables, ev)
}
event.DispatchUpdate(ev, "finished")
return
}
// UpdateSrvShard creates the SrvShard object based on the global ShardInfo,
// and writes it to the given cell.
func UpdateSrvShard(ctx context.Context, ts topo.Server, cell string, si *topo.ShardInfo) error {
srvShard := &topo.SrvShard{
Name: si.ShardName(),
KeyRange: si.KeyRange,
MasterCell: si.MasterAlias.Cell,
}
return ts.UpdateSrvShard(ctx, cell, si.Keyspace(), si.ShardName(), srvShard)
}
func (wr *Wrangler) shardReplicationStatuses(ctx context.Context, shardInfo *topo.ShardInfo) ([]*topo.TabletInfo, []*myproto.ReplicationStatus, error) {
// FIXME(msolomon) this assumes no hierarchical replication, which is currently the case.
tabletMap, err := wr.ts.GetTabletMapForShard(ctx, shardInfo.Keyspace(), shardInfo.ShardName())
if err != nil {
return nil, nil, err
}
tablets := topotools.CopyMapValues(tabletMap, []*topo.TabletInfo{}).([]*topo.TabletInfo)
stats, err := wr.tabletReplicationStatuses(ctx, tablets)
return tablets, stats, err
}
func (wr *Wrangler) shardReplicationPositions(shardInfo *topo.ShardInfo) ([]*topo.TabletInfo, []*mysqlctl.ReplicationPosition, error) {
// FIXME(msolomon) this assumes no hierarchical replication, which is currently the case.
tabletMap, err := GetTabletMapForShard(wr.ts, shardInfo.Keyspace(), shardInfo.ShardName())
if err != nil {
return nil, nil, err
}
tablets := CopyMapValues(tabletMap, []*topo.TabletInfo{}).([]*topo.TabletInfo)
positions, err := wr.tabletReplicationPositions(tablets)
return tablets, positions, err
}
func (zkts *Server) UpdateShard(si *topo.ShardInfo) error {
shardPath := path.Join(globalKeyspacesPath, si.Keyspace(), "shards", si.ShardName())
_, err := zkts.zconn.Set(shardPath, jscfg.ToJson(si.Shard), -1)
if err != nil {
if zookeeper.IsError(err, zookeeper.ZNONODE) {
err = topo.ErrNoNode
}
}
return err
}
// updateShardCellsAndMaster will update the 'Cells' and possibly
// MasterAlias records for the shard, if needed.
func (wr *Wrangler) updateShardCellsAndMaster(ctx context.Context, si *topo.ShardInfo, tabletAlias topo.TabletAlias, tabletType topo.TabletType, force bool) error {
// See if we need to update the Shard:
// - add the tablet's cell to the shard's Cells if needed
// - change the master if needed
shardUpdateRequired := false
if !si.HasCell(tabletAlias.Cell) {
shardUpdateRequired = true
}
if tabletType == topo.TYPE_MASTER && si.MasterAlias != tabletAlias {
shardUpdateRequired = true
}
if !shardUpdateRequired {
return nil
}
actionNode := actionnode.UpdateShard()
keyspace := si.Keyspace()
shard := si.ShardName()
lockPath, err := wr.lockShard(ctx, keyspace, shard, actionNode)
if err != nil {
return err
}
// re-read the shard with the lock
si, err = wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err)
}
// update it
wasUpdated := false
if !si.HasCell(tabletAlias.Cell) {
si.Cells = append(si.Cells, tabletAlias.Cell)
wasUpdated = true
}
if tabletType == topo.TYPE_MASTER && si.MasterAlias != tabletAlias {
if !si.MasterAlias.IsZero() && !force {
return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, keyspace, shard))
}
si.MasterAlias = tabletAlias
wasUpdated = true
}
if wasUpdated {
// write it back
if err := topo.UpdateShard(ctx, wr.ts, si); err != nil {
return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err)
}
}
// and unlock
return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err)
}
func (wr *Wrangler) migrateServedFrom(ctx context.Context, ki *topo.KeyspaceInfo, destinationShard *topo.ShardInfo, servedType topodatapb.TabletType, cells []string, reverse bool, filteredReplicationWaitTime time.Duration) (err error) {
// re-read and update keyspace info record
ki, err = wr.ts.GetKeyspace(ctx, ki.KeyspaceName())
if err != nil {
return err
}
if reverse {
ki.UpdateServedFromMap(servedType, cells, destinationShard.SourceShards[0].Keyspace, false, nil)
} else {
ki.UpdateServedFromMap(servedType, cells, destinationShard.SourceShards[0].Keyspace, true, destinationShard.Cells)
}
// re-read and check the destination shard
destinationShard, err = wr.ts.GetShard(ctx, destinationShard.Keyspace(), destinationShard.ShardName())
if err != nil {
return err
}
if len(destinationShard.SourceShards) != 1 {
return fmt.Errorf("Destination shard %v/%v is not a vertical split target", destinationShard.Keyspace(), destinationShard.ShardName())
}
tables := destinationShard.SourceShards[0].Tables
// read the source shard, we'll need its master, and we'll need to
// update the blacklisted tables.
var sourceShard *topo.ShardInfo
sourceShard, err = wr.ts.GetShard(ctx, destinationShard.SourceShards[0].Keyspace, destinationShard.SourceShards[0].Shard)
if err != nil {
return err
}
ev := &events.MigrateServedFrom{
KeyspaceName: ki.KeyspaceName(),
SourceShard: *sourceShard,
DestinationShard: *destinationShard,
ServedType: servedType,
Reverse: reverse,
}
event.DispatchUpdate(ev, "start")
defer func() {
if err != nil {
event.DispatchUpdate(ev, "failed: "+err.Error())
}
}()
if servedType == topodatapb.TabletType_MASTER {
err = wr.masterMigrateServedFrom(ctx, ki, sourceShard, destinationShard, tables, ev, filteredReplicationWaitTime)
} else {
err = wr.replicaMigrateServedFrom(ctx, ki, sourceShard, destinationShard, servedType, cells, reverse, tables, ev)
}
event.DispatchUpdate(ev, "finished")
return
}
func (tee *Tee) UpdateShard(si *topo.ShardInfo) error {
if err := tee.primary.UpdateShard(si); err != nil {
// failed on primary, not updating secondary
return err
}
if err := tee.secondary.UpdateShard(si); err != nil {
// not critical enough to fail
log.Warningf("secondary.UpdateShard(%v,%v) failed: %v", si.Keyspace(), si.ShardName(), err)
}
return nil
}
func (wr *Wrangler) applySchemaShard(ctx context.Context, shardInfo *topo.ShardInfo, preflight *myproto.SchemaChangeResult, masterTabletAlias *pb.TabletAlias, change string, newParentTabletAlias *pb.TabletAlias, simple, force bool, waitSlaveTimeout time.Duration) (*myproto.SchemaChangeResult, error) {
// find all the shards we need to handle
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, shardInfo.Keyspace(), shardInfo.ShardName())
if err != nil {
return nil, err
}
// build the array of tabletStatus we're going to use
statusArray := make([]*tabletStatus, 0, len(aliases)-1)
for _, alias := range aliases {
if alias == masterTabletAlias {
// we skip the master
continue
}
ti, err := wr.ts.GetTablet(ctx, alias)
if err != nil {
return nil, err
}
statusArray = append(statusArray, &tabletStatus{ti: ti})
}
// get schema on all tablets.
log.Infof("Getting schema on all tablets for shard %v/%v", shardInfo.Keyspace(), shardInfo.ShardName())
wg := &sync.WaitGroup{}
for _, status := range statusArray {
wg.Add(1)
go func(status *tabletStatus) {
status.beforeSchema, status.lastError = wr.tmc.GetSchema(ctx, status.ti, nil, nil, false)
wg.Done()
}(status)
}
wg.Wait()
// quick check for errors
for _, status := range statusArray {
if status.lastError != nil {
return nil, fmt.Errorf("Error getting schema on tablet %v: %v", status.ti.Alias, status.lastError)
}
}
// simple or complex?
if simple {
return wr.applySchemaShardSimple(ctx, statusArray, preflight, masterTabletAlias, change, force)
}
return wr.applySchemaShardComplex(ctx, statusArray, shardInfo, preflight, masterTabletAlias, change, newParentTabletAlias, force, waitSlaveTimeout)
}
// updateShardCellsAndMaster will update the 'Cells' and possibly
// MasterAlias records for the shard, if needed.
func (wr *Wrangler) updateShardCellsAndMaster(ctx context.Context, si *topo.ShardInfo, tabletAlias *topodatapb.TabletAlias, tabletType topodatapb.TabletType, allowMasterOverride bool) error {
// See if we need to update the Shard:
// - add the tablet's cell to the shard's Cells if needed
// - change the master if needed
shardUpdateRequired := false
if !si.HasCell(tabletAlias.Cell) {
shardUpdateRequired = true
}
if tabletType == topodatapb.TabletType_MASTER && !topoproto.TabletAliasEqual(si.MasterAlias, tabletAlias) {
shardUpdateRequired = true
}
if !shardUpdateRequired {
return nil
}
// we do need to update the shard, lock it to not interfere with
// reparenting operations.
actionNode := actionnode.UpdateShard()
keyspace := si.Keyspace()
shard := si.ShardName()
lockPath, err := wr.lockShard(ctx, keyspace, shard, actionNode)
if err != nil {
return err
}
// run the update
_, err = wr.ts.UpdateShardFields(ctx, keyspace, shard, func(s *topodatapb.Shard) error {
wasUpdated := false
if !topoproto.ShardHasCell(s, tabletAlias.Cell) {
s.Cells = append(s.Cells, tabletAlias.Cell)
wasUpdated = true
}
if tabletType == topodatapb.TabletType_MASTER && !topoproto.TabletAliasEqual(s.MasterAlias, tabletAlias) {
if !topoproto.TabletAliasIsZero(s.MasterAlias) && !allowMasterOverride {
return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", topoproto.TabletAliasString(s.MasterAlias), keyspace, shard)
}
s.MasterAlias = tabletAlias
wasUpdated = true
}
if !wasUpdated {
return topo.ErrNoUpdateNeeded
}
return nil
})
return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err)
}
func (wr *Wrangler) validateReplication(shardInfo *topo.ShardInfo, tabletMap map[topo.TabletAlias]*topo.TabletInfo, results chan<- vresult) {
masterTablet, ok := tabletMap[shardInfo.MasterAlias]
if !ok {
results <- vresult{shardInfo.MasterAlias.String(), fmt.Errorf("master not in tablet map")}
return
}
slaveList, err := wr.ai.GetSlaves(masterTablet, wr.ActionTimeout())
if err != nil {
results <- vresult{shardInfo.MasterAlias.String(), err}
return
}
if len(slaveList) == 0 {
results <- vresult{shardInfo.MasterAlias.String(), fmt.Errorf("no slaves found")}
return
}
// Some addresses don't resolve in all locations, just use IP address
if err != nil {
results <- vresult{shardInfo.MasterAlias.String(), fmt.Errorf("resolve slaves failed: %v", err)}
return
}
tabletIpMap := make(map[string]*topo.Tablet)
slaveIpMap := make(map[string]bool)
for _, tablet := range tabletMap {
tabletIpMap[normalizeIP(tablet.IPAddr)] = tablet.Tablet
}
// See if every slave is in the replication graph.
for _, slaveAddr := range slaveList {
if tabletIpMap[normalizeIP(slaveAddr)] == nil {
results <- vresult{shardInfo.Keyspace() + "/" + shardInfo.ShardName(), fmt.Errorf("slave not in replication graph: %v (mysql instance without vttablet?)", slaveAddr)}
}
slaveIpMap[normalizeIP(slaveAddr)] = true
}
// See if every entry in the replication graph is connected to the master.
for _, tablet := range tabletMap {
if !tablet.IsSlaveType() {
continue
}
if !slaveIpMap[normalizeIP(tablet.IPAddr)] {
results <- vresult{tablet.Alias.String(), fmt.Errorf("slave not replicating: %v %q", tablet.IPAddr, slaveList)}
}
}
}
func (zkts *Server) UpdateShard(si *topo.ShardInfo) error {
shardPath := path.Join(globalKeyspacesPath, si.Keyspace(), "shards", si.ShardName())
_, err := zkts.zconn.Set(shardPath, jscfg.ToJson(si.Shard), -1)
if err != nil {
if zookeeper.IsError(err, zookeeper.ZNONODE) {
err = topo.ErrNoNode
}
return err
}
event.Dispatch(&events.ShardChange{
ShardInfo: *si,
Status: "updated",
})
return nil
}
// UpdateShard is part of the topo.Server interface
func (zkts *Server) UpdateShard(ctx context.Context, si *topo.ShardInfo, existingVersion int64) (int64, error) {
shardPath := path.Join(globalKeyspacesPath, si.Keyspace(), "shards", si.ShardName())
stat, err := zkts.zconn.Set(shardPath, jscfg.ToJSON(si.Shard), int(existingVersion))
if err != nil {
if zookeeper.IsError(err, zookeeper.ZNONODE) {
err = topo.ErrNoNode
}
return -1, err
}
event.Dispatch(&events.ShardChange{
ShardInfo: *si,
Status: "updated",
})
return int64(stat.Version()), nil
}
func (wr *Wrangler) validateReplication(ctx context.Context, shardInfo *topo.ShardInfo, tabletMap map[topodatapb.TabletAlias]*topo.TabletInfo, results chan<- error) {
if shardInfo.MasterAlias == nil {
results <- fmt.Errorf("no master in shard record %v/%v", shardInfo.Keyspace(), shardInfo.ShardName())
return
}
masterTabletInfo, ok := tabletMap[*shardInfo.MasterAlias]
if !ok {
results <- fmt.Errorf("master %v not in tablet map", topoproto.TabletAliasString(shardInfo.MasterAlias))
return
}
slaveList, err := wr.tmc.GetSlaves(ctx, masterTabletInfo.Tablet)
if err != nil {
results <- fmt.Errorf("GetSlaves(%v) failed: %v", masterTabletInfo, err)
return
}
if len(slaveList) == 0 {
results <- fmt.Errorf("no slaves of tablet %v found", topoproto.TabletAliasString(shardInfo.MasterAlias))
return
}
tabletIPMap := make(map[string]*topodatapb.Tablet)
slaveIPMap := make(map[string]bool)
for _, tablet := range tabletMap {
tabletIPMap[normalizeIP(tablet.Ip)] = tablet.Tablet
}
// See if every slave is in the replication graph.
for _, slaveAddr := range slaveList {
if tabletIPMap[normalizeIP(slaveAddr)] == nil {
results <- fmt.Errorf("slave %v not in replication graph for shard %v/%v (mysql instance without vttablet?)", slaveAddr, shardInfo.Keyspace(), shardInfo.ShardName())
}
slaveIPMap[normalizeIP(slaveAddr)] = true
}
// See if every entry in the replication graph is connected to the master.
for _, tablet := range tabletMap {
if !tablet.IsSlaveType() {
continue
}
if !slaveIPMap[normalizeIP(tablet.Ip)] {
results <- fmt.Errorf("slave %v not replicating: %v slave list: %q", topoproto.TabletAliasString(tablet.Alias), tablet.Ip, slaveList)
}
}
}
// Update shard file with new master, replicas, etc.
//
// Re-read from TopologyServer to make sure we are using the side
// effects of all actions.
//
// This function should only be used with an action lock on the shard
// - otherwise the consistency of the serving graph data can't be
// guaranteed.
func (wr *Wrangler) rebuildShard(keyspace, shard string, options rebuildShardOptions) error {
log.Infof("rebuildShard %v/%v", keyspace, shard)
// read the existing shard info. It has to exist.
var (
shardInfo *topo.ShardInfo
err error
)
if options.Critical {
shardInfo, err = wr.ts.GetShardCritical(keyspace, shard)
} else {
shardInfo, err = wr.ts.GetShard(keyspace, shard)
}
if err != nil {
return err
}
tabletMap, err := GetTabletMapForShardByCell(wr.ts, keyspace, shard, options.Cells)
if err != nil {
if options.IgnorePartialResult && err == topo.ErrPartialResult {
log.Warningf("rebuildShard: got topo.ErrPartialResult from GetTabletMapForShard, but skipping error as it was expected")
} else {
return err
}
}
tablets := make([]*topo.TabletInfo, 0, len(tabletMap))
for _, ti := range tabletMap {
if ti.Keyspace != shardInfo.Keyspace() || ti.Shard != shardInfo.ShardName() {
return fmt.Errorf("CRITICAL: tablet %v is in replication graph for shard %v/%v but belongs to shard %v:%v (maybe remove its replication path in shard %v/%v)", ti.Alias, keyspace, shard, ti.Keyspace, ti.Shard, keyspace, shard)
}
if !ti.IsInReplicationGraph() {
// only valid case is a scrapped master in the
// catastrophic reparent case
if ti.Parent.Uid != topo.NO_TABLET {
log.Warningf("Tablet %v should not be in the replication graph, please investigate (it will be ignored in the rebuild)", ti.Alias)
}
}
tablets = append(tablets, ti)
}
return wr.rebuildShardSrvGraph(shardInfo, tablets, options.Cells)
}
// UpdateShard implements topo.Server.
func (s *Server) UpdateShard(ctx context.Context, si *topo.ShardInfo, existingVersion int64) (int64, error) {
data := jscfg.ToJSON(si.Shard)
resp, err := s.getGlobal().CompareAndSwap(shardFilePath(si.Keyspace(), si.ShardName()),
data, 0 /* ttl */, "" /* prevValue */, uint64(existingVersion))
if err != nil {
return -1, convertError(err)
}
if resp.Node == nil {
return -1, ErrBadResponse
}
event.Dispatch(&events.ShardChange{
ShardInfo: *si,
Status: "updated",
})
return int64(resp.Node.ModifiedIndex), nil
}
// updateShardCellsAndMaster will update the 'Cells' and possibly
// MasterAlias records for the shard, if needed.
func (wr *Wrangler) updateShardCellsAndMaster(ctx context.Context, si *topo.ShardInfo, tabletAlias *topodatapb.TabletAlias, tabletType topodatapb.TabletType, allowMasterOverride bool) error {
// See if we need to update the Shard:
// - add the tablet's cell to the shard's Cells if needed
// - change the master if needed
shardUpdateRequired := false
if !si.HasCell(tabletAlias.Cell) {
shardUpdateRequired = true
}
if tabletType == topodatapb.TabletType_MASTER && !topoproto.TabletAliasEqual(si.MasterAlias, tabletAlias) {
shardUpdateRequired = true
}
if !shardUpdateRequired {
return nil
}
// run the update
_, err := wr.ts.UpdateShardFields(ctx, si.Keyspace(), si.ShardName(), func(s *topo.ShardInfo) error {
wasUpdated := false
if !s.HasCell(tabletAlias.Cell) {
s.Cells = append(s.Cells, tabletAlias.Cell)
wasUpdated = true
}
if tabletType == topodatapb.TabletType_MASTER && !topoproto.TabletAliasEqual(s.MasterAlias, tabletAlias) {
if !topoproto.TabletAliasIsZero(s.MasterAlias) && !allowMasterOverride {
return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", topoproto.TabletAliasString(s.MasterAlias), si.Keyspace(), si.ShardName())
}
s.MasterAlias = tabletAlias
wasUpdated = true
}
if !wasUpdated {
return topo.ErrNoUpdateNeeded
}
return nil
})
return err
}
// RefreshTablesByShard calls RefreshState on all the tables of a
// given type in a shard. It would work for the master, but the
// discovery wouldn't be very efficient.
func (wr *Wrangler) RefreshTablesByShard(ctx context.Context, si *topo.ShardInfo, tabletType topodatapb.TabletType, cells []string) error {
wr.Logger().Infof("RefreshTablesByShard called on shard %v/%v", si.Keyspace(), si.ShardName())
tabletMap, err := wr.ts.GetTabletMapForShardByCell(ctx, si.Keyspace(), si.ShardName(), cells)
switch err {
case nil:
// keep going
case topo.ErrPartialResult:
wr.Logger().Warningf("RefreshTablesByShard: got partial result for shard %v/%v, may not refresh all tablets everywhere", si.Keyspace(), si.ShardName())
default:
return err
}
// ignore errors in this phase
wg := sync.WaitGroup{}
for _, ti := range tabletMap {
if ti.Type != tabletType {
continue
}
wg.Add(1)
go func(ti *topo.TabletInfo) {
wr.Logger().Infof("Calling RefreshState on tablet %v", ti.AliasString())
// Setting an upper bound timeout to fail faster in case of an error.
// Using 60 seconds because RefreshState should not take more than 30 seconds.
// (RefreshState will restart the tablet's QueryService and most time will be spent on the shutdown, i.e. waiting up to 30 seconds on transactions (see Config.TransactionTimeout)).
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
if err := wr.tmc.RefreshState(ctx, ti); err != nil {
wr.Logger().Warningf("RefreshTablesByShard: failed to refresh %v: %v", ti.AliasString(), err)
}
cancel()
wg.Done()
}(ti)
}
wg.Wait()
return nil
}
func (wr *Wrangler) applySchemaShardComplex(ctx context.Context, statusArray []*tabletStatus, shardInfo *topo.ShardInfo, preflight *myproto.SchemaChangeResult, masterTabletAlias *pb.TabletAlias, change string, newParentTabletAlias *pb.TabletAlias, force bool, waitSlaveTimeout time.Duration) (*myproto.SchemaChangeResult, error) {
// apply the schema change to all replica / slave tablets
for _, status := range statusArray {
// if already applied, we skip this guy
diffs := myproto.DiffSchemaToArray("after", preflight.AfterSchema, topo.TabletAliasString(status.ti.Alias), status.beforeSchema)
if len(diffs) == 0 {
log.Infof("Tablet %v already has the AfterSchema, skipping", status.ti.Alias)
continue
}
// make sure the before schema matches
diffs = myproto.DiffSchemaToArray("master", preflight.BeforeSchema, topo.TabletAliasString(status.ti.Alias), status.beforeSchema)
if len(diffs) > 0 {
if force {
log.Warningf("Tablet %v has inconsistent schema, ignoring: %v", status.ti.Alias, strings.Join(diffs, "\n"))
} else {
return nil, fmt.Errorf("Tablet %v has inconsistent schema: %v", status.ti.Alias, strings.Join(diffs, "\n"))
}
}
// take this guy out of the serving graph if necessary
ti, err := wr.ts.GetTablet(ctx, status.ti.Alias)
if err != nil {
return nil, err
}
typeChangeRequired := ti.IsInServingGraph()
if typeChangeRequired {
// note we want to update the serving graph there
err = wr.changeTypeInternal(ctx, ti.Alias, pb.TabletType_SCHEMA_UPGRADE)
if err != nil {
return nil, err
}
}
// apply the schema change
log.Infof("Applying schema change to slave %v in complex mode", status.ti.Alias)
sc := &myproto.SchemaChange{Sql: change, Force: force, AllowReplication: false, BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}
_, err = wr.ApplySchema(ctx, status.ti.Alias, sc)
if err != nil {
return nil, err
}
// put this guy back into the serving graph
if typeChangeRequired {
err = wr.changeTypeInternal(ctx, ti.Alias, ti.Tablet.Type)
if err != nil {
return nil, err
}
}
}
// if newParentTabletAlias is passed in, use that as the new master
if !topo.TabletAliasIsZero(newParentTabletAlias) {
log.Infof("Reparenting with new master set to %v", newParentTabletAlias)
oldMasterAlias := shardInfo.MasterAlias
// Create reusable Reparent event with available info
ev := &events.Reparent{}
if err := wr.plannedReparentShardLocked(ctx, ev, shardInfo.Keyspace(), shardInfo.ShardName(), newParentTabletAlias, waitSlaveTimeout); err != nil {
return nil, err
}
// Here we would apply the schema change to the old
// master, but we just scrap it, to be consistent
// with the previous implementation of the reparent.
// (this code will be refactored at some point anyway)
if err := wr.Scrap(ctx, oldMasterAlias, false, false); err != nil {
wr.Logger().Warningf("Scrapping old master %v from shard %v/%v failed: %v", oldMasterAlias, shardInfo.Keyspace(), shardInfo.ShardName(), err)
}
}
return &myproto.SchemaChangeResult{BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}, nil
}
func (wr *Wrangler) applySchemaShardComplex(statusArray []*TabletStatus, shardInfo *topo.ShardInfo, preflight *mysqlctl.SchemaChangeResult, masterTabletAlias topo.TabletAlias, change string, newParentTabletAlias topo.TabletAlias, force bool) (*mysqlctl.SchemaChangeResult, error) {
// apply the schema change to all replica / slave tablets
for _, status := range statusArray {
// if already applied, we skip this guy
diffs := mysqlctl.DiffSchemaToArray("after", preflight.AfterSchema, status.ti.Alias().String(), status.beforeSchema)
if len(diffs) == 0 {
relog.Info("Tablet %v already has the AfterSchema, skipping", status.ti.Alias())
continue
}
// make sure the before schema matches
diffs = mysqlctl.DiffSchemaToArray("master", preflight.BeforeSchema, status.ti.Alias().String(), status.beforeSchema)
if len(diffs) > 0 {
if force {
relog.Warning("Tablet %v has inconsistent schema, ignoring: %v", status.ti.Alias(), strings.Join(diffs, "\n"))
} else {
return nil, fmt.Errorf("Tablet %v has inconsistent schema: %v", status.ti.Alias(), strings.Join(diffs, "\n"))
}
}
// take this guy out of the serving graph if necessary
ti, err := wr.ts.GetTablet(status.ti.Alias())
if err != nil {
return nil, err
}
typeChangeRequired := ti.Tablet.IsServingType()
if typeChangeRequired {
// note we want to update the serving graph there
err = wr.changeTypeInternal(ti.Alias(), topo.TYPE_SCHEMA_UPGRADE)
if err != nil {
return nil, err
}
}
// apply the schema change
relog.Info("Applying schema change to slave %v in complex mode", status.ti.Alias())
sc := &mysqlctl.SchemaChange{Sql: change, Force: force, AllowReplication: false, BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}
_, err = wr.ApplySchema(status.ti.Alias(), sc)
if err != nil {
return nil, err
}
// put this guy back into the serving graph
if typeChangeRequired {
err = wr.changeTypeInternal(ti.Alias(), ti.Tablet.Type)
if err != nil {
return nil, err
}
}
}
// if newParentTabletAlias is passed in, use that as the new master
if newParentTabletAlias != (topo.TabletAlias{}) {
relog.Info("Reparenting with new master set to %v", newParentTabletAlias)
tabletMap, err := GetTabletMapForShard(wr.ts, shardInfo.Keyspace(), shardInfo.ShardName())
if err != nil {
return nil, err
}
slaveTabletMap, foundMaster, err := slaveTabletMap(tabletMap)
if err != nil {
return nil, err
}
newMasterTablet, err := wr.ts.GetTablet(newParentTabletAlias)
if err != nil {
return nil, err
}
err = wr.reparentShardGraceful(slaveTabletMap, foundMaster, newMasterTablet /*leaveMasterReadOnly*/, false)
if err != nil {
return nil, err
}
// Here we would apply the schema change to the old
// master, but after a reparent it's in Scrap state,
// so no need to. When/if reparent leaves the
// original master in a different state (like replica
// or rdonly), then we should apply the schema there
// too.
relog.Info("Skipping schema change on old master %v in complex mode, it's been Scrapped", masterTabletAlias)
}
return &mysqlctl.SchemaChangeResult{BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}, nil
}
func (wr *Wrangler) applySchemaShard(shardInfo *topo.ShardInfo, preflight *mysqlctl.SchemaChangeResult, masterTabletAlias topo.TabletAlias, change string, newParentTabletAlias topo.TabletAlias, simple, force bool) (*mysqlctl.SchemaChangeResult, error) {
// find all the shards we need to handle
aliases, err := topo.FindAllTabletAliasesInShard(wr.ts, shardInfo.Keyspace(), shardInfo.ShardName())
if err != nil {
return nil, err
}
// build the array of TabletStatus we're going to use
statusArray := make([]*TabletStatus, 0, len(aliases)-1)
for _, alias := range aliases {
if alias == masterTabletAlias {
// we skip the master
continue
}
ti, err := wr.ts.GetTablet(alias)
if err != nil {
return nil, err
}
if ti.Type == topo.TYPE_LAG {
// lag tablets are usually behind, not replicating,
// and a general pain. So let's just skip them
// all together.
// TODO(alainjobart) figure out other types to skip:
// ValidateSchemaShard only does the serving types.
// We do everything in the replication graph
// but LAG. This seems fine for now.
relog.Info("Skipping tablet %v as it is LAG", ti.Alias())
continue
}
statusArray = append(statusArray, &TabletStatus{ti: ti})
}
// get schema on all tablets.
relog.Info("Getting schema on all tablets for shard %v/%v", shardInfo.Keyspace(), shardInfo.ShardName())
wg := &sync.WaitGroup{}
for _, status := range statusArray {
wg.Add(1)
go func(status *TabletStatus) {
status.beforeSchema, status.lastError = wr.GetSchemaTablet(status.ti, nil, false)
wg.Done()
}(status)
}
wg.Wait()
// quick check for errors
for _, status := range statusArray {
if status.lastError != nil {
return nil, fmt.Errorf("Error getting schema on tablet %v: %v", status.ti.Alias(), status.lastError)
}
}
// simple or complex?
if simple {
return wr.applySchemaShardSimple(statusArray, preflight, masterTabletAlias, change, force)
}
return wr.applySchemaShardComplex(statusArray, shardInfo, preflight, masterTabletAlias, change, newParentTabletAlias, force)
}
// rebuildCellSrvShard computes and writes the serving graph data to a
// single cell
func rebuildCellSrvShard(ctx context.Context, log logutil.Logger, ts topo.Server, si *topo.ShardInfo, cell string) (err error) {
log.Infof("rebuildCellSrvShard %v/%v in cell %v", si.Keyspace(), si.ShardName(), cell)
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
// Read existing EndPoints node versions, so we know if any
// changes sneak in after we read the tablets.
versions, err := getEndPointsVersions(ctx, ts, cell, si.Keyspace(), si.ShardName())
// Get all tablets in this cell/shard.
tablets, err := ts.GetTabletMapForShardByCell(ctx, si.Keyspace(), si.ShardName(), []string{cell})
if err != nil {
if err != topo.ErrPartialResult {
return err
}
log.Warningf("Got ErrPartialResult from topo.GetTabletMapForShardByCell(%v), some tablets may not be added properly to serving graph", cell)
}
// Build up the serving graph from scratch.
serving := make(map[pb.TabletType]*pb.EndPoints)
for _, tablet := range tablets {
// Only add serving types.
if !tablet.IsInServingGraph() {
continue
}
// Check the Keyspace and Shard for the tablet are right.
if tablet.Keyspace != si.Keyspace() || tablet.Shard != si.ShardName() {
return fmt.Errorf("CRITICAL: tablet %v is in replication graph for shard %v/%v but belongs to shard %v:%v", tablet.Alias, si.Keyspace(), si.ShardName(), tablet.Keyspace, tablet.Shard)
}
// Add the tablet to the list.
endpoints, ok := serving[tablet.Type]
if !ok {
endpoints = topo.NewEndPoints()
serving[tablet.Type] = endpoints
}
entry, err := topo.TabletEndPoint(tablet.Tablet)
if err != nil {
log.Warningf("EndPointForTablet failed for tablet %v: %v", tablet.Alias, err)
continue
}
endpoints.Entries = append(endpoints.Entries, entry)
}
wg := sync.WaitGroup{}
fatalErrs := concurrency.AllErrorRecorder{}
retryErrs := concurrency.AllErrorRecorder{}
// Write nodes that should exist.
for tabletType, endpoints := range serving {
wg.Add(1)
go func(tabletType pb.TabletType, endpoints *pb.EndPoints) {
defer wg.Done()
log.Infof("saving serving graph for cell %v shard %v/%v tabletType %v", cell, si.Keyspace(), si.ShardName(), tabletType)
version, ok := versions[tabletType]
if !ok {
// This type didn't exist when we first checked.
// Try to create, but only if it still doesn't exist.
if err := ts.CreateEndPoints(ctx, cell, si.Keyspace(), si.ShardName(), tabletType, endpoints); err != nil {
log.Warningf("CreateEndPoints(%v, %v, %v) failed during rebuild: %v", cell, si, tabletType, err)
switch err {
case topo.ErrNodeExists:
retryErrs.RecordError(err)
default:
fatalErrs.RecordError(err)
}
}
return
}
// Update only if the version matches.
if err := ts.UpdateEndPoints(ctx, cell, si.Keyspace(), si.ShardName(), tabletType, endpoints, version); err != nil {
log.Warningf("UpdateEndPoints(%v, %v, %v) failed during rebuild: %v", cell, si, tabletType, err)
switch err {
case topo.ErrBadVersion, topo.ErrNoNode:
retryErrs.RecordError(err)
default:
fatalErrs.RecordError(err)
}
}
}(tabletType, endpoints)
}
// Delete nodes that shouldn't exist.
for tabletType, version := range versions {
if _, ok := serving[tabletType]; !ok {
wg.Add(1)
go func(tabletType pb.TabletType, version int64) {
defer wg.Done()
log.Infof("removing stale db type from serving graph: %v", tabletType)
if err := ts.DeleteEndPoints(ctx, cell, si.Keyspace(), si.ShardName(), tabletType, version); err != nil && err != topo.ErrNoNode {
log.Warningf("DeleteEndPoints(%v, %v, %v) failed during rebuild: %v", cell, si, tabletType, err)
switch err {
case topo.ErrNoNode:
// Someone else deleted it, which is fine.
case topo.ErrBadVersion:
retryErrs.RecordError(err)
default:
fatalErrs.RecordError(err)
}
}
}(tabletType, version)
}
}
// Update srvShard object
wg.Add(1)
go func() {
defer wg.Done()
log.Infof("updating shard serving graph in cell %v for %v/%v", cell, si.Keyspace(), si.ShardName())
if err := UpdateSrvShard(ctx, ts, cell, si); err != nil {
fatalErrs.RecordError(err)
log.Warningf("writing serving data in cell %v for %v/%v failed: %v", cell, si.Keyspace(), si.ShardName(), err)
}
}()
wg.Wait()
// If there are any fatal errors, give up.
if fatalErrs.HasErrors() {
return fatalErrs.Error()
}
// If there are any retry errors, try again.
if retryErrs.HasErrors() {
continue
}
// Otherwise, success!
return nil
}
}
func (wr *Wrangler) migrateServedFrom(ki *topo.KeyspaceInfo, si *topo.ShardInfo, servedType topo.TabletType, reverse bool) (err error) {
// re-read and update keyspace info record
ki, err = wr.ts.GetKeyspace(ki.KeyspaceName())
if err != nil {
return err
}
if reverse {
if _, ok := ki.ServedFrom[servedType]; ok {
return fmt.Errorf("Destination Keyspace %s is not serving type %v", ki.KeyspaceName(), servedType)
}
ki.ServedFrom[servedType] = si.SourceShards[0].Keyspace
} else {
if _, ok := ki.ServedFrom[servedType]; !ok {
return fmt.Errorf("Destination Keyspace %s is already serving type %v", ki.KeyspaceName(), servedType)
}
delete(ki.ServedFrom, servedType)
}
// re-read and check the destination shard
si, err = wr.ts.GetShard(si.Keyspace(), si.ShardName())
if err != nil {
return err
}
if len(si.SourceShards) != 1 {
return fmt.Errorf("Destination shard %v/%v is not a vertical split target", si.Keyspace(), si.ShardName())
}
tables := si.SourceShards[0].Tables
// read the source shard, we'll need its master
sourceShard, err := wr.ts.GetShard(si.SourceShards[0].Keyspace, si.SourceShards[0].Shard)
if err != nil {
return err
}
ev := &events.MigrateServedFrom{
Keyspace: *ki,
SourceShard: *sourceShard,
DestinationShard: *si,
ServedType: servedType,
Reverse: reverse,
}
event.DispatchUpdate(ev, "start")
defer func() {
if err != nil {
event.DispatchUpdate(ev, "failed: "+err.Error())
}
}()
// For master type migration, need to:
// - switch the source shard to read-only
// - gather the replication point
// - wait for filtered replication to catch up before we continue
// - disable filtered replication after the fact
var sourceMasterTabletInfo *topo.TabletInfo
if servedType == topo.TYPE_MASTER {
// set master to read-only
event.DispatchUpdate(ev, "setting source shard master to read-only")
actionPath, err := wr.ai.SetReadOnly(sourceShard.MasterAlias)
if err != nil {
return err
}
if err := wr.WaitForCompletion(actionPath); err != nil {
return err
}
// get the position
event.DispatchUpdate(ev, "getting master position")
sourceMasterTabletInfo, err = wr.ts.GetTablet(sourceShard.MasterAlias)
if err != nil {
return err
}
masterPosition, err := wr.ai.MasterPosition(sourceMasterTabletInfo, wr.ActionTimeout())
if err != nil {
return err
}
// wait for it
event.DispatchUpdate(ev, "waiting for destination master to catch up to source master")
if err := wr.ai.WaitBlpPosition(si.MasterAlias, blproto.BlpPosition{
Uid: 0,
Position: masterPosition,
}, wr.ActionTimeout()); err != nil {
return err
}
// and clear the shard record
si.SourceShards = nil
}
// All is good, we can save the keyspace and shard (if needed) now
event.DispatchUpdate(ev, "updating keyspace")
if err = topo.UpdateKeyspace(wr.ts, ki); err != nil {
return err
}
event.DispatchUpdate(ev, "updating destination shard")
if servedType == topo.TYPE_MASTER {
if err := topo.UpdateShard(wr.ts, si); err != nil {
return err
}
}
// Tell the new shards masters they can now be read-write.
// Invoking a remote action will also make the tablet stop filtered
// replication.
event.DispatchUpdate(ev, "setting destination shard masters read-write")
if servedType == topo.TYPE_MASTER {
if err := wr.makeMastersReadWrite([]*topo.ShardInfo{si}); err != nil {
return err
}
}
// Now blacklist the table list on the right servers
event.DispatchUpdate(ev, "setting blacklisted tables on source shard")
if servedType == topo.TYPE_MASTER {
if err := wr.ai.SetBlacklistedTables(sourceMasterTabletInfo, tables, wr.ActionTimeout()); err != nil {
return err
}
} else {
// We use the list of tables that are replicating
// for the blacklist. In case of a reverse move, we clear the
// blacklist.
if reverse {
tables = nil
}
if err := wr.SetBlacklistedTablesByShard(sourceShard.Keyspace(), sourceShard.ShardName(), servedType, tables); err != nil {
return err
}
}
event.DispatchUpdate(ev, "finished")
return nil
}
// Write serving graph data to the cells
func (wr *Wrangler) rebuildShardSrvGraph(shardInfo *topo.ShardInfo, tablets []*topo.TabletInfo, cells []string) error {
log.Infof("rebuildShardSrvGraph %v/%v", shardInfo.Keyspace(), shardInfo.ShardName())
// Get all existing db types so they can be removed if nothing
// had been editted. This applies to all cells, which can't
// be determined until you walk through all the tablets.
//
// existingDbTypeLocations is a map:
// key: {cell,keyspace,shard,tabletType}
// value: true
existingDbTypeLocations := make(map[cellKeyspaceShardType]bool)
// Update db type addresses in the serving graph
//
// locationAddrsMap is a map:
// key: {cell,keyspace,shard,tabletType}
// value: topo.EndPoints (list of server records)
locationAddrsMap := make(map[cellKeyspaceShardType]*topo.EndPoints)
// we keep track of the existingDbTypeLocations we've already looked at
knownShardLocations := make(map[cellKeyspaceShard]bool)
for _, tablet := range tablets {
// only look at tablets in the cells we want to rebuild
if !topo.InCellList(tablet.Tablet.Alias.Cell, cells) {
continue
}
// this is {cell,keyspace,shard}
// we'll get the children to find the existing types
shardLocation := cellKeyspaceShard{tablet.Tablet.Alias.Cell, tablet.Tablet.Keyspace, tablet.Shard}
// only need to do this once per cell
if !knownShardLocations[shardLocation] {
log.Infof("Getting tablet types on cell %v for %v/%v", tablet.Tablet.Alias.Cell, tablet.Tablet.Keyspace, tablet.Shard)
tabletTypes, err := wr.ts.GetSrvTabletTypesPerShard(tablet.Tablet.Alias.Cell, tablet.Tablet.Keyspace, tablet.Shard)
if err != nil {
if err != topo.ErrNoNode {
return err
}
} else {
for _, tabletType := range tabletTypes {
existingDbTypeLocations[cellKeyspaceShardType{tablet.Tablet.Alias.Cell, tablet.Tablet.Keyspace, tablet.Shard, tabletType}] = true
}
}
knownShardLocations[shardLocation] = true
}
// Check IsInServingGraph after we have populated
// existingDbTypeLocations so we properly prune data
// if the definition of serving type changes.
if !tablet.IsInServingGraph() {
continue
}
location := cellKeyspaceShardType{tablet.Tablet.Alias.Cell, tablet.Keyspace, tablet.Shard, tablet.Type}
addrs, ok := locationAddrsMap[location]
if !ok {
addrs = topo.NewEndPoints()
locationAddrsMap[location] = addrs
}
entry, err := tabletmanager.EndPointForTablet(tablet.Tablet)
if err != nil {
log.Warningf("EndPointForTablet failed for tablet %v: %v", tablet.Alias, err)
continue
}
addrs.Entries = append(addrs.Entries, *entry)
}
// we're gonna parallelize a lot here
rec := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
// write all the {cell,keyspace,shard,type}
// nodes everywhere we want them
for location, addrs := range locationAddrsMap {
wg.Add(1)
go func(location cellKeyspaceShardType, addrs *topo.EndPoints) {
log.Infof("saving serving graph for cell %v shard %v/%v tabletType %v", location.cell, location.keyspace, location.shard, location.tabletType)
if err := wr.ts.UpdateEndPoints(location.cell, location.keyspace, location.shard, location.tabletType, addrs); err != nil {
rec.RecordError(fmt.Errorf("writing endpoints for cell %v shard %v/%v tabletType %v failed: %v", location.cell, location.keyspace, location.shard, location.tabletType, err))
}
wg.Done()
}(location, addrs)
}
// Delete any pre-existing paths that were not updated by this process.
// That's the existingDbTypeLocations - locationAddrsMap
for dbTypeLocation := range existingDbTypeLocations {
if _, ok := locationAddrsMap[dbTypeLocation]; !ok {
cell := dbTypeLocation.cell
if !topo.InCellList(cell, cells) {
continue
}
wg.Add(1)
go func(dbTypeLocation cellKeyspaceShardType) {
log.Infof("removing stale db type from serving graph: %v", dbTypeLocation)
if err := wr.ts.DeleteSrvTabletType(dbTypeLocation.cell, dbTypeLocation.keyspace, dbTypeLocation.shard, dbTypeLocation.tabletType); err != nil {
log.Warningf("unable to remove stale db type %v from serving graph: %v", dbTypeLocation, err)
}
wg.Done()
}(dbTypeLocation)
}
}
// wait until we're done with the background stuff to do the rest
// FIXME(alainjobart) this wouldn't be necessary if UpdateSrvShard
// below was creating the zookeeper nodes recursively.
wg.Wait()
if err := rec.Error(); err != nil {
return err
}
// Update per-shard information per cell-specific serving path.
//
// srvShardByPath is a map:
// key: {cell,keyspace,shard}
// value: topo.SrvShard
// this will create all the SrvShard objects
srvShardByPath := make(map[cellKeyspaceShard]*topo.SrvShard)
for location := range locationAddrsMap {
// location will be {cell,keyspace,shard,type}
srvShardPath := cellKeyspaceShard{location.cell, location.keyspace, location.shard}
srvShard, ok := srvShardByPath[srvShardPath]
if !ok {
srvShard = &topo.SrvShard{
KeyRange: shardInfo.KeyRange,
ServedTypes: shardInfo.ServedTypes,
TabletTypes: make([]topo.TabletType, 0, 2),
}
srvShardByPath[srvShardPath] = srvShard
}
foundType := false
for _, t := range srvShard.TabletTypes {
if t == location.tabletType {
foundType = true
}
}
if !foundType {
srvShard.TabletTypes = append(srvShard.TabletTypes, location.tabletType)
}
}
// Save the shard entries
for cks, srvShard := range srvShardByPath {
wg.Add(1)
go func(cks cellKeyspaceShard, srvShard *topo.SrvShard) {
log.Infof("updating shard serving graph in cell %v for %v/%v", cks.cell, cks.keyspace, cks.shard)
if err := wr.ts.UpdateSrvShard(cks.cell, cks.keyspace, cks.shard, srvShard); err != nil {
rec.RecordError(fmt.Errorf("writing serving data in cell %v for %v/%v failed: %v", cks.cell, cks.keyspace, cks.shard, err))
}
wg.Done()
}(cks, srvShard)
}
wg.Wait()
return rec.Error()
}
func (wr *Wrangler) applySchemaShard(ctx context.Context, shardInfo *topo.ShardInfo, preflight *tmutils.SchemaChangeResult, masterTabletAlias *topodatapb.TabletAlias, change string, newParentTabletAlias *topodatapb.TabletAlias, force bool, waitSlaveTimeout time.Duration) (*tmutils.SchemaChangeResult, error) {
// find all the shards we need to handle
aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, shardInfo.Keyspace(), shardInfo.ShardName())
if err != nil {
return nil, err
}
// build the array of tabletStatus we're going to use
statusArray := make([]*tabletStatus, 0, len(aliases)-1)
for _, alias := range aliases {
if alias == masterTabletAlias {
// we skip the master
continue
}
ti, err := wr.ts.GetTablet(ctx, alias)
if err != nil {
return nil, err
}
statusArray = append(statusArray, &tabletStatus{ti: ti})
}
// get schema on all tablets.
log.Infof("Getting schema on all tablets for shard %v/%v", shardInfo.Keyspace(), shardInfo.ShardName())
wg := &sync.WaitGroup{}
for _, status := range statusArray {
wg.Add(1)
go func(status *tabletStatus) {
status.beforeSchema, status.lastError = wr.tmc.GetSchema(ctx, status.ti, nil, nil, false)
wg.Done()
}(status)
}
wg.Wait()
// quick check for errors
for _, status := range statusArray {
if status.lastError != nil {
return nil, fmt.Errorf("Error getting schema on tablet %v: %v", status.ti.AliasString(), status.lastError)
}
}
// check all tablets have the same schema as the master's
// BeforeSchema. If not, we shouldn't proceed
log.Infof("Checking schema on all tablets")
for _, status := range statusArray {
diffs := tmutils.DiffSchemaToArray("master", preflight.BeforeSchema, topoproto.TabletAliasString(status.ti.Alias), status.beforeSchema)
if len(diffs) > 0 {
if force {
log.Warningf("Tablet %v has inconsistent schema, ignoring: %v", status.ti.AliasString(), strings.Join(diffs, "\n"))
} else {
return nil, fmt.Errorf("Tablet %v has inconsistent schema: %v", status.ti.AliasString(), strings.Join(diffs, "\n"))
}
}
}
// we're good, just send to the master
log.Infof("Applying schema change to master")
sc := &tmutils.SchemaChange{
SQL: change,
Force: force,
AllowReplication: true,
BeforeSchema: preflight.BeforeSchema,
AfterSchema: preflight.AfterSchema,
}
return wr.ApplySchema(ctx, masterTabletAlias, sc)
}
// UpdateShard is part of the topo.Server interface
func (tee *Tee) UpdateShard(ctx context.Context, si *topo.ShardInfo, existingVersion int64) (newVersion int64, err error) {
if newVersion, err = tee.primary.UpdateShard(ctx, si, existingVersion); err != nil {
// failed on primary, not updating secondary
return
}
// if we have a mapping between shard version in first topo
// and shard version in second topo, replace the version number.
// if not, this will probably fail and log.
tee.mu.Lock()
svm, ok := tee.shardVersionMapping[si.Keyspace()+"/"+si.ShardName()]
if ok && svm.readFromVersion == existingVersion {
existingVersion = svm.readFromSecondVersion
delete(tee.shardVersionMapping, si.Keyspace()+"/"+si.ShardName())
}
tee.mu.Unlock()
if newVersion2, serr := tee.secondary.UpdateShard(ctx, si, existingVersion); serr != nil {
// not critical enough to fail
if serr == topo.ErrNoNode {
// the shard doesn't exist on the secondary, let's
// just create it
if serr = tee.secondary.CreateShard(ctx, si.Keyspace(), si.ShardName(), si.Shard); serr != nil {
log.Warningf("secondary.CreateShard(%v,%v) failed (after UpdateShard returned ErrNoNode): %v", si.Keyspace(), si.ShardName(), serr)
} else {
log.Infof("secondary.UpdateShard(%v, %v) failed with ErrNoNode, CreateShard then worked.", si.Keyspace(), si.ShardName())
si, gerr := tee.secondary.GetShard(ctx, si.Keyspace(), si.ShardName())
if gerr != nil {
log.Warningf("Failed to re-read shard(%v, %v) after creating it on secondary: %v", si.Keyspace(), si.ShardName(), gerr)
} else {
tee.mu.Lock()
tee.shardVersionMapping[si.Keyspace()+"/"+si.ShardName()] = versionMapping{
readFromVersion: newVersion,
readFromSecondVersion: si.Version(),
}
tee.mu.Unlock()
}
}
} else {
log.Warningf("secondary.UpdateShard(%v, %v) failed: %v", si.Keyspace(), si.ShardName(), serr)
}
} else {
tee.mu.Lock()
tee.shardVersionMapping[si.Keyspace()+"/"+si.ShardName()] = versionMapping{
readFromVersion: newVersion,
readFromSecondVersion: newVersion2,
}
tee.mu.Unlock()
}
return
}