// UpdateShardReplicationRecord is a low level function to add / update an // entry to the ShardReplication object. func UpdateShardReplicationRecord(ctx context.Context, ts Server, keyspace, shard string, tabletAlias, parent TabletAlias) error { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UpdateShardReplicationFields") span.Annotate("keyspace", keyspace) span.Annotate("shard", shard) span.Annotate("tablet", tabletAlias.String()) defer span.Finish() return ts.UpdateShardReplicationFields(tabletAlias.Cell, keyspace, shard, func(sr *ShardReplication) error { // not very efficient, but easy to read links := make([]ReplicationLink, 0, len(sr.ReplicationLinks)+1) found := false for _, link := range sr.ReplicationLinks { if link.TabletAlias == tabletAlias { if found { log.Warningf("Found a second ReplicationLink for tablet %v, deleting it", tabletAlias) continue } found = true // update the master link.Parent = parent } links = append(links, link) } if !found { links = append(links, ReplicationLink{TabletAlias: tabletAlias, Parent: parent}) } sr.ReplicationLinks = links return nil }) }
// UnlockShard unlocks a previously locked shard. func (n *ActionNode) UnlockShard(ctx context.Context, ts topo.Server, keyspace, shard string, lockPath string, actionError error) error { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UnlockShardForAction") span.Annotate("action", n.Action) span.Annotate("keyspace", keyspace) span.Annotate("shard", shard) defer span.Finish() // first update the actionNode if actionError != nil { log.Infof("Unlocking shard %v/%v for action %v with error %v", keyspace, shard, n.Action, actionError) n.Error = actionError.Error() n.State = ACTION_STATE_FAILED } else { log.Infof("Unlocking shard %v/%v for successful action %v", keyspace, shard, n.Action) n.Error = "" n.State = ACTION_STATE_DONE } err := ts.UnlockShardForAction(keyspace, shard, lockPath, n.ToJson()) if actionError != nil { if err != nil { // this will be masked log.Warningf("UnlockShardForAction failed: %v", err) } return actionError } return err }
// GetTabletMap tries to read all the tablets in the provided list, // and returns them all in a map. // If error is ErrPartialResult, the results in the dictionary are // incomplete, meaning some tablets couldn't be read. func GetTabletMap(ctx context.Context, ts Server, tabletAliases []TabletAlias) (map[TabletAlias]*TabletInfo, error) { span := trace.NewSpanFromContext(ctx) span.StartLocal("topo.GetTabletMap") span.Annotate("num_tablets", len(tabletAliases)) defer span.Finish() wg := sync.WaitGroup{} mutex := sync.Mutex{} tabletMap := make(map[TabletAlias]*TabletInfo) var someError error for _, tabletAlias := range tabletAliases { wg.Add(1) go func(tabletAlias TabletAlias) { defer wg.Done() tabletInfo, err := ts.GetTablet(tabletAlias) mutex.Lock() if err != nil { log.Warningf("%v: %v", tabletAlias, err) // There can be data races removing nodes - ignore them for now. if err != ErrNoNode { someError = ErrPartialResult } } else { tabletMap[tabletAlias] = tabletInfo } mutex.Unlock() }(tabletAlias) } wg.Wait() return tabletMap, someError }
// refreshTablet needs to be run after an action may have changed the current // state of the tablet. func (agent *ActionAgent) refreshTablet(ctx context.Context, reason string) error { log.Infof("Executing post-action state refresh") span := trace.NewSpanFromContext(ctx) span.StartLocal("ActionAgent.refreshTablet") span.Annotate("reason", reason) defer span.Finish() ctx = trace.NewContext(ctx, span) // Save the old tablet so callbacks can have a better idea of // the precise nature of the transition. oldTablet := agent.Tablet().Tablet // Actions should have side effects on the tablet, so reload the data. if err := agent.readTablet(); err != nil { log.Warningf("Failed rereading tablet after %v - services may be inconsistent: %v", reason, err) return fmt.Errorf("Failed rereading tablet after %v: %v", reason, err) } if updatedTablet := agent.checkTabletMysqlPort(agent.Tablet()); updatedTablet != nil { agent.mutex.Lock() agent._tablet = updatedTablet agent.mutex.Unlock() } if err := agent.updateState(ctx, oldTablet, reason); err != nil { return err } log.Infof("Done with post-action state refresh") return nil }
// LockKeyspace will lock the keyspace in the topology server. // UnlockKeyspace should be called if this returns no error. func (n *ActionNode) LockKeyspace(ctx context.Context, ts topo.Server, keyspace string, lockTimeout time.Duration, interrupted chan struct{}) (lockPath string, err error) { log.Infof("Locking keyspace %v for action %v", keyspace, n.Action) span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.LockKeyspaceForAction") span.Annotate("action", n.Action) span.Annotate("keyspace", keyspace) defer span.Finish() return ts.LockKeyspaceForAction(keyspace, n.ToJson(), lockTimeout, interrupted) }
// LockSrvShard will lock the serving shard in the topology server. // UnlockSrvShard should be called if this returns no error. func (n *ActionNode) LockSrvShard(ctx context.Context, ts topo.Server, cell, keyspace, shard string, lockTimeout time.Duration, interrupted chan struct{}) (lockPath string, err error) { log.Infof("Locking serving shard %v/%v/%v for action %v", cell, keyspace, shard, n.Action) span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.LockSrvShardForAction") span.Annotate("action", n.Action) span.Annotate("keyspace", keyspace) span.Annotate("shard", shard) span.Annotate("cell", cell) defer span.Finish() return ts.LockSrvShardForAction(cell, keyspace, shard, n.ToJson(), lockTimeout, interrupted) }
// UpdateTablet updates the tablet data only - not associated replication paths. func UpdateTablet(ctx context.Context, ts Server, tablet *TabletInfo) error { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UpdateTablet") span.Annotate("tablet", tablet.Alias.String()) defer span.Finish() var version int64 = -1 if tablet.version != 0 { version = tablet.version } newVersion, err := ts.UpdateTablet(tablet, version) if err == nil { tablet.version = newVersion } return err }
// UpdateShard updates the shard data, with the right version func UpdateShard(ctx context.Context, ts Server, si *ShardInfo) error { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UpdateShard") span.Annotate("keyspace", si.Keyspace()) span.Annotate("shard", si.ShardName()) defer span.Finish() var version int64 = -1 if si.version != 0 { version = si.version } newVersion, err := ts.UpdateShard(si, version) if err == nil { si.version = newVersion } return err }
// Go invokes the function asynchronously. It returns the Call structure representing // the invocation. The done channel will signal when the call is complete by returning // the same Call object. If done is nil, Go will allocate a new channel. // If non-nil, done must be buffered or Go will deliberately crash. func (client *Client) Go(ctx context.Context, serviceMethod string, args interface{}, reply interface{}, done chan *Call) *Call { span := trace.NewSpanFromContext(ctx) span.StartClient(serviceMethod) defer span.Finish() call := new(Call) call.ServiceMethod = serviceMethod call.Args = args call.Reply = reply if done == nil { done = make(chan *Call, 10) // buffered. } else { // If caller passes done != nil, it must arrange that // done has enough buffer for the number of simultaneous // RPCs that will be using that channel. If the channel // is totally unbuffered, it's best not to run at all. if cap(done) == 0 { log.Panic("rpc: done channel is unbuffered") } } call.Done = done client.send(call) return call }
// Update shard file with new master, replicas, etc. // // Re-read from TopologyServer to make sure we are using the side // effects of all actions. // // This function locks individual SvrShard paths, so it doesn't need a lock // on the shard. func RebuildShard(ctx context.Context, log logutil.Logger, ts topo.Server, keyspace, shard string, cells []string, timeout time.Duration, interrupted chan struct{}) (*topo.ShardInfo, error) { log.Infof("RebuildShard %v/%v", keyspace, shard) span := trace.NewSpanFromContext(ctx) span.StartLocal("topotools.RebuildShard") defer span.Finish() ctx = trace.NewContext(ctx, span) // read the existing shard info. It has to exist. shardInfo, err := ts.GetShard(keyspace, shard) if err != nil { return nil, err } // rebuild all cells in parallel wg := sync.WaitGroup{} rec := concurrency.AllErrorRecorder{} for _, cell := range shardInfo.Cells { // skip this cell if we shouldn't rebuild it if !topo.InCellList(cell, cells) { continue } // start with the master if it's in the current cell tabletsAsMap := make(map[topo.TabletAlias]bool) if shardInfo.MasterAlias.Cell == cell { tabletsAsMap[shardInfo.MasterAlias] = true } wg.Add(1) go func(cell string) { defer wg.Done() // Lock the SrvShard so we don't race with other rebuilds of the same // shard in the same cell (e.g. from our peer tablets). actionNode := actionnode.RebuildSrvShard() lockPath, err := actionNode.LockSrvShard(ctx, ts, cell, keyspace, shard, timeout, interrupted) if err != nil { rec.RecordError(err) return } // read the ShardReplication object to find tablets sri, err := ts.GetShardReplication(cell, keyspace, shard) if err != nil { rec.RecordError(fmt.Errorf("GetShardReplication(%v, %v, %v) failed: %v", cell, keyspace, shard, err)) return } // add all relevant tablets to the map for _, rl := range sri.ReplicationLinks { tabletsAsMap[rl.TabletAlias] = true if rl.Parent.Cell == cell { tabletsAsMap[rl.Parent] = true } } // convert the map to a list aliases := make([]topo.TabletAlias, 0, len(tabletsAsMap)) for a := range tabletsAsMap { aliases = append(aliases, a) } // read all the Tablet records tablets, err := topo.GetTabletMap(ctx, ts, aliases) switch err { case nil: // keep going, we're good case topo.ErrPartialResult: log.Warningf("Got ErrPartialResult from topo.GetTabletMap in cell %v, some tablets may not be added properly to serving graph", cell) default: rec.RecordError(fmt.Errorf("GetTabletMap in cell %v failed: %v", cell, err)) return } // write the data we need to rebuildErr := rebuildCellSrvShard(ctx, log, ts, shardInfo, cell, tablets) // and unlock if err := actionNode.UnlockSrvShard(ctx, ts, cell, keyspace, shard, lockPath, rebuildErr); err != nil { rec.RecordError(err) } }(cell) } wg.Wait() return shardInfo, rec.Error() }
// rebuildCellSrvShard computes and writes the serving graph data to a // single cell func rebuildCellSrvShard(ctx context.Context, log logutil.Logger, ts topo.Server, shardInfo *topo.ShardInfo, cell string, tablets map[topo.TabletAlias]*topo.TabletInfo) error { log.Infof("rebuildCellSrvShard %v/%v in cell %v", shardInfo.Keyspace(), shardInfo.ShardName(), cell) // Get all existing db types so they can be removed if nothing // had been edited. existingTabletTypes, err := ts.GetSrvTabletTypesPerShard(cell, shardInfo.Keyspace(), shardInfo.ShardName()) if err != nil { if err != topo.ErrNoNode { return err } } // Update db type addresses in the serving graph // // locationAddrsMap is a map: // key: tabletType // value: EndPoints (list of server records) locationAddrsMap := make(map[topo.TabletType]*topo.EndPoints) for _, tablet := range tablets { if !tablet.IsInReplicationGraph() { // only valid case is a scrapped master in the // catastrophic reparent case if tablet.Parent.Uid != topo.NO_TABLET { log.Warningf("Tablet %v should not be in the replication graph, please investigate (it is being ignored in the rebuild)", tablet.Alias) } continue } // Check IsInServingGraph, we don't want to add tablets that // are not serving if !tablet.IsInServingGraph() { continue } // Check the Keyspace and Shard for the tablet are right if tablet.Keyspace != shardInfo.Keyspace() || tablet.Shard != shardInfo.ShardName() { return fmt.Errorf("CRITICAL: tablet %v is in replication graph for shard %v/%v but belongs to shard %v:%v", tablet.Alias, shardInfo.Keyspace(), shardInfo.ShardName(), tablet.Keyspace, tablet.Shard) } // Add the tablet to the list addrs, ok := locationAddrsMap[tablet.Type] if !ok { addrs = topo.NewEndPoints() locationAddrsMap[tablet.Type] = addrs } entry, err := tablet.Tablet.EndPoint() if err != nil { log.Warningf("EndPointForTablet failed for tablet %v: %v", tablet.Alias, err) continue } addrs.Entries = append(addrs.Entries, *entry) } // we're gonna parallelize a lot here: // - writing all the tabletTypes records // - removing the unused records // - writing SrvShard rec := concurrency.AllErrorRecorder{} wg := sync.WaitGroup{} // write all the EndPoints nodes everywhere we want them for tabletType, addrs := range locationAddrsMap { wg.Add(1) go func(tabletType topo.TabletType, addrs *topo.EndPoints) { log.Infof("saving serving graph for cell %v shard %v/%v tabletType %v", cell, shardInfo.Keyspace(), shardInfo.ShardName(), tabletType) span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UpdateEndPoints") span.Annotate("tablet_type", string(tabletType)) if err := ts.UpdateEndPoints(cell, shardInfo.Keyspace(), shardInfo.ShardName(), tabletType, addrs); err != nil { rec.RecordError(fmt.Errorf("writing endpoints for cell %v shard %v/%v tabletType %v failed: %v", cell, shardInfo.Keyspace(), shardInfo.ShardName(), tabletType, err)) } span.Finish() wg.Done() }(tabletType, addrs) } // Delete any pre-existing paths that were not updated by this process. // That's the existingTabletTypes - locationAddrsMap for _, tabletType := range existingTabletTypes { if _, ok := locationAddrsMap[tabletType]; !ok { wg.Add(1) go func(tabletType topo.TabletType) { log.Infof("removing stale db type from serving graph: %v", tabletType) span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.DeleteEndPoints") span.Annotate("tablet_type", string(tabletType)) if err := ts.DeleteEndPoints(cell, shardInfo.Keyspace(), shardInfo.ShardName(), tabletType); err != nil { log.Warningf("unable to remove stale db type %v from serving graph: %v", tabletType, err) } span.Finish() wg.Done() }(tabletType) } } // Update srvShard object wg.Add(1) go func() { log.Infof("updating shard serving graph in cell %v for %v/%v", cell, shardInfo.Keyspace(), shardInfo.ShardName()) srvShard := &topo.SrvShard{ Name: shardInfo.ShardName(), KeyRange: shardInfo.KeyRange, ServedTypes: shardInfo.GetServedTypesPerCell(cell), MasterCell: shardInfo.MasterAlias.Cell, TabletTypes: make([]topo.TabletType, 0, len(locationAddrsMap)), } for tabletType := range locationAddrsMap { srvShard.TabletTypes = append(srvShard.TabletTypes, tabletType) } span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UpdateSrvShard") span.Annotate("keyspace", shardInfo.Keyspace()) span.Annotate("shard", shardInfo.ShardName()) span.Annotate("cell", cell) if err := ts.UpdateSrvShard(cell, shardInfo.Keyspace(), shardInfo.ShardName(), srvShard); err != nil { rec.RecordError(fmt.Errorf("writing serving data in cell %v for %v/%v failed: %v", cell, shardInfo.Keyspace(), shardInfo.ShardName(), err)) } span.Finish() wg.Done() }() wg.Wait() return rec.Error() }
// FindAllTabletAliasesInShard uses the replication graph to find all the // tablet aliases in the given shard. // It can return ErrPartialResult if some cells were not fetched, // in which case the result only contains the cells that were fetched. func FindAllTabletAliasesInShardByCell(ctx context.Context, ts Server, keyspace, shard string, cells []string) ([]TabletAlias, error) { span := trace.NewSpanFromContext(ctx) span.StartLocal("topo.FindAllTabletAliasesInShardbyCell") span.Annotate("keyspace", keyspace) span.Annotate("shard", shard) span.Annotate("num_cells", len(cells)) defer span.Finish() // read the shard information to find the cells si, err := ts.GetShard(keyspace, shard) if err != nil { return nil, err } resultAsMap := make(map[TabletAlias]bool) if !si.MasterAlias.IsZero() { if InCellList(si.MasterAlias.Cell, cells) { resultAsMap[si.MasterAlias] = true } } // read the replication graph in each cell and add all found tablets wg := sync.WaitGroup{} mutex := sync.Mutex{} rec := concurrency.AllErrorRecorder{} for _, cell := range si.Cells { if !InCellList(cell, cells) { continue } wg.Add(1) go func(cell string) { defer wg.Done() sri, err := ts.GetShardReplication(cell, keyspace, shard) if err != nil { rec.RecordError(fmt.Errorf("GetShardReplication(%v, %v, %v) failed: %v", cell, keyspace, shard, err)) return } mutex.Lock() for _, rl := range sri.ReplicationLinks { resultAsMap[rl.TabletAlias] = true if !rl.Parent.IsZero() && InCellList(rl.Parent.Cell, cells) { resultAsMap[rl.Parent] = true } } mutex.Unlock() }(cell) } wg.Wait() err = nil if rec.HasErrors() { log.Warningf("FindAllTabletAliasesInShard(%v,%v): got partial result: %v", keyspace, shard, rec.Error()) err = ErrPartialResult } result := make([]TabletAlias, 0, len(resultAsMap)) for a := range resultAsMap { result = append(result, a) } return result, err }
// changeCallback is run after every action that might // have changed something in the tablet record. func (agent *ActionAgent) changeCallback(ctx context.Context, oldTablet, newTablet *topo.Tablet) error { span := trace.NewSpanFromContext(ctx) span.StartLocal("ActionAgent.changeCallback") defer span.Finish() allowQuery := newTablet.IsRunningQueryService() // Read the shard to get SourceShards / TabletControlMap if // we're going to use it. var shardInfo *topo.ShardInfo var tabletControl *topo.TabletControl var blacklistedTables []string var err error if allowQuery { shardInfo, err = agent.TopoServer.GetShard(newTablet.Keyspace, newTablet.Shard) if err != nil { log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err) } else { if newTablet.Type == topo.TYPE_MASTER { allowQuery = len(shardInfo.SourceShards) == 0 } if tc, ok := shardInfo.TabletControlMap[newTablet.Type]; ok { if topo.InCellList(newTablet.Alias.Cell, tc.Cells) { if tc.DisableQueryService { allowQuery = false } blacklistedTables = tc.BlacklistedTables tabletControl = tc } } } } // Read the keyspace on masters to get ShardingColumnType, // for binlog replication, only if source shards are set. var keyspaceInfo *topo.KeyspaceInfo if newTablet.Type == topo.TYPE_MASTER && shardInfo != nil && len(shardInfo.SourceShards) > 0 { keyspaceInfo, err = agent.TopoServer.GetKeyspace(newTablet.Keyspace) if err != nil { log.Errorf("Cannot read keyspace for this tablet %v: %v", newTablet.Alias, err) keyspaceInfo = nil } } if allowQuery { // There are a few transitions when we're // going to need to restart the query service: // - transitioning from replica to master, so clients // that were already connected don't keep on using // the master as replica or rdonly. // - having different parameters for the query // service. It needs to stop and restart with the // new parameters. That includes: // - changing KeyRange // - changing the BlacklistedTables list if (newTablet.Type == topo.TYPE_MASTER && oldTablet.Type != topo.TYPE_MASTER) || (newTablet.KeyRange != oldTablet.KeyRange) || !reflect.DeepEqual(blacklistedTables, agent.BlacklistedTables()) { agent.disallowQueries() } if err := agent.allowQueries(newTablet, blacklistedTables); err != nil { log.Errorf("Cannot start query service: %v", err) } } else { agent.disallowQueries() } // save the tabletControl we've been using, so the background // healthcheck makes the same decisions as we've been making. agent.setTabletControl(tabletControl) // update stream needs to be started or stopped too if agent.DBConfigs != nil { if topo.IsRunningUpdateStream(newTablet.Type) { binlog.EnableUpdateStreamService(agent.DBConfigs.App.DbName, agent.Mysqld) } else { binlog.DisableUpdateStreamService() } } statsType.Set(string(newTablet.Type)) statsKeyspace.Set(newTablet.Keyspace) statsShard.Set(newTablet.Shard) statsKeyRangeStart.Set(string(newTablet.KeyRange.Start.Hex())) statsKeyRangeEnd.Set(string(newTablet.KeyRange.End.Hex())) // See if we need to start or stop any binlog player if agent.BinlogPlayerMap != nil { if newTablet.Type == topo.TYPE_MASTER { agent.BinlogPlayerMap.RefreshMap(newTablet, keyspaceInfo, shardInfo) } else { agent.BinlogPlayerMap.StopAllPlayersAndReset() } } return nil }