// UpdateShardReplicationRecord is a low level function to add / update an
// entry to the ShardReplication object.
func UpdateShardReplicationRecord(ctx context.Context, ts Server, keyspace, shard string, tabletAlias, parent TabletAlias) error {
span := trace.NewSpanFromContext(ctx)
span.StartClient("TopoServer.UpdateShardReplicationFields")
span.Annotate("keyspace", keyspace)
span.Annotate("shard", shard)
span.Annotate("tablet", tabletAlias.String())
defer span.Finish()
return ts.UpdateShardReplicationFields(tabletAlias.Cell, keyspace, shard, func(sr *ShardReplication) error {
// not very efficient, but easy to read
links := make([]ReplicationLink, 0, len(sr.ReplicationLinks)+1)
found := false
for _, link := range sr.ReplicationLinks {
if link.TabletAlias == tabletAlias {
if found {
log.Warningf("Found a second ReplicationLink for tablet %v, deleting it", tabletAlias)
continue
}
found = true
// update the master
link.Parent = parent
}
links = append(links, link)
}
if !found {
links = append(links, ReplicationLink{TabletAlias: tabletAlias, Parent: parent})
}
sr.ReplicationLinks = links
return nil
})
}
// UnlockShard unlocks a previously locked shard.
func (n *ActionNode) UnlockShard(ctx context.Context, ts topo.Server, keyspace, shard string, lockPath string, actionError error) error {
span := trace.NewSpanFromContext(ctx)
span.StartClient("TopoServer.UnlockShardForAction")
span.Annotate("action", n.Action)
span.Annotate("keyspace", keyspace)
span.Annotate("shard", shard)
defer span.Finish()
// first update the actionNode
if actionError != nil {
log.Infof("Unlocking shard %v/%v for action %v with error %v", keyspace, shard, n.Action, actionError)
n.Error = actionError.Error()
n.State = ACTION_STATE_FAILED
} else {
log.Infof("Unlocking shard %v/%v for successful action %v", keyspace, shard, n.Action)
n.Error = ""
n.State = ACTION_STATE_DONE
}
err := ts.UnlockShardForAction(keyspace, shard, lockPath, n.ToJson())
if actionError != nil {
if err != nil {
// this will be masked
log.Warningf("UnlockShardForAction failed: %v", err)
}
return actionError
}
return err
}
// GetTabletMap tries to read all the tablets in the provided list,
// and returns them all in a map.
// If error is ErrPartialResult, the results in the dictionary are
// incomplete, meaning some tablets couldn't be read.
func GetTabletMap(ctx context.Context, ts Server, tabletAliases []TabletAlias) (map[TabletAlias]*TabletInfo, error) {
span := trace.NewSpanFromContext(ctx)
span.StartLocal("topo.GetTabletMap")
span.Annotate("num_tablets", len(tabletAliases))
defer span.Finish()
wg := sync.WaitGroup{}
mutex := sync.Mutex{}
tabletMap := make(map[TabletAlias]*TabletInfo)
var someError error
for _, tabletAlias := range tabletAliases {
wg.Add(1)
go func(tabletAlias TabletAlias) {
defer wg.Done()
tabletInfo, err := ts.GetTablet(tabletAlias)
mutex.Lock()
if err != nil {
log.Warningf("%v: %v", tabletAlias, err)
// There can be data races removing nodes - ignore them for now.
if err != ErrNoNode {
someError = ErrPartialResult
}
} else {
tabletMap[tabletAlias] = tabletInfo
}
mutex.Unlock()
}(tabletAlias)
}
wg.Wait()
return tabletMap, someError
}
// refreshTablet needs to be run after an action may have changed the current
// state of the tablet.
func (agent *ActionAgent) refreshTablet(ctx context.Context, reason string) error {
log.Infof("Executing post-action state refresh")
span := trace.NewSpanFromContext(ctx)
span.StartLocal("ActionAgent.refreshTablet")
span.Annotate("reason", reason)
defer span.Finish()
ctx = trace.NewContext(ctx, span)
// Save the old tablet so callbacks can have a better idea of
// the precise nature of the transition.
oldTablet := agent.Tablet().Tablet
// Actions should have side effects on the tablet, so reload the data.
if err := agent.readTablet(); err != nil {
log.Warningf("Failed rereading tablet after %v - services may be inconsistent: %v", reason, err)
return fmt.Errorf("Failed rereading tablet after %v: %v", reason, err)
}
if updatedTablet := agent.checkTabletMysqlPort(agent.Tablet()); updatedTablet != nil {
agent.mutex.Lock()
agent._tablet = updatedTablet
agent.mutex.Unlock()
}
if err := agent.updateState(ctx, oldTablet, reason); err != nil {
return err
}
log.Infof("Done with post-action state refresh")
return nil
}
// LockKeyspace will lock the keyspace in the topology server.
// UnlockKeyspace should be called if this returns no error.
func (n *ActionNode) LockKeyspace(ctx context.Context, ts topo.Server, keyspace string, lockTimeout time.Duration, interrupted chan struct{}) (lockPath string, err error) {
log.Infof("Locking keyspace %v for action %v", keyspace, n.Action)
span := trace.NewSpanFromContext(ctx)
span.StartClient("TopoServer.LockKeyspaceForAction")
span.Annotate("action", n.Action)
span.Annotate("keyspace", keyspace)
defer span.Finish()
return ts.LockKeyspaceForAction(keyspace, n.ToJson(), lockTimeout, interrupted)
}
// LockSrvShard will lock the serving shard in the topology server.
// UnlockSrvShard should be called if this returns no error.
func (n *ActionNode) LockSrvShard(ctx context.Context, ts topo.Server, cell, keyspace, shard string, lockTimeout time.Duration, interrupted chan struct{}) (lockPath string, err error) {
log.Infof("Locking serving shard %v/%v/%v for action %v", cell, keyspace, shard, n.Action)
span := trace.NewSpanFromContext(ctx)
span.StartClient("TopoServer.LockSrvShardForAction")
span.Annotate("action", n.Action)
span.Annotate("keyspace", keyspace)
span.Annotate("shard", shard)
span.Annotate("cell", cell)
defer span.Finish()
return ts.LockSrvShardForAction(cell, keyspace, shard, n.ToJson(), lockTimeout, interrupted)
}
// UpdateTablet updates the tablet data only - not associated replication paths.
func UpdateTablet(ctx context.Context, ts Server, tablet *TabletInfo) error {
span := trace.NewSpanFromContext(ctx)
span.StartClient("TopoServer.UpdateTablet")
span.Annotate("tablet", tablet.Alias.String())
defer span.Finish()
var version int64 = -1
if tablet.version != 0 {
version = tablet.version
}
newVersion, err := ts.UpdateTablet(tablet, version)
if err == nil {
tablet.version = newVersion
}
return err
}
// UpdateShard updates the shard data, with the right version
func UpdateShard(ctx context.Context, ts Server, si *ShardInfo) error {
span := trace.NewSpanFromContext(ctx)
span.StartClient("TopoServer.UpdateShard")
span.Annotate("keyspace", si.Keyspace())
span.Annotate("shard", si.ShardName())
defer span.Finish()
var version int64 = -1
if si.version != 0 {
version = si.version
}
newVersion, err := ts.UpdateShard(si, version)
if err == nil {
si.version = newVersion
}
return err
}
// Go invokes the function asynchronously. It returns the Call structure representing
// the invocation. The done channel will signal when the call is complete by returning
// the same Call object. If done is nil, Go will allocate a new channel.
// If non-nil, done must be buffered or Go will deliberately crash.
func (client *Client) Go(ctx context.Context, serviceMethod string, args interface{}, reply interface{}, done chan *Call) *Call {
span := trace.NewSpanFromContext(ctx)
span.StartClient(serviceMethod)
defer span.Finish()
call := new(Call)
call.ServiceMethod = serviceMethod
call.Args = args
call.Reply = reply
if done == nil {
done = make(chan *Call, 10) // buffered.
} else {
// If caller passes done != nil, it must arrange that
// done has enough buffer for the number of simultaneous
// RPCs that will be using that channel. If the channel
// is totally unbuffered, it's best not to run at all.
if cap(done) == 0 {
log.Panic("rpc: done channel is unbuffered")
}
}
call.Done = done
client.send(call)
return call
}
// Update shard file with new master, replicas, etc.
//
// Re-read from TopologyServer to make sure we are using the side
// effects of all actions.
//
// This function locks individual SvrShard paths, so it doesn't need a lock
// on the shard.
func RebuildShard(ctx context.Context, log logutil.Logger, ts topo.Server, keyspace, shard string, cells []string, timeout time.Duration, interrupted chan struct{}) (*topo.ShardInfo, error) {
log.Infof("RebuildShard %v/%v", keyspace, shard)
span := trace.NewSpanFromContext(ctx)
span.StartLocal("topotools.RebuildShard")
defer span.Finish()
ctx = trace.NewContext(ctx, span)
// read the existing shard info. It has to exist.
shardInfo, err := ts.GetShard(keyspace, shard)
if err != nil {
return nil, err
}
// rebuild all cells in parallel
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, cell := range shardInfo.Cells {
// skip this cell if we shouldn't rebuild it
if !topo.InCellList(cell, cells) {
continue
}
// start with the master if it's in the current cell
tabletsAsMap := make(map[topo.TabletAlias]bool)
if shardInfo.MasterAlias.Cell == cell {
tabletsAsMap[shardInfo.MasterAlias] = true
}
wg.Add(1)
go func(cell string) {
defer wg.Done()
// Lock the SrvShard so we don't race with other rebuilds of the same
// shard in the same cell (e.g. from our peer tablets).
actionNode := actionnode.RebuildSrvShard()
lockPath, err := actionNode.LockSrvShard(ctx, ts, cell, keyspace, shard, timeout, interrupted)
if err != nil {
rec.RecordError(err)
return
}
// read the ShardReplication object to find tablets
sri, err := ts.GetShardReplication(cell, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardReplication(%v, %v, %v) failed: %v", cell, keyspace, shard, err))
return
}
// add all relevant tablets to the map
for _, rl := range sri.ReplicationLinks {
tabletsAsMap[rl.TabletAlias] = true
if rl.Parent.Cell == cell {
tabletsAsMap[rl.Parent] = true
}
}
// convert the map to a list
aliases := make([]topo.TabletAlias, 0, len(tabletsAsMap))
for a := range tabletsAsMap {
aliases = append(aliases, a)
}
// read all the Tablet records
tablets, err := topo.GetTabletMap(ctx, ts, aliases)
switch err {
case nil:
// keep going, we're good
case topo.ErrPartialResult:
log.Warningf("Got ErrPartialResult from topo.GetTabletMap in cell %v, some tablets may not be added properly to serving graph", cell)
default:
rec.RecordError(fmt.Errorf("GetTabletMap in cell %v failed: %v", cell, err))
return
}
// write the data we need to
rebuildErr := rebuildCellSrvShard(ctx, log, ts, shardInfo, cell, tablets)
// and unlock
if err := actionNode.UnlockSrvShard(ctx, ts, cell, keyspace, shard, lockPath, rebuildErr); err != nil {
rec.RecordError(err)
}
}(cell)
}
wg.Wait()
return shardInfo, rec.Error()
}
// rebuildCellSrvShard computes and writes the serving graph data to a
// single cell
func rebuildCellSrvShard(ctx context.Context, log logutil.Logger, ts topo.Server, shardInfo *topo.ShardInfo, cell string, tablets map[topo.TabletAlias]*topo.TabletInfo) error {
log.Infof("rebuildCellSrvShard %v/%v in cell %v", shardInfo.Keyspace(), shardInfo.ShardName(), cell)
// Get all existing db types so they can be removed if nothing
// had been edited.
existingTabletTypes, err := ts.GetSrvTabletTypesPerShard(cell, shardInfo.Keyspace(), shardInfo.ShardName())
if err != nil {
if err != topo.ErrNoNode {
return err
}
}
// Update db type addresses in the serving graph
//
// locationAddrsMap is a map:
// key: tabletType
// value: EndPoints (list of server records)
locationAddrsMap := make(map[topo.TabletType]*topo.EndPoints)
for _, tablet := range tablets {
if !tablet.IsInReplicationGraph() {
// only valid case is a scrapped master in the
// catastrophic reparent case
if tablet.Parent.Uid != topo.NO_TABLET {
log.Warningf("Tablet %v should not be in the replication graph, please investigate (it is being ignored in the rebuild)", tablet.Alias)
}
continue
}
// Check IsInServingGraph, we don't want to add tablets that
// are not serving
if !tablet.IsInServingGraph() {
continue
}
// Check the Keyspace and Shard for the tablet are right
if tablet.Keyspace != shardInfo.Keyspace() || tablet.Shard != shardInfo.ShardName() {
return fmt.Errorf("CRITICAL: tablet %v is in replication graph for shard %v/%v but belongs to shard %v:%v", tablet.Alias, shardInfo.Keyspace(), shardInfo.ShardName(), tablet.Keyspace, tablet.Shard)
}
// Add the tablet to the list
addrs, ok := locationAddrsMap[tablet.Type]
if !ok {
addrs = topo.NewEndPoints()
locationAddrsMap[tablet.Type] = addrs
}
entry, err := tablet.Tablet.EndPoint()
if err != nil {
log.Warningf("EndPointForTablet failed for tablet %v: %v", tablet.Alias, err)
continue
}
addrs.Entries = append(addrs.Entries, *entry)
}
// we're gonna parallelize a lot here:
// - writing all the tabletTypes records
// - removing the unused records
// - writing SrvShard
rec := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
// write all the EndPoints nodes everywhere we want them
for tabletType, addrs := range locationAddrsMap {
wg.Add(1)
go func(tabletType topo.TabletType, addrs *topo.EndPoints) {
log.Infof("saving serving graph for cell %v shard %v/%v tabletType %v", cell, shardInfo.Keyspace(), shardInfo.ShardName(), tabletType)
span := trace.NewSpanFromContext(ctx)
span.StartClient("TopoServer.UpdateEndPoints")
span.Annotate("tablet_type", string(tabletType))
if err := ts.UpdateEndPoints(cell, shardInfo.Keyspace(), shardInfo.ShardName(), tabletType, addrs); err != nil {
rec.RecordError(fmt.Errorf("writing endpoints for cell %v shard %v/%v tabletType %v failed: %v", cell, shardInfo.Keyspace(), shardInfo.ShardName(), tabletType, err))
}
span.Finish()
wg.Done()
}(tabletType, addrs)
}
// Delete any pre-existing paths that were not updated by this process.
// That's the existingTabletTypes - locationAddrsMap
for _, tabletType := range existingTabletTypes {
if _, ok := locationAddrsMap[tabletType]; !ok {
wg.Add(1)
go func(tabletType topo.TabletType) {
log.Infof("removing stale db type from serving graph: %v", tabletType)
span := trace.NewSpanFromContext(ctx)
span.StartClient("TopoServer.DeleteEndPoints")
span.Annotate("tablet_type", string(tabletType))
if err := ts.DeleteEndPoints(cell, shardInfo.Keyspace(), shardInfo.ShardName(), tabletType); err != nil {
log.Warningf("unable to remove stale db type %v from serving graph: %v", tabletType, err)
}
span.Finish()
wg.Done()
}(tabletType)
}
}
// Update srvShard object
wg.Add(1)
go func() {
log.Infof("updating shard serving graph in cell %v for %v/%v", cell, shardInfo.Keyspace(), shardInfo.ShardName())
srvShard := &topo.SrvShard{
Name: shardInfo.ShardName(),
KeyRange: shardInfo.KeyRange,
ServedTypes: shardInfo.GetServedTypesPerCell(cell),
MasterCell: shardInfo.MasterAlias.Cell,
TabletTypes: make([]topo.TabletType, 0, len(locationAddrsMap)),
}
for tabletType := range locationAddrsMap {
srvShard.TabletTypes = append(srvShard.TabletTypes, tabletType)
}
span := trace.NewSpanFromContext(ctx)
span.StartClient("TopoServer.UpdateSrvShard")
span.Annotate("keyspace", shardInfo.Keyspace())
span.Annotate("shard", shardInfo.ShardName())
span.Annotate("cell", cell)
if err := ts.UpdateSrvShard(cell, shardInfo.Keyspace(), shardInfo.ShardName(), srvShard); err != nil {
rec.RecordError(fmt.Errorf("writing serving data in cell %v for %v/%v failed: %v", cell, shardInfo.Keyspace(), shardInfo.ShardName(), err))
}
span.Finish()
wg.Done()
}()
wg.Wait()
return rec.Error()
}
// FindAllTabletAliasesInShard uses the replication graph to find all the
// tablet aliases in the given shard.
// It can return ErrPartialResult if some cells were not fetched,
// in which case the result only contains the cells that were fetched.
func FindAllTabletAliasesInShardByCell(ctx context.Context, ts Server, keyspace, shard string, cells []string) ([]TabletAlias, error) {
span := trace.NewSpanFromContext(ctx)
span.StartLocal("topo.FindAllTabletAliasesInShardbyCell")
span.Annotate("keyspace", keyspace)
span.Annotate("shard", shard)
span.Annotate("num_cells", len(cells))
defer span.Finish()
// read the shard information to find the cells
si, err := ts.GetShard(keyspace, shard)
if err != nil {
return nil, err
}
resultAsMap := make(map[TabletAlias]bool)
if !si.MasterAlias.IsZero() {
if InCellList(si.MasterAlias.Cell, cells) {
resultAsMap[si.MasterAlias] = true
}
}
// read the replication graph in each cell and add all found tablets
wg := sync.WaitGroup{}
mutex := sync.Mutex{}
rec := concurrency.AllErrorRecorder{}
for _, cell := range si.Cells {
if !InCellList(cell, cells) {
continue
}
wg.Add(1)
go func(cell string) {
defer wg.Done()
sri, err := ts.GetShardReplication(cell, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardReplication(%v, %v, %v) failed: %v", cell, keyspace, shard, err))
return
}
mutex.Lock()
for _, rl := range sri.ReplicationLinks {
resultAsMap[rl.TabletAlias] = true
if !rl.Parent.IsZero() && InCellList(rl.Parent.Cell, cells) {
resultAsMap[rl.Parent] = true
}
}
mutex.Unlock()
}(cell)
}
wg.Wait()
err = nil
if rec.HasErrors() {
log.Warningf("FindAllTabletAliasesInShard(%v,%v): got partial result: %v", keyspace, shard, rec.Error())
err = ErrPartialResult
}
result := make([]TabletAlias, 0, len(resultAsMap))
for a := range resultAsMap {
result = append(result, a)
}
return result, err
}
// changeCallback is run after every action that might
// have changed something in the tablet record.
func (agent *ActionAgent) changeCallback(ctx context.Context, oldTablet, newTablet *topo.Tablet) error {
span := trace.NewSpanFromContext(ctx)
span.StartLocal("ActionAgent.changeCallback")
defer span.Finish()
allowQuery := newTablet.IsRunningQueryService()
// Read the shard to get SourceShards / TabletControlMap if
// we're going to use it.
var shardInfo *topo.ShardInfo
var tabletControl *topo.TabletControl
var blacklistedTables []string
var err error
if allowQuery {
shardInfo, err = agent.TopoServer.GetShard(newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err)
} else {
if newTablet.Type == topo.TYPE_MASTER {
allowQuery = len(shardInfo.SourceShards) == 0
}
if tc, ok := shardInfo.TabletControlMap[newTablet.Type]; ok {
if topo.InCellList(newTablet.Alias.Cell, tc.Cells) {
if tc.DisableQueryService {
allowQuery = false
}
blacklistedTables = tc.BlacklistedTables
tabletControl = tc
}
}
}
}
// Read the keyspace on masters to get ShardingColumnType,
// for binlog replication, only if source shards are set.
var keyspaceInfo *topo.KeyspaceInfo
if newTablet.Type == topo.TYPE_MASTER && shardInfo != nil && len(shardInfo.SourceShards) > 0 {
keyspaceInfo, err = agent.TopoServer.GetKeyspace(newTablet.Keyspace)
if err != nil {
log.Errorf("Cannot read keyspace for this tablet %v: %v", newTablet.Alias, err)
keyspaceInfo = nil
}
}
if allowQuery {
// There are a few transitions when we're
// going to need to restart the query service:
// - transitioning from replica to master, so clients
// that were already connected don't keep on using
// the master as replica or rdonly.
// - having different parameters for the query
// service. It needs to stop and restart with the
// new parameters. That includes:
// - changing KeyRange
// - changing the BlacklistedTables list
if (newTablet.Type == topo.TYPE_MASTER &&
oldTablet.Type != topo.TYPE_MASTER) ||
(newTablet.KeyRange != oldTablet.KeyRange) ||
!reflect.DeepEqual(blacklistedTables, agent.BlacklistedTables()) {
agent.disallowQueries()
}
if err := agent.allowQueries(newTablet, blacklistedTables); err != nil {
log.Errorf("Cannot start query service: %v", err)
}
} else {
agent.disallowQueries()
}
// save the tabletControl we've been using, so the background
// healthcheck makes the same decisions as we've been making.
agent.setTabletControl(tabletControl)
// update stream needs to be started or stopped too
if agent.DBConfigs != nil {
if topo.IsRunningUpdateStream(newTablet.Type) {
binlog.EnableUpdateStreamService(agent.DBConfigs.App.DbName, agent.Mysqld)
} else {
binlog.DisableUpdateStreamService()
}
}
statsType.Set(string(newTablet.Type))
statsKeyspace.Set(newTablet.Keyspace)
statsShard.Set(newTablet.Shard)
statsKeyRangeStart.Set(string(newTablet.KeyRange.Start.Hex()))
statsKeyRangeEnd.Set(string(newTablet.KeyRange.End.Hex()))
// See if we need to start or stop any binlog player
if agent.BinlogPlayerMap != nil {
if newTablet.Type == topo.TYPE_MASTER {
agent.BinlogPlayerMap.RefreshMap(newTablet, keyspaceInfo, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
}
return nil
}