// findCellsForRebuild will find all the cells in the given keyspace
// and create an entry if the map for them
func (wr *Wrangler) findCellsForRebuild(ki *topo.KeyspaceInfo, keyspace string, shards []string, cells []string, srvKeyspaceMap map[string]*topo.SrvKeyspace) error {
er := concurrency.FirstErrorRecorder{}
mu := sync.Mutex{}
wg := sync.WaitGroup{}
for _, shard := range shards {
wg.Add(1)
go func(shard string) {
if si, err := wr.ts.GetShard(keyspace, shard); err != nil {
er.RecordError(fmt.Errorf("GetShard(%v,%v) failed: %v", keyspace, shard, err))
} else {
mu.Lock()
for _, cell := range si.Cells {
if !topo.InCellList(cell, cells) {
continue
}
if _, ok := srvKeyspaceMap[cell]; !ok {
srvKeyspaceMap[cell] = &topo.SrvKeyspace{
Shards: make([]topo.SrvShard, 0, 16),
ShardingColumnName: ki.ShardingColumnName,
ShardingColumnType: ki.ShardingColumnType,
ServedFrom: ki.ServedFrom,
}
}
}
mu.Unlock()
}
wg.Done()
}(shard)
}
wg.Wait()
return er.Error()
}
// RebuildShard updates the SrvShard objects and underlying serving graph.
//
// Re-read from TopologyServer to make sure we are using the side
// effects of all actions.
//
// This function will start each cell over from the beginning on ErrBadVersion,
// so it doesn't need a lock on the shard.
func RebuildShard(ctx context.Context, log logutil.Logger, ts topo.Server, keyspace, shard string, cells []string, lockTimeout time.Duration) (*topo.ShardInfo, error) {
log.Infof("RebuildShard %v/%v", keyspace, shard)
span := trace.NewSpanFromContext(ctx)
span.StartLocal("topotools.RebuildShard")
defer span.Finish()
ctx = trace.NewContext(ctx, span)
// read the existing shard info. It has to exist.
shardInfo, err := ts.GetShard(ctx, keyspace, shard)
if err != nil {
return nil, err
}
// rebuild all cells in parallel
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, cell := range shardInfo.Cells {
// skip this cell if we shouldn't rebuild it
if !topo.InCellList(cell, cells) {
continue
}
wg.Add(1)
go func(cell string) {
defer wg.Done()
rec.RecordError(rebuildCellSrvShard(ctx, log, ts, shardInfo, cell))
}(cell)
}
wg.Wait()
return shardInfo, rec.Error()
}
func (wr *Wrangler) removeShardCell(keyspace, shard, cell string, force bool) error {
shardInfo, err := wr.ts.GetShardCritical(keyspace, shard)
if err != nil {
return err
}
// check the cell is in the list already
if !topo.InCellList(cell, shardInfo.Cells) {
return fmt.Errorf("cell %v in not in shard info", cell)
}
// check the master alias is not in the cell
if shardInfo.MasterAlias.Cell == cell {
return fmt.Errorf("master %v is in the cell '%v' we want to remove", shardInfo.MasterAlias, cell)
}
// get the ShardReplication object in the cell
sri, err := wr.ts.GetShardReplication(cell, keyspace, shard)
switch err {
case nil:
if len(sri.ReplicationLinks) > 0 {
return fmt.Errorf("cell %v has %v possible tablets in replication graph", cell, len(sri.ReplicationLinks))
}
// ShardReplication object is now useless, remove it
if err := wr.ts.DeleteShardReplication(cell, keyspace, shard); err != nil {
return fmt.Errorf("error deleting ShardReplication object in cell %v: %v", cell, err)
}
// we keep going
case topo.ErrNoNode:
// no ShardReplication object, we keep going
default:
// we can't get the object, assume topo server is down there,
// so we look at force flag
if !force {
return err
}
log.Warningf("Cannot get ShardReplication from cell %v, assuming cell topo server is down, and forcing the removal", cell)
}
// now we can update the shard
log.Infof("Removing cell %v from shard %v/%v", cell, keyspace, shard)
newCells := make([]string, 0, len(shardInfo.Cells)-1)
for _, c := range shardInfo.Cells {
if c != cell {
newCells = append(newCells, c)
}
}
shardInfo.Cells = newCells
return wr.ts.UpdateShard(shardInfo)
}
// findCellsForRebuild will find all the cells in the given keyspace
// and create an entry if the map for them
func findCellsForRebuild(ki *topo.KeyspaceInfo, shardMap map[string]*topo.ShardInfo, cells []string, srvKeyspaceMap map[string]*topodatapb.SrvKeyspace) {
for _, si := range shardMap {
for _, cell := range si.Cells {
if !topo.InCellList(cell, cells) {
continue
}
if _, ok := srvKeyspaceMap[cell]; !ok {
srvKeyspaceMap[cell] = &topodatapb.SrvKeyspace{
ShardingColumnName: ki.ShardingColumnName,
ShardingColumnType: ki.ShardingColumnType,
ServedFrom: ki.ComputeCellServedFrom(cell),
}
}
}
}
}
// findCellsForRebuild will find all the cells in the given keyspace
// and create an entry if the map for them
func (wr *Wrangler) findCellsForRebuild(ki *topo.KeyspaceInfo, shardMap map[string]*topo.ShardInfo, cells []string, srvKeyspaceMap map[string]*topo.SrvKeyspace) {
for _, si := range shardMap {
for _, cell := range si.Cells {
if !topo.InCellList(cell, cells) {
continue
}
if _, ok := srvKeyspaceMap[cell]; !ok {
srvKeyspaceMap[cell] = &topo.SrvKeyspace{
ShardingColumnName: ki.ShardingColumnName,
ShardingColumnType: key.ProtoToKeyspaceIdType(ki.ShardingColumnType),
ServedFrom: ki.ComputeCellServedFrom(cell),
SplitShardCount: ki.SplitShardCount,
}
}
}
}
}
// changeCallback is run after every action that might
// have changed something in the tablet record.
func (agent *ActionAgent) changeCallback(oldTablet, newTablet *topo.Tablet) error {
allowQuery := newTablet.IsRunningQueryService()
// Read the shard to get SourceShards / TabletControlMap if
// we're going to use it.
var shardInfo *topo.ShardInfo
var tabletControl *topo.TabletControl
var blacklistedTables []string
var err error
if allowQuery {
shardInfo, err = agent.TopoServer.GetShard(newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err)
} else {
if newTablet.Type == topo.TYPE_MASTER {
allowQuery = len(shardInfo.SourceShards) == 0
}
if tc, ok := shardInfo.TabletControlMap[newTablet.Type]; ok {
if topo.InCellList(newTablet.Alias.Cell, tc.Cells) {
if tc.DisableQueryService {
allowQuery = false
}
blacklistedTables = tc.BlacklistedTables
tabletControl = tc
}
}
}
}
// Read the keyspace on masters to get ShardingColumnType,
// for binlog replication, only if source shards are set.
var keyspaceInfo *topo.KeyspaceInfo
if newTablet.Type == topo.TYPE_MASTER && shardInfo != nil && len(shardInfo.SourceShards) > 0 {
keyspaceInfo, err = agent.TopoServer.GetKeyspace(newTablet.Keyspace)
if err != nil {
log.Errorf("Cannot read keyspace for this tablet %v: %v", newTablet.Alias, err)
keyspaceInfo = nil
}
}
if allowQuery {
// There are a few transitions when we're
// going to need to restart the query service:
// - transitioning from replica to master, so clients
// that were already connected don't keep on using
// the master as replica or rdonly.
// - having different parameters for the query
// service. It needs to stop and restart with the
// new parameters. That includes:
// - changing KeyRange
// - changing the BlacklistedTables list
if (newTablet.Type == topo.TYPE_MASTER &&
oldTablet.Type != topo.TYPE_MASTER) ||
(newTablet.KeyRange != oldTablet.KeyRange) ||
!reflect.DeepEqual(blacklistedTables, agent.BlacklistedTables()) {
agent.disallowQueries()
}
if err := agent.allowQueries(newTablet, blacklistedTables); err != nil {
log.Errorf("Cannot start query service: %v", err)
}
} else {
agent.disallowQueries()
}
// save the tabletControl we've been using, so the background
// healthcheck makes the same decisions as we've been making.
agent.setTabletControl(tabletControl)
// update stream needs to be started or stopped too
if agent.DBConfigs != nil {
if topo.IsRunningUpdateStream(newTablet.Type) {
binlog.EnableUpdateStreamService(agent.DBConfigs.App.DbName, agent.Mysqld)
} else {
binlog.DisableUpdateStreamService()
}
}
statsType.Set(string(newTablet.Type))
statsKeyspace.Set(newTablet.Keyspace)
statsShard.Set(newTablet.Shard)
statsKeyRangeStart.Set(string(newTablet.KeyRange.Start.Hex()))
statsKeyRangeEnd.Set(string(newTablet.KeyRange.End.Hex()))
// See if we need to start or stop any binlog player
if agent.BinlogPlayerMap != nil {
if newTablet.Type == topo.TYPE_MASTER {
agent.BinlogPlayerMap.RefreshMap(newTablet, keyspaceInfo, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
}
return nil
}
func (wr *Wrangler) removeShardCell(ctx context.Context, keyspace, shard, cell string, force, recursive bool) error {
shardInfo, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return err
}
// check the cell is in the list already
if !topo.InCellList(cell, shardInfo.Cells) {
return fmt.Errorf("cell %v in not in shard info", cell)
}
// check the master alias is not in the cell
if shardInfo.MasterAlias.Cell == cell {
return fmt.Errorf("master %v is in the cell '%v' we want to remove", shardInfo.MasterAlias, cell)
}
// get the ShardReplication object in the cell
sri, err := wr.ts.GetShardReplication(ctx, cell, keyspace, shard)
switch err {
case nil:
if recursive {
wr.Logger().Infof("Deleting all tablets in shard %v/%v", keyspace, shard)
for _, node := range sri.Nodes {
// We don't care about scrapping or updating the replication graph,
// because we're about to delete the entire replication graph.
wr.Logger().Infof("Deleting tablet %v", node.TabletAlias)
if err := wr.TopoServer().DeleteTablet(ctx, topo.ProtoToTabletAlias(node.TabletAlias)); err != nil && err != topo.ErrNoNode {
return fmt.Errorf("can't delete tablet %v: %v", node.TabletAlias, err)
}
}
} else if len(sri.Nodes) > 0 {
return fmt.Errorf("cell %v has %v possible tablets in replication graph", cell, len(sri.Nodes))
}
// ShardReplication object is now useless, remove it
if err := wr.ts.DeleteShardReplication(ctx, cell, keyspace, shard); err != nil && err != topo.ErrNoNode {
return fmt.Errorf("error deleting ShardReplication object in cell %v: %v", cell, err)
}
// Rebuild the shard serving graph to reflect the tablets we deleted.
// This must be done before removing the cell from the global shard record,
// since this cell will be skipped by all future rebuilds.
if _, err := wr.RebuildShardGraph(ctx, keyspace, shard, []string{cell}); err != nil {
return fmt.Errorf("can't rebuild serving graph for shard %v/%v in cell %v: %v", keyspace, shard, cell, err)
}
// we keep going
case topo.ErrNoNode:
// no ShardReplication object, we keep going
default:
// we can't get the object, assume topo server is down there,
// so we look at force flag
if !force {
return err
}
wr.Logger().Warningf("Cannot get ShardReplication from cell %v, assuming cell topo server is down, and forcing the removal", cell)
}
// now we can update the shard
wr.Logger().Infof("Removing cell %v from shard %v/%v", cell, keyspace, shard)
newCells := make([]string, 0, len(shardInfo.Cells)-1)
for _, c := range shardInfo.Cells {
if c != cell {
newCells = append(newCells, c)
}
}
shardInfo.Cells = newCells
return topo.UpdateShard(ctx, wr.ts, shardInfo)
}
// Write serving graph data to the cells
func (wr *Wrangler) rebuildShardSrvGraph(shardInfo *topo.ShardInfo, tablets []*topo.TabletInfo, cells []string) error {
log.Infof("rebuildShardSrvGraph %v/%v", shardInfo.Keyspace(), shardInfo.ShardName())
// Get all existing db types so they can be removed if nothing
// had been editted. This applies to all cells, which can't
// be determined until you walk through all the tablets.
//
// existingDbTypeLocations is a map:
// key: {cell,keyspace,shard,tabletType}
// value: true
existingDbTypeLocations := make(map[cellKeyspaceShardType]bool)
// Update db type addresses in the serving graph
//
// locationAddrsMap is a map:
// key: {cell,keyspace,shard,tabletType}
// value: topo.EndPoints (list of server records)
locationAddrsMap := make(map[cellKeyspaceShardType]*topo.EndPoints)
// we keep track of the existingDbTypeLocations we've already looked at
knownShardLocations := make(map[cellKeyspaceShard]bool)
for _, tablet := range tablets {
// only look at tablets in the cells we want to rebuild
if !topo.InCellList(tablet.Tablet.Alias.Cell, cells) {
continue
}
// this is {cell,keyspace,shard}
// we'll get the children to find the existing types
shardLocation := cellKeyspaceShard{tablet.Tablet.Alias.Cell, tablet.Tablet.Keyspace, tablet.Shard}
// only need to do this once per cell
if !knownShardLocations[shardLocation] {
log.Infof("Getting tablet types on cell %v for %v/%v", tablet.Tablet.Alias.Cell, tablet.Tablet.Keyspace, tablet.Shard)
tabletTypes, err := wr.ts.GetSrvTabletTypesPerShard(tablet.Tablet.Alias.Cell, tablet.Tablet.Keyspace, tablet.Shard)
if err != nil {
if err != topo.ErrNoNode {
return err
}
} else {
for _, tabletType := range tabletTypes {
existingDbTypeLocations[cellKeyspaceShardType{tablet.Tablet.Alias.Cell, tablet.Tablet.Keyspace, tablet.Shard, tabletType}] = true
}
}
knownShardLocations[shardLocation] = true
}
// Check IsInServingGraph after we have populated
// existingDbTypeLocations so we properly prune data
// if the definition of serving type changes.
if !tablet.IsInServingGraph() {
continue
}
location := cellKeyspaceShardType{tablet.Tablet.Alias.Cell, tablet.Keyspace, tablet.Shard, tablet.Type}
addrs, ok := locationAddrsMap[location]
if !ok {
addrs = topo.NewEndPoints()
locationAddrsMap[location] = addrs
}
entry, err := tabletmanager.EndPointForTablet(tablet.Tablet)
if err != nil {
log.Warningf("EndPointForTablet failed for tablet %v: %v", tablet.Alias, err)
continue
}
addrs.Entries = append(addrs.Entries, *entry)
}
// we're gonna parallelize a lot here
rec := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
// write all the {cell,keyspace,shard,type}
// nodes everywhere we want them
for location, addrs := range locationAddrsMap {
wg.Add(1)
go func(location cellKeyspaceShardType, addrs *topo.EndPoints) {
log.Infof("saving serving graph for cell %v shard %v/%v tabletType %v", location.cell, location.keyspace, location.shard, location.tabletType)
if err := wr.ts.UpdateEndPoints(location.cell, location.keyspace, location.shard, location.tabletType, addrs); err != nil {
rec.RecordError(fmt.Errorf("writing endpoints for cell %v shard %v/%v tabletType %v failed: %v", location.cell, location.keyspace, location.shard, location.tabletType, err))
}
wg.Done()
}(location, addrs)
}
// Delete any pre-existing paths that were not updated by this process.
// That's the existingDbTypeLocations - locationAddrsMap
for dbTypeLocation := range existingDbTypeLocations {
if _, ok := locationAddrsMap[dbTypeLocation]; !ok {
cell := dbTypeLocation.cell
if !topo.InCellList(cell, cells) {
continue
}
wg.Add(1)
go func(dbTypeLocation cellKeyspaceShardType) {
log.Infof("removing stale db type from serving graph: %v", dbTypeLocation)
if err := wr.ts.DeleteSrvTabletType(dbTypeLocation.cell, dbTypeLocation.keyspace, dbTypeLocation.shard, dbTypeLocation.tabletType); err != nil {
log.Warningf("unable to remove stale db type %v from serving graph: %v", dbTypeLocation, err)
}
wg.Done()
}(dbTypeLocation)
}
}
// wait until we're done with the background stuff to do the rest
// FIXME(alainjobart) this wouldn't be necessary if UpdateSrvShard
// below was creating the zookeeper nodes recursively.
wg.Wait()
if err := rec.Error(); err != nil {
return err
}
// Update per-shard information per cell-specific serving path.
//
// srvShardByPath is a map:
// key: {cell,keyspace,shard}
// value: topo.SrvShard
// this will create all the SrvShard objects
srvShardByPath := make(map[cellKeyspaceShard]*topo.SrvShard)
for location := range locationAddrsMap {
// location will be {cell,keyspace,shard,type}
srvShardPath := cellKeyspaceShard{location.cell, location.keyspace, location.shard}
srvShard, ok := srvShardByPath[srvShardPath]
if !ok {
srvShard = &topo.SrvShard{
KeyRange: shardInfo.KeyRange,
ServedTypes: shardInfo.ServedTypes,
TabletTypes: make([]topo.TabletType, 0, 2),
}
srvShardByPath[srvShardPath] = srvShard
}
foundType := false
for _, t := range srvShard.TabletTypes {
if t == location.tabletType {
foundType = true
}
}
if !foundType {
srvShard.TabletTypes = append(srvShard.TabletTypes, location.tabletType)
}
}
// Save the shard entries
for cks, srvShard := range srvShardByPath {
wg.Add(1)
go func(cks cellKeyspaceShard, srvShard *topo.SrvShard) {
log.Infof("updating shard serving graph in cell %v for %v/%v", cks.cell, cks.keyspace, cks.shard)
if err := wr.ts.UpdateSrvShard(cks.cell, cks.keyspace, cks.shard, srvShard); err != nil {
rec.RecordError(fmt.Errorf("writing serving data in cell %v for %v/%v failed: %v", cks.cell, cks.keyspace, cks.shard, err))
}
wg.Done()
}(cks, srvShard)
}
wg.Wait()
return rec.Error()
}
// changeCallback is run after every action that might
// have changed something in the tablet record.
func (agent *ActionAgent) changeCallback(ctx context.Context, oldTablet, newTablet *pbt.Tablet) error {
span := trace.NewSpanFromContext(ctx)
span.StartLocal("ActionAgent.changeCallback")
defer span.Finish()
allowQuery := topo.IsRunningQueryService(newTablet.Type)
// Read the shard to get SourceShards / TabletControlMap if
// we're going to use it.
var shardInfo *topo.ShardInfo
var tabletControl *pbt.Shard_TabletControl
var blacklistedTables []string
var err error
var disallowQueryReason string
if allowQuery {
shardInfo, err = agent.TopoServer.GetShard(ctx, newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err)
} else {
if newTablet.Type == pbt.TabletType_MASTER {
if len(shardInfo.SourceShards) > 0 {
allowQuery = false
disallowQueryReason = "old master is still in shard info"
}
}
if tc := shardInfo.GetTabletControl(newTablet.Type); tc != nil {
if topo.InCellList(newTablet.Alias.Cell, tc.Cells) {
if tc.DisableQueryService {
allowQuery = false
disallowQueryReason = "query service disabled by tablet control"
}
blacklistedTables = tc.BlacklistedTables
tabletControl = tc
}
}
}
} else {
disallowQueryReason = fmt.Sprintf("not a serving tablet type(%v)", newTablet.Type)
}
// Read the keyspace on masters to get ShardingColumnType,
// for binlog replication, only if source shards are set.
var keyspaceInfo *topo.KeyspaceInfo
if newTablet.Type == pbt.TabletType_MASTER && shardInfo != nil && len(shardInfo.SourceShards) > 0 {
keyspaceInfo, err = agent.TopoServer.GetKeyspace(ctx, newTablet.Keyspace)
if err != nil {
log.Errorf("Cannot read keyspace for this tablet %v: %v", newTablet.Alias, err)
keyspaceInfo = nil
}
}
if allowQuery {
// There are a few transitions when we need to restart the query service:
switch {
// If either InitMaster or InitSlave was called, because those calls
// (or a prior call to ResetReplication) may have silently broken the
// rowcache invalidator by executing RESET MASTER.
// Note that we don't care about fixing it after ResetReplication itself
// since that call breaks everything on purpose, and we don't expect
// anything to start working until either InitMaster or InitSlave.
case agent.initReplication:
agent.initReplication = false
agent.stopQueryService("initialize replication")
// Transitioning from replica to master, so clients that were already
// connected don't keep on using the master as replica or rdonly.
case newTablet.Type == pbt.TabletType_MASTER && oldTablet.Type != pbt.TabletType_MASTER:
agent.stopQueryService("tablet promoted to master")
// Having different parameters for the query service.
// It needs to stop and restart with the new parameters.
// That includes:
// - changing KeyRange
// - changing the BlacklistedTables list
case (newTablet.KeyRange != oldTablet.KeyRange),
!reflect.DeepEqual(blacklistedTables, agent.BlacklistedTables()):
agent.stopQueryService("keyrange/blacklistedtables changed")
}
if err := agent.allowQueries(newTablet, blacklistedTables); err != nil {
log.Errorf("Cannot start query service: %v", err)
}
} else {
agent.stopQueryService(disallowQueryReason)
}
// save the tabletControl we've been using, so the background
// healthcheck makes the same decisions as we've been making.
agent.setTabletControl(tabletControl)
// update stream needs to be started or stopped too
if agent.DBConfigs != nil {
if topo.IsRunningUpdateStream(newTablet.Type) {
binlog.EnableUpdateStreamService(agent.DBConfigs.App.DbName, agent.MysqlDaemon)
} else {
binlog.DisableUpdateStreamService()
}
}
statsType.Set(strings.ToLower(newTablet.Type.String()))
statsKeyspace.Set(newTablet.Keyspace)
statsShard.Set(newTablet.Shard)
if newTablet.KeyRange != nil {
statsKeyRangeStart.Set(hex.EncodeToString(newTablet.KeyRange.Start))
statsKeyRangeEnd.Set(hex.EncodeToString(newTablet.KeyRange.End))
} else {
statsKeyRangeStart.Set("")
statsKeyRangeEnd.Set("")
}
// See if we need to start or stop any binlog player
if agent.BinlogPlayerMap != nil {
if newTablet.Type == pbt.TabletType_MASTER {
agent.BinlogPlayerMap.RefreshMap(agent.batchCtx, newTablet, keyspaceInfo, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
}
return nil
}
// changeCallback is run after every action that might
// have changed something in the tablet record or in the topology.
//
// It owns making changes to the BinlogPlayerMap. The input for this is the
// tablet type (has to be master), and the shard's SourceShards.
//
// It owns updating the blacklisted tables.
//
// It owns updating the stats record for 'TabletType'.
//
// It owns starting and stopping the update stream service.
//
// It owns reading the TabletControl for the current tablet, and storing it.
func (agent *ActionAgent) changeCallback(ctx context.Context, oldTablet, newTablet *topodatapb.Tablet) {
span := trace.NewSpanFromContext(ctx)
span.StartLocal("ActionAgent.changeCallback")
defer span.Finish()
allowQuery := topo.IsRunningQueryService(newTablet.Type)
broadcastHealth := false
runUpdateStream := allowQuery
// Read the shard to get SourceShards / TabletControlMap if
// we're going to use it.
var shardInfo *topo.ShardInfo
var err error
var disallowQueryReason string
var blacklistedTables []string
updateBlacklistedTables := true
if allowQuery {
shardInfo, err = agent.TopoServer.GetShard(ctx, newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err)
updateBlacklistedTables = false
} else {
if newTablet.Type == topodatapb.TabletType_MASTER {
if len(shardInfo.SourceShards) > 0 {
allowQuery = false
disallowQueryReason = "master tablet with filtered replication on"
}
}
if tc := shardInfo.GetTabletControl(newTablet.Type); tc != nil {
if topo.InCellList(newTablet.Alias.Cell, tc.Cells) {
if tc.DisableQueryService {
allowQuery = false
disallowQueryReason = "TabletControl.DisableQueryService set"
}
blacklistedTables = tc.BlacklistedTables
}
}
}
} else {
disallowQueryReason = fmt.Sprintf("not a serving tablet type(%v)", newTablet.Type)
}
agent.setServicesDesiredState(disallowQueryReason, runUpdateStream)
if updateBlacklistedTables {
if err := agent.loadBlacklistRules(newTablet, blacklistedTables); err != nil {
// FIXME(alainjobart) how to handle this error?
log.Errorf("Cannot update blacklisted tables rule: %v", err)
} else {
agent.setBlacklistedTables(blacklistedTables)
}
}
if allowQuery {
// Query service should be running.
if oldTablet.Type == topodatapb.TabletType_REPLICA &&
newTablet.Type == topodatapb.TabletType_MASTER {
// When promoting from replica to master, allow both master and replica
// queries to be served during gracePeriod.
if _, err := agent.QueryServiceControl.SetServingType(newTablet.Type,
true, []topodatapb.TabletType{oldTablet.Type}); err == nil {
// If successful, broadcast to vtgate and then wait.
agent.broadcastHealth()
time.Sleep(*gracePeriod)
} else {
log.Errorf("Can't start query service for MASTER+REPLICA mode: %v", err)
}
}
if stateChanged, err := agent.QueryServiceControl.SetServingType(newTablet.Type, true, nil); err == nil {
// If the state changed, broadcast to vtgate.
// (e.g. this happens when the tablet was already master, but it just
// changed from NOT_SERVING to SERVING due to
// "vtctl MigrateServedFrom ... master".)
if stateChanged {
broadcastHealth = true
}
} else {
runUpdateStream = false
log.Errorf("Cannot start query service: %v", err)
}
} else {
// Query service should be stopped.
if topo.IsSubjectToLameduck(oldTablet.Type) &&
newTablet.Type == topodatapb.TabletType_SPARE &&
*gracePeriod > 0 {
// When a non-MASTER serving type is going SPARE,
// put query service in lameduck during gracePeriod.
agent.lameduck(disallowQueryReason)
}
log.Infof("Disabling query service on type change, reason: %v", disallowQueryReason)
if stateChanged, err := agent.QueryServiceControl.SetServingType(newTablet.Type, false, nil); err == nil {
// If the state changed, broadcast to vtgate.
// (e.g. this happens when the tablet was already master, but it just
// changed from SERVING to NOT_SERVING because filtered replication was
// enabled.)
if stateChanged {
broadcastHealth = true
}
} else {
log.Errorf("SetServingType(serving=false) failed: %v", err)
}
}
// update stream needs to be started or stopped too
if topo.IsRunningUpdateStream(newTablet.Type) && runUpdateStream {
agent.UpdateStream.Enable()
} else {
agent.UpdateStream.Disable()
}
// upate the stats to our current type
if agent.exportStats {
agent.statsTabletType.Set(topoproto.TabletTypeLString(newTablet.Type))
}
// See if we need to start or stop any binlog player
if agent.BinlogPlayerMap != nil {
if newTablet.Type == topodatapb.TabletType_MASTER {
agent.BinlogPlayerMap.RefreshMap(agent.batchCtx, newTablet, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
}
// Broadcast health changes to vtgate immediately.
if broadcastHealth {
agent.broadcastHealth()
}
}
// changeCallback is run after every action that might
// have changed something in the tablet record or in the topology.
func (agent *ActionAgent) changeCallback(ctx context.Context, oldTablet, newTablet *pbt.Tablet) error {
span := trace.NewSpanFromContext(ctx)
span.StartLocal("ActionAgent.changeCallback")
defer span.Finish()
allowQuery := topo.IsRunningQueryService(newTablet.Type)
// Read the shard to get SourceShards / TabletControlMap if
// we're going to use it.
var shardInfo *topo.ShardInfo
var tabletControl *pbt.Shard_TabletControl
var blacklistedTables []string
var err error
var disallowQueryReason string
if allowQuery {
shardInfo, err = agent.TopoServer.GetShard(ctx, newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err)
} else {
if newTablet.Type == pbt.TabletType_MASTER {
if len(shardInfo.SourceShards) > 0 {
allowQuery = false
disallowQueryReason = "master tablet with filtered replication on"
}
}
if tc := shardInfo.GetTabletControl(newTablet.Type); tc != nil {
if topo.InCellList(newTablet.Alias.Cell, tc.Cells) {
if tc.DisableQueryService {
allowQuery = false
disallowQueryReason = "query service disabled by tablet control"
}
blacklistedTables = tc.BlacklistedTables
tabletControl = tc
}
}
}
} else {
disallowQueryReason = fmt.Sprintf("not a serving tablet type(%v)", newTablet.Type)
}
if allowQuery {
if err := agent.allowQueries(newTablet, blacklistedTables); err != nil {
log.Errorf("Cannot start query service: %v", err)
}
} else {
agent.disallowQueries(newTablet, disallowQueryReason)
}
// save the tabletControl we've been using, so the background
// healthcheck makes the same decisions as we've been making.
agent.setTabletControl(tabletControl)
// update stream needs to be started or stopped too
if topo.IsRunningUpdateStream(newTablet.Type) {
agent.UpdateStream.Enable()
} else {
agent.UpdateStream.Disable()
}
statsType.Set(strings.ToLower(newTablet.Type.String()))
statsKeyspace.Set(newTablet.Keyspace)
statsShard.Set(newTablet.Shard)
if newTablet.KeyRange != nil {
statsKeyRangeStart.Set(hex.EncodeToString(newTablet.KeyRange.Start))
statsKeyRangeEnd.Set(hex.EncodeToString(newTablet.KeyRange.End))
} else {
statsKeyRangeStart.Set("")
statsKeyRangeEnd.Set("")
}
// See if we need to start or stop any binlog player
if agent.BinlogPlayerMap != nil {
if newTablet.Type == pbt.TabletType_MASTER {
// Read the keyspace on masters to get
// ShardingColumnType, for binlog replication,
// only if source shards are set.
var keyspaceInfo *topo.KeyspaceInfo
if shardInfo != nil && len(shardInfo.SourceShards) > 0 {
keyspaceInfo, err = agent.TopoServer.GetKeyspace(ctx, newTablet.Keyspace)
if err != nil {
keyspaceInfo = nil
}
}
agent.BinlogPlayerMap.RefreshMap(agent.batchCtx, newTablet, keyspaceInfo, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
}
return nil
}
// Update shard file with new master, replicas, etc.
//
// Re-read from TopologyServer to make sure we are using the side
// effects of all actions.
//
// This function locks individual SvrShard paths, so it doesn't need a lock
// on the shard.
func RebuildShard(log logutil.Logger, ts topo.Server, keyspace, shard string, cells []string, timeout time.Duration, interrupted chan struct{}) error {
log.Infof("RebuildShard %v/%v", keyspace, shard)
// read the existing shard info. It has to exist.
shardInfo, err := ts.GetShard(keyspace, shard)
if err != nil {
return err
}
// rebuild all cells in parallel
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, cell := range shardInfo.Cells {
// skip this cell if we shouldn't rebuild it
if !topo.InCellList(cell, cells) {
continue
}
// start with the master if it's in the current cell
tabletsAsMap := make(map[topo.TabletAlias]bool)
if shardInfo.MasterAlias.Cell == cell {
tabletsAsMap[shardInfo.MasterAlias] = true
}
wg.Add(1)
go func(cell string) {
defer wg.Done()
// read the ShardReplication object to find tablets
sri, err := ts.GetShardReplication(cell, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardReplication(%v, %v, %v) failed: %v", cell, keyspace, shard, err))
return
}
// add all relevant tablets to the map
for _, rl := range sri.ReplicationLinks {
tabletsAsMap[rl.TabletAlias] = true
if rl.Parent.Cell == cell {
tabletsAsMap[rl.Parent] = true
}
}
// convert the map to a list
aliases := make([]topo.TabletAlias, 0, len(tabletsAsMap))
for a := range tabletsAsMap {
aliases = append(aliases, a)
}
// read all the Tablet records
tablets, err := topo.GetTabletMap(ts, aliases)
switch err {
case nil:
// keep going, we're good
case topo.ErrPartialResult:
log.Warningf("Got ErrPartialResult from topo.GetTabletMap in cell %v, some tablets may not be added properly to serving graph", cell)
default:
rec.RecordError(fmt.Errorf("GetTabletMap in cell %v failed: %v", cell, err))
return
}
// Lock the SrvShard so we write a consistent data set.
actionNode := actionnode.RebuildSrvShard()
lockPath, err := actionNode.LockSrvShard(ts, cell, keyspace, shard, timeout, interrupted)
if err != nil {
rec.RecordError(err)
return
}
// write the data we need to
rebuildErr := rebuildCellSrvShard(log, ts, shardInfo, cell, tablets)
// and unlock
if err := actionNode.UnlockSrvShard(ts, cell, keyspace, shard, lockPath, rebuildErr); err != nil {
rec.RecordError(err)
}
}(cell)
}
wg.Wait()
return rec.Error()
}
// RemoveShardCell will remove a cell from the Cells list in a shard.
//
// It will first check the shard has no tablets there. If 'force' is
// specified, it will remove the cell even when the tablet map cannot
// be retrieved. This is intended to be used when a cell is completely
// down and its topology server cannot even be reached.
//
// If 'recursive' is specified, it will delete any tablets in the cell/shard,
// with the assumption that the tablet processes have already been terminated.
func (wr *Wrangler) RemoveShardCell(ctx context.Context, keyspace, shard, cell string, force, recursive bool) error {
shardInfo, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return err
}
// check the cell is in the list already
if !topo.InCellList(cell, shardInfo.Cells) {
return fmt.Errorf("cell %v in not in shard info", cell)
}
// check the master alias is not in the cell
if shardInfo.MasterAlias != nil && shardInfo.MasterAlias.Cell == cell {
return fmt.Errorf("master %v is in the cell '%v' we want to remove", topoproto.TabletAliasString(shardInfo.MasterAlias), cell)
}
// get the ShardReplication object in the cell
sri, err := wr.ts.GetShardReplication(ctx, cell, keyspace, shard)
switch err {
case nil:
if recursive {
wr.Logger().Infof("Deleting all tablets in shard %v/%v", keyspace, shard)
for _, node := range sri.Nodes {
// We don't care about scrapping or updating the replication graph,
// because we're about to delete the entire replication graph.
wr.Logger().Infof("Deleting tablet %v", topoproto.TabletAliasString(node.TabletAlias))
if err := wr.TopoServer().DeleteTablet(ctx, node.TabletAlias); err != nil && err != topo.ErrNoNode {
return fmt.Errorf("can't delete tablet %v: %v", topoproto.TabletAliasString(node.TabletAlias), err)
}
}
} else if len(sri.Nodes) > 0 {
return fmt.Errorf("cell %v has %v possible tablets in replication graph", cell, len(sri.Nodes))
}
// ShardReplication object is now useless, remove it
if err := wr.ts.DeleteShardReplication(ctx, cell, keyspace, shard); err != nil && err != topo.ErrNoNode {
return fmt.Errorf("error deleting ShardReplication object in cell %v: %v", cell, err)
}
// we keep going
case topo.ErrNoNode:
// no ShardReplication object, we keep going
default:
// we can't get the object, assume topo server is down there,
// so we look at force flag
if !force {
return err
}
wr.Logger().Warningf("Cannot get ShardReplication from cell %v, assuming cell topo server is down, and forcing the removal", cell)
}
// now we can update the shard
wr.Logger().Infof("Removing cell %v from shard %v/%v", cell, keyspace, shard)
_, err = wr.ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error {
// since no lock is taken, protect against corner cases.
if len(si.Cells) == 0 {
return topo.ErrNoUpdateNeeded
}
var newCells []string
for _, c := range si.Cells {
if c != cell {
newCells = append(newCells, c)
}
}
si.Cells = newCells
return nil
})
return err
}
// changeCallback is run after every action that might
// have changed something in the tablet record or in the topology.
//
// It owns making changes to the BinlogPlayerMap. The input for this is the
// tablet type (has to be master), and the shard's SourceShards.
//
// It owns updating the blacklisted tables.
//
// It owns updating the stats record for 'TabletType'.
//
// It owns starting and stopping the update stream service.
//
// It owns reading the TabletControl for the current tablet, and storing it.
func (agent *ActionAgent) changeCallback(ctx context.Context, oldTablet, newTablet *topodatapb.Tablet) error {
span := trace.NewSpanFromContext(ctx)
span.StartLocal("ActionAgent.changeCallback")
defer span.Finish()
allowQuery := topo.IsRunningQueryService(newTablet.Type)
// Read the shard to get SourceShards / TabletControlMap if
// we're going to use it.
var shardInfo *topo.ShardInfo
var tabletControl *topodatapb.Shard_TabletControl
var err error
var disallowQueryReason string
var blacklistedTables []string
updateBlacklistedTables := true
if allowQuery {
shardInfo, err = agent.TopoServer.GetShard(ctx, newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err)
updateBlacklistedTables = false
} else {
if newTablet.Type == topodatapb.TabletType_MASTER {
if len(shardInfo.SourceShards) > 0 {
allowQuery = false
disallowQueryReason = "master tablet with filtered replication on"
}
}
if tc := shardInfo.GetTabletControl(newTablet.Type); tc != nil {
if topo.InCellList(newTablet.Alias.Cell, tc.Cells) {
if tc.DisableQueryService {
allowQuery = false
disallowQueryReason = "query service disabled by tablet control"
}
blacklistedTables = tc.BlacklistedTables
tabletControl = tc
}
}
}
} else {
disallowQueryReason = fmt.Sprintf("not a serving tablet type(%v)", newTablet.Type)
}
if updateBlacklistedTables {
if err := agent.loadBlacklistRules(newTablet, blacklistedTables); err != nil {
// FIXME(alainjobart) how to handle this error?
log.Errorf("Cannot update blacklisted tables rule: %v", err)
}
}
if allowQuery {
// Query service should be running.
if oldTablet.Type == topodatapb.TabletType_REPLICA &&
newTablet.Type == topodatapb.TabletType_MASTER {
// When promoting from replica to master, allow both master and replica
// queries to be served during gracePeriod.
if err := agent.QueryServiceControl.SetServingType(newTablet.Type, true,
[]topodatapb.TabletType{oldTablet.Type}); err != nil {
log.Errorf("Can't start query service for MASTER+REPLICA mode: %v", err)
} else {
// If successful, broadcast to vtgate and then wait.
agent.broadcastHealth()
time.Sleep(*gracePeriod)
}
}
if err := agent.allowQueries(newTablet.Type); err != nil {
log.Errorf("Cannot start query service: %v", err)
}
} else {
// Query service should be stopped.
if (oldTablet.Type == topodatapb.TabletType_REPLICA ||
oldTablet.Type == topodatapb.TabletType_RDONLY) &&
newTablet.Type == topodatapb.TabletType_SPARE {
// When a non-MASTER serving type is going SPARE,
// put query service in lameduck during gracePeriod.
agent.enterLameduck(disallowQueryReason)
agent.broadcastHealth()
time.Sleep(*gracePeriod)
}
agent.disallowQueries(newTablet.Type, disallowQueryReason)
}
// save the tabletControl we've been using, so the background
// healthcheck makes the same decisions as we've been making.
agent.setTabletControl(tabletControl)
// update stream needs to be started or stopped too
if topo.IsRunningUpdateStream(newTablet.Type) {
agent.UpdateStream.Enable()
} else {
agent.UpdateStream.Disable()
}
// upate the stats to our current type
if agent.exportStats {
agent.statsTabletType.Set(strings.ToLower(newTablet.Type.String()))
}
// See if we need to start or stop any binlog player
if agent.BinlogPlayerMap != nil {
if newTablet.Type == topodatapb.TabletType_MASTER {
agent.BinlogPlayerMap.RefreshMap(agent.batchCtx, newTablet, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
}
return nil
}