// changeCallback is run after every action that might
// have changed something in the tablet record.
func (agent *ActionAgent) changeCallback(oldTablet, newTablet topo.Tablet) {
allowQuery := true
var shardInfo *topo.ShardInfo
var keyspaceInfo *topo.KeyspaceInfo
if newTablet.Type == topo.TYPE_MASTER {
// read the shard to get SourceShards
var err error
shardInfo, err = agent.TopoServer.GetShard(newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v: %v", newTablet.Alias, err)
} else {
allowQuery = len(shardInfo.SourceShards) == 0
}
// read the keyspace to get ShardingColumnType
keyspaceInfo, err = agent.TopoServer.GetKeyspace(newTablet.Keyspace)
if err != nil {
log.Errorf("Cannot read keyspace for this tablet %v: %v", newTablet.Alias, err)
keyspaceInfo = nil
}
}
if newTablet.IsRunningQueryService() && allowQuery {
// There are a few transitions when we're
// going to need to restart the query service:
// - transitioning from replica to master, so clients
// that were already connected don't keep on using
// the master as replica or rdonly.
// - having different parameters for the query
// service. It needs to stop and restart with the
// new parameters. That includes:
// - changing KeyRange
// - changing the BlacklistedTables list
if (newTablet.Type == topo.TYPE_MASTER &&
oldTablet.Type != topo.TYPE_MASTER) ||
(newTablet.KeyRange != oldTablet.KeyRange) ||
!reflect.DeepEqual(newTablet.BlacklistedTables, oldTablet.BlacklistedTables) {
agent.disallowQueries()
}
if err := agent.allowQueries(&newTablet); err != nil {
log.Errorf("Cannot start query service: %v", err)
}
// Disable before enabling to force existing streams to stop.
if agent.DBConfigs != nil {
binlog.DisableUpdateStreamService()
binlog.EnableUpdateStreamService(agent.DBConfigs.App.DbName, agent.Mysqld)
}
} else {
agent.disallowQueries()
if agent.DBConfigs != nil {
binlog.DisableUpdateStreamService()
}
}
statsType.Set(string(newTablet.Type))
statsKeyspace.Set(newTablet.Keyspace)
statsShard.Set(newTablet.Shard)
statsKeyRangeStart.Set(string(newTablet.KeyRange.Start.Hex()))
statsKeyRangeEnd.Set(string(newTablet.KeyRange.End.Hex()))
// See if we need to start or stop any binlog player
if agent.BinlogPlayerMap != nil {
if newTablet.Type == topo.TYPE_MASTER {
agent.BinlogPlayerMap.RefreshMap(newTablet, keyspaceInfo, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
}
}
// changeCallback is run after every action that might
// have changed something in the tablet record.
func (agent *ActionAgent) changeCallback(oldTablet, newTablet *topo.Tablet) error {
allowQuery := newTablet.IsRunningQueryService()
// Read the shard to get SourceShards / TabletControlMap if
// we're going to use it.
var shardInfo *topo.ShardInfo
var tabletControl *topo.TabletControl
var blacklistedTables []string
var err error
if allowQuery {
shardInfo, err = agent.TopoServer.GetShard(newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err)
} else {
if newTablet.Type == topo.TYPE_MASTER {
allowQuery = len(shardInfo.SourceShards) == 0
}
if tc, ok := shardInfo.TabletControlMap[newTablet.Type]; ok {
if topo.InCellList(newTablet.Alias.Cell, tc.Cells) {
if tc.DisableQueryService {
allowQuery = false
}
blacklistedTables = tc.BlacklistedTables
tabletControl = tc
}
}
}
}
// Read the keyspace on masters to get ShardingColumnType,
// for binlog replication, only if source shards are set.
var keyspaceInfo *topo.KeyspaceInfo
if newTablet.Type == topo.TYPE_MASTER && shardInfo != nil && len(shardInfo.SourceShards) > 0 {
keyspaceInfo, err = agent.TopoServer.GetKeyspace(newTablet.Keyspace)
if err != nil {
log.Errorf("Cannot read keyspace for this tablet %v: %v", newTablet.Alias, err)
keyspaceInfo = nil
}
}
if allowQuery {
// There are a few transitions when we're
// going to need to restart the query service:
// - transitioning from replica to master, so clients
// that were already connected don't keep on using
// the master as replica or rdonly.
// - having different parameters for the query
// service. It needs to stop and restart with the
// new parameters. That includes:
// - changing KeyRange
// - changing the BlacklistedTables list
if (newTablet.Type == topo.TYPE_MASTER &&
oldTablet.Type != topo.TYPE_MASTER) ||
(newTablet.KeyRange != oldTablet.KeyRange) ||
!reflect.DeepEqual(blacklistedTables, agent.BlacklistedTables()) {
agent.disallowQueries()
}
if err := agent.allowQueries(newTablet, blacklistedTables); err != nil {
log.Errorf("Cannot start query service: %v", err)
}
} else {
agent.disallowQueries()
}
// save the tabletControl we've been using, so the background
// healthcheck makes the same decisions as we've been making.
agent.setTabletControl(tabletControl)
// update stream needs to be started or stopped too
if agent.DBConfigs != nil {
if topo.IsRunningUpdateStream(newTablet.Type) {
binlog.EnableUpdateStreamService(agent.DBConfigs.App.DbName, agent.Mysqld)
} else {
binlog.DisableUpdateStreamService()
}
}
statsType.Set(string(newTablet.Type))
statsKeyspace.Set(newTablet.Keyspace)
statsShard.Set(newTablet.Shard)
statsKeyRangeStart.Set(string(newTablet.KeyRange.Start.Hex()))
statsKeyRangeEnd.Set(string(newTablet.KeyRange.End.Hex()))
// See if we need to start or stop any binlog player
if agent.BinlogPlayerMap != nil {
if newTablet.Type == topo.TYPE_MASTER {
agent.BinlogPlayerMap.RefreshMap(newTablet, keyspaceInfo, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
}
return nil
}
// changeCallback is run after every action that might
// have changed something in the tablet record.
func (agent *ActionAgent) changeCallback(ctx context.Context, oldTablet, newTablet *pbt.Tablet) error {
span := trace.NewSpanFromContext(ctx)
span.StartLocal("ActionAgent.changeCallback")
defer span.Finish()
allowQuery := topo.IsRunningQueryService(newTablet.Type)
// Read the shard to get SourceShards / TabletControlMap if
// we're going to use it.
var shardInfo *topo.ShardInfo
var tabletControl *pbt.Shard_TabletControl
var blacklistedTables []string
var err error
var disallowQueryReason string
if allowQuery {
shardInfo, err = agent.TopoServer.GetShard(ctx, newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err)
} else {
if newTablet.Type == pbt.TabletType_MASTER {
if len(shardInfo.SourceShards) > 0 {
allowQuery = false
disallowQueryReason = "old master is still in shard info"
}
}
if tc := shardInfo.GetTabletControl(newTablet.Type); tc != nil {
if topo.InCellList(newTablet.Alias.Cell, tc.Cells) {
if tc.DisableQueryService {
allowQuery = false
disallowQueryReason = "query service disabled by tablet control"
}
blacklistedTables = tc.BlacklistedTables
tabletControl = tc
}
}
}
} else {
disallowQueryReason = fmt.Sprintf("not a serving tablet type(%v)", newTablet.Type)
}
// Read the keyspace on masters to get ShardingColumnType,
// for binlog replication, only if source shards are set.
var keyspaceInfo *topo.KeyspaceInfo
if newTablet.Type == pbt.TabletType_MASTER && shardInfo != nil && len(shardInfo.SourceShards) > 0 {
keyspaceInfo, err = agent.TopoServer.GetKeyspace(ctx, newTablet.Keyspace)
if err != nil {
log.Errorf("Cannot read keyspace for this tablet %v: %v", newTablet.Alias, err)
keyspaceInfo = nil
}
}
if allowQuery {
// There are a few transitions when we need to restart the query service:
switch {
// If either InitMaster or InitSlave was called, because those calls
// (or a prior call to ResetReplication) may have silently broken the
// rowcache invalidator by executing RESET MASTER.
// Note that we don't care about fixing it after ResetReplication itself
// since that call breaks everything on purpose, and we don't expect
// anything to start working until either InitMaster or InitSlave.
case agent.initReplication:
agent.initReplication = false
agent.stopQueryService("initialize replication")
// Transitioning from replica to master, so clients that were already
// connected don't keep on using the master as replica or rdonly.
case newTablet.Type == pbt.TabletType_MASTER && oldTablet.Type != pbt.TabletType_MASTER:
agent.stopQueryService("tablet promoted to master")
// Having different parameters for the query service.
// It needs to stop and restart with the new parameters.
// That includes:
// - changing KeyRange
// - changing the BlacklistedTables list
case (newTablet.KeyRange != oldTablet.KeyRange),
!reflect.DeepEqual(blacklistedTables, agent.BlacklistedTables()):
agent.stopQueryService("keyrange/blacklistedtables changed")
}
if err := agent.allowQueries(newTablet, blacklistedTables); err != nil {
log.Errorf("Cannot start query service: %v", err)
}
} else {
agent.stopQueryService(disallowQueryReason)
}
// save the tabletControl we've been using, so the background
// healthcheck makes the same decisions as we've been making.
agent.setTabletControl(tabletControl)
// update stream needs to be started or stopped too
if agent.DBConfigs != nil {
if topo.IsRunningUpdateStream(newTablet.Type) {
binlog.EnableUpdateStreamService(agent.DBConfigs.App.DbName, agent.MysqlDaemon)
} else {
binlog.DisableUpdateStreamService()
}
}
statsType.Set(strings.ToLower(newTablet.Type.String()))
statsKeyspace.Set(newTablet.Keyspace)
statsShard.Set(newTablet.Shard)
if newTablet.KeyRange != nil {
statsKeyRangeStart.Set(hex.EncodeToString(newTablet.KeyRange.Start))
statsKeyRangeEnd.Set(hex.EncodeToString(newTablet.KeyRange.End))
} else {
statsKeyRangeStart.Set("")
statsKeyRangeEnd.Set("")
}
// See if we need to start or stop any binlog player
if agent.BinlogPlayerMap != nil {
if newTablet.Type == pbt.TabletType_MASTER {
agent.BinlogPlayerMap.RefreshMap(agent.batchCtx, newTablet, keyspaceInfo, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
}
return nil
}
// InitAgent initializes the agent within vttablet.
func InitAgent(
tabletAlias topo.TabletAlias,
dbcfgs *dbconfigs.DBConfigs,
mycnf *mysqlctl.Mycnf,
port, securePort int,
overridesFile string,
) (agent *tabletmanager.ActionAgent, err error) {
schemaOverrides := loadSchemaOverrides(overridesFile)
topoServer := topo.GetServer()
mysqld := mysqlctl.NewMysqld(mycnf, &dbcfgs.Dba, &dbcfgs.Repl)
statsType := stats.NewString("TabletType")
statsKeyspace := stats.NewString("TabletKeyspace")
statsShard := stats.NewString("TabletShard")
statsKeyRangeStart := stats.NewString("TabletKeyRangeStart")
statsKeyRangeEnd := stats.NewString("TabletKeyRangeEnd")
agent, err = tabletmanager.NewActionAgent(topoServer, tabletAlias, mysqld)
if err != nil {
return nil, err
}
// Start the binlog player services, not playing at start.
agent.BinlogPlayerMap = tabletmanager.NewBinlogPlayerMap(topoServer, &dbcfgs.App.ConnectionParams, mysqld)
tabletmanager.RegisterBinlogPlayerMap(agent.BinlogPlayerMap)
// Action agent listens to changes in zookeeper and makes
// modifications to this tablet.
agent.AddChangeCallback(func(oldTablet, newTablet topo.Tablet) {
allowQuery := true
var shardInfo *topo.ShardInfo
var keyspaceInfo *topo.KeyspaceInfo
if newTablet.Type == topo.TYPE_MASTER {
// read the shard to get SourceShards
shardInfo, err = topoServer.GetShard(newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v: %v", newTablet.Alias, err)
} else {
allowQuery = len(shardInfo.SourceShards) == 0
}
// read the keyspace to get ShardingColumnType
keyspaceInfo, err = topoServer.GetKeyspace(newTablet.Keyspace)
switch err {
case nil:
// continue
case topo.ErrNoNode:
// backward compatible mode
keyspaceInfo = topo.NewKeyspaceInfo(newTablet.Keyspace, &topo.Keyspace{})
default:
log.Errorf("Cannot read keyspace for this tablet %v: %v", newTablet.Alias, err)
keyspaceInfo = nil
}
}
if newTablet.IsRunningQueryService() && allowQuery {
if dbcfgs.App.DbName == "" {
dbcfgs.App.DbName = newTablet.DbName()
}
dbcfgs.App.Keyspace = newTablet.Keyspace
dbcfgs.App.Shard = newTablet.Shard
if newTablet.Type != topo.TYPE_MASTER {
dbcfgs.App.EnableInvalidator = true
} else {
dbcfgs.App.EnableInvalidator = false
}
// Transitioning from replica to master, first disconnect
// existing connections. "false" indicateds that clients must
// re-resolve their endpoint before reconnecting.
if newTablet.Type == topo.TYPE_MASTER && oldTablet.Type != topo.TYPE_MASTER {
ts.DisallowQueries()
}
qrs := ts.LoadCustomRules()
if newTablet.KeyRange.IsPartial() {
qr := ts.NewQueryRule("enforce keyspace_id range", "keyspace_id_not_in_range", ts.QR_FAIL_QUERY)
qr.AddPlanCond(sqlparser.PLAN_INSERT_PK)
err = qr.AddBindVarCond("keyspace_id", true, true, ts.QR_NOTIN, newTablet.KeyRange)
if err != nil {
log.Warningf("Unable to add keyspace rule: %v", err)
} else {
qrs.Add(qr)
}
}
ts.AllowQueries(&dbcfgs.App, schemaOverrides, qrs, mysqld)
// Disable before enabling to force existing streams to stop.
binlog.DisableUpdateStreamService()
binlog.EnableUpdateStreamService(dbcfgs)
} else {
ts.DisallowQueries()
binlog.DisableUpdateStreamService()
}
statsType.Set(string(newTablet.Type))
statsKeyspace.Set(newTablet.Keyspace)
statsShard.Set(newTablet.Shard)
statsKeyRangeStart.Set(string(newTablet.KeyRange.Start.Hex()))
statsKeyRangeEnd.Set(string(newTablet.KeyRange.End.Hex()))
// See if we need to start or stop any binlog player
if newTablet.Type == topo.TYPE_MASTER {
agent.BinlogPlayerMap.RefreshMap(newTablet, keyspaceInfo, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
})
if err := agent.Start(mysqld.Port(), port, securePort); err != nil {
return nil, err
}
// register the RPC services from the agent
agent.RegisterQueryService()
return agent, nil
}
// changeCallback is run after every action that might
// have changed something in the tablet record.
func (agent *ActionAgent) changeCallback(ctx context.Context, oldTablet, newTablet *pbt.Tablet) error {
span := trace.NewSpanFromContext(ctx)
span.StartLocal("ActionAgent.changeCallback")
defer span.Finish()
allowQuery := topo.IsRunningQueryService(newTablet.Type)
// Read the shard to get SourceShards / TabletControlMap if
// we're going to use it.
var shardInfo *topo.ShardInfo
var tabletControl *pbt.Shard_TabletControl
var blacklistedTables []string
var err error
var disallowQueryReason string
if allowQuery {
shardInfo, err = agent.TopoServer.GetShard(ctx, newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err)
} else {
if newTablet.Type == pbt.TabletType_MASTER {
if len(shardInfo.SourceShards) > 0 {
allowQuery = false
disallowQueryReason = "old master is still in shard info"
}
}
if tc := shardInfo.GetTabletControl(newTablet.Type); tc != nil {
if topo.InCellList(newTablet.Alias.Cell, tc.Cells) {
if tc.DisableQueryService {
allowQuery = false
disallowQueryReason = "query service disabled by tablet control"
}
blacklistedTables = tc.BlacklistedTables
tabletControl = tc
}
}
}
} else {
disallowQueryReason = fmt.Sprintf("not a serving tablet type(%v)", newTablet.Type)
}
if allowQuery {
if err := agent.allowQueries(newTablet, blacklistedTables); err != nil {
log.Errorf("Cannot start query service: %v", err)
}
} else {
agent.disallowQueries(newTablet, disallowQueryReason)
}
// save the tabletControl we've been using, so the background
// healthcheck makes the same decisions as we've been making.
agent.setTabletControl(tabletControl)
// update stream needs to be started or stopped too
if topo.IsRunningUpdateStream(newTablet.Type) {
binlog.EnableUpdateStreamService(agent.DBConfigs.App.DbName, agent.MysqlDaemon)
} else {
binlog.DisableUpdateStreamService()
}
statsType.Set(strings.ToLower(newTablet.Type.String()))
statsKeyspace.Set(newTablet.Keyspace)
statsShard.Set(newTablet.Shard)
if newTablet.KeyRange != nil {
statsKeyRangeStart.Set(hex.EncodeToString(newTablet.KeyRange.Start))
statsKeyRangeEnd.Set(hex.EncodeToString(newTablet.KeyRange.End))
} else {
statsKeyRangeStart.Set("")
statsKeyRangeEnd.Set("")
}
// See if we need to start or stop any binlog player
if agent.BinlogPlayerMap != nil {
if newTablet.Type == pbt.TabletType_MASTER {
// Read the keyspace on masters to get
// ShardingColumnType, for binlog replication,
// only if source shards are set.
var keyspaceInfo *topo.KeyspaceInfo
if shardInfo != nil && len(shardInfo.SourceShards) > 0 {
keyspaceInfo, err = agent.TopoServer.GetKeyspace(ctx, newTablet.Keyspace)
if err != nil {
keyspaceInfo = nil
}
}
agent.BinlogPlayerMap.RefreshMap(agent.batchCtx, newTablet, keyspaceInfo, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
}
return nil
}
// InitAgent initializes the agent within vttablet.
func InitAgent(
tabletAlias topo.TabletAlias,
dbcfgs *dbconfigs.DBConfigs,
mycnf *mysqlctl.Mycnf,
port, securePort int,
overridesFile string,
) (agent *tabletmanager.ActionAgent, err error) {
schemaOverrides := loadSchemaOverrides(overridesFile)
topoServer := topo.GetServer()
mysqld := mysqlctl.NewMysqld(mycnf, &dbcfgs.Dba, &dbcfgs.Repl)
statsType := stats.NewString("TabletType")
statsKeyspace := stats.NewString("TabletKeyspace")
statsShard := stats.NewString("TabletShard")
statsKeyRangeStart := stats.NewString("TabletKeyRangeStart")
statsKeyRangeEnd := stats.NewString("TabletKeyRangeEnd")
agent, err = tabletmanager.NewActionAgent(topoServer, tabletAlias, mysqld)
if err != nil {
return nil, err
}
// Start the binlog player services, not playing at start.
agent.BinlogPlayerMap = tabletmanager.NewBinlogPlayerMap(topoServer, &dbcfgs.App.ConnectionParams, mysqld)
tabletmanager.RegisterBinlogPlayerMap(agent.BinlogPlayerMap)
// Action agent listens to changes in zookeeper and makes
// modifications to this tablet.
agent.AddChangeCallback(func(oldTablet, newTablet topo.Tablet) {
allowQuery := true
var shardInfo *topo.ShardInfo
var keyspaceInfo *topo.KeyspaceInfo
if newTablet.Type == topo.TYPE_MASTER {
// read the shard to get SourceShards
shardInfo, err = topoServer.GetShard(newTablet.Keyspace, newTablet.Shard)
if err != nil {
log.Errorf("Cannot read shard for this tablet %v: %v", newTablet.Alias, err)
} else {
allowQuery = len(shardInfo.SourceShards) == 0
}
// read the keyspace to get ShardingColumnType
keyspaceInfo, err = topoServer.GetKeyspace(newTablet.Keyspace)
switch err {
case nil:
// continue
case topo.ErrNoNode:
// backward compatible mode
keyspaceInfo = topo.NewKeyspaceInfo(newTablet.Keyspace, &topo.Keyspace{})
default:
log.Errorf("Cannot read keyspace for this tablet %v: %v", newTablet.Alias, err)
keyspaceInfo = nil
}
}
if newTablet.IsRunningQueryService() && allowQuery {
if dbcfgs.App.DbName == "" {
dbcfgs.App.DbName = newTablet.DbName()
}
dbcfgs.App.Keyspace = newTablet.Keyspace
dbcfgs.App.Shard = newTablet.Shard
if newTablet.Type != topo.TYPE_MASTER {
dbcfgs.App.EnableInvalidator = true
} else {
dbcfgs.App.EnableInvalidator = false
}
// There are a few transitions when we're
// going to need to restart the query service:
// - transitioning from replica to master, so clients
// that were already connected don't keep on using
// the master as replica or rdonly.
// - having different parameters for the query
// service. It needs to stop and restart with the
// new parameters. That includes:
// - changing KeyRange
// - changing the BlacklistedTables list
if (newTablet.Type == topo.TYPE_MASTER &&
oldTablet.Type != topo.TYPE_MASTER) ||
(newTablet.KeyRange != oldTablet.KeyRange) ||
!reflect.DeepEqual(newTablet.BlacklistedTables, oldTablet.BlacklistedTables) {
ts.DisallowQueries()
}
qrs := ts.LoadCustomRules()
if newTablet.KeyRange.IsPartial() {
qr := ts.NewQueryRule("enforce keyspace_id range", "keyspace_id_not_in_range", ts.QR_FAIL_QUERY)
qr.AddPlanCond(sqlparser.PLAN_INSERT_PK)
err = qr.AddBindVarCond("keyspace_id", true, true, ts.QR_NOTIN, newTablet.KeyRange)
if err != nil {
log.Warningf("Unable to add keyspace rule: %v", err)
} else {
qrs.Add(qr)
}
}
if len(newTablet.BlacklistedTables) > 0 {
log.Infof("Blacklisting tables %v", strings.Join(newTablet.BlacklistedTables, ", "))
qr := ts.NewQueryRule("enforce blacklisted tables", "blacklisted_table", ts.QR_FAIL_QUERY)
for _, t := range newTablet.BlacklistedTables {
qr.AddTableCond(t)
}
qrs.Add(qr)
}
ts.AllowQueries(&dbcfgs.App, schemaOverrides, qrs, mysqld)
// Disable before enabling to force existing streams to stop.
binlog.DisableUpdateStreamService()
binlog.EnableUpdateStreamService(dbcfgs)
} else {
ts.DisallowQueries()
binlog.DisableUpdateStreamService()
}
statsType.Set(string(newTablet.Type))
statsKeyspace.Set(newTablet.Keyspace)
statsShard.Set(newTablet.Shard)
statsKeyRangeStart.Set(string(newTablet.KeyRange.Start.Hex()))
statsKeyRangeEnd.Set(string(newTablet.KeyRange.End.Hex()))
// See if we need to start or stop any binlog player
if newTablet.Type == topo.TYPE_MASTER {
agent.BinlogPlayerMap.RefreshMap(newTablet, keyspaceInfo, shardInfo)
} else {
agent.BinlogPlayerMap.StopAllPlayersAndReset()
}
})
if err := agent.Start(mysqld.Port(), port, securePort); err != nil {
return nil, err
}
// register the RPC services from the agent
agent.RegisterQueryService()
// start health check if needed
initHeathCheck(agent)
return agent, nil
}