// loadKeyspaceAndBlacklistRules does what the name suggests:
// 1. load and build keyrange query rules
// 2. load and build blacklist query rules
func (agent *ActionAgent) loadKeyspaceAndBlacklistRules(tablet *pbt.Tablet, blacklistedTables []string) (err error) {
// Keyrange rules
keyrangeRules := tabletserver.NewQueryRules()
if key.KeyRangeIsPartial(tablet.KeyRange) {
log.Infof("Restricting to keyrange: %v", tablet.KeyRange)
dmlPlans := []struct {
planID planbuilder.PlanType
onAbsent bool
}{
{planbuilder.PlanInsertPK, true},
{planbuilder.PlanInsertSubquery, true},
{planbuilder.PlanPassDML, false},
{planbuilder.PlanDMLPK, false},
{planbuilder.PlanDMLSubquery, false},
{planbuilder.PlanUpsertPK, false},
}
for _, plan := range dmlPlans {
qr := tabletserver.NewQueryRule(
fmt.Sprintf("enforce keyspace_id range for %v", plan.planID),
fmt.Sprintf("keyspace_id_not_in_range_%v", plan.planID),
tabletserver.QRFail,
)
qr.AddPlanCond(plan.planID)
err := qr.AddBindVarCond("keyspace_id", plan.onAbsent, true, tabletserver.QRNotIn, tablet.KeyRange)
if err != nil {
return fmt.Errorf("Unable to add keyspace rule: %v", err)
}
keyrangeRules.Add(qr)
}
}
// Blacklisted tables
blacklistRules := tabletserver.NewQueryRules()
if len(blacklistedTables) > 0 {
// tables, first resolve wildcards
tables, err := mysqlctl.ResolveTables(agent.MysqlDaemon, topoproto.TabletDbName(tablet), blacklistedTables)
if err != nil {
return err
}
log.Infof("Blacklisting tables %v", strings.Join(tables, ", "))
qr := tabletserver.NewQueryRule("enforce blacklisted tables", "blacklisted_table", tabletserver.QRFailRetry)
for _, t := range tables {
qr.AddTableCond(t)
}
blacklistRules.Add(qr)
}
// Push all three sets of QueryRules to TabletServerRpcService
loadRuleErr := agent.QueryServiceControl.SetQueryRules(keyrangeQueryRules, keyrangeRules)
if loadRuleErr != nil {
log.Warningf("Fail to load query rule set %s: %s", keyrangeQueryRules, loadRuleErr)
}
loadRuleErr = agent.QueryServiceControl.SetQueryRules(blacklistQueryRules, blacklistRules)
if loadRuleErr != nil {
log.Warningf("Fail to load query rule set %s: %s", blacklistQueryRules, loadRuleErr)
}
return nil
}
// initializeKeyRangeRule will create and set the key range rules
func (agent *ActionAgent) initializeKeyRangeRule(ctx context.Context, keyspace string, keyRange *topodatapb.KeyRange) error {
// check we have a partial key range
if !key.KeyRangeIsPartial(keyRange) {
log.Infof("Tablet covers the full KeyRange, not adding KeyRange query rule")
return nil
}
// read the keyspace to get the sharding column name
keyspaceInfo, err := agent.TopoServer.GetKeyspace(ctx, keyspace)
if err != nil {
return fmt.Errorf("cannot read keyspace %v to get sharding key: %v", keyspace, err)
}
if keyspaceInfo.ShardingColumnName == "" {
log.Infof("Keyspace %v has an empty ShardingColumnName, not adding KeyRange query rule", keyspace)
return nil
}
// create the rules
log.Infof("Restricting to keyrange: %v", key.KeyRangeString(keyRange))
keyrangeRules := tabletserver.NewQueryRules()
dmlPlans := []struct {
planID planbuilder.PlanType
onAbsent bool
}{
{planbuilder.PlanInsertPK, true},
{planbuilder.PlanInsertSubquery, true},
{planbuilder.PlanPassDML, false},
{planbuilder.PlanDMLPK, false},
{planbuilder.PlanDMLSubquery, false},
{planbuilder.PlanUpsertPK, false},
}
for _, plan := range dmlPlans {
qr := tabletserver.NewQueryRule(
fmt.Sprintf("enforce %v range for %v", keyspaceInfo.ShardingColumnName, plan.planID),
fmt.Sprintf("%v_not_in_range_%v", keyspaceInfo.ShardingColumnName, plan.planID),
tabletserver.QRFail,
)
qr.AddPlanCond(plan.planID)
err := qr.AddBindVarCond(keyspaceInfo.ShardingColumnName, plan.onAbsent, true, tabletserver.QRNotIn, keyRange)
if err != nil {
return fmt.Errorf("Unable to add key range rule: %v", err)
}
keyrangeRules.Add(qr)
}
// and load them
agent.QueryServiceControl.RegisterQueryRuleSource(keyrangeQueryRules)
if err := agent.QueryServiceControl.SetQueryRules(keyrangeQueryRules, keyrangeRules); err != nil {
return fmt.Errorf("failed to load query rule set %s: %s", keyrangeQueryRules, err)
}
return nil
}
// This maps a list of keyranges to shard names.
func resolveKeyRangeToShards(allShards []*topodatapb.ShardReference, matches map[string]bool, kr *topodatapb.KeyRange) {
if !key.KeyRangeIsPartial(kr) {
for _, shard := range allShards {
matches[shard.Name] = true
}
return
}
for _, shard := range allShards {
if key.KeyRangesIntersect(kr, shard.KeyRange) {
matches[shard.Name] = true
}
}
}
// SourceShardAsHTML returns a HTML version of the object.
func SourceShardAsHTML(source *topodatapb.Shard_SourceShard) template.HTML {
result := fmt.Sprintf("<b>Uid</b>: %v</br>\n<b>Source</b>: %v/%v</br>\n", source.Uid, source.Keyspace, source.Shard)
if key.KeyRangeIsPartial(source.KeyRange) {
result += fmt.Sprintf("<b>KeyRange</b>: %v-%v</br>\n",
hex.EncodeToString(source.KeyRange.Start),
hex.EncodeToString(source.KeyRange.End))
}
if len(source.Tables) > 0 {
result += fmt.Sprintf("<b>Tables</b>: %v</br>\n",
strings.Join(source.Tables, " "))
}
return template.HTML(result)
}
// This maps a list of keyranges to shard names.
func resolveKeyRangeToShards(allShards []*pb.ShardReference, kr *pb.KeyRange) ([]string, error) {
shards := make([]string, 0, 1)
if !key.KeyRangeIsPartial(kr) {
for j := 0; j < len(allShards); j++ {
shards = append(shards, allShards[j].Name)
}
return shards, nil
}
for j := 0; j < len(allShards); j++ {
shard := allShards[j]
if key.KeyRangesIntersect(kr, shard.KeyRange) {
shards = append(shards, shard.Name)
}
}
return shards, nil
}
// Start validates and updates the topology records for the tablet, and performs
// the initial state change callback to start tablet services.
// If initUpdateStream is set, update stream service will also be registered.
func (agent *ActionAgent) Start(ctx context.Context, mysqlPort, vtPort, gRPCPort int32, initUpdateStream bool) error {
var err error
if _, err = agent.updateTabletFromTopo(ctx); err != nil {
return err
}
// find our hostname as fully qualified, and IP
hostname := *tabletHostname
if hostname == "" {
hostname, err = netutil.FullyQualifiedHostname()
if err != nil {
return err
}
}
ipAddrs, err := net.LookupHost(hostname)
if err != nil {
return err
}
ipAddr := ipAddrs[0]
// Update bind addr for mysql and query service in the tablet node.
f := func(tablet *pb.Tablet) error {
tablet.Hostname = hostname
tablet.Ip = ipAddr
if tablet.PortMap == nil {
tablet.PortMap = make(map[string]int32)
}
if mysqlPort != 0 {
// only overwrite mysql port if we know it, otherwise
// leave it as is.
tablet.PortMap["mysql"] = mysqlPort
}
if vtPort != 0 {
tablet.PortMap["vt"] = vtPort
} else {
delete(tablet.PortMap, "vt")
}
delete(tablet.PortMap, "vts")
if gRPCPort != 0 {
tablet.PortMap["grpc"] = gRPCPort
} else {
delete(tablet.PortMap, "grpc")
}
return nil
}
if err := agent.TopoServer.UpdateTabletFields(ctx, agent.Tablet().Alias, f); err != nil {
return err
}
// Reread to get the changes we just made
tablet, err := agent.updateTabletFromTopo(ctx)
if err != nil {
return err
}
if err = agent.verifyTopology(ctx); err != nil {
return err
}
if err = agent.verifyServingAddrs(ctx); err != nil {
return err
}
// get and fix the dbname if necessary
if !agent.DBConfigs.IsZero() {
// Only for real instances
// Update our DB config to match the info we have in the tablet
if agent.DBConfigs.App.DbName == "" {
agent.DBConfigs.App.DbName = topoproto.TabletDbName(tablet.Tablet)
}
agent.DBConfigs.App.Keyspace = tablet.Keyspace
agent.DBConfigs.App.Shard = tablet.Shard
}
// create and register the RPC services from UpdateStream
// (it needs the dbname, so it has to be delayed up to here,
// but it has to be before updateState below that may use it)
if initUpdateStream {
us := binlog.NewUpdateStream(agent.MysqlDaemon, agent.DBConfigs.App.DbName)
us.RegisterService()
agent.UpdateStream = us
}
// initialize tablet server
if err := agent.QueryServiceControl.InitDBConfig(pbq.Target{
Keyspace: tablet.Keyspace,
Shard: tablet.Shard,
TabletType: tablet.Type,
}, agent.DBConfigs, agent.SchemaOverrides, agent.MysqlDaemon); err != nil {
return fmt.Errorf("failed to InitDBConfig: %v", err)
}
// export a few static variables
if agent.exportStats {
statsKeyspace := stats.NewString("TabletKeyspace")
statsShard := stats.NewString("TabletShard")
statsKeyRangeStart := stats.NewString("TabletKeyRangeStart")
statsKeyRangeEnd := stats.NewString("TabletKeyRangeEnd")
statsKeyspace.Set(tablet.Keyspace)
statsShard.Set(tablet.Shard)
if key.KeyRangeIsPartial(tablet.KeyRange) {
statsKeyRangeStart.Set(hex.EncodeToString(tablet.KeyRange.Start))
statsKeyRangeEnd.Set(hex.EncodeToString(tablet.KeyRange.End))
}
}
// initialize the key range query rule
if err := agent.initializeKeyRangeRule(ctx, tablet.Keyspace, tablet.KeyRange); err != nil {
return err
}
// and update our state
oldTablet := &pb.Tablet{}
if err = agent.updateState(ctx, oldTablet, "Start"); err != nil {
log.Warningf("Initial updateState failed, will need a state change before running properly: %v", err)
}
return nil
}
// Start validates and updates the topology records for the tablet, and performs
// the initial state change callback to start tablet services.
// If initUpdateStream is set, update stream service will also be registered.
func (agent *ActionAgent) Start(ctx context.Context, mysqlPort, vtPort, gRPCPort int32, initUpdateStream bool) error {
// find our hostname as fully qualified, and IP
hostname := *tabletHostname
if hostname == "" {
var err error
hostname, err = netutil.FullyQualifiedHostname()
if err != nil {
return err
}
}
ipAddrs, err := net.LookupHost(hostname)
if err != nil {
return err
}
ipAddr := ipAddrs[0]
// Update bind addr for mysql and query service in the tablet node.
f := func(tablet *topodatapb.Tablet) error {
tablet.Hostname = hostname
tablet.Ip = ipAddr
if tablet.PortMap == nil {
tablet.PortMap = make(map[string]int32)
}
if mysqlPort != 0 {
// only overwrite mysql port if we know it, otherwise
// leave it as is.
tablet.PortMap["mysql"] = mysqlPort
}
if vtPort != 0 {
tablet.PortMap["vt"] = vtPort
} else {
delete(tablet.PortMap, "vt")
}
delete(tablet.PortMap, "vts")
if gRPCPort != 0 {
tablet.PortMap["grpc"] = gRPCPort
} else {
delete(tablet.PortMap, "grpc")
}
// Save the original tablet for ownership tests later.
agent.initialTablet = tablet
return nil
}
if _, err := agent.TopoServer.UpdateTabletFields(ctx, agent.TabletAlias, f); err != nil {
return err
}
// Verify the topology is correct.
agent.verifyTopology(ctx)
// Get and fix the dbname if necessary, only for real instances.
if !agent.DBConfigs.IsZero() {
dbname := topoproto.TabletDbName(agent.initialTablet)
// Update our DB config to match the info we have in the tablet
if agent.DBConfigs.App.DbName == "" {
agent.DBConfigs.App.DbName = dbname
}
if agent.DBConfigs.Filtered.DbName == "" {
agent.DBConfigs.Filtered.DbName = dbname
}
}
// create and register the RPC services from UpdateStream
// (it needs the dbname, so it has to be delayed up to here,
// but it has to be before updateState below that may use it)
if initUpdateStream {
us := binlog.NewUpdateStream(agent.MysqlDaemon, agent.DBConfigs.App.DbName)
agent.UpdateStream = us
servenv.OnRun(func() {
us.RegisterService()
})
}
servenv.OnTerm(func() {
// Disable UpdateStream (if any) upon entering lameduck.
// We do this regardless of initUpdateStream, since agent.UpdateStream
// may have been set from elsewhere.
if agent.UpdateStream != nil {
agent.UpdateStream.Disable()
}
})
// initialize tablet server
if err := agent.QueryServiceControl.InitDBConfig(querypb.Target{
Keyspace: agent.initialTablet.Keyspace,
Shard: agent.initialTablet.Shard,
TabletType: agent.initialTablet.Type,
}, agent.DBConfigs, agent.MysqlDaemon); err != nil {
return fmt.Errorf("failed to InitDBConfig: %v", err)
}
// export a few static variables
if agent.exportStats {
statsKeyspace := stats.NewString("TabletKeyspace")
statsShard := stats.NewString("TabletShard")
statsKeyRangeStart := stats.NewString("TabletKeyRangeStart")
statsKeyRangeEnd := stats.NewString("TabletKeyRangeEnd")
statsKeyspace.Set(agent.initialTablet.Keyspace)
statsShard.Set(agent.initialTablet.Shard)
if key.KeyRangeIsPartial(agent.initialTablet.KeyRange) {
statsKeyRangeStart.Set(hex.EncodeToString(agent.initialTablet.KeyRange.Start))
statsKeyRangeEnd.Set(hex.EncodeToString(agent.initialTablet.KeyRange.End))
}
}
// Initialize the current tablet to match our current running
// state: Has most field filled in, but type is UNKNOWN.
// Subsequents calls to updateState or refreshTablet
// will then work as expected.
startingTablet := proto.Clone(agent.initialTablet).(*topodatapb.Tablet)
startingTablet.Type = topodatapb.TabletType_UNKNOWN
agent.setTablet(startingTablet)
// run a background task to rebuild the SrvKeyspace in our cell/keyspace
// if it doesn't exist yet
go agent.maybeRebuildKeyspace(agent.initialTablet.Alias.Cell, agent.initialTablet.Keyspace)
return nil
}
// Start validates and updates the topology records for the tablet, and performs
// the initial state change callback to start tablet services.
// If initUpdateStream is set, update stream service will also be registered.
func (agent *ActionAgent) Start(ctx context.Context, mysqlPort, vtPort, gRPCPort int32, initUpdateStream bool) error {
var err error
if _, err = agent.updateTabletFromTopo(ctx); err != nil {
return err
}
// find our hostname as fully qualified, and IP
hostname := *tabletHostname
if hostname == "" {
hostname, err = netutil.FullyQualifiedHostname()
if err != nil {
return err
}
}
ipAddrs, err := net.LookupHost(hostname)
if err != nil {
return err
}
ipAddr := ipAddrs[0]
// Update bind addr for mysql and query service in the tablet node.
f := func(tablet *topodatapb.Tablet) error {
tablet.Hostname = hostname
tablet.Ip = ipAddr
if tablet.PortMap == nil {
tablet.PortMap = make(map[string]int32)
}
if mysqlPort != 0 {
// only overwrite mysql port if we know it, otherwise
// leave it as is.
tablet.PortMap["mysql"] = mysqlPort
}
if vtPort != 0 {
tablet.PortMap["vt"] = vtPort
} else {
delete(tablet.PortMap, "vt")
}
delete(tablet.PortMap, "vts")
if gRPCPort != 0 {
tablet.PortMap["grpc"] = gRPCPort
} else {
delete(tablet.PortMap, "grpc")
}
return nil
}
if _, err := agent.TopoServer.UpdateTabletFields(ctx, agent.Tablet().Alias, f); err != nil {
return err
}
// Reread to get the changes we just made
tablet, err := agent.updateTabletFromTopo(ctx)
if err != nil {
return err
}
// Save the original tablet record as it is now (at startup).
agent.initialTablet = proto.Clone(tablet).(*topodatapb.Tablet)
if err = agent.verifyTopology(ctx); err != nil {
return err
}
// get and fix the dbname if necessary
if !agent.DBConfigs.IsZero() {
// Only for real instances
// Update our DB config to match the info we have in the tablet
if agent.DBConfigs.App.DbName == "" {
agent.DBConfigs.App.DbName = topoproto.TabletDbName(tablet)
}
if agent.DBConfigs.Filtered.DbName == "" {
agent.DBConfigs.Filtered.DbName = topoproto.TabletDbName(tablet)
}
agent.DBConfigs.App.Keyspace = tablet.Keyspace
agent.DBConfigs.App.Shard = tablet.Shard
}
// create and register the RPC services from UpdateStream
// (it needs the dbname, so it has to be delayed up to here,
// but it has to be before updateState below that may use it)
if initUpdateStream {
us := binlog.NewUpdateStream(agent.MysqlDaemon, agent.DBConfigs.App.DbName)
agent.UpdateStream = us
servenv.OnRun(func() {
us.RegisterService()
})
}
servenv.OnTerm(func() {
// Disable UpdateStream (if any) upon entering lameduck.
// We do this regardless of initUpdateStream, since agent.UpdateStream
// may have been set from elsewhere.
if agent.UpdateStream != nil {
agent.UpdateStream.Disable()
}
})
// initialize tablet server
if err := agent.QueryServiceControl.InitDBConfig(querypb.Target{
Keyspace: tablet.Keyspace,
Shard: tablet.Shard,
TabletType: tablet.Type,
}, agent.DBConfigs, agent.MysqlDaemon); err != nil {
return fmt.Errorf("failed to InitDBConfig: %v", err)
}
// export a few static variables
if agent.exportStats {
statsKeyspace := stats.NewString("TabletKeyspace")
statsShard := stats.NewString("TabletShard")
statsKeyRangeStart := stats.NewString("TabletKeyRangeStart")
statsKeyRangeEnd := stats.NewString("TabletKeyRangeEnd")
statsKeyspace.Set(tablet.Keyspace)
statsShard.Set(tablet.Shard)
if key.KeyRangeIsPartial(tablet.KeyRange) {
statsKeyRangeStart.Set(hex.EncodeToString(tablet.KeyRange.Start))
statsKeyRangeEnd.Set(hex.EncodeToString(tablet.KeyRange.End))
}
}
// initialize the key range query rule
if err := agent.initializeKeyRangeRule(ctx, tablet.Keyspace, tablet.KeyRange); err != nil {
return err
}
// update our state
oldTablet := &topodatapb.Tablet{}
agent.updateState(ctx, oldTablet, "Start")
// run a background task to rebuild the SrvKeyspace in our cell/keyspace
// if it doesn't exist yet
go agent.maybeRebuildKeyspace(tablet.Alias.Cell, tablet.Keyspace)
return nil
}