// loadBlacklistRules loads and builds the blacklist query rules
func (agent *ActionAgent) loadBlacklistRules(tablet *topodatapb.Tablet, blacklistedTables []string) (err error) {
blacklistRules := tabletserver.NewQueryRules()
if len(blacklistedTables) > 0 {
// tables, first resolve wildcards
tables, err := mysqlctl.ResolveTables(agent.MysqlDaemon, topoproto.TabletDbName(tablet), blacklistedTables)
if err != nil {
return err
}
// Verify that at least one table matches the wildcards, so
// that we don't add a rule to blacklist all tables
if len(tables) > 0 {
log.Infof("Blacklisting tables %v", strings.Join(tables, ", "))
qr := tabletserver.NewQueryRule("enforce blacklisted tables", "blacklisted_table", tabletserver.QRFailRetry)
for _, t := range tables {
qr.AddTableCond(t)
}
blacklistRules.Add(qr)
}
}
loadRuleErr := agent.QueryServiceControl.SetQueryRules(blacklistQueryRules, blacklistRules)
if loadRuleErr != nil {
log.Warningf("Fail to load query rule set %s: %s", blacklistQueryRules, loadRuleErr)
}
return nil
}
// loadKeyspaceAndBlacklistRules does what the name suggests:
// 1. load and build keyrange query rules
// 2. load and build blacklist query rules
func (agent *ActionAgent) loadKeyspaceAndBlacklistRules(tablet *pbt.Tablet, blacklistedTables []string) (err error) {
// Keyrange rules
keyrangeRules := tabletserver.NewQueryRules()
if key.KeyRangeIsPartial(tablet.KeyRange) {
log.Infof("Restricting to keyrange: %v", tablet.KeyRange)
dmlPlans := []struct {
planID planbuilder.PlanType
onAbsent bool
}{
{planbuilder.PlanInsertPK, true},
{planbuilder.PlanInsertSubquery, true},
{planbuilder.PlanPassDML, false},
{planbuilder.PlanDMLPK, false},
{planbuilder.PlanDMLSubquery, false},
{planbuilder.PlanUpsertPK, false},
}
for _, plan := range dmlPlans {
qr := tabletserver.NewQueryRule(
fmt.Sprintf("enforce keyspace_id range for %v", plan.planID),
fmt.Sprintf("keyspace_id_not_in_range_%v", plan.planID),
tabletserver.QRFail,
)
qr.AddPlanCond(plan.planID)
err := qr.AddBindVarCond("keyspace_id", plan.onAbsent, true, tabletserver.QRNotIn, tablet.KeyRange)
if err != nil {
return fmt.Errorf("Unable to add keyspace rule: %v", err)
}
keyrangeRules.Add(qr)
}
}
// Blacklisted tables
blacklistRules := tabletserver.NewQueryRules()
if len(blacklistedTables) > 0 {
// tables, first resolve wildcards
tables, err := mysqlctl.ResolveTables(agent.MysqlDaemon, topoproto.TabletDbName(tablet), blacklistedTables)
if err != nil {
return err
}
log.Infof("Blacklisting tables %v", strings.Join(tables, ", "))
qr := tabletserver.NewQueryRule("enforce blacklisted tables", "blacklisted_table", tabletserver.QRFailRetry)
for _, t := range tables {
qr.AddTableCond(t)
}
blacklistRules.Add(qr)
}
// Push all three sets of QueryRules to TabletServerRpcService
loadRuleErr := agent.QueryServiceControl.SetQueryRules(keyrangeQueryRules, keyrangeRules)
if loadRuleErr != nil {
log.Warningf("Fail to load query rule set %s: %s", keyrangeQueryRules, loadRuleErr)
}
loadRuleErr = agent.QueryServiceControl.SetQueryRules(blacklistQueryRules, blacklistRules)
if loadRuleErr != nil {
log.Warningf("Fail to load query rule set %s: %s", blacklistQueryRules, loadRuleErr)
}
return nil
}
// Iteration is a single iteration for the player: get the current status,
// try to play, and plays until interrupted, or until an error occurs.
func (bpc *BinlogPlayerController) Iteration() (err error) {
defer func() {
if x := recover(); x != nil {
log.Errorf("%v: caught panic: %v\n%s", bpc, x, tb.Stack(4))
err = fmt.Errorf("panic: %v", x)
}
}()
// Apply any special settings necessary for playback of binlogs.
// We do it on every iteration to be sure, in case MySQL was restarted.
if err := bpc.mysqld.EnableBinlogPlayback(); err != nil {
// We failed to apply the required settings, so we shouldn't keep going.
return err
}
// create the db connection, connect it
vtClient := bpc.vtClientFactory()
if err := vtClient.Connect(); err != nil {
return fmt.Errorf("can't connect to database: %v", err)
}
defer vtClient.Close()
// Read the start position
startPosition, flags, err := binlogplayer.ReadStartPosition(vtClient, bpc.sourceShard.Uid)
if err != nil {
return fmt.Errorf("can't read startPosition: %v", err)
}
// if we shouldn't start, we just error out and try again later
if strings.Index(flags, binlogplayer.BlpFlagDontStart) != -1 {
return fmt.Errorf("not starting because flag '%v' is set", binlogplayer.BlpFlagDontStart)
}
// wait for the endpoint set (usefull for the first run at least, fast for next runs)
if err := discovery.WaitForEndPoints(bpc.healthCheck, bpc.cell, bpc.sourceShard.Keyspace, bpc.sourceShard.Shard, []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != nil {
return fmt.Errorf("error waiting for endpoints for %v %v %v: %v", bpc.cell, bpc.sourceShard.String(), topodatapb.TabletType_REPLICA, err)
}
// Find the server list from the health check
addrs := bpc.healthCheck.GetEndPointStatsFromTarget(bpc.sourceShard.Keyspace, bpc.sourceShard.Shard, topodatapb.TabletType_REPLICA)
if len(addrs) == 0 {
return fmt.Errorf("can't find any source tablet for %v %v %v", bpc.cell, bpc.sourceShard.String(), topodatapb.TabletType_REPLICA)
}
newServerIndex := rand.Intn(len(addrs))
endPoint := addrs[newServerIndex].EndPoint
// save our current server
bpc.playerMutex.Lock()
bpc.sourceTablet = &topodatapb.TabletAlias{
Cell: bpc.cell,
Uid: endPoint.Uid,
}
bpc.lastError = nil
bpc.playerMutex.Unlock()
// check which kind of replication we're doing, tables or keyrange
if len(bpc.sourceShard.Tables) > 0 {
// tables, first resolve wildcards
tables, err := mysqlctl.ResolveTables(bpc.mysqld, bpc.dbName, bpc.sourceShard.Tables)
if err != nil {
return fmt.Errorf("failed to resolve table names: %v", err)
}
// tables, just get them
player, err := binlogplayer.NewBinlogPlayerTables(vtClient, endPoint, tables, bpc.sourceShard.Uid, startPosition, bpc.stopPosition, bpc.binlogPlayerStats)
if err != nil {
return fmt.Errorf("NewBinlogPlayerTables failed: %v", err)
}
return player.ApplyBinlogEvents(bpc.ctx)
}
// the data we have to replicate is the intersection of the
// source keyrange and our keyrange
overlap, err := key.KeyRangesOverlap(bpc.sourceShard.KeyRange, bpc.keyRange)
if err != nil {
return fmt.Errorf("Source shard %v doesn't overlap destination shard %v", bpc.sourceShard.KeyRange, bpc.keyRange)
}
player, err := binlogplayer.NewBinlogPlayerKeyRange(vtClient, endPoint, overlap, bpc.sourceShard.Uid, startPosition, bpc.stopPosition, bpc.binlogPlayerStats)
if err != nil {
return fmt.Errorf("NewBinlogPlayerKeyRange failed: %v", err)
}
return player.ApplyBinlogEvents(bpc.ctx)
}
// Iteration is a single iteration for the player: get the current status,
// try to play, and plays until interrupted, or until an error occurs.
func (bpc *BinlogPlayerController) Iteration() (err error) {
defer func() {
if x := recover(); x != nil {
log.Errorf("%v: caught panic: %v\n%s", bpc, x, tb.Stack(4))
err = fmt.Errorf("panic: %v", x)
}
}()
// Check if the context is still good.
select {
case <-bpc.ctx.Done():
if bpc.ctx.Err() == context.Canceled {
// We were stopped. Break out of Loop().
return nil
}
return fmt.Errorf("giving up since the context is done: %v", bpc.ctx.Err())
default:
}
// Apply any special settings necessary for playback of binlogs.
// We do it on every iteration to be sure, in case MySQL was restarted.
if err := bpc.mysqld.EnableBinlogPlayback(); err != nil {
// We failed to apply the required settings, so we shouldn't keep going.
return err
}
// create the db connection, connect it
vtClient := bpc.vtClientFactory()
if err := vtClient.Connect(); err != nil {
return fmt.Errorf("can't connect to database: %v", err)
}
defer vtClient.Close()
// Read the start position
startPosition, flags, err := binlogplayer.ReadStartPosition(vtClient, bpc.sourceShard.Uid)
if err != nil {
return fmt.Errorf("can't read startPosition: %v", err)
}
// if we shouldn't start, we just error out and try again later
if strings.Index(flags, binlogplayer.BlpFlagDontStart) != -1 {
return fmt.Errorf("not starting because flag '%v' is set", binlogplayer.BlpFlagDontStart)
}
// wait for the tablet set (usefull for the first run at least, fast for next runs)
if err := bpc.tabletStatsCache.WaitForTablets(bpc.ctx, bpc.cell, bpc.sourceShard.Keyspace, bpc.sourceShard.Shard, []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != nil {
return fmt.Errorf("error waiting for tablets for %v %v %v: %v", bpc.cell, bpc.sourceShard.String(), topodatapb.TabletType_REPLICA, err)
}
// Find the server list from the health check.
// Note: We cannot use tsc.GetHealthyTabletStats() here because it does
// not return non-serving tablets. We must include non-serving tablets because
// REPLICA source tablets may not be serving anymore because their traffic was
// already migrated to the destination shards.
addrs := discovery.RemoveUnhealthyTablets(bpc.tabletStatsCache.GetTabletStats(bpc.sourceShard.Keyspace, bpc.sourceShard.Shard, topodatapb.TabletType_REPLICA))
if len(addrs) == 0 {
return fmt.Errorf("can't find any healthy source tablet for %v %v %v", bpc.cell, bpc.sourceShard.String(), topodatapb.TabletType_REPLICA)
}
newServerIndex := rand.Intn(len(addrs))
tablet := addrs[newServerIndex].Tablet
// save our current server
bpc.playerMutex.Lock()
bpc.sourceTablet = tablet.Alias
bpc.lastError = nil
bpc.playerMutex.Unlock()
// check which kind of replication we're doing, tables or keyrange
if len(bpc.sourceShard.Tables) > 0 {
// tables, first resolve wildcards
tables, err := mysqlctl.ResolveTables(bpc.mysqld, bpc.dbName, bpc.sourceShard.Tables)
if err != nil {
return fmt.Errorf("failed to resolve table names: %v", err)
}
// tables, just get them
player, err := binlogplayer.NewBinlogPlayerTables(vtClient, tablet, tables, bpc.sourceShard.Uid, startPosition, bpc.stopPosition, bpc.binlogPlayerStats)
if err != nil {
return fmt.Errorf("NewBinlogPlayerTables failed: %v", err)
}
return player.ApplyBinlogEvents(bpc.ctx)
}
// the data we have to replicate is the intersection of the
// source keyrange and our keyrange
overlap, err := key.KeyRangesOverlap(bpc.sourceShard.KeyRange, bpc.keyRange)
if err != nil {
return fmt.Errorf("Source shard %v doesn't overlap destination shard %v", bpc.sourceShard.KeyRange, bpc.keyRange)
}
player, err := binlogplayer.NewBinlogPlayerKeyRange(vtClient, tablet, overlap, bpc.sourceShard.Uid, startPosition, bpc.stopPosition, bpc.binlogPlayerStats)
if err != nil {
return fmt.Errorf("NewBinlogPlayerKeyRange failed: %v", err)
}
return player.ApplyBinlogEvents(bpc.ctx)
}
// Iteration is a single iteration for the player: get the current status,
// try to play, and plays until interrupted, or until an error occurs.
func (bpc *BinlogPlayerController) Iteration() (err error) {
defer func() {
if x := recover(); x != nil {
log.Errorf("%v: caught panic: %v", bpc, x)
err = fmt.Errorf("panic: %v", x)
}
}()
// Apply any special settings necessary for playback of binlogs.
// We do it on every iteration to be sure, in case MySQL was restarted.
if err := bpc.mysqld.EnableBinlogPlayback(); err != nil {
// We failed to apply the required settings, so we shouldn't keep going.
return err
}
// create the db connection, connect it
vtClient := binlogplayer.NewDbClient(bpc.dbConfig)
if err := vtClient.Connect(); err != nil {
return fmt.Errorf("can't connect to database: %v", err)
}
defer vtClient.Close()
// Read the start position
startPosition, flags, err := binlogplayer.ReadStartPosition(vtClient, bpc.sourceShard.Uid)
if err != nil {
return fmt.Errorf("can't read startPosition: %v", err)
}
// if we shouldn't start, we just error out and try again later
if strings.Index(flags, binlogplayer.BlpFlagDontStart) != -1 {
return fmt.Errorf("not starting because flag '%v' is set", binlogplayer.BlpFlagDontStart)
}
// Find the server list for the source shard in our cell
addrs, _, err := bpc.ts.GetEndPoints(bpc.ctx, bpc.cell, bpc.sourceShard.Keyspace, bpc.sourceShard.Shard, topo.TYPE_REPLICA)
if err != nil {
// If this calls fails because the context was canceled,
// we need to return nil.
select {
case <-bpc.ctx.Done():
if bpc.ctx.Err() == context.Canceled {
return nil
}
default:
}
return fmt.Errorf("can't find any source tablet for %v %v %v: %v", bpc.cell, bpc.sourceShard.String(), topo.TYPE_REPLICA, err)
}
if len(addrs.Entries) == 0 {
return fmt.Errorf("empty source tablet list for %v %v %v", bpc.cell, bpc.sourceShard.String(), topo.TYPE_REPLICA)
}
newServerIndex := rand.Intn(len(addrs.Entries))
endPoint := addrs.Entries[newServerIndex]
// save our current server
bpc.playerMutex.Lock()
bpc.sourceTablet = topo.TabletAlias{
Cell: bpc.cell,
Uid: addrs.Entries[newServerIndex].Uid,
}
bpc.lastError = nil
bpc.playerMutex.Unlock()
// check which kind of replication we're doing, tables or keyrange
if len(bpc.sourceShard.Tables) > 0 {
// tables, first resolve wildcards
tables, err := mysqlctl.ResolveTables(bpc.mysqld, bpc.dbName, bpc.sourceShard.Tables)
if err != nil {
return fmt.Errorf("failed to resolve table names: %v", err)
}
// tables, just get them
player := binlogplayer.NewBinlogPlayerTables(vtClient, endPoint, tables, startPosition, bpc.stopPosition, bpc.binlogPlayerStats)
return player.ApplyBinlogEvents(bpc.ctx)
}
// the data we have to replicate is the intersection of the
// source keyrange and our keyrange
overlap, err := key.KeyRangesOverlap3(bpc.sourceShard.KeyRange, bpc.keyRange)
if err != nil {
return fmt.Errorf("Source shard %v doesn't overlap destination shard %v", bpc.sourceShard.KeyRange, bpc.keyRange)
}
player := binlogplayer.NewBinlogPlayerKeyRange(vtClient, endPoint, bpc.keyspaceIDType, overlap, startPosition, bpc.stopPosition, bpc.binlogPlayerStats)
return player.ApplyBinlogEvents(bpc.ctx)
}