// findDestinationMasters finds for each destination shard the current master.
func (scw *SplitCloneWorker) findDestinationMasters(ctx context.Context) error {
scw.setState(WorkerStateFindTargets)
// Make sure we find a master for each destination shard and log it.
scw.wr.Logger().Infof("Finding a MASTER tablet for each destination shard...")
for _, si := range scw.destinationShards {
waitCtx, waitCancel := context.WithTimeout(ctx, *waitForHealthyTabletsTimeout)
defer waitCancel()
if err := discovery.WaitForTablets(waitCtx, scw.healthCheck,
scw.cell, si.Keyspace(), si.ShardName(), []topodatapb.TabletType{topodatapb.TabletType_MASTER}); err != nil {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v (in cell: %v): %v", si.Keyspace(), si.ShardName(), scw.cell, err)
}
masters := discovery.GetCurrentMaster(
scw.healthCheck.GetTabletStatsFromTarget(si.Keyspace(), si.ShardName(), topodatapb.TabletType_MASTER))
if len(masters) == 0 {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v (in cell: %v) in HealthCheck: empty TabletStats list", si.Keyspace(), si.ShardName(), scw.cell)
}
master := masters[0]
// Get the MySQL database name of the tablet.
keyspaceAndShard := topoproto.KeyspaceShardString(si.Keyspace(), si.ShardName())
scw.destinationDbNames[keyspaceAndShard] = topoproto.TabletDbName(master.Tablet)
// TODO(mberlin): Verify on the destination master that the
// _vt.blp_checkpoint table has the latest schema.
scw.wr.Logger().Infof("Using tablet %v as destination master for %v/%v", topoproto.TabletAliasString(master.Tablet.Alias), si.Keyspace(), si.ShardName())
}
scw.wr.Logger().Infof("NOTE: The used master of a destination shard might change over the course of the copy e.g. due to a reparent. The HealthCheck module will track and log master changes and any error message will always refer the actually used master address.")
return nil
}
// fetchWithRetries will attempt to run ExecuteFetch for a single command, with
// a reasonably small timeout.
// If will keep retrying the ExecuteFetch (for a finite but longer duration) if
// it fails due to a timeout or a retriable application error.
//
// executeFetchWithRetries will always get the current MASTER tablet from the
// healthcheck instance. If no MASTER is available, it will keep retrying.
func (e *executor) fetchWithRetries(ctx context.Context, command string) error {
retryDuration := 2 * time.Hour
// We should keep retrying up until the retryCtx runs out.
retryCtx, retryCancel := context.WithTimeout(ctx, retryDuration)
defer retryCancel()
// Is this current attempt a retry of a previous attempt?
isRetry := false
for {
var master *discovery.TabletStats
var err error
// Get the current master from the HealthCheck.
masters := discovery.GetCurrentMaster(
e.healthCheck.GetTabletStatsFromTarget(e.keyspace, e.shard, topodatapb.TabletType_MASTER))
if len(masters) == 0 {
e.wr.Logger().Warningf("ExecuteFetch failed for keyspace/shard %v/%v because no MASTER is available; will retry until there is MASTER again", e.keyspace, e.shard)
statsRetryCount.Add(1)
statsRetryCounters.Add(retryCategoryNoMasterAvailable, 1)
goto retry
}
master = masters[0]
// Block if we are throttled.
if e.throttler != nil {
for {
backoff := e.throttler.Throttle(e.threadID)
if backoff == throttler.NotThrottled {
break
}
statsThrottledCounters.Add(e.statsKey, 1)
time.Sleep(backoff)
}
}
// Run the command (in a block since goto above does not allow to introduce
// new variables until the label is reached.)
{
tryCtx, cancel := context.WithTimeout(retryCtx, 2*time.Minute)
_, err = e.wr.TabletManagerClient().ExecuteFetchAsApp(tryCtx, master.Tablet, command, 0)
cancel()
if err == nil {
// success!
return nil
}
succeeded, finalErr := e.checkError(tryCtx, err, isRetry, master)
if succeeded {
// We can ignore the error and don't have to retry.
return nil
}
if finalErr != nil {
// Non-retryable error.
return finalErr
}
}
retry:
masterAlias := "no-master-was-available"
if master != nil {
masterAlias = topoproto.TabletAliasString(master.Tablet.Alias)
}
tabletString := fmt.Sprintf("%v (%v/%v)", masterAlias, e.keyspace, e.shard)
select {
case <-retryCtx.Done():
if retryCtx.Err() == context.DeadlineExceeded {
return fmt.Errorf("failed to connect to destination tablet %v after retrying for %v", tabletString, retryDuration)
}
return fmt.Errorf("interrupted while trying to run %v on tablet %v", command, tabletString)
case <-time.After(*executeFetchRetryTime):
// Retry 30s after the failure using the current master seen by the HealthCheck.
}
isRetry = true
}
}
// findTargets phase:
// - find one rdonly in the source shard
// - mark it as 'worker' pointing back to us
// - get the aliases of all the targets
func (vscw *VerticalSplitCloneWorker) findTargets(ctx context.Context) error {
vscw.setState(WorkerStateFindTargets)
// find an appropriate tablet in the source shard
var err error
vscw.sourceAlias, err = FindWorkerTablet(ctx, vscw.wr, vscw.cleaner, nil /* healthCheck */, vscw.cell, vscw.sourceKeyspace, "0", vscw.minHealthyRdonlyTablets)
if err != nil {
return fmt.Errorf("FindWorkerTablet() failed for %v/%v/0: %v", vscw.cell, vscw.sourceKeyspace, err)
}
vscw.wr.Logger().Infof("Using tablet %v as the source", topoproto.TabletAliasString(vscw.sourceAlias))
// get the tablet info for it
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
ti, err := vscw.wr.TopoServer().GetTablet(shortCtx, vscw.sourceAlias)
cancel()
if err != nil {
return fmt.Errorf("cannot read tablet %v: %v", topoproto.TabletAliasString(vscw.sourceAlias), err)
}
vscw.sourceTablet = ti.Tablet
// stop replication on it
shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout)
err = vscw.wr.TabletManagerClient().StopSlave(shortCtx, vscw.sourceTablet)
cancel()
if err != nil {
return fmt.Errorf("cannot stop replication on tablet %v", topoproto.TabletAliasString(vscw.sourceAlias))
}
wrangler.RecordStartSlaveAction(vscw.cleaner, vscw.sourceTablet)
// Initialize healthcheck and add destination shards to it.
vscw.healthCheck = discovery.NewHealthCheck(*remoteActionsTimeout, *healthcheckRetryDelay, *healthCheckTimeout)
watcher := discovery.NewShardReplicationWatcher(vscw.wr.TopoServer(), vscw.healthCheck,
vscw.cell, vscw.destinationKeyspace, vscw.destinationShard,
*healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency)
vscw.destinationShardWatchers = append(vscw.destinationShardWatchers, watcher)
// Make sure we find a master for each destination shard and log it.
vscw.wr.Logger().Infof("Finding a MASTER tablet for each destination shard...")
waitCtx, waitCancel := context.WithTimeout(ctx, 10*time.Second)
defer waitCancel()
if err := discovery.WaitForTablets(waitCtx, vscw.healthCheck,
vscw.cell, vscw.destinationKeyspace, vscw.destinationShard, []topodatapb.TabletType{topodatapb.TabletType_MASTER}); err != nil {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v: %v", vscw.destinationKeyspace, vscw.destinationShard, err)
}
masters := discovery.GetCurrentMaster(
vscw.healthCheck.GetTabletStatsFromTarget(vscw.destinationKeyspace, vscw.destinationShard, topodatapb.TabletType_MASTER))
if len(masters) == 0 {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v in HealthCheck: empty TabletStats list", vscw.destinationKeyspace, vscw.destinationShard)
}
master := masters[0]
// Get the MySQL database name of the tablet.
keyspaceAndShard := topoproto.KeyspaceShardString(vscw.destinationKeyspace, vscw.destinationShard)
vscw.destinationDbNames[keyspaceAndShard] = topoproto.TabletDbName(master.Tablet)
// TODO(mberlin): Verify on the destination master that the
// _vt.blp_checkpoint table has the latest schema.
vscw.wr.Logger().Infof("Using tablet %v as destination master for %v/%v", topoproto.TabletAliasString(master.Tablet.Alias), vscw.destinationKeyspace, vscw.destinationShard)
vscw.wr.Logger().Infof("NOTE: The used master of a destination shard might change over the course of the copy e.g. due to a reparent. The HealthCheck module will track and log master changes and any error message will always refer the actually used master address.")
return nil
}
// findTargets phase:
// - find one rdonly in the source shard
// - mark it as 'worker' pointing back to us
// - get the aliases of all the targets
func (scw *LegacySplitCloneWorker) findTargets(ctx context.Context) error {
scw.setState(WorkerStateFindTargets)
var err error
// find an appropriate tablet in the source shards
scw.sourceAliases = make([]*topodatapb.TabletAlias, len(scw.sourceShards))
for i, si := range scw.sourceShards {
scw.sourceAliases[i], err = FindWorkerTablet(ctx, scw.wr, scw.cleaner, scw.healthCheck, scw.cell, si.Keyspace(), si.ShardName(), scw.minHealthyRdonlyTablets)
if err != nil {
return fmt.Errorf("FindWorkerTablet() failed for %v/%v/%v: %v", scw.cell, si.Keyspace(), si.ShardName(), err)
}
scw.wr.Logger().Infof("Using tablet %v as source for %v/%v", topoproto.TabletAliasString(scw.sourceAliases[i]), si.Keyspace(), si.ShardName())
}
// get the tablet info for them, and stop their replication
scw.sourceTablets = make([]*topodatapb.Tablet, len(scw.sourceAliases))
for i, alias := range scw.sourceAliases {
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
ti, err := scw.wr.TopoServer().GetTablet(shortCtx, alias)
cancel()
if err != nil {
return fmt.Errorf("cannot read tablet %v: %v", topoproto.TabletAliasString(alias), err)
}
scw.sourceTablets[i] = ti.Tablet
shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout)
err = scw.wr.TabletManagerClient().StopSlave(shortCtx, scw.sourceTablets[i])
cancel()
if err != nil {
return fmt.Errorf("cannot stop replication on tablet %v", topoproto.TabletAliasString(alias))
}
wrangler.RecordStartSlaveAction(scw.cleaner, scw.sourceTablets[i])
}
// Initialize healthcheck and add destination shards to it.
scw.healthCheck = discovery.NewHealthCheck(*remoteActionsTimeout, *healthcheckRetryDelay, *healthCheckTimeout)
for _, si := range scw.destinationShards {
watcher := discovery.NewShardReplicationWatcher(scw.wr.TopoServer(), scw.healthCheck,
scw.cell, si.Keyspace(), si.ShardName(),
*healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency)
scw.destinationShardWatchers = append(scw.destinationShardWatchers, watcher)
}
// Make sure we find a master for each destination shard and log it.
scw.wr.Logger().Infof("Finding a MASTER tablet for each destination shard...")
for _, si := range scw.destinationShards {
waitCtx, waitCancel := context.WithTimeout(ctx, 10*time.Second)
defer waitCancel()
if err := discovery.WaitForTablets(waitCtx, scw.healthCheck,
scw.cell, si.Keyspace(), si.ShardName(), []topodatapb.TabletType{topodatapb.TabletType_MASTER}); err != nil {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v: %v", si.Keyspace(), si.ShardName(), err)
}
masters := discovery.GetCurrentMaster(
scw.healthCheck.GetTabletStatsFromTarget(si.Keyspace(), si.ShardName(), topodatapb.TabletType_MASTER))
if len(masters) == 0 {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v in HealthCheck: empty TabletStats list", si.Keyspace(), si.ShardName())
}
master := masters[0]
// Get the MySQL database name of the tablet.
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
ti, err := scw.wr.TopoServer().GetTablet(shortCtx, master.Tablet.Alias)
cancel()
if err != nil {
return fmt.Errorf("cannot get the TabletInfo for destination master (%v) to find out its db name: %v", topoproto.TabletAliasString(master.Tablet.Alias), err)
}
keyspaceAndShard := topoproto.KeyspaceShardString(si.Keyspace(), si.ShardName())
scw.destinationDbNames[keyspaceAndShard] = ti.DbName()
// TODO(mberlin): Verify on the destination master that the
// _vt.blp_checkpoint table has the latest schema.
scw.wr.Logger().Infof("Using tablet %v as destination master for %v/%v", topoproto.TabletAliasString(master.Tablet.Alias), si.Keyspace(), si.ShardName())
}
scw.wr.Logger().Infof("NOTE: The used master of a destination shard might change over the course of the copy e.g. due to a reparent. The HealthCheck module will track and log master changes and any error message will always refer the actually used master address.")
// Set up the throttler for each destination shard.
for _, si := range scw.destinationShards {
keyspaceAndShard := topoproto.KeyspaceShardString(si.Keyspace(), si.ShardName())
t, err := throttler.NewThrottler(
keyspaceAndShard, "transactions", scw.destinationWriterCount, scw.maxTPS, throttler.ReplicationLagModuleDisabled)
if err != nil {
return fmt.Errorf("cannot instantiate throttler: %v", err)
}
scw.destinationThrottlers[keyspaceAndShard] = t
}
return nil
}