// findDestinationMasters finds for each destination shard the current master.
func (scw *SplitCloneWorker) findDestinationMasters(ctx context.Context) error {
scw.setState(WorkerStateFindTargets)
// Make sure we find a master for each destination shard and log it.
scw.wr.Logger().Infof("Finding a MASTER tablet for each destination shard...")
for _, si := range scw.destinationShards {
waitCtx, waitCancel := context.WithTimeout(ctx, *waitForHealthyTabletsTimeout)
defer waitCancel()
if err := discovery.WaitForTablets(waitCtx, scw.healthCheck,
scw.cell, si.Keyspace(), si.ShardName(), []topodatapb.TabletType{topodatapb.TabletType_MASTER}); err != nil {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v (in cell: %v): %v", si.Keyspace(), si.ShardName(), scw.cell, err)
}
masters := discovery.GetCurrentMaster(
scw.healthCheck.GetTabletStatsFromTarget(si.Keyspace(), si.ShardName(), topodatapb.TabletType_MASTER))
if len(masters) == 0 {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v (in cell: %v) in HealthCheck: empty TabletStats list", si.Keyspace(), si.ShardName(), scw.cell)
}
master := masters[0]
// Get the MySQL database name of the tablet.
keyspaceAndShard := topoproto.KeyspaceShardString(si.Keyspace(), si.ShardName())
scw.destinationDbNames[keyspaceAndShard] = topoproto.TabletDbName(master.Tablet)
// TODO(mberlin): Verify on the destination master that the
// _vt.blp_checkpoint table has the latest schema.
scw.wr.Logger().Infof("Using tablet %v as destination master for %v/%v", topoproto.TabletAliasString(master.Tablet.Alias), si.Keyspace(), si.ShardName())
}
scw.wr.Logger().Infof("NOTE: The used master of a destination shard might change over the course of the copy e.g. due to a reparent. The HealthCheck module will track and log master changes and any error message will always refer the actually used master address.")
return nil
}
func waitForHealthyRdonlyTablets(ctx context.Context, wr *wrangler.Wrangler, healthCheck discovery.HealthCheck, cell, keyspace, shard string, minHealthyRdonlyTablets int, timeout time.Duration) ([]*discovery.TabletStats, error) {
busywaitCtx, busywaitCancel := context.WithTimeout(ctx, timeout)
defer busywaitCancel()
start := time.Now()
deadlineForLog, _ := busywaitCtx.Deadline()
wr.Logger().Infof("Waiting for enough healthy RDONLY tablets to become available in (%v,%v/%v). required: %v Waiting up to %.1f seconds.",
cell, keyspace, shard, minHealthyRdonlyTablets, deadlineForLog.Sub(time.Now()).Seconds())
// Wait for at least one RDONLY tablet initially before checking the list.
if err := discovery.WaitForTablets(busywaitCtx, healthCheck, cell, keyspace, shard, []topodatapb.TabletType{topodatapb.TabletType_RDONLY}); err != nil {
return nil, fmt.Errorf("error waiting for RDONLY tablets for (%v,%v/%v): %v", cell, keyspace, shard, err)
}
var healthyTablets []*discovery.TabletStats
for {
select {
case <-busywaitCtx.Done():
return nil, fmt.Errorf("not enough healthy RDONLY tablets to choose from in (%v,%v/%v), have %v healthy ones, need at least %v Context error: %v",
cell, keyspace, shard, len(healthyTablets), minHealthyRdonlyTablets, busywaitCtx.Err())
default:
}
healthyTablets = discovery.RemoveUnhealthyTablets(
healthCheck.GetTabletStatsFromTarget(keyspace, shard, topodatapb.TabletType_RDONLY))
if len(healthyTablets) >= minHealthyRdonlyTablets {
break
}
deadlineForLog, _ := busywaitCtx.Deadline()
wr.Logger().Infof("Waiting for enough healthy RDONLY tablets to become available (%v,%v/%v). available: %v required: %v Waiting up to %.1f more seconds.",
cell, keyspace, shard, len(healthyTablets), minHealthyRdonlyTablets, deadlineForLog.Sub(time.Now()).Seconds())
// Block for 1 second because 2 seconds is the -health_check_interval flag value in integration tests.
timer := time.NewTimer(1 * time.Second)
select {
case <-busywaitCtx.Done():
timer.Stop()
case <-timer.C:
}
}
wr.Logger().Infof("At least %v healthy RDONLY tablets are available in (%v,%v/%v) (required: %v). Took %.1f seconds to find this out.",
len(healthyTablets), cell, keyspace, shard, minHealthyRdonlyTablets, time.Now().Sub(start).Seconds())
return healthyTablets, nil
}
// Iteration is a single iteration for the player: get the current status,
// try to play, and plays until interrupted, or until an error occurs.
func (bpc *BinlogPlayerController) Iteration() (err error) {
defer func() {
if x := recover(); x != nil {
log.Errorf("%v: caught panic: %v\n%s", bpc, x, tb.Stack(4))
err = fmt.Errorf("panic: %v", x)
}
}()
// Check if the context is still good.
select {
case <-bpc.ctx.Done():
if bpc.ctx.Err() == context.Canceled {
// We were stopped. Break out of Loop().
return nil
}
return fmt.Errorf("giving up since the context is done: %v", bpc.ctx.Err())
default:
}
// Apply any special settings necessary for playback of binlogs.
// We do it on every iteration to be sure, in case MySQL was restarted.
if err := bpc.mysqld.EnableBinlogPlayback(); err != nil {
// We failed to apply the required settings, so we shouldn't keep going.
return err
}
// create the db connection, connect it
vtClient := bpc.vtClientFactory()
if err := vtClient.Connect(); err != nil {
return fmt.Errorf("can't connect to database: %v", err)
}
defer vtClient.Close()
// Read the start position
startPosition, flags, err := binlogplayer.ReadStartPosition(vtClient, bpc.sourceShard.Uid)
if err != nil {
return fmt.Errorf("can't read startPosition: %v", err)
}
// if we shouldn't start, we just error out and try again later
if strings.Index(flags, binlogplayer.BlpFlagDontStart) != -1 {
return fmt.Errorf("not starting because flag '%v' is set", binlogplayer.BlpFlagDontStart)
}
// wait for the tablet set (usefull for the first run at least, fast for next runs)
if err := discovery.WaitForTablets(bpc.ctx, bpc.healthCheck, bpc.cell, bpc.sourceShard.Keyspace, bpc.sourceShard.Shard, []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != nil {
return fmt.Errorf("error waiting for tablets for %v %v %v: %v", bpc.cell, bpc.sourceShard.String(), topodatapb.TabletType_REPLICA, err)
}
// Find the server list from the health check
addrs := discovery.RemoveUnhealthyTablets(
bpc.healthCheck.GetTabletStatsFromTarget(bpc.sourceShard.Keyspace, bpc.sourceShard.Shard, topodatapb.TabletType_REPLICA))
if len(addrs) == 0 {
return fmt.Errorf("can't find any healthy source tablet for %v %v %v", bpc.cell, bpc.sourceShard.String(), topodatapb.TabletType_REPLICA)
}
newServerIndex := rand.Intn(len(addrs))
tablet := addrs[newServerIndex].Tablet
// save our current server
bpc.playerMutex.Lock()
bpc.sourceTablet = tablet.Alias
bpc.lastError = nil
bpc.playerMutex.Unlock()
// check which kind of replication we're doing, tables or keyrange
if len(bpc.sourceShard.Tables) > 0 {
// tables, first resolve wildcards
tables, err := mysqlctl.ResolveTables(bpc.mysqld, bpc.dbName, bpc.sourceShard.Tables)
if err != nil {
return fmt.Errorf("failed to resolve table names: %v", err)
}
// tables, just get them
player, err := binlogplayer.NewBinlogPlayerTables(vtClient, tablet, tables, bpc.sourceShard.Uid, startPosition, bpc.stopPosition, bpc.binlogPlayerStats)
if err != nil {
return fmt.Errorf("NewBinlogPlayerTables failed: %v", err)
}
return player.ApplyBinlogEvents(bpc.ctx)
}
// the data we have to replicate is the intersection of the
// source keyrange and our keyrange
overlap, err := key.KeyRangesOverlap(bpc.sourceShard.KeyRange, bpc.keyRange)
if err != nil {
return fmt.Errorf("Source shard %v doesn't overlap destination shard %v", bpc.sourceShard.KeyRange, bpc.keyRange)
}
player, err := binlogplayer.NewBinlogPlayerKeyRange(vtClient, tablet, overlap, bpc.sourceShard.Uid, startPosition, bpc.stopPosition, bpc.binlogPlayerStats)
if err != nil {
return fmt.Errorf("NewBinlogPlayerKeyRange failed: %v", err)
}
return player.ApplyBinlogEvents(bpc.ctx)
}
// findTargets phase:
// - find one rdonly in the source shard
// - mark it as 'worker' pointing back to us
// - get the aliases of all the targets
func (vscw *VerticalSplitCloneWorker) findTargets(ctx context.Context) error {
vscw.setState(WorkerStateFindTargets)
// find an appropriate tablet in the source shard
var err error
vscw.sourceAlias, err = FindWorkerTablet(ctx, vscw.wr, vscw.cleaner, nil /* healthCheck */, vscw.cell, vscw.sourceKeyspace, "0", vscw.minHealthyRdonlyTablets)
if err != nil {
return fmt.Errorf("FindWorkerTablet() failed for %v/%v/0: %v", vscw.cell, vscw.sourceKeyspace, err)
}
vscw.wr.Logger().Infof("Using tablet %v as the source", topoproto.TabletAliasString(vscw.sourceAlias))
// get the tablet info for it
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
ti, err := vscw.wr.TopoServer().GetTablet(shortCtx, vscw.sourceAlias)
cancel()
if err != nil {
return fmt.Errorf("cannot read tablet %v: %v", topoproto.TabletAliasString(vscw.sourceAlias), err)
}
vscw.sourceTablet = ti.Tablet
// stop replication on it
shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout)
err = vscw.wr.TabletManagerClient().StopSlave(shortCtx, vscw.sourceTablet)
cancel()
if err != nil {
return fmt.Errorf("cannot stop replication on tablet %v", topoproto.TabletAliasString(vscw.sourceAlias))
}
wrangler.RecordStartSlaveAction(vscw.cleaner, vscw.sourceTablet)
// Initialize healthcheck and add destination shards to it.
vscw.healthCheck = discovery.NewHealthCheck(*remoteActionsTimeout, *healthcheckRetryDelay, *healthCheckTimeout)
watcher := discovery.NewShardReplicationWatcher(vscw.wr.TopoServer(), vscw.healthCheck,
vscw.cell, vscw.destinationKeyspace, vscw.destinationShard,
*healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency)
vscw.destinationShardWatchers = append(vscw.destinationShardWatchers, watcher)
// Make sure we find a master for each destination shard and log it.
vscw.wr.Logger().Infof("Finding a MASTER tablet for each destination shard...")
waitCtx, waitCancel := context.WithTimeout(ctx, 10*time.Second)
defer waitCancel()
if err := discovery.WaitForTablets(waitCtx, vscw.healthCheck,
vscw.cell, vscw.destinationKeyspace, vscw.destinationShard, []topodatapb.TabletType{topodatapb.TabletType_MASTER}); err != nil {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v: %v", vscw.destinationKeyspace, vscw.destinationShard, err)
}
masters := discovery.GetCurrentMaster(
vscw.healthCheck.GetTabletStatsFromTarget(vscw.destinationKeyspace, vscw.destinationShard, topodatapb.TabletType_MASTER))
if len(masters) == 0 {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v in HealthCheck: empty TabletStats list", vscw.destinationKeyspace, vscw.destinationShard)
}
master := masters[0]
// Get the MySQL database name of the tablet.
keyspaceAndShard := topoproto.KeyspaceShardString(vscw.destinationKeyspace, vscw.destinationShard)
vscw.destinationDbNames[keyspaceAndShard] = topoproto.TabletDbName(master.Tablet)
// TODO(mberlin): Verify on the destination master that the
// _vt.blp_checkpoint table has the latest schema.
vscw.wr.Logger().Infof("Using tablet %v as destination master for %v/%v", topoproto.TabletAliasString(master.Tablet.Alias), vscw.destinationKeyspace, vscw.destinationShard)
vscw.wr.Logger().Infof("NOTE: The used master of a destination shard might change over the course of the copy e.g. due to a reparent. The HealthCheck module will track and log master changes and any error message will always refer the actually used master address.")
return nil
}
func (wr *Wrangler) waitForDrainInCell(ctx context.Context, cell, keyspace, shard string, servedType topodatapb.TabletType,
retryDelay, healthCheckTopologyRefresh, healthcheckRetryDelay, healthCheckTimeout time.Duration) error {
hc := discovery.NewHealthCheck(healthCheckTimeout /* connectTimeout */, healthcheckRetryDelay, healthCheckTimeout)
defer hc.Close()
watcher := discovery.NewShardReplicationWatcher(wr.TopoServer(), hc, cell, keyspace, shard, healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency)
defer watcher.Stop()
if err := discovery.WaitForTablets(ctx, hc, cell, keyspace, shard, []topodatapb.TabletType{servedType}); err != nil {
return fmt.Errorf("%v: error waiting for initial %v tablets for %v/%v: %v", cell, servedType, keyspace, shard, err)
}
wr.Logger().Infof("%v: Waiting for %.1f seconds to make sure that the discovery module retrieves healthcheck information from all tablets.",
cell, healthCheckTimeout.Seconds())
// Wait at least for -vtctl_healthcheck_timeout to elapse to make sure that we
// see all healthy tablets. Otherwise, we might miss some tablets.
// It's safe to wait not longer for this because we would only miss slow
// tablets and vtgate would not serve from such tablets anyway.
time.Sleep(healthCheckTimeout)
// Now check the QPS rate of all tablets until the timeout expires.
startTime := time.Now()
for {
// map key: tablet uid
drainedHealthyTablets := make(map[uint32]*discovery.TabletStats)
notDrainedHealtyTablets := make(map[uint32]*discovery.TabletStats)
healthyTablets := discovery.RemoveUnhealthyTablets(
hc.GetTabletStatsFromTarget(keyspace, shard, servedType))
for _, ts := range healthyTablets {
if ts.Stats.Qps == 0.0 {
drainedHealthyTablets[ts.Tablet.Alias.Uid] = ts
} else {
notDrainedHealtyTablets[ts.Tablet.Alias.Uid] = ts
}
}
if len(drainedHealthyTablets) == len(healthyTablets) {
wr.Logger().Infof("%v: All %d healthy tablets were drained after %.1f seconds (not counting %.1f seconds for the initial wait).",
cell, len(healthyTablets), time.Now().Sub(startTime).Seconds(), healthCheckTimeout.Seconds())
break
}
// Continue waiting, sleep in between.
deadlineString := ""
if d, ok := ctx.Deadline(); ok {
deadlineString = fmt.Sprintf(" up to %.1f more seconds", d.Sub(time.Now()).Seconds())
}
wr.Logger().Infof("%v: Waiting%v for all healthy tablets to be drained (%d/%d done).",
cell, deadlineString, len(drainedHealthyTablets), len(healthyTablets))
timer := time.NewTimer(retryDelay)
select {
case <-ctx.Done():
timer.Stop()
var l []string
for _, ts := range notDrainedHealtyTablets {
l = append(l, formatTabletStats(ts))
}
return fmt.Errorf("%v: WaitForDrain failed for %v tablets in %v/%v. Only %d/%d tablets were drained. err: %v List of tablets which were not drained: %v",
cell, servedType, keyspace, shard, len(drainedHealthyTablets), len(healthyTablets), ctx.Err(), strings.Join(l, ";"))
case <-timer.C:
}
}
return nil
}
// findTargets phase:
// - find one rdonly in the source shard
// - mark it as 'worker' pointing back to us
// - get the aliases of all the targets
func (scw *LegacySplitCloneWorker) findTargets(ctx context.Context) error {
scw.setState(WorkerStateFindTargets)
var err error
// find an appropriate tablet in the source shards
scw.sourceAliases = make([]*topodatapb.TabletAlias, len(scw.sourceShards))
for i, si := range scw.sourceShards {
scw.sourceAliases[i], err = FindWorkerTablet(ctx, scw.wr, scw.cleaner, scw.healthCheck, scw.cell, si.Keyspace(), si.ShardName(), scw.minHealthyRdonlyTablets)
if err != nil {
return fmt.Errorf("FindWorkerTablet() failed for %v/%v/%v: %v", scw.cell, si.Keyspace(), si.ShardName(), err)
}
scw.wr.Logger().Infof("Using tablet %v as source for %v/%v", topoproto.TabletAliasString(scw.sourceAliases[i]), si.Keyspace(), si.ShardName())
}
// get the tablet info for them, and stop their replication
scw.sourceTablets = make([]*topodatapb.Tablet, len(scw.sourceAliases))
for i, alias := range scw.sourceAliases {
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
ti, err := scw.wr.TopoServer().GetTablet(shortCtx, alias)
cancel()
if err != nil {
return fmt.Errorf("cannot read tablet %v: %v", topoproto.TabletAliasString(alias), err)
}
scw.sourceTablets[i] = ti.Tablet
shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout)
err = scw.wr.TabletManagerClient().StopSlave(shortCtx, scw.sourceTablets[i])
cancel()
if err != nil {
return fmt.Errorf("cannot stop replication on tablet %v", topoproto.TabletAliasString(alias))
}
wrangler.RecordStartSlaveAction(scw.cleaner, scw.sourceTablets[i])
}
// Initialize healthcheck and add destination shards to it.
scw.healthCheck = discovery.NewHealthCheck(*remoteActionsTimeout, *healthcheckRetryDelay, *healthCheckTimeout)
for _, si := range scw.destinationShards {
watcher := discovery.NewShardReplicationWatcher(scw.wr.TopoServer(), scw.healthCheck,
scw.cell, si.Keyspace(), si.ShardName(),
*healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency)
scw.destinationShardWatchers = append(scw.destinationShardWatchers, watcher)
}
// Make sure we find a master for each destination shard and log it.
scw.wr.Logger().Infof("Finding a MASTER tablet for each destination shard...")
for _, si := range scw.destinationShards {
waitCtx, waitCancel := context.WithTimeout(ctx, 10*time.Second)
defer waitCancel()
if err := discovery.WaitForTablets(waitCtx, scw.healthCheck,
scw.cell, si.Keyspace(), si.ShardName(), []topodatapb.TabletType{topodatapb.TabletType_MASTER}); err != nil {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v: %v", si.Keyspace(), si.ShardName(), err)
}
masters := discovery.GetCurrentMaster(
scw.healthCheck.GetTabletStatsFromTarget(si.Keyspace(), si.ShardName(), topodatapb.TabletType_MASTER))
if len(masters) == 0 {
return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v in HealthCheck: empty TabletStats list", si.Keyspace(), si.ShardName())
}
master := masters[0]
// Get the MySQL database name of the tablet.
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
ti, err := scw.wr.TopoServer().GetTablet(shortCtx, master.Tablet.Alias)
cancel()
if err != nil {
return fmt.Errorf("cannot get the TabletInfo for destination master (%v) to find out its db name: %v", topoproto.TabletAliasString(master.Tablet.Alias), err)
}
keyspaceAndShard := topoproto.KeyspaceShardString(si.Keyspace(), si.ShardName())
scw.destinationDbNames[keyspaceAndShard] = ti.DbName()
// TODO(mberlin): Verify on the destination master that the
// _vt.blp_checkpoint table has the latest schema.
scw.wr.Logger().Infof("Using tablet %v as destination master for %v/%v", topoproto.TabletAliasString(master.Tablet.Alias), si.Keyspace(), si.ShardName())
}
scw.wr.Logger().Infof("NOTE: The used master of a destination shard might change over the course of the copy e.g. due to a reparent. The HealthCheck module will track and log master changes and any error message will always refer the actually used master address.")
// Set up the throttler for each destination shard.
for _, si := range scw.destinationShards {
keyspaceAndShard := topoproto.KeyspaceShardString(si.Keyspace(), si.ShardName())
t, err := throttler.NewThrottler(
keyspaceAndShard, "transactions", scw.destinationWriterCount, scw.maxTPS, throttler.ReplicationLagModuleDisabled)
if err != nil {
return fmt.Errorf("cannot instantiate throttler: %v", err)
}
scw.destinationThrottlers[keyspaceAndShard] = t
}
return nil
}
// TestRealtimeStatsWithQueryService uses fakeTablets and the fakeQueryService to
// copy the environment needed for the HealthCheck object.
func TestRealtimeStatsWithQueryService(t *testing.T) {
// Set up testing keyspace with 2 tablets within 2 cells.
keyspace := "ks"
shard := "-80"
tabletType := topodatapb.TabletType_REPLICA.String()
ctx := context.Background()
db := fakesqldb.Register()
ts := zktestserver.New(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
if err := ts.CreateKeyspace(context.Background(), keyspace, &topodatapb.Keyspace{
ShardingColumnName: "keyspace_id",
ShardingColumnType: topodatapb.KeyspaceIdType_UINT64,
}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
t1 := testlib.NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_REPLICA, db,
testlib.TabletKeyspaceShard(t, keyspace, shard))
t2 := testlib.NewFakeTablet(t, wr, "cell2", 1, topodatapb.TabletType_REPLICA, db,
testlib.TabletKeyspaceShard(t, keyspace, shard))
for _, ft := range []*(testlib.FakeTablet){t1, t2} {
ft.StartActionLoop(t, wr)
defer ft.StopActionLoop(t)
}
target := querypb.Target{
Keyspace: keyspace,
Shard: shard,
TabletType: topodatapb.TabletType_REPLICA,
}
fqs1 := fakes.NewStreamHealthQueryService(target)
fqs2 := fakes.NewStreamHealthQueryService(target)
grpcqueryservice.Register(t1.RPCServer, fqs1)
grpcqueryservice.Register(t2.RPCServer, fqs2)
fqs1.AddDefaultHealthResponse()
realtimeStats, err := newRealtimeStats(ts)
if err != nil {
t.Fatalf("newRealtimeStats error: %v", err)
}
if err := discovery.WaitForTablets(ctx, realtimeStats.healthCheck, "cell1", keyspace, shard, []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != nil {
t.Fatalf("waitForTablets failed: %v", err)
}
// Test 1: tablet1's stats should be updated with the one received by the HealthCheck object.
result := realtimeStats.tabletStatuses("cell1", keyspace, shard, tabletType)
got := result["0"].Stats
want := &querypb.RealtimeStats{
SecondsBehindMaster: 1,
}
if !proto.Equal(got, want) {
t.Errorf("got: %v, want: %v", got, want)
}
// Test 2: tablet1's stats should be updated with the new one received by the HealthCheck object.
fqs1.AddHealthResponseWithQPS(2.0)
want2 := &querypb.RealtimeStats{
SecondsBehindMaster: 1,
Qps: 2.0,
}
if err := checkStats(realtimeStats, "0", "cell1", keyspace, shard, tabletType, want2); err != nil {
t.Errorf("%v", err)
}
// Test 3: tablet2's stats should be updated with the one received by the HealthCheck object,
// leaving tablet1's stats unchanged.
fqs2.AddHealthResponseWithQPS(3.0)
want3 := &querypb.RealtimeStats{
SecondsBehindMaster: 1,
Qps: 3.0,
}
if err := checkStats(realtimeStats, "1", "cell2", keyspace, shard, tabletType, want3); err != nil {
t.Errorf("%v", err)
}
if err := checkStats(realtimeStats, "0", "cell1", keyspace, shard, tabletType, want2); err != nil {
t.Errorf("%v", err)
}
}
