// DeleteTablet removes a tablet from a shard. // - if allowMaster is set, we can Delete a master tablet (and clear // its record from the Shard record if it was the master). // - if skipRebuild is set, we do not rebuild the serving graph. func (wr *Wrangler) DeleteTablet(ctx context.Context, tabletAlias *topodatapb.TabletAlias, allowMaster, skipRebuild bool) error { // load the tablet, see if we'll need to rebuild ti, err := wr.ts.GetTablet(ctx, tabletAlias) if err != nil { return err } rebuildRequired := ti.IsInServingGraph() wasMaster := ti.Type == topodatapb.TabletType_MASTER if wasMaster && !allowMaster { return fmt.Errorf("cannot delete tablet %v as it is a master, use allow_master flag", topoproto.TabletAliasString(tabletAlias)) } // remove the record and its replication graph entry if err := topotools.DeleteTablet(ctx, wr.ts, ti.Tablet); err != nil { return err } // update the Shard object if the master was scrapped. // we lock the shard to not conflict with reparent operations. if wasMaster { actionNode := actionnode.UpdateShard() lockPath, err := wr.lockShard(ctx, ti.Keyspace, ti.Shard, actionNode) if err != nil { return err } // update the shard record's master if _, err := wr.ts.UpdateShardFields(ctx, ti.Keyspace, ti.Shard, func(s *topodatapb.Shard) error { if !topoproto.TabletAliasEqual(s.MasterAlias, tabletAlias) { wr.Logger().Warningf("Deleting master %v from shard %v/%v but master in Shard object was %v", topoproto.TabletAliasString(tabletAlias), ti.Keyspace, ti.Shard, topoproto.TabletAliasString(s.MasterAlias)) return topo.ErrNoUpdateNeeded } s.MasterAlias = nil return nil }); err != nil { return wr.unlockShard(ctx, ti.Keyspace, ti.Shard, actionNode, lockPath, err) } // and unlock if err := wr.unlockShard(ctx, ti.Keyspace, ti.Shard, actionNode, lockPath, err); err != nil { return err } } // and rebuild the original shard if needed if !rebuildRequired { wr.Logger().Infof("Rebuild not required") return nil } if skipRebuild { wr.Logger().Warningf("Rebuild required, but skipping it") return nil } _, err = wr.RebuildShardGraph(ctx, ti.Keyspace, ti.Shard, []string{ti.Alias.Cell}) return err }
// SourceShardAdd will add a new SourceShard inside a shard func (wr *Wrangler) SourceShardAdd(ctx context.Context, keyspace, shard string, uid uint32, skeyspace, sshard string, keyRange key.KeyRange, tables []string) error { actionNode := actionnode.UpdateShard() lockPath, err := wr.lockShard(ctx, keyspace, shard, actionNode) if err != nil { return err } err = wr.sourceShardAdd(ctx, keyspace, shard, uid, skeyspace, sshard, keyRange, tables) return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err) }
// SourceShardDelete will delete a SourceShard inside a shard, by index. func (wr *Wrangler) SourceShardDelete(ctx context.Context, keyspace, shard string, uid uint32) error { actionNode := actionnode.UpdateShard() lockPath, err := wr.lockShard(ctx, keyspace, shard, actionNode) if err != nil { return err } err = wr.sourceShardDelete(ctx, keyspace, shard, uid) return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err) }
// RemoveShardCell will remove a cell from the Cells list in a shard. // // It will first check the shard has no tablets there. If 'force' is // specified, it will remove the cell even when the tablet map cannot // be retrieved. This is intended to be used when a cell is completely // down and its topology server cannot even be reached. // // If 'recursive' is specified, it will delete any tablets in the cell/shard, // with the assumption that the tablet processes have already been terminated. func (wr *Wrangler) RemoveShardCell(ctx context.Context, keyspace, shard, cell string, force, recursive bool) error { actionNode := actionnode.UpdateShard() lockPath, err := wr.lockShard(ctx, keyspace, shard, actionNode) if err != nil { return err } err = wr.removeShardCell(ctx, keyspace, shard, cell, force, recursive) return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err) }
// SetShardBlacklistedTables changes the BlacklistedTablesMap // parameter of a shard. It does not rebuild any serving graph or do // any consistency check. func (wr *Wrangler) SetShardBlacklistedTables(keyspace, shard string, tabletType topo.TabletType, tables []string) error { actionNode := actionnode.UpdateShard() lockPath, err := wr.lockShard(keyspace, shard, actionNode) if err != nil { return err } err = wr.setShardBlacklistedTables(keyspace, shard, tabletType, tables) return wr.unlockShard(keyspace, shard, actionNode, lockPath, err) }
// updateShardCellsAndMaster will update the 'Cells' and possibly // MasterAlias records for the shard, if needed. func (wr *Wrangler) updateShardCellsAndMaster(ctx context.Context, si *topo.ShardInfo, tabletAlias topo.TabletAlias, tabletType topo.TabletType, force bool) error { // See if we need to update the Shard: // - add the tablet's cell to the shard's Cells if needed // - change the master if needed shardUpdateRequired := false if !si.HasCell(tabletAlias.Cell) { shardUpdateRequired = true } if tabletType == topo.TYPE_MASTER && si.MasterAlias != tabletAlias { shardUpdateRequired = true } if !shardUpdateRequired { return nil } actionNode := actionnode.UpdateShard() keyspace := si.Keyspace() shard := si.ShardName() lockPath, err := wr.lockShard(ctx, keyspace, shard, actionNode) if err != nil { return err } // re-read the shard with the lock si, err = wr.ts.GetShard(ctx, keyspace, shard) if err != nil { return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err) } // update it wasUpdated := false if !si.HasCell(tabletAlias.Cell) { si.Cells = append(si.Cells, tabletAlias.Cell) wasUpdated = true } if tabletType == topo.TYPE_MASTER && si.MasterAlias != tabletAlias { if !si.MasterAlias.IsZero() && !force { return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, keyspace, shard)) } si.MasterAlias = tabletAlias wasUpdated = true } if wasUpdated { // write it back if err := topo.UpdateShard(ctx, wr.ts, si); err != nil { return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err) } } // and unlock return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err) }
// SetShardTabletControl changes the TabletControl records // for a shard. It does not rebuild any serving graph or do // cross-shard consistency check. // - if disableQueryService is set, tables has to be empty // - if disableQueryService is not set, and tables is empty, we remove // the TabletControl record for the cells func (wr *Wrangler) SetShardTabletControl(ctx context.Context, keyspace, shard string, tabletType topo.TabletType, cells []string, remove, disableQueryService bool, tables []string) error { if disableQueryService && len(tables) > 0 { return fmt.Errorf("SetShardTabletControl cannot have both DisableQueryService set and tables set") } actionNode := actionnode.UpdateShard() lockPath, err := wr.lockShard(ctx, keyspace, shard, actionNode) if err != nil { return err } err = wr.setShardTabletControl(ctx, keyspace, shard, tabletType, cells, remove, disableQueryService, tables) return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err) }
// updateShardCellsAndMaster will update the 'Cells' and possibly // MasterAlias records for the shard, if needed. func (wr *Wrangler) updateShardCellsAndMaster(ctx context.Context, si *topo.ShardInfo, tabletAlias *topodatapb.TabletAlias, tabletType topodatapb.TabletType, allowMasterOverride bool) error { // See if we need to update the Shard: // - add the tablet's cell to the shard's Cells if needed // - change the master if needed shardUpdateRequired := false if !si.HasCell(tabletAlias.Cell) { shardUpdateRequired = true } if tabletType == topodatapb.TabletType_MASTER && !topoproto.TabletAliasEqual(si.MasterAlias, tabletAlias) { shardUpdateRequired = true } if !shardUpdateRequired { return nil } // we do need to update the shard, lock it to not interfere with // reparenting operations. actionNode := actionnode.UpdateShard() keyspace := si.Keyspace() shard := si.ShardName() lockPath, err := wr.lockShard(ctx, keyspace, shard, actionNode) if err != nil { return err } // run the update _, err = wr.ts.UpdateShardFields(ctx, keyspace, shard, func(s *topodatapb.Shard) error { wasUpdated := false if !topoproto.ShardHasCell(s, tabletAlias.Cell) { s.Cells = append(s.Cells, tabletAlias.Cell) wasUpdated = true } if tabletType == topodatapb.TabletType_MASTER && !topoproto.TabletAliasEqual(s.MasterAlias, tabletAlias) { if !topoproto.TabletAliasIsZero(s.MasterAlias) && !allowMasterOverride { return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", topoproto.TabletAliasString(s.MasterAlias), keyspace, shard) } s.MasterAlias = tabletAlias wasUpdated = true } if !wasUpdated { return topo.ErrNoUpdateNeeded } return nil }) return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err) }
// InitTablet initializes the tablet record if necessary. func (agent *ActionAgent) InitTablet(port, gRPCPort int32) error { // only enabled if one of init_tablet_type (when healthcheck // is disabled) or init_keyspace (when healthcheck is enabled) // is passed in, then check other parameters if *initTabletType == "" && *initKeyspace == "" { return nil } // figure out our default target type var tabletType topodatapb.TabletType if *initTabletType != "" { if *targetTabletType != "" { log.Fatalf("cannot specify both target_tablet_type and init_tablet_type parameters (as they might conflict)") } // use the type specified on the command line var err error tabletType, err = topoproto.ParseTabletType(*initTabletType) if err != nil { log.Fatalf("Invalid init tablet type %v: %v", *initTabletType, err) } if tabletType == topodatapb.TabletType_MASTER { // We disallow MASTER, so we don't have to change // shard.MasterAlias, and deal with the corner cases. log.Fatalf("init_tablet_type cannot be %v", tabletType) } } else if *targetTabletType != "" { if strings.ToUpper(*targetTabletType) == topodatapb.TabletType_name[int32(topodatapb.TabletType_MASTER)] { log.Fatalf("target_tablet_type cannot be '%v'. Use '%v' instead.", tabletType, topodatapb.TabletType_REPLICA) } // use spare, the healthcheck will turn us into what // we need to be eventually tabletType = topodatapb.TabletType_SPARE } else { log.Fatalf("if init tablet is enabled, one of init_tablet_type or target_tablet_type needs to be specified") } // create a context for this whole operation ctx, cancel := context.WithTimeout(agent.batchCtx, *initTimeout) defer cancel() // since we're assigned to a shard, make sure it exists, see if // we are its master, and update its cells list if necessary if *initKeyspace == "" || *initShard == "" { log.Fatalf("if init tablet is enabled and the target type is not idle, init_keyspace and init_shard also need to be specified") } shard, _, err := topo.ValidateShardName(*initShard) if err != nil { log.Fatalf("cannot validate shard name: %v", err) } log.Infof("Reading shard record %v/%v", *initKeyspace, shard) // read the shard, create it if necessary si, err := topotools.GetOrCreateShard(ctx, agent.TopoServer, *initKeyspace, shard) if err != nil { return fmt.Errorf("InitTablet cannot GetOrCreateShard shard: %v", err) } if si.MasterAlias != nil && topoproto.TabletAliasEqual(si.MasterAlias, agent.TabletAlias) { // we are the current master for this shard (probably // means the master tablet process was just restarted), // so InitTablet as master. tabletType = topodatapb.TabletType_MASTER } // See if we need to add the tablet's cell to the shard's cell // list. If we do, it has to be under the shard lock. if !si.HasCell(agent.TabletAlias.Cell) { actionNode := actionnode.UpdateShard() lockPath, err := actionNode.LockShard(ctx, agent.TopoServer, *initKeyspace, shard) if err != nil { return fmt.Errorf("LockShard(%v/%v) failed: %v", *initKeyspace, shard, err) } // re-read the shard with the lock si, err = agent.TopoServer.GetShard(ctx, *initKeyspace, shard) if err != nil { return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err) } // see if we really need to update it now if !si.HasCell(agent.TabletAlias.Cell) { si.Cells = append(si.Cells, agent.TabletAlias.Cell) // write it back if err := agent.TopoServer.UpdateShard(ctx, si); err != nil { return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err) } } // and unlock if err := actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, nil); err != nil { return err } } log.Infof("Initializing the tablet for type %v", tabletType) // figure out the hostname hostname := *tabletHostname if hostname != "" { log.Infof("Using hostname: %v from -tablet_hostname flag.", hostname) } else { hostname, err := netutil.FullyQualifiedHostname() if err != nil { return err } log.Infof("Using detected machine hostname: %v To change this, fix your machine network configuration or override it with -tablet_hostname.", hostname) } // create and populate tablet record tablet := &topodatapb.Tablet{ Alias: agent.TabletAlias, Hostname: hostname, PortMap: make(map[string]int32), Keyspace: *initKeyspace, Shard: *initShard, Type: tabletType, DbNameOverride: *initDbNameOverride, Tags: initTags, } if port != 0 { tablet.PortMap["vt"] = port } if gRPCPort != 0 { tablet.PortMap["grpc"] = gRPCPort } if err := topo.TabletComplete(tablet); err != nil { return fmt.Errorf("InitTablet TabletComplete failed: %v", err) } // now try to create the record err = agent.TopoServer.CreateTablet(ctx, tablet) switch err { case nil: // it worked, we're good, can update the replication graph if err := topo.UpdateTabletReplicationData(ctx, agent.TopoServer, tablet); err != nil { return fmt.Errorf("UpdateTabletReplicationData failed: %v", err) } case topo.ErrNodeExists: // The node already exists, will just try to update // it. So we read it first. oldTablet, err := agent.TopoServer.GetTablet(ctx, tablet.Alias) if err != nil { return fmt.Errorf("InitTablet failed to read existing tablet record: %v", err) } // Sanity check the keyspace and shard if oldTablet.Keyspace != tablet.Keyspace || oldTablet.Shard != tablet.Shard { return fmt.Errorf("InitTablet failed because existing tablet keyspace and shard %v/%v differ from the provided ones %v/%v", oldTablet.Keyspace, oldTablet.Shard, tablet.Keyspace, tablet.Shard) } // And overwrite the rest *(oldTablet.Tablet) = *tablet if err := agent.TopoServer.UpdateTablet(ctx, oldTablet); err != nil { return fmt.Errorf("UpdateTablet failed: %v", err) } // Note we don't need to UpdateTabletReplicationData // as the tablet already existed with the right data // in the replication graph default: return fmt.Errorf("CreateTablet failed: %v", err) } // and now update the serving graph. Note we do that in any case, // to clean any inaccurate record from any part of the serving graph. if err := topotools.UpdateTabletEndpoints(ctx, agent.TopoServer, tablet); err != nil { return fmt.Errorf("UpdateTabletEndpoints failed: %v", err) } return nil }
// Scrap a tablet. If force is used, we write to topo.Server // directly and don't remote-execute the command. // // If we scrap the master for a shard, we will clear its record // from the Shard object (only if that was the right master) func (wr *Wrangler) Scrap(ctx context.Context, tabletAlias topo.TabletAlias, force, skipRebuild bool) error { // load the tablet, see if we'll need to rebuild ti, err := wr.ts.GetTablet(ctx, tabletAlias) if err != nil { return err } rebuildRequired := ti.IsInServingGraph() wasMaster := ti.Type == topo.TYPE_MASTER if force { err = topotools.Scrap(ctx, wr.ts, ti.Alias, force) } else { err = wr.tmc.Scrap(ctx, ti) } if err != nil { return err } if !rebuildRequired { wr.Logger().Infof("Rebuild not required") return nil } if skipRebuild { wr.Logger().Warningf("Rebuild required, but skipping it") return nil } // update the Shard object if the master was scrapped if wasMaster { actionNode := actionnode.UpdateShard() lockPath, err := wr.lockShard(ctx, ti.Keyspace, ti.Shard, actionNode) if err != nil { return err } // read the shard with the lock si, err := wr.ts.GetShard(ctx, ti.Keyspace, ti.Shard) if err != nil { return wr.unlockShard(ctx, ti.Keyspace, ti.Shard, actionNode, lockPath, err) } // update it if the right alias is there if topo.TabletAliasEqual(si.MasterAlias, topo.TabletAliasToProto(tabletAlias)) { si.MasterAlias = nil // write it back if err := topo.UpdateShard(ctx, wr.ts, si); err != nil { return wr.unlockShard(ctx, ti.Keyspace, ti.Shard, actionNode, lockPath, err) } } else { wr.Logger().Warningf("Scrapping master %v from shard %v/%v but master in Shard object was %v", tabletAlias, ti.Keyspace, ti.Shard, si.MasterAlias) } // and unlock if err := wr.unlockShard(ctx, ti.Keyspace, ti.Shard, actionNode, lockPath, err); err != nil { return err } } // and rebuild the original shard _, err = wr.RebuildShardGraph(ctx, ti.Keyspace, ti.Shard, []string{ti.Alias.Cell}) return err }
// Scrap a tablet. If force is used, we write to topo.Server // directly and don't remote-execute the command. // // If we scrap the master for a shard, we will clear its record // from the Shard object (only if that was the right master) func (wr *Wrangler) Scrap(tabletAlias topo.TabletAlias, force, skipRebuild bool) (actionPath string, err error) { // load the tablet, see if we'll need to rebuild ti, err := wr.ts.GetTablet(tabletAlias) if err != nil { return "", err } rebuildRequired := ti.Tablet.IsInServingGraph() wasMaster := ti.Type == topo.TYPE_MASTER if force { err = topotools.Scrap(wr.ts, ti.Alias, force) } else { actionPath, err = wr.ai.Scrap(ti.Alias) } if err != nil { return "", err } if !rebuildRequired { log.Infof("Rebuild not required") return } if skipRebuild { log.Warningf("Rebuild required, but skipping it") return } // wait for the remote Scrap if necessary if actionPath != "" { err = wr.WaitForCompletion(actionPath) if err != nil { return "", err } } // update the Shard object if the master was scrapped if wasMaster { actionNode := actionnode.UpdateShard() lockPath, err := wr.lockShard(ti.Keyspace, ti.Shard, actionNode) if err != nil { return "", err } // read the shard with the lock si, err := wr.ts.GetShard(ti.Keyspace, ti.Shard) if err != nil { return "", wr.unlockShard(ti.Keyspace, ti.Shard, actionNode, lockPath, err) } // update it if the right alias is there if si.MasterAlias == tabletAlias { si.MasterAlias = topo.TabletAlias{} // write it back if err := topo.UpdateShard(wr.ts, si); err != nil { return "", wr.unlockShard(ti.Keyspace, ti.Shard, actionNode, lockPath, err) } } else { log.Warningf("Scrapping master %v from shard %v/%v but master in Shard object was %v", tabletAlias, ti.Keyspace, ti.Shard, si.MasterAlias) } // and unlock if err := wr.unlockShard(ti.Keyspace, ti.Shard, actionNode, lockPath, err); err != nil { return "", err } } // and rebuild the original shard / keyspace return "", wr.RebuildShardGraph(ti.Keyspace, ti.Shard, []string{ti.Alias.Cell}) }
// InitTablet creates or updates a tablet. If no parent is specified // in the tablet, and the tablet has a slave type, we will find the // appropriate parent. If createShardAndKeyspace is true and the // parent keyspace or shard don't exist, they will be created. If // update is true, and a tablet with the same ID exists, update it. // If Force is true, and a tablet with the same ID already exists, it // will be scrapped and deleted, and then recreated. func (wr *Wrangler) InitTablet(tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error { if err := tablet.Complete(); err != nil { return err } if tablet.IsInReplicationGraph() { // create the parent keyspace and shard if needed if createShardAndKeyspace { if err := wr.ts.CreateKeyspace(tablet.Keyspace, &topo.Keyspace{}); err != nil && err != topo.ErrNodeExists { return err } if err := topo.CreateShard(wr.ts, tablet.Keyspace, tablet.Shard); err != nil && err != topo.ErrNodeExists { return err } } // get the shard, checks a couple things si, err := wr.ts.GetShard(tablet.Keyspace, tablet.Shard) if err != nil { return fmt.Errorf("missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard") } if si.KeyRange != tablet.KeyRange { return fmt.Errorf("shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange) } if tablet.Type == topo.TYPE_MASTER && !si.MasterAlias.IsZero() && si.MasterAlias != tablet.Alias && !force { return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, tablet.Keyspace, tablet.Shard) } // see if we specified a parent, otherwise get it from the shard if tablet.Parent.IsZero() && tablet.Type.IsSlaveType() { if si.MasterAlias.IsZero() { return fmt.Errorf("trying to create tablet %v in shard %v/%v without a master", tablet.Alias, tablet.Keyspace, tablet.Shard) } tablet.Parent = si.MasterAlias } // See if we need to update the Shard: // - add the tablet's cell to the shard's Cells if needed // - change the master if needed shardUpdateRequired := false if !si.HasCell(tablet.Alias.Cell) { shardUpdateRequired = true } if tablet.Type == topo.TYPE_MASTER && si.MasterAlias != tablet.Alias { shardUpdateRequired = true } if shardUpdateRequired { actionNode := actionnode.UpdateShard() lockPath, err := wr.lockShard(tablet.Keyspace, tablet.Shard, actionNode) if err != nil { return err } // re-read the shard with the lock si, err = wr.ts.GetShard(tablet.Keyspace, tablet.Shard) if err != nil { return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err) } // update it wasUpdated := false if !si.HasCell(tablet.Alias.Cell) { si.Cells = append(si.Cells, tablet.Alias.Cell) wasUpdated = true } if tablet.Type == topo.TYPE_MASTER && si.MasterAlias != tablet.Alias { if !si.MasterAlias.IsZero() && !force { return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, tablet.Keyspace, tablet.Shard)) } si.MasterAlias = tablet.Alias wasUpdated = true } if wasUpdated { // write it back if err := wr.ts.UpdateShard(si); err != nil { return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err) } } // and unlock if err := wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err); err != nil { return err } // also create the cell's ShardReplication if err := wr.ts.CreateShardReplication(tablet.Alias.Cell, tablet.Keyspace, tablet.Shard, &topo.ShardReplication{}); err != nil && err != topo.ErrNodeExists { return err } } } err := topo.CreateTablet(wr.ts, tablet) if err != nil && err == topo.ErrNodeExists { // Try to update nicely, but if it fails fall back to force behavior. if update || force { oldTablet, err := wr.ts.GetTablet(tablet.Alias) if err != nil { log.Warningf("failed reading tablet %v: %v", tablet.Alias, err) } else { if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard { *(oldTablet.Tablet) = *tablet if err := topo.UpdateTablet(wr.ts, oldTablet); err != nil { log.Warningf("failed updating tablet %v: %v", tablet.Alias, err) // now fall through the Scrap case } else { if !tablet.IsInReplicationGraph() { return nil } if err := topo.CreateTabletReplicationData(wr.ts, tablet); err != nil { log.Warningf("failed updating tablet replication data for %v: %v", tablet.Alias, err) // now fall through the Scrap case } else { return nil } } } } } if force { if _, err = wr.Scrap(tablet.Alias, force, false); err != nil { log.Errorf("failed scrapping tablet %v: %v", tablet.Alias, err) return err } if err := wr.ts.DeleteTablet(tablet.Alias); err != nil { // we ignore this log.Errorf("failed deleting tablet %v: %v", tablet.Alias, err) } return topo.CreateTablet(wr.ts, tablet) } } return err }