func createTestAgent(ctx context.Context, t *testing.T) *ActionAgent {
ts := zktopo.NewTestServer(t, []string{cell})
if err := ts.CreateKeyspace(ctx, keyspace, &pb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
if err := topo.CreateShard(ctx, ts, keyspace, shard); err != nil {
t.Fatalf("CreateShard failed: %v", err)
}
port := 1234
tablet := &topo.Tablet{
Alias: tabletAlias,
Hostname: "host",
Portmap: map[string]int{
"vt": port,
},
IPAddr: "1.0.0.1",
Keyspace: keyspace,
Shard: shard,
Type: topo.TYPE_SPARE,
}
if err := topo.CreateTablet(ctx, ts, tablet); err != nil {
t.Fatalf("CreateTablet failed: %v", err)
}
mysqlDaemon := &mysqlctl.FakeMysqlDaemon{MysqlPort: 3306}
agent := NewTestActionAgent(ctx, ts, tabletAlias, port, 0, mysqlDaemon)
agent.BinlogPlayerMap = NewBinlogPlayerMap(ts, nil, nil)
agent.HealthReporter = &fakeHealthCheck{}
return agent
}
func addTablet(ctx context.Context, t *testing.T, ts topo.Server, uid int, cell string, tabletType topo.TabletType) *topo.TabletInfo {
tablet := &topo.Tablet{
Alias: topo.TabletAlias{Cell: cell, Uid: uint32(uid)},
Hostname: fmt.Sprintf("%vbsr%v", cell, uid),
IPAddr: fmt.Sprintf("212.244.218.%v", uid),
Portmap: map[string]int{
"vt": 3333 + 10*uid,
"mysql": 3334 + 10*uid,
},
Keyspace: testKeyspace,
Type: tabletType,
Shard: testShard,
}
if err := topo.CreateTablet(ctx, ts, tablet); err != nil {
t.Fatalf("CreateTablet: %v", err)
}
ti, err := ts.GetTablet(ctx, tablet.Alias)
if err != nil {
t.Fatalf("GetTablet: %v", err)
}
return ti
}
// InitTablet creates or updates a tablet. If no parent is specified
// in the tablet, and the tablet has a slave type, we will find the
// appropriate parent. If createShardAndKeyspace is true and the
// parent keyspace or shard don't exist, they will be created. If
// update is true, and a tablet with the same ID exists, update it.
// If Force is true, and a tablet with the same ID already exists, it
// will be scrapped and deleted, and then recreated.
func (wr *Wrangler) InitTablet(ctx context.Context, tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error {
if err := topo.TabletComplete(tablet); err != nil {
return err
}
if topo.IsInReplicationGraph(tablet.Type) {
// get the shard, possibly creating it
var err error
var si *topo.ShardInfo
if createShardAndKeyspace {
// create the parent keyspace and shard if needed
si, err = topotools.GetOrCreateShard(ctx, wr.ts, tablet.Keyspace, tablet.Shard)
} else {
si, err = wr.ts.GetShard(ctx, tablet.Keyspace, tablet.Shard)
if err == topo.ErrNoNode {
return fmt.Errorf("missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard")
}
}
// get the shard, checks a couple things
if err != nil {
return fmt.Errorf("cannot get (or create) shard %v/%v: %v", tablet.Keyspace, tablet.Shard, err)
}
if key.ProtoToKeyRange(si.KeyRange) != tablet.KeyRange {
return fmt.Errorf("shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange)
}
if tablet.Type == topo.TYPE_MASTER && !topo.TabletAliasIsZero(si.MasterAlias) && topo.ProtoToTabletAlias(si.MasterAlias) != tablet.Alias && !force {
return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, tablet.Keyspace, tablet.Shard)
}
// update the shard record if needed
if err := wr.updateShardCellsAndMaster(ctx, si, topo.TabletAliasToProto(tablet.Alias), topo.TabletTypeToProto(tablet.Type), force); err != nil {
return err
}
}
err := topo.CreateTablet(ctx, wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update || force {
oldTablet, err := wr.ts.GetTablet(ctx, tablet.Alias)
if err != nil {
wr.Logger().Warningf("failed reading tablet %v: %v", tablet.Alias, err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
if err := topo.UpdateTablet(ctx, wr.ts, oldTablet); err != nil {
wr.Logger().Warningf("failed updating tablet %v: %v", tablet.Alias, err)
// now fall through the Scrap case
} else {
if !topo.IsInReplicationGraph(tablet.Type) {
return nil
}
if err := topo.UpdateTabletReplicationData(ctx, wr.ts, tablet); err != nil {
wr.Logger().Warningf("failed updating tablet replication data for %v: %v", tablet.Alias, err)
// now fall through the Scrap case
} else {
return nil
}
}
}
}
}
if force {
if err = wr.Scrap(ctx, tablet.Alias, force, false); err != nil {
wr.Logger().Errorf("failed scrapping tablet %v: %v", tablet.Alias, err)
return err
}
if err := wr.ts.DeleteTablet(ctx, tablet.Alias); err != nil {
// we ignore this
wr.Logger().Errorf("failed deleting tablet %v: %v", tablet.Alias, err)
}
return topo.CreateTablet(ctx, wr.ts, tablet)
}
}
return err
}
func createSetup(t *testing.T) (topo.Server, topo.Server) {
fromConn := fakezk.NewConn()
fromTS := zktopo.NewServer(fromConn)
toConn := fakezk.NewConn()
toTS := zktopo.NewServer(toConn)
for _, zkPath := range []string{"/zk/test_cell/vt", "/zk/global/vt"} {
if _, err := zk.CreateRecursive(fromConn, zkPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
t.Fatalf("cannot init fromTS: %v", err)
}
}
// create a keyspace and a couple tablets
if err := fromTS.CreateKeyspace("test_keyspace"); err != nil {
t.Fatalf("cannot create keyspace: %v", err)
}
if err := topo.CreateTablet(fromTS, &topo.Tablet{
Cell: "test_cell",
Uid: 123,
Parent: topo.TabletAlias{},
Addr: "masterhost:8101",
SecureAddr: "masterhost:8102",
MysqlAddr: "masterhost:3306",
MysqlIpAddr: "1.2.3.4:3306",
Keyspace: "test_keyspace",
Shard: "0",
Type: topo.TYPE_MASTER,
State: topo.STATE_READ_WRITE,
DbNameOverride: "",
KeyRange: key.KeyRange{},
}); err != nil {
t.Fatalf("cannot create master tablet: %v", err)
}
if err := topo.CreateTablet(fromTS, &topo.Tablet{
Cell: "test_cell",
Uid: 234,
Parent: topo.TabletAlias{
Cell: "test_cell",
Uid: 123,
},
Addr: "slavehost:8101",
SecureAddr: "slavehost:8102",
MysqlAddr: "slavehost:3306",
MysqlIpAddr: "2.3.4.5:3306",
Keyspace: "test_keyspace",
Shard: "0",
Type: topo.TYPE_REPLICA,
State: topo.STATE_READ_ONLY,
DbNameOverride: "",
KeyRange: key.KeyRange{},
}); err != nil {
t.Fatalf("cannot create slave tablet: %v", err)
}
os.Setenv("ZK_CLIENT_CONFIG", "test_zk_client.json")
cells, err := fromTS.GetKnownCells()
if err != nil {
t.Fatalf("fromTS.GetKnownCells: %v", err)
}
log.Infof("Cells: %v", cells)
return fromTS, toTS
}
// InitTablet creates or updates a tablet. If no parent is specified
// in the tablet, and the tablet has a slave type, we will find the
// appropriate parent. If createShardAndKeyspace is true and the
// parent keyspace or shard don't exist, they will be created. If
// update is true, and a tablet with the same ID exists, update it.
// If Force is true, and a tablet with the same ID already exists, it
// will be scrapped and deleted, and then recreated.
func (wr *Wrangler) InitTablet(tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error {
if err := tablet.Complete(); err != nil {
return err
}
if tablet.IsInReplicationGraph() {
// create the parent keyspace and shard if needed
if createShardAndKeyspace {
if err := wr.ts.CreateKeyspace(tablet.Keyspace, &topo.Keyspace{}); err != nil && err != topo.ErrNodeExists {
return err
}
if err := topo.CreateShard(wr.ts, tablet.Keyspace, tablet.Shard); err != nil && err != topo.ErrNodeExists {
return err
}
}
// get the shard, checks a couple things
si, err := wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return fmt.Errorf("missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard")
}
if si.KeyRange != tablet.KeyRange {
return fmt.Errorf("shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange)
}
if tablet.Type == topo.TYPE_MASTER && !si.MasterAlias.IsZero() && si.MasterAlias != tablet.Alias && !force {
return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, tablet.Keyspace, tablet.Shard)
}
// see if we specified a parent, otherwise get it from the shard
if tablet.Parent.IsZero() && tablet.Type.IsSlaveType() {
if si.MasterAlias.IsZero() {
return fmt.Errorf("trying to create tablet %v in shard %v/%v without a master", tablet.Alias, tablet.Keyspace, tablet.Shard)
}
tablet.Parent = si.MasterAlias
}
// update the shard record if needed
if err := wr.updateShardCellsAndMaster(si, tablet.Alias, tablet.Type, force); err != nil {
return err
}
}
err := topo.CreateTablet(wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update || force {
oldTablet, err := wr.ts.GetTablet(tablet.Alias)
if err != nil {
log.Warningf("failed reading tablet %v: %v", tablet.Alias, err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
if err := topo.UpdateTablet(wr.ts, oldTablet); err != nil {
log.Warningf("failed updating tablet %v: %v", tablet.Alias, err)
// now fall through the Scrap case
} else {
if !tablet.IsInReplicationGraph() {
return nil
}
if err := topo.CreateTabletReplicationData(wr.ts, tablet); err != nil {
log.Warningf("failed updating tablet replication data for %v: %v", tablet.Alias, err)
// now fall through the Scrap case
} else {
return nil
}
}
}
}
}
if force {
if _, err = wr.Scrap(tablet.Alias, force, false); err != nil {
log.Errorf("failed scrapping tablet %v: %v", tablet.Alias, err)
return err
}
if err := wr.ts.DeleteTablet(tablet.Alias); err != nil {
// we ignore this
log.Errorf("failed deleting tablet %v: %v", tablet.Alias, err)
}
return topo.CreateTablet(wr.ts, tablet)
}
}
return err
}
// InitTablet initializes the tablet record if necessary.
func (agent *ActionAgent) InitTablet(port, securePort, gRPCPort int) error {
// only enabled if one of init_tablet_type (when healthcheck
// is disabled) or init_keyspace (when healthcheck is enabled)
// is passed in, then check other parameters
if *initTabletType == "" && *initKeyspace == "" {
return nil
}
// figure out our default target type
var tabletType topo.TabletType
if *initTabletType != "" {
if *targetTabletType != "" {
log.Fatalf("cannot specify both target_tablet_type and init_tablet_type parameters (as they might conflict)")
}
// use the type specified on the command line
tabletType = topo.TabletType(*initTabletType)
if !topo.IsTypeInList(tabletType, topo.AllTabletTypes) {
log.Fatalf("InitTablet encountered unknown init_tablet_type '%v'", *initTabletType)
}
if tabletType == topo.TYPE_MASTER || tabletType == topo.TYPE_SCRAP {
// We disallow TYPE_MASTER, so we don't have to change
// shard.MasterAlias, and deal with the corner cases.
// We also disallow TYPE_SCRAP, obviously.
log.Fatalf("init_tablet_type cannot be %v", tabletType)
}
} else if *targetTabletType != "" {
if tabletType := topo.TabletType(*targetTabletType); tabletType == topo.TYPE_MASTER {
log.Fatalf("target_tablet_type cannot be '%v'. Use '%v' instead.", tabletType, topo.TYPE_REPLICA)
}
// use spare, the healthcheck will turn us into what
// we need to be eventually
tabletType = topo.TYPE_SPARE
} else {
log.Fatalf("if init tablet is enabled, one of init_tablet_type or target_tablet_type needs to be specified")
}
// create a context for this whole operation
ctx, cancel := context.WithTimeout(agent.batchCtx, *initTimeout)
defer cancel()
// if we're assigned to a shard, make sure it exists, see if
// we are its master, and update its cells list if necessary
if tabletType != topo.TYPE_IDLE {
if *initKeyspace == "" || *initShard == "" {
log.Fatalf("if init tablet is enabled and the target type is not idle, init_keyspace and init_shard also need to be specified")
}
shard, _, err := topo.ValidateShardName(*initShard)
if err != nil {
log.Fatalf("cannot validate shard name: %v", err)
}
log.Infof("Reading shard record %v/%v", *initKeyspace, shard)
// read the shard, create it if necessary
si, err := topotools.GetOrCreateShard(ctx, agent.TopoServer, *initKeyspace, shard)
if err != nil {
return fmt.Errorf("InitTablet cannot GetOrCreateShard shard: %v", err)
}
if si.MasterAlias == agent.TabletAlias {
// we are the current master for this shard (probably
// means the master tablet process was just restarted),
// so InitTablet as master.
tabletType = topo.TYPE_MASTER
}
// See if we need to add the tablet's cell to the shard's cell
// list. If we do, it has to be under the shard lock.
if !si.HasCell(agent.TabletAlias.Cell) {
actionNode := actionnode.UpdateShard()
lockPath, err := actionNode.LockShard(ctx, agent.TopoServer, *initKeyspace, shard)
if err != nil {
return fmt.Errorf("LockShard(%v/%v) failed: %v", *initKeyspace, shard, err)
}
// re-read the shard with the lock
si, err = agent.TopoServer.GetShard(ctx, *initKeyspace, shard)
if err != nil {
return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err)
}
// see if we really need to update it now
if !si.HasCell(agent.TabletAlias.Cell) {
si.Cells = append(si.Cells, agent.TabletAlias.Cell)
// write it back
if err := topo.UpdateShard(ctx, agent.TopoServer, si); err != nil {
return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err)
}
}
// and unlock
if err := actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, nil); err != nil {
return err
}
}
}
log.Infof("Initializing the tablet for type %v", tabletType)
// figure out the hostname
hostname := *tabletHostname
if hostname == "" {
var err error
hostname, err = netutil.FullyQualifiedHostname()
if err != nil {
return err
}
}
// create and populate tablet record
tablet := &topo.Tablet{
Alias: agent.TabletAlias,
Hostname: hostname,
Portmap: make(map[string]int),
Keyspace: *initKeyspace,
Shard: *initShard,
Type: tabletType,
DbNameOverride: *initDbNameOverride,
Tags: initTags,
}
if port != 0 {
tablet.Portmap["vt"] = port
}
if securePort != 0 {
tablet.Portmap["vts"] = securePort
}
if gRPCPort != 0 {
tablet.Portmap["grpc"] = gRPCPort
}
if err := tablet.Complete(); err != nil {
return fmt.Errorf("InitTablet tablet.Complete failed: %v", err)
}
// now try to create the record
err := topo.CreateTablet(ctx, agent.TopoServer, tablet)
switch err {
case nil:
// it worked, we're good, can update the replication graph
if tablet.IsInReplicationGraph() {
if err := topo.UpdateTabletReplicationData(ctx, agent.TopoServer, tablet); err != nil {
return fmt.Errorf("UpdateTabletReplicationData failed: %v", err)
}
}
case topo.ErrNodeExists:
// The node already exists, will just try to update
// it. So we read it first.
oldTablet, err := agent.TopoServer.GetTablet(ctx, tablet.Alias)
if err != nil {
fmt.Errorf("InitTablet failed to read existing tablet record: %v", err)
}
// Sanity check the keyspace and shard
if oldTablet.Keyspace != tablet.Keyspace || oldTablet.Shard != tablet.Shard {
return fmt.Errorf("InitTablet failed because existing tablet keyspace and shard %v/%v differ from the provided ones %v/%v", oldTablet.Keyspace, oldTablet.Shard, tablet.Keyspace, tablet.Shard)
}
// And overwrite the rest
*(oldTablet.Tablet) = *tablet
if err := topo.UpdateTablet(ctx, agent.TopoServer, oldTablet); err != nil {
return fmt.Errorf("UpdateTablet failed: %v", err)
}
// Note we don't need to UpdateTabletReplicationData
// as the tablet already existed with the right data
// in the replication graph
default:
return fmt.Errorf("CreateTablet failed: %v", err)
}
// and now update the serving graph. Note we do that in any case,
// to clean any inaccurate record from any part of the serving graph.
if tabletType != topo.TYPE_IDLE {
if err := topotools.UpdateTabletEndpoints(ctx, agent.TopoServer, tablet); err != nil {
return fmt.Errorf("UpdateTabletEndpoints failed: %v", err)
}
}
return nil
}
// InitTablet creates or updates a tablet. If no parent is specified
// in the tablet, and the tablet has a slave type, we will find the
// appropriate parent. If createShardAndKeyspace is true and the
// parent keyspace or shard don't exist, they will be created. If
// update is true, and a tablet with the same ID exists, update it.
// If Force is true, and a tablet with the same ID already exists, it
// will be scrapped and deleted, and then recreated.
func (wr *Wrangler) InitTablet(tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error {
if err := tablet.Complete(); err != nil {
return err
}
if tablet.IsInReplicationGraph() {
// create the parent keyspace and shard if needed
if createShardAndKeyspace {
if err := wr.ts.CreateKeyspace(tablet.Keyspace); err != nil && err != topo.ErrNodeExists {
return err
}
if err := topo.CreateShard(wr.ts, tablet.Keyspace, tablet.Shard); err != nil && err != topo.ErrNodeExists {
return err
}
}
// get the shard, checks a couple things
si, err := wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return fmt.Errorf("missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard")
}
if si.KeyRange != tablet.KeyRange {
return fmt.Errorf("shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange)
}
if tablet.Type == topo.TYPE_MASTER && !si.MasterAlias.IsZero() && si.MasterAlias != tablet.Alias() && !force {
return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, tablet.Keyspace, tablet.Shard)
}
// see if we specified a parent, otherwise get it from the shard
if tablet.Parent.IsZero() && tablet.Type.IsSlaveType() {
if si.MasterAlias.IsZero() {
return fmt.Errorf("trying to create tablet %v in shard %v/%v without a master", tablet.Alias(), tablet.Keyspace, tablet.Shard)
}
tablet.Parent = si.MasterAlias
}
// See if we need to update the Shard:
// - add the tablet's cell to the shard's Cells if needed
// - change the master if needed
shardUpdateRequired := false
if !si.HasCell(tablet.Cell) {
shardUpdateRequired = true
}
if tablet.Type == topo.TYPE_MASTER && si.MasterAlias != tablet.Alias() {
shardUpdateRequired = true
}
if shardUpdateRequired {
actionNode := wr.ai.UpdateShard()
lockPath, err := wr.lockShard(tablet.Keyspace, tablet.Shard, actionNode)
if err != nil {
return err
}
// re-read the shard with the lock
si, err = wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err)
}
// update it
wasUpdated := false
if !si.HasCell(tablet.Cell) {
si.Cells = append(si.Cells, tablet.Cell)
wasUpdated = true
}
if tablet.Type == topo.TYPE_MASTER && si.MasterAlias != tablet.Alias() {
if !si.MasterAlias.IsZero() && !force {
return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, tablet.Keyspace, tablet.Shard))
}
si.MasterAlias = tablet.Alias()
wasUpdated = true
}
if wasUpdated {
// write it back
if err := wr.ts.UpdateShard(si); err != nil {
return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err)
}
}
// and unlock
if err := wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err); err != nil {
return err
}
// also create the cell's ShardReplication
if err := wr.ts.CreateShardReplication(tablet.Cell, tablet.Keyspace, tablet.Shard, &topo.ShardReplication{}); err != nil && err != topo.ErrNodeExists {
return err
}
}
}
err := topo.CreateTablet(wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update || force {
oldTablet, err := wr.ts.GetTablet(tablet.Alias())
if err != nil {
log.Warningf("failed reading tablet %v: %v", tablet.Alias(), err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
if err := topo.UpdateTablet(wr.ts, oldTablet); err != nil {
log.Warningf("failed updating tablet %v: %v", tablet.Alias(), err)
// now fall through the Scrap case
} else {
if !tablet.IsInReplicationGraph() {
return nil
}
if err := topo.CreateTabletReplicationData(wr.ts, tablet); err != nil {
log.Warningf("failed updating tablet replication data for %v: %v", tablet.Alias(), err)
// now fall through the Scrap case
} else {
return nil
}
}
}
}
}
if force {
if _, err = wr.Scrap(tablet.Alias(), force, false); err != nil {
log.Errorf("failed scrapping tablet %v: %v", tablet.Alias(), err)
return err
}
if err := wr.ts.DeleteTablet(tablet.Alias()); err != nil {
// we ignore this
log.Errorf("failed deleting tablet %v: %v", tablet.Alias(), err)
}
return topo.CreateTablet(wr.ts, tablet)
}
}
return err
}
// InitTablet creates or updates a tablet. If no parent is specified
// in the tablet, and the tablet has a slave type, we will find the
// appropriate parent. If createShardAndKeyspace is true and the
// parent keyspace or shard don't exist, they will be created. If
// update is true, and a tablet with the same ID exists, update it.
// If Force is true, and a tablet with the same ID already exists, it
// will be scrapped and deleted, and then recreated.
func (wr *Wrangler) InitTablet(tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error {
if err := tablet.Complete(); err != nil {
return err
}
if tablet.Parent.IsZero() && tablet.Type.IsSlaveType() {
parentAlias, err := wr.getMasterAlias(tablet.Keyspace, tablet.Shard)
if err != nil {
return err
}
tablet.Parent = parentAlias
}
if tablet.IsInReplicationGraph() {
// create the parent keyspace and shard if needed
if createShardAndKeyspace {
if err := wr.ts.CreateKeyspace(tablet.Keyspace); err != nil && err != topo.ErrNodeExists {
return err
}
if err := topo.CreateShard(wr.ts, tablet.Keyspace, tablet.Shard); err != nil && err != topo.ErrNodeExists {
return err
}
}
// get the shard, checks KeyRange is the same
si, err := wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return fmt.Errorf("Missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard")
}
if si.KeyRange != tablet.KeyRange {
return fmt.Errorf("Shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange)
}
}
err := topo.CreateTablet(wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update {
oldTablet, err := wr.ts.GetTablet(tablet.Alias())
if err != nil {
relog.Warning("failed reading tablet %v: %v", tablet.Alias(), err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
err := topo.UpdateTablet(wr.ts, oldTablet)
if err != nil {
relog.Warning("failed updating tablet %v: %v", tablet.Alias(), err)
} else {
return nil
}
}
}
}
if force {
if _, err = wr.Scrap(tablet.Alias(), force, false); err != nil {
relog.Error("failed scrapping tablet %v: %v", tablet.Alias(), err)
return err
}
if err := wr.ts.DeleteTablet(tablet.Alias()); err != nil {
// we ignore this
relog.Error("failed deleting tablet %v: %v", tablet.Alias(), err)
}
return topo.CreateTablet(wr.ts, tablet)
}
}
return err
}
func createSetup(ctx context.Context, t *testing.T) (topo.Server, topo.Server) {
fromConn := fakezk.NewConn()
fromTS := zktopo.NewServer(fromConn)
toConn := fakezk.NewConn()
toTS := zktopo.NewServer(toConn)
for _, zkPath := range []string{"/zk/test_cell/vt", "/zk/global/vt"} {
if _, err := zk.CreateRecursive(fromConn, zkPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
t.Fatalf("cannot init fromTS: %v", err)
}
}
// create a keyspace and a couple tablets
if err := fromTS.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil {
t.Fatalf("cannot create keyspace: %v", err)
}
if err := fromTS.CreateShard(ctx, "test_keyspace", "0", &pb.Shard{Cells: []string{"test_cell"}}); err != nil {
t.Fatalf("cannot create shard: %v", err)
}
if err := topo.CreateTablet(ctx, fromTS, &topo.Tablet{
Alias: topo.TabletAlias{
Cell: "test_cell",
Uid: 123,
},
Hostname: "masterhost",
IPAddr: "1.2.3.4",
Portmap: map[string]int{
"vt": 8101,
"gprc": 8102,
"mysql": 3306,
},
Keyspace: "test_keyspace",
Shard: "0",
Type: topo.TYPE_MASTER,
DbNameOverride: "",
KeyRange: key.KeyRange{},
}); err != nil {
t.Fatalf("cannot create master tablet: %v", err)
}
if err := topo.CreateTablet(ctx, fromTS, &topo.Tablet{
Alias: topo.TabletAlias{
Cell: "test_cell",
Uid: 234,
},
IPAddr: "2.3.4.5",
Portmap: map[string]int{
"vt": 8101,
"grpc": 8102,
"mysql": 3306,
},
Hostname: "slavehost",
Keyspace: "test_keyspace",
Shard: "0",
Type: topo.TYPE_REPLICA,
DbNameOverride: "",
KeyRange: key.KeyRange{},
}); err != nil {
t.Fatalf("cannot create slave tablet: %v", err)
}
os.Setenv("ZK_CLIENT_CONFIG", testfiles.Locate("topo_helpers_test_zk_client.json"))
cells, err := fromTS.GetKnownCells(ctx)
if err != nil {
t.Fatalf("fromTS.GetKnownCells: %v", err)
}
log.Infof("Cells: %v", cells)
return fromTS, toTS
}
// InitTablet creates or updates a tablet. If no parent is specified
// in the tablet, and the tablet has a slave type, we will find the
// appropriate parent. If createShardAndKeyspace is true and the
// parent keyspace or shard don't exist, they will be created. If
// update is true, and a tablet with the same ID exists, update it.
// If Force is true, and a tablet with the same ID already exists, it
// will be scrapped and deleted, and then recreated.
func (wr *Wrangler) InitTablet(tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error {
if err := tablet.Complete(); err != nil {
return err
}
if tablet.Parent.IsZero() && tablet.Type.IsSlaveType() {
parentAlias, err := wr.getMasterAlias(tablet.Keyspace, tablet.Shard)
if err != nil {
return err
}
tablet.Parent = parentAlias
}
if tablet.IsInReplicationGraph() {
// create the parent keyspace and shard if needed
if createShardAndKeyspace {
if err := wr.ts.CreateKeyspace(tablet.Keyspace); err != nil && err != topo.ErrNodeExists {
return err
}
if err := topo.CreateShard(wr.ts, tablet.Keyspace, tablet.Shard); err != nil && err != topo.ErrNodeExists {
return err
}
}
// get the shard, checks KeyRange is the same
si, err := wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return fmt.Errorf("Missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard")
}
if si.KeyRange != tablet.KeyRange {
return fmt.Errorf("Shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange)
}
// add the tablet's cell to the shard cell if needed
if !si.HasCell(tablet.Cell) {
actionNode := wr.ai.UpdateShard()
lockPath, err := wr.lockShard(tablet.Keyspace, tablet.Shard, actionNode)
if err != nil {
return err
}
// re-read the shard with the lock
si, err = wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err)
}
// update it
if !si.HasCell(tablet.Cell) {
si.Cells = append(si.Cells, tablet.Cell)
// write it back
if err := wr.ts.UpdateShard(si); err != nil {
return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err)
}
}
// and unlock
if err := wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err); err != nil {
return err
}
// also create the cell's ShardReplication
if err := wr.ts.CreateShardReplication(tablet.Cell, tablet.Keyspace, tablet.Shard, &topo.ShardReplication{}); err != nil && err != topo.ErrNodeExists {
return err
}
}
}
err := topo.CreateTablet(wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update {
oldTablet, err := wr.ts.GetTablet(tablet.Alias())
if err != nil {
log.Warningf("failed reading tablet %v: %v", tablet.Alias(), err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
err := topo.UpdateTablet(wr.ts, oldTablet)
if err != nil {
log.Warningf("failed updating tablet %v: %v", tablet.Alias(), err)
} else {
return nil
}
}
}
}
if force {
if _, err = wr.Scrap(tablet.Alias(), force, false); err != nil {
log.Errorf("failed scrapping tablet %v: %v", tablet.Alias(), err)
return err
}
if err := wr.ts.DeleteTablet(tablet.Alias()); err != nil {
// we ignore this
log.Errorf("failed deleting tablet %v: %v", tablet.Alias(), err)
}
return topo.CreateTablet(wr.ts, tablet)
}
}
return err
}
func TestAPI(t *testing.T) {
ctx := context.Background()
cells := []string{"cell1", "cell2"}
ts := zktopo.NewTestServer(t, cells)
actionRepo = NewActionRepository(ts)
initAPI(ctx, ts, actionRepo)
server := httptest.NewServer(nil)
defer server.Close()
// Populate topo.
ts.CreateKeyspace(ctx, "ks1", &pb.Keyspace{ShardingColumnName: "shardcol"})
ts.CreateShard(ctx, "ks1", "-80", &pb.Shard{
Cells: cells,
KeyRange: &pb.KeyRange{Start: nil, End: []byte{0x80}},
})
ts.CreateShard(ctx, "ks1", "80-", &pb.Shard{
Cells: cells,
KeyRange: &pb.KeyRange{Start: []byte{0x80}, End: nil},
})
topo.CreateTablet(ctx, ts, &pb.Tablet{
Alias: &pb.TabletAlias{Cell: "cell1", Uid: 100},
Keyspace: "ks1",
Shard: "-80",
Type: pb.TabletType_REPLICA,
KeyRange: &pb.KeyRange{Start: nil, End: []byte{0x80}},
PortMap: map[string]int32{"vt": 100},
})
topo.CreateTablet(ctx, ts, &pb.Tablet{
Alias: &pb.TabletAlias{Cell: "cell2", Uid: 200},
Keyspace: "ks1",
Shard: "-80",
Type: pb.TabletType_REPLICA,
KeyRange: &pb.KeyRange{Start: nil, End: []byte{0x80}},
PortMap: map[string]int32{"vt": 200},
})
topotools.RebuildShard(ctx, logutil.NewConsoleLogger(), ts, "ks1", "-80", cells, 10*time.Second)
// Populate fake actions.
actionRepo.RegisterKeyspaceAction("TestKeyspaceAction",
func(ctx context.Context, wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "TestKeyspaceAction Result", nil
})
actionRepo.RegisterShardAction("TestShardAction",
func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "TestShardAction Result", nil
})
actionRepo.RegisterTabletAction("TestTabletAction", "",
func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *pb.TabletAlias, r *http.Request) (string, error) {
return "TestTabletAction Result", nil
})
// Test cases.
table := []struct {
method, path, want string
}{
// Cells
{"GET", "cells", `["cell1","cell2"]`},
// Keyspaces
{"GET", "keyspaces", `["ks1"]`},
{"GET", "keyspaces/ks1", `{
"sharding_column_name": "shardcol"
}`},
{"POST", "keyspaces/ks1?action=TestKeyspaceAction", `{
"Name": "TestKeyspaceAction",
"Parameters": "ks1",
"Output": "TestKeyspaceAction Result",
"Error": false
}`},
// Shards
{"GET", "shards/ks1/", `["-80","80-"]`},
{"GET", "shards/ks1/-80", `{
"key_range": {"end":"gA=="},
"cells": ["cell1", "cell2"]
}`},
{"POST", "shards/ks1/-80?action=TestShardAction", `{
"Name": "TestShardAction",
"Parameters": "ks1/-80",
"Output": "TestShardAction Result",
"Error": false
}`},
// Tablets
{"GET", "tablets/?shard=ks1%2F-80", `[
{"cell":"cell1","uid":100},
{"cell":"cell2","uid":200}
]`},
{"GET", "tablets/?cell=cell1", `[
{"cell":"cell1","uid":100}
]`},
{"GET", "tablets/?shard=ks1%2F-80&cell=cell2", `[
{"cell":"cell2","uid":200}
]`},
{"GET", "tablets/cell1-100", `{
"alias": {"cell": "cell1", "uid": 100},
"port_map": {"vt": 100},
"keyspace": "ks1",
"shard": "-80",
"key_range": {"end": "gA=="},
"type": 3
}`},
{"POST", "tablets/cell1-100?action=TestTabletAction", `{
"Name": "TestTabletAction",
"Parameters": "cell1-0000000100",
"Output": "TestTabletAction Result",
"Error": false
}`},
// EndPoints
{"GET", "endpoints/cell1/ks1/-80/", `[3]`},
{"GET", "endpoints/cell1/ks1/-80/replica", `{
"entries": [{
"uid": 100,
"port_map": {"vt": 100}
}]
}`},
}
for _, in := range table {
var resp *http.Response
var err error
switch in.method {
case "GET":
resp, err = http.Get(server.URL + apiPrefix + in.path)
case "POST":
resp, err = http.Post(server.URL+apiPrefix+in.path, "", nil)
default:
t.Errorf("[%v] unknown method: %v", in.path, in.method)
continue
}
if err != nil {
t.Errorf("[%v] http error: %v", in.path, err)
continue
}
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
t.Errorf("[%v] ioutil.ReadAll(resp.Body) error: %v", in.path, err)
continue
}
if got, want := compactJSON(body), compactJSON([]byte(in.want)); got != want {
t.Errorf("[%v] got %v, want %v", in.path, got, want)
}
}
}
// InitTablet will create or update a tablet. If not parent is
// specified, and the tablet created is a slave type, we will find the
// appropriate parent.
func (wr *Wrangler) InitTablet(tabletAlias topo.TabletAlias, hostname, mysqlPort, port, keyspace, shardId, tabletType string, parentAlias topo.TabletAlias, dbNameOverride string, force, update bool) error {
// if shardId contains a '-', we assume it's a range-based shard,
// so we try to extract the KeyRange.
var keyRange key.KeyRange
var err error
if strings.Contains(shardId, "-") {
parts := strings.Split(shardId, "-")
if len(parts) != 2 {
return fmt.Errorf("Invalid shardId, can only contains one '-': %v", shardId)
}
keyRange.Start, err = key.HexKeyspaceId(parts[0]).Unhex()
if err != nil {
return err
}
keyRange.End, err = key.HexKeyspaceId(parts[1]).Unhex()
if err != nil {
return err
}
shardId = strings.ToUpper(shardId)
}
if parentAlias == (topo.TabletAlias{}) && topo.TabletType(tabletType) != topo.TYPE_MASTER && topo.TabletType(tabletType) != topo.TYPE_IDLE {
parentAlias, err = wr.getMasterAlias(keyspace, shardId)
if err != nil {
return err
}
}
tablet, err := topo.NewTablet(tabletAlias.Cell, tabletAlias.Uid, parentAlias, fmt.Sprintf("%v:%v", hostname, port), fmt.Sprintf("%v:%v", hostname, mysqlPort), keyspace, shardId, topo.TabletType(tabletType))
if err != nil {
return err
}
tablet.DbNameOverride = dbNameOverride
tablet.KeyRange = keyRange
err = topo.CreateTablet(wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update {
oldTablet, err := wr.ts.GetTablet(tabletAlias)
if err != nil {
relog.Warning("failed reading tablet %v: %v", tabletAlias, err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
err := topo.UpdateTablet(wr.ts, oldTablet)
if err != nil {
relog.Warning("failed updating tablet %v: %v", tabletAlias, err)
} else {
return nil
}
}
}
}
if force {
if _, err = wr.Scrap(tabletAlias, force, false); err != nil {
relog.Error("failed scrapping tablet %v: %v", tabletAlias, err)
return err
}
if err = wr.ts.DeleteTablet(tabletAlias); err != nil {
// we ignore this
relog.Error("failed deleting tablet %v: %v", tabletAlias, err)
}
err = topo.CreateTablet(wr.ts, tablet)
}
}
return err
}