func createTestAgent(ctx context.Context, t *testing.T) *ActionAgent {
ts := zktopo.NewTestServer(t, []string{cell})
if err := ts.CreateKeyspace(ctx, keyspace, &pb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
if err := topo.CreateShard(ctx, ts, keyspace, shard); err != nil {
t.Fatalf("CreateShard failed: %v", err)
}
port := 1234
tablet := &topo.Tablet{
Alias: tabletAlias,
Hostname: "host",
Portmap: map[string]int{
"vt": port,
},
IPAddr: "1.0.0.1",
Keyspace: keyspace,
Shard: shard,
Type: topo.TYPE_SPARE,
}
if err := topo.CreateTablet(ctx, ts, tablet); err != nil {
t.Fatalf("CreateTablet failed: %v", err)
}
mysqlDaemon := &mysqlctl.FakeMysqlDaemon{MysqlPort: 3306}
agent := NewTestActionAgent(ctx, ts, tabletAlias, port, 0, mysqlDaemon)
agent.BinlogPlayerMap = NewBinlogPlayerMap(ts, nil, nil)
agent.HealthReporter = &fakeHealthCheck{}
return agent
}
// GetOrCreateShard will return the shard object, or create one if it doesn't
// already exist. Note the shard creation is protected by a keyspace Lock.
func GetOrCreateShard(ctx context.Context, ts topo.Server, keyspace, shard string) (*topo.ShardInfo, error) {
si, finalErr := ts.GetShard(ctx, keyspace, shard)
if finalErr == topo.ErrNoNode {
// create the keyspace, maybe it already exists
if err := ts.CreateKeyspace(ctx, keyspace, &topo.Keyspace{}); err != nil && err != topo.ErrNodeExists {
return nil, fmt.Errorf("CreateKeyspace(%v) failed: %v", keyspace, err)
}
// now we can lock the keyspace
node := actionnode.KeyspaceCreateShard()
lockPath, err := node.LockKeyspace(ctx, ts, keyspace)
if err != nil {
return nil, fmt.Errorf("LockKeyspace failed: %v", err)
}
// now try to create within the lock, may already exist
if err := topo.CreateShard(ctx, ts, keyspace, shard); err != nil && err != topo.ErrNodeExists {
return nil, node.UnlockKeyspace(ctx, ts, keyspace, lockPath, fmt.Errorf("CreateShard(%v/%v) failed: %v", keyspace, shard, err))
}
// try to read the shard again, maybe someone created it
// in between the original GetShard and the LockKeyspace
si, finalErr = ts.GetShard(ctx, keyspace, shard)
// and unlock
if err := node.UnlockKeyspace(ctx, ts, keyspace, lockPath, finalErr); err != nil {
return nil, fmt.Errorf("UnlockKeyspace failed: %v", err)
}
}
return si, finalErr
}
// CopyShards will create the shards in the destination topo
func CopyShards(fromTS, toTS topo.Server, deleteKeyspaceShards bool) {
keyspaces, err := fromTS.GetKeyspaces()
if err != nil {
log.Fatalf("fromTS.GetKeyspaces: %v", err)
}
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
shards, err := fromTS.GetShardNames(keyspace)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardNames(%v): %v", keyspace, err))
return
}
if deleteKeyspaceShards {
if err := toTS.DeleteKeyspaceShards(keyspace); err != nil {
rec.RecordError(fmt.Errorf("DeleteKeyspaceShards(%v): %v", keyspace, err))
return
}
}
for _, shard := range shards {
wg.Add(1)
go func(keyspace, shard string) {
defer wg.Done()
if err := topo.CreateShard(toTS, keyspace, shard); err != nil {
if err == topo.ErrNodeExists {
log.Warningf("shard %v/%v already exists", keyspace, shard)
} else {
rec.RecordError(fmt.Errorf("CreateShard(%v, %v): %v", keyspace, shard, err))
return
}
}
si, err := fromTS.GetShard(keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShard(%v, %v): %v", keyspace, shard, err))
return
}
if err := toTS.UpdateShard(si); err != nil {
rec.RecordError(fmt.Errorf("UpdateShard(%v, %v): %v", keyspace, shard, err))
}
}(keyspace, shard)
}
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
log.Fatalf("copyShards failed: %v", rec.Error())
}
}
// CheckShardLock checks we can take a shard lock
func CheckShardLock(ctx context.Context, t *testing.T, ts topo.Server) {
if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
if err := topo.CreateShard(ctx, ts, "test_keyspace", "10-20"); err != nil {
t.Fatalf("CreateShard: %v", err)
}
checkShardLockTimeout(ctx, t, ts)
checkShardLockMissing(ctx, t, ts)
checkShardLockUnblocks(ctx, t, ts)
}
// CreateShard will create the shard, while holding the keyspace lock
func CreateShard(ctx context.Context, ts topo.Server, keyspace, shard string) error {
// Lock the keyspace
node := actionnode.KeyspaceCreateShard()
lockPath, err := node.LockKeyspace(ctx, ts, keyspace)
if err != nil {
return fmt.Errorf("LockKeyspace failed: %v", err)
}
// now try to create within the lock, may already exist
err = topo.CreateShard(ctx, ts, keyspace, shard)
// and unlock and return
return node.UnlockKeyspace(ctx, ts, keyspace, lockPath, err)
}
func TestReparentTablet(t *testing.T) {
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
// create shard and tablets
if err := topo.CreateShard(ctx, ts, "test_keyspace", "0"); err != nil {
t.Fatalf("CreateShard failed: %v", err)
}
master := NewFakeTablet(t, wr, "cell1", 1, pb.TabletType_MASTER)
slave := NewFakeTablet(t, wr, "cell1", 2, pb.TabletType_REPLICA)
// mark the master inside the shard
si, err := ts.GetShard(ctx, "test_keyspace", "0")
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
si.MasterAlias = master.Tablet.Alias
if err := topo.UpdateShard(ctx, ts, si); err != nil {
t.Fatalf("UpdateShard failed: %v", err)
}
// master action loop (to initialize host and port)
master.StartActionLoop(t, wr)
defer master.StopActionLoop(t)
// slave loop
slave.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", master.Tablet.Hostname, master.Tablet.PortMap["mysql"])
slave.FakeMysqlDaemon.SetMasterCommandsResult = []string{"set master cmd 1"}
slave.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"set master cmd 1",
}
slave.StartActionLoop(t, wr)
defer slave.StopActionLoop(t)
// run ReparentTablet
if err := wr.ReparentTablet(ctx, slave.Tablet.Alias); err != nil {
t.Fatalf("ReparentTablet failed: %v", err)
}
// check what was run
if err := slave.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("slave.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
}
func CheckShardLock(t *testing.T, ts topo.Server) {
if err := ts.CreateKeyspace("test_keyspace", &topo.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
if err := topo.CreateShard(ts, "test_keyspace", "10-20"); err != nil {
t.Fatalf("CreateShard: %v", err)
}
interrupted := make(chan struct{}, 1)
lockPath, err := ts.LockShardForAction("test_keyspace", "10-20", "fake-content", 5*time.Second, interrupted)
if err != nil {
t.Fatalf("LockShardForAction: %v", err)
}
// test we can't take the lock again
if _, err := ts.LockShardForAction("test_keyspace", "10-20", "unused-fake-content", time.Second/2, interrupted); err != topo.ErrTimeout {
t.Errorf("LockShardForAction(again): %v", err)
}
// test we can interrupt taking the lock
go func() {
time.Sleep(time.Second / 2)
close(interrupted)
}()
if _, err := ts.LockShardForAction("test_keyspace", "10-20", "unused-fake-content", 5*time.Second, interrupted); err != topo.ErrInterrupted {
t.Errorf("LockShardForAction(interrupted): %v", err)
}
if err := ts.UnlockShardForAction("test_keyspace", "10-20", lockPath, "fake-results"); err != nil {
t.Errorf("UnlockShardForAction(): %v", err)
}
// test we can't unlock again
if err := ts.UnlockShardForAction("test_keyspace", "10-20", lockPath, "fake-results"); err == nil {
t.Error("UnlockShardForAction(again) worked")
}
// test we can't lock a non-existing shard
interrupted = make(chan struct{}, 1)
if _, err := ts.LockShardForAction("test_keyspace", "20-30", "fake-content", 5*time.Second, interrupted); err == nil {
t.Fatalf("LockShardForAction(test_keyspace/20-30) worked for non-existing shard")
}
}
func CheckShard(t *testing.T, ts topo.Server) {
if err := ts.CreateKeyspace("test_keyspace", &topo.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
if err := topo.CreateShard(ts, "test_keyspace", "b0-c0"); err != nil {
t.Fatalf("CreateShard: %v", err)
}
if err := topo.CreateShard(ts, "test_keyspace", "b0-c0"); err != topo.ErrNodeExists {
t.Errorf("CreateShard called second time, got: %v", err)
}
if _, err := ts.GetShard("test_keyspace", "666"); err != topo.ErrNoNode {
t.Errorf("GetShard(666): %v", err)
}
shardInfo, err := ts.GetShard("test_keyspace", "b0-c0")
if err != nil {
t.Errorf("GetShard: %v", err)
}
if want := newKeyRange("b0-c0"); shardInfo.KeyRange != want {
t.Errorf("shardInfo.KeyRange: want %v, got %v", want, shardInfo.KeyRange)
}
master := topo.TabletAlias{Cell: "ny", Uid: 1}
shardInfo.MasterAlias = master
shardInfo.KeyRange = newKeyRange("b0-c0")
shardInfo.ServedTypes = []topo.TabletType{topo.TYPE_MASTER, topo.TYPE_REPLICA, topo.TYPE_RDONLY}
shardInfo.SourceShards = []topo.SourceShard{
topo.SourceShard{
Uid: 1,
Keyspace: "source_ks",
Shard: "b8-c0",
KeyRange: newKeyRange("b8-c0"),
Tables: []string{"table1", "table2"},
},
}
if err := topo.UpdateShard(ts, shardInfo); err != nil {
t.Errorf("UpdateShard: %v", err)
}
shardInfo, err = ts.GetShard("test_keyspace", "b0-c0")
if err != nil {
t.Errorf("GetShard: %v", err)
}
if shardInfo.MasterAlias != master {
t.Errorf("after UpdateShard: shardInfo.MasterAlias got %v", shardInfo.MasterAlias)
}
if shardInfo.KeyRange != newKeyRange("b0-c0") {
t.Errorf("after UpdateShard: shardInfo.KeyRange got %v", shardInfo.KeyRange)
}
if len(shardInfo.ServedTypes) != 3 || shardInfo.ServedTypes[0] != topo.TYPE_MASTER || shardInfo.ServedTypes[1] != topo.TYPE_REPLICA || shardInfo.ServedTypes[2] != topo.TYPE_RDONLY {
t.Errorf("after UpdateShard: shardInfo.ServedTypes got %v", shardInfo.ServedTypes)
}
if len(shardInfo.SourceShards) != 1 ||
shardInfo.SourceShards[0].Uid != 1 ||
shardInfo.SourceShards[0].Keyspace != "source_ks" ||
shardInfo.SourceShards[0].Shard != "b8-c0" ||
shardInfo.SourceShards[0].KeyRange != newKeyRange("b8-c0") ||
len(shardInfo.SourceShards[0].Tables) != 2 ||
shardInfo.SourceShards[0].Tables[0] != "table1" ||
shardInfo.SourceShards[0].Tables[1] != "table2" {
t.Errorf("after UpdateShard: shardInfo.SourceShards got %v", shardInfo.SourceShards)
}
shards, err := ts.GetShardNames("test_keyspace")
if err != nil {
t.Errorf("GetShardNames: %v", err)
}
if len(shards) != 1 || shards[0] != "b0-c0" {
t.Errorf(`GetShardNames: want [ "b0-c0" ], got %v`, shards)
}
if _, err := ts.GetShardNames("test_keyspace666"); err != topo.ErrNoNode {
t.Errorf("GetShardNames(666): %v", err)
}
}
func TestSplitDiff(t *testing.T) {
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
// We need to use FakeTabletManagerClient because we don't have a good way to fake the binlog player yet,
// which is necessary for synchronizing replication.
wr := wrangler.New(logutil.NewConsoleLogger(), ts, faketmclient.NewFakeTabletManagerClient(), time.Second)
ctx := context.Background()
sourceMaster := testlib.NewFakeTablet(t, wr, "cell1", 0,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-80"))
sourceRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 1,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80"))
sourceRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 2,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80"))
leftMaster := testlib.NewFakeTablet(t, wr, "cell1", 10,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-40"))
leftRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 11,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-40"))
leftRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 12,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-40"))
for _, ft := range []*testlib.FakeTablet{sourceMaster, sourceRdonly1, sourceRdonly2, leftMaster, leftRdonly1, leftRdonly2} {
ft.StartActionLoop(t, wr)
defer ft.StopActionLoop(t)
}
// add the topo and schema data we'll need
if err := topo.CreateShard(ctx, ts, "ks", "80-"); err != nil {
t.Fatalf("CreateShard(\"-80\") failed: %v", err)
}
wr.SetSourceShards(ctx, "ks", "-40", []topo.TabletAlias{sourceRdonly1.Tablet.Alias}, nil)
if err := wr.SetKeyspaceShardingInfo(ctx, "ks", "keyspace_id", pbt.KeyspaceIdType_UINT64, 4, false); err != nil {
t.Fatalf("SetKeyspaceShardingInfo failed: %v", err)
}
if err := wr.RebuildKeyspaceGraph(ctx, "ks", nil, true); err != nil {
t.Fatalf("RebuildKeyspaceGraph failed: %v", err)
}
excludedTable := "excludedTable1"
gwrk := NewSplitDiffWorker(wr, "cell1", "ks", "-40", []string{excludedTable})
wrk := gwrk.(*SplitDiffWorker)
for _, rdonly := range []*testlib.FakeTablet{sourceRdonly1, sourceRdonly2, leftRdonly1, leftRdonly2} {
// In reality, the destinations *shouldn't* have identical data to the source - instead, we should see
// the data split into left and right. However, if we do that in this test, we would really just be
// testing our fake SQL logic, since we do the data filtering in SQL.
// To simplify things, just assume that both sides have identical data.
rdonly.FakeMysqlDaemon.Schema = &myproto.SchemaDefinition{
DatabaseSchema: "",
TableDefinitions: []*myproto.TableDefinition{
&myproto.TableDefinition{
Name: "table1",
Columns: []string{"id", "msg", "keyspace_id"},
PrimaryKeyColumns: []string{"id"},
Type: myproto.TableBaseTable,
},
&myproto.TableDefinition{
Name: excludedTable,
Columns: []string{"id", "msg", "keyspace_id"},
PrimaryKeyColumns: []string{"id"},
Type: myproto.TableBaseTable,
},
&myproto.TableDefinition{
Name: "view1",
Type: myproto.TableView,
},
},
}
}
grpcqueryservice.RegisterForTest(leftRdonly1.RPCServer, &destinationSqlQuery{t: t, excludedTable: excludedTable})
grpcqueryservice.RegisterForTest(leftRdonly2.RPCServer, &destinationSqlQuery{t: t, excludedTable: excludedTable})
grpcqueryservice.RegisterForTest(sourceRdonly1.RPCServer, &sourceSqlQuery{t: t, excludedTable: excludedTable})
grpcqueryservice.RegisterForTest(sourceRdonly2.RPCServer, &sourceSqlQuery{t: t, excludedTable: excludedTable})
err := wrk.Run(ctx)
status := wrk.StatusAsText()
t.Logf("Got status: %v", status)
if err != nil || wrk.State != WorkerStateDone {
t.Errorf("Worker run failed")
}
}
// InitTablet creates or updates a tablet. If no parent is specified
// in the tablet, and the tablet has a slave type, we will find the
// appropriate parent. If createShardAndKeyspace is true and the
// parent keyspace or shard don't exist, they will be created. If
// update is true, and a tablet with the same ID exists, update it.
// If Force is true, and a tablet with the same ID already exists, it
// will be scrapped and deleted, and then recreated.
func (wr *Wrangler) InitTablet(tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error {
if err := tablet.Complete(); err != nil {
return err
}
if tablet.IsInReplicationGraph() {
// create the parent keyspace and shard if needed
if createShardAndKeyspace {
if err := wr.ts.CreateKeyspace(tablet.Keyspace); err != nil && err != topo.ErrNodeExists {
return err
}
if err := topo.CreateShard(wr.ts, tablet.Keyspace, tablet.Shard); err != nil && err != topo.ErrNodeExists {
return err
}
}
// get the shard, checks a couple things
si, err := wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return fmt.Errorf("missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard")
}
if si.KeyRange != tablet.KeyRange {
return fmt.Errorf("shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange)
}
if tablet.Type == topo.TYPE_MASTER && !si.MasterAlias.IsZero() && si.MasterAlias != tablet.Alias() && !force {
return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, tablet.Keyspace, tablet.Shard)
}
// see if we specified a parent, otherwise get it from the shard
if tablet.Parent.IsZero() && tablet.Type.IsSlaveType() {
if si.MasterAlias.IsZero() {
return fmt.Errorf("trying to create tablet %v in shard %v/%v without a master", tablet.Alias(), tablet.Keyspace, tablet.Shard)
}
tablet.Parent = si.MasterAlias
}
// See if we need to update the Shard:
// - add the tablet's cell to the shard's Cells if needed
// - change the master if needed
shardUpdateRequired := false
if !si.HasCell(tablet.Cell) {
shardUpdateRequired = true
}
if tablet.Type == topo.TYPE_MASTER && si.MasterAlias != tablet.Alias() {
shardUpdateRequired = true
}
if shardUpdateRequired {
actionNode := wr.ai.UpdateShard()
lockPath, err := wr.lockShard(tablet.Keyspace, tablet.Shard, actionNode)
if err != nil {
return err
}
// re-read the shard with the lock
si, err = wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err)
}
// update it
wasUpdated := false
if !si.HasCell(tablet.Cell) {
si.Cells = append(si.Cells, tablet.Cell)
wasUpdated = true
}
if tablet.Type == topo.TYPE_MASTER && si.MasterAlias != tablet.Alias() {
if !si.MasterAlias.IsZero() && !force {
return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, tablet.Keyspace, tablet.Shard))
}
si.MasterAlias = tablet.Alias()
wasUpdated = true
}
if wasUpdated {
// write it back
if err := wr.ts.UpdateShard(si); err != nil {
return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err)
}
}
// and unlock
if err := wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err); err != nil {
return err
}
// also create the cell's ShardReplication
if err := wr.ts.CreateShardReplication(tablet.Cell, tablet.Keyspace, tablet.Shard, &topo.ShardReplication{}); err != nil && err != topo.ErrNodeExists {
return err
}
}
}
err := topo.CreateTablet(wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update || force {
oldTablet, err := wr.ts.GetTablet(tablet.Alias())
if err != nil {
log.Warningf("failed reading tablet %v: %v", tablet.Alias(), err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
if err := topo.UpdateTablet(wr.ts, oldTablet); err != nil {
log.Warningf("failed updating tablet %v: %v", tablet.Alias(), err)
// now fall through the Scrap case
} else {
if !tablet.IsInReplicationGraph() {
return nil
}
if err := topo.CreateTabletReplicationData(wr.ts, tablet); err != nil {
log.Warningf("failed updating tablet replication data for %v: %v", tablet.Alias(), err)
// now fall through the Scrap case
} else {
return nil
}
}
}
}
}
if force {
if _, err = wr.Scrap(tablet.Alias(), force, false); err != nil {
log.Errorf("failed scrapping tablet %v: %v", tablet.Alias(), err)
return err
}
if err := wr.ts.DeleteTablet(tablet.Alias()); err != nil {
// we ignore this
log.Errorf("failed deleting tablet %v: %v", tablet.Alias(), err)
}
return topo.CreateTablet(wr.ts, tablet)
}
}
return err
}
func CheckShard(t *testing.T, ts topo.Server) {
if err := ts.CreateKeyspace("test_keyspace", &topo.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
if err := topo.CreateShard(ts, "test_keyspace", "b0-c0"); err != nil {
t.Fatalf("CreateShard: %v", err)
}
if err := topo.CreateShard(ts, "test_keyspace", "b0-c0"); err != topo.ErrNodeExists {
t.Errorf("CreateShard called second time, got: %v", err)
}
if _, err := ts.GetShard("test_keyspace", "666"); err != topo.ErrNoNode {
t.Errorf("GetShard(666): %v", err)
}
shardInfo, err := ts.GetShard("test_keyspace", "b0-c0")
if err != nil {
t.Errorf("GetShard: %v", err)
}
if want := newKeyRange("b0-c0"); shardInfo.KeyRange != want {
t.Errorf("shardInfo.KeyRange: want %v, got %v", want, shardInfo.KeyRange)
}
master := topo.TabletAlias{Cell: "ny", Uid: 1}
shardInfo.MasterAlias = master
shardInfo.KeyRange = newKeyRange("b0-c0")
shardInfo.ServedTypes = []topo.TabletType{topo.TYPE_MASTER, topo.TYPE_REPLICA, topo.TYPE_RDONLY}
shardInfo.SourceShards = []topo.SourceShard{
topo.SourceShard{
Uid: 1,
Keyspace: "source_ks",
Shard: "b8-c0",
KeyRange: newKeyRange("b8-c0"),
Tables: []string{"table1", "table2"},
},
}
shardInfo.BlacklistedTablesMap = map[topo.TabletType][]string{
topo.TYPE_MASTER: []string{"black1", "black2"},
topo.TYPE_REPLICA: []string{"black3", "black4"},
}
if err := topo.UpdateShard(ts, shardInfo); err != nil {
t.Errorf("UpdateShard: %v", err)
}
updatedShardInfo, err := ts.GetShard("test_keyspace", "b0-c0")
if err != nil {
t.Fatalf("GetShard: %v", err)
}
if eq, err := shardEqual(shardInfo.Shard, updatedShardInfo.Shard); err != nil {
t.Errorf("cannot compare shards: %v", err)
} else if !eq {
t.Errorf("put and got shards are not identical:\n%#v\n%#v", shardInfo.Shard, updatedShardInfo.Shard)
}
// test GetShardNames
shards, err := ts.GetShardNames("test_keyspace")
if err != nil {
t.Errorf("GetShardNames: %v", err)
}
if len(shards) != 1 || shards[0] != "b0-c0" {
t.Errorf(`GetShardNames: want [ "b0-c0" ], got %v`, shards)
}
if _, err := ts.GetShardNames("test_keyspace666"); err != topo.ErrNoNode {
t.Errorf("GetShardNames(666): %v", err)
}
}
// This is a quick and dirty tool to resurrect the TopologyServer data from the
// canonical data stored in the tablet nodes.
//
// cells: local vt cells to scan for all tablets
// keyspaces: list of keyspaces to rebuild
func (wr *Wrangler) RebuildReplicationGraph(cells []string, keyspaces []string) error {
if cells == nil || len(cells) == 0 {
return fmt.Errorf("must specify cells to rebuild replication graph")
}
if keyspaces == nil || len(keyspaces) == 0 {
return fmt.Errorf("must specify keyspaces to rebuild replication graph")
}
allTablets := make([]*topo.TabletInfo, 0, 1024)
for _, cell := range cells {
tablets, err := GetAllTablets(wr.ts, cell)
if err != nil {
return err
}
allTablets = append(allTablets, tablets...)
}
for _, keyspace := range keyspaces {
log.V(6).Infof("delete keyspace shards: %v", keyspace)
if err := wr.ts.DeleteKeyspaceShards(keyspace); err != nil {
return err
}
}
keyspacesToRebuild := make(map[string]bool)
shardsCreated := make(map[string]bool)
hasErr := false
mu := sync.Mutex{}
wg := sync.WaitGroup{}
for _, ti := range allTablets {
wg.Add(1)
go func(ti *topo.TabletInfo) {
defer wg.Done()
if !ti.IsInReplicationGraph() {
return
}
if !strInList(keyspaces, ti.Keyspace) {
return
}
mu.Lock()
keyspacesToRebuild[ti.Keyspace] = true
shardPath := ti.Keyspace + "/" + ti.Shard
if !shardsCreated[shardPath] {
if err := topo.CreateShard(wr.ts, ti.Keyspace, ti.Shard); err != nil && err != topo.ErrNodeExists {
log.Warningf("failed re-creating shard %v: %v", shardPath, err)
hasErr = true
} else {
shardsCreated[shardPath] = true
}
}
mu.Unlock()
err := topo.CreateTabletReplicationData(wr.ts, ti.Tablet)
if err != nil {
mu.Lock()
hasErr = true
mu.Unlock()
log.Warningf("failed creating replication path: %v", err)
}
}(ti)
}
wg.Wait()
for keyspace := range keyspacesToRebuild {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
if err := wr.RebuildKeyspaceGraph(keyspace, nil); err != nil {
mu.Lock()
hasErr = true
mu.Unlock()
log.Warningf("RebuildKeyspaceGraph(%v) failed: %v", keyspace, err)
return
}
}(keyspace)
}
wg.Wait()
if hasErr {
return fmt.Errorf("some errors occurred rebuilding replication graph, consult log")
}
return nil
}
// InitTablet creates or updates a tablet. If no parent is specified
// in the tablet, and the tablet has a slave type, we will find the
// appropriate parent. If createShardAndKeyspace is true and the
// parent keyspace or shard don't exist, they will be created. If
// update is true, and a tablet with the same ID exists, update it.
// If Force is true, and a tablet with the same ID already exists, it
// will be scrapped and deleted, and then recreated.
func (wr *Wrangler) InitTablet(tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error {
if err := tablet.Complete(); err != nil {
return err
}
if tablet.Parent.IsZero() && tablet.Type.IsSlaveType() {
parentAlias, err := wr.getMasterAlias(tablet.Keyspace, tablet.Shard)
if err != nil {
return err
}
tablet.Parent = parentAlias
}
if tablet.IsInReplicationGraph() {
// create the parent keyspace and shard if needed
if createShardAndKeyspace {
if err := wr.ts.CreateKeyspace(tablet.Keyspace); err != nil && err != topo.ErrNodeExists {
return err
}
if err := topo.CreateShard(wr.ts, tablet.Keyspace, tablet.Shard); err != nil && err != topo.ErrNodeExists {
return err
}
}
// get the shard, checks KeyRange is the same
si, err := wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return fmt.Errorf("Missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard")
}
if si.KeyRange != tablet.KeyRange {
return fmt.Errorf("Shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange)
}
}
err := topo.CreateTablet(wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update {
oldTablet, err := wr.ts.GetTablet(tablet.Alias())
if err != nil {
relog.Warning("failed reading tablet %v: %v", tablet.Alias(), err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
err := topo.UpdateTablet(wr.ts, oldTablet)
if err != nil {
relog.Warning("failed updating tablet %v: %v", tablet.Alias(), err)
} else {
return nil
}
}
}
}
if force {
if _, err = wr.Scrap(tablet.Alias(), force, false); err != nil {
relog.Error("failed scrapping tablet %v: %v", tablet.Alias(), err)
return err
}
if err := wr.ts.DeleteTablet(tablet.Alias()); err != nil {
// we ignore this
relog.Error("failed deleting tablet %v: %v", tablet.Alias(), err)
}
return topo.CreateTablet(wr.ts, tablet)
}
}
return err
}
func TestSplitClone(t *testing.T) {
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, time.Minute, time.Second)
sourceMaster := testlib.NewFakeTablet(t, wr, "cell1", 0,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-80"))
sourceRdonly := testlib.NewFakeTablet(t, wr, "cell1", 1,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80"),
testlib.TabletParent(sourceMaster.Tablet.Alias))
leftMaster := testlib.NewFakeTablet(t, wr, "cell1", 10,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-40"))
leftRdonly := testlib.NewFakeTablet(t, wr, "cell1", 11,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-40"),
testlib.TabletParent(leftMaster.Tablet.Alias))
rightMaster := testlib.NewFakeTablet(t, wr, "cell1", 20,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "40-80"))
rightRdonly := testlib.NewFakeTablet(t, wr, "cell1", 21,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "40-80"),
testlib.TabletParent(rightMaster.Tablet.Alias))
for _, ft := range []*testlib.FakeTablet{sourceMaster, sourceRdonly, leftMaster, leftRdonly, rightMaster, rightRdonly} {
ft.StartActionLoop(t, wr)
defer ft.StopActionLoop(t)
}
// add the topo and schema data we'll need
if err := topo.CreateShard(ts, "ks", "80-"); err != nil {
t.Fatalf("CreateShard(\"-80\") failed: %v", err)
}
if err := wr.SetKeyspaceShardingInfo("ks", "keyspace_id", key.KIT_UINT64, 4, false); err != nil {
t.Fatalf("SetKeyspaceShardingInfo failed: %v", err)
}
if err := wr.RebuildKeyspaceGraph("ks", nil); err != nil {
t.Fatalf("RebuildKeyspaceGraph failed: %v", err)
}
sourceRdonly.FakeMysqlDaemon.Schema = &myproto.SchemaDefinition{
DatabaseSchema: "CREATE DATABASE `{{.DatabaseName}}` /*!40100 DEFAULT CHARACTER SET utf8 */",
TableDefinitions: []*myproto.TableDefinition{
&myproto.TableDefinition{
Name: "table1",
Schema: "CREATE TABLE `resharding1` (\n `id` bigint(20) NOT NULL AUTO_INCREMENT,\n `msg` varchar(64) DEFAULT NULL,\n `keyspace_id` bigint(20) unsigned NOT NULL,\n PRIMARY KEY (`id`),\n KEY `by_msg` (`msg`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8",
Columns: []string{"id", "msg", "keyspace_id"},
PrimaryKeyColumns: []string{"id"},
Type: myproto.TABLE_BASE_TABLE,
DataLength: 2048,
RowCount: 100,
},
},
Version: "unused",
}
sourceRdonly.FakeMysqlDaemon.DbaConnectionFactory = SourceRdonlyFactory(t)
sourceRdonly.FakeMysqlDaemon.CurrentSlaveStatus = &myproto.ReplicationStatus{
Position: myproto.ReplicationPosition{
GTIDSet: myproto.MariadbGTID{Domain: 12, Server: 34, Sequence: 5678},
},
}
sourceRdonly.RpcServer.Register(&SqlQuery{t: t})
leftMaster.FakeMysqlDaemon.DbaConnectionFactory = DestinationsFactory(t, 50)
leftRdonly.FakeMysqlDaemon.DbaConnectionFactory = DestinationsFactory(t, 50)
rightMaster.FakeMysqlDaemon.DbaConnectionFactory = DestinationsFactory(t, 50)
rightRdonly.FakeMysqlDaemon.DbaConnectionFactory = DestinationsFactory(t, 50)
wrk := NewSplitCloneWorker(wr, "cell1", "ks", "-80", nil, "populateBlpCheckpoint", 10, 1, 10).(*SplitCloneWorker)
wrk.Run()
status := wrk.StatusAsText()
t.Logf("Got status: %v", status)
if wrk.err != nil || wrk.state != stateSCDone {
t.Errorf("Worker run failed")
}
}
// InitTablet creates or updates a tablet. If no parent is specified
// in the tablet, and the tablet has a slave type, we will find the
// appropriate parent. If createShardAndKeyspace is true and the
// parent keyspace or shard don't exist, they will be created. If
// update is true, and a tablet with the same ID exists, update it.
// If Force is true, and a tablet with the same ID already exists, it
// will be scrapped and deleted, and then recreated.
func (wr *Wrangler) InitTablet(tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error {
if err := tablet.Complete(); err != nil {
return err
}
if tablet.Parent.IsZero() && tablet.Type.IsSlaveType() {
parentAlias, err := wr.getMasterAlias(tablet.Keyspace, tablet.Shard)
if err != nil {
return err
}
tablet.Parent = parentAlias
}
if tablet.IsInReplicationGraph() {
// create the parent keyspace and shard if needed
if createShardAndKeyspace {
if err := wr.ts.CreateKeyspace(tablet.Keyspace); err != nil && err != topo.ErrNodeExists {
return err
}
if err := topo.CreateShard(wr.ts, tablet.Keyspace, tablet.Shard); err != nil && err != topo.ErrNodeExists {
return err
}
}
// get the shard, checks KeyRange is the same
si, err := wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return fmt.Errorf("Missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard")
}
if si.KeyRange != tablet.KeyRange {
return fmt.Errorf("Shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange)
}
// add the tablet's cell to the shard cell if needed
if !si.HasCell(tablet.Cell) {
actionNode := wr.ai.UpdateShard()
lockPath, err := wr.lockShard(tablet.Keyspace, tablet.Shard, actionNode)
if err != nil {
return err
}
// re-read the shard with the lock
si, err = wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err)
}
// update it
if !si.HasCell(tablet.Cell) {
si.Cells = append(si.Cells, tablet.Cell)
// write it back
if err := wr.ts.UpdateShard(si); err != nil {
return wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err)
}
}
// and unlock
if err := wr.unlockShard(tablet.Keyspace, tablet.Shard, actionNode, lockPath, err); err != nil {
return err
}
// also create the cell's ShardReplication
if err := wr.ts.CreateShardReplication(tablet.Cell, tablet.Keyspace, tablet.Shard, &topo.ShardReplication{}); err != nil && err != topo.ErrNodeExists {
return err
}
}
}
err := topo.CreateTablet(wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update {
oldTablet, err := wr.ts.GetTablet(tablet.Alias())
if err != nil {
log.Warningf("failed reading tablet %v: %v", tablet.Alias(), err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
err := topo.UpdateTablet(wr.ts, oldTablet)
if err != nil {
log.Warningf("failed updating tablet %v: %v", tablet.Alias(), err)
} else {
return nil
}
}
}
}
if force {
if _, err = wr.Scrap(tablet.Alias(), force, false); err != nil {
log.Errorf("failed scrapping tablet %v: %v", tablet.Alias(), err)
return err
}
if err := wr.ts.DeleteTablet(tablet.Alias()); err != nil {
// we ignore this
log.Errorf("failed deleting tablet %v: %v", tablet.Alias(), err)
}
return topo.CreateTablet(wr.ts, tablet)
}
}
return err
}
func CheckReplicationPaths(t *testing.T, ts topo.Server) {
if _, err := ts.GetReplicationPaths("test_keyspace", "-10", "/"); err != topo.ErrNoNode {
t.Errorf("GetReplicationPaths(bad shard): %v", err)
}
if err := ts.CreateKeyspace("test_keyspace"); err != nil {
t.Errorf("CreateKeyspace: %v", err)
}
if err := topo.CreateShard(ts, "test_keyspace", "-10"); err != nil {
t.Errorf("CreateShard: %v", err)
}
if paths, err := ts.GetReplicationPaths("test_keyspace", "-10", "/"); err != nil || len(paths) != 0 {
t.Errorf("GetReplicationPaths(empty shard): %v, %v", err, paths)
}
if _, err := ts.GetReplicationPaths("test_keyspace", "-10", "/666"); err != topo.ErrNoNode {
t.Errorf("GetReplicationPaths(non-existing path): %v", err)
}
if err := ts.CreateReplicationPath("test_keyspace", "-10", "/cell-0000000001"); err != nil {
t.Errorf("CreateReplicationPath: %v", err)
}
if err := ts.CreateReplicationPath("test_keyspace", "-10", "/cell-0000000001"); err != topo.ErrNodeExists {
t.Errorf("CreateReplicationPath(again): %v", err)
}
if paths, err := ts.GetReplicationPaths("test_keyspace", "-10", "/"); err != nil || len(paths) != 1 || paths[0].String() != "cell-0000000001" {
t.Errorf("GetReplicationPaths(root): %v, %v", err, paths)
}
if err := ts.CreateReplicationPath("test_keyspace", "-10", "/cell-0000000001/cell-0000000002"); err != nil {
t.Errorf("CreateReplicationPath(2): %v", err)
}
if err := ts.CreateReplicationPath("test_keyspace", "-10", "/cell-0000000001/cell-0000000003"); err != nil {
t.Errorf("CreateReplicationPath(3): %v", err)
}
if paths, err := ts.GetReplicationPaths("test_keyspace", "-10", "/cell-0000000001"); err != nil || len(paths) != 2 || paths[0].String() != "cell-0000000002" || paths[1].String() != "cell-0000000003" {
t.Errorf("GetReplicationPaths(master): %v, %v", err, paths)
}
if err := ts.DeleteReplicationPath("test_keyspace", "-10", "/cell-0000000001"); err != topo.ErrNotEmpty {
t.Errorf("DeleteReplicationPath(master with slaves): %v", err)
}
if err := ts.DeleteReplicationPath("test_keyspace", "-10", "/cell-0000000001/cell-0000000002"); err != nil {
t.Errorf("DeleteReplicationPath(slave1): %v", err)
}
if paths, err := ts.GetReplicationPaths("test_keyspace", "-10", "/cell-0000000001"); err != nil || len(paths) != 1 || paths[0].String() != "cell-0000000003" {
t.Errorf("GetReplicationPaths(master): %v, %v", err, paths)
}
if err := ts.DeleteReplicationPath("test_keyspace", "-10", "/cell-0000000001/cell-0000000003"); err != nil {
t.Errorf("DeleteReplicationPath(slave2): %v", err)
}
if paths, err := ts.GetReplicationPaths("test_keyspace", "-10", "/cell-0000000001"); err != nil || len(paths) != 0 {
t.Errorf("GetReplicationPaths(master): %v, %v", err, paths)
}
if err := ts.DeleteReplicationPath("test_keyspace", "-10", "/cell-0000000001"); err != nil {
t.Errorf("DeleteReplicationPath(master): %v", err)
}
if paths, err := ts.GetReplicationPaths("test_keyspace", "-10", "/"); err != nil || len(paths) != 0 {
t.Errorf("GetReplicationPaths(root): %v, %v", err, paths)
}
}
// InitTablet creates or updates a tablet. If no parent is specified
// in the tablet, and the tablet has a slave type, we will find the
// appropriate parent. If createShardAndKeyspace is true and the
// parent keyspace or shard don't exist, they will be created. If
// update is true, and a tablet with the same ID exists, update it.
// If Force is true, and a tablet with the same ID already exists, it
// will be scrapped and deleted, and then recreated.
func (wr *Wrangler) InitTablet(tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error {
if err := tablet.Complete(); err != nil {
return err
}
if tablet.IsInReplicationGraph() {
// create the parent keyspace and shard if needed
if createShardAndKeyspace {
if err := wr.ts.CreateKeyspace(tablet.Keyspace, &topo.Keyspace{}); err != nil && err != topo.ErrNodeExists {
return err
}
if err := topo.CreateShard(wr.ts, tablet.Keyspace, tablet.Shard); err != nil && err != topo.ErrNodeExists {
return err
}
}
// get the shard, checks a couple things
si, err := wr.ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return fmt.Errorf("missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard")
}
if si.KeyRange != tablet.KeyRange {
return fmt.Errorf("shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange)
}
if tablet.Type == topo.TYPE_MASTER && !si.MasterAlias.IsZero() && si.MasterAlias != tablet.Alias && !force {
return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, tablet.Keyspace, tablet.Shard)
}
// see if we specified a parent, otherwise get it from the shard
if tablet.Parent.IsZero() && tablet.Type.IsSlaveType() {
if si.MasterAlias.IsZero() {
return fmt.Errorf("trying to create tablet %v in shard %v/%v without a master", tablet.Alias, tablet.Keyspace, tablet.Shard)
}
tablet.Parent = si.MasterAlias
}
// update the shard record if needed
if err := wr.updateShardCellsAndMaster(si, tablet.Alias, tablet.Type, force); err != nil {
return err
}
}
err := topo.CreateTablet(wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update || force {
oldTablet, err := wr.ts.GetTablet(tablet.Alias)
if err != nil {
log.Warningf("failed reading tablet %v: %v", tablet.Alias, err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
if err := topo.UpdateTablet(wr.ts, oldTablet); err != nil {
log.Warningf("failed updating tablet %v: %v", tablet.Alias, err)
// now fall through the Scrap case
} else {
if !tablet.IsInReplicationGraph() {
return nil
}
if err := topo.CreateTabletReplicationData(wr.ts, tablet); err != nil {
log.Warningf("failed updating tablet replication data for %v: %v", tablet.Alias, err)
// now fall through the Scrap case
} else {
return nil
}
}
}
}
}
if force {
if _, err = wr.Scrap(tablet.Alias, force, false); err != nil {
log.Errorf("failed scrapping tablet %v: %v", tablet.Alias, err)
return err
}
if err := wr.ts.DeleteTablet(tablet.Alias); err != nil {
// we ignore this
log.Errorf("failed deleting tablet %v: %v", tablet.Alias, err)
}
return topo.CreateTablet(wr.ts, tablet)
}
}
return err
}
func TestShardReplicationStatuses(t *testing.T) {
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
// create shard and tablets
if err := topo.CreateShard(ctx, ts, "test_keyspace", "0"); err != nil {
t.Fatalf("CreateShard failed: %v", err)
}
master := NewFakeTablet(t, wr, "cell1", 1, pb.TabletType_MASTER)
slave := NewFakeTablet(t, wr, "cell1", 2, pb.TabletType_REPLICA)
// mark the master inside the shard
si, err := ts.GetShard(ctx, "test_keyspace", "0")
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
si.MasterAlias = master.Tablet.Alias
if err := topo.UpdateShard(ctx, ts, si); err != nil {
t.Fatalf("UpdateShard failed: %v", err)
}
// master action loop (to initialize host and port)
master.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
GTIDSet: myproto.MariadbGTID{
Domain: 5,
Server: 456,
Sequence: 892,
},
}
master.StartActionLoop(t, wr)
defer master.StopActionLoop(t)
// slave loop
slave.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
GTIDSet: myproto.MariadbGTID{
Domain: 5,
Server: 456,
Sequence: 890,
},
}
slave.FakeMysqlDaemon.CurrentMasterHost = master.Tablet.Hostname
slave.FakeMysqlDaemon.CurrentMasterPort = int(master.Tablet.PortMap["mysql"])
slave.StartActionLoop(t, wr)
defer slave.StopActionLoop(t)
// run ShardReplicationStatuses
ti, rs, err := wr.ShardReplicationStatuses(ctx, "test_keyspace", "0")
if err != nil {
t.Fatalf("ShardReplicationStatuses failed: %v", err)
}
// check result (make master first in the array)
if len(ti) != 2 || len(rs) != 2 {
t.Fatalf("ShardReplicationStatuses returned wrong results: %v %v", ti, rs)
}
if topo.TabletAliasEqual(ti[0].Alias, slave.Tablet.Alias) {
ti[0], ti[1] = ti[1], ti[0]
rs[0], rs[1] = rs[1], rs[0]
}
if !topo.TabletAliasEqual(ti[0].Alias, master.Tablet.Alias) ||
!topo.TabletAliasEqual(ti[1].Alias, slave.Tablet.Alias) ||
rs[0].MasterHost != "" ||
rs[1].MasterHost != master.Tablet.Hostname {
t.Fatalf("ShardReplicationStatuses returend wrong results: %v %v", ti, rs)
}
}
func testSplitClone(t *testing.T, strategy string) {
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
sourceMaster := testlib.NewFakeTablet(t, wr, "cell1", 0,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-80"))
sourceRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 1,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80"))
sourceRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 2,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80"))
leftMaster := testlib.NewFakeTablet(t, wr, "cell1", 10,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-40"))
leftRdonly := testlib.NewFakeTablet(t, wr, "cell1", 11,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-40"))
rightMaster := testlib.NewFakeTablet(t, wr, "cell1", 20,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "40-80"))
rightRdonly := testlib.NewFakeTablet(t, wr, "cell1", 21,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "40-80"))
for _, ft := range []*testlib.FakeTablet{sourceMaster, sourceRdonly1, sourceRdonly2, leftMaster, leftRdonly, rightMaster, rightRdonly} {
ft.StartActionLoop(t, wr)
defer ft.StopActionLoop(t)
}
// add the topo and schema data we'll need
ctx := context.Background()
if err := topo.CreateShard(ctx, ts, "ks", "80-"); err != nil {
t.Fatalf("CreateShard(\"-80\") failed: %v", err)
}
if err := wr.SetKeyspaceShardingInfo(ctx, "ks", "keyspace_id", key.KIT_UINT64, 4, false); err != nil {
t.Fatalf("SetKeyspaceShardingInfo failed: %v", err)
}
if err := wr.RebuildKeyspaceGraph(ctx, "ks", nil, true); err != nil {
t.Fatalf("RebuildKeyspaceGraph failed: %v", err)
}
gwrk, err := NewSplitCloneWorker(wr, "cell1", "ks", "-80", nil, strategy, 10 /*sourceReaderCount*/, 4 /*destinationPackCount*/, 1 /*minTableSizeForSplit*/, 10 /*destinationWriterCount*/)
if err != nil {
t.Errorf("Worker creation failed: %v", err)
}
wrk := gwrk.(*SplitCloneWorker)
for _, sourceRdonly := range []*testlib.FakeTablet{sourceRdonly1, sourceRdonly2} {
sourceRdonly.FakeMysqlDaemon.Schema = &myproto.SchemaDefinition{
DatabaseSchema: "",
TableDefinitions: []*myproto.TableDefinition{
&myproto.TableDefinition{
Name: "table1",
Columns: []string{"id", "msg", "keyspace_id"},
PrimaryKeyColumns: []string{"id"},
Type: myproto.TableBaseTable,
// This informs how many rows we can pack into a single insert
DataLength: 2048,
},
},
}
sourceRdonly.FakeMysqlDaemon.DbAppConnectionFactory = SourceRdonlyFactory(t)
sourceRdonly.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
GTIDSet: myproto.MariadbGTID{Domain: 12, Server: 34, Sequence: 5678},
}
sourceRdonly.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP SLAVE",
"START SLAVE",
}
sourceRdonly.RPCServer.Register(gorpcqueryservice.New(&testQueryService{t: t}))
}
// We read 100 source rows. sourceReaderCount is set to 10, so
// we'll have 100/10=10 rows per table chunk.
// destinationPackCount is set to 4, so we take 4 source rows
// at once. So we'll process 4 + 4 + 2 rows to get to 10.
// That means 3 insert statements on each target (each
// containing half of the rows, i.e. 2 + 2 + 1 rows). So 3 * 10
// = 30 insert statements on each destination.
leftMaster.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
leftRdonly.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
rightMaster.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
rightRdonly.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
// Only wait 1 ms between retries, so that the test passes faster
*executeFetchRetryTime = (1 * time.Millisecond)
err = wrk.Run(ctx)
status := wrk.StatusAsText()
t.Logf("Got status: %v", status)
if err != nil || wrk.State != WorkerStateDone {
t.Errorf("Worker run failed")
}
if statsDestinationAttemptedResolves.String() != "3" {
t.Errorf("Wrong statsDestinationAttemptedResolves: wanted %v, got %v", "3", statsDestinationAttemptedResolves.String())
}
if statsDestinationActualResolves.String() != "1" {
t.Errorf("Wrong statsDestinationActualResolves: wanted %v, got %v", "1", statsDestinationActualResolves.String())
}
if statsRetryCounters.String() != "{\"ReadOnly\": 2}" {
t.Errorf("Wrong statsRetryCounters: wanted %v, got %v", "{\"ReadOnly\": 2}", statsRetryCounters.String())
}
}
// CheckShard verifies the Shard operations work correctly
func CheckShard(ctx context.Context, t *testing.T, ts topo.Server) {
if err := ts.CreateKeyspace(ctx, "test_keyspace", &topo.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != nil {
t.Fatalf("CreateShard: %v", err)
}
if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != topo.ErrNodeExists {
t.Errorf("CreateShard called second time, got: %v", err)
}
// Delete shard and see if we can re-create it.
if err := ts.DeleteShard(ctx, "test_keyspace", "b0-c0"); err != nil {
t.Fatalf("DeleteShard: %v", err)
}
if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != nil {
t.Fatalf("CreateShard: %v", err)
}
// Delete ALL shards.
if err := ts.DeleteKeyspaceShards(ctx, "test_keyspace"); err != nil {
t.Fatalf("DeleteKeyspaceShards: %v", err)
}
if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != nil {
t.Fatalf("CreateShard: %v", err)
}
if _, err := topo.GetShard(ctx, ts, "test_keyspace", "666"); err != topo.ErrNoNode {
t.Errorf("GetShard(666): %v", err)
}
shardInfo, err := topo.GetShard(ctx, ts, "test_keyspace", "b0-c0")
if err != nil {
t.Errorf("GetShard: %v", err)
}
if want := newKeyRange("b0-c0"); shardInfo.KeyRange != want {
t.Errorf("shardInfo.KeyRange: want %v, got %v", want, shardInfo.KeyRange)
}
master := topo.TabletAlias{Cell: "ny", Uid: 1}
shardInfo.MasterAlias = master
shardInfo.KeyRange = newKeyRange("b0-c0")
shardInfo.ServedTypesMap = map[topo.TabletType]*topo.ShardServedType{
topo.TYPE_MASTER: &topo.ShardServedType{},
topo.TYPE_REPLICA: &topo.ShardServedType{Cells: []string{"c1"}},
topo.TYPE_RDONLY: &topo.ShardServedType{},
}
shardInfo.SourceShards = []topo.SourceShard{
topo.SourceShard{
Uid: 1,
Keyspace: "source_ks",
Shard: "b8-c0",
KeyRange: newKeyRange("b8-c0"),
Tables: []string{"table1", "table2"},
},
}
shardInfo.TabletControlMap = map[topo.TabletType]*topo.TabletControl{
topo.TYPE_MASTER: &topo.TabletControl{
Cells: []string{"c1", "c2"},
BlacklistedTables: []string{"black1", "black2"},
},
topo.TYPE_REPLICA: &topo.TabletControl{
DisableQueryService: true,
},
}
if err := topo.UpdateShard(ctx, ts, shardInfo); err != nil {
t.Errorf("UpdateShard: %v", err)
}
other := topo.TabletAlias{Cell: "ny", Uid: 82873}
_, err = topo.UpdateShardFields(ctx, ts, "test_keyspace", "b0-c0", func(shard *topo.Shard) error {
shard.MasterAlias = other
return nil
})
if err != nil {
t.Fatalf("UpdateShardFields error: %v", err)
}
si, err := topo.GetShard(ctx, ts, "test_keyspace", "b0-c0")
if err != nil {
t.Fatalf("GetShard: %v", err)
}
if si.MasterAlias != other {
t.Fatalf("shard.MasterAlias = %v, want %v", si.MasterAlias, other)
}
_, err = topo.UpdateShardFields(ctx, ts, "test_keyspace", "b0-c0", func(shard *topo.Shard) error {
shard.MasterAlias = master
return nil
})
if err != nil {
t.Fatalf("UpdateShardFields error: %v", err)
}
updatedShardInfo, err := topo.GetShard(ctx, ts, "test_keyspace", "b0-c0")
if err != nil {
t.Fatalf("GetShard: %v", err)
}
if eq, err := shardEqual(shardInfo.Shard, updatedShardInfo.Shard); err != nil {
t.Errorf("cannot compare shards: %v", err)
} else if !eq {
t.Errorf("put and got shards are not identical:\n%#v\n%#v", shardInfo.Shard, updatedShardInfo.Shard)
}
// test GetShardNames
shards, err := ts.GetShardNames(ctx, "test_keyspace")
if err != nil {
t.Errorf("GetShardNames: %v", err)
}
if len(shards) != 1 || shards[0] != "b0-c0" {
t.Errorf(`GetShardNames: want [ "b0-c0" ], got %v`, shards)
}
if _, err := ts.GetShardNames(ctx, "test_keyspace666"); err != topo.ErrNoNode {
t.Errorf("GetShardNames(666): %v", err)
}
// test ValidateShard
if err := ts.ValidateShard(ctx, "test_keyspace", "b0-c0"); err != nil {
t.Errorf("ValidateShard(test_keyspace, b0-c0) failed: %v", err)
}
}