// TestInitMasterShardChecks makes sure the safety checks work
func TestInitMasterShardChecks(t *testing.T) {
ctx := context.Background()
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
master := NewFakeTablet(t, wr, "cell1", 0, pb.TabletType_MASTER, db)
// InitShardMaster with an unknown tablet
if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, &pb.TabletAlias{
Cell: master.Tablet.Alias.Cell,
Uid: master.Tablet.Alias.Uid + 1,
}, false /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is not in the shard") {
t.Errorf("InitShardMaster with unknown alias returned wrong error: %v", err)
}
// InitShardMaster with two masters in the shard, no force flag
// (master2 needs to run InitTablet with -force, as it is the second
// master in the same shard)
master2 := NewFakeTablet(t, wr, "cell1", 1, pb.TabletType_MASTER, db, ForceInitTablet())
if err := wr.InitShardMaster(ctx, master2.Tablet.Keyspace, master2.Tablet.Shard, master2.Tablet.Alias, false /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is not the only master in the shard") {
t.Errorf("InitShardMaster with two masters returned wrong error: %v", err)
}
// InitShardMaster where the new master fails (use force flag
// as we have 2 masters). We force the failure by making the
// SQL commands executed on the master unexpected by the test fixture
master.StartActionLoop(t, wr)
defer master.StopActionLoop(t)
master2.StartActionLoop(t, wr)
defer master2.StopActionLoop(t)
if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, master.Tablet.Alias, true /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "unexpected extra query") {
t.Errorf("InitShardMaster with new master failing in new master InitMaster returned wrong error: %v", err)
}
}
func createTestAgent(ctx context.Context, t *testing.T) *ActionAgent {
ts := zktopo.NewTestServer(t, []string{"cell1"})
if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
if err := ts.CreateShard(ctx, "test_keyspace", "0"); err != nil {
t.Fatalf("CreateShard failed: %v", err)
}
port := int32(1234)
tablet := &pb.Tablet{
Alias: tabletAlias,
Hostname: "host",
PortMap: map[string]int32{
"vt": port,
},
Ip: "1.0.0.1",
Keyspace: "test_keyspace",
Shard: "0",
Type: pb.TabletType_SPARE,
}
if err := ts.CreateTablet(ctx, tablet); err != nil {
t.Fatalf("CreateTablet failed: %v", err)
}
mysqlDaemon := &mysqlctl.FakeMysqlDaemon{MysqlPort: 3306}
agent := NewTestActionAgent(ctx, ts, tabletAlias, port, 0, mysqlDaemon)
agent.BinlogPlayerMap = NewBinlogPlayerMap(ts, nil, nil)
agent.HealthReporter = &fakeHealthCheck{}
return agent
}
// TestEmergencyReparentShardMasterElectNotBest tries to emergency reparent
// to a host that is not the latest in replication position.
func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) {
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
// Create a master, a couple good slaves
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA)
moreAdvancedSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA)
// new master
newMaster.FakeMysqlDaemon.Replicating = true
newMaster.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
GTIDSet: myproto.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 456,
},
}
newMaster.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP SLAVE",
}
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// old master, will be scrapped
oldMaster.StartActionLoop(t, wr)
defer oldMaster.StopActionLoop(t)
// more advanced slave
moreAdvancedSlave.FakeMysqlDaemon.Replicating = true
moreAdvancedSlave.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
GTIDSet: myproto.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
}
moreAdvancedSlave.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP SLAVE",
}
moreAdvancedSlave.StartActionLoop(t, wr)
defer moreAdvancedSlave.StopActionLoop(t)
// run EmergencyReparentShard
if err := wr.EmergencyReparentShard(ctx, newMaster.Tablet.Keyspace, newMaster.Tablet.Shard, newMaster.Tablet.Alias, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is more advanced than master elect tablet") {
t.Fatalf("EmergencyReparentShard returned the wrong error: %v", err)
}
// check what was run
if err := newMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("newMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
if err := moreAdvancedSlave.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("moreAdvancedSlave.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
}
// TestGetOrCreateShard will create / get 100 shards in a keyspace
// for a long time in parallel, making sure the locking and everything
// works correctly.
func TestGetOrCreateShard(t *testing.T) {
ctx := context.Background()
cells := []string{"test_cell"}
// Set up topology.
ts := zktopo.NewTestServer(t, cells)
// and do massive parallel GetOrCreateShard
keyspace := "test_keyspace"
wg := sync.WaitGroup{}
rand.Seed(time.Now().UnixNano())
for i := 0; i < 100; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
for j := 0; j < 100; j++ {
index := rand.Intn(10)
shard := fmt.Sprintf("%v", index)
si, err := GetOrCreateShard(ctx, ts, keyspace, shard)
if err != nil {
t.Errorf("GetOrCreateShard(%v, %v) failed: %v", i, shard, err)
}
if si.ShardName() != shard {
t.Errorf("si.ShardName() is wrong, got %v expected %v", si.ShardName(), shard)
}
}
}(i)
}
wg.Wait()
}
func TestSlaveWasRestarted(t *testing.T) {
ts := zktopo.NewTestServer(t, []string{"cell1"})
wr := New(ts, time.Minute, time.Second)
wr.UseRPCs = false
// Create an old master, a new master, two good slaves, one bad slave
oldMasterAlias := createTestTablet(t, wr, "cell1", 0, topo.TYPE_MASTER, topo.TabletAlias{})
newMasterAlias := createTestTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA, oldMasterAlias)
slaveAlias := createTestTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA, oldMasterAlias)
slaveMySQLDaemon := &mysqlctl.FakeMysqlDaemon{
MasterAddr: "101.0.0.1:3301",
MysqlPort: 3302,
}
done := make(chan struct{}, 1)
startFakeTabletActionLoop(t, wr, slaveAlias, slaveMySQLDaemon, done)
defer close(done)
if err := wr.SlaveWasRestarted(slaveAlias, newMasterAlias, false); err != nil {
t.Fatalf("SlaveWasRestarted %v: %v", slaveAlias, err)
}
tablet, err := wr.ts.GetTablet(slaveAlias)
if err != nil {
t.Fatalf("GetTablet %v: %v", slaveAlias, err)
}
if want, got := newMasterAlias, tablet.Parent; want != got {
t.Errorf("parent of %v: want %v, got %v", tablet, want, got)
}
}
func createTestAgent(ctx context.Context, t *testing.T) *ActionAgent {
ts := zktopo.NewTestServer(t, []string{cell})
if err := ts.CreateKeyspace(ctx, keyspace, &pb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
if err := topo.CreateShard(ctx, ts, keyspace, shard); err != nil {
t.Fatalf("CreateShard failed: %v", err)
}
port := 1234
tablet := &topo.Tablet{
Alias: tabletAlias,
Hostname: "host",
Portmap: map[string]int{
"vt": port,
},
IPAddr: "1.0.0.1",
Keyspace: keyspace,
Shard: shard,
Type: topo.TYPE_SPARE,
}
if err := topo.CreateTablet(ctx, ts, tablet); err != nil {
t.Fatalf("CreateTablet failed: %v", err)
}
mysqlDaemon := &mysqlctl.FakeMysqlDaemon{MysqlPort: 3306}
agent := NewTestActionAgent(ctx, ts, tabletAlias, port, 0, mysqlDaemon)
agent.BinlogPlayerMap = NewBinlogPlayerMap(ts, nil, nil)
agent.HealthReporter = &fakeHealthCheck{}
return agent
}
func TestTabletExternallyReparentedFailedOldMaster(t *testing.T) {
tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */)
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
// Create an old master, a new master, and a good slave.
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA)
goodSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA)
// Reparent to a replica, and pretend the old master is not responding.
// On the elected master, we will respond to
// TabletActionSlaveWasPromoted
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// On the old master, we will only get a
// TabletActionSlaveWasRestarted call, let's just not
// respond to it at all
// On the good slave, we will respond to
// TabletActionSlaveWasRestarted.
goodSlave.StartActionLoop(t, wr)
defer goodSlave.StopActionLoop(t)
// The reparent should work as expected here
t.Logf("TabletExternallyReparented(new master) expecting success")
tmc := tmclient.NewTabletManagerClient()
ti, err := ts.GetTablet(ctx, newMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
waitID := makeWaitID()
if err := tmc.TabletExternallyReparented(context.Background(), ti, waitID); err != nil {
t.Fatalf("TabletExternallyReparented(replica) failed: %v", err)
}
waitForExternalReparent(t, waitID)
// Now double-check the serving graph is good.
// Should only have one good replica left.
addrs, _, err := ts.GetEndPoints(ctx, "cell1", "test_keyspace", "0", topo.TYPE_REPLICA)
if err != nil {
t.Fatalf("GetEndPoints failed at the end: %v", err)
}
if len(addrs.Entries) != 1 {
t.Fatalf("GetEndPoints has too many entries: %v", addrs)
}
// check the old master was converted to spare
tablet, err := ts.GetTablet(ctx, oldMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet(%v) failed: %v", oldMaster.Tablet.Alias, err)
}
if tablet.Type != topo.TYPE_SPARE {
t.Fatalf("old master should be spare but is: %v", tablet.Type)
}
}
func TestKeyspaceCache(t *testing.T) {
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
if err := ts.CreateKeyspace(ctx, "ks1", &pb.Keyspace{
ShardingColumnName: "sharding_key",
}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
if err := ts.CreateKeyspace(ctx, "ks2", &pb.Keyspace{
SplitShardCount: 10,
}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
kc := newKeyspaceCache(ts)
var k Keyspace
expectedK := Keyspace{
KeyspaceName: "ks1",
Keyspace: &pb.Keyspace{
ShardingColumnName: "sharding_key",
},
}
testVersionedObjectCacheMap(t, kc, "ks1", &k, &expectedK)
k = Keyspace{}
expectedK = Keyspace{
KeyspaceName: "ks2",
Keyspace: &pb.Keyspace{
SplitShardCount: 10,
},
}
testVersionedObjectCacheMap(t, kc, "ks2", &k, &expectedK)
}
func TestShardNamesCache(t *testing.T) {
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
if err := ts.CreateKeyspace(ctx, "ks1", &pb.Keyspace{
ShardingColumnName: "sharding_key",
}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
if err := ts.CreateShard(ctx, "ks1", "s1", &pb.Shard{
Cells: []string{"cell1", "cell2"},
}); err != nil {
t.Fatalf("CreateShard failed: %v", err)
}
if err := ts.CreateShard(ctx, "ks1", "s2", &pb.Shard{
MasterAlias: &pb.TabletAlias{
Cell: "cell1",
Uid: 12,
},
}); err != nil {
t.Fatalf("CreateShard failed: %v", err)
}
snc := newShardNamesCache(ts)
var sn ShardNames
expectedSN := ShardNames{
KeyspaceName: "ks1",
ShardNames: []string{"s1", "s2"},
}
testVersionedObjectCacheMap(t, snc, "ks1", &sn, &expectedSN)
}
// TestCreateShard tests a few cases for CreateShard
func TestCreateShard(t *testing.T) {
ctx := context.Background()
cells := []string{"test_cell"}
// Set up topology.
ts := zktopo.NewTestServer(t, cells)
keyspace := "test_keyspace"
shard := "0"
// create shard in a non-existing keyspace
if err := CreateShard(ctx, ts, keyspace, shard); err == nil {
t.Fatalf("CreateShard(invalid keyspace) didn't fail")
}
// create keyspace
if err := ts.CreateKeyspace(ctx, keyspace, &pb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
// create shard should now work
if err := CreateShard(ctx, ts, keyspace, shard); err != nil {
t.Fatalf("CreateShard failed: %v", err)
}
}
func TestShardExternallyReparented(t *testing.T) {
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := New(ts, time.Minute, time.Second)
// Create a master and a replica
oldMasterAlias := createTestTablet(t, wr, "cell1", 0, topo.TYPE_MASTER, topo.TabletAlias{})
newMasterAlias := createTestTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA, oldMasterAlias)
goodSlaveAlias1 := createTestTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA, oldMasterAlias)
goodSlaveAlias2 := createTestTablet(t, wr, "cell2", 3, topo.TYPE_REPLICA, oldMasterAlias)
badSlaveAlias := createTestTablet(t, wr, "cell1", 4, topo.TYPE_REPLICA, oldMasterAlias)
// First test: reparent to the same master, make sure it works
// as expected.
if err := wr.ShardExternallyReparented("test_keyspace", "0", oldMasterAlias, false, 80); err == nil {
t.Fatalf("ShardExternallyReparented(same master) should have failed")
} else {
if !strings.Contains(err.Error(), "already master") {
t.Fatalf("ShardExternallyReparented(same master) should have failed with an error that contains 'already master' but got: %v", err)
}
}
// Second test: reparent to the replica, and pretend the old
// master is still good to go.
done := make(chan struct{}, 1)
// On the elected master, we will only respond to
// TABLET_ACTION_SLAVE_WAS_PROMOTED, no need for a MysqlDaemon
startFakeTabletActionLoop(t, wr, newMasterAlias, nil, done)
// On the old master, we will only respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED.
oldMasterMysqlDaemon := &mysqlctl.FakeMysqlDaemon{
MasterAddr: "101.0.0.1:3301",
}
startFakeTabletActionLoop(t, wr, oldMasterAlias, oldMasterMysqlDaemon, done)
// On the good slaves, we will respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED.
goodSlaveMysqlDaemon1 := &mysqlctl.FakeMysqlDaemon{
MasterAddr: "101.0.0.1:3301",
}
startFakeTabletActionLoop(t, wr, goodSlaveAlias1, goodSlaveMysqlDaemon1, done)
goodSlaveMysqlDaemon2 := &mysqlctl.FakeMysqlDaemon{
MasterAddr: "101.0.0.1:3301",
}
startFakeTabletActionLoop(t, wr, goodSlaveAlias2, goodSlaveMysqlDaemon2, done)
// On the bad slave, we will respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED.
badSlaveMysqlDaemon := &mysqlctl.FakeMysqlDaemon{
MasterAddr: "234.0.0.1:3301",
}
startFakeTabletActionLoop(t, wr, badSlaveAlias, badSlaveMysqlDaemon, done)
if err := wr.ShardExternallyReparented("test_keyspace", "0", newMasterAlias, false, 60); err != nil {
t.Fatalf("ShardExternallyReparented(replica) failed: %v", err)
}
close(done)
}
func TestShardExternallyReparentedFailedOldMaster(t *testing.T) {
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, time.Minute, time.Second)
// Create an old master, a new master, two good slaves
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA,
TabletParent(oldMaster.Tablet.Alias))
goodSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA,
TabletParent(oldMaster.Tablet.Alias))
// Reparent to a replica, and pretend the old master is not responding
// On the elected master, we will respond to
// TABLET_ACTION_SLAVE_WAS_PROMOTED
newMaster.FakeMysqlDaemon.MasterAddr = ""
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// On the old master, we will only get a
// TABLET_ACTION_SLAVE_WAS_RESTARTED call, let's just not
// respond to it at all
// On the good slave, we will respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED.
goodSlave.FakeMysqlDaemon.MasterAddr = newMaster.Tablet.MysqlIpAddr()
goodSlave.StartActionLoop(t, wr)
defer goodSlave.StopActionLoop(t)
// The reparent should work as expected here
t.Logf("ShardExternallyReparented(new master) expecting success")
if err := wr.ShardExternallyReparented("test_keyspace", "0", newMaster.Tablet.Alias); err != nil {
t.Fatalf("ShardExternallyReparented(replica) failed: %v", err)
}
// Now double-check the serving graph is good.
// Should only have one good replica left.
addrs, err := ts.GetEndPoints("cell1", "test_keyspace", "0", topo.TYPE_REPLICA)
if err != nil {
t.Fatalf("GetEndPoints failed at the end: %v", err)
}
if len(addrs.Entries) != 1 {
t.Fatalf("GetEndPoints has too many entries: %v", addrs)
}
// check the old master was converted to spare
tablet, err := ts.GetTablet(oldMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet(%v) failed: %v", oldMaster.Tablet.Alias, err)
}
if tablet.Type != topo.TYPE_SPARE {
t.Fatalf("old master should be spare but is: %v", tablet.Type)
}
if tablet.Parent != newMaster.Tablet.Alias {
t.Fatalf("old master has the wrong master, got %v expected %v", tablet.Parent, newMaster.Tablet.Alias)
}
}
func TestKnownCellsCache(t *testing.T) {
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
kcc := newKnownCellsCache(ts)
var kc KnownCells
expectedKc := KnownCells{
Cells: []string{"cell1", "cell2"},
}
testVersionedObjectCache(t, kcc, &kc, &expectedKc)
}
func TestTabletData(t *testing.T) {
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
if err := ts.CreateKeyspace(context.Background(), "ks", &topodatapb.Keyspace{
ShardingColumnName: "keyspace_id",
ShardingColumnType: topodatapb.KeyspaceIdType_UINT64,
}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
tablet1 := testlib.NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db, testlib.TabletKeyspaceShard(t, "ks", "-80"))
tablet1.StartActionLoop(t, wr)
defer tablet1.StopActionLoop(t)
shsq := newStreamHealthTabletServer(t)
grpcqueryservice.RegisterForTest(tablet1.RPCServer, shsq)
thc := newTabletHealthCache(ts)
stats := &querypb.RealtimeStats{
HealthError: "testHealthError",
SecondsBehindMaster: 72,
CpuUsage: 1.1,
}
// Keep broadcasting until the first result goes through.
stop := make(chan struct{})
go func() {
for {
select {
case <-stop:
return
default:
shsq.BroadcastHealth(42, stats)
}
}
}()
// Start streaming and wait for the first result.
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
result, err := thc.Get(ctx, tablet1.Tablet.Alias)
cancel()
close(stop)
if err != nil {
t.Fatalf("thc.Get failed: %v", err)
}
if got, want := result.RealtimeStats, stats; !proto.Equal(got, want) {
t.Errorf("RealtimeStats = %#v, want %#v", got, want)
}
}
// New creates a topology fixture.
func New(t *testing.T, cells []string) *Fixture {
ts := zktopo.NewTestServer(t, cells)
wr := wrangler.New(logutil.NewConsoleLogger(), ts, 1*time.Second, 1*time.Second)
return &Fixture{
T: t,
Topo: ts,
Wrangler: wr,
done: make(chan struct{}, 1),
tablets: make(map[int]*tabletPack),
}
}
// New creates a topology fixture.
func New(t *testing.T, cells []string) *Fixture {
ts := zktopo.NewTestServer(t, cells)
wr := wrangler.New(ts, 1*time.Second, 1*time.Second)
wr.UseRPCs = false
return &Fixture{
T: t,
Topo: ts,
Wrangler: wr,
done: make(chan struct{}, 1),
tablets: make(map[int]*tabletPack),
}
}
// TestShardExternallyReparentedContinueOnUnexpectedMaster makes sure
// that we ignore mysql's master if the flag is set
func TestShardExternallyReparentedContinueOnUnexpectedMaster(t *testing.T) {
ts := zktopo.NewTestServer(t, []string{"cell1"})
wr := New(ts, time.Minute, time.Second)
wr.UseRPCs = false
// Create an old master, a new master, two good slaves, one bad slave
oldMasterAlias := createTestTablet(t, wr, "cell1", 0, topo.TYPE_MASTER, topo.TabletAlias{})
newMasterAlias := createTestTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA, oldMasterAlias)
goodSlaveAlias := createTestTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA, oldMasterAlias)
done := make(chan struct{}, 1)
// On the elected master, we will respond to
// TABLET_ACTION_SLAVE_WAS_PROMOTED, so we need a MysqlDaemon
// that returns no master, and the new port (as returned by mysql)
newMasterMysqlDaemon := &mysqlctl.FakeMysqlDaemon{
MasterAddr: "",
MysqlPort: 3301,
}
startFakeTabletActionLoop(t, wr, newMasterAlias, newMasterMysqlDaemon, done)
// On the old master, we will only respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED and point to a bad host
oldMasterMysqlDaemon := &mysqlctl.FakeMysqlDaemon{
MasterAddr: "1.2.3.4:6666",
MysqlPort: 3300,
}
startFakeTabletActionLoop(t, wr, oldMasterAlias, oldMasterMysqlDaemon, done)
// On the good slaves, we will respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED and point to a bad host
goodSlaveMysqlDaemon := &mysqlctl.FakeMysqlDaemon{
MasterAddr: "1.2.3.4:6666",
MysqlPort: 3302,
}
startFakeTabletActionLoop(t, wr, goodSlaveAlias, goodSlaveMysqlDaemon, done)
// This tests the good case, where everything works as planned
t.Logf("ShardExternallyReparented(new master) expecting success")
// temporary failure still:
if err := wr.ShardExternallyReparented("test_keyspace", "0", newMasterAlias, false, true, 60); err == nil {
t.Fatal("ShardExternallyReparented(replica) should have failed")
}
// if err := wr.ShardExternallyReparented("test_keyspace", "0", newMasterAlias, false, true, 60); err != nil {
// t.Fatalf("ShardExternallyReparented(replica) failed: %v", err)
// }
close(done)
}
// TestTabletExternallyReparentedWithDifferentMysqlPort makes sure
// that if mysql is restarted on the master-elect tablet and has a different
// port, we pick it up correctly.
func TestTabletExternallyReparentedWithDifferentMysqlPort(t *testing.T) {
tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */)
ctx := context.Background()
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
// Create an old master, a new master, two good slaves, one bad slave
oldMaster := NewFakeTablet(t, wr, "cell1", 0, pb.TabletType_MASTER, db)
newMaster := NewFakeTablet(t, wr, "cell1", 1, pb.TabletType_REPLICA, db)
goodSlave := NewFakeTablet(t, wr, "cell1", 2, pb.TabletType_REPLICA, db)
// Now we're restarting mysql on a different port, 3301->3303
// but without updating the Tablet record in topology.
// On the elected master, we will respond to
// TabletActionSlaveWasPromoted, so we need a MysqlDaemon
// that returns no master, and the new port (as returned by mysql)
newMaster.FakeMysqlDaemon.MysqlPort = 3303
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// On the old master, we will only respond to
// TabletActionSlaveWasRestarted and point to the new mysql port
oldMaster.StartActionLoop(t, wr)
defer oldMaster.StopActionLoop(t)
// On the good slaves, we will respond to
// TabletActionSlaveWasRestarted and point to the new mysql port
goodSlave.StartActionLoop(t, wr)
defer goodSlave.StopActionLoop(t)
// This tests the good case, where everything works as planned
t.Logf("TabletExternallyReparented(new master) expecting success")
tmc := tmclient.NewTabletManagerClient()
ti, err := ts.GetTablet(ctx, newMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
waitID := makeWaitID()
if err := tmc.TabletExternallyReparented(context.Background(), ti, waitID); err != nil {
t.Fatalf("TabletExternallyReparented(replica) failed: %v", err)
}
waitForExternalReparent(t, waitID)
}
// TestShardExternallyReparentedWithDifferentMysqlPort makes sure
// that if mysql is restarted on the master-elect tablet and has a different
// port, we pick it up correctly.
func TestShardExternallyReparentedWithDifferentMysqlPort(t *testing.T) {
ts := zktopo.NewTestServer(t, []string{"cell1"})
wr := New(ts, time.Minute, time.Second)
wr.UseRPCs = false
// Create an old master, a new master, two good slaves, one bad slave
oldMasterAlias := createTestTablet(t, wr, "cell1", 0, topo.TYPE_MASTER, topo.TabletAlias{})
newMasterAlias := createTestTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA, oldMasterAlias)
goodSlaveAlias := createTestTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA, oldMasterAlias)
done := make(chan struct{}, 1)
// Now we're restarting mysql on a different port, 3301->3302
// but without updating the Tablet record in topology.
// On the elected master, we will respond to
// TABLET_ACTION_SLAVE_WAS_PROMOTED, so we need a MysqlDaemon
// that returns no master, and the new port (as returned by mysql)
newMasterMysqlDaemon := &mysqlctl.FakeMysqlDaemon{
MasterAddr: "",
MysqlPort: 3302,
}
startFakeTabletActionLoop(t, wr, newMasterAlias, newMasterMysqlDaemon, done)
// On the old master, we will only respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED and point to the new mysql port
oldMasterMysqlDaemon := &mysqlctl.FakeMysqlDaemon{
MasterAddr: "101.0.0.1:3302",
MysqlPort: 3300,
}
startFakeTabletActionLoop(t, wr, oldMasterAlias, oldMasterMysqlDaemon, done)
// On the good slaves, we will respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED and point to the new mysql port
goodSlaveMysqlDaemon := &mysqlctl.FakeMysqlDaemon{
MasterAddr: "101.0.0.1:3302",
MysqlPort: 3302,
}
startFakeTabletActionLoop(t, wr, goodSlaveAlias, goodSlaveMysqlDaemon, done)
// This tests the good case, where everything works as planned
t.Logf("ShardExternallyReparented(new master) expecting success")
if err := wr.ShardExternallyReparented("test_keyspace", "0", newMasterAlias, false, false, 60); err != nil {
t.Fatalf("ShardExternallyReparented(replica) failed: %v", err)
}
close(done)
}
func TestKeyspacesCache(t *testing.T) {
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
if err := ts.CreateKeyspace(ctx, "ks1", &pb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
if err := ts.CreateKeyspace(ctx, "ks2", &pb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
kc := newKeyspacesCache(ts)
var k Keyspaces
expectedK := Keyspaces{
Keyspaces: []string{"ks1", "ks2"},
}
testVersionedObjectCache(t, kc, &k, &expectedK)
}
func TestReparentTablet(t *testing.T) {
ctx := context.Background()
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
// create shard and tablets
if err := ts.CreateShard(ctx, "test_keyspace", "0"); err != nil {
t.Fatalf("CreateShard failed: %v", err)
}
master := NewFakeTablet(t, wr, "cell1", 1, pb.TabletType_MASTER, db)
slave := NewFakeTablet(t, wr, "cell1", 2, pb.TabletType_REPLICA, db)
// mark the master inside the shard
si, err := ts.GetShard(ctx, "test_keyspace", "0")
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
si.MasterAlias = master.Tablet.Alias
if err := ts.UpdateShard(ctx, si); err != nil {
t.Fatalf("UpdateShard failed: %v", err)
}
// master action loop (to initialize host and port)
master.StartActionLoop(t, wr)
defer master.StopActionLoop(t)
// slave loop
slave.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", master.Tablet.Hostname, master.Tablet.PortMap["mysql"])
slave.FakeMysqlDaemon.SetMasterCommandsResult = []string{"set master cmd 1"}
slave.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"set master cmd 1",
}
slave.StartActionLoop(t, wr)
defer slave.StopActionLoop(t)
// run ReparentTablet
if err := wr.ReparentTablet(ctx, slave.Tablet.Alias); err != nil {
t.Fatalf("ReparentTablet failed: %v", err)
}
// check what was run
if err := slave.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("slave.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
}
func TestVersion(t *testing.T) {
// We need to run this test with the /debug/vars version of the
// plugin.
wrangler.ResetDebugVarsGetVersion()
// Initialize our environment
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
vp := NewVtctlPipe(t, ts)
defer vp.Close()
// couple tablets is enough
sourceMaster := NewFakeTablet(t, wr, "cell1", 10, pb.TabletType_MASTER, db,
TabletKeyspaceShard(t, "source", "0"),
StartHTTPServer())
sourceReplica := NewFakeTablet(t, wr, "cell1", 11, pb.TabletType_REPLICA, db,
TabletKeyspaceShard(t, "source", "0"),
StartHTTPServer())
// sourceMaster loop
sourceMasterGitRev := "fake git rev"
sourceMaster.StartActionLoop(t, wr)
sourceMaster.HTTPServer.Handler.(*http.ServeMux).HandleFunc("/debug/vars", expvarHandler(&sourceMasterGitRev))
defer sourceMaster.StopActionLoop(t)
// sourceReplica loop
sourceReplicaGitRev := "fake git rev"
sourceReplica.StartActionLoop(t, wr)
sourceReplica.HTTPServer.Handler.(*http.ServeMux).HandleFunc("/debug/vars", expvarHandler(&sourceReplicaGitRev))
defer sourceReplica.StopActionLoop(t)
// test when versions are the same
if err := vp.Run([]string{"ValidateVersionKeyspace", sourceMaster.Tablet.Keyspace}); err != nil {
t.Fatalf("ValidateVersionKeyspace(same) failed: %v", err)
}
// test when versions are different
sourceReplicaGitRev = "different fake git rev"
if err := vp.Run([]string{"ValidateVersionKeyspace", sourceMaster.Tablet.Keyspace}); err == nil || !strings.Contains(err.Error(), "is different than slave") {
t.Fatalf("ValidateVersionKeyspace(different) returned an unexpected error: %v", err)
}
}
func TestCellShardTabletsCache(t *testing.T) {
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
if err := ts.UpdateShardReplicationFields(ctx, "cell1", "ks1", "s1", func(sr *pb.ShardReplication) error {
sr.Nodes = []*pb.ShardReplication_Node{
&pb.ShardReplication_Node{
TabletAlias: &pb.TabletAlias{
Cell: "cell1",
Uid: 12,
},
},
&pb.ShardReplication_Node{
TabletAlias: &pb.TabletAlias{
Cell: "cell1",
Uid: 13,
},
},
}
return nil
}); err != nil {
t.Fatalf("UpdateShardReplicationFields failed: %v", err)
}
cstc := newCellShardTabletsCache(ts)
var cst CellShardTablets
expectedCST := CellShardTablets{
Cell: "cell1",
KeyspaceName: "ks1",
ShardName: "s1",
TabletAliases: []topo.TabletAlias{
topo.TabletAlias{
Cell: "cell1",
Uid: 12,
},
topo.TabletAlias{
Cell: "cell1",
Uid: 13,
},
},
}
testVersionedObjectCacheMap(t, cstc, "cell1/ks1/s1", &cst, &expectedCST)
}
func TestHandleExplorerRedirect(t *testing.T) {
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1"})
if err := ts.CreateTablet(ctx, &topodatapb.Tablet{
Alias: &topodatapb.TabletAlias{
Cell: "cell1",
Uid: 123,
},
Keyspace: "test_keyspace",
Shard: "123-456",
}); err != nil {
t.Fatalf("CreateTablet failed: %v", err)
}
table := map[string]string{
"/explorers/redirect?type=keyspace&keyspace=test_keyspace": "/app/#/keyspaces/",
"/explorers/redirect?type=shard&keyspace=test_keyspace&shard=-80": "/app/#/shard/test_keyspace/-80",
"/explorers/redirect?type=srv_keyspace&keyspace=test_keyspace&cell=cell1": "/app/#/keyspaces/",
"/explorers/redirect?type=srv_shard&keyspace=test_keyspace&shard=-80&cell=cell1": "/app/#/shard/test_keyspace/-80",
"/explorers/redirect?type=srv_type&keyspace=test_keyspace&shard=-80&cell=cell1&tablet_type=replica": "/app/#/shard/test_keyspace/-80",
"/explorers/redirect?type=tablet&alias=cell1-123": "/app/#/shard/test_keyspace/123-456",
"/explorers/redirect?type=replication&keyspace=test_keyspace&shard=-80&cell=cell1": "/app/#/shard/test_keyspace/-80",
}
for input, want := range table {
request, err := http.NewRequest("GET", input, nil)
if err != nil {
t.Fatalf("NewRequest error: %v", err)
}
if err := request.ParseForm(); err != nil {
t.Fatalf("ParseForm error: %v", err)
}
got, err := handleExplorerRedirect(ctx, ts, request)
if err != nil {
t.Fatalf("handleExplorerRedirect error: %v", err)
}
if got != want {
t.Errorf("handlExplorerRedirect(%#v) = %#v, want %#v", input, got, want)
}
}
}
// TestShardExternallyReparentedWithDifferentMysqlPort makes sure
// that if mysql is restarted on the master-elect tablet and has a different
// port, we pick it up correctly.
func TestShardExternallyReparentedWithDifferentMysqlPort(t *testing.T) {
ts := zktopo.NewTestServer(t, []string{"cell1"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, time.Minute, time.Second)
wr.UseRPCs = false
// Create an old master, a new master, two good slaves, one bad slave
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA,
TabletParent(oldMaster.Tablet.Alias))
goodSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA,
TabletParent(oldMaster.Tablet.Alias))
// Now we're restarting mysql on a different port, 3301->3303
// but without updating the Tablet record in topology.
// On the elected master, we will respond to
// TABLET_ACTION_SLAVE_WAS_PROMOTED, so we need a MysqlDaemon
// that returns no master, and the new port (as returned by mysql)
newMaster.FakeMysqlDaemon.MasterAddr = ""
newMaster.FakeMysqlDaemon.MysqlPort = 3303
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// On the old master, we will only respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED and point to the new mysql port
oldMaster.FakeMysqlDaemon.MasterAddr = "101.0.0.1:3303"
oldMaster.StartActionLoop(t, wr)
defer oldMaster.StopActionLoop(t)
// On the good slaves, we will respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED and point to the new mysql port
goodSlave.FakeMysqlDaemon.MasterAddr = "101.0.0.1:3303"
goodSlave.StartActionLoop(t, wr)
defer goodSlave.StopActionLoop(t)
// This tests the good case, where everything works as planned
t.Logf("ShardExternallyReparented(new master) expecting success")
if err := wr.ShardExternallyReparented("test_keyspace", "0", newMaster.Tablet.Alias); err != nil {
t.Fatalf("ShardExternallyReparented(replica) failed: %v", err)
}
}
// TestTabletExternallyReparentedContinueOnUnexpectedMaster makes sure
// that we ignore mysql's master if the flag is set
func TestTabletExternallyReparentedContinueOnUnexpectedMaster(t *testing.T) {
tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */)
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
// Create an old master, a new master, two good slaves, one bad slave
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA)
goodSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA)
// On the elected master, we will respond to
// TabletActionSlaveWasPromoted, so we need a MysqlDaemon
// that returns no master, and the new port (as returned by mysql)
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// On the old master, we will only respond to
// TabletActionSlaveWasRestarted and point to a bad host
oldMaster.StartActionLoop(t, wr)
defer oldMaster.StopActionLoop(t)
// On the good slave, we will respond to
// TabletActionSlaveWasRestarted and point to a bad host
goodSlave.StartActionLoop(t, wr)
defer goodSlave.StopActionLoop(t)
// This tests the good case, where everything works as planned
t.Logf("TabletExternallyReparented(new master) expecting success")
tmc := tmclient.NewTabletManagerClient()
ti, err := ts.GetTablet(ctx, newMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if err := tmc.TabletExternallyReparented(context.Background(), ti, ""); err != nil {
t.Fatalf("TabletExternallyReparented(replica) failed: %v", err)
}
}
// TestCreateShardCustomSharding checks ServedTypes is set correctly
// when creating multiple custom sharding shards
func TestCreateShardCustomSharding(t *testing.T) {
ctx := context.Background()
cells := []string{"test_cell"}
// Set up topology.
ts := zktopo.NewTestServer(t, cells)
// create keyspace
keyspace := "test_keyspace"
if err := ts.CreateKeyspace(ctx, keyspace, &pb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
// create first shard in keyspace
shard0 := "0"
if err := CreateShard(ctx, ts, keyspace, shard0); err != nil {
t.Fatalf("CreateShard(shard0) failed: %v", err)
}
if si, err := ts.GetShard(ctx, keyspace, shard0); err != nil {
t.Fatalf("GetShard(shard0) failed: %v", err)
} else {
if len(si.ServedTypes) != 3 {
t.Fatalf("shard0 should have all 3 served types")
}
}
// create second shard in keyspace
shard1 := "1"
if err := CreateShard(ctx, ts, keyspace, shard1); err != nil {
t.Fatalf("CreateShard(shard1) failed: %v", err)
}
if si, err := ts.GetShard(ctx, keyspace, shard1); err != nil {
t.Fatalf("GetShard(shard1) failed: %v", err)
} else {
if len(si.ServedTypes) != 3 {
t.Fatalf("shard1 should have all 3 served types")
}
}
}
// TestShardExternallyReparentedContinueOnUnexpectedMaster makes sure
// that we ignore mysql's master if the flag is set
func TestShardExternallyReparentedContinueOnUnexpectedMaster(t *testing.T) {
ts := zktopo.NewTestServer(t, []string{"cell1"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, time.Minute, time.Second)
wr.UseRPCs = false
// Create an old master, a new master, two good slaves, one bad slave
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA,
TabletParent(oldMaster.Tablet.Alias))
goodSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA,
TabletParent(oldMaster.Tablet.Alias))
// On the elected master, we will respond to
// TABLET_ACTION_SLAVE_WAS_PROMOTED, so we need a MysqlDaemon
// that returns no master, and the new port (as returned by mysql)
newMaster.FakeMysqlDaemon.MasterAddr = ""
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// On the old master, we will only respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED and point to a bad host
oldMaster.FakeMysqlDaemon.MasterAddr = "1.2.3.4:6666"
oldMaster.StartActionLoop(t, wr)
defer oldMaster.StopActionLoop(t)
// On the good slave, we will respond to
// TABLET_ACTION_SLAVE_WAS_RESTARTED and point to a bad host
goodSlave.FakeMysqlDaemon.MasterAddr = "1.2.3.4:6666"
goodSlave.StartActionLoop(t, wr)
defer goodSlave.StopActionLoop(t)
// This tests the good case, where everything works as planned
t.Logf("ShardExternallyReparented(new master) expecting success")
// temporary failure still:
if err := wr.ShardExternallyReparented("test_keyspace", "0", newMaster.Tablet.Alias); err != nil {
t.Fatalf("ShardExternallyReparented(replica) failed: %v", err)
}
}
func testSplitClone(t *testing.T, strategy string) {
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
sourceMaster := testlib.NewFakeTablet(t, wr, "cell1", 0,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-80"))
sourceRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 1,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80"))
sourceRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 2,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80"))
leftMaster := testlib.NewFakeTablet(t, wr, "cell1", 10,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-40"))
leftRdonly := testlib.NewFakeTablet(t, wr, "cell1", 11,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-40"))
rightMaster := testlib.NewFakeTablet(t, wr, "cell1", 20,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "40-80"))
rightRdonly := testlib.NewFakeTablet(t, wr, "cell1", 21,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "40-80"))
for _, ft := range []*testlib.FakeTablet{sourceMaster, sourceRdonly1, sourceRdonly2, leftMaster, leftRdonly, rightMaster, rightRdonly} {
ft.StartActionLoop(t, wr)
defer ft.StopActionLoop(t)
}
// add the topo and schema data we'll need
ctx := context.Background()
if err := topo.CreateShard(ctx, ts, "ks", "80-"); err != nil {
t.Fatalf("CreateShard(\"-80\") failed: %v", err)
}
if err := wr.SetKeyspaceShardingInfo(ctx, "ks", "keyspace_id", key.KIT_UINT64, 4, false); err != nil {
t.Fatalf("SetKeyspaceShardingInfo failed: %v", err)
}
if err := wr.RebuildKeyspaceGraph(ctx, "ks", nil, true); err != nil {
t.Fatalf("RebuildKeyspaceGraph failed: %v", err)
}
gwrk, err := NewSplitCloneWorker(wr, "cell1", "ks", "-80", nil, strategy, 10 /*sourceReaderCount*/, 4 /*destinationPackCount*/, 1 /*minTableSizeForSplit*/, 10 /*destinationWriterCount*/)
if err != nil {
t.Errorf("Worker creation failed: %v", err)
}
wrk := gwrk.(*SplitCloneWorker)
for _, sourceRdonly := range []*testlib.FakeTablet{sourceRdonly1, sourceRdonly2} {
sourceRdonly.FakeMysqlDaemon.Schema = &myproto.SchemaDefinition{
DatabaseSchema: "",
TableDefinitions: []*myproto.TableDefinition{
&myproto.TableDefinition{
Name: "table1",
Columns: []string{"id", "msg", "keyspace_id"},
PrimaryKeyColumns: []string{"id"},
Type: myproto.TableBaseTable,
// This informs how many rows we can pack into a single insert
DataLength: 2048,
},
},
}
sourceRdonly.FakeMysqlDaemon.DbAppConnectionFactory = SourceRdonlyFactory(t)
sourceRdonly.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
GTIDSet: myproto.MariadbGTID{Domain: 12, Server: 34, Sequence: 5678},
}
sourceRdonly.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP SLAVE",
"START SLAVE",
}
sourceRdonly.RPCServer.Register(gorpcqueryservice.New(&testQueryService{t: t}))
}
// We read 100 source rows. sourceReaderCount is set to 10, so
// we'll have 100/10=10 rows per table chunk.
// destinationPackCount is set to 4, so we take 4 source rows
// at once. So we'll process 4 + 4 + 2 rows to get to 10.
// That means 3 insert statements on each target (each
// containing half of the rows, i.e. 2 + 2 + 1 rows). So 3 * 10
// = 30 insert statements on each destination.
leftMaster.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
leftRdonly.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
rightMaster.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
rightRdonly.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
// Only wait 1 ms between retries, so that the test passes faster
*executeFetchRetryTime = (1 * time.Millisecond)
err = wrk.Run(ctx)
status := wrk.StatusAsText()
t.Logf("Got status: %v", status)
if err != nil || wrk.State != WorkerStateDone {
t.Errorf("Worker run failed")
}
if statsDestinationAttemptedResolves.String() != "3" {
t.Errorf("Wrong statsDestinationAttemptedResolves: wanted %v, got %v", "3", statsDestinationAttemptedResolves.String())
}
if statsDestinationActualResolves.String() != "1" {
t.Errorf("Wrong statsDestinationActualResolves: wanted %v, got %v", "1", statsDestinationActualResolves.String())
}
if statsRetryCounters.String() != "{\"ReadOnly\": 2}" {
t.Errorf("Wrong statsRetryCounters: wanted %v, got %v", "{\"ReadOnly\": 2}", statsRetryCounters.String())
}
}
func TestPermissions(t *testing.T) {
// Initialize our environment
ctx := context.Background()
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
vp := NewVtctlPipe(t, ts)
defer vp.Close()
master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db)
replica := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db)
// mark the master inside the shard
si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard)
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
si.MasterAlias = master.Tablet.Alias
if err := ts.UpdateShard(ctx, si); err != nil {
t.Fatalf("UpdateShard failed: %v", err)
}
// master will be asked for permissions
master.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{
"SELECT * FROM mysql.user": {
Fields: []*querypb.Field{
{
Name: "Host",
Type: sqltypes.Char,
},
{
Name: "User",
Type: sqltypes.Char,
},
{
Name: "Password",
Type: sqltypes.Char,
},
{
Name: "Select_priv",
Type: sqltypes.Char,
},
{
Name: "Insert_priv",
Type: sqltypes.Char,
},
{
Name: "Update_priv",
Type: sqltypes.Char,
},
{
Name: "Delete_priv",
Type: sqltypes.Char,
},
{
Name: "Create_priv",
Type: sqltypes.Char,
},
{
Name: "Drop_priv",
Type: sqltypes.Char,
},
{
Name: "Reload_priv",
Type: sqltypes.Char,
},
{
Name: "Shutdown_priv",
Type: sqltypes.Char,
},
{
Name: "Process_priv",
Type: sqltypes.Char,
},
{
Name: "File_priv",
Type: sqltypes.Char,
},
{
Name: "Grant_priv",
Type: sqltypes.Char,
},
{
Name: "References_priv",
Type: sqltypes.Char,
},
{
Name: "Index_priv",
Type: sqltypes.Char,
},
{
Name: "Alter_priv",
Type: sqltypes.Char,
},
{
Name: "Show_db_priv",
Type: sqltypes.Char,
},
{
Name: "Super_priv",
Type: sqltypes.Char,
},
{
Name: "Create_tmp_table_priv",
Type: sqltypes.Char,
},
{
Name: "Lock_tables_priv",
Type: sqltypes.Char,
},
{
Name: "Execute_priv",
Type: sqltypes.Char,
},
{
Name: "Repl_slave_priv",
Type: sqltypes.Char,
},
{
Name: "Repl_client_priv",
Type: sqltypes.Char,
},
{
Name: "Create_view_priv",
Type: sqltypes.Char,
},
{
Name: "Show_view_priv",
Type: sqltypes.Char,
},
{
Name: "Create_routine_priv",
Type: sqltypes.Char,
},
{
Name: "Alter_routine_priv",
Type: sqltypes.Char,
},
{
Name: "Create_user_priv",
Type: sqltypes.Char,
},
{
Name: "Event_priv",
Type: sqltypes.Char,
},
{
Name: "Trigger_priv",
Type: sqltypes.Char,
},
{
Name: "Create_tablespace_priv",
Type: sqltypes.Char,
},
{
Name: "ssl_type",
Type: sqltypes.Char,
},
{
Name: "ssl_cipher",
Type: 252,
},
{
Name: "x509_issuer",
Type: 252,
},
{
Name: "x509_subject",
Type: 252,
},
{
Name: "max_questions",
Type: 3,
},
{
Name: "max_updates",
Type: 3,
},
{
Name: "max_connections",
Type: 3,
},
{
Name: "max_user_connections",
Type: 3,
},
{
Name: "plugin",
Type: sqltypes.Char,
},
{
Name: "authentication_string",
Type: 252,
},
{
Name: "password_expired",
Type: sqltypes.Char,
},
{
Name: "is_role",
Type: sqltypes.Char,
}},
RowsAffected: 0x6,
InsertID: 0x0,
Rows: [][]sqltypes.Value{
{
sqltypes.MakeString([]byte("test_host1")),
sqltypes.MakeString([]byte("test_user1")),
sqltypes.MakeString([]byte("test_password1")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N"))},
{
sqltypes.MakeString([]byte("test_host2")),
sqltypes.MakeString([]byte("test_user2")),
sqltypes.MakeString([]byte("test_password2")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N"))},
{
sqltypes.MakeString([]byte("test_host3")),
sqltypes.MakeString([]byte("test_user3")),
sqltypes.MakeString([]byte("test_password3")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N"))},
{
sqltypes.MakeString([]byte("test_host4")),
sqltypes.MakeString([]byte("test_user4")),
sqltypes.MakeString([]byte("test_password4")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
},
},
},
"SELECT * FROM mysql.db": {
Fields: []*querypb.Field{
{
Name: "Host",
Type: sqltypes.Char,
},
{
Name: "Db",
Type: sqltypes.Char,
},
{
Name: "User",
Type: sqltypes.Char,
},
{
Name: "Select_priv",
Type: sqltypes.Char,
},
{
Name: "Insert_priv",
Type: sqltypes.Char,
},
{
Name: "Update_priv",
Type: sqltypes.Char,
},
{
Name: "Delete_priv",
Type: sqltypes.Char,
},
{
Name: "Create_priv",
Type: sqltypes.Char,
},
{
Name: "Drop_priv",
Type: sqltypes.Char,
},
{
Name: "Grant_priv",
Type: sqltypes.Char,
},
{
Name: "References_priv",
Type: sqltypes.Char,
},
{
Name: "Index_priv",
Type: sqltypes.Char,
},
{
Name: "Alter_priv",
Type: sqltypes.Char,
},
{
Name: "Create_tmp_table_priv",
Type: sqltypes.Char,
},
{
Name: "Lock_tables_priv",
Type: sqltypes.Char,
},
{
Name: "Create_view_priv",
Type: sqltypes.Char,
},
{
Name: "Show_view_priv",
Type: sqltypes.Char,
},
{
Name: "Create_routine_priv",
Type: sqltypes.Char,
},
{
Name: "Alter_routine_priv",
Type: sqltypes.Char,
},
{
Name: "Execute_priv",
Type: sqltypes.Char,
},
{
Name: "Event_priv",
Type: sqltypes.Char,
},
{
Name: "Trigger_priv",
Type: sqltypes.Char,
},
},
RowsAffected: 0,
InsertID: 0,
Rows: [][]sqltypes.Value{
{
sqltypes.MakeString([]byte("test_host")),
sqltypes.MakeString([]byte("test_db")),
sqltypes.MakeString([]byte("test_user")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
},
},
},
}
master.StartActionLoop(t, wr)
defer master.StopActionLoop(t)
// Make a two-level-deep copy, so we can make them diverge later.
user := *master.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"]
user.Fields = append([]*querypb.Field{}, user.Fields...)
// replica will be asked for permissions
replica.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{
"SELECT * FROM mysql.user": &user,
"SELECT * FROM mysql.db": master.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.db"],
}
replica.StartActionLoop(t, wr)
defer replica.StopActionLoop(t)
// Overwrite with the correct value to make sure it passes.
replica.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"].Fields[0] = &querypb.Field{
Name: "Host",
Type: sqltypes.Char,
}
// run ValidatePermissionsKeyspace, this should work
if err := vp.Run([]string{"ValidatePermissionsKeyspace", master.Tablet.Keyspace}); err != nil {
t.Fatalf("ValidatePermissionsKeyspace failed: %v", err)
}
// modify one field, this should fail
replica.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"].Fields[0] = &querypb.Field{
Name: "Wrong",
Type: sqltypes.Char,
}
// run ValidatePermissionsKeyspace again, this should now fail
if err := vp.Run([]string{"ValidatePermissionsKeyspace", master.Tablet.Keyspace}); err == nil || !strings.Contains(err.Error(), "has an extra user") {
t.Fatalf("ValidatePermissionsKeyspace has unexpected err: %v", err)
}
}