func TestTabletExternallyReparentedFailedOldMaster(t *testing.T) {
tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */)
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
// Create an old master, a new master, and a good slave.
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA)
goodSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA)
// Reparent to a replica, and pretend the old master is not responding.
// On the elected master, we will respond to
// TabletActionSlaveWasPromoted
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// On the old master, we will only get a
// TabletActionSlaveWasRestarted call, let's just not
// respond to it at all
// On the good slave, we will respond to
// TabletActionSlaveWasRestarted.
goodSlave.StartActionLoop(t, wr)
defer goodSlave.StopActionLoop(t)
// The reparent should work as expected here
t.Logf("TabletExternallyReparented(new master) expecting success")
tmc := tmclient.NewTabletManagerClient()
ti, err := ts.GetTablet(ctx, newMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
waitID := makeWaitID()
if err := tmc.TabletExternallyReparented(context.Background(), ti, waitID); err != nil {
t.Fatalf("TabletExternallyReparented(replica) failed: %v", err)
}
waitForExternalReparent(t, waitID)
// Now double-check the serving graph is good.
// Should only have one good replica left.
addrs, _, err := ts.GetEndPoints(ctx, "cell1", "test_keyspace", "0", topo.TYPE_REPLICA)
if err != nil {
t.Fatalf("GetEndPoints failed at the end: %v", err)
}
if len(addrs.Entries) != 1 {
t.Fatalf("GetEndPoints has too many entries: %v", addrs)
}
// check the old master was converted to spare
tablet, err := ts.GetTablet(ctx, oldMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet(%v) failed: %v", oldMaster.Tablet.Alias, err)
}
if tablet.Type != topo.TYPE_SPARE {
t.Fatalf("old master should be spare but is: %v", tablet.Type)
}
}
func TestTabletExternallyReparentedFailedOldMaster(t *testing.T) {
tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */)
ctx := context.Background()
db := fakesqldb.Register()
ts := zktestserver.New(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
// Create an old master, a new master, and a good slave.
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db)
goodSlave := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db)
// Reparent to a replica, and pretend the old master is not responding.
// On the elected master, we will respond to
// TabletActionSlaveWasPromoted
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// On the old master, we will only get a
// TabletActionSlaveWasRestarted call, let's just not
// respond to it at all
// On the good slave, we will respond to
// TabletActionSlaveWasRestarted.
goodSlave.StartActionLoop(t, wr)
defer goodSlave.StopActionLoop(t)
// The reparent should work as expected here
t.Logf("TabletExternallyReparented(new master) expecting success")
tmc := tmclient.NewTabletManagerClient()
ti, err := ts.GetTablet(ctx, newMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
waitID := makeWaitID()
if err := tmc.TabletExternallyReparented(context.Background(), ti.Tablet, waitID); err != nil {
t.Fatalf("TabletExternallyReparented(replica) failed: %v", err)
}
waitForExternalReparent(t, waitID)
// check the old master was converted to replica
tablet, err := ts.GetTablet(ctx, oldMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet(%v) failed: %v", oldMaster.Tablet.Alias, err)
}
if tablet.Type != topodatapb.TabletType_REPLICA {
t.Fatalf("old master should be spare but is: %v", tablet.Type)
}
}
// ExecuteVtctlCommand is part of the pb.VtctlServer interface
func (s *VtctlServer) ExecuteVtctlCommand(args *pb.ExecuteVtctlCommandRequest, stream pbs.Vtctl_ExecuteVtctlCommandServer) (err error) {
defer servenv.HandlePanic("vtctl", &err)
// create a logger, send the result back to the caller
logstream := logutil.NewChannelLogger(10)
logger := logutil.NewTeeLogger(logstream, logutil.NewConsoleLogger())
// send logs to the caller
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
for e := range logstream {
// Note we don't interrupt the loop here, as
// we still need to flush and finish the
// command, even if the channel to the client
// has been broken. We'll just keep trying.
stream.Send(&pb.ExecuteVtctlCommandResponse{
Event: logutil.LoggerEventToProto(&e),
})
}
wg.Done()
}()
// create the wrangler
wr := wrangler.New(logger, s.ts, tmclient.NewTabletManagerClient(), time.Duration(args.LockTimeout))
// execute the command
err = vtctl.RunCommand(stream.Context(), wr, args.Args)
// close the log channel, and wait for them all to be sent
close(logstream)
wg.Wait()
return err
}
// TestInitMasterShardChecks makes sure the safety checks work
func TestInitMasterShardChecks(t *testing.T) {
ctx := context.Background()
db := fakesqldb.Register()
ts := zktestserver.New(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db)
// InitShardMaster with an unknown tablet
if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, &topodatapb.TabletAlias{
Cell: master.Tablet.Alias.Cell,
Uid: master.Tablet.Alias.Uid + 1,
}, false /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is not in the shard") {
t.Errorf("InitShardMaster with unknown alias returned wrong error: %v", err)
}
// InitShardMaster with two masters in the shard, no force flag
// (master2 needs to run InitTablet with -force, as it is the second
// master in the same shard)
master2 := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_MASTER, db, ForceInitTablet())
if err := wr.InitShardMaster(ctx, master2.Tablet.Keyspace, master2.Tablet.Shard, master2.Tablet.Alias, false /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is not the only master in the shard") {
t.Errorf("InitShardMaster with two masters returned wrong error: %v", err)
}
// InitShardMaster where the new master fails (use force flag
// as we have 2 masters). We force the failure by making the
// SQL commands executed on the master unexpected by the test fixture
master.StartActionLoop(t, wr)
defer master.StopActionLoop(t)
master2.StartActionLoop(t, wr)
defer master2.StopActionLoop(t)
if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, master.Tablet.Alias, true /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "unexpected extra query") {
t.Errorf("InitShardMaster with new master failing in new master InitMaster returned wrong error: %v", err)
}
}
// ApplyTabletAction applies the provided action to the tablet.
func (ar *ActionRepository) ApplyTabletAction(ctx context.Context, actionName string, tabletAlias *topodatapb.TabletAlias, r *http.Request) *ActionResult {
result := &ActionResult{
Name: actionName,
Parameters: topoproto.TabletAliasString(tabletAlias),
}
action, ok := ar.tabletActions[actionName]
if !ok {
result.error("Unknown tablet action")
return result
}
// check the role
if action.role != "" {
if err := acl.CheckAccessHTTP(r, action.role); err != nil {
result.error("Access denied")
return result
}
}
// run the action
ctx, cancel := context.WithTimeout(ctx, *actionTimeout)
wr := wrangler.New(logutil.NewConsoleLogger(), ar.ts, tmclient.NewTabletManagerClient())
output, err := action.method(ctx, wr, tabletAlias, r)
cancel()
if err != nil {
result.error(err.Error())
return result
}
result.Output = output
return result
}
// ApplyShardAction applies the provided action to the shard.
func (ar *ActionRepository) ApplyShardAction(ctx context.Context, actionName, keyspace, shard string, r *http.Request) *ActionResult {
// if the shard name contains a '-', we assume it's the
// name for a ranged based shard, so we lower case it.
if strings.Contains(shard, "-") {
shard = strings.ToLower(shard)
}
result := &ActionResult{Name: actionName, Parameters: keyspace + "/" + shard}
action, ok := ar.shardActions[actionName]
if !ok {
result.error("Unknown shard action")
return result
}
ctx, cancel := context.WithTimeout(ctx, *actionTimeout)
wr := wrangler.New(logutil.NewConsoleLogger(), ar.ts, tmclient.NewTabletManagerClient())
output, err := action(ctx, wr, keyspace, shard, r)
cancel()
if err != nil {
result.error(err.Error())
return result
}
result.Output = output
return result
}
// TestNewRestartableResultReader tests the correct error handling e.g.
// if the connection to a tablet fails due to a canceled context.
func TestNewRestartableResultReader(t *testing.T) {
wantErr := errors.New("restartable_result_reader_test.go: context canceled")
tabletconn.RegisterDialer("fake_dialer", func(tablet *topodatapb.Tablet, timeout time.Duration) (tabletconn.TabletConn, error) {
return nil, wantErr
})
protocol := flag.CommandLine.Lookup("tablet_protocol").Value.String()
flag.Set("tablet_protocol", "fake_dialer")
// Restore the previous flag value after the test.
defer flag.Set("tablet_protocol", protocol)
// Create dependencies e.g. a "singleTabletProvider" instance.
ts := zktestserver.New(t, []string{"cell1"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
alias := &topodatapb.TabletAlias{
Cell: "cell1",
Uid: 1,
}
tablet := &topodatapb.Tablet{
Keyspace: "ks1",
Shard: "-80",
Alias: alias,
}
ctx := context.Background()
if err := ts.CreateTablet(ctx, tablet); err != nil {
t.Fatalf("CreateTablet failed: %v", err)
}
tp := newSingleTabletProvider(ctx, ts, alias)
_, err := NewRestartableResultReader(ctx, wr.Logger(), tp, nil /* td */, chunk{}, false)
if err == nil || !strings.Contains(err.Error(), wantErr.Error()) {
t.Fatalf("NewRestartableResultReader() should have failed because the context is canceled: %v", err)
}
}
// TestEmergencyReparentShardMasterElectNotBest tries to emergency reparent
// to a host that is not the latest in replication position.
func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) {
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
// Create a master, a couple good slaves
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA)
moreAdvancedSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA)
// new master
newMaster.FakeMysqlDaemon.Replicating = true
newMaster.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
GTIDSet: myproto.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 456,
},
}
newMaster.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP SLAVE",
}
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// old master, will be scrapped
oldMaster.StartActionLoop(t, wr)
defer oldMaster.StopActionLoop(t)
// more advanced slave
moreAdvancedSlave.FakeMysqlDaemon.Replicating = true
moreAdvancedSlave.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
GTIDSet: myproto.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
}
moreAdvancedSlave.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP SLAVE",
}
moreAdvancedSlave.StartActionLoop(t, wr)
defer moreAdvancedSlave.StopActionLoop(t)
// run EmergencyReparentShard
if err := wr.EmergencyReparentShard(ctx, newMaster.Tablet.Keyspace, newMaster.Tablet.Shard, newMaster.Tablet.Alias, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is more advanced than master elect tablet") {
t.Fatalf("EmergencyReparentShard returned the wrong error: %v", err)
}
// check what was run
if err := newMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("newMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
if err := moreAdvancedSlave.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("moreAdvancedSlave.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
}
// TestTabletExternallyReparentedWithDifferentMysqlPort makes sure
// that if mysql is restarted on the master-elect tablet and has a different
// port, we pick it up correctly.
func TestTabletExternallyReparentedWithDifferentMysqlPort(t *testing.T) {
tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */)
ctx := context.Background()
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
// Create an old master, a new master, two good slaves, one bad slave
oldMaster := NewFakeTablet(t, wr, "cell1", 0, pb.TabletType_MASTER, db)
newMaster := NewFakeTablet(t, wr, "cell1", 1, pb.TabletType_REPLICA, db)
goodSlave := NewFakeTablet(t, wr, "cell1", 2, pb.TabletType_REPLICA, db)
// Now we're restarting mysql on a different port, 3301->3303
// but without updating the Tablet record in topology.
// On the elected master, we will respond to
// TabletActionSlaveWasPromoted, so we need a MysqlDaemon
// that returns no master, and the new port (as returned by mysql)
newMaster.FakeMysqlDaemon.MysqlPort = 3303
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// On the old master, we will only respond to
// TabletActionSlaveWasRestarted and point to the new mysql port
oldMaster.StartActionLoop(t, wr)
defer oldMaster.StopActionLoop(t)
// On the good slaves, we will respond to
// TabletActionSlaveWasRestarted and point to the new mysql port
goodSlave.StartActionLoop(t, wr)
defer goodSlave.StopActionLoop(t)
// This tests the good case, where everything works as planned
t.Logf("TabletExternallyReparented(new master) expecting success")
tmc := tmclient.NewTabletManagerClient()
ti, err := ts.GetTablet(ctx, newMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
waitID := makeWaitID()
if err := tmc.TabletExternallyReparented(context.Background(), ti, waitID); err != nil {
t.Fatalf("TabletExternallyReparented(replica) failed: %v", err)
}
waitForExternalReparent(t, waitID)
}
// TestSplitCloneV2_RetryDueToReparent tests that vtworker correctly failovers
// during a reparent.
// NOTE: worker.py is an end-to-end test which tests this as well.
func TestSplitCloneV2_RetryDueToReparent(t *testing.T) {
tc := &splitCloneTestCase{t: t}
tc.setUp(false /* v3 */)
defer tc.tearDown()
// Provoke a reparent just before the copy finishes.
// leftReplica will take over for the last, 30th, insert and the BLP checkpoint.
tc.leftReplicaFakeDb.addExpectedQuery("INSERT INTO `vt_ks`.`table1` (`id`, `msg`, `keyspace_id`) VALUES (*", nil)
expectBlpCheckpointCreationQueries(tc.leftReplicaFakeDb)
// Do not let leftMaster succeed the 30th write.
tc.leftMasterFakeDb.deleteAllEntriesAfterIndex(28)
tc.leftMasterFakeDb.addExpectedQuery("INSERT INTO `vt_ks`.`table1` (`id`, `msg`, `keyspace_id`) VALUES (*", errReadOnly)
tc.leftMasterFakeDb.enableInfinite()
// When vtworker encounters the readonly error on leftMaster, do the reparent.
tc.leftMasterFakeDb.getEntry(29).AfterFunc = func() {
// Reparent from leftMaster to leftReplica.
// NOTE: This step is actually not necessary due to our fakes which bypass
// a lot of logic. Let's keep it for correctness though.
ti, err := tc.ts.GetTablet(context.Background(), tc.leftReplica.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
tmc := tmclient.NewTabletManagerClient()
if err := tmc.TabletExternallyReparented(context.Background(), ti.Tablet, "wait id 1"); err != nil {
t.Fatalf("TabletExternallyReparented(replica) failed: %v", err)
}
// Update targets in fake query service and send out a new health response.
tc.leftMasterQs.UpdateType(topodatapb.TabletType_REPLICA)
tc.leftMasterQs.AddDefaultHealthResponse()
tc.leftReplicaQs.UpdateType(topodatapb.TabletType_MASTER)
tc.leftReplicaQs.AddDefaultHealthResponse()
// After this, vtworker will retry. The following situations can occur:
// 1. HealthCheck picked up leftReplica as new MASTER
// => retry will succeed.
// 2. HealthCheck picked up no changes (leftMaster remains MASTER)
// => retry will hit leftMaster which keeps responding with readonly err.
// 3. HealthCheck picked up leftMaster as REPLICA, but leftReplica is still
// a REPLICA.
// => vtworker has no MASTER to go to and will keep retrying.
}
// Only wait 1 ms between retries, so that the test passes faster.
*executeFetchRetryTime = 1 * time.Millisecond
// Run the vtworker command.
if err := runCommand(t, tc.wi, tc.wi.wr, tc.defaultWorkerArgs); err != nil {
t.Fatal(err)
}
wantRetryCountMin := int64(1)
if got := statsRetryCount.Get(); got < wantRetryCountMin {
t.Errorf("Wrong statsRetryCounter: got %v, wanted >= %v", got, wantRetryCountMin)
}
}
// StartActionLoop will start the action loop for a fake tablet,
// using ft.FakeMysqlDaemon as the backing mysqld.
func (ft *FakeTablet) StartActionLoop(t *testing.T, wr *wrangler.Wrangler) {
if ft.Agent != nil {
t.Fatalf("Agent for %v is already running", ft.Tablet.Alias)
}
// Listen on a random port for gRPC
var err error
ft.Listener, err = net.Listen("tcp", ":0")
if err != nil {
t.Fatalf("Cannot listen: %v", err)
}
gRPCPort := int32(ft.Listener.Addr().(*net.TCPAddr).Port)
// if needed, listen on a random port for HTTP
vtPort := ft.Tablet.PortMap["vt"]
if ft.StartHTTPServer {
ft.HTTPListener, err = net.Listen("tcp", ":0")
if err != nil {
t.Fatalf("Cannot listen on http port: %v", err)
}
handler := http.NewServeMux()
ft.HTTPServer = http.Server{
Handler: handler,
}
go ft.HTTPServer.Serve(ft.HTTPListener)
vtPort = int32(ft.HTTPListener.Addr().(*net.TCPAddr).Port)
}
// create a test agent on that port, and re-read the record
// (it has new ports and IP)
ft.Agent = tabletmanager.NewTestActionAgent(context.Background(), wr.TopoServer(), ft.Tablet.Alias, vtPort, gRPCPort, ft.FakeMysqlDaemon)
ft.Tablet = ft.Agent.Tablet()
// create the gRPC server
ft.RPCServer = grpc.NewServer()
grpctmserver.RegisterForTest(ft.RPCServer, ft.Agent)
go ft.RPCServer.Serve(ft.Listener)
// and wait for it to serve, so we don't start using it before it's
// ready.
timeout := 5 * time.Second
step := 10 * time.Millisecond
c := tmclient.NewTabletManagerClient()
for timeout >= 0 {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
err := c.Ping(ctx, topo.NewTabletInfo(ft.Agent.Tablet(), -1))
cancel()
if err == nil {
break
}
time.Sleep(step)
timeout -= step
}
if timeout < 0 {
panic("StartActionLoop failed.")
}
}
func TestTabletData(t *testing.T) {
db := fakesqldb.Register()
ts := zktestserver.New(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
if err := ts.CreateKeyspace(context.Background(), "ks", &topodatapb.Keyspace{
ShardingColumnName: "keyspace_id",
ShardingColumnType: topodatapb.KeyspaceIdType_UINT64,
}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
tablet1 := testlib.NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db, testlib.TabletKeyspaceShard(t, "ks", "-80"))
tablet1.StartActionLoop(t, wr)
defer tablet1.StopActionLoop(t)
shsq := newStreamHealthTabletServer(t)
grpcqueryservice.Register(tablet1.RPCServer, shsq)
thc := newTabletHealthCache(ts)
stats := &querypb.RealtimeStats{
HealthError: "testHealthError",
SecondsBehindMaster: 72,
CpuUsage: 1.1,
}
// Keep broadcasting until the first result goes through.
stop := make(chan struct{})
go func() {
for {
select {
case <-stop:
return
default:
shsq.BroadcastHealth(42, stats)
}
}
}()
// Start streaming and wait for the first result.
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
result, err := thc.Get(ctx, tablet1.Tablet.Alias)
cancel()
close(stop)
if err != nil {
t.Fatalf("thc.Get failed: %v", err)
}
if got, want := result.RealtimeStats, stats; !proto.Equal(got, want) {
t.Errorf("RealtimeStats = %#v, want %#v", got, want)
}
}
// TestTabletExternallyReparentedContinueOnUnexpectedMaster makes sure
// that we ignore mysql's master if the flag is set
func TestTabletExternallyReparentedContinueOnUnexpectedMaster(t *testing.T) {
tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */)
ctx := context.Background()
ts := zktopo.NewTestServer(t, []string{"cell1"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
// Create an old master, a new master, two good slaves, one bad slave
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA)
goodSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA)
// On the elected master, we will respond to
// TabletActionSlaveWasPromoted, so we need a MysqlDaemon
// that returns no master, and the new port (as returned by mysql)
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// On the old master, we will only respond to
// TabletActionSlaveWasRestarted and point to a bad host
oldMaster.StartActionLoop(t, wr)
defer oldMaster.StopActionLoop(t)
// On the good slave, we will respond to
// TabletActionSlaveWasRestarted and point to a bad host
goodSlave.StartActionLoop(t, wr)
defer goodSlave.StopActionLoop(t)
// This tests the good case, where everything works as planned
t.Logf("TabletExternallyReparented(new master) expecting success")
tmc := tmclient.NewTabletManagerClient()
ti, err := ts.GetTablet(ctx, newMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if err := tmc.TabletExternallyReparented(context.Background(), ti, ""); err != nil {
t.Fatalf("TabletExternallyReparented(replica) failed: %v", err)
}
}
func TestReparentTablet(t *testing.T) {
ctx := context.Background()
db := fakesqldb.Register()
ts := zktestserver.New(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
// create shard and tablets
if err := ts.CreateShard(ctx, "test_keyspace", "0"); err != nil {
t.Fatalf("CreateShard failed: %v", err)
}
master := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_MASTER, db)
slave := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db)
// mark the master inside the shard
si, err := ts.GetShard(ctx, "test_keyspace", "0")
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
si.MasterAlias = master.Tablet.Alias
if err := ts.UpdateShard(ctx, si); err != nil {
t.Fatalf("UpdateShard failed: %v", err)
}
// master action loop (to initialize host and port)
master.StartActionLoop(t, wr)
defer master.StopActionLoop(t)
// slave loop
slave.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", master.Tablet.Hostname, master.Tablet.PortMap["mysql"])
slave.FakeMysqlDaemon.SetMasterCommandsResult = []string{"set master cmd 1"}
slave.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"set master cmd 1",
}
slave.StartActionLoop(t, wr)
defer slave.StopActionLoop(t)
// run ReparentTablet
if err := wr.ReparentTablet(ctx, slave.Tablet.Alias); err != nil {
t.Fatalf("ReparentTablet failed: %v", err)
}
// check what was run
if err := slave.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("slave.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
checkSemiSyncEnabled(t, false, true, slave)
}
func main() {
defer exit.RecoverAll()
defer logutil.Flush()
flag.Parse()
args := flag.Args()
if len(args) == 0 {
flag.Usage()
exit.Return(1)
}
action := args[0]
startMsg := fmt.Sprintf("USER=%v SUDO_USER=%v %v", os.Getenv("USER"), os.Getenv("SUDO_USER"), strings.Join(os.Args, " "))
if syslogger, err := syslog.New(syslog.LOG_INFO, "vtctl "); err == nil {
syslogger.Info(startMsg)
} else {
log.Warningf("cannot connect to syslog: %v", err)
}
topoServer := topo.GetServer()
defer topo.CloseServers()
ctx, cancel := context.WithTimeout(context.Background(), *waitTime)
wr := wrangler.New(logutil.NewConsoleLogger(), topoServer, tmclient.NewTabletManagerClient(), *lockWaitTimeout)
installSignalHandlers(cancel)
for _, f := range initFuncs {
f()
}
err := vtctl.RunCommand(ctx, wr, args)
cancel()
switch err {
case vtctl.ErrUnknownCommand:
flag.Usage()
exit.Return(1)
case nil:
// keep going
default:
log.Errorf("action failed: %v %v", action, err)
exit.Return(255)
}
}
func TestVersion(t *testing.T) {
// We need to run this test with the /debug/vars version of the
// plugin.
wrangler.ResetDebugVarsGetVersion()
// Initialize our environment
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
vp := NewVtctlPipe(t, ts)
defer vp.Close()
// couple tablets is enough
sourceMaster := NewFakeTablet(t, wr, "cell1", 10, pb.TabletType_MASTER, db,
TabletKeyspaceShard(t, "source", "0"),
StartHTTPServer())
sourceReplica := NewFakeTablet(t, wr, "cell1", 11, pb.TabletType_REPLICA, db,
TabletKeyspaceShard(t, "source", "0"),
StartHTTPServer())
// sourceMaster loop
sourceMasterGitRev := "fake git rev"
sourceMaster.StartActionLoop(t, wr)
sourceMaster.HTTPServer.Handler.(*http.ServeMux).HandleFunc("/debug/vars", expvarHandler(&sourceMasterGitRev))
defer sourceMaster.StopActionLoop(t)
// sourceReplica loop
sourceReplicaGitRev := "fake git rev"
sourceReplica.StartActionLoop(t, wr)
sourceReplica.HTTPServer.Handler.(*http.ServeMux).HandleFunc("/debug/vars", expvarHandler(&sourceReplicaGitRev))
defer sourceReplica.StopActionLoop(t)
// test when versions are the same
if err := vp.Run([]string{"ValidateVersionKeyspace", sourceMaster.Tablet.Keyspace}); err != nil {
t.Fatalf("ValidateVersionKeyspace(same) failed: %v", err)
}
// test when versions are different
sourceReplicaGitRev = "different fake git rev"
if err := vp.Run([]string{"ValidateVersionKeyspace", sourceMaster.Tablet.Keyspace}); err == nil || !strings.Contains(err.Error(), "is different than slave") {
t.Fatalf("ValidateVersionKeyspace(different) returned an unexpected error: %v", err)
}
}
// ExecuteVtctlCommand is part of the vtctldatapb.VtctlServer interface
func (s *VtctlServer) ExecuteVtctlCommand(args *vtctldatapb.ExecuteVtctlCommandRequest, stream vtctlservicepb.Vtctl_ExecuteVtctlCommandServer) (err error) {
defer servenv.HandlePanic("vtctl", &err)
// create a logger, send the result back to the caller
logstream := logutil.NewCallbackLogger(func(e *logutilpb.Event) {
// If the client disconnects, we will just fail
// to send the log events, but won't interrupt
// the command.
stream.Send(&vtctldatapb.ExecuteVtctlCommandResponse{
Event: e,
})
})
logger := logutil.NewTeeLogger(logstream, logutil.NewConsoleLogger())
// create the wrangler
wr := wrangler.New(logger, s.ts, tmclient.NewTabletManagerClient())
// execute the command
return vtctl.RunCommand(stream.Context(), wr, args.Args)
}
// ApplyKeyspaceAction applies the provided action to the keyspace.
func (ar *ActionRepository) ApplyKeyspaceAction(ctx context.Context, actionName, keyspace string, r *http.Request) *ActionResult {
result := &ActionResult{Name: actionName, Parameters: keyspace}
action, ok := ar.keyspaceActions[actionName]
if !ok {
result.error("Unknown keyspace action")
return result
}
ctx, cancel := context.WithTimeout(ctx, *actionTimeout)
wr := wrangler.New(logutil.NewConsoleLogger(), ar.ts, tmclient.NewTabletManagerClient())
output, err := action(ctx, wr, keyspace, r)
cancel()
if err != nil {
result.error(err.Error())
return result
}
result.Output = output
return result
}
func newReplica(lagUpdateInterval, degrationInterval, degrationDuration time.Duration) *replica {
t := &testing.T{}
ts := zktestserver.New(t, []string{"cell1"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
db := fakesqldb.Register()
fakeTablet := testlib.NewFakeTablet(t, wr, "cell1", 0,
topodatapb.TabletType_REPLICA, db, testlib.TabletKeyspaceShard(t, "ks", "-80"))
fakeTablet.StartActionLoop(t, wr)
target := querypb.Target{
Keyspace: "ks",
Shard: "-80",
TabletType: topodatapb.TabletType_REPLICA,
}
qs := fakes.NewStreamHealthQueryService(target)
grpcqueryservice.Register(fakeTablet.RPCServer, qs)
throttler, err := throttler.NewThrottler("replica", "TPS", 1, *rate, throttler.ReplicationLagModuleDisabled)
if err != nil {
log.Fatal(err)
}
var nextDegration time.Time
if degrationInterval != time.Duration(0) {
nextDegration = time.Now().Add(degrationInterval)
}
r := &replica{
fakeTablet: fakeTablet,
qs: qs,
throttler: throttler,
replicationStream: make(chan time.Time, 1*1024*1024),
lagUpdateInterval: lagUpdateInterval,
degrationInterval: degrationInterval,
degrationDuration: degrationDuration,
nextDegration: nextDegration,
stopChan: make(chan struct{}),
}
r.wg.Add(1)
go r.processReplicationStream()
return r
}
func initSchema() {
// Start schema manager service if needed.
if *schemaChangeDir != "" {
interval := 60
if *schemaChangeCheckInterval > 0 {
interval = *schemaChangeCheckInterval
}
timer := timer.NewTimer(time.Duration(interval) * time.Second)
controllerFactory, err :=
schemamanager.GetControllerFactory(*schemaChangeController)
if err != nil {
log.Fatalf("unable to get a controller factory, error: %v", err)
}
timer.Start(func() {
controller, err := controllerFactory(map[string]string{
schemamanager.SchemaChangeDirName: *schemaChangeDir,
schemamanager.SchemaChangeUser: *schemaChangeUser,
})
if err != nil {
log.Errorf("failed to get controller, error: %v", err)
return
}
ctx := context.Background()
err = schemamanager.Run(
ctx,
controller,
schemamanager.NewTabletExecutor(
tmclient.NewTabletManagerClient(), ts),
)
if err != nil {
log.Errorf("Schema change failed, error: %v", err)
}
})
servenv.OnClose(func() { timer.Stop() })
}
}
// ExecuteVtctlCommand is the server side method that will execute the query,
// and stream the results.
func (s *VtctlServer) ExecuteVtctlCommand(ctx context.Context, query *gorpcproto.ExecuteVtctlCommandArgs, sendReply func(interface{}) error) (err error) {
defer vtctl.HandlePanic(&err)
// create a logger, send the result back to the caller
logstream := logutil.NewChannelLogger(10)
logger := logutil.NewTeeLogger(logstream, logutil.NewConsoleLogger())
// send logs to the caller
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
for e := range logstream {
// Note we don't interrupt the loop here, as
// we still need to flush and finish the
// command, even if the channel to the client
// has been broken. We'll just keep trying.
sendReply(&e)
}
wg.Done()
}()
// create the wrangler
wr := wrangler.New(logger, s.ts, tmclient.NewTabletManagerClient(), query.LockTimeout)
// FIXME(alainjobart) use a single context, copy the source info from it
ctx, cancel := context.WithTimeout(context.TODO(), query.ActionTimeout)
// execute the command
err = vtctl.RunCommand(ctx, wr, query.Args)
cancel()
// close the log channel, and wait for them all to be sent
close(logstream)
wg.Wait()
return err
}
func testSplitClone(t *testing.T, strategy string) {
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
sourceMaster := testlib.NewFakeTablet(t, wr, "cell1", 0,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-80"))
sourceRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 1,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80"))
sourceRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 2,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80"))
leftMaster := testlib.NewFakeTablet(t, wr, "cell1", 10,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-40"))
leftRdonly := testlib.NewFakeTablet(t, wr, "cell1", 11,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-40"))
rightMaster := testlib.NewFakeTablet(t, wr, "cell1", 20,
topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "40-80"))
rightRdonly := testlib.NewFakeTablet(t, wr, "cell1", 21,
topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "40-80"))
for _, ft := range []*testlib.FakeTablet{sourceMaster, sourceRdonly1, sourceRdonly2, leftMaster, leftRdonly, rightMaster, rightRdonly} {
ft.StartActionLoop(t, wr)
defer ft.StopActionLoop(t)
}
// add the topo and schema data we'll need
ctx := context.Background()
if err := topo.CreateShard(ctx, ts, "ks", "80-"); err != nil {
t.Fatalf("CreateShard(\"-80\") failed: %v", err)
}
if err := wr.SetKeyspaceShardingInfo(ctx, "ks", "keyspace_id", key.KIT_UINT64, 4, false); err != nil {
t.Fatalf("SetKeyspaceShardingInfo failed: %v", err)
}
if err := wr.RebuildKeyspaceGraph(ctx, "ks", nil, true); err != nil {
t.Fatalf("RebuildKeyspaceGraph failed: %v", err)
}
gwrk, err := NewSplitCloneWorker(wr, "cell1", "ks", "-80", nil, strategy, 10 /*sourceReaderCount*/, 4 /*destinationPackCount*/, 1 /*minTableSizeForSplit*/, 10 /*destinationWriterCount*/)
if err != nil {
t.Errorf("Worker creation failed: %v", err)
}
wrk := gwrk.(*SplitCloneWorker)
for _, sourceRdonly := range []*testlib.FakeTablet{sourceRdonly1, sourceRdonly2} {
sourceRdonly.FakeMysqlDaemon.Schema = &myproto.SchemaDefinition{
DatabaseSchema: "",
TableDefinitions: []*myproto.TableDefinition{
&myproto.TableDefinition{
Name: "table1",
Columns: []string{"id", "msg", "keyspace_id"},
PrimaryKeyColumns: []string{"id"},
Type: myproto.TableBaseTable,
// This informs how many rows we can pack into a single insert
DataLength: 2048,
},
},
}
sourceRdonly.FakeMysqlDaemon.DbAppConnectionFactory = SourceRdonlyFactory(t)
sourceRdonly.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
GTIDSet: myproto.MariadbGTID{Domain: 12, Server: 34, Sequence: 5678},
}
sourceRdonly.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP SLAVE",
"START SLAVE",
}
sourceRdonly.RPCServer.Register(gorpcqueryservice.New(&testQueryService{t: t}))
}
// We read 100 source rows. sourceReaderCount is set to 10, so
// we'll have 100/10=10 rows per table chunk.
// destinationPackCount is set to 4, so we take 4 source rows
// at once. So we'll process 4 + 4 + 2 rows to get to 10.
// That means 3 insert statements on each target (each
// containing half of the rows, i.e. 2 + 2 + 1 rows). So 3 * 10
// = 30 insert statements on each destination.
leftMaster.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
leftRdonly.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
rightMaster.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
rightRdonly.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30)
// Only wait 1 ms between retries, so that the test passes faster
*executeFetchRetryTime = (1 * time.Millisecond)
err = wrk.Run(ctx)
status := wrk.StatusAsText()
t.Logf("Got status: %v", status)
if err != nil || wrk.State != WorkerStateDone {
t.Errorf("Worker run failed")
}
if statsDestinationAttemptedResolves.String() != "3" {
t.Errorf("Wrong statsDestinationAttemptedResolves: wanted %v, got %v", "3", statsDestinationAttemptedResolves.String())
}
if statsDestinationActualResolves.String() != "1" {
t.Errorf("Wrong statsDestinationActualResolves: wanted %v, got %v", "1", statsDestinationActualResolves.String())
}
if statsRetryCounters.String() != "{\"ReadOnly\": 2}" {
t.Errorf("Wrong statsRetryCounters: wanted %v, got %v", "{\"ReadOnly\": 2}", statsRetryCounters.String())
}
}
func TestPermissions(t *testing.T) {
// Initialize our environment
ctx := context.Background()
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
vp := NewVtctlPipe(t, ts)
defer vp.Close()
master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db)
replica := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db)
// mark the master inside the shard
si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard)
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
si.MasterAlias = master.Tablet.Alias
if err := ts.UpdateShard(ctx, si); err != nil {
t.Fatalf("UpdateShard failed: %v", err)
}
// master will be asked for permissions
master.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{
"SELECT * FROM mysql.user": {
Fields: []*querypb.Field{
{
Name: "Host",
Type: sqltypes.Char,
},
{
Name: "User",
Type: sqltypes.Char,
},
{
Name: "Password",
Type: sqltypes.Char,
},
{
Name: "Select_priv",
Type: sqltypes.Char,
},
{
Name: "Insert_priv",
Type: sqltypes.Char,
},
{
Name: "Update_priv",
Type: sqltypes.Char,
},
{
Name: "Delete_priv",
Type: sqltypes.Char,
},
{
Name: "Create_priv",
Type: sqltypes.Char,
},
{
Name: "Drop_priv",
Type: sqltypes.Char,
},
{
Name: "Reload_priv",
Type: sqltypes.Char,
},
{
Name: "Shutdown_priv",
Type: sqltypes.Char,
},
{
Name: "Process_priv",
Type: sqltypes.Char,
},
{
Name: "File_priv",
Type: sqltypes.Char,
},
{
Name: "Grant_priv",
Type: sqltypes.Char,
},
{
Name: "References_priv",
Type: sqltypes.Char,
},
{
Name: "Index_priv",
Type: sqltypes.Char,
},
{
Name: "Alter_priv",
Type: sqltypes.Char,
},
{
Name: "Show_db_priv",
Type: sqltypes.Char,
},
{
Name: "Super_priv",
Type: sqltypes.Char,
},
{
Name: "Create_tmp_table_priv",
Type: sqltypes.Char,
},
{
Name: "Lock_tables_priv",
Type: sqltypes.Char,
},
{
Name: "Execute_priv",
Type: sqltypes.Char,
},
{
Name: "Repl_slave_priv",
Type: sqltypes.Char,
},
{
Name: "Repl_client_priv",
Type: sqltypes.Char,
},
{
Name: "Create_view_priv",
Type: sqltypes.Char,
},
{
Name: "Show_view_priv",
Type: sqltypes.Char,
},
{
Name: "Create_routine_priv",
Type: sqltypes.Char,
},
{
Name: "Alter_routine_priv",
Type: sqltypes.Char,
},
{
Name: "Create_user_priv",
Type: sqltypes.Char,
},
{
Name: "Event_priv",
Type: sqltypes.Char,
},
{
Name: "Trigger_priv",
Type: sqltypes.Char,
},
{
Name: "Create_tablespace_priv",
Type: sqltypes.Char,
},
{
Name: "ssl_type",
Type: sqltypes.Char,
},
{
Name: "ssl_cipher",
Type: 252,
},
{
Name: "x509_issuer",
Type: 252,
},
{
Name: "x509_subject",
Type: 252,
},
{
Name: "max_questions",
Type: 3,
},
{
Name: "max_updates",
Type: 3,
},
{
Name: "max_connections",
Type: 3,
},
{
Name: "max_user_connections",
Type: 3,
},
{
Name: "plugin",
Type: sqltypes.Char,
},
{
Name: "authentication_string",
Type: 252,
},
{
Name: "password_expired",
Type: sqltypes.Char,
},
{
Name: "is_role",
Type: sqltypes.Char,
}},
RowsAffected: 0x6,
InsertID: 0x0,
Rows: [][]sqltypes.Value{
{
sqltypes.MakeString([]byte("test_host1")),
sqltypes.MakeString([]byte("test_user1")),
sqltypes.MakeString([]byte("test_password1")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N"))},
{
sqltypes.MakeString([]byte("test_host2")),
sqltypes.MakeString([]byte("test_user2")),
sqltypes.MakeString([]byte("test_password2")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N"))},
{
sqltypes.MakeString([]byte("test_host3")),
sqltypes.MakeString([]byte("test_user3")),
sqltypes.MakeString([]byte("test_password3")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N"))},
{
sqltypes.MakeString([]byte("test_host4")),
sqltypes.MakeString([]byte("test_user4")),
sqltypes.MakeString([]byte("test_password4")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("0")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("N")),
},
},
},
"SELECT * FROM mysql.db": {
Fields: []*querypb.Field{
{
Name: "Host",
Type: sqltypes.Char,
},
{
Name: "Db",
Type: sqltypes.Char,
},
{
Name: "User",
Type: sqltypes.Char,
},
{
Name: "Select_priv",
Type: sqltypes.Char,
},
{
Name: "Insert_priv",
Type: sqltypes.Char,
},
{
Name: "Update_priv",
Type: sqltypes.Char,
},
{
Name: "Delete_priv",
Type: sqltypes.Char,
},
{
Name: "Create_priv",
Type: sqltypes.Char,
},
{
Name: "Drop_priv",
Type: sqltypes.Char,
},
{
Name: "Grant_priv",
Type: sqltypes.Char,
},
{
Name: "References_priv",
Type: sqltypes.Char,
},
{
Name: "Index_priv",
Type: sqltypes.Char,
},
{
Name: "Alter_priv",
Type: sqltypes.Char,
},
{
Name: "Create_tmp_table_priv",
Type: sqltypes.Char,
},
{
Name: "Lock_tables_priv",
Type: sqltypes.Char,
},
{
Name: "Create_view_priv",
Type: sqltypes.Char,
},
{
Name: "Show_view_priv",
Type: sqltypes.Char,
},
{
Name: "Create_routine_priv",
Type: sqltypes.Char,
},
{
Name: "Alter_routine_priv",
Type: sqltypes.Char,
},
{
Name: "Execute_priv",
Type: sqltypes.Char,
},
{
Name: "Event_priv",
Type: sqltypes.Char,
},
{
Name: "Trigger_priv",
Type: sqltypes.Char,
},
},
RowsAffected: 0,
InsertID: 0,
Rows: [][]sqltypes.Value{
{
sqltypes.MakeString([]byte("test_host")),
sqltypes.MakeString([]byte("test_db")),
sqltypes.MakeString([]byte("test_user")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("N")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
sqltypes.MakeString([]byte("Y")),
},
},
},
}
master.StartActionLoop(t, wr)
defer master.StopActionLoop(t)
// Make a two-level-deep copy, so we can make them diverge later.
user := *master.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"]
user.Fields = append([]*querypb.Field{}, user.Fields...)
// replica will be asked for permissions
replica.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{
"SELECT * FROM mysql.user": &user,
"SELECT * FROM mysql.db": master.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.db"],
}
replica.StartActionLoop(t, wr)
defer replica.StopActionLoop(t)
// Overwrite with the correct value to make sure it passes.
replica.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"].Fields[0] = &querypb.Field{
Name: "Host",
Type: sqltypes.Char,
}
// run ValidatePermissionsKeyspace, this should work
if err := vp.Run([]string{"ValidatePermissionsKeyspace", master.Tablet.Keyspace}); err != nil {
t.Fatalf("ValidatePermissionsKeyspace failed: %v", err)
}
// modify one field, this should fail
replica.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"].Fields[0] = &querypb.Field{
Name: "Wrong",
Type: sqltypes.Char,
}
// run ValidatePermissionsKeyspace again, this should now fail
if err := vp.Run([]string{"ValidatePermissionsKeyspace", master.Tablet.Keyspace}); err == nil || !strings.Contains(err.Error(), "has an extra user") {
t.Fatalf("ValidatePermissionsKeyspace has unexpected err: %v", err)
}
}
// NewFakeTabletManagerClient should be used to create a new FakeTabletManagerClient.
// There is intentionally no init in this file with a call to RegisterTabletManagerClientFactory.
// There shouldn't be any legitimate use-case where we would want to start a vitess cluster
// with a FakeTMC, and we don't want to do it by accident.
func NewFakeTabletManagerClient() tmclient.TabletManagerClient {
return &FakeTabletManagerClient{
tmc: tmclient.NewTabletManagerClient(),
}
}
func TestTabletExternallyReparented(t *testing.T) {
tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */)
ctx := context.Background()
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
vp := NewVtctlPipe(t, ts)
defer vp.Close()
// Create an old master, a new master, two good slaves, one bad slave
oldMaster := NewFakeTablet(t, wr, "cell1", 0, pb.TabletType_MASTER, db)
newMaster := NewFakeTablet(t, wr, "cell1", 1, pb.TabletType_REPLICA, db)
goodSlave1 := NewFakeTablet(t, wr, "cell1", 2, pb.TabletType_REPLICA, db)
goodSlave2 := NewFakeTablet(t, wr, "cell2", 3, pb.TabletType_REPLICA, db)
badSlave := NewFakeTablet(t, wr, "cell1", 4, pb.TabletType_REPLICA, db)
// Add a new Cell to the Shard, that doesn't map to any read topo cell,
// to simulate a data center being unreachable.
si, err := ts.GetShard(ctx, "test_keyspace", "0")
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
si.Cells = append(si.Cells, "cell666")
if err := ts.UpdateShard(ctx, si); err != nil {
t.Fatalf("UpdateShard failed: %v", err)
}
// Slightly unrelated test: make sure we can find the tablets
// even with a datacenter being down.
tabletMap, err := ts.GetTabletMapForShardByCell(ctx, "test_keyspace", "0", []string{"cell1"})
if err != nil {
t.Fatalf("GetTabletMapForShardByCell should have worked but got: %v", err)
}
master, err := topotools.FindTabletByIPAddrAndPort(tabletMap, oldMaster.Tablet.Ip, "vt", oldMaster.Tablet.PortMap["vt"])
if err != nil || !topoproto.TabletAliasEqual(&master, oldMaster.Tablet.Alias) {
t.Fatalf("FindTabletByIPAddrAndPort(master) failed: %v %v", err, master)
}
slave1, err := topotools.FindTabletByIPAddrAndPort(tabletMap, goodSlave1.Tablet.Ip, "vt", goodSlave1.Tablet.PortMap["vt"])
if err != nil || !topoproto.TabletAliasEqual(&slave1, goodSlave1.Tablet.Alias) {
t.Fatalf("FindTabletByIPAddrAndPort(slave1) failed: %v %v", err, master)
}
slave2, err := topotools.FindTabletByIPAddrAndPort(tabletMap, goodSlave2.Tablet.Ip, "vt", goodSlave2.Tablet.PortMap["vt"])
if err != topo.ErrNoNode {
t.Fatalf("FindTabletByIPAddrAndPort(slave2) worked: %v %v", err, slave2)
}
// Make sure the master is not exported in other cells
tabletMap, err = ts.GetTabletMapForShardByCell(ctx, "test_keyspace", "0", []string{"cell2"})
master, err = topotools.FindTabletByIPAddrAndPort(tabletMap, oldMaster.Tablet.Ip, "vt", oldMaster.Tablet.PortMap["vt"])
if err != topo.ErrNoNode {
t.Fatalf("FindTabletByIPAddrAndPort(master) worked in cell2: %v %v", err, master)
}
tabletMap, err = ts.GetTabletMapForShard(ctx, "test_keyspace", "0")
if err != topo.ErrPartialResult {
t.Fatalf("GetTabletMapForShard should have returned ErrPartialResult but got: %v", err)
}
master, err = topotools.FindTabletByIPAddrAndPort(tabletMap, oldMaster.Tablet.Ip, "vt", oldMaster.Tablet.PortMap["vt"])
if err != nil || !topoproto.TabletAliasEqual(&master, oldMaster.Tablet.Alias) {
t.Fatalf("FindTabletByIPAddrAndPort(master) failed: %v %v", err, master)
}
// On the elected master, we will respond to
// TabletActionSlaveWasPromoted
newMaster.StartActionLoop(t, wr)
defer newMaster.StopActionLoop(t)
// On the old master, we will only respond to
// TabletActionSlaveWasRestarted.
oldMaster.StartActionLoop(t, wr)
defer oldMaster.StopActionLoop(t)
// On the good slaves, we will respond to
// TabletActionSlaveWasRestarted.
goodSlave1.StartActionLoop(t, wr)
defer goodSlave1.StopActionLoop(t)
goodSlave2.StartActionLoop(t, wr)
defer goodSlave2.StopActionLoop(t)
// On the bad slave, we will respond to
// TabletActionSlaveWasRestarted with bad data.
badSlave.StartActionLoop(t, wr)
defer badSlave.StopActionLoop(t)
// First test: reparent to the same master, make sure it works
// as expected.
tmc := tmclient.NewTabletManagerClient()
ti, err := ts.GetTablet(ctx, oldMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if err := vp.Run([]string{"TabletExternallyReparented", topoproto.TabletAliasString(oldMaster.Tablet.Alias)}); err != nil {
t.Fatalf("TabletExternallyReparented(same master) should have worked: %v", err)
}
// Second test: reparent to a replica, and pretend the old
// master is still good to go.
// This tests a bad case; the new designated master is a slave,
// but we should do what we're told anyway
ti, err = ts.GetTablet(ctx, goodSlave1.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
waitID := makeWaitID()
if err := tmc.TabletExternallyReparented(context.Background(), ti, waitID); err != nil {
t.Fatalf("TabletExternallyReparented(slave) error: %v", err)
}
waitForExternalReparent(t, waitID)
// This tests the good case, where everything works as planned
t.Logf("TabletExternallyReparented(new master) expecting success")
ti, err = ts.GetTablet(ctx, newMaster.Tablet.Alias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
waitID = makeWaitID()
if err := tmc.TabletExternallyReparented(context.Background(), ti, waitID); err != nil {
t.Fatalf("TabletExternallyReparented(replica) failed: %v", err)
}
waitForExternalReparent(t, waitID)
// Now double-check the serving graph is good.
// Should only have one good replica left.
addrs, _, err := ts.GetEndPoints(ctx, "cell1", "test_keyspace", "0", pb.TabletType_REPLICA)
if err != nil {
t.Fatalf("GetEndPoints failed at the end: %v", err)
}
if len(addrs.Entries) != 1 {
t.Fatalf("GetEndPoints has too many entries: %v", addrs)
}
}
func TestMigrateServedTypes(t *testing.T) {
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
vp := NewVtctlPipe(t, ts)
defer vp.Close()
// create keyspace
if err := ts.CreateKeyspace(context.Background(), "ks", &pb.Keyspace{
ShardingColumnName: "keyspace_id",
ShardingColumnType: pb.KeyspaceIdType_UINT64,
}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
// create the source shard
sourceMaster := NewFakeTablet(t, wr, "cell1", 10, pb.TabletType_MASTER, db,
TabletKeyspaceShard(t, "ks", "0"))
sourceReplica := NewFakeTablet(t, wr, "cell1", 11, pb.TabletType_REPLICA, db,
TabletKeyspaceShard(t, "ks", "0"))
sourceRdonly := NewFakeTablet(t, wr, "cell1", 12, pb.TabletType_RDONLY, db,
TabletKeyspaceShard(t, "ks", "0"))
// create the first destination shard
dest1Master := NewFakeTablet(t, wr, "cell1", 20, pb.TabletType_MASTER, db,
TabletKeyspaceShard(t, "ks", "-80"))
dest1Replica := NewFakeTablet(t, wr, "cell1", 21, pb.TabletType_REPLICA, db,
TabletKeyspaceShard(t, "ks", "-80"))
dest1Rdonly := NewFakeTablet(t, wr, "cell1", 22, pb.TabletType_RDONLY, db,
TabletKeyspaceShard(t, "ks", "-80"))
// create the second destination shard
dest2Master := NewFakeTablet(t, wr, "cell1", 30, pb.TabletType_MASTER, db,
TabletKeyspaceShard(t, "ks", "80-"))
dest2Replica := NewFakeTablet(t, wr, "cell1", 31, pb.TabletType_REPLICA, db,
TabletKeyspaceShard(t, "ks", "80-"))
dest2Rdonly := NewFakeTablet(t, wr, "cell1", 32, pb.TabletType_RDONLY, db,
TabletKeyspaceShard(t, "ks", "80-"))
// double check the shards have the right served types
checkShardServedTypes(t, ts, "0", 3)
checkShardServedTypes(t, ts, "-80", 0)
checkShardServedTypes(t, ts, "80-", 0)
// sourceRdonly will see the refresh
sourceRdonly.StartActionLoop(t, wr)
defer sourceRdonly.StopActionLoop(t)
// sourceReplica will see the refresh
sourceReplica.StartActionLoop(t, wr)
defer sourceReplica.StopActionLoop(t)
// sourceMaster will see the refresh, and has to respond to it
// also will be asked about its replication position.
sourceMaster.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
GTIDSet: myproto.MariadbGTID{
Domain: 5,
Server: 456,
Sequence: 892,
},
}
sourceMaster.StartActionLoop(t, wr)
defer sourceMaster.StopActionLoop(t)
// dest1Rdonly will see the refresh
dest1Rdonly.StartActionLoop(t, wr)
defer dest1Rdonly.StopActionLoop(t)
// dest1Replica will see the refresh
dest1Replica.StartActionLoop(t, wr)
defer dest1Replica.StopActionLoop(t)
// dest1Master will see the refresh, and has to respond to it.
// It will also need to respond to WaitBlpPosition, saying it's already caught up.
dest1Master.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*mproto.QueryResult{
"SELECT pos, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=0": &mproto.QueryResult{
Rows: [][]sqltypes.Value{
[]sqltypes.Value{
sqltypes.MakeString([]byte(myproto.EncodeReplicationPosition(sourceMaster.FakeMysqlDaemon.CurrentMasterPosition))),
sqltypes.MakeString([]byte("")),
},
},
},
}
dest1Master.StartActionLoop(t, wr)
defer dest1Master.StopActionLoop(t)
// dest2Rdonly will see the refresh
dest2Rdonly.StartActionLoop(t, wr)
defer dest2Rdonly.StopActionLoop(t)
// dest2Replica will see the refresh
dest2Replica.StartActionLoop(t, wr)
defer dest2Replica.StopActionLoop(t)
// dest2Master will see the refresh, and has to respond to it.
// It will also need to respond to WaitBlpPosition, saying it's already caught up.
dest2Master.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*mproto.QueryResult{
"SELECT pos, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=0": &mproto.QueryResult{
Rows: [][]sqltypes.Value{
[]sqltypes.Value{
sqltypes.MakeString([]byte(myproto.EncodeReplicationPosition(sourceMaster.FakeMysqlDaemon.CurrentMasterPosition))),
sqltypes.MakeString([]byte("")),
},
},
},
}
dest2Master.StartActionLoop(t, wr)
defer dest2Master.StopActionLoop(t)
// simulate the clone, by fixing the dest shard record
if err := vp.Run([]string{"SourceShardAdd", "--key_range=-", "ks/-80", "0", "ks/0"}); err != nil {
t.Fatalf("SourceShardAdd failed: %v", err)
}
if err := vp.Run([]string{"SourceShardAdd", "--key_range=-", "ks/80-", "0", "ks/0"}); err != nil {
t.Fatalf("SourceShardAdd failed: %v", err)
}
// migrate rdonly over
if err := vp.Run([]string{"MigrateServedTypes", "ks/0", "rdonly"}); err != nil {
t.Fatalf("MigrateServedType(rdonly) failed: %v", err)
}
checkShardServedTypes(t, ts, "0", 2)
checkShardServedTypes(t, ts, "-80", 1)
checkShardServedTypes(t, ts, "80-", 1)
// migrate replica over
if err := vp.Run([]string{"MigrateServedTypes", "ks/0", "replica"}); err != nil {
t.Fatalf("MigrateServedType(replica) failed: %v", err)
}
checkShardServedTypes(t, ts, "0", 1)
checkShardServedTypes(t, ts, "-80", 2)
checkShardServedTypes(t, ts, "80-", 2)
// migrate master over
if err := vp.Run([]string{"MigrateServedTypes", "ks/0", "master"}); err != nil {
t.Fatalf("MigrateServedType(master) failed: %v", err)
}
checkShardServedTypes(t, ts, "0", 0)
checkShardServedTypes(t, ts, "-80", 3)
checkShardServedTypes(t, ts, "80-", 3)
}
func TestTabletData(t *testing.T) {
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
tablet1 := testlib.NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-80"))
tablet1.StartActionLoop(t, wr)
defer tablet1.StopActionLoop(t)
thc := newTabletHealthCache(ts, tmclient.NewTabletManagerClient())
// get the first result, it's not containing any data but the alias
result, err := thc.get(tablet1.Tablet.Alias)
if err != nil {
t.Fatalf("thc.get failed: %v", err)
}
var unpacked TabletHealth
if err := json.Unmarshal(result, &unpacked); err != nil {
t.Fatalf("bad json: %v", err)
}
if unpacked.HealthStreamReply.Tablet.Alias != tablet1.Tablet.Alias {
t.Fatalf("wrong alias: %v", &unpacked)
}
if unpacked.Version != 1 {
t.Errorf("wrong version, got %v was expecting 1", unpacked.Version)
}
// wait for the streaming RPC to be established
timeout := 5 * time.Second
for {
if tablet1.Agent.HealthStreamMapSize() > 0 {
break
}
timeout -= 10 * time.Millisecond
if timeout < 0 {
t.Fatalf("timeout waiting for streaming RPC to be established")
}
time.Sleep(10 * time.Millisecond)
}
// feed some data from the tablet, with just a data marker
hsr := &actionnode.HealthStreamReply{
BinlogPlayerMapSize: 42,
}
tablet1.Agent.BroadcastHealthStreamReply(hsr)
// and wait for the cache to pick it up
timeout = 5 * time.Second
for {
result, err = thc.get(tablet1.Tablet.Alias)
if err != nil {
t.Fatalf("thc.get failed: %v", err)
}
if err := json.Unmarshal(result, &unpacked); err != nil {
t.Fatalf("bad json: %v", err)
}
if unpacked.HealthStreamReply.BinlogPlayerMapSize == 42 {
if unpacked.Version != 2 {
t.Errorf("wrong version, got %v was expecting 2", unpacked.Version)
}
break
}
timeout -= 10 * time.Millisecond
if timeout < 0 {
t.Fatalf("timeout waiting for streaming RPC to be established")
}
time.Sleep(10 * time.Millisecond)
}
}
// TestInitMasterShardOneSlaveFails makes sure that if one slave fails to
// proceed, the action completes anyway
func TestInitMasterShardOneSlaveFails(t *testing.T) {
ctx := context.Background()
db := fakesqldb.Register()
ts := zktestserver.New(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
// Create a master, a couple slaves
master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db)
goodSlave := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db)
badSlave := NewFakeTablet(t, wr, "cell2", 2, topodatapb.TabletType_REPLICA, db)
// Master: set a plausible ReplicationPosition to return,
// and expect to add entry in _vt.reparent_journal
master.FakeMysqlDaemon.CurrentMasterPosition = replication.Position{
GTIDSet: replication.MariadbGTID{
Domain: 5,
Server: 456,
Sequence: 890,
},
}
master.FakeMysqlDaemon.ReadOnly = true
master.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"CREATE DATABASE IF NOT EXISTS _vt",
"SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal",
"CREATE DATABASE IF NOT EXISTS _vt",
"SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal",
"SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES",
}
master.StartActionLoop(t, wr)
defer master.StopActionLoop(t)
// goodSlave: expect to be re-parented
goodSlave.FakeMysqlDaemon.ReadOnly = true
goodSlave.FakeMysqlDaemon.SetSlavePositionCommandsPos = master.FakeMysqlDaemon.CurrentMasterPosition
goodSlave.FakeMysqlDaemon.SetSlavePositionCommandsResult = []string{"cmd1"}
goodSlave.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", master.Tablet.Hostname, master.Tablet.PortMap["mysql"])
goodSlave.FakeMysqlDaemon.SetMasterCommandsResult = []string{"set master cmd 1"}
goodSlave.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"cmd1",
"set master cmd 1",
"START SLAVE",
}
goodSlave.StartActionLoop(t, wr)
defer goodSlave.StopActionLoop(t)
// badSlave: insert an error by failing the master hostname input
// on purpose
badSlave.FakeMysqlDaemon.ReadOnly = true
badSlave.FakeMysqlDaemon.SetSlavePositionCommandsPos = master.FakeMysqlDaemon.CurrentMasterPosition
badSlave.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", "", master.Tablet.PortMap["mysql"])
badSlave.StartActionLoop(t, wr)
defer badSlave.StopActionLoop(t)
// also change the master alias in the Shard object, to make sure it
// is set back.
si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard)
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
si.MasterAlias.Uid++
if err := ts.UpdateShard(ctx, si); err != nil {
t.Fatalf("UpdateShard failed: %v", err)
}
// run InitShardMaster without force, it fails because master is
// changing.
if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, master.Tablet.Alias, false /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is not the shard master") {
t.Errorf("InitShardMaster with mismatched new master returned wrong error: %v", err)
}
// run InitShardMaster
if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, master.Tablet.Alias, true /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "wrong input for SetMasterCommands") {
t.Errorf("InitShardMaster with one failed slave returned wrong error: %v", err)
}
// check what was run: master should still be good
if master.FakeMysqlDaemon.ReadOnly {
t.Errorf("master was not turned read-write")
}
si, err = ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard)
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
if !topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) {
t.Errorf("unexpected shard master alias, got %v expected %v", si.MasterAlias, master.Tablet.Alias)
}
}
// TestInitMasterShard is the good scenario test, where everything
// works as planned
func TestInitMasterShard(t *testing.T) {
ctx := context.Background()
db := fakesqldb.Register()
ts := zktestserver.New(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
vp := NewVtctlPipe(t, ts)
defer vp.Close()
db.AddQuery("CREATE DATABASE IF NOT EXISTS `vt_test_keyspace`", &sqltypes.Result{})
// Create a master, a couple good slaves
master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db)
goodSlave1 := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db)
goodSlave2 := NewFakeTablet(t, wr, "cell2", 2, topodatapb.TabletType_REPLICA, db)
// Master: set a plausible ReplicationPosition to return,
// and expect to add entry in _vt.reparent_journal
master.FakeMysqlDaemon.CurrentMasterPosition = replication.Position{
GTIDSet: replication.MariadbGTID{
Domain: 5,
Server: 456,
Sequence: 890,
},
}
master.FakeMysqlDaemon.ReadOnly = true
master.FakeMysqlDaemon.ResetReplicationResult = []string{"reset rep 1"}
master.FakeMysqlDaemon.SetSlavePositionCommandsResult = []string{"new master shouldn't use this"}
master.FakeMysqlDaemon.SetMasterCommandsResult = []string{"new master shouldn't use this"}
master.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"reset rep 1",
"CREATE DATABASE IF NOT EXISTS _vt",
"SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal",
"CREATE DATABASE IF NOT EXISTS _vt",
"SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal",
"SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES",
}
master.StartActionLoop(t, wr)
defer master.StopActionLoop(t)
// Slave1: expect to be reset and re-parented
goodSlave1.FakeMysqlDaemon.ReadOnly = true
goodSlave1.FakeMysqlDaemon.ResetReplicationResult = []string{"reset rep 1"}
goodSlave1.FakeMysqlDaemon.SetSlavePositionCommandsPos = master.FakeMysqlDaemon.CurrentMasterPosition
goodSlave1.FakeMysqlDaemon.SetSlavePositionCommandsResult = []string{"cmd1"}
goodSlave1.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", master.Tablet.Hostname, master.Tablet.PortMap["mysql"])
goodSlave1.FakeMysqlDaemon.SetMasterCommandsResult = []string{"set master cmd 1"}
goodSlave1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"reset rep 1",
"cmd1",
"set master cmd 1",
"START SLAVE",
}
goodSlave1.StartActionLoop(t, wr)
defer goodSlave1.StopActionLoop(t)
// Slave2: expect to be re-parented
goodSlave2.FakeMysqlDaemon.ReadOnly = true
goodSlave2.FakeMysqlDaemon.ResetReplicationResult = []string{"reset rep 2"}
goodSlave2.FakeMysqlDaemon.SetSlavePositionCommandsPos = master.FakeMysqlDaemon.CurrentMasterPosition
goodSlave2.FakeMysqlDaemon.SetSlavePositionCommandsResult = []string{"cmd1", "cmd2"}
goodSlave2.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", master.Tablet.Hostname, master.Tablet.PortMap["mysql"])
goodSlave2.FakeMysqlDaemon.SetMasterCommandsResult = []string{"set master cmd 1"}
goodSlave2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"reset rep 2",
"cmd1",
"cmd2",
"set master cmd 1",
"START SLAVE",
}
goodSlave2.StartActionLoop(t, wr)
defer goodSlave2.StopActionLoop(t)
// run InitShardMaster
if err := vp.Run([]string{"InitShardMaster", "-wait_slave_timeout", "10s", master.Tablet.Keyspace + "/" + master.Tablet.Shard, topoproto.TabletAliasString(master.Tablet.Alias)}); err != nil {
t.Fatalf("InitShardMaster failed: %v", err)
}
// check what was run
if master.FakeMysqlDaemon.ReadOnly {
t.Errorf("master was not turned read-write")
}
si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard)
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
if !topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) {
t.Errorf("unexpected shard master alias, got %v expected %v", si.MasterAlias, master.Tablet.Alias)
}
if err := master.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("master.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
if err := goodSlave1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("goodSlave1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
if err := goodSlave2.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("goodSlave2.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
}
func testWaitForDrain(t *testing.T, desc, cells string, drain drainDirective, expectedErrors []string) {
const keyspace = "ks"
const shard = "-80"
db := fakesqldb.Register()
ts := zktestserver.New(t, []string{"cell1", "cell2"})
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
flag.Set("vtctl_healthcheck_timeout", "0.25s")
vp := NewVtctlPipe(t, ts)
defer vp.Close()
// Create keyspace.
if err := ts.CreateKeyspace(context.Background(), keyspace, &topodatapb.Keyspace{
ShardingColumnName: "keyspace_id",
ShardingColumnType: topodatapb.KeyspaceIdType_UINT64,
}); err != nil {
t.Fatalf("CreateKeyspace failed: %v", err)
}
t1 := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_REPLICA, db,
TabletKeyspaceShard(t, keyspace, shard))
t2 := NewFakeTablet(t, wr, "cell2", 1, topodatapb.TabletType_REPLICA, db,
TabletKeyspaceShard(t, keyspace, shard))
for _, ft := range []*FakeTablet{t1, t2} {
ft.StartActionLoop(t, wr)
defer ft.StopActionLoop(t)
}
target := querypb.Target{
Keyspace: keyspace,
Shard: shard,
TabletType: topodatapb.TabletType_REPLICA,
}
fqs1 := newFakeQueryService(target)
fqs2 := newFakeQueryService(target)
grpcqueryservice.RegisterForTest(t1.RPCServer, fqs1)
grpcqueryservice.RegisterForTest(t2.RPCServer, fqs2)
// Run vtctl WaitForDrain and react depending on its output.
timeout := "0.5s"
if len(expectedErrors) == 0 {
// Tests with a positive outcome should have a more generous timeout to
// avoid flakyness.
timeout = "30s"
}
stream, err := vp.RunAndStreamOutput(
[]string{"WaitForDrain", "-cells", cells, "-retry_delay", "100ms", "-timeout", timeout,
keyspace + "/" + shard, topodatapb.TabletType_REPLICA.String()})
if err != nil {
t.Fatalf("VtctlPipe.RunAndStreamOutput() failed: %v", err)
}
// QPS = 1.0. Tablets are not drained yet.
fqs1.addHealthResponse(1.0)
fqs2.addHealthResponse(1.0)
var le *logutilpb.Event
for {
le, err = stream.Recv()
if err != nil {
break
}
line := logutil.EventString(le)
t.Logf(line)
if strings.Contains(line, "for all healthy tablets to be drained") {
t.Log("Successfully waited for WaitForDrain to be blocked because tablets have a QPS rate > 0.0")
break
} else {
t.Log("waiting for WaitForDrain to see a QPS rate > 0.0")
}
}
if drain&DrainCell1 != 0 {
fqs1.addHealthResponse(0.0)
} else {
fqs1.addHealthResponse(2.0)
}
if drain&DrainCell2 != 0 {
fqs2.addHealthResponse(0.0)
} else {
fqs2.addHealthResponse(2.0)
}
// If a cell was drained, rate should go below <0.0 now.
// If not all selected cells were drained, this will end after "-timeout".
for {
le, err = stream.Recv()
if err == nil {
vp.t.Logf(logutil.EventString(le))
} else {
break
}
}
if len(expectedErrors) == 0 {
if err != io.EOF {
t.Fatalf("TestWaitForDrain: %v: no error expected but got: %v", desc, err)
}
// else: Success.
} else {
if err == nil || err == io.EOF {
t.Fatalf("TestWaitForDrain: %v: error expected but got none", desc)
}
for _, errString := range expectedErrors {
if !strings.Contains(err.Error(), errString) {
t.Fatalf("TestWaitForDrain: %v: error does not include expected string. got: %v want: %v", desc, err, errString)
}
}
// Success.
}
}