func TestTabletExternallyReparentedFailedOldMaster(t *testing.T) { tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */) ctx := context.Background() ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) // Create an old master, a new master, and a good slave. oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER) newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA) goodSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA) // Reparent to a replica, and pretend the old master is not responding. // On the elected master, we will respond to // TabletActionSlaveWasPromoted newMaster.StartActionLoop(t, wr) defer newMaster.StopActionLoop(t) // On the old master, we will only get a // TabletActionSlaveWasRestarted call, let's just not // respond to it at all // On the good slave, we will respond to // TabletActionSlaveWasRestarted. goodSlave.StartActionLoop(t, wr) defer goodSlave.StopActionLoop(t) // The reparent should work as expected here t.Logf("TabletExternallyReparented(new master) expecting success") tmc := tmclient.NewTabletManagerClient() ti, err := ts.GetTablet(ctx, newMaster.Tablet.Alias) if err != nil { t.Fatalf("GetTablet failed: %v", err) } waitID := makeWaitID() if err := tmc.TabletExternallyReparented(context.Background(), ti, waitID); err != nil { t.Fatalf("TabletExternallyReparented(replica) failed: %v", err) } waitForExternalReparent(t, waitID) // Now double-check the serving graph is good. // Should only have one good replica left. addrs, _, err := ts.GetEndPoints(ctx, "cell1", "test_keyspace", "0", topo.TYPE_REPLICA) if err != nil { t.Fatalf("GetEndPoints failed at the end: %v", err) } if len(addrs.Entries) != 1 { t.Fatalf("GetEndPoints has too many entries: %v", addrs) } // check the old master was converted to spare tablet, err := ts.GetTablet(ctx, oldMaster.Tablet.Alias) if err != nil { t.Fatalf("GetTablet(%v) failed: %v", oldMaster.Tablet.Alias, err) } if tablet.Type != topo.TYPE_SPARE { t.Fatalf("old master should be spare but is: %v", tablet.Type) } }
func TestTabletExternallyReparentedFailedOldMaster(t *testing.T) { tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */) ctx := context.Background() db := fakesqldb.Register() ts := zktestserver.New(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) // Create an old master, a new master, and a good slave. oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db) newMaster := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db) goodSlave := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db) // Reparent to a replica, and pretend the old master is not responding. // On the elected master, we will respond to // TabletActionSlaveWasPromoted newMaster.StartActionLoop(t, wr) defer newMaster.StopActionLoop(t) // On the old master, we will only get a // TabletActionSlaveWasRestarted call, let's just not // respond to it at all // On the good slave, we will respond to // TabletActionSlaveWasRestarted. goodSlave.StartActionLoop(t, wr) defer goodSlave.StopActionLoop(t) // The reparent should work as expected here t.Logf("TabletExternallyReparented(new master) expecting success") tmc := tmclient.NewTabletManagerClient() ti, err := ts.GetTablet(ctx, newMaster.Tablet.Alias) if err != nil { t.Fatalf("GetTablet failed: %v", err) } waitID := makeWaitID() if err := tmc.TabletExternallyReparented(context.Background(), ti.Tablet, waitID); err != nil { t.Fatalf("TabletExternallyReparented(replica) failed: %v", err) } waitForExternalReparent(t, waitID) // check the old master was converted to replica tablet, err := ts.GetTablet(ctx, oldMaster.Tablet.Alias) if err != nil { t.Fatalf("GetTablet(%v) failed: %v", oldMaster.Tablet.Alias, err) } if tablet.Type != topodatapb.TabletType_REPLICA { t.Fatalf("old master should be spare but is: %v", tablet.Type) } }
// ExecuteVtctlCommand is part of the pb.VtctlServer interface func (s *VtctlServer) ExecuteVtctlCommand(args *pb.ExecuteVtctlCommandRequest, stream pbs.Vtctl_ExecuteVtctlCommandServer) (err error) { defer servenv.HandlePanic("vtctl", &err) // create a logger, send the result back to the caller logstream := logutil.NewChannelLogger(10) logger := logutil.NewTeeLogger(logstream, logutil.NewConsoleLogger()) // send logs to the caller wg := sync.WaitGroup{} wg.Add(1) go func() { for e := range logstream { // Note we don't interrupt the loop here, as // we still need to flush and finish the // command, even if the channel to the client // has been broken. We'll just keep trying. stream.Send(&pb.ExecuteVtctlCommandResponse{ Event: logutil.LoggerEventToProto(&e), }) } wg.Done() }() // create the wrangler wr := wrangler.New(logger, s.ts, tmclient.NewTabletManagerClient(), time.Duration(args.LockTimeout)) // execute the command err = vtctl.RunCommand(stream.Context(), wr, args.Args) // close the log channel, and wait for them all to be sent close(logstream) wg.Wait() return err }
// TestInitMasterShardChecks makes sure the safety checks work func TestInitMasterShardChecks(t *testing.T) { ctx := context.Background() db := fakesqldb.Register() ts := zktestserver.New(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db) // InitShardMaster with an unknown tablet if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, &topodatapb.TabletAlias{ Cell: master.Tablet.Alias.Cell, Uid: master.Tablet.Alias.Uid + 1, }, false /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is not in the shard") { t.Errorf("InitShardMaster with unknown alias returned wrong error: %v", err) } // InitShardMaster with two masters in the shard, no force flag // (master2 needs to run InitTablet with -force, as it is the second // master in the same shard) master2 := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_MASTER, db, ForceInitTablet()) if err := wr.InitShardMaster(ctx, master2.Tablet.Keyspace, master2.Tablet.Shard, master2.Tablet.Alias, false /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is not the only master in the shard") { t.Errorf("InitShardMaster with two masters returned wrong error: %v", err) } // InitShardMaster where the new master fails (use force flag // as we have 2 masters). We force the failure by making the // SQL commands executed on the master unexpected by the test fixture master.StartActionLoop(t, wr) defer master.StopActionLoop(t) master2.StartActionLoop(t, wr) defer master2.StopActionLoop(t) if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, master.Tablet.Alias, true /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "unexpected extra query") { t.Errorf("InitShardMaster with new master failing in new master InitMaster returned wrong error: %v", err) } }
// ApplyTabletAction applies the provided action to the tablet. func (ar *ActionRepository) ApplyTabletAction(ctx context.Context, actionName string, tabletAlias *topodatapb.TabletAlias, r *http.Request) *ActionResult { result := &ActionResult{ Name: actionName, Parameters: topoproto.TabletAliasString(tabletAlias), } action, ok := ar.tabletActions[actionName] if !ok { result.error("Unknown tablet action") return result } // check the role if action.role != "" { if err := acl.CheckAccessHTTP(r, action.role); err != nil { result.error("Access denied") return result } } // run the action ctx, cancel := context.WithTimeout(ctx, *actionTimeout) wr := wrangler.New(logutil.NewConsoleLogger(), ar.ts, tmclient.NewTabletManagerClient()) output, err := action.method(ctx, wr, tabletAlias, r) cancel() if err != nil { result.error(err.Error()) return result } result.Output = output return result }
// ApplyShardAction applies the provided action to the shard. func (ar *ActionRepository) ApplyShardAction(ctx context.Context, actionName, keyspace, shard string, r *http.Request) *ActionResult { // if the shard name contains a '-', we assume it's the // name for a ranged based shard, so we lower case it. if strings.Contains(shard, "-") { shard = strings.ToLower(shard) } result := &ActionResult{Name: actionName, Parameters: keyspace + "/" + shard} action, ok := ar.shardActions[actionName] if !ok { result.error("Unknown shard action") return result } ctx, cancel := context.WithTimeout(ctx, *actionTimeout) wr := wrangler.New(logutil.NewConsoleLogger(), ar.ts, tmclient.NewTabletManagerClient()) output, err := action(ctx, wr, keyspace, shard, r) cancel() if err != nil { result.error(err.Error()) return result } result.Output = output return result }
// TestNewRestartableResultReader tests the correct error handling e.g. // if the connection to a tablet fails due to a canceled context. func TestNewRestartableResultReader(t *testing.T) { wantErr := errors.New("restartable_result_reader_test.go: context canceled") tabletconn.RegisterDialer("fake_dialer", func(tablet *topodatapb.Tablet, timeout time.Duration) (tabletconn.TabletConn, error) { return nil, wantErr }) protocol := flag.CommandLine.Lookup("tablet_protocol").Value.String() flag.Set("tablet_protocol", "fake_dialer") // Restore the previous flag value after the test. defer flag.Set("tablet_protocol", protocol) // Create dependencies e.g. a "singleTabletProvider" instance. ts := zktestserver.New(t, []string{"cell1"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) alias := &topodatapb.TabletAlias{ Cell: "cell1", Uid: 1, } tablet := &topodatapb.Tablet{ Keyspace: "ks1", Shard: "-80", Alias: alias, } ctx := context.Background() if err := ts.CreateTablet(ctx, tablet); err != nil { t.Fatalf("CreateTablet failed: %v", err) } tp := newSingleTabletProvider(ctx, ts, alias) _, err := NewRestartableResultReader(ctx, wr.Logger(), tp, nil /* td */, chunk{}, false) if err == nil || !strings.Contains(err.Error(), wantErr.Error()) { t.Fatalf("NewRestartableResultReader() should have failed because the context is canceled: %v", err) } }
// TestEmergencyReparentShardMasterElectNotBest tries to emergency reparent // to a host that is not the latest in replication position. func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) { ctx := context.Background() ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) // Create a master, a couple good slaves oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER) newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA) moreAdvancedSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA) // new master newMaster.FakeMysqlDaemon.Replicating = true newMaster.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{ GTIDSet: myproto.MariadbGTID{ Domain: 2, Server: 123, Sequence: 456, }, } newMaster.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", } newMaster.StartActionLoop(t, wr) defer newMaster.StopActionLoop(t) // old master, will be scrapped oldMaster.StartActionLoop(t, wr) defer oldMaster.StopActionLoop(t) // more advanced slave moreAdvancedSlave.FakeMysqlDaemon.Replicating = true moreAdvancedSlave.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{ GTIDSet: myproto.MariadbGTID{ Domain: 2, Server: 123, Sequence: 457, }, } moreAdvancedSlave.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", } moreAdvancedSlave.StartActionLoop(t, wr) defer moreAdvancedSlave.StopActionLoop(t) // run EmergencyReparentShard if err := wr.EmergencyReparentShard(ctx, newMaster.Tablet.Keyspace, newMaster.Tablet.Shard, newMaster.Tablet.Alias, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is more advanced than master elect tablet") { t.Fatalf("EmergencyReparentShard returned the wrong error: %v", err) } // check what was run if err := newMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Fatalf("newMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Fatalf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } if err := moreAdvancedSlave.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Fatalf("moreAdvancedSlave.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } }
// TestTabletExternallyReparentedWithDifferentMysqlPort makes sure // that if mysql is restarted on the master-elect tablet and has a different // port, we pick it up correctly. func TestTabletExternallyReparentedWithDifferentMysqlPort(t *testing.T) { tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */) ctx := context.Background() db := fakesqldb.Register() ts := zktopo.NewTestServer(t, []string{"cell1"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) // Create an old master, a new master, two good slaves, one bad slave oldMaster := NewFakeTablet(t, wr, "cell1", 0, pb.TabletType_MASTER, db) newMaster := NewFakeTablet(t, wr, "cell1", 1, pb.TabletType_REPLICA, db) goodSlave := NewFakeTablet(t, wr, "cell1", 2, pb.TabletType_REPLICA, db) // Now we're restarting mysql on a different port, 3301->3303 // but without updating the Tablet record in topology. // On the elected master, we will respond to // TabletActionSlaveWasPromoted, so we need a MysqlDaemon // that returns no master, and the new port (as returned by mysql) newMaster.FakeMysqlDaemon.MysqlPort = 3303 newMaster.StartActionLoop(t, wr) defer newMaster.StopActionLoop(t) // On the old master, we will only respond to // TabletActionSlaveWasRestarted and point to the new mysql port oldMaster.StartActionLoop(t, wr) defer oldMaster.StopActionLoop(t) // On the good slaves, we will respond to // TabletActionSlaveWasRestarted and point to the new mysql port goodSlave.StartActionLoop(t, wr) defer goodSlave.StopActionLoop(t) // This tests the good case, where everything works as planned t.Logf("TabletExternallyReparented(new master) expecting success") tmc := tmclient.NewTabletManagerClient() ti, err := ts.GetTablet(ctx, newMaster.Tablet.Alias) if err != nil { t.Fatalf("GetTablet failed: %v", err) } waitID := makeWaitID() if err := tmc.TabletExternallyReparented(context.Background(), ti, waitID); err != nil { t.Fatalf("TabletExternallyReparented(replica) failed: %v", err) } waitForExternalReparent(t, waitID) }
// TestSplitCloneV2_RetryDueToReparent tests that vtworker correctly failovers // during a reparent. // NOTE: worker.py is an end-to-end test which tests this as well. func TestSplitCloneV2_RetryDueToReparent(t *testing.T) { tc := &splitCloneTestCase{t: t} tc.setUp(false /* v3 */) defer tc.tearDown() // Provoke a reparent just before the copy finishes. // leftReplica will take over for the last, 30th, insert and the BLP checkpoint. tc.leftReplicaFakeDb.addExpectedQuery("INSERT INTO `vt_ks`.`table1` (`id`, `msg`, `keyspace_id`) VALUES (*", nil) expectBlpCheckpointCreationQueries(tc.leftReplicaFakeDb) // Do not let leftMaster succeed the 30th write. tc.leftMasterFakeDb.deleteAllEntriesAfterIndex(28) tc.leftMasterFakeDb.addExpectedQuery("INSERT INTO `vt_ks`.`table1` (`id`, `msg`, `keyspace_id`) VALUES (*", errReadOnly) tc.leftMasterFakeDb.enableInfinite() // When vtworker encounters the readonly error on leftMaster, do the reparent. tc.leftMasterFakeDb.getEntry(29).AfterFunc = func() { // Reparent from leftMaster to leftReplica. // NOTE: This step is actually not necessary due to our fakes which bypass // a lot of logic. Let's keep it for correctness though. ti, err := tc.ts.GetTablet(context.Background(), tc.leftReplica.Tablet.Alias) if err != nil { t.Fatalf("GetTablet failed: %v", err) } tmc := tmclient.NewTabletManagerClient() if err := tmc.TabletExternallyReparented(context.Background(), ti.Tablet, "wait id 1"); err != nil { t.Fatalf("TabletExternallyReparented(replica) failed: %v", err) } // Update targets in fake query service and send out a new health response. tc.leftMasterQs.UpdateType(topodatapb.TabletType_REPLICA) tc.leftMasterQs.AddDefaultHealthResponse() tc.leftReplicaQs.UpdateType(topodatapb.TabletType_MASTER) tc.leftReplicaQs.AddDefaultHealthResponse() // After this, vtworker will retry. The following situations can occur: // 1. HealthCheck picked up leftReplica as new MASTER // => retry will succeed. // 2. HealthCheck picked up no changes (leftMaster remains MASTER) // => retry will hit leftMaster which keeps responding with readonly err. // 3. HealthCheck picked up leftMaster as REPLICA, but leftReplica is still // a REPLICA. // => vtworker has no MASTER to go to and will keep retrying. } // Only wait 1 ms between retries, so that the test passes faster. *executeFetchRetryTime = 1 * time.Millisecond // Run the vtworker command. if err := runCommand(t, tc.wi, tc.wi.wr, tc.defaultWorkerArgs); err != nil { t.Fatal(err) } wantRetryCountMin := int64(1) if got := statsRetryCount.Get(); got < wantRetryCountMin { t.Errorf("Wrong statsRetryCounter: got %v, wanted >= %v", got, wantRetryCountMin) } }
// StartActionLoop will start the action loop for a fake tablet, // using ft.FakeMysqlDaemon as the backing mysqld. func (ft *FakeTablet) StartActionLoop(t *testing.T, wr *wrangler.Wrangler) { if ft.Agent != nil { t.Fatalf("Agent for %v is already running", ft.Tablet.Alias) } // Listen on a random port for gRPC var err error ft.Listener, err = net.Listen("tcp", ":0") if err != nil { t.Fatalf("Cannot listen: %v", err) } gRPCPort := int32(ft.Listener.Addr().(*net.TCPAddr).Port) // if needed, listen on a random port for HTTP vtPort := ft.Tablet.PortMap["vt"] if ft.StartHTTPServer { ft.HTTPListener, err = net.Listen("tcp", ":0") if err != nil { t.Fatalf("Cannot listen on http port: %v", err) } handler := http.NewServeMux() ft.HTTPServer = http.Server{ Handler: handler, } go ft.HTTPServer.Serve(ft.HTTPListener) vtPort = int32(ft.HTTPListener.Addr().(*net.TCPAddr).Port) } // create a test agent on that port, and re-read the record // (it has new ports and IP) ft.Agent = tabletmanager.NewTestActionAgent(context.Background(), wr.TopoServer(), ft.Tablet.Alias, vtPort, gRPCPort, ft.FakeMysqlDaemon) ft.Tablet = ft.Agent.Tablet() // create the gRPC server ft.RPCServer = grpc.NewServer() grpctmserver.RegisterForTest(ft.RPCServer, ft.Agent) go ft.RPCServer.Serve(ft.Listener) // and wait for it to serve, so we don't start using it before it's // ready. timeout := 5 * time.Second step := 10 * time.Millisecond c := tmclient.NewTabletManagerClient() for timeout >= 0 { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) err := c.Ping(ctx, topo.NewTabletInfo(ft.Agent.Tablet(), -1)) cancel() if err == nil { break } time.Sleep(step) timeout -= step } if timeout < 0 { panic("StartActionLoop failed.") } }
func TestTabletData(t *testing.T) { db := fakesqldb.Register() ts := zktestserver.New(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) if err := ts.CreateKeyspace(context.Background(), "ks", &topodatapb.Keyspace{ ShardingColumnName: "keyspace_id", ShardingColumnType: topodatapb.KeyspaceIdType_UINT64, }); err != nil { t.Fatalf("CreateKeyspace failed: %v", err) } tablet1 := testlib.NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db, testlib.TabletKeyspaceShard(t, "ks", "-80")) tablet1.StartActionLoop(t, wr) defer tablet1.StopActionLoop(t) shsq := newStreamHealthTabletServer(t) grpcqueryservice.Register(tablet1.RPCServer, shsq) thc := newTabletHealthCache(ts) stats := &querypb.RealtimeStats{ HealthError: "testHealthError", SecondsBehindMaster: 72, CpuUsage: 1.1, } // Keep broadcasting until the first result goes through. stop := make(chan struct{}) go func() { for { select { case <-stop: return default: shsq.BroadcastHealth(42, stats) } } }() // Start streaming and wait for the first result. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) result, err := thc.Get(ctx, tablet1.Tablet.Alias) cancel() close(stop) if err != nil { t.Fatalf("thc.Get failed: %v", err) } if got, want := result.RealtimeStats, stats; !proto.Equal(got, want) { t.Errorf("RealtimeStats = %#v, want %#v", got, want) } }
// TestTabletExternallyReparentedContinueOnUnexpectedMaster makes sure // that we ignore mysql's master if the flag is set func TestTabletExternallyReparentedContinueOnUnexpectedMaster(t *testing.T) { tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */) ctx := context.Background() ts := zktopo.NewTestServer(t, []string{"cell1"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) // Create an old master, a new master, two good slaves, one bad slave oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER) newMaster := NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_REPLICA) goodSlave := NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_REPLICA) // On the elected master, we will respond to // TabletActionSlaveWasPromoted, so we need a MysqlDaemon // that returns no master, and the new port (as returned by mysql) newMaster.StartActionLoop(t, wr) defer newMaster.StopActionLoop(t) // On the old master, we will only respond to // TabletActionSlaveWasRestarted and point to a bad host oldMaster.StartActionLoop(t, wr) defer oldMaster.StopActionLoop(t) // On the good slave, we will respond to // TabletActionSlaveWasRestarted and point to a bad host goodSlave.StartActionLoop(t, wr) defer goodSlave.StopActionLoop(t) // This tests the good case, where everything works as planned t.Logf("TabletExternallyReparented(new master) expecting success") tmc := tmclient.NewTabletManagerClient() ti, err := ts.GetTablet(ctx, newMaster.Tablet.Alias) if err != nil { t.Fatalf("GetTablet failed: %v", err) } if err := tmc.TabletExternallyReparented(context.Background(), ti, ""); err != nil { t.Fatalf("TabletExternallyReparented(replica) failed: %v", err) } }
func TestReparentTablet(t *testing.T) { ctx := context.Background() db := fakesqldb.Register() ts := zktestserver.New(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) // create shard and tablets if err := ts.CreateShard(ctx, "test_keyspace", "0"); err != nil { t.Fatalf("CreateShard failed: %v", err) } master := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_MASTER, db) slave := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db) // mark the master inside the shard si, err := ts.GetShard(ctx, "test_keyspace", "0") if err != nil { t.Fatalf("GetShard failed: %v", err) } si.MasterAlias = master.Tablet.Alias if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard failed: %v", err) } // master action loop (to initialize host and port) master.StartActionLoop(t, wr) defer master.StopActionLoop(t) // slave loop slave.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", master.Tablet.Hostname, master.Tablet.PortMap["mysql"]) slave.FakeMysqlDaemon.SetMasterCommandsResult = []string{"set master cmd 1"} slave.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "set master cmd 1", } slave.StartActionLoop(t, wr) defer slave.StopActionLoop(t) // run ReparentTablet if err := wr.ReparentTablet(ctx, slave.Tablet.Alias); err != nil { t.Fatalf("ReparentTablet failed: %v", err) } // check what was run if err := slave.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Fatalf("slave.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } checkSemiSyncEnabled(t, false, true, slave) }
func main() { defer exit.RecoverAll() defer logutil.Flush() flag.Parse() args := flag.Args() if len(args) == 0 { flag.Usage() exit.Return(1) } action := args[0] startMsg := fmt.Sprintf("USER=%v SUDO_USER=%v %v", os.Getenv("USER"), os.Getenv("SUDO_USER"), strings.Join(os.Args, " ")) if syslogger, err := syslog.New(syslog.LOG_INFO, "vtctl "); err == nil { syslogger.Info(startMsg) } else { log.Warningf("cannot connect to syslog: %v", err) } topoServer := topo.GetServer() defer topo.CloseServers() ctx, cancel := context.WithTimeout(context.Background(), *waitTime) wr := wrangler.New(logutil.NewConsoleLogger(), topoServer, tmclient.NewTabletManagerClient(), *lockWaitTimeout) installSignalHandlers(cancel) for _, f := range initFuncs { f() } err := vtctl.RunCommand(ctx, wr, args) cancel() switch err { case vtctl.ErrUnknownCommand: flag.Usage() exit.Return(1) case nil: // keep going default: log.Errorf("action failed: %v %v", action, err) exit.Return(255) } }
func TestVersion(t *testing.T) { // We need to run this test with the /debug/vars version of the // plugin. wrangler.ResetDebugVarsGetVersion() // Initialize our environment db := fakesqldb.Register() ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) vp := NewVtctlPipe(t, ts) defer vp.Close() // couple tablets is enough sourceMaster := NewFakeTablet(t, wr, "cell1", 10, pb.TabletType_MASTER, db, TabletKeyspaceShard(t, "source", "0"), StartHTTPServer()) sourceReplica := NewFakeTablet(t, wr, "cell1", 11, pb.TabletType_REPLICA, db, TabletKeyspaceShard(t, "source", "0"), StartHTTPServer()) // sourceMaster loop sourceMasterGitRev := "fake git rev" sourceMaster.StartActionLoop(t, wr) sourceMaster.HTTPServer.Handler.(*http.ServeMux).HandleFunc("/debug/vars", expvarHandler(&sourceMasterGitRev)) defer sourceMaster.StopActionLoop(t) // sourceReplica loop sourceReplicaGitRev := "fake git rev" sourceReplica.StartActionLoop(t, wr) sourceReplica.HTTPServer.Handler.(*http.ServeMux).HandleFunc("/debug/vars", expvarHandler(&sourceReplicaGitRev)) defer sourceReplica.StopActionLoop(t) // test when versions are the same if err := vp.Run([]string{"ValidateVersionKeyspace", sourceMaster.Tablet.Keyspace}); err != nil { t.Fatalf("ValidateVersionKeyspace(same) failed: %v", err) } // test when versions are different sourceReplicaGitRev = "different fake git rev" if err := vp.Run([]string{"ValidateVersionKeyspace", sourceMaster.Tablet.Keyspace}); err == nil || !strings.Contains(err.Error(), "is different than slave") { t.Fatalf("ValidateVersionKeyspace(different) returned an unexpected error: %v", err) } }
// ExecuteVtctlCommand is part of the vtctldatapb.VtctlServer interface func (s *VtctlServer) ExecuteVtctlCommand(args *vtctldatapb.ExecuteVtctlCommandRequest, stream vtctlservicepb.Vtctl_ExecuteVtctlCommandServer) (err error) { defer servenv.HandlePanic("vtctl", &err) // create a logger, send the result back to the caller logstream := logutil.NewCallbackLogger(func(e *logutilpb.Event) { // If the client disconnects, we will just fail // to send the log events, but won't interrupt // the command. stream.Send(&vtctldatapb.ExecuteVtctlCommandResponse{ Event: e, }) }) logger := logutil.NewTeeLogger(logstream, logutil.NewConsoleLogger()) // create the wrangler wr := wrangler.New(logger, s.ts, tmclient.NewTabletManagerClient()) // execute the command return vtctl.RunCommand(stream.Context(), wr, args.Args) }
// ApplyKeyspaceAction applies the provided action to the keyspace. func (ar *ActionRepository) ApplyKeyspaceAction(ctx context.Context, actionName, keyspace string, r *http.Request) *ActionResult { result := &ActionResult{Name: actionName, Parameters: keyspace} action, ok := ar.keyspaceActions[actionName] if !ok { result.error("Unknown keyspace action") return result } ctx, cancel := context.WithTimeout(ctx, *actionTimeout) wr := wrangler.New(logutil.NewConsoleLogger(), ar.ts, tmclient.NewTabletManagerClient()) output, err := action(ctx, wr, keyspace, r) cancel() if err != nil { result.error(err.Error()) return result } result.Output = output return result }
func newReplica(lagUpdateInterval, degrationInterval, degrationDuration time.Duration) *replica { t := &testing.T{} ts := zktestserver.New(t, []string{"cell1"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) db := fakesqldb.Register() fakeTablet := testlib.NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_REPLICA, db, testlib.TabletKeyspaceShard(t, "ks", "-80")) fakeTablet.StartActionLoop(t, wr) target := querypb.Target{ Keyspace: "ks", Shard: "-80", TabletType: topodatapb.TabletType_REPLICA, } qs := fakes.NewStreamHealthQueryService(target) grpcqueryservice.Register(fakeTablet.RPCServer, qs) throttler, err := throttler.NewThrottler("replica", "TPS", 1, *rate, throttler.ReplicationLagModuleDisabled) if err != nil { log.Fatal(err) } var nextDegration time.Time if degrationInterval != time.Duration(0) { nextDegration = time.Now().Add(degrationInterval) } r := &replica{ fakeTablet: fakeTablet, qs: qs, throttler: throttler, replicationStream: make(chan time.Time, 1*1024*1024), lagUpdateInterval: lagUpdateInterval, degrationInterval: degrationInterval, degrationDuration: degrationDuration, nextDegration: nextDegration, stopChan: make(chan struct{}), } r.wg.Add(1) go r.processReplicationStream() return r }
func initSchema() { // Start schema manager service if needed. if *schemaChangeDir != "" { interval := 60 if *schemaChangeCheckInterval > 0 { interval = *schemaChangeCheckInterval } timer := timer.NewTimer(time.Duration(interval) * time.Second) controllerFactory, err := schemamanager.GetControllerFactory(*schemaChangeController) if err != nil { log.Fatalf("unable to get a controller factory, error: %v", err) } timer.Start(func() { controller, err := controllerFactory(map[string]string{ schemamanager.SchemaChangeDirName: *schemaChangeDir, schemamanager.SchemaChangeUser: *schemaChangeUser, }) if err != nil { log.Errorf("failed to get controller, error: %v", err) return } ctx := context.Background() err = schemamanager.Run( ctx, controller, schemamanager.NewTabletExecutor( tmclient.NewTabletManagerClient(), ts), ) if err != nil { log.Errorf("Schema change failed, error: %v", err) } }) servenv.OnClose(func() { timer.Stop() }) } }
// ExecuteVtctlCommand is the server side method that will execute the query, // and stream the results. func (s *VtctlServer) ExecuteVtctlCommand(ctx context.Context, query *gorpcproto.ExecuteVtctlCommandArgs, sendReply func(interface{}) error) (err error) { defer vtctl.HandlePanic(&err) // create a logger, send the result back to the caller logstream := logutil.NewChannelLogger(10) logger := logutil.NewTeeLogger(logstream, logutil.NewConsoleLogger()) // send logs to the caller wg := sync.WaitGroup{} wg.Add(1) go func() { for e := range logstream { // Note we don't interrupt the loop here, as // we still need to flush and finish the // command, even if the channel to the client // has been broken. We'll just keep trying. sendReply(&e) } wg.Done() }() // create the wrangler wr := wrangler.New(logger, s.ts, tmclient.NewTabletManagerClient(), query.LockTimeout) // FIXME(alainjobart) use a single context, copy the source info from it ctx, cancel := context.WithTimeout(context.TODO(), query.ActionTimeout) // execute the command err = vtctl.RunCommand(ctx, wr, query.Args) cancel() // close the log channel, and wait for them all to be sent close(logstream) wg.Wait() return err }
func testSplitClone(t *testing.T, strategy string) { ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) sourceMaster := testlib.NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-80")) sourceRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80")) sourceRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80")) leftMaster := testlib.NewFakeTablet(t, wr, "cell1", 10, topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-40")) leftRdonly := testlib.NewFakeTablet(t, wr, "cell1", 11, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-40")) rightMaster := testlib.NewFakeTablet(t, wr, "cell1", 20, topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "40-80")) rightRdonly := testlib.NewFakeTablet(t, wr, "cell1", 21, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "40-80")) for _, ft := range []*testlib.FakeTablet{sourceMaster, sourceRdonly1, sourceRdonly2, leftMaster, leftRdonly, rightMaster, rightRdonly} { ft.StartActionLoop(t, wr) defer ft.StopActionLoop(t) } // add the topo and schema data we'll need ctx := context.Background() if err := topo.CreateShard(ctx, ts, "ks", "80-"); err != nil { t.Fatalf("CreateShard(\"-80\") failed: %v", err) } if err := wr.SetKeyspaceShardingInfo(ctx, "ks", "keyspace_id", key.KIT_UINT64, 4, false); err != nil { t.Fatalf("SetKeyspaceShardingInfo failed: %v", err) } if err := wr.RebuildKeyspaceGraph(ctx, "ks", nil, true); err != nil { t.Fatalf("RebuildKeyspaceGraph failed: %v", err) } gwrk, err := NewSplitCloneWorker(wr, "cell1", "ks", "-80", nil, strategy, 10 /*sourceReaderCount*/, 4 /*destinationPackCount*/, 1 /*minTableSizeForSplit*/, 10 /*destinationWriterCount*/) if err != nil { t.Errorf("Worker creation failed: %v", err) } wrk := gwrk.(*SplitCloneWorker) for _, sourceRdonly := range []*testlib.FakeTablet{sourceRdonly1, sourceRdonly2} { sourceRdonly.FakeMysqlDaemon.Schema = &myproto.SchemaDefinition{ DatabaseSchema: "", TableDefinitions: []*myproto.TableDefinition{ &myproto.TableDefinition{ Name: "table1", Columns: []string{"id", "msg", "keyspace_id"}, PrimaryKeyColumns: []string{"id"}, Type: myproto.TableBaseTable, // This informs how many rows we can pack into a single insert DataLength: 2048, }, }, } sourceRdonly.FakeMysqlDaemon.DbAppConnectionFactory = SourceRdonlyFactory(t) sourceRdonly.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{ GTIDSet: myproto.MariadbGTID{Domain: 12, Server: 34, Sequence: 5678}, } sourceRdonly.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "START SLAVE", } sourceRdonly.RPCServer.Register(gorpcqueryservice.New(&testQueryService{t: t})) } // We read 100 source rows. sourceReaderCount is set to 10, so // we'll have 100/10=10 rows per table chunk. // destinationPackCount is set to 4, so we take 4 source rows // at once. So we'll process 4 + 4 + 2 rows to get to 10. // That means 3 insert statements on each target (each // containing half of the rows, i.e. 2 + 2 + 1 rows). So 3 * 10 // = 30 insert statements on each destination. leftMaster.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30) leftRdonly.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30) rightMaster.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30) rightRdonly.FakeMysqlDaemon.DbAppConnectionFactory = DestinationsFactory(t, 30) // Only wait 1 ms between retries, so that the test passes faster *executeFetchRetryTime = (1 * time.Millisecond) err = wrk.Run(ctx) status := wrk.StatusAsText() t.Logf("Got status: %v", status) if err != nil || wrk.State != WorkerStateDone { t.Errorf("Worker run failed") } if statsDestinationAttemptedResolves.String() != "3" { t.Errorf("Wrong statsDestinationAttemptedResolves: wanted %v, got %v", "3", statsDestinationAttemptedResolves.String()) } if statsDestinationActualResolves.String() != "1" { t.Errorf("Wrong statsDestinationActualResolves: wanted %v, got %v", "1", statsDestinationActualResolves.String()) } if statsRetryCounters.String() != "{\"ReadOnly\": 2}" { t.Errorf("Wrong statsRetryCounters: wanted %v, got %v", "{\"ReadOnly\": 2}", statsRetryCounters.String()) } }
func TestPermissions(t *testing.T) { // Initialize our environment ctx := context.Background() db := fakesqldb.Register() ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) vp := NewVtctlPipe(t, ts) defer vp.Close() master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db) replica := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db) // mark the master inside the shard si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard) if err != nil { t.Fatalf("GetShard failed: %v", err) } si.MasterAlias = master.Tablet.Alias if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard failed: %v", err) } // master will be asked for permissions master.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{ "SELECT * FROM mysql.user": { Fields: []*querypb.Field{ { Name: "Host", Type: sqltypes.Char, }, { Name: "User", Type: sqltypes.Char, }, { Name: "Password", Type: sqltypes.Char, }, { Name: "Select_priv", Type: sqltypes.Char, }, { Name: "Insert_priv", Type: sqltypes.Char, }, { Name: "Update_priv", Type: sqltypes.Char, }, { Name: "Delete_priv", Type: sqltypes.Char, }, { Name: "Create_priv", Type: sqltypes.Char, }, { Name: "Drop_priv", Type: sqltypes.Char, }, { Name: "Reload_priv", Type: sqltypes.Char, }, { Name: "Shutdown_priv", Type: sqltypes.Char, }, { Name: "Process_priv", Type: sqltypes.Char, }, { Name: "File_priv", Type: sqltypes.Char, }, { Name: "Grant_priv", Type: sqltypes.Char, }, { Name: "References_priv", Type: sqltypes.Char, }, { Name: "Index_priv", Type: sqltypes.Char, }, { Name: "Alter_priv", Type: sqltypes.Char, }, { Name: "Show_db_priv", Type: sqltypes.Char, }, { Name: "Super_priv", Type: sqltypes.Char, }, { Name: "Create_tmp_table_priv", Type: sqltypes.Char, }, { Name: "Lock_tables_priv", Type: sqltypes.Char, }, { Name: "Execute_priv", Type: sqltypes.Char, }, { Name: "Repl_slave_priv", Type: sqltypes.Char, }, { Name: "Repl_client_priv", Type: sqltypes.Char, }, { Name: "Create_view_priv", Type: sqltypes.Char, }, { Name: "Show_view_priv", Type: sqltypes.Char, }, { Name: "Create_routine_priv", Type: sqltypes.Char, }, { Name: "Alter_routine_priv", Type: sqltypes.Char, }, { Name: "Create_user_priv", Type: sqltypes.Char, }, { Name: "Event_priv", Type: sqltypes.Char, }, { Name: "Trigger_priv", Type: sqltypes.Char, }, { Name: "Create_tablespace_priv", Type: sqltypes.Char, }, { Name: "ssl_type", Type: sqltypes.Char, }, { Name: "ssl_cipher", Type: 252, }, { Name: "x509_issuer", Type: 252, }, { Name: "x509_subject", Type: 252, }, { Name: "max_questions", Type: 3, }, { Name: "max_updates", Type: 3, }, { Name: "max_connections", Type: 3, }, { Name: "max_user_connections", Type: 3, }, { Name: "plugin", Type: sqltypes.Char, }, { Name: "authentication_string", Type: 252, }, { Name: "password_expired", Type: sqltypes.Char, }, { Name: "is_role", Type: sqltypes.Char, }}, RowsAffected: 0x6, InsertID: 0x0, Rows: [][]sqltypes.Value{ { sqltypes.MakeString([]byte("test_host1")), sqltypes.MakeString([]byte("test_user1")), sqltypes.MakeString([]byte("test_password1")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N"))}, { sqltypes.MakeString([]byte("test_host2")), sqltypes.MakeString([]byte("test_user2")), sqltypes.MakeString([]byte("test_password2")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N"))}, { sqltypes.MakeString([]byte("test_host3")), sqltypes.MakeString([]byte("test_user3")), sqltypes.MakeString([]byte("test_password3")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N"))}, { sqltypes.MakeString([]byte("test_host4")), sqltypes.MakeString([]byte("test_user4")), sqltypes.MakeString([]byte("test_password4")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("0")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("N")), }, }, }, "SELECT * FROM mysql.db": { Fields: []*querypb.Field{ { Name: "Host", Type: sqltypes.Char, }, { Name: "Db", Type: sqltypes.Char, }, { Name: "User", Type: sqltypes.Char, }, { Name: "Select_priv", Type: sqltypes.Char, }, { Name: "Insert_priv", Type: sqltypes.Char, }, { Name: "Update_priv", Type: sqltypes.Char, }, { Name: "Delete_priv", Type: sqltypes.Char, }, { Name: "Create_priv", Type: sqltypes.Char, }, { Name: "Drop_priv", Type: sqltypes.Char, }, { Name: "Grant_priv", Type: sqltypes.Char, }, { Name: "References_priv", Type: sqltypes.Char, }, { Name: "Index_priv", Type: sqltypes.Char, }, { Name: "Alter_priv", Type: sqltypes.Char, }, { Name: "Create_tmp_table_priv", Type: sqltypes.Char, }, { Name: "Lock_tables_priv", Type: sqltypes.Char, }, { Name: "Create_view_priv", Type: sqltypes.Char, }, { Name: "Show_view_priv", Type: sqltypes.Char, }, { Name: "Create_routine_priv", Type: sqltypes.Char, }, { Name: "Alter_routine_priv", Type: sqltypes.Char, }, { Name: "Execute_priv", Type: sqltypes.Char, }, { Name: "Event_priv", Type: sqltypes.Char, }, { Name: "Trigger_priv", Type: sqltypes.Char, }, }, RowsAffected: 0, InsertID: 0, Rows: [][]sqltypes.Value{ { sqltypes.MakeString([]byte("test_host")), sqltypes.MakeString([]byte("test_db")), sqltypes.MakeString([]byte("test_user")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("N")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), sqltypes.MakeString([]byte("Y")), }, }, }, } master.StartActionLoop(t, wr) defer master.StopActionLoop(t) // Make a two-level-deep copy, so we can make them diverge later. user := *master.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"] user.Fields = append([]*querypb.Field{}, user.Fields...) // replica will be asked for permissions replica.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{ "SELECT * FROM mysql.user": &user, "SELECT * FROM mysql.db": master.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.db"], } replica.StartActionLoop(t, wr) defer replica.StopActionLoop(t) // Overwrite with the correct value to make sure it passes. replica.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"].Fields[0] = &querypb.Field{ Name: "Host", Type: sqltypes.Char, } // run ValidatePermissionsKeyspace, this should work if err := vp.Run([]string{"ValidatePermissionsKeyspace", master.Tablet.Keyspace}); err != nil { t.Fatalf("ValidatePermissionsKeyspace failed: %v", err) } // modify one field, this should fail replica.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"].Fields[0] = &querypb.Field{ Name: "Wrong", Type: sqltypes.Char, } // run ValidatePermissionsKeyspace again, this should now fail if err := vp.Run([]string{"ValidatePermissionsKeyspace", master.Tablet.Keyspace}); err == nil || !strings.Contains(err.Error(), "has an extra user") { t.Fatalf("ValidatePermissionsKeyspace has unexpected err: %v", err) } }
// NewFakeTabletManagerClient should be used to create a new FakeTabletManagerClient. // There is intentionally no init in this file with a call to RegisterTabletManagerClientFactory. // There shouldn't be any legitimate use-case where we would want to start a vitess cluster // with a FakeTMC, and we don't want to do it by accident. func NewFakeTabletManagerClient() tmclient.TabletManagerClient { return &FakeTabletManagerClient{ tmc: tmclient.NewTabletManagerClient(), } }
func TestTabletExternallyReparented(t *testing.T) { tabletmanager.SetReparentFlags(time.Minute /* finalizeTimeout */) ctx := context.Background() db := fakesqldb.Register() ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) vp := NewVtctlPipe(t, ts) defer vp.Close() // Create an old master, a new master, two good slaves, one bad slave oldMaster := NewFakeTablet(t, wr, "cell1", 0, pb.TabletType_MASTER, db) newMaster := NewFakeTablet(t, wr, "cell1", 1, pb.TabletType_REPLICA, db) goodSlave1 := NewFakeTablet(t, wr, "cell1", 2, pb.TabletType_REPLICA, db) goodSlave2 := NewFakeTablet(t, wr, "cell2", 3, pb.TabletType_REPLICA, db) badSlave := NewFakeTablet(t, wr, "cell1", 4, pb.TabletType_REPLICA, db) // Add a new Cell to the Shard, that doesn't map to any read topo cell, // to simulate a data center being unreachable. si, err := ts.GetShard(ctx, "test_keyspace", "0") if err != nil { t.Fatalf("GetShard failed: %v", err) } si.Cells = append(si.Cells, "cell666") if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard failed: %v", err) } // Slightly unrelated test: make sure we can find the tablets // even with a datacenter being down. tabletMap, err := ts.GetTabletMapForShardByCell(ctx, "test_keyspace", "0", []string{"cell1"}) if err != nil { t.Fatalf("GetTabletMapForShardByCell should have worked but got: %v", err) } master, err := topotools.FindTabletByIPAddrAndPort(tabletMap, oldMaster.Tablet.Ip, "vt", oldMaster.Tablet.PortMap["vt"]) if err != nil || !topoproto.TabletAliasEqual(&master, oldMaster.Tablet.Alias) { t.Fatalf("FindTabletByIPAddrAndPort(master) failed: %v %v", err, master) } slave1, err := topotools.FindTabletByIPAddrAndPort(tabletMap, goodSlave1.Tablet.Ip, "vt", goodSlave1.Tablet.PortMap["vt"]) if err != nil || !topoproto.TabletAliasEqual(&slave1, goodSlave1.Tablet.Alias) { t.Fatalf("FindTabletByIPAddrAndPort(slave1) failed: %v %v", err, master) } slave2, err := topotools.FindTabletByIPAddrAndPort(tabletMap, goodSlave2.Tablet.Ip, "vt", goodSlave2.Tablet.PortMap["vt"]) if err != topo.ErrNoNode { t.Fatalf("FindTabletByIPAddrAndPort(slave2) worked: %v %v", err, slave2) } // Make sure the master is not exported in other cells tabletMap, err = ts.GetTabletMapForShardByCell(ctx, "test_keyspace", "0", []string{"cell2"}) master, err = topotools.FindTabletByIPAddrAndPort(tabletMap, oldMaster.Tablet.Ip, "vt", oldMaster.Tablet.PortMap["vt"]) if err != topo.ErrNoNode { t.Fatalf("FindTabletByIPAddrAndPort(master) worked in cell2: %v %v", err, master) } tabletMap, err = ts.GetTabletMapForShard(ctx, "test_keyspace", "0") if err != topo.ErrPartialResult { t.Fatalf("GetTabletMapForShard should have returned ErrPartialResult but got: %v", err) } master, err = topotools.FindTabletByIPAddrAndPort(tabletMap, oldMaster.Tablet.Ip, "vt", oldMaster.Tablet.PortMap["vt"]) if err != nil || !topoproto.TabletAliasEqual(&master, oldMaster.Tablet.Alias) { t.Fatalf("FindTabletByIPAddrAndPort(master) failed: %v %v", err, master) } // On the elected master, we will respond to // TabletActionSlaveWasPromoted newMaster.StartActionLoop(t, wr) defer newMaster.StopActionLoop(t) // On the old master, we will only respond to // TabletActionSlaveWasRestarted. oldMaster.StartActionLoop(t, wr) defer oldMaster.StopActionLoop(t) // On the good slaves, we will respond to // TabletActionSlaveWasRestarted. goodSlave1.StartActionLoop(t, wr) defer goodSlave1.StopActionLoop(t) goodSlave2.StartActionLoop(t, wr) defer goodSlave2.StopActionLoop(t) // On the bad slave, we will respond to // TabletActionSlaveWasRestarted with bad data. badSlave.StartActionLoop(t, wr) defer badSlave.StopActionLoop(t) // First test: reparent to the same master, make sure it works // as expected. tmc := tmclient.NewTabletManagerClient() ti, err := ts.GetTablet(ctx, oldMaster.Tablet.Alias) if err != nil { t.Fatalf("GetTablet failed: %v", err) } if err := vp.Run([]string{"TabletExternallyReparented", topoproto.TabletAliasString(oldMaster.Tablet.Alias)}); err != nil { t.Fatalf("TabletExternallyReparented(same master) should have worked: %v", err) } // Second test: reparent to a replica, and pretend the old // master is still good to go. // This tests a bad case; the new designated master is a slave, // but we should do what we're told anyway ti, err = ts.GetTablet(ctx, goodSlave1.Tablet.Alias) if err != nil { t.Fatalf("GetTablet failed: %v", err) } waitID := makeWaitID() if err := tmc.TabletExternallyReparented(context.Background(), ti, waitID); err != nil { t.Fatalf("TabletExternallyReparented(slave) error: %v", err) } waitForExternalReparent(t, waitID) // This tests the good case, where everything works as planned t.Logf("TabletExternallyReparented(new master) expecting success") ti, err = ts.GetTablet(ctx, newMaster.Tablet.Alias) if err != nil { t.Fatalf("GetTablet failed: %v", err) } waitID = makeWaitID() if err := tmc.TabletExternallyReparented(context.Background(), ti, waitID); err != nil { t.Fatalf("TabletExternallyReparented(replica) failed: %v", err) } waitForExternalReparent(t, waitID) // Now double-check the serving graph is good. // Should only have one good replica left. addrs, _, err := ts.GetEndPoints(ctx, "cell1", "test_keyspace", "0", pb.TabletType_REPLICA) if err != nil { t.Fatalf("GetEndPoints failed at the end: %v", err) } if len(addrs.Entries) != 1 { t.Fatalf("GetEndPoints has too many entries: %v", addrs) } }
func TestMigrateServedTypes(t *testing.T) { db := fakesqldb.Register() ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) vp := NewVtctlPipe(t, ts) defer vp.Close() // create keyspace if err := ts.CreateKeyspace(context.Background(), "ks", &pb.Keyspace{ ShardingColumnName: "keyspace_id", ShardingColumnType: pb.KeyspaceIdType_UINT64, }); err != nil { t.Fatalf("CreateKeyspace failed: %v", err) } // create the source shard sourceMaster := NewFakeTablet(t, wr, "cell1", 10, pb.TabletType_MASTER, db, TabletKeyspaceShard(t, "ks", "0")) sourceReplica := NewFakeTablet(t, wr, "cell1", 11, pb.TabletType_REPLICA, db, TabletKeyspaceShard(t, "ks", "0")) sourceRdonly := NewFakeTablet(t, wr, "cell1", 12, pb.TabletType_RDONLY, db, TabletKeyspaceShard(t, "ks", "0")) // create the first destination shard dest1Master := NewFakeTablet(t, wr, "cell1", 20, pb.TabletType_MASTER, db, TabletKeyspaceShard(t, "ks", "-80")) dest1Replica := NewFakeTablet(t, wr, "cell1", 21, pb.TabletType_REPLICA, db, TabletKeyspaceShard(t, "ks", "-80")) dest1Rdonly := NewFakeTablet(t, wr, "cell1", 22, pb.TabletType_RDONLY, db, TabletKeyspaceShard(t, "ks", "-80")) // create the second destination shard dest2Master := NewFakeTablet(t, wr, "cell1", 30, pb.TabletType_MASTER, db, TabletKeyspaceShard(t, "ks", "80-")) dest2Replica := NewFakeTablet(t, wr, "cell1", 31, pb.TabletType_REPLICA, db, TabletKeyspaceShard(t, "ks", "80-")) dest2Rdonly := NewFakeTablet(t, wr, "cell1", 32, pb.TabletType_RDONLY, db, TabletKeyspaceShard(t, "ks", "80-")) // double check the shards have the right served types checkShardServedTypes(t, ts, "0", 3) checkShardServedTypes(t, ts, "-80", 0) checkShardServedTypes(t, ts, "80-", 0) // sourceRdonly will see the refresh sourceRdonly.StartActionLoop(t, wr) defer sourceRdonly.StopActionLoop(t) // sourceReplica will see the refresh sourceReplica.StartActionLoop(t, wr) defer sourceReplica.StopActionLoop(t) // sourceMaster will see the refresh, and has to respond to it // also will be asked about its replication position. sourceMaster.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{ GTIDSet: myproto.MariadbGTID{ Domain: 5, Server: 456, Sequence: 892, }, } sourceMaster.StartActionLoop(t, wr) defer sourceMaster.StopActionLoop(t) // dest1Rdonly will see the refresh dest1Rdonly.StartActionLoop(t, wr) defer dest1Rdonly.StopActionLoop(t) // dest1Replica will see the refresh dest1Replica.StartActionLoop(t, wr) defer dest1Replica.StopActionLoop(t) // dest1Master will see the refresh, and has to respond to it. // It will also need to respond to WaitBlpPosition, saying it's already caught up. dest1Master.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*mproto.QueryResult{ "SELECT pos, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=0": &mproto.QueryResult{ Rows: [][]sqltypes.Value{ []sqltypes.Value{ sqltypes.MakeString([]byte(myproto.EncodeReplicationPosition(sourceMaster.FakeMysqlDaemon.CurrentMasterPosition))), sqltypes.MakeString([]byte("")), }, }, }, } dest1Master.StartActionLoop(t, wr) defer dest1Master.StopActionLoop(t) // dest2Rdonly will see the refresh dest2Rdonly.StartActionLoop(t, wr) defer dest2Rdonly.StopActionLoop(t) // dest2Replica will see the refresh dest2Replica.StartActionLoop(t, wr) defer dest2Replica.StopActionLoop(t) // dest2Master will see the refresh, and has to respond to it. // It will also need to respond to WaitBlpPosition, saying it's already caught up. dest2Master.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*mproto.QueryResult{ "SELECT pos, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=0": &mproto.QueryResult{ Rows: [][]sqltypes.Value{ []sqltypes.Value{ sqltypes.MakeString([]byte(myproto.EncodeReplicationPosition(sourceMaster.FakeMysqlDaemon.CurrentMasterPosition))), sqltypes.MakeString([]byte("")), }, }, }, } dest2Master.StartActionLoop(t, wr) defer dest2Master.StopActionLoop(t) // simulate the clone, by fixing the dest shard record if err := vp.Run([]string{"SourceShardAdd", "--key_range=-", "ks/-80", "0", "ks/0"}); err != nil { t.Fatalf("SourceShardAdd failed: %v", err) } if err := vp.Run([]string{"SourceShardAdd", "--key_range=-", "ks/80-", "0", "ks/0"}); err != nil { t.Fatalf("SourceShardAdd failed: %v", err) } // migrate rdonly over if err := vp.Run([]string{"MigrateServedTypes", "ks/0", "rdonly"}); err != nil { t.Fatalf("MigrateServedType(rdonly) failed: %v", err) } checkShardServedTypes(t, ts, "0", 2) checkShardServedTypes(t, ts, "-80", 1) checkShardServedTypes(t, ts, "80-", 1) // migrate replica over if err := vp.Run([]string{"MigrateServedTypes", "ks/0", "replica"}); err != nil { t.Fatalf("MigrateServedType(replica) failed: %v", err) } checkShardServedTypes(t, ts, "0", 1) checkShardServedTypes(t, ts, "-80", 2) checkShardServedTypes(t, ts, "80-", 2) // migrate master over if err := vp.Run([]string{"MigrateServedTypes", "ks/0", "master"}); err != nil { t.Fatalf("MigrateServedType(master) failed: %v", err) } checkShardServedTypes(t, ts, "0", 0) checkShardServedTypes(t, ts, "-80", 3) checkShardServedTypes(t, ts, "80-", 3) }
func TestTabletData(t *testing.T) { ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) tablet1 := testlib.NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-80")) tablet1.StartActionLoop(t, wr) defer tablet1.StopActionLoop(t) thc := newTabletHealthCache(ts, tmclient.NewTabletManagerClient()) // get the first result, it's not containing any data but the alias result, err := thc.get(tablet1.Tablet.Alias) if err != nil { t.Fatalf("thc.get failed: %v", err) } var unpacked TabletHealth if err := json.Unmarshal(result, &unpacked); err != nil { t.Fatalf("bad json: %v", err) } if unpacked.HealthStreamReply.Tablet.Alias != tablet1.Tablet.Alias { t.Fatalf("wrong alias: %v", &unpacked) } if unpacked.Version != 1 { t.Errorf("wrong version, got %v was expecting 1", unpacked.Version) } // wait for the streaming RPC to be established timeout := 5 * time.Second for { if tablet1.Agent.HealthStreamMapSize() > 0 { break } timeout -= 10 * time.Millisecond if timeout < 0 { t.Fatalf("timeout waiting for streaming RPC to be established") } time.Sleep(10 * time.Millisecond) } // feed some data from the tablet, with just a data marker hsr := &actionnode.HealthStreamReply{ BinlogPlayerMapSize: 42, } tablet1.Agent.BroadcastHealthStreamReply(hsr) // and wait for the cache to pick it up timeout = 5 * time.Second for { result, err = thc.get(tablet1.Tablet.Alias) if err != nil { t.Fatalf("thc.get failed: %v", err) } if err := json.Unmarshal(result, &unpacked); err != nil { t.Fatalf("bad json: %v", err) } if unpacked.HealthStreamReply.BinlogPlayerMapSize == 42 { if unpacked.Version != 2 { t.Errorf("wrong version, got %v was expecting 2", unpacked.Version) } break } timeout -= 10 * time.Millisecond if timeout < 0 { t.Fatalf("timeout waiting for streaming RPC to be established") } time.Sleep(10 * time.Millisecond) } }
// TestInitMasterShardOneSlaveFails makes sure that if one slave fails to // proceed, the action completes anyway func TestInitMasterShardOneSlaveFails(t *testing.T) { ctx := context.Background() db := fakesqldb.Register() ts := zktestserver.New(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) // Create a master, a couple slaves master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db) goodSlave := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db) badSlave := NewFakeTablet(t, wr, "cell2", 2, topodatapb.TabletType_REPLICA, db) // Master: set a plausible ReplicationPosition to return, // and expect to add entry in _vt.reparent_journal master.FakeMysqlDaemon.CurrentMasterPosition = replication.Position{ GTIDSet: replication.MariadbGTID{ Domain: 5, Server: 456, Sequence: 890, }, } master.FakeMysqlDaemon.ReadOnly = true master.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "CREATE DATABASE IF NOT EXISTS _vt", "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", "CREATE DATABASE IF NOT EXISTS _vt", "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", } master.StartActionLoop(t, wr) defer master.StopActionLoop(t) // goodSlave: expect to be re-parented goodSlave.FakeMysqlDaemon.ReadOnly = true goodSlave.FakeMysqlDaemon.SetSlavePositionCommandsPos = master.FakeMysqlDaemon.CurrentMasterPosition goodSlave.FakeMysqlDaemon.SetSlavePositionCommandsResult = []string{"cmd1"} goodSlave.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", master.Tablet.Hostname, master.Tablet.PortMap["mysql"]) goodSlave.FakeMysqlDaemon.SetMasterCommandsResult = []string{"set master cmd 1"} goodSlave.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "cmd1", "set master cmd 1", "START SLAVE", } goodSlave.StartActionLoop(t, wr) defer goodSlave.StopActionLoop(t) // badSlave: insert an error by failing the master hostname input // on purpose badSlave.FakeMysqlDaemon.ReadOnly = true badSlave.FakeMysqlDaemon.SetSlavePositionCommandsPos = master.FakeMysqlDaemon.CurrentMasterPosition badSlave.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", "", master.Tablet.PortMap["mysql"]) badSlave.StartActionLoop(t, wr) defer badSlave.StopActionLoop(t) // also change the master alias in the Shard object, to make sure it // is set back. si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard) if err != nil { t.Fatalf("GetShard failed: %v", err) } si.MasterAlias.Uid++ if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard failed: %v", err) } // run InitShardMaster without force, it fails because master is // changing. if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, master.Tablet.Alias, false /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is not the shard master") { t.Errorf("InitShardMaster with mismatched new master returned wrong error: %v", err) } // run InitShardMaster if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, master.Tablet.Alias, true /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "wrong input for SetMasterCommands") { t.Errorf("InitShardMaster with one failed slave returned wrong error: %v", err) } // check what was run: master should still be good if master.FakeMysqlDaemon.ReadOnly { t.Errorf("master was not turned read-write") } si, err = ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard) if err != nil { t.Fatalf("GetShard failed: %v", err) } if !topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { t.Errorf("unexpected shard master alias, got %v expected %v", si.MasterAlias, master.Tablet.Alias) } }
// TestInitMasterShard is the good scenario test, where everything // works as planned func TestInitMasterShard(t *testing.T) { ctx := context.Background() db := fakesqldb.Register() ts := zktestserver.New(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) vp := NewVtctlPipe(t, ts) defer vp.Close() db.AddQuery("CREATE DATABASE IF NOT EXISTS `vt_test_keyspace`", &sqltypes.Result{}) // Create a master, a couple good slaves master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db) goodSlave1 := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db) goodSlave2 := NewFakeTablet(t, wr, "cell2", 2, topodatapb.TabletType_REPLICA, db) // Master: set a plausible ReplicationPosition to return, // and expect to add entry in _vt.reparent_journal master.FakeMysqlDaemon.CurrentMasterPosition = replication.Position{ GTIDSet: replication.MariadbGTID{ Domain: 5, Server: 456, Sequence: 890, }, } master.FakeMysqlDaemon.ReadOnly = true master.FakeMysqlDaemon.ResetReplicationResult = []string{"reset rep 1"} master.FakeMysqlDaemon.SetSlavePositionCommandsResult = []string{"new master shouldn't use this"} master.FakeMysqlDaemon.SetMasterCommandsResult = []string{"new master shouldn't use this"} master.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "reset rep 1", "CREATE DATABASE IF NOT EXISTS _vt", "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", "CREATE DATABASE IF NOT EXISTS _vt", "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", } master.StartActionLoop(t, wr) defer master.StopActionLoop(t) // Slave1: expect to be reset and re-parented goodSlave1.FakeMysqlDaemon.ReadOnly = true goodSlave1.FakeMysqlDaemon.ResetReplicationResult = []string{"reset rep 1"} goodSlave1.FakeMysqlDaemon.SetSlavePositionCommandsPos = master.FakeMysqlDaemon.CurrentMasterPosition goodSlave1.FakeMysqlDaemon.SetSlavePositionCommandsResult = []string{"cmd1"} goodSlave1.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", master.Tablet.Hostname, master.Tablet.PortMap["mysql"]) goodSlave1.FakeMysqlDaemon.SetMasterCommandsResult = []string{"set master cmd 1"} goodSlave1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "reset rep 1", "cmd1", "set master cmd 1", "START SLAVE", } goodSlave1.StartActionLoop(t, wr) defer goodSlave1.StopActionLoop(t) // Slave2: expect to be re-parented goodSlave2.FakeMysqlDaemon.ReadOnly = true goodSlave2.FakeMysqlDaemon.ResetReplicationResult = []string{"reset rep 2"} goodSlave2.FakeMysqlDaemon.SetSlavePositionCommandsPos = master.FakeMysqlDaemon.CurrentMasterPosition goodSlave2.FakeMysqlDaemon.SetSlavePositionCommandsResult = []string{"cmd1", "cmd2"} goodSlave2.FakeMysqlDaemon.SetMasterCommandsInput = fmt.Sprintf("%v:%v", master.Tablet.Hostname, master.Tablet.PortMap["mysql"]) goodSlave2.FakeMysqlDaemon.SetMasterCommandsResult = []string{"set master cmd 1"} goodSlave2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "reset rep 2", "cmd1", "cmd2", "set master cmd 1", "START SLAVE", } goodSlave2.StartActionLoop(t, wr) defer goodSlave2.StopActionLoop(t) // run InitShardMaster if err := vp.Run([]string{"InitShardMaster", "-wait_slave_timeout", "10s", master.Tablet.Keyspace + "/" + master.Tablet.Shard, topoproto.TabletAliasString(master.Tablet.Alias)}); err != nil { t.Fatalf("InitShardMaster failed: %v", err) } // check what was run if master.FakeMysqlDaemon.ReadOnly { t.Errorf("master was not turned read-write") } si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard) if err != nil { t.Fatalf("GetShard failed: %v", err) } if !topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { t.Errorf("unexpected shard master alias, got %v expected %v", si.MasterAlias, master.Tablet.Alias) } if err := master.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Fatalf("master.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } if err := goodSlave1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Fatalf("goodSlave1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } if err := goodSlave2.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Fatalf("goodSlave2.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } }
func testWaitForDrain(t *testing.T, desc, cells string, drain drainDirective, expectedErrors []string) { const keyspace = "ks" const shard = "-80" db := fakesqldb.Register() ts := zktestserver.New(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) flag.Set("vtctl_healthcheck_timeout", "0.25s") vp := NewVtctlPipe(t, ts) defer vp.Close() // Create keyspace. if err := ts.CreateKeyspace(context.Background(), keyspace, &topodatapb.Keyspace{ ShardingColumnName: "keyspace_id", ShardingColumnType: topodatapb.KeyspaceIdType_UINT64, }); err != nil { t.Fatalf("CreateKeyspace failed: %v", err) } t1 := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_REPLICA, db, TabletKeyspaceShard(t, keyspace, shard)) t2 := NewFakeTablet(t, wr, "cell2", 1, topodatapb.TabletType_REPLICA, db, TabletKeyspaceShard(t, keyspace, shard)) for _, ft := range []*FakeTablet{t1, t2} { ft.StartActionLoop(t, wr) defer ft.StopActionLoop(t) } target := querypb.Target{ Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_REPLICA, } fqs1 := newFakeQueryService(target) fqs2 := newFakeQueryService(target) grpcqueryservice.RegisterForTest(t1.RPCServer, fqs1) grpcqueryservice.RegisterForTest(t2.RPCServer, fqs2) // Run vtctl WaitForDrain and react depending on its output. timeout := "0.5s" if len(expectedErrors) == 0 { // Tests with a positive outcome should have a more generous timeout to // avoid flakyness. timeout = "30s" } stream, err := vp.RunAndStreamOutput( []string{"WaitForDrain", "-cells", cells, "-retry_delay", "100ms", "-timeout", timeout, keyspace + "/" + shard, topodatapb.TabletType_REPLICA.String()}) if err != nil { t.Fatalf("VtctlPipe.RunAndStreamOutput() failed: %v", err) } // QPS = 1.0. Tablets are not drained yet. fqs1.addHealthResponse(1.0) fqs2.addHealthResponse(1.0) var le *logutilpb.Event for { le, err = stream.Recv() if err != nil { break } line := logutil.EventString(le) t.Logf(line) if strings.Contains(line, "for all healthy tablets to be drained") { t.Log("Successfully waited for WaitForDrain to be blocked because tablets have a QPS rate > 0.0") break } else { t.Log("waiting for WaitForDrain to see a QPS rate > 0.0") } } if drain&DrainCell1 != 0 { fqs1.addHealthResponse(0.0) } else { fqs1.addHealthResponse(2.0) } if drain&DrainCell2 != 0 { fqs2.addHealthResponse(0.0) } else { fqs2.addHealthResponse(2.0) } // If a cell was drained, rate should go below <0.0 now. // If not all selected cells were drained, this will end after "-timeout". for { le, err = stream.Recv() if err == nil { vp.t.Logf(logutil.EventString(le)) } else { break } } if len(expectedErrors) == 0 { if err != io.EOF { t.Fatalf("TestWaitForDrain: %v: no error expected but got: %v", desc, err) } // else: Success. } else { if err == nil || err == io.EOF { t.Fatalf("TestWaitForDrain: %v: error expected but got none", desc) } for _, errString := range expectedErrors { if !strings.Contains(err.Error(), errString) { t.Fatalf("TestWaitForDrain: %v: error does not include expected string. got: %v want: %v", desc, err, errString) } } // Success. } }