func newFakeTabletManagerClient() *fakeTabletManagerClient { return &fakeTabletManagerClient{ TabletManagerClient: faketmclient.NewFakeTabletManagerClient(), preflightSchemas: make(map[string]*proto.SchemaChangeResult), schemaDefinitions: make(map[string]*proto.SchemaDefinition), } }
func TestVerticalSplitDiff(t *testing.T) { db := fakesqldb.Register() ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) ctx := context.Background() wi := NewInstance(ctx, ts, "cell1", time.Second) sourceMaster := testlib.NewFakeTablet(t, wi.wr, "cell1", 0, pbt.TabletType_MASTER, db, testlib.TabletKeyspaceShard(t, "source_ks", "0")) sourceRdonly1 := testlib.NewFakeTablet(t, wi.wr, "cell1", 1, pbt.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "source_ks", "0")) sourceRdonly2 := testlib.NewFakeTablet(t, wi.wr, "cell1", 2, pbt.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "source_ks", "0")) // Create the destination keyspace with the appropriate ServedFromMap ki := &pbt.Keyspace{ ServedFroms: []*pbt.Keyspace_ServedFrom{ &pbt.Keyspace_ServedFrom{ TabletType: pbt.TabletType_MASTER, Keyspace: "source_ks", }, &pbt.Keyspace_ServedFrom{ TabletType: pbt.TabletType_REPLICA, Keyspace: "source_ks", }, &pbt.Keyspace_ServedFrom{ TabletType: pbt.TabletType_RDONLY, Keyspace: "source_ks", }, }, } wi.wr.TopoServer().CreateKeyspace(ctx, "destination_ks", ki) destMaster := testlib.NewFakeTablet(t, wi.wr, "cell1", 10, pbt.TabletType_MASTER, db, testlib.TabletKeyspaceShard(t, "destination_ks", "0")) destRdonly1 := testlib.NewFakeTablet(t, wi.wr, "cell1", 11, pbt.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "destination_ks", "0")) destRdonly2 := testlib.NewFakeTablet(t, wi.wr, "cell1", 12, pbt.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "destination_ks", "0")) for _, ft := range []*testlib.FakeTablet{sourceMaster, sourceRdonly1, sourceRdonly2, destMaster, destRdonly1, destRdonly2} { ft.StartActionLoop(t, wi.wr) defer ft.StopActionLoop(t) } wi.wr.SetSourceShards(ctx, "destination_ks", "0", []*pbt.TabletAlias{sourceRdonly1.Tablet.Alias}, []string{"moving.*", "view1"}) // add the topo and schema data we'll need if err := wi.wr.RebuildKeyspaceGraph(ctx, "source_ks", nil, true); err != nil { t.Fatalf("RebuildKeyspaceGraph failed: %v", err) } if err := wi.wr.RebuildKeyspaceGraph(ctx, "destination_ks", nil, true); err != nil { t.Fatalf("RebuildKeyspaceGraph failed: %v", err) } // We need to use FakeTabletManagerClient because we don't // have a good way to fake the binlog player yet, which is // necessary for synchronizing replication. wr := wrangler.New(logutil.NewConsoleLogger(), ts, faketmclient.NewFakeTabletManagerClient()) excludedTable := "excludedTable1" subFlags := flag.NewFlagSet("VerticalSplitDiff", flag.ContinueOnError) gwrk, err := commandVerticalSplitDiff(wi, wr, subFlags, []string{ "-exclude_tables", excludedTable, "destination_ks/0", }) if err != nil { t.Fatalf("commandVerticalSplitDiff failed: %v", err) } wrk := gwrk.(*VerticalSplitDiffWorker) for _, rdonly := range []*testlib.FakeTablet{sourceRdonly1, sourceRdonly2, destRdonly1, destRdonly2} { // both source and destination should be identical (for schema and data returned) rdonly.FakeMysqlDaemon.Schema = &myproto.SchemaDefinition{ DatabaseSchema: "", TableDefinitions: []*myproto.TableDefinition{ &myproto.TableDefinition{ Name: "moving1", Columns: []string{"id", "msg"}, PrimaryKeyColumns: []string{"id"}, Type: myproto.TableBaseTable, }, &myproto.TableDefinition{ Name: excludedTable, Columns: []string{"id", "msg"}, PrimaryKeyColumns: []string{"id"}, Type: myproto.TableBaseTable, }, &myproto.TableDefinition{ Name: "view1", Type: myproto.TableView, }, }, } grpcqueryservice.RegisterForTest(rdonly.RPCServer, &verticalDiffTabletServer{t: t, excludedTable: excludedTable}) } err = wrk.Run(ctx) status := wrk.StatusAsText() t.Logf("Got status: %v", status) if err != nil || wrk.State != WorkerStateDone { t.Errorf("Worker run failed") } }
func TestSplitDiff(t *testing.T) { db := fakesqldb.Register() ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) ctx := context.Background() wi := NewInstance(ctx, ts, "cell1", time.Second) if err := ts.CreateKeyspace(context.Background(), "ks", &topodatapb.Keyspace{ ShardingColumnName: "keyspace_id", ShardingColumnType: topodatapb.KeyspaceIdType_UINT64, }); err != nil { t.Fatalf("CreateKeyspace failed: %v", err) } sourceMaster := testlib.NewFakeTablet(t, wi.wr, "cell1", 0, topodatapb.TabletType_MASTER, db, testlib.TabletKeyspaceShard(t, "ks", "-80")) sourceRdonly1 := testlib.NewFakeTablet(t, wi.wr, "cell1", 1, topodatapb.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "ks", "-80")) sourceRdonly2 := testlib.NewFakeTablet(t, wi.wr, "cell1", 2, topodatapb.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "ks", "-80")) leftMaster := testlib.NewFakeTablet(t, wi.wr, "cell1", 10, topodatapb.TabletType_MASTER, db, testlib.TabletKeyspaceShard(t, "ks", "-40")) leftRdonly1 := testlib.NewFakeTablet(t, wi.wr, "cell1", 11, topodatapb.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "ks", "-40")) leftRdonly2 := testlib.NewFakeTablet(t, wi.wr, "cell1", 12, topodatapb.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "ks", "-40")) for _, ft := range []*testlib.FakeTablet{sourceMaster, sourceRdonly1, sourceRdonly2, leftMaster, leftRdonly1, leftRdonly2} { ft.StartActionLoop(t, wi.wr) defer ft.StopActionLoop(t) } // add the topo and schema data we'll need if err := ts.CreateShard(ctx, "ks", "80-"); err != nil { t.Fatalf("CreateShard(\"-80\") failed: %v", err) } wi.wr.SetSourceShards(ctx, "ks", "-40", []*topodatapb.TabletAlias{sourceRdonly1.Tablet.Alias}, nil) if err := wi.wr.SetKeyspaceShardingInfo(ctx, "ks", "keyspace_id", topodatapb.KeyspaceIdType_UINT64, 4, false); err != nil { t.Fatalf("SetKeyspaceShardingInfo failed: %v", err) } if err := wi.wr.RebuildKeyspaceGraph(ctx, "ks", nil, true); err != nil { t.Fatalf("RebuildKeyspaceGraph failed: %v", err) } subFlags := flag.NewFlagSet("SplitDiff", flag.ContinueOnError) // We need to use FakeTabletManagerClient because we don't // have a good way to fake the binlog player yet, which is // necessary for synchronizing replication. wr := wrangler.New(logutil.NewConsoleLogger(), ts, faketmclient.NewFakeTabletManagerClient()) excludedTable := "excludedTable1" gwrk, err := commandSplitDiff(wi, wr, subFlags, []string{ "-exclude_tables", excludedTable, "ks/-40", }) if err != nil { t.Fatalf("commandSplitDiff failed: %v", err) } wrk := gwrk.(*SplitDiffWorker) for _, rdonly := range []*testlib.FakeTablet{sourceRdonly1, sourceRdonly2, leftRdonly1, leftRdonly2} { // In reality, the destinations *shouldn't* have identical data to the source - instead, we should see // the data split into left and right. However, if we do that in this test, we would really just be // testing our fake SQL logic, since we do the data filtering in SQL. // To simplify things, just assume that both sides have identical data. rdonly.FakeMysqlDaemon.Schema = &tabletmanagerdatapb.SchemaDefinition{ DatabaseSchema: "", TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "table1", Columns: []string{"id", "msg", "keyspace_id"}, PrimaryKeyColumns: []string{"id"}, Type: tmutils.TableBaseTable, }, { Name: excludedTable, Columns: []string{"id", "msg", "keyspace_id"}, PrimaryKeyColumns: []string{"id"}, Type: tmutils.TableBaseTable, }, { Name: "view1", Type: tmutils.TableView, }, }, } } grpcqueryservice.RegisterForTest(leftRdonly1.RPCServer, &destinationTabletServer{t: t, excludedTable: excludedTable}) grpcqueryservice.RegisterForTest(leftRdonly2.RPCServer, &destinationTabletServer{t: t, excludedTable: excludedTable}) grpcqueryservice.RegisterForTest(sourceRdonly1.RPCServer, &sourceTabletServer{t: t, excludedTable: excludedTable}) grpcqueryservice.RegisterForTest(sourceRdonly2.RPCServer, &sourceTabletServer{t: t, excludedTable: excludedTable}) err = wrk.Run(ctx) status := wrk.StatusAsText() t.Logf("Got status: %v", status) if err != nil || wrk.State != WorkerStateDone { t.Errorf("Worker run failed") } }
// TODO(aaijazi): Create a test in which source and destination data does not match // TODO(aaijazi): This test is reallly slow; investigate why. func TestSqlDiffer(t *testing.T) { db := fakesqldb.Register() ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) // We need to use FakeTabletManagerClient because we don't have a good way to fake the binlog player yet, // which is necessary for synchronizing replication. wr := wrangler.New(logutil.NewConsoleLogger(), ts, faketmclient.NewFakeTabletManagerClient(), time.Second) ctx := context.Background() supersetMaster := testlib.NewFakeTablet(t, wr, "cell1", 0, pbt.TabletType_MASTER, db, testlib.TabletKeyspaceShard(t, "source_ks", "0")) supersetRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 1, pbt.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "source_ks", "0")) supersetRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 2, pbt.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "source_ks", "0")) subsetMaster := testlib.NewFakeTablet(t, wr, "cell1", 10, pbt.TabletType_MASTER, db, testlib.TabletKeyspaceShard(t, "destination_ks", "0")) subsetRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 11, pbt.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "destination_ks", "0")) subsetRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 12, pbt.TabletType_RDONLY, db, testlib.TabletKeyspaceShard(t, "destination_ks", "0")) for _, ft := range []*testlib.FakeTablet{supersetMaster, supersetRdonly1, supersetRdonly2, subsetMaster, subsetRdonly1, subsetRdonly2} { ft.StartActionLoop(t, wr) defer ft.StopActionLoop(t) } wr.SetSourceShards(ctx, "destination_ks", "0", []*pbt.TabletAlias{supersetRdonly1.Tablet.Alias}, []string{"moving.*", "view1"}) // add the topo and schema data we'll need if err := wr.RebuildKeyspaceGraph(ctx, "source_ks", nil, true); err != nil { t.Fatalf("RebuildKeyspaceGraph failed: %v", err) } if err := wr.RebuildKeyspaceGraph(ctx, "destination_ks", nil, true); err != nil { t.Fatalf("RebuildKeyspaceGraph failed: %v", err) } supersetSourceSpec := SourceSpec{"source_ks", "0", "SELECT *", supersetRdonly1.Tablet.Alias} subsetSourceSpec := SourceSpec{"destination_ks", "0", "SELECT *", subsetRdonly1.Tablet.Alias} gwrk := NewSQLDiffWorker(wr, "cell1", supersetSourceSpec, subsetSourceSpec) wrk := gwrk.(*SQLDiffWorker) for _, rdonly := range []*testlib.FakeTablet{supersetRdonly1, supersetRdonly2, subsetRdonly1, subsetRdonly2} { rdonly.FakeMysqlDaemon.Schema = &myproto.SchemaDefinition{ DatabaseSchema: "", TableDefinitions: []*myproto.TableDefinition{ &myproto.TableDefinition{ Name: "moving1", Columns: []string{"id", "msg"}, PrimaryKeyColumns: []string{"id"}, Type: myproto.TableBaseTable, }, &myproto.TableDefinition{ Name: "view1", Type: myproto.TableView, }, }, } grpcqueryservice.RegisterForTest(rdonly.RPCServer, &sqlDifferTabletServer{t: t}) } err := wrk.Run(ctx) status := wrk.StatusAsText() t.Logf("Got status: %v", status) if err != nil || wrk.State != WorkerStateDone { t.Errorf("Worker run failed") } }
func TestVerticalSplitDiff(t *testing.T) { ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) // We need to use FakeTabletManagerClient because we don't have a good way to fake the binlog player yet, // which is necessary for synchronizing replication. wr := wrangler.New(logutil.NewConsoleLogger(), ts, faketmclient.NewFakeTabletManagerClient(), time.Second) ctx := context.Background() sourceMaster := testlib.NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "source_ks", "0")) sourceRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "source_ks", "0")) sourceRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "source_ks", "0")) // Create the destination keyspace with the appropriate ServedFromMap ki := &topo.Keyspace{} ki.ServedFromMap = map[topo.TabletType]*topo.KeyspaceServedFrom{ topo.TYPE_MASTER: &topo.KeyspaceServedFrom{Keyspace: "source_ks"}, topo.TYPE_REPLICA: &topo.KeyspaceServedFrom{Keyspace: "source_ks"}, topo.TYPE_RDONLY: &topo.KeyspaceServedFrom{Keyspace: "source_ks"}, } wr.TopoServer().CreateKeyspace(ctx, "destination_ks", ki) destMaster := testlib.NewFakeTablet(t, wr, "cell1", 10, topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "destination_ks", "0")) destRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 11, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "destination_ks", "0")) destRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 12, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "destination_ks", "0")) for _, ft := range []*testlib.FakeTablet{sourceMaster, sourceRdonly1, sourceRdonly2, destMaster, destRdonly1, destRdonly2} { ft.StartActionLoop(t, wr) defer ft.StopActionLoop(t) } wr.SetSourceShards(ctx, "destination_ks", "0", []topo.TabletAlias{sourceRdonly1.Tablet.Alias}, []string{"moving.*", "view1"}) // add the topo and schema data we'll need if err := wr.RebuildKeyspaceGraph(ctx, "source_ks", nil, true); err != nil { t.Fatalf("RebuildKeyspaceGraph failed: %v", err) } if err := wr.RebuildKeyspaceGraph(ctx, "destination_ks", nil, true); err != nil { t.Fatalf("RebuildKeyspaceGraph failed: %v", err) } excludedTable := "excludedTable1" gwrk := NewVerticalSplitDiffWorker(wr, "cell1", "destination_ks", "0", []string{excludedTable}) wrk := gwrk.(*VerticalSplitDiffWorker) for _, rdonly := range []*testlib.FakeTablet{sourceRdonly1, sourceRdonly2, destRdonly1, destRdonly2} { // both source and destination should be identical (for schema and data returned) rdonly.FakeMysqlDaemon.Schema = &myproto.SchemaDefinition{ DatabaseSchema: "", TableDefinitions: []*myproto.TableDefinition{ &myproto.TableDefinition{ Name: "moving1", Columns: []string{"id", "msg"}, PrimaryKeyColumns: []string{"id"}, Type: myproto.TableBaseTable, }, &myproto.TableDefinition{ Name: excludedTable, Columns: []string{"id", "msg"}, PrimaryKeyColumns: []string{"id"}, Type: myproto.TableBaseTable, }, &myproto.TableDefinition{ Name: "view1", Type: myproto.TableView, }, }, } rdonly.RPCServer.Register(gorpcqueryservice.New(&verticalDiffSqlQuery{t: t, excludedTable: excludedTable})) } err := wrk.Run(ctx) status := wrk.StatusAsText() t.Logf("Got status: %v", status) if err != nil || wrk.State != WorkerStateDone { t.Errorf("Worker run failed") } }
func newFakeTMCTopo(ts topo.Server) tmclient.TabletManagerClient { return &fakeTMCTopo{ TabletManagerClient: faketmclient.NewFakeTabletManagerClient(), server: ts, } }
func TestSplitDiff(t *testing.T) { ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) // We need to use FakeTabletManagerClient because we don't have a good way to fake the binlog player yet, // which is necessary for synchronizing replication. wr := wrangler.New(logutil.NewConsoleLogger(), ts, faketmclient.NewFakeTabletManagerClient(), time.Second) ctx := context.Background() sourceMaster := testlib.NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-80")) sourceRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 1, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80")) sourceRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 2, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-80")) leftMaster := testlib.NewFakeTablet(t, wr, "cell1", 10, topo.TYPE_MASTER, testlib.TabletKeyspaceShard(t, "ks", "-40")) leftRdonly1 := testlib.NewFakeTablet(t, wr, "cell1", 11, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-40")) leftRdonly2 := testlib.NewFakeTablet(t, wr, "cell1", 12, topo.TYPE_RDONLY, testlib.TabletKeyspaceShard(t, "ks", "-40")) for _, ft := range []*testlib.FakeTablet{sourceMaster, sourceRdonly1, sourceRdonly2, leftMaster, leftRdonly1, leftRdonly2} { ft.StartActionLoop(t, wr) defer ft.StopActionLoop(t) } // add the topo and schema data we'll need if err := topo.CreateShard(ctx, ts, "ks", "80-"); err != nil { t.Fatalf("CreateShard(\"-80\") failed: %v", err) } wr.SetSourceShards(ctx, "ks", "-40", []topo.TabletAlias{sourceRdonly1.Tablet.Alias}, nil) if err := wr.SetKeyspaceShardingInfo(ctx, "ks", "keyspace_id", key.KIT_UINT64, 4, false); err != nil { t.Fatalf("SetKeyspaceShardingInfo failed: %v", err) } if err := wr.RebuildKeyspaceGraph(ctx, "ks", nil, true); err != nil { t.Fatalf("RebuildKeyspaceGraph failed: %v", err) } excludedTable := "excludedTable1" gwrk := NewSplitDiffWorker(wr, "cell1", "ks", "-40", []string{excludedTable}) wrk := gwrk.(*SplitDiffWorker) for _, rdonly := range []*testlib.FakeTablet{sourceRdonly1, sourceRdonly2, leftRdonly1, leftRdonly2} { // In reality, the destinations *shouldn't* have identical data to the source - instead, we should see // the data split into left and right. However, if we do that in this test, we would really just be // testing our fake SQL logic, since we do the data filtering in SQL. // To simplify things, just assume that both sides have identical data. rdonly.FakeMysqlDaemon.Schema = &myproto.SchemaDefinition{ DatabaseSchema: "", TableDefinitions: []*myproto.TableDefinition{ &myproto.TableDefinition{ Name: "table1", Columns: []string{"id", "msg", "keyspace_id"}, PrimaryKeyColumns: []string{"id"}, Type: myproto.TableBaseTable, }, &myproto.TableDefinition{ Name: excludedTable, Columns: []string{"id", "msg", "keyspace_id"}, PrimaryKeyColumns: []string{"id"}, Type: myproto.TableBaseTable, }, &myproto.TableDefinition{ Name: "view1", Type: myproto.TableView, }, }, } } leftRdonly1.RPCServer.Register(gorpcqueryservice.New(&destinationSqlQuery{t: t, excludedTable: excludedTable})) leftRdonly2.RPCServer.Register(gorpcqueryservice.New(&destinationSqlQuery{t: t, excludedTable: excludedTable})) sourceRdonly1.RPCServer.Register(gorpcqueryservice.New(&sourceSqlQuery{t: t, excludedTable: excludedTable})) sourceRdonly2.RPCServer.Register(gorpcqueryservice.New(&sourceSqlQuery{t: t, excludedTable: excludedTable})) err := wrk.Run(ctx) status := wrk.StatusAsText() t.Logf("Got status: %v", status) if err != nil || wrk.State != WorkerStateDone { t.Errorf("Worker run failed") } }