Beispiel #1
0
func TestBinlogPlayerMapVerticalSplit(t *testing.T) {
	ts := zktestserver.New(t, []string{"cell1"})
	ctx := context.Background()

	// create the keyspaces, with one shard each
	if err := ts.CreateKeyspace(ctx, "source", &topodatapb.Keyspace{}); err != nil {
		t.Fatalf("CreateKeyspace failed: %v", err)
	}
	if err := ts.CreateKeyspace(ctx, "destination", &topodatapb.Keyspace{}); err != nil {
		t.Fatalf("CreateKeyspace failed: %v", err)
	}
	for _, keyspace := range []string{"source", "destination"} {
		if err := ts.CreateShard(ctx, keyspace, "0"); err != nil {
			t.Fatalf("CreateShard failed: %v", err)
		}
	}

	// create one replica remote tablet in source keyspace, we will
	// use it as a source for filtered replication.
	createSourceTablet(t, "test_vertical", ts, "source", "0")

	// register a binlog player factory that will return the instances
	// we want
	clientSyncChannel := make(chan *fakeBinlogClient)
	binlogplayer.RegisterClientFactory("test_vertical", func() binlogplayer.Client {
		return <-clientSyncChannel
	})
	flag.Lookup("binlog_player_protocol").Value.Set("test_vertical")

	// create the BinlogPlayerMap on the local tablet
	// (note that local tablet is never in the topology, we don't
	// need it there at all)
	// The schema will be used to resolve the table wildcards.
	mysqlDaemon := &mysqlctl.FakeMysqlDaemon{
		MysqlPort: 3306,
		Schema: &tabletmanagerdatapb.SchemaDefinition{
			DatabaseSchema: "",
			TableDefinitions: []*tabletmanagerdatapb.TableDefinition{
				{
					Name:              "table1",
					Columns:           []string{"id", "msg", "keyspace_id"},
					PrimaryKeyColumns: []string{"id"},
					Type:              tmutils.TableBaseTable,
				},
				{
					Name:              "funtables_one",
					Columns:           []string{"id", "msg", "keyspace_id"},
					PrimaryKeyColumns: []string{"id"},
					Type:              tmutils.TableBaseTable,
				},
				{
					Name:              "excluded_table",
					Columns:           []string{"id", "msg", "keyspace_id"},
					PrimaryKeyColumns: []string{"id"},
					Type:              tmutils.TableBaseTable,
				},
			},
		},
	}
	vtClientSyncChannel := make(chan *binlogplayer.VtClientMock)
	bpm := NewBinlogPlayerMap(ts, mysqlDaemon, func() binlogplayer.VtClient {
		return <-vtClientSyncChannel
	})

	tablet := &topodatapb.Tablet{
		Alias: &topodatapb.TabletAlias{
			Cell: "cell1",
			Uid:  1,
		},
		Keyspace: "destination",
		Shard:    "0",
	}

	si, err := ts.GetShard(ctx, "destination", "0")
	if err != nil {
		t.Fatalf("GetShard failed: %v", err)
	}
	si.SourceShards = []*topodatapb.Shard_SourceShard{
		{
			Uid:      1,
			Keyspace: "source",
			Shard:    "0",
			Tables: []string{
				"table1",
				"funtables_*",
			},
		},
	}
	if err := ts.UpdateShard(ctx, si); err != nil {
		t.Fatalf("UpdateShard failed: %v", err)
	}

	// now we have a source, adding players
	bpm.RefreshMap(ctx, tablet, si)
	if !bpm.isRunningFilteredReplication() {
		t.Errorf("isRunningFilteredReplication should be true")
	}

	// write a mocked vtClientMock that will be used to read the
	// start position at first. Note this also synchronizes the player,
	// so we can then check mysqlDaemon.BinlogPlayerEnabled.
	vtClientMock := binlogplayer.NewVtClientMock()
	vtClientMock.Result = &sqltypes.Result{
		Fields:       nil,
		RowsAffected: 1,
		InsertID:     0,
		Rows: [][]sqltypes.Value{
			{
				sqltypes.MakeString([]byte("MariaDB/0-1-1234")),
				sqltypes.MakeString([]byte("")),
			},
		},
	}
	vtClientSyncChannel <- vtClientMock
	if !mysqlDaemon.BinlogPlayerEnabled {
		t.Errorf("mysqlDaemon.BinlogPlayerEnabled should be true")
	}

	// the client will then try to connect to the remote tablet.
	// give it what it needs.
	fbc := newFakeBinlogClient(t, 100)
	fbc.expectedTables = "table1,funtables_one"
	clientSyncChannel <- fbc

	// now we can feed an event through the fake connection
	vtClientMock.CommitChannel = make(chan []string)
	fbc.tablesChannel <- &binlogdatapb.BinlogTransaction{
		Statements: []*binlogdatapb.BinlogTransaction_Statement{
			{
				Category: binlogdatapb.BinlogTransaction_Statement_BL_DML,
				Sql:      "INSERT INTO tablet VALUES(1)",
			},
		},
		Timestamp:     72,
		TransactionId: "MariaDB/0-1-1235",
	}

	// and make sure it results in a committed statement
	sql := <-vtClientMock.CommitChannel
	if len(sql) != 5 ||
		sql[0] != "SELECT pos, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=1" ||
		sql[1] != "BEGIN" ||
		!strings.HasPrefix(sql[2], "UPDATE _vt.blp_checkpoint SET pos='MariaDB/0-1-1235', time_updated=") ||
		!strings.HasSuffix(sql[2], ", transaction_timestamp=72 WHERE source_shard_uid=1") ||
		sql[3] != "INSERT INTO tablet VALUES(1)" ||
		sql[4] != "COMMIT" {
		t.Errorf("Got wrong SQL: %#v", sql)
	}

	// ask for status, make sure we got what we expect
	s := bpm.Status()
	if s.State != "Running" ||
		len(s.Controllers) != 1 ||
		s.Controllers[0].Index != 1 ||
		s.Controllers[0].State != "Running" ||
		s.Controllers[0].SourceShard.Keyspace != "source" ||
		s.Controllers[0].SourceShard.Shard != "0" ||
		s.Controllers[0].LastError != "" {
		t.Errorf("unexpected state: %v %v", s, s.Controllers[0])
	}

	// check BlpPositionList API from BinlogPlayerMap
	checkBlpPositionList(t, bpm, vtClientSyncChannel)

	// now stop the binlog player map.
	// this will stop the player, which will cancel its context,
	// and exit the fake streaming connection.
	bpm.Stop()
	s = bpm.Status()
	if s.State != "Stopped" ||
		len(s.Controllers) != 1 ||
		s.Controllers[0].State != "Stopped" {
		t.Errorf("unexpected state: %v", s)
	}
}
Beispiel #2
0
// Registration as a factory
func init() {
	binlogplayer.RegisterClientFactory("grpc", func() binlogplayer.Client {
		return &client{}
	})
}
Beispiel #3
0
func TestBinlogPlayerMapHorizontalSplitStopStartUntil(t *testing.T) {
	ts := zktestserver.New(t, []string{"cell1"})
	ctx := context.Background()

	// create the keyspace, a full set of covering shards,
	// and a new split destination shard.
	if err := ts.CreateKeyspace(ctx, "ks", &topodatapb.Keyspace{
		ShardingColumnType: topodatapb.KeyspaceIdType_UINT64,
		ShardingColumnName: "sharding_key",
	}); err != nil {
		t.Fatalf("CreateKeyspace failed: %v", err)
	}
	for _, shard := range []string{"-80", "80-", "40-60"} {
		if err := ts.CreateShard(ctx, "ks", shard); err != nil {
			t.Fatalf("CreateShard failed: %v", err)
		}
	}

	// create one replica remote tablet in source shard, we will
	// use it as a source for filtered replication.
	createSourceTablet(t, "test_horizontal_until", ts, "ks", "-80")

	// register a binlog player factory that will return the instances
	// we want
	clientSyncChannel := make(chan *fakeBinlogClient)
	binlogplayer.RegisterClientFactory("test_horizontal_until", func() binlogplayer.Client {
		return <-clientSyncChannel
	})
	flag.Lookup("binlog_player_protocol").Value.Set("test_horizontal_until")

	// create the BinlogPlayerMap on the local tablet
	// (note that local tablet is never in the topology, we don't
	// need it there at all)
	mysqlDaemon := &mysqlctl.FakeMysqlDaemon{MysqlPort: 3306}
	vtClientSyncChannel := make(chan *binlogplayer.VtClientMock)
	bpm := NewBinlogPlayerMap(ts, mysqlDaemon, func() binlogplayer.VtClient {
		return <-vtClientSyncChannel
	})

	tablet := &topodatapb.Tablet{
		Alias: &topodatapb.TabletAlias{
			Cell: "cell1",
			Uid:  1,
		},
		KeyRange: &topodatapb.KeyRange{
			Start: []byte{0x40},
			End:   []byte{0x60},
		},
		Keyspace: "ks",
		Shard:    "40-60",
	}

	si, err := ts.GetShard(ctx, "ks", "40-60")
	if err != nil {
		t.Fatalf("GetShard failed: %v", err)
	}
	si.SourceShards = []*topodatapb.Shard_SourceShard{
		{
			Uid:      1,
			Keyspace: "ks",
			Shard:    "-80",
			KeyRange: &topodatapb.KeyRange{
				End: []byte{0x80},
			},
		},
	}
	if err := ts.UpdateShard(ctx, si); err != nil {
		t.Fatalf("UpdateShard failed: %v", err)
	}

	// now we have a source, adding players
	bpm.RefreshMap(ctx, tablet, si)
	if !bpm.isRunningFilteredReplication() {
		t.Errorf("isRunningFilteredReplication should be true")
	}

	// write a mocked vtClientMock that will be used to read the
	// start position at first. Note this also synchronizes the player,
	// so we can then check mysqlDaemon.BinlogPlayerEnabled.
	vtClientMock := binlogplayer.NewVtClientMock()
	vtClientMock.Result = &sqltypes.Result{
		Fields:       nil,
		RowsAffected: 1,
		InsertID:     0,
		Rows: [][]sqltypes.Value{
			{
				sqltypes.MakeString([]byte("MariaDB/0-1-1234")),
				sqltypes.MakeString([]byte("")),
			},
		},
	}
	vtClientSyncChannel <- vtClientMock
	if !mysqlDaemon.BinlogPlayerEnabled {
		t.Errorf("mysqlDaemon.BinlogPlayerEnabled should be true")
	}

	// the client will then try to connect to the remote tablet.
	// give it what it needs.
	fbc := newFakeBinlogClient(t, 100)
	fbc.expectedKeyRange = "40-60"
	clientSyncChannel <- fbc

	// now stop the map
	bpm.Stop()
	s := bpm.Status()
	if s.State != "Stopped" ||
		len(s.Controllers) != 1 ||
		s.Controllers[0].State != "Stopped" {
		t.Errorf("unexpected state: %v", s)
	}

	// in the background, start a function that will do what's needed
	wg := sync.WaitGroup{}
	wg.Add(1)
	go func() {
		// the client will first try to read the current position again
		vtClientSyncChannel <- vtClientMock

		// the client will then try to connect to the remote tablet.
		// give it what it needs.
		fbc := newFakeBinlogClient(t, 100)
		fbc.expectedKeyRange = "40-60"
		clientSyncChannel <- fbc

		// feed an event through the fake connection
		vtClientMock.CommitChannel = make(chan []string)
		fbc.keyRangeChannel <- &binlogdatapb.BinlogTransaction{
			Statements: []*binlogdatapb.BinlogTransaction_Statement{
				{
					Category: binlogdatapb.BinlogTransaction_Statement_BL_DML,
					Sql:      "INSERT INTO tablet VALUES(1)",
				},
			},
			Timestamp:     72,
			TransactionId: "MariaDB/0-1-1235",
		}

		// and make sure it results in a committed statement
		sql := <-vtClientMock.CommitChannel
		if len(sql) != 6 ||
			sql[0] != "SELECT pos, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=1" ||
			sql[1] != "SELECT pos, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=1" ||
			sql[2] != "BEGIN" ||
			!strings.HasPrefix(sql[3], "UPDATE _vt.blp_checkpoint SET pos='MariaDB/0-1-1235', time_updated=") ||
			!strings.HasSuffix(sql[3], ", transaction_timestamp=72 WHERE source_shard_uid=1") ||
			sql[4] != "INSERT INTO tablet VALUES(1)" ||
			sql[5] != "COMMIT" {
			t.Errorf("Got wrong SQL: %#v", sql)
		}
		wg.Done()
	}()

	// now restart the map until we get the right BlpPosition
	mysqlDaemon.BinlogPlayerEnabled = false
	ctx1, _ := context.WithTimeout(ctx, 5*time.Second)
	if err := bpm.RunUntil(ctx1, []*tabletmanagerdatapb.BlpPosition{
		{
			Uid:      1,
			Position: "MariaDB/0-1-1235",
		},
	}, 5*time.Second); err != nil {
		t.Fatalf("RunUntil failed: %v", err)
	}

	// make sure the background function is done
	wg.Wait()

	// ask for status, make sure we got what we expect
	s = bpm.Status()
	if s.State != "Stopped" ||
		len(s.Controllers) != 1 ||
		s.Controllers[0].Index != 1 ||
		s.Controllers[0].State != "Stopped" ||
		s.Controllers[0].LastError != "" {
		t.Errorf("unexpected state: %v", s)
	}

	// check BlpPositionList API from BinlogPlayerMap
	checkBlpPositionList(t, bpm, vtClientSyncChannel)
}