func CheckServingGraph(t *testing.T, ts topo.Server) {
cell := getLocalCell(t, ts)
// test individual cell/keyspace/shard/type entries
if _, err := ts.GetSrvTabletTypesPerShard(cell, "test_keyspace", "-10"); err != topo.ErrNoNode {
t.Errorf("GetSrvTabletTypesPerShard(invalid): %v", err)
}
if _, err := ts.GetEndPoints(cell, "test_keyspace", "-10", topo.TYPE_MASTER); err != topo.ErrNoNode {
t.Errorf("GetEndPoints(invalid): %v", err)
}
endPoints := topo.EndPoints{
Entries: []topo.EndPoint{
topo.EndPoint{
Uid: 1,
Host: "host1",
NamedPortMap: map[string]int{"_vt": 1234, "_mysql": 1235, "_vts": 1236},
},
},
}
if err := ts.UpdateEndPoints(cell, "test_keyspace", "-10", topo.TYPE_MASTER, &endPoints); err != nil {
t.Errorf("UpdateEndPoints(master): %v", err)
}
if types, err := ts.GetSrvTabletTypesPerShard(cell, "test_keyspace", "-10"); err != nil || len(types) != 1 || types[0] != topo.TYPE_MASTER {
t.Errorf("GetSrvTabletTypesPerShard(1): %v %v", err, types)
}
addrs, err := ts.GetEndPoints(cell, "test_keyspace", "-10", topo.TYPE_MASTER)
if err != nil {
t.Errorf("GetEndPoints: %v", err)
}
if len(addrs.Entries) != 1 || addrs.Entries[0].Uid != 1 {
t.Errorf("GetEndPoints(1): %v", addrs)
}
if pm := addrs.Entries[0].NamedPortMap; pm["_vt"] != 1234 || pm["_mysql"] != 1235 || pm["_vts"] != 1236 {
t.Errorf("GetSrcTabletType(1).NamedPortmap: want %v, got %v", endPoints.Entries[0].NamedPortMap, pm)
}
if err := ts.UpdateTabletEndpoint(cell, "test_keyspace", "-10", topo.TYPE_REPLICA, &topo.EndPoint{Uid: 2, Host: "host2"}); err != nil {
t.Errorf("UpdateTabletEndpoint(invalid): %v", err)
}
if err := ts.UpdateTabletEndpoint(cell, "test_keyspace", "-10", topo.TYPE_MASTER, &topo.EndPoint{Uid: 1, Host: "host2"}); err != nil {
t.Errorf("UpdateTabletEndpoint(master): %v", err)
}
if addrs, err := ts.GetEndPoints(cell, "test_keyspace", "-10", topo.TYPE_MASTER); err != nil || len(addrs.Entries) != 1 || addrs.Entries[0].Uid != 1 {
t.Errorf("GetEndPoints(2): %v %v", err, addrs)
}
if err := ts.UpdateTabletEndpoint(cell, "test_keyspace", "-10", topo.TYPE_MASTER, &topo.EndPoint{Uid: 3, Host: "host3"}); err != nil {
t.Errorf("UpdateTabletEndpoint(master): %v", err)
}
if addrs, err := ts.GetEndPoints(cell, "test_keyspace", "-10", topo.TYPE_MASTER); err != nil || len(addrs.Entries) != 2 {
t.Errorf("GetEndPoints(2): %v %v", err, addrs)
}
if err := ts.DeleteSrvTabletType(cell, "test_keyspace", "-10", topo.TYPE_REPLICA); err != topo.ErrNoNode {
t.Errorf("DeleteSrvTabletType(unknown): %v", err)
}
if err := ts.DeleteSrvTabletType(cell, "test_keyspace", "-10", topo.TYPE_MASTER); err != nil {
t.Errorf("DeleteSrvTabletType(master): %v", err)
}
// test cell/keyspace/shard entries (SrvShard)
srvShard := topo.SrvShard{
ServedTypes: []topo.TabletType{topo.TYPE_MASTER},
TabletTypes: []topo.TabletType{topo.TYPE_REPLICA, topo.TYPE_RDONLY},
}
if err := ts.UpdateSrvShard(cell, "test_keyspace", "-10", &srvShard); err != nil {
t.Errorf("UpdateSrvShard(1): %v", err)
}
if _, err := ts.GetSrvShard(cell, "test_keyspace", "666"); err != topo.ErrNoNode {
t.Errorf("GetSrvShard(invalid): %v", err)
}
if s, err := ts.GetSrvShard(cell, "test_keyspace", "-10"); err != nil ||
len(s.ServedTypes) != 1 ||
s.ServedTypes[0] != topo.TYPE_MASTER ||
len(s.TabletTypes) != 2 ||
s.TabletTypes[0] != topo.TYPE_REPLICA ||
s.TabletTypes[1] != topo.TYPE_RDONLY {
t.Errorf("GetSrvShard(valid): %v", err)
}
// test cell/keyspace entries (SrvKeyspace)
srvKeyspace := topo.SrvKeyspace{
Partitions: map[topo.TabletType]*topo.KeyspacePartition{
topo.TYPE_MASTER: &topo.KeyspacePartition{
Shards: []topo.SrvShard{
topo.SrvShard{
ServedTypes: []topo.TabletType{topo.TYPE_MASTER},
},
},
},
},
TabletTypes: []topo.TabletType{topo.TYPE_MASTER},
}
if err := ts.UpdateSrvKeyspace(cell, "test_keyspace", &srvKeyspace); err != nil {
t.Errorf("UpdateSrvKeyspace(1): %v", err)
}
if _, err := ts.GetSrvKeyspace(cell, "test_keyspace666"); err != topo.ErrNoNode {
t.Errorf("GetSrvKeyspace(invalid): %v", err)
}
if s, err := ts.GetSrvKeyspace(cell, "test_keyspace"); err != nil ||
len(s.TabletTypes) != 1 ||
s.TabletTypes[0] != topo.TYPE_MASTER ||
len(s.Partitions) != 1 ||
len(s.Partitions[topo.TYPE_MASTER].Shards) != 1 ||
len(s.Partitions[topo.TYPE_MASTER].Shards[0].ServedTypes) != 1 ||
s.Partitions[topo.TYPE_MASTER].Shards[0].ServedTypes[0] != topo.TYPE_MASTER {
t.Errorf("GetSrvKeyspace(valid): %v", err)
}
if k, err := ts.GetSrvKeyspaceNames(cell); err != nil || len(k) != 1 || k[0] != "test_keyspace" {
t.Errorf("GetSrvKeyspaceNames(): %v", err)
}
}
// CheckServingGraph makes sure the serving graph functions work properly.
func CheckServingGraph(ctx context.Context, t *testing.T, ts topo.Server) {
cell := getLocalCell(ctx, t, ts)
// test individual cell/keyspace/shard/type entries
if _, err := ts.GetSrvTabletTypesPerShard(ctx, cell, "test_keyspace", "-10"); err != topo.ErrNoNode {
t.Errorf("GetSrvTabletTypesPerShard(invalid): %v", err)
}
if _, _, err := ts.GetEndPoints(ctx, cell, "test_keyspace", "-10", topo.TYPE_MASTER); err != topo.ErrNoNode {
t.Errorf("GetEndPoints(invalid): %v", err)
}
endPoints := &pb.EndPoints{
Entries: []*pb.EndPoint{
&pb.EndPoint{
Uid: 1,
Host: "host1",
PortMap: map[string]int32{
"vt": 1234,
"mysql": 1235,
"grpc": 1236,
},
},
},
}
if err := ts.CreateEndPoints(ctx, cell, "test_keyspace", "-10", topo.TYPE_MASTER, endPoints); err != nil {
t.Fatalf("CreateEndPoints(master): %v", err)
}
// Try to create again.
if err := ts.CreateEndPoints(ctx, cell, "test_keyspace", "-10", topo.TYPE_MASTER, endPoints); err != topo.ErrNodeExists {
t.Fatalf("CreateEndPoints(master): err = %v, want topo.ErrNodeExists", err)
}
// Get version.
_, version, err := ts.GetEndPoints(ctx, cell, "test_keyspace", "-10", topo.TYPE_MASTER)
if err != nil {
t.Fatalf("GetEndPoints(master): %v", err)
}
// Make a change.
tmp := endPoints.Entries[0].Uid
endPoints.Entries[0].Uid = tmp + 1
if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", topo.TYPE_MASTER, endPoints, -1); err != nil {
t.Fatalf("UpdateEndPoints(master): %v", err)
}
endPoints.Entries[0].Uid = tmp
// Try to delete with the wrong version.
if err := ts.DeleteEndPoints(ctx, cell, "test_keyspace", "-10", topo.TYPE_MASTER, version); err != topo.ErrBadVersion {
t.Fatalf("DeleteEndPoints: err = %v, want topo.ErrBadVersion", err)
}
// Delete with the correct version.
_, version, err = ts.GetEndPoints(ctx, cell, "test_keyspace", "-10", topo.TYPE_MASTER)
if err != nil {
t.Fatalf("GetEndPoints(master): %v", err)
}
if err := ts.DeleteEndPoints(ctx, cell, "test_keyspace", "-10", topo.TYPE_MASTER, version); err != nil {
t.Fatalf("DeleteEndPoints: %v", err)
}
// Recreate it with an unconditional update.
if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", topo.TYPE_MASTER, endPoints, -1); err != nil {
t.Fatalf("UpdateEndPoints(master): %v", err)
}
if types, err := ts.GetSrvTabletTypesPerShard(ctx, cell, "test_keyspace", "-10"); err != nil || len(types) != 1 || types[0] != topo.TYPE_MASTER {
t.Errorf("GetSrvTabletTypesPerShard(1): %v %v", err, types)
}
// Delete it unconditionally.
if err := ts.DeleteEndPoints(ctx, cell, "test_keyspace", "-10", topo.TYPE_MASTER, -1); err != nil {
t.Fatalf("DeleteEndPoints: %v", err)
}
// Delete the SrvShard.
if err := ts.DeleteSrvShard(ctx, cell, "test_keyspace", "-10"); err != nil {
t.Fatalf("DeleteSrvShard: %v", err)
}
if _, err := ts.GetSrvShard(ctx, cell, "test_keyspace", "-10"); err != topo.ErrNoNode {
t.Errorf("GetSrvShard(deleted) got %v, want ErrNoNode", err)
}
// Re-add endpoints.
if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", topo.TYPE_MASTER, endPoints, -1); err != nil {
t.Fatalf("UpdateEndPoints(master): %v", err)
}
addrs, version, err := ts.GetEndPoints(ctx, cell, "test_keyspace", "-10", topo.TYPE_MASTER)
if err != nil {
t.Errorf("GetEndPoints: %v", err)
}
if len(addrs.Entries) != 1 || addrs.Entries[0].Uid != 1 {
t.Errorf("GetEndPoints(1): %v", addrs)
}
if pm := addrs.Entries[0].PortMap; pm["vt"] != 1234 || pm["mysql"] != 1235 || pm["grpc"] != 1236 {
t.Errorf("GetSrcTabletType(1).PortMap: want %v, got %v", endPoints.Entries[0].PortMap, pm)
}
// Update with the wrong version.
if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", topo.TYPE_MASTER, endPoints, version+1); err != topo.ErrBadVersion {
t.Fatalf("UpdateEndPoints(master): err = %v, want topo.ErrBadVersion", err)
}
// Update with the right version.
if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", topo.TYPE_MASTER, endPoints, version); err != nil {
t.Fatalf("UpdateEndPoints(master): %v", err)
}
// Update existing EndPoints unconditionally.
if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", topo.TYPE_MASTER, endPoints, -1); err != nil {
t.Fatalf("UpdateEndPoints(master): %v", err)
}
if err := ts.DeleteEndPoints(ctx, cell, "test_keyspace", "-10", topo.TYPE_REPLICA, -1); err != topo.ErrNoNode {
t.Errorf("DeleteEndPoints(unknown): %v", err)
}
if err := ts.DeleteEndPoints(ctx, cell, "test_keyspace", "-10", topo.TYPE_MASTER, -1); err != nil {
t.Errorf("DeleteEndPoints(master): %v", err)
}
// test cell/keyspace/shard entries (SrvShard)
srvShard := &pb.SrvShard{
Name: "-10",
KeyRange: newKeyRange3("-10"),
MasterCell: "test",
}
if err := ts.UpdateSrvShard(ctx, cell, "test_keyspace", "-10", srvShard); err != nil {
t.Fatalf("UpdateSrvShard(1): %v", err)
}
if _, err := ts.GetSrvShard(ctx, cell, "test_keyspace", "666"); err != topo.ErrNoNode {
t.Errorf("GetSrvShard(invalid): %v", err)
}
if s, err := ts.GetSrvShard(ctx, cell, "test_keyspace", "-10"); err != nil ||
s.Name != "-10" ||
!key.KeyRangeEqual(s.KeyRange, newKeyRange3("-10")) ||
s.MasterCell != "test" {
t.Errorf("GetSrvShard(valid): %v", err)
}
// test cell/keyspace entries (SrvKeyspace)
srvKeyspace := topo.SrvKeyspace{
Partitions: map[topo.TabletType]*topo.KeyspacePartition{
topo.TYPE_MASTER: &topo.KeyspacePartition{
ShardReferences: []topo.ShardReference{
topo.ShardReference{
Name: "-80",
KeyRange: newKeyRange("-80"),
},
},
},
},
ShardingColumnName: "video_id",
ShardingColumnType: key.KIT_UINT64,
ServedFrom: map[topo.TabletType]string{
topo.TYPE_REPLICA: "other_keyspace",
},
}
if err := ts.UpdateSrvKeyspace(ctx, cell, "test_keyspace", &srvKeyspace); err != nil {
t.Errorf("UpdateSrvKeyspace(1): %v", err)
}
if _, err := ts.GetSrvKeyspace(ctx, cell, "test_keyspace666"); err != topo.ErrNoNode {
t.Errorf("GetSrvKeyspace(invalid): %v", err)
}
if k, err := ts.GetSrvKeyspace(ctx, cell, "test_keyspace"); err != nil ||
len(k.Partitions) != 1 ||
len(k.Partitions[topo.TYPE_MASTER].ShardReferences) != 1 ||
k.Partitions[topo.TYPE_MASTER].ShardReferences[0].Name != "-80" ||
k.Partitions[topo.TYPE_MASTER].ShardReferences[0].KeyRange != newKeyRange("-80") ||
k.ShardingColumnName != "video_id" ||
k.ShardingColumnType != key.KIT_UINT64 ||
k.ServedFrom[topo.TYPE_REPLICA] != "other_keyspace" {
t.Errorf("GetSrvKeyspace(valid): %v %v", err, k)
}
if k, err := ts.GetSrvKeyspaceNames(ctx, cell); err != nil || len(k) != 1 || k[0] != "test_keyspace" {
t.Errorf("GetSrvKeyspaceNames(): %v", err)
}
// check that updating a SrvKeyspace out of the blue works
if err := ts.UpdateSrvKeyspace(ctx, cell, "unknown_keyspace_so_far", &srvKeyspace); err != nil {
t.Fatalf("UpdateSrvKeyspace(2): %v", err)
}
if k, err := ts.GetSrvKeyspace(ctx, cell, "unknown_keyspace_so_far"); err != nil ||
len(k.Partitions) != 1 ||
len(k.Partitions[topo.TYPE_MASTER].ShardReferences) != 1 ||
k.Partitions[topo.TYPE_MASTER].ShardReferences[0].Name != "-80" ||
k.Partitions[topo.TYPE_MASTER].ShardReferences[0].KeyRange != newKeyRange("-80") ||
k.ShardingColumnName != "video_id" ||
k.ShardingColumnType != key.KIT_UINT64 ||
k.ServedFrom[topo.TYPE_REPLICA] != "other_keyspace" {
t.Errorf("GetSrvKeyspace(out of the blue): %v %v", err, *k)
}
// Delete the SrvKeyspace.
if err := ts.DeleteSrvKeyspace(ctx, cell, "unknown_keyspace_so_far"); err != nil {
t.Fatalf("DeleteSrvShard: %v", err)
}
if _, err := ts.GetSrvKeyspace(ctx, cell, "unknown_keyspace_so_far"); err != topo.ErrNoNode {
t.Errorf("GetSrvKeyspace(deleted) got %v, want ErrNoNode", err)
}
}
// rebuildKeyspace should only be used with an action lock on the keyspace
// - otherwise the consistency of the serving graph data can't be
// guaranteed.
//
// Take data from the global keyspace and rebuild the local serving
// copies in each cell.
func rebuildKeyspace(ctx context.Context, log logutil.Logger, ts topo.Server, keyspace string, cells []string, rebuildSrvShards bool) error {
log.Infof("rebuildKeyspace %v", keyspace)
ki, err := ts.GetKeyspace(ctx, keyspace)
if err != nil {
return err
}
var shardCache map[string]*topo.ShardInfo
if rebuildSrvShards {
shards, err := ts.GetShardNames(ctx, keyspace)
if err != nil {
return nil
}
// Rebuild all shards in parallel, save the shards
shardCache = make(map[string]*topo.ShardInfo)
wg := sync.WaitGroup{}
mu := sync.Mutex{}
rec := concurrency.FirstErrorRecorder{}
for _, shard := range shards {
wg.Add(1)
go func(shard string) {
if shardInfo, err := RebuildShard(ctx, log, ts, keyspace, shard, cells); err != nil {
rec.RecordError(fmt.Errorf("RebuildShard failed: %v/%v %v", keyspace, shard, err))
} else {
mu.Lock()
shardCache[shard] = shardInfo
mu.Unlock()
}
wg.Done()
}(shard)
}
wg.Wait()
if rec.HasErrors() {
return rec.Error()
}
} else {
shardCache, err = ts.FindAllShardsInKeyspace(ctx, keyspace)
if err != nil {
return err
}
}
// Build the list of cells to work on: we get the union
// of all the Cells of all the Shards, limited to the provided cells.
//
// srvKeyspaceMap is a map:
// key: cell
// value: topo.SrvKeyspace object being built
srvKeyspaceMap := make(map[string]*topodatapb.SrvKeyspace)
findCellsForRebuild(ki, shardCache, cells, srvKeyspaceMap)
// Then we add the cells from the keyspaces we might be 'ServedFrom'.
for _, ksf := range ki.ServedFroms {
servedFromShards, err := ts.FindAllShardsInKeyspace(ctx, ksf.Keyspace)
if err != nil {
return err
}
findCellsForRebuild(ki, servedFromShards, cells, srvKeyspaceMap)
}
// for each entry in the srvKeyspaceMap map, we do the following:
// - read the SrvShard structures for each shard / cell
// - if not present, build an empty one from global Shard
// - compute the union of the db types (replica, master, ...)
// - sort the shards in the list by range
// - check the ranges are compatible (no hole, covers everything)
for cell, srvKeyspace := range srvKeyspaceMap {
for _, si := range shardCache {
servedTypes := si.GetServedTypesPerCell(cell)
// for each type this shard is supposed to serve,
// add it to srvKeyspace.Partitions
for _, tabletType := range servedTypes {
partition := topoproto.SrvKeyspaceGetPartition(srvKeyspace, tabletType)
if partition == nil {
partition = &topodatapb.SrvKeyspace_KeyspacePartition{
ServedType: tabletType,
}
srvKeyspace.Partitions = append(srvKeyspace.Partitions, partition)
}
partition.ShardReferences = append(partition.ShardReferences, &topodatapb.ShardReference{
Name: si.ShardName(),
KeyRange: si.KeyRange,
})
}
}
if err := orderAndCheckPartitions(cell, srvKeyspace); err != nil {
return err
}
}
// and then finally save the keyspace objects
for cell, srvKeyspace := range srvKeyspaceMap {
log.Infof("updating keyspace serving graph in cell %v for %v", cell, keyspace)
if err := ts.UpdateSrvKeyspace(ctx, cell, keyspace, srvKeyspace); err != nil {
return fmt.Errorf("writing serving data failed: %v", err)
}
}
return nil
}
// CheckWatchSrvKeyspace makes sure WatchSrvKeyspace works as expected
func CheckWatchSrvKeyspace(ctx context.Context, t *testing.T, ts topo.Server) {
cell := getLocalCell(ctx, t, ts)
keyspace := "test_keyspace"
// start watching, should get nil first
notifications, stopWatching, err := ts.WatchSrvKeyspace(ctx, cell, keyspace)
if err != nil {
t.Fatalf("WatchSrvKeyspace failed: %v", err)
}
sk, ok := <-notifications
if !ok || sk != nil {
t.Fatalf("first value is wrong: %v %v", sk, ok)
}
// update the SrvKeyspace, should get a notification
srvKeyspace := &topo.SrvKeyspace{
ShardingColumnName: "test_column",
Partitions: map[topo.TabletType]*topo.KeyspacePartition{
topo.TYPE_RDONLY: &topo.KeyspacePartition{
ShardReferences: []topo.ShardReference{
topo.ShardReference{
Name: "0",
},
},
},
},
ServedFrom: map[topo.TabletType]string{
topo.TYPE_MASTER: "other_keyspace",
},
}
if err := ts.UpdateSrvKeyspace(ctx, cell, keyspace, srvKeyspace); err != nil {
t.Fatalf("UpdateSrvKeyspace failed: %v", err)
}
for {
sk, ok := <-notifications
if !ok {
t.Fatalf("watch channel is closed???")
}
if sk == nil {
// duplicate notification of the first value, that's OK
continue
}
// non-empty value, that one should be ours
if !reflect.DeepEqual(sk, srvKeyspace) {
t.Fatalf("first value is wrong: got %v expected %v", sk, srvKeyspace)
}
break
}
// delete the SrvKeyspace, should get a notification
if err := ts.DeleteSrvKeyspace(ctx, cell, keyspace); err != nil {
t.Fatalf("DeleteSrvKeyspace failed: %v", err)
}
for {
sk, ok := <-notifications
if !ok {
t.Fatalf("watch channel is closed???")
}
if sk == nil {
break
}
// duplicate notification of the first value, that's OK,
// but value better be good.
if !reflect.DeepEqual(srvKeyspace, sk) {
t.Fatalf("duplicate notification value is bad: %v", sk)
}
}
// re-create the value, a bit different, should get a notification
srvKeyspace.SplitShardCount = 2
if err := ts.UpdateSrvKeyspace(ctx, cell, keyspace, srvKeyspace); err != nil {
t.Fatalf("UpdateSrvKeyspace failed: %v", err)
}
for {
sk, ok := <-notifications
if !ok {
t.Fatalf("watch channel is closed???")
}
if sk == nil {
// duplicate notification of the closed value, that's OK
continue
}
// non-empty value, that one should be ours
if !reflect.DeepEqual(srvKeyspace, sk) {
t.Fatalf("value after delete / re-create is wrong: %v %v", sk, ok)
}
break
}
// close the stopWatching channel, should eventually get a closed
// notifications channel too
close(stopWatching)
for {
sk, ok := <-notifications
if !ok {
break
}
if !reflect.DeepEqual(srvKeyspace, sk) {
t.Fatalf("duplicate notification value is bad: %v", sk)
}
}
}
