// waitForInitialValue waits for the initial value of
// /keyspaces/test_keyspace/SrvKeyspace to appear, and match the
// provided srvKeyspace.
func waitForInitialValue(t *testing.T, ts topo.Impl, cell string, srvKeyspace *topodatapb.SrvKeyspace) (changes <-chan *topo.WatchData, cancel func()) {
var current *topo.WatchData
ctx := context.Background()
start := time.Now()
for {
current, changes, cancel = ts.Watch(ctx, cell, "/keyspaces/test_keyspace/SrvKeyspace")
if current.Err == topo.ErrNoNode {
// hasn't appeared yet
if time.Now().Sub(start) > 10*time.Second {
t.Fatalf("time out waiting for file to appear")
}
time.Sleep(10 * time.Millisecond)
continue
}
if current.Err != nil {
t.Fatalf("watch failed: %v", current.Err)
}
// we got a valid result
break
}
got := &topodatapb.SrvKeyspace{}
if err := proto.Unmarshal(current.Contents, got); err != nil {
t.Fatalf("cannot proto-unmarshal data: %v", err)
}
if !proto.Equal(got, srvKeyspace) {
t.Fatalf("got bad data: %v expected: %v", got, srvKeyspace)
}
return changes, cancel
}
func checkKeyspaceLockTimeout(ctx context.Context, t *testing.T, ts topo.Impl) {
lockPath, err := ts.LockKeyspaceForAction(ctx, "test_keyspace", "fake-content")
if err != nil {
t.Fatalf("LockKeyspaceForAction: %v", err)
}
// test we can't take the lock again
fastCtx, cancel := context.WithTimeout(ctx, timeUntilLockIsTaken)
if _, err := ts.LockKeyspaceForAction(fastCtx, "test_keyspace", "unused-fake-content"); err != topo.ErrTimeout {
t.Fatalf("LockKeyspaceForAction(again): %v", err)
}
cancel()
// test we can interrupt taking the lock
interruptCtx, cancel := context.WithCancel(ctx)
go func() {
time.Sleep(timeUntilLockIsTaken)
cancel()
}()
if _, err := ts.LockKeyspaceForAction(interruptCtx, "test_keyspace", "unused-fake-content"); err != topo.ErrInterrupted {
t.Fatalf("LockKeyspaceForAction(interrupted): %v", err)
}
if err := ts.UnlockKeyspaceForAction(ctx, "test_keyspace", lockPath, "fake-results"); err != nil {
t.Fatalf("UnlockKeyspaceForAction(): %v", err)
}
// test we can't unlock again
if err := ts.UnlockKeyspaceForAction(ctx, "test_keyspace", lockPath, "fake-results"); err == nil {
t.Fatalf("UnlockKeyspaceForAction(again) worked")
}
}
// CheckKeyspaceLock checks we can take a keyspace lock as expected.
func CheckKeyspaceLock(ctx context.Context, t *testing.T, ts topo.Impl) {
if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
checkKeyspaceLockTimeout(ctx, t, ts)
checkKeyspaceLockMissing(ctx, t, ts)
checkKeyspaceLockUnblocks(ctx, t, ts)
}
func getLocalCell(ctx context.Context, t *testing.T, ts topo.Impl) string {
cells, err := ts.GetKnownCells(ctx)
if err != nil {
t.Fatalf("GetKnownCells: %v", err)
}
if len(cells) < 1 {
t.Fatalf("provided topo.Impl doesn't have enough cells (need at least 1): %v", cells)
}
return cells[0]
}
// CopyShardReplications will create the ShardReplication objects in
// the destination topo
func CopyShardReplications(ctx context.Context, fromTS, toTS topo.Impl) {
keyspaces, err := fromTS.GetKeyspaces(ctx)
if err != nil {
log.Fatalf("fromTS.GetKeyspaces: %v", err)
}
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
shards, err := fromTS.GetShardNames(ctx, keyspace)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardNames(%v): %v", keyspace, err))
return
}
for _, shard := range shards {
wg.Add(1)
go func(keyspace, shard string) {
defer wg.Done()
// read the source shard to get the cells
s, _, err := fromTS.GetShard(ctx, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShard(%v, %v): %v", keyspace, shard, err))
return
}
for _, cell := range s.Cells {
sri, err := fromTS.GetShardReplication(ctx, cell, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardReplication(%v, %v, %v): %v", cell, keyspace, shard, err))
continue
}
if err := toTS.UpdateShardReplicationFields(ctx, cell, keyspace, shard, func(oldSR *topodatapb.ShardReplication) error {
*oldSR = *sri.ShardReplication
return nil
}); err != nil {
rec.RecordError(fmt.Errorf("UpdateShardReplicationFields(%v, %v, %v): %v", cell, keyspace, shard, err))
}
}
}(keyspace, shard)
}
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
log.Fatalf("copyShards failed: %v", rec.Error())
}
}
// CopyTablets will create the tablets in the destination topo
func CopyTablets(ctx context.Context, fromTS, toTS topo.Impl) {
cells, err := fromTS.GetKnownCells(ctx)
if err != nil {
log.Fatalf("fromTS.GetKnownCells: %v", err)
}
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, cell := range cells {
wg.Add(1)
go func(cell string) {
defer wg.Done()
tabletAliases, err := fromTS.GetTabletsByCell(ctx, cell)
if err != nil {
rec.RecordError(fmt.Errorf("GetTabletsByCell(%v): %v", cell, err))
} else {
for _, tabletAlias := range tabletAliases {
wg.Add(1)
go func(tabletAlias *topodatapb.TabletAlias) {
defer wg.Done()
// read the source tablet
tablet, _, err := fromTS.GetTablet(ctx, tabletAlias)
if err != nil {
rec.RecordError(fmt.Errorf("GetTablet(%v): %v", tabletAlias, err))
return
}
// try to create the destination
err = toTS.CreateTablet(ctx, tablet)
if err == topo.ErrNodeExists {
// update the destination tablet
log.Warningf("tablet %v already exists, updating it", tabletAlias)
_, err = toTS.UpdateTabletFields(ctx, tablet.Alias, func(t *topodatapb.Tablet) error {
*t = *tablet
return nil
})
}
if err != nil {
rec.RecordError(fmt.Errorf("CreateTablet(%v): %v", tabletAlias, err))
return
}
}(tabletAlias)
}
}
}(cell)
}
wg.Wait()
if rec.HasErrors() {
log.Fatalf("copyTablets failed: %v", rec.Error())
}
}
// checkWatchInterrupt tests we can interrupt a watch.
func checkWatchInterrupt(t *testing.T, ts topo.Impl) {
ctx := context.Background()
cell := getLocalCell(ctx, t, ts)
// create some data
srvKeyspace := &topodatapb.SrvKeyspace{
ShardingColumnName: "user_id",
}
if err := ts.UpdateSrvKeyspace(ctx, cell, "test_keyspace", srvKeyspace); err != nil {
t.Fatalf("UpdateSrvKeyspace(1): %v", err)
}
// Start watching, it should work.
changes, cancel := waitForInitialValue(t, ts, cell, srvKeyspace)
// Now cancel the watch.
cancel()
// Make sure we get the topo.ErrInterrupted notification eventually.
for {
wd, ok := <-changes
if !ok {
t.Fatalf("watch channel unexpectedly closed")
}
if wd.Err == topo.ErrInterrupted {
// good
break
}
if wd.Err != nil {
t.Fatalf("bad error returned for deletion: %v", wd.Err)
}
// we got something, better be the right value
got := &topodatapb.SrvKeyspace{}
if err := proto.Unmarshal(wd.Contents, got); err != nil {
t.Fatalf("cannot proto-unmarshal data: %v", err)
}
if got.ShardingColumnName == "user_id" {
// good value
continue
}
t.Fatalf("got unknown SrvKeyspace waiting for deletion: %v", got)
}
// Now the channel should be closed.
if wd, ok := <-changes; ok {
t.Fatalf("got unexpected event after error: %v", wd)
}
// And calling cancel() again should just work.
cancel()
}
// CheckShardLock checks we can take a shard lock
func CheckShardLock(ctx context.Context, t *testing.T, ts topo.Impl) {
if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
if err := ts.CreateShard(ctx, "test_keyspace", "10-20", &pb.Shard{
KeyRange: newKeyRange3("10-20"),
}); err != nil {
t.Fatalf("CreateShard: %v", err)
}
checkShardLockTimeout(ctx, t, ts)
checkShardLockMissing(ctx, t, ts)
checkShardLockUnblocks(ctx, t, ts)
}
func checkListDir(ctx context.Context, t *testing.T, ts topo.Impl, cell string, dirPath string, expected []string) {
entries, err := ts.ListDir(ctx, cell, dirPath)
switch err {
case topo.ErrNoNode:
if len(expected) != 0 {
t.Errorf("ListDir(%v) returned ErrNoNode but was expecting %v", dirPath, expected)
}
case nil:
if !reflect.DeepEqual(entries, expected) {
t.Errorf("ListDir(%v) returned %v but was expecting %v", dirPath, entries, expected)
}
default:
t.Errorf("ListDir(%v) returned unexpected error: %v", dirPath, err)
}
}
// checkKeyspaceLock checks we can take a keyspace lock as expected.
func checkKeyspaceLock(t *testing.T, ts topo.Impl) {
ctx := context.Background()
if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
t.Log("=== checkKeyspaceLockTimeout")
checkKeyspaceLockTimeout(ctx, t, ts)
t.Log("=== checkKeyspaceLockMissing")
checkKeyspaceLockMissing(ctx, t, ts)
t.Log("=== checkKeyspaceLockUnblocks")
checkKeyspaceLockUnblocks(ctx, t, ts)
}
// checkShardLock checks we can take a shard lock
func checkShardLock(t *testing.T, ts topo.Impl) {
ctx := context.Background()
if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
if err := ts.CreateShard(ctx, "test_keyspace", "10-20", &topodatapb.Shard{
KeyRange: newKeyRange("10-20"),
}); err != nil {
t.Fatalf("CreateShard: %v", err)
}
t.Log("=== checkShardLockTimeout")
checkShardLockTimeout(ctx, t, ts)
t.Log("=== checkShardLockMissing")
checkShardLockMissing(ctx, t, ts)
t.Log("=== checkShardLockUnblocks")
checkShardLockUnblocks(ctx, t, ts)
}
// CopyKeyspaces will create the keyspaces in the destination topo
func CopyKeyspaces(ctx context.Context, fromTS, toTS topo.Impl) {
keyspaces, err := fromTS.GetKeyspaces(ctx)
if err != nil {
log.Fatalf("GetKeyspaces: %v", err)
}
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
k, _, err := fromTS.GetKeyspace(ctx, keyspace)
if err != nil {
rec.RecordError(fmt.Errorf("GetKeyspace(%v): %v", keyspace, err))
return
}
if err := toTS.CreateKeyspace(ctx, keyspace, k); err != nil {
if err == topo.ErrNodeExists {
log.Warningf("keyspace %v already exists", keyspace)
} else {
rec.RecordError(fmt.Errorf("CreateKeyspace(%v): %v", keyspace, err))
}
}
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
log.Fatalf("copyKeyspaces failed: %v", rec.Error())
}
}
// checkSrvShardLockUnblocks makes sure that a routine waiting on a lock
// is unblocked when another routine frees the lock
func checkSrvShardLockUnblocks(ctx context.Context, t *testing.T, ts topo.Impl) {
cell := getLocalCell(ctx, t, ts)
unblock := make(chan struct{})
finished := make(chan struct{})
// as soon as we're unblocked, we try to lock the shard
go func() {
<-unblock
lockPath, err := ts.LockSrvShardForAction(ctx, cell, "test_keyspace", "10-20", "fake-content")
if err != nil {
t.Fatalf("LockSrvShardForAction(test, test_keyspace, 10-20) failed: %v", err)
}
if err = ts.UnlockSrvShardForAction(ctx, cell, "test_keyspace", "10-20", lockPath, "fake-results"); err != nil {
t.Fatalf("UnlockSrvShardForAction(test, test_keyspace, 10-20): %v", err)
}
close(finished)
}()
// lock the shard
lockPath2, err := ts.LockSrvShardForAction(ctx, cell, "test_keyspace", "10-20", "fake-content")
if err != nil {
t.Fatalf("LockSrvShardForAction(test, test_keyspace, 10-20) failed: %v", err)
}
// unblock the go routine so it starts waiting
close(unblock)
// sleep for a while so we're sure the go routine is blocking
time.Sleep(timeUntilLockIsTaken)
if err = ts.UnlockSrvShardForAction(ctx, cell, "test_keyspace", "10-20", lockPath2, "fake-results"); err != nil {
t.Fatalf("UnlockSrvShardForAction(test, test_keyspace, 10-20): %v", err)
}
timeout := time.After(10 * time.Second)
select {
case <-finished:
case <-timeout:
t.Fatalf("unlocking timed out")
}
}
// checkSrvVSchema tests the SrvVSchema methods (other than watch).
func checkSrvVSchema(t *testing.T, ts topo.Impl) {
ctx := context.Background()
cell := getLocalCell(ctx, t, ts)
// check GetSrvVSchema returns topo.ErrNoNode if no SrvVSchema
if _, err := ts.GetSrvVSchema(ctx, cell); err != topo.ErrNoNode {
t.Errorf("GetSrvVSchema(not set): %v", err)
}
srvVSchema := &vschemapb.SrvVSchema{
Keyspaces: map[string]*vschemapb.Keyspace{
"test_keyspace": {
Sharded: true,
},
},
}
if err := ts.UpdateSrvVSchema(ctx, cell, srvVSchema); err != nil {
t.Errorf("UpdateSrvVSchema(1): %v", err)
}
if v, err := ts.GetSrvVSchema(ctx, cell); err != nil || !proto.Equal(srvVSchema, v) {
t.Errorf("GetSrvVSchema(valid): %v %v", err, v)
}
}
func checkShardLockMissing(ctx context.Context, t *testing.T, ts topo.Impl) {
// test we can't lock a non-existing shard
if _, err := ts.LockShardForAction(ctx, "test_keyspace", "20-30", "fake-content"); err == nil {
t.Fatalf("LockShardForAction(test_keyspace/20-30) worked for non-existing shard")
}
}
// TopoServerTestSuite runs the full topo.Impl test suite.
// The factory method should return a topo server that has a single cell
// called 'test'.
func TopoServerTestSuite(t *testing.T, factory func() topo.Impl) {
var ts topo.Impl
t.Log("=== checkKeyspace")
ts = factory()
checkKeyspace(t, ts)
ts.Close()
t.Log("=== checkShard")
ts = factory()
checkShard(t, ts)
ts.Close()
t.Log("=== checkTablet")
ts = factory()
checkTablet(t, ts)
ts.Close()
t.Log("=== checkShardReplication")
ts = factory()
checkShardReplication(t, ts)
ts.Close()
t.Log("=== checkSrvKeyspace")
ts = factory()
checkSrvKeyspace(t, ts)
ts.Close()
t.Log("=== checkWatchSrvKeyspace")
ts = factory()
checkWatchSrvKeyspace(t, ts)
ts.Close()
t.Log("=== checkSrvVSchema")
ts = factory()
checkSrvVSchema(t, ts)
ts.Close()
t.Log("=== checkWatchSrvVSchema")
ts = factory()
checkWatchSrvVSchema(t, ts)
ts.Close()
t.Log("=== checkKeyspaceLock")
ts = factory()
checkKeyspaceLock(t, ts)
ts.Close()
t.Log("=== checkShardLock")
ts = factory()
checkShardLock(t, ts)
ts.Close()
t.Log("=== checkVSchema")
ts = factory()
checkVSchema(t, ts)
ts.Close()
}
// CopyShards will create the shards in the destination topo
func CopyShards(ctx context.Context, fromTS, toTS topo.Impl, deleteKeyspaceShards bool) {
keyspaces, err := fromTS.GetKeyspaces(ctx)
if err != nil {
log.Fatalf("fromTS.GetKeyspaces: %v", err)
}
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
shards, err := fromTS.GetShardNames(ctx, keyspace)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardNames(%v): %v", keyspace, err))
return
}
if deleteKeyspaceShards {
if err := toTS.DeleteKeyspaceShards(ctx, keyspace); err != nil {
rec.RecordError(fmt.Errorf("DeleteKeyspaceShards(%v): %v", keyspace, err))
return
}
}
for _, shard := range shards {
wg.Add(1)
go func(keyspace, shard string) {
defer wg.Done()
if err := toTS.CreateShard(ctx, keyspace, shard, &topodatapb.Shard{}); err != nil {
if err == topo.ErrNodeExists {
log.Warningf("shard %v/%v already exists", keyspace, shard)
} else {
rec.RecordError(fmt.Errorf("CreateShard(%v, %v): %v", keyspace, shard, err))
return
}
}
s, _, err := fromTS.GetShard(ctx, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShard(%v, %v): %v", keyspace, shard, err))
return
}
_, toV, err := toTS.GetShard(ctx, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("toTS.GetShard(%v, %v): %v", keyspace, shard, err))
return
}
if _, err := toTS.UpdateShard(ctx, keyspace, shard, s, toV); err != nil {
rec.RecordError(fmt.Errorf("UpdateShard(%v, %v): %v", keyspace, shard, err))
}
}(keyspace, shard)
}
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
log.Fatalf("copyShards failed: %v", rec.Error())
}
}
// checkSrvKeyspace tests the SrvKeyspace methods (other than watch).
func checkSrvKeyspace(t *testing.T, ts topo.Impl) {
ctx := context.Background()
cell := getLocalCell(ctx, t, ts)
// test cell/keyspace entries (SrvKeyspace)
srvKeyspace := &topodatapb.SrvKeyspace{
Partitions: []*topodatapb.SrvKeyspace_KeyspacePartition{
{
ServedType: topodatapb.TabletType_MASTER,
ShardReferences: []*topodatapb.ShardReference{
{
Name: "-80",
KeyRange: &topodatapb.KeyRange{
End: []byte{0x80},
},
},
},
},
},
ShardingColumnName: "video_id",
ShardingColumnType: topodatapb.KeyspaceIdType_UINT64,
ServedFrom: []*topodatapb.SrvKeyspace_ServedFrom{
{
TabletType: topodatapb.TabletType_REPLICA,
Keyspace: "other_keyspace",
},
},
}
if err := ts.UpdateSrvKeyspace(ctx, cell, "test_keyspace", srvKeyspace); err != nil {
t.Errorf("UpdateSrvKeyspace(1): %v", err)
}
if _, err := ts.GetSrvKeyspace(ctx, cell, "test_keyspace666"); err != topo.ErrNoNode {
t.Errorf("GetSrvKeyspace(invalid): %v", err)
}
if k, err := ts.GetSrvKeyspace(ctx, cell, "test_keyspace"); err != nil || !proto.Equal(srvKeyspace, k) {
t.Errorf("GetSrvKeyspace(valid): %v %v", err, k)
}
if k, err := ts.GetSrvKeyspaceNames(ctx, cell); err != nil || len(k) != 1 || k[0] != "test_keyspace" {
t.Errorf("GetSrvKeyspaceNames(): %v", err)
}
// check that updating a SrvKeyspace out of the blue works
if err := ts.UpdateSrvKeyspace(ctx, cell, "unknown_keyspace_so_far", srvKeyspace); err != nil {
t.Fatalf("UpdateSrvKeyspace(2): %v", err)
}
if k, err := ts.GetSrvKeyspace(ctx, cell, "unknown_keyspace_so_far"); err != nil || !proto.Equal(srvKeyspace, k) {
t.Errorf("GetSrvKeyspace(out of the blue): %v %v", err, *k)
}
// Delete the SrvKeyspace.
if err := ts.DeleteSrvKeyspace(ctx, cell, "unknown_keyspace_so_far"); err != nil {
t.Fatalf("DeleteSrvKeyspace: %v", err)
}
if _, err := ts.GetSrvKeyspace(ctx, cell, "unknown_keyspace_so_far"); err != topo.ErrNoNode {
t.Errorf("GetSrvKeyspace(deleted) got %v, want ErrNoNode", err)
}
}
func checkSrvShardLockGeneral(ctx context.Context, t *testing.T, ts topo.Impl) {
cell := getLocalCell(ctx, t, ts)
// make sure we can create the lock even if no directory exists
lockPath, err := ts.LockSrvShardForAction(ctx, cell, "test_keyspace", "10-20", "fake-content")
if err != nil {
t.Fatalf("LockSrvShardForAction: %v", err)
}
if err := ts.UnlockSrvShardForAction(ctx, cell, "test_keyspace", "10-20", lockPath, "fake-results"); err != nil {
t.Fatalf("UnlockShardForAction: %v", err)
}
// now take the lock again after the root exists
lockPath, err = ts.LockSrvShardForAction(ctx, cell, "test_keyspace", "10-20", "fake-content")
if err != nil {
t.Fatalf("LockSrvShardForAction: %v", err)
}
// test we can't take the lock again
fastCtx, cancel := context.WithTimeout(ctx, timeUntilLockIsTaken)
if _, err := ts.LockSrvShardForAction(fastCtx, cell, "test_keyspace", "10-20", "unused-fake-content"); err != topo.ErrTimeout {
t.Fatalf("LockSrvShardForAction(again): %v", err)
}
cancel()
// test we can interrupt taking the lock
interruptCtx, cancel := context.WithCancel(ctx)
go func() {
time.Sleep(timeUntilLockIsTaken)
cancel()
}()
if _, err := ts.LockSrvShardForAction(interruptCtx, cell, "test_keyspace", "10-20", "unused-fake-content"); err != topo.ErrInterrupted {
t.Fatalf("LockSrvShardForAction(interrupted): %v", err)
}
// unlock now
if err := ts.UnlockSrvShardForAction(ctx, cell, "test_keyspace", "10-20", lockPath, "fake-results"); err != nil {
t.Fatalf("UnlockSrvShardForAction(): %v", err)
}
// test we can't unlock again
if err := ts.UnlockSrvShardForAction(ctx, cell, "test_keyspace", "10-20", lockPath, "fake-results"); err == nil {
t.Error("UnlockSrvShardForAction(again) worked")
}
}
// checkShard verifies the Shard operations work correctly
func checkShard(t *testing.T, ts topo.Impl) {
ctx := context.Background()
tts := topo.Server{Impl: ts}
if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
shard := &topodatapb.Shard{
KeyRange: newKeyRange("b0-c0"),
}
if err := ts.CreateShard(ctx, "test_keyspace", "b0-c0", shard); err != nil {
t.Fatalf("CreateShard: %v", err)
}
if err := ts.CreateShard(ctx, "test_keyspace", "b0-c0", shard); err != topo.ErrNodeExists {
t.Errorf("CreateShard called second time, got: %v", err)
}
// Delete shard and see if we can re-create it.
if err := ts.DeleteShard(ctx, "test_keyspace", "b0-c0"); err != nil {
t.Fatalf("DeleteShard: %v", err)
}
if err := ts.CreateShard(ctx, "test_keyspace", "b0-c0", shard); err != nil {
t.Fatalf("CreateShard: %v", err)
}
// Delete ALL shards.
if err := ts.DeleteKeyspaceShards(ctx, "test_keyspace"); err != nil {
t.Fatalf("DeleteKeyspaceShards: %v", err)
}
if err := ts.CreateShard(ctx, "test_keyspace", "b0-c0", shard); err != nil {
t.Fatalf("CreateShard: %v", err)
}
if _, _, err := ts.GetShard(ctx, "test_keyspace", "666"); err != topo.ErrNoNode {
t.Errorf("GetShard(666): %v", err)
}
shard, version, err := ts.GetShard(ctx, "test_keyspace", "b0-c0")
if err != nil {
t.Errorf("GetShard: %v", err)
}
if want := newKeyRange("b0-c0"); !key.KeyRangeEqual(shard.KeyRange, want) {
t.Errorf("shard.KeyRange: want %v, got %v", want, shard.KeyRange)
}
master := &topodatapb.TabletAlias{Cell: "ny", Uid: 1}
shard.MasterAlias = master
shard.KeyRange = newKeyRange("b0-c0")
shard.ServedTypes = []*topodatapb.Shard_ServedType{
{
TabletType: topodatapb.TabletType_MASTER,
},
{
TabletType: topodatapb.TabletType_REPLICA,
Cells: []string{"c1"},
},
{
TabletType: topodatapb.TabletType_RDONLY,
},
}
shard.SourceShards = []*topodatapb.Shard_SourceShard{
{
Uid: 1,
Keyspace: "source_ks",
Shard: "b8-c0",
KeyRange: newKeyRange("b8-c0"),
Tables: []string{"table1", "table2"},
},
}
shard.TabletControls = []*topodatapb.Shard_TabletControl{
{
TabletType: topodatapb.TabletType_MASTER,
Cells: []string{"c1", "c2"},
BlacklistedTables: []string{"black1", "black2"},
},
{
TabletType: topodatapb.TabletType_REPLICA,
DisableQueryService: true,
},
}
if _, err := ts.UpdateShard(ctx, "test_keyspace", "b0-c0", shard, version); err != nil {
t.Errorf("UpdateShard: %v", err)
}
other := &topodatapb.TabletAlias{Cell: "ny", Uid: 82873}
_, err = tts.UpdateShardFields(ctx, "test_keyspace", "b0-c0", func(si *topo.ShardInfo) error {
si.MasterAlias = other
return nil
})
if err != nil {
t.Fatalf("UpdateShardFields error: %v", err)
}
s, _, err := ts.GetShard(ctx, "test_keyspace", "b0-c0")
if err != nil {
t.Fatalf("GetShard: %v", err)
}
if *s.MasterAlias != *other {
t.Fatalf("shard.MasterAlias = %v, want %v", s.MasterAlias, other)
}
// unconditional shard update
_, err = ts.UpdateShard(ctx, "test_keyspace", "b0-c0", shard, -1)
if err != nil {
t.Fatalf("UpdateShard(-1) error: %v", err)
}
updatedShard, _, err := ts.GetShard(ctx, "test_keyspace", "b0-c0")
if err != nil {
t.Fatalf("GetShard: %v", err)
}
if eq, err := shardEqual(shard, updatedShard); err != nil {
t.Errorf("cannot compare shards: %v", err)
} else if !eq {
t.Errorf("put and got shards are not identical:\n%#v\n%#v", shard, updatedShard)
}
// test GetShardNames
shards, err := ts.GetShardNames(ctx, "test_keyspace")
if err != nil {
t.Errorf("GetShardNames: %v", err)
}
if len(shards) != 1 || shards[0] != "b0-c0" {
t.Errorf(`GetShardNames: want [ "b0-c0" ], got %v`, shards)
}
if _, err := ts.GetShardNames(ctx, "test_keyspace666"); err != topo.ErrNoNode {
t.Errorf("GetShardNames(666): %v", err)
}
// test ValidateShard
if err := ts.ValidateShard(ctx, "test_keyspace", "b0-c0"); err != nil {
t.Errorf("ValidateShard(test_keyspace, b0-c0) failed: %v", err)
}
}
// checkVSchema runs the tests on the VSchema part of the API
func checkVSchema(t *testing.T, ts topo.Impl) {
ctx := context.Background()
if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
shard := &topodatapb.Shard{
KeyRange: newKeyRange("b0-c0"),
}
if err := ts.CreateShard(ctx, "test_keyspace", "b0-c0", shard); err != nil {
t.Fatalf("CreateShard: %v", err)
}
got, err := ts.GetVSchema(ctx, "test_keyspace")
want := &vschemapb.Keyspace{}
if err != topo.ErrNoNode {
t.Error(err)
}
err = ts.SaveVSchema(ctx, "test_keyspace", &vschemapb.Keyspace{
Sharded: true,
Vindexes: map[string]*vschemapb.Vindex{
"stfu1": {
Type: "stfu",
Params: map[string]string{
"stfu1": "1",
},
Owner: "t1",
},
"stln1": {
Type: "stln",
Owner: "t1",
},
},
Tables: map[string]*vschemapb.Table{
"t1": {
ColumnVindexes: []*vschemapb.ColumnVindex{
{
Column: "c1",
Name: "stfu1",
}, {
Column: "c2",
Name: "stln1",
},
},
},
},
})
if err != nil {
t.Error(err)
}
got, err = ts.GetVSchema(ctx, "test_keyspace")
if err != nil {
t.Error(err)
}
want = &vschemapb.Keyspace{
Sharded: true,
Vindexes: map[string]*vschemapb.Vindex{
"stfu1": {
Type: "stfu",
Params: map[string]string{
"stfu1": "1",
},
Owner: "t1",
},
"stln1": {
Type: "stln",
Owner: "t1",
},
},
Tables: map[string]*vschemapb.Table{
"t1": {
ColumnVindexes: []*vschemapb.ColumnVindex{
{
Column: "c1",
Name: "stfu1",
}, {
Column: "c2",
Name: "stln1",
},
},
},
},
}
if !proto.Equal(got, want) {
t.Errorf("GetVSchema: %s, want %s", got, want)
}
err = ts.SaveVSchema(ctx, "test_keyspace", &vschemapb.Keyspace{})
if err != nil {
t.Error(err)
}
got, err = ts.GetVSchema(ctx, "test_keyspace")
if err != nil {
t.Error(err)
}
want = &vschemapb.Keyspace{}
if !proto.Equal(got, want) {
t.Errorf("GetVSchema: %s, want %s", got, want)
}
// Make sure the vschema is not returned as a shard name,
// because they share the same directory location.
shards, err := ts.GetShardNames(ctx, "test_keyspace")
if err != nil {
t.Errorf("GetShardNames: %v", err)
}
if len(shards) != 1 || shards[0] != "b0-c0" {
t.Errorf(`GetShardNames: want [ "b0-c0" ], got %v`, shards)
}
}
// checkKeyspace tests the keyspace part of the API
func checkKeyspace(t *testing.T, ts topo.Impl) {
ctx := context.Background()
keyspaces, err := ts.GetKeyspaces(ctx)
if err != nil {
t.Errorf("GetKeyspaces(empty): %v", err)
}
if len(keyspaces) != 0 {
t.Errorf("len(GetKeyspaces()) != 0: %v", keyspaces)
}
if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil {
t.Errorf("CreateKeyspace: %v", err)
}
if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != topo.ErrNodeExists {
t.Errorf("CreateKeyspace(again) is not ErrNodeExists: %v", err)
}
// Delete and re-create.
if err := ts.DeleteKeyspace(ctx, "test_keyspace"); err != nil {
t.Errorf("DeleteKeyspace: %v", err)
}
if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil {
t.Errorf("CreateKeyspace: %v", err)
}
keyspaces, err = ts.GetKeyspaces(ctx)
if err != nil {
t.Errorf("GetKeyspaces: %v", err)
}
if len(keyspaces) != 1 || keyspaces[0] != "test_keyspace" {
t.Errorf("GetKeyspaces: want %v, got %v", []string{"test_keyspace"}, keyspaces)
}
k := &topodatapb.Keyspace{
ShardingColumnName: "user_id",
ShardingColumnType: topodatapb.KeyspaceIdType_UINT64,
ServedFroms: []*topodatapb.Keyspace_ServedFrom{
{
TabletType: topodatapb.TabletType_REPLICA,
Cells: []string{"c1", "c2"},
Keyspace: "test_keyspace3",
},
{
TabletType: topodatapb.TabletType_MASTER,
Cells: nil,
Keyspace: "test_keyspace3",
},
},
}
if err := ts.CreateKeyspace(ctx, "test_keyspace2", k); err != nil {
t.Errorf("CreateKeyspace: %v", err)
}
keyspaces, err = ts.GetKeyspaces(ctx)
if err != nil {
t.Errorf("GetKeyspaces: %v", err)
}
if len(keyspaces) != 2 ||
keyspaces[0] != "test_keyspace" ||
keyspaces[1] != "test_keyspace2" {
t.Errorf("GetKeyspaces: want %v, got %v", []string{"test_keyspace", "test_keyspace2"}, keyspaces)
}
// re-read and update.
storedK, storedVersion, err := ts.GetKeyspace(ctx, "test_keyspace2")
if err != nil {
t.Fatalf("GetKeyspace: %v", err)
}
storedK.ShardingColumnName = "other_id"
var newServedFroms []*topodatapb.Keyspace_ServedFrom
for _, ksf := range storedK.ServedFroms {
if ksf.TabletType == topodatapb.TabletType_MASTER {
continue
}
if ksf.TabletType == topodatapb.TabletType_REPLICA {
ksf.Keyspace = "test_keyspace4"
}
newServedFroms = append(newServedFroms, ksf)
}
storedK.ServedFroms = newServedFroms
_, err = ts.UpdateKeyspace(ctx, "test_keyspace2", storedK, storedVersion)
if err != nil {
t.Fatalf("UpdateKeyspace: %v", err)
}
// unconditional update
storedK.ShardingColumnType = topodatapb.KeyspaceIdType_BYTES
_, err = ts.UpdateKeyspace(ctx, "test_keyspace2", storedK, -1)
if err != nil {
t.Fatalf("UpdateKeyspace(-1): %v", err)
}
storedK, storedVersion, err = ts.GetKeyspace(ctx, "test_keyspace2")
if err != nil {
t.Fatalf("GetKeyspace: %v", err)
}
if storedK.ShardingColumnName != "other_id" ||
storedK.ShardingColumnType != topodatapb.KeyspaceIdType_BYTES ||
len(storedK.ServedFroms) != 1 ||
storedK.ServedFroms[0].TabletType != topodatapb.TabletType_REPLICA ||
storedK.ServedFroms[0].Keyspace != "test_keyspace4" {
t.Errorf("GetKeyspace: unexpected keyspace, got %v", *storedK)
}
}
// CheckWatchSrvKeyspace makes sure WatchSrvKeyspace works as expected
func CheckWatchSrvKeyspace(ctx context.Context, t *testing.T, ts topo.Impl) {
cell := getLocalCell(ctx, t, ts)
keyspace := "test_keyspace"
// start watching, should get nil first
notifications, stopWatching, err := ts.WatchSrvKeyspace(ctx, cell, keyspace)
if err != nil {
t.Fatalf("WatchSrvKeyspace failed: %v", err)
}
sk, ok := <-notifications
if !ok || sk != nil {
t.Fatalf("first value is wrong: %v %v", sk, ok)
}
// update the SrvKeyspace, should get a notification
srvKeyspace := &topodatapb.SrvKeyspace{
ShardingColumnName: "test_column",
Partitions: []*topodatapb.SrvKeyspace_KeyspacePartition{
&topodatapb.SrvKeyspace_KeyspacePartition{
ServedType: topodatapb.TabletType_RDONLY,
ShardReferences: []*topodatapb.ShardReference{
&topodatapb.ShardReference{
Name: "0",
},
},
},
},
ServedFrom: []*topodatapb.SrvKeyspace_ServedFrom{
&topodatapb.SrvKeyspace_ServedFrom{
TabletType: topodatapb.TabletType_MASTER,
Keyspace: "other_keyspace",
},
},
}
if err := ts.UpdateSrvKeyspace(ctx, cell, keyspace, srvKeyspace); err != nil {
t.Fatalf("UpdateSrvKeyspace failed: %v", err)
}
for {
sk, ok := <-notifications
if !ok {
t.Fatalf("watch channel is closed???")
}
if sk == nil {
// duplicate notification of the first value, that's OK
continue
}
// non-empty value, that one should be ours
if !reflect.DeepEqual(sk, srvKeyspace) {
t.Fatalf("first value is wrong: got %v expected %v", sk, srvKeyspace)
}
break
}
// delete the SrvKeyspace, should get a notification
if err := ts.DeleteSrvKeyspace(ctx, cell, keyspace); err != nil {
t.Fatalf("DeleteSrvKeyspace failed: %v", err)
}
for {
sk, ok := <-notifications
if !ok {
t.Fatalf("watch channel is closed???")
}
if sk == nil {
break
}
// duplicate notification of the first value, that's OK,
// but value better be good.
if !reflect.DeepEqual(srvKeyspace, sk) {
t.Fatalf("duplicate notification value is bad: %v", sk)
}
}
// re-create the value, a bit different, should get a notification
srvKeyspace.SplitShardCount = 2
if err := ts.UpdateSrvKeyspace(ctx, cell, keyspace, srvKeyspace); err != nil {
t.Fatalf("UpdateSrvKeyspace failed: %v", err)
}
for {
sk, ok := <-notifications
if !ok {
t.Fatalf("watch channel is closed???")
}
if sk == nil {
// duplicate notification of the closed value, that's OK
continue
}
// non-empty value, that one should be ours
if !reflect.DeepEqual(srvKeyspace, sk) {
t.Fatalf("value after delete / re-create is wrong: %v %v", sk, ok)
}
break
}
// close the stopWatching channel, should eventually get a closed
// notifications channel too
close(stopWatching)
for {
sk, ok := <-notifications
if !ok {
break
}
if !reflect.DeepEqual(srvKeyspace, sk) {
t.Fatalf("duplicate notification value is bad: %v", sk)
}
}
}
// CheckKeyspace tests the keyspace part of the API
func CheckKeyspace(ctx context.Context, t *testing.T, ts topo.Impl) {
keyspaces, err := ts.GetKeyspaces(ctx)
if err != nil {
t.Errorf("GetKeyspaces(empty): %v", err)
}
if len(keyspaces) != 0 {
t.Errorf("len(GetKeyspaces()) != 0: %v", keyspaces)
}
if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil {
t.Errorf("CreateKeyspace: %v", err)
}
if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != topo.ErrNodeExists {
t.Errorf("CreateKeyspace(again) is not ErrNodeExists: %v", err)
}
// Delete and re-create.
if err := ts.DeleteKeyspace(ctx, "test_keyspace"); err != nil {
t.Errorf("DeleteKeyspace: %v", err)
}
if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil {
t.Errorf("CreateKeyspace: %v", err)
}
keyspaces, err = ts.GetKeyspaces(ctx)
if err != nil {
t.Errorf("GetKeyspaces: %v", err)
}
if len(keyspaces) != 1 || keyspaces[0] != "test_keyspace" {
t.Errorf("GetKeyspaces: want %v, got %v", []string{"test_keyspace"}, keyspaces)
}
k := &pb.Keyspace{
ShardingColumnName: "user_id",
ShardingColumnType: pb.KeyspaceIdType_UINT64,
ServedFroms: []*pb.Keyspace_ServedFrom{
&pb.Keyspace_ServedFrom{
TabletType: pb.TabletType_REPLICA,
Cells: []string{"c1", "c2"},
Keyspace: "test_keyspace3",
},
&pb.Keyspace_ServedFrom{
TabletType: pb.TabletType_MASTER,
Cells: nil,
Keyspace: "test_keyspace3",
},
},
SplitShardCount: 64,
}
if err := ts.CreateKeyspace(ctx, "test_keyspace2", k); err != nil {
t.Errorf("CreateKeyspace: %v", err)
}
keyspaces, err = ts.GetKeyspaces(ctx)
if err != nil {
t.Errorf("GetKeyspaces: %v", err)
}
if len(keyspaces) != 2 ||
keyspaces[0] != "test_keyspace" ||
keyspaces[1] != "test_keyspace2" {
t.Errorf("GetKeyspaces: want %v, got %v", []string{"test_keyspace", "test_keyspace2"}, keyspaces)
}
// Call delete shards and make sure the keyspace still exists.
if err := ts.DeleteKeyspaceShards(ctx, "test_keyspace2"); err != nil {
t.Errorf("DeleteKeyspaceShards: %v", err)
}
storedK, storedVersion, err := ts.GetKeyspace(ctx, "test_keyspace2")
if err != nil {
t.Fatalf("GetKeyspace: %v", err)
}
if !reflect.DeepEqual(storedK, k) {
t.Fatalf("returned keyspace doesn't match: got %v expected %v", storedK, k)
}
storedK.ShardingColumnName = "other_id"
storedK.ShardingColumnType = pb.KeyspaceIdType_BYTES
var newServedFroms []*pb.Keyspace_ServedFrom
for _, ksf := range storedK.ServedFroms {
if ksf.TabletType == pb.TabletType_MASTER {
continue
}
if ksf.TabletType == pb.TabletType_REPLICA {
ksf.Keyspace = "test_keyspace4"
}
newServedFroms = append(newServedFroms, ksf)
}
storedK.ServedFroms = newServedFroms
_, err = ts.UpdateKeyspace(ctx, "test_keyspace2", storedK, storedVersion)
if err != nil {
t.Fatalf("UpdateKeyspace: %v", err)
}
storedK, storedVersion, err = ts.GetKeyspace(ctx, "test_keyspace2")
if err != nil {
t.Fatalf("GetKeyspace: %v", err)
}
if storedK.ShardingColumnName != "other_id" ||
storedK.ShardingColumnType != pb.KeyspaceIdType_BYTES ||
len(storedK.ServedFroms) != 1 ||
storedK.ServedFroms[0].TabletType != pb.TabletType_REPLICA ||
storedK.ServedFroms[0].Keyspace != "test_keyspace4" {
t.Errorf("GetKeyspace: unexpected keyspace, got %v", *storedK)
}
}
func checkDirectoryInCell(t *testing.T, ts topo.Impl, cell string) {
t.Logf("=== checkDirectoryInCell %v", cell)
ctx := context.Background()
// ListDir root: nothing
checkListDir(ctx, t, ts, cell, "/", nil)
// Create a topolevel entry
version, err := ts.Create(ctx, cell, "/MyFile", []byte{'a'})
if err != nil {
t.Fatalf("cannot create toplevel file: %v", err)
}
// ListDir should return it.
checkListDir(ctx, t, ts, cell, "/", []string{"MyFile"})
// Delete it, it should be gone.
if err := ts.Delete(ctx, cell, "/MyFile", version); err != nil {
t.Fatalf("cannot delete toplevel file: %v", err)
}
checkListDir(ctx, t, ts, cell, "/", nil)
// Create a file 3 layers down.
version, err = ts.Create(ctx, cell, "/types/name/MyFile", []byte{'a'})
if err != nil {
t.Fatalf("cannot create deep file: %v", err)
}
// Check listing at all levels.
checkListDir(ctx, t, ts, cell, "/", []string{"types"})
checkListDir(ctx, t, ts, cell, "/types/", []string{"name"})
checkListDir(ctx, t, ts, cell, "/types/name/", []string{"MyFile"})
// Add a second file
version2, err := ts.Create(ctx, cell, "/types/othername/MyFile", []byte{'a'})
if err != nil {
t.Fatalf("cannot create deep file2: %v", err)
}
// Check entries at all levels
checkListDir(ctx, t, ts, cell, "/", []string{"types"})
checkListDir(ctx, t, ts, cell, "/types/", []string{"name", "othername"})
checkListDir(ctx, t, ts, cell, "/types/name/", []string{"MyFile"})
checkListDir(ctx, t, ts, cell, "/types/othername/", []string{"MyFile"})
// Delete the first file, expect all lists to return the second one.
if err := ts.Delete(ctx, cell, "/types/name/MyFile", version); err != nil {
t.Fatalf("cannot delete deep file: %v", err)
}
checkListDir(ctx, t, ts, cell, "/", []string{"types"})
checkListDir(ctx, t, ts, cell, "/types/", []string{"othername"})
checkListDir(ctx, t, ts, cell, "/types/name/", nil)
checkListDir(ctx, t, ts, cell, "/types/othername/", []string{"MyFile"})
// Delete the second file, expect all lists to return nothing.
if err := ts.Delete(ctx, cell, "/types/othername/MyFile", version2); err != nil {
t.Fatalf("cannot delete second deep file: %v", err)
}
for _, dir := range []string{"/", "/types/", "/types/name/", "/types/othername/"} {
checkListDir(ctx, t, ts, cell, dir, nil)
}
}
// checkKeyspaceLockMissing makes sure we can't lock a non-existing keyspace
func checkKeyspaceLockMissing(ctx context.Context, t *testing.T, ts topo.Impl) {
if _, err := ts.LockKeyspaceForAction(ctx, "test_keyspace_666", "fake-content"); err == nil {
t.Fatalf("LockKeyspaceForAction(test_keyspace_666) worked for non-existing keyspace")
}
}
// CheckServingGraph makes sure the serving graph functions work properly.
func CheckServingGraph(ctx context.Context, t *testing.T, ts topo.Impl) {
cell := getLocalCell(ctx, t, ts)
// test individual cell/keyspace/shard/type entries
if _, err := ts.GetSrvTabletTypesPerShard(ctx, cell, "test_keyspace", "-10"); err != topo.ErrNoNode {
t.Errorf("GetSrvTabletTypesPerShard(invalid): %v", err)
}
if _, _, err := ts.GetEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER); err != topo.ErrNoNode {
t.Errorf("GetEndPoints(invalid): %v", err)
}
endPoints := &topodatapb.EndPoints{
Entries: []*topodatapb.EndPoint{
&topodatapb.EndPoint{
Uid: 1,
Host: "host1",
PortMap: map[string]int32{
"vt": 1234,
"mysql": 1235,
"grpc": 1236,
},
},
},
}
if err := ts.CreateEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, endPoints); err != nil {
t.Fatalf("CreateEndPoints(master): %v", err)
}
// Try to create again.
if err := ts.CreateEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, endPoints); err != topo.ErrNodeExists {
t.Fatalf("CreateEndPoints(master): err = %v, want topo.ErrNodeExists", err)
}
// Get version.
_, version, err := ts.GetEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER)
if err != nil {
t.Fatalf("GetEndPoints(master): %v", err)
}
// Make a change.
tmp := endPoints.Entries[0].Uid
endPoints.Entries[0].Uid = tmp + 1
if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, endPoints, -1); err != nil {
t.Fatalf("UpdateEndPoints(master): %v", err)
}
endPoints.Entries[0].Uid = tmp
// Try to delete with the wrong version.
if err := ts.DeleteEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, version); err != topo.ErrBadVersion {
t.Fatalf("DeleteEndPoints: err = %v, want topo.ErrBadVersion", err)
}
// Delete with the correct version.
_, version, err = ts.GetEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER)
if err != nil {
t.Fatalf("GetEndPoints(master): %v", err)
}
if err := ts.DeleteEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, version); err != nil {
t.Fatalf("DeleteEndPoints: %v", err)
}
// Recreate it with an unconditional update.
if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, endPoints, -1); err != nil {
t.Fatalf("UpdateEndPoints(master): %v", err)
}
if types, err := ts.GetSrvTabletTypesPerShard(ctx, cell, "test_keyspace", "-10"); err != nil || len(types) != 1 || types[0] != topodatapb.TabletType_MASTER {
t.Errorf("GetSrvTabletTypesPerShard(1): %v %v", err, types)
}
// Delete it unconditionally.
if err := ts.DeleteEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, -1); err != nil {
t.Fatalf("DeleteEndPoints: %v", err)
}
// Delete the SrvShard.
if err := ts.DeleteSrvShard(ctx, cell, "test_keyspace", "-10"); err != nil {
t.Fatalf("DeleteSrvShard: %v", err)
}
if _, err := ts.GetSrvShard(ctx, cell, "test_keyspace", "-10"); err != topo.ErrNoNode {
t.Errorf("GetSrvShard(deleted) got %v, want ErrNoNode", err)
}
// Re-add endpoints.
if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, endPoints, -1); err != nil {
t.Fatalf("UpdateEndPoints(master): %v", err)
}
addrs, version, err := ts.GetEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER)
if err != nil {
t.Errorf("GetEndPoints: %v", err)
}
if len(addrs.Entries) != 1 || addrs.Entries[0].Uid != 1 {
t.Errorf("GetEndPoints(1): %v", addrs)
}
if pm := addrs.Entries[0].PortMap; pm["vt"] != 1234 || pm["mysql"] != 1235 || pm["grpc"] != 1236 {
t.Errorf("GetSrcTabletType(1).PortMap: want %v, got %v", endPoints.Entries[0].PortMap, pm)
}
// Update with the wrong version.
if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, endPoints, version+1); err != topo.ErrBadVersion {
t.Fatalf("UpdateEndPoints(master): err = %v, want topo.ErrBadVersion", err)
}
// Update with the right version.
if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, endPoints, version); err != nil {
t.Fatalf("UpdateEndPoints(master): %v", err)
}
// Update existing EndPoints unconditionally.
if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, endPoints, -1); err != nil {
t.Fatalf("UpdateEndPoints(master): %v", err)
}
if err := ts.DeleteEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_REPLICA, -1); err != topo.ErrNoNode {
t.Errorf("DeleteEndPoints(unknown): %v", err)
}
if err := ts.DeleteEndPoints(ctx, cell, "test_keyspace", "-10", topodatapb.TabletType_MASTER, -1); err != nil {
t.Errorf("DeleteEndPoints(master): %v", err)
}
// test cell/keyspace/shard entries (SrvShard)
srvShard := &topodatapb.SrvShard{
Name: "-10",
KeyRange: newKeyRange("-10"),
MasterCell: "test",
}
if err := ts.UpdateSrvShard(ctx, cell, "test_keyspace", "-10", srvShard); err != nil {
t.Fatalf("UpdateSrvShard(1): %v", err)
}
if _, err := ts.GetSrvShard(ctx, cell, "test_keyspace", "666"); err != topo.ErrNoNode {
t.Errorf("GetSrvShard(invalid): %v", err)
}
if s, err := ts.GetSrvShard(ctx, cell, "test_keyspace", "-10"); err != nil ||
s.Name != "-10" ||
!key.KeyRangeEqual(s.KeyRange, newKeyRange("-10")) ||
s.MasterCell != "test" {
t.Errorf("GetSrvShard(valid): %v", err)
}
// test cell/keyspace entries (SrvKeyspace)
srvKeyspace := topodatapb.SrvKeyspace{
Partitions: []*topodatapb.SrvKeyspace_KeyspacePartition{
&topodatapb.SrvKeyspace_KeyspacePartition{
ServedType: topodatapb.TabletType_MASTER,
ShardReferences: []*topodatapb.ShardReference{
&topodatapb.ShardReference{
Name: "-80",
KeyRange: &topodatapb.KeyRange{
End: []byte{0x80},
},
},
},
},
},
ShardingColumnName: "video_id",
ShardingColumnType: topodatapb.KeyspaceIdType_UINT64,
ServedFrom: []*topodatapb.SrvKeyspace_ServedFrom{
&topodatapb.SrvKeyspace_ServedFrom{
TabletType: topodatapb.TabletType_REPLICA,
Keyspace: "other_keyspace",
},
},
}
if err := ts.UpdateSrvKeyspace(ctx, cell, "test_keyspace", &srvKeyspace); err != nil {
t.Errorf("UpdateSrvKeyspace(1): %v", err)
}
if _, err := ts.GetSrvKeyspace(ctx, cell, "test_keyspace666"); err != topo.ErrNoNode {
t.Errorf("GetSrvKeyspace(invalid): %v", err)
}
if k, err := ts.GetSrvKeyspace(ctx, cell, "test_keyspace"); err != nil ||
len(k.Partitions) != 1 ||
k.Partitions[0].ServedType != topodatapb.TabletType_MASTER ||
len(k.Partitions[0].ShardReferences) != 1 ||
k.Partitions[0].ShardReferences[0].Name != "-80" ||
key.KeyRangeString(k.Partitions[0].ShardReferences[0].KeyRange) != "-80" ||
k.ShardingColumnName != "video_id" ||
k.ShardingColumnType != topodatapb.KeyspaceIdType_UINT64 ||
len(k.ServedFrom) != 1 ||
k.ServedFrom[0].TabletType != topodatapb.TabletType_REPLICA ||
k.ServedFrom[0].Keyspace != "other_keyspace" {
t.Errorf("GetSrvKeyspace(valid): %v %v", err, k)
}
if k, err := ts.GetSrvKeyspaceNames(ctx, cell); err != nil || len(k) != 1 || k[0] != "test_keyspace" {
t.Errorf("GetSrvKeyspaceNames(): %v", err)
}
// check that updating a SrvKeyspace out of the blue works
if err := ts.UpdateSrvKeyspace(ctx, cell, "unknown_keyspace_so_far", &srvKeyspace); err != nil {
t.Fatalf("UpdateSrvKeyspace(2): %v", err)
}
if k, err := ts.GetSrvKeyspace(ctx, cell, "unknown_keyspace_so_far"); err != nil ||
len(k.Partitions) != 1 ||
k.Partitions[0].ServedType != topodatapb.TabletType_MASTER ||
len(k.Partitions[0].ShardReferences) != 1 ||
k.Partitions[0].ShardReferences[0].Name != "-80" ||
key.KeyRangeString(k.Partitions[0].ShardReferences[0].KeyRange) != "-80" ||
k.ShardingColumnName != "video_id" ||
k.ShardingColumnType != topodatapb.KeyspaceIdType_UINT64 ||
len(k.ServedFrom) != 1 ||
k.ServedFrom[0].TabletType != topodatapb.TabletType_REPLICA ||
k.ServedFrom[0].Keyspace != "other_keyspace" {
t.Errorf("GetSrvKeyspace(out of the blue): %v %v", err, *k)
}
// Delete the SrvKeyspace.
if err := ts.DeleteSrvKeyspace(ctx, cell, "unknown_keyspace_so_far"); err != nil {
t.Fatalf("DeleteSrvShard: %v", err)
}
if _, err := ts.GetSrvKeyspace(ctx, cell, "unknown_keyspace_so_far"); err != topo.ErrNoNode {
t.Errorf("GetSrvKeyspace(deleted) got %v, want ErrNoNode", err)
}
}
// CheckVSchema runs the tests on the VSchema part of the API
func CheckVSchema(ctx context.Context, t *testing.T, ts topo.Impl) {
if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
shard := &topodatapb.Shard{
KeyRange: newKeyRange("b0-c0"),
}
if err := ts.CreateShard(ctx, "test_keyspace", "b0-c0", shard); err != nil {
t.Fatalf("CreateShard: %v", err)
}
got, err := ts.GetVSchema(ctx, "test_keyspace")
if err != nil {
t.Error(err)
}
want := "{}"
if got != want {
t.Errorf("GetVSchema: %s, want %s", got, want)
}
err = ts.SaveVSchema(ctx, "test_keyspace", `{ "Sharded": true }`)
if err != nil {
t.Error(err)
}
got, err = ts.GetVSchema(ctx, "test_keyspace")
if err != nil {
t.Error(err)
}
want = `{ "Sharded": true }`
if got != want {
t.Errorf("GetVSchema: %s, want %s", got, want)
}
err = ts.SaveVSchema(ctx, "test_keyspace", `{ "Sharded": false }`)
if err != nil {
t.Error(err)
}
got, err = ts.GetVSchema(ctx, "test_keyspace")
if err != nil {
t.Error(err)
}
want = `{ "Sharded": false }`
if got != want {
t.Errorf("GetVSchema: %s, want %s", got, want)
}
err = ts.SaveVSchema(ctx, "test_keyspace", "invalid")
want = "Unmarshal failed:"
if err == nil || !strings.HasPrefix(err.Error(), want) {
t.Errorf("SaveVSchema: %v, must start with %s", err, want)
}
// Make sure the vschema is not returned as a shard name,
// because they share the same directory location.
shards, err := ts.GetShardNames(ctx, "test_keyspace")
if err != nil {
t.Errorf("GetShardNames: %v", err)
}
if len(shards) != 1 || shards[0] != "b0-c0" {
t.Errorf(`GetShardNames: want [ "b0-c0" ], got %v`, shards)
}
}
// CheckTablet verifies the topo server API is correct for managing tablets.
func CheckTablet(ctx context.Context, t *testing.T, ts topo.Impl) {
tts := topo.Server{Impl: ts}
cell := getLocalCell(ctx, t, ts)
tablet := &topodatapb.Tablet{
Alias: &topodatapb.TabletAlias{Cell: cell, Uid: 1},
Hostname: "localhost",
Ip: "10.11.12.13",
PortMap: map[string]int32{
"vt": 3333,
"mysql": 3334,
},
Tags: map[string]string{"tag": "value"},
Keyspace: "test_keyspace",
Type: topodatapb.TabletType_MASTER,
KeyRange: newKeyRange("-10"),
}
if err := ts.CreateTablet(ctx, tablet); err != nil {
t.Errorf("CreateTablet: %v", err)
}
if err := ts.CreateTablet(ctx, tablet); err != topo.ErrNodeExists {
t.Errorf("CreateTablet(again): %v", err)
}
if _, _, err := ts.GetTablet(ctx, &topodatapb.TabletAlias{Cell: cell, Uid: 666}); err != topo.ErrNoNode {
t.Errorf("GetTablet(666): %v", err)
}
nt, nv, err := ts.GetTablet(ctx, tablet.Alias)
if err != nil {
t.Errorf("GetTablet %v: %v", tablet.Alias, err)
}
if eq, err := tabletEqual(nt, tablet); err != nil {
t.Errorf("cannot compare tablets: %v", err)
} else if !eq {
t.Errorf("put and got tablets are not identical:\n%#v\n%#v", tablet, t)
}
if _, err := ts.GetTabletsByCell(ctx, "666"); err != topo.ErrNoNode {
t.Errorf("GetTabletsByCell(666): %v", err)
}
inCell, err := ts.GetTabletsByCell(ctx, cell)
if err != nil {
t.Errorf("GetTabletsByCell: %v", err)
}
if len(inCell) != 1 || *inCell[0] != *tablet.Alias {
t.Errorf("GetTabletsByCell: want [%v], got %v", tablet.Alias, inCell)
}
nt.Hostname = "remotehost"
if _, err := ts.UpdateTablet(ctx, nt, nv); err != nil {
t.Errorf("UpdateTablet: %v", err)
}
nt, nv, err = ts.GetTablet(ctx, tablet.Alias)
if err != nil {
t.Errorf("GetTablet %v: %v", tablet.Alias, err)
}
if want := "remotehost"; nt.Hostname != want {
t.Errorf("nt.Hostname: want %v, got %v", want, nt.Hostname)
}
// unconditional tablet update
nt.Hostname = "remotehost2"
if _, err := ts.UpdateTablet(ctx, nt, -1); err != nil {
t.Errorf("UpdateTablet(-1): %v", err)
}
nt, nv, err = ts.GetTablet(ctx, tablet.Alias)
if err != nil {
t.Errorf("GetTablet %v: %v", tablet.Alias, err)
}
if want := "remotehost2"; nt.Hostname != want {
t.Errorf("nt.Hostname: want %v, got %v", want, nt.Hostname)
}
// test UpdateTabletFields works
updatedTablet, err := tts.UpdateTabletFields(ctx, tablet.Alias, func(t *topodatapb.Tablet) error {
t.Hostname = "anotherhost"
return nil
})
if err != nil {
t.Errorf("UpdateTabletFields: %v", err)
}
if got, want := updatedTablet.Hostname, "anotherhost"; got != want {
t.Errorf("updatedTablet.Hostname = %q, want %q", got, want)
}
nt, nv, err = ts.GetTablet(ctx, tablet.Alias)
if err != nil {
t.Errorf("GetTablet %v: %v", tablet.Alias, err)
}
if got, want := nt.Hostname, "anotherhost"; got != want {
t.Errorf("nt.Hostname = %q, want %q", got, want)
}
// test UpdateTabletFields that returns ErrNoUpdateNeeded works
if _, err := tts.UpdateTabletFields(ctx, tablet.Alias, func(t *topodatapb.Tablet) error {
return topo.ErrNoUpdateNeeded
}); err != nil {
t.Errorf("UpdateTabletFields: %v", err)
}
if nnt, nnv, nnerr := ts.GetTablet(ctx, tablet.Alias); nnv != nv {
t.Errorf("GetTablet %v: %v %v %v", tablet.Alias, nnt, nnv, nnerr)
}
if want := "anotherhost"; nt.Hostname != want {
t.Errorf("nt.Hostname: want %v, got %v", want, nt.Hostname)
}
if err := ts.DeleteTablet(ctx, tablet.Alias); err != nil {
t.Errorf("DeleteTablet: %v", err)
}
if err := ts.DeleteTablet(ctx, tablet.Alias); err != topo.ErrNoNode {
t.Errorf("DeleteTablet(again): %v", err)
}
if _, _, err := ts.GetTablet(ctx, tablet.Alias); err != topo.ErrNoNode {
t.Errorf("GetTablet: expected error, tablet was deleted: %v", err)
}
}
// checkShardReplication tests ShardReplication objects
func checkShardReplication(t *testing.T, ts topo.Impl) {
ctx := context.Background()
cell := getLocalCell(ctx, t, ts)
if _, err := ts.GetShardReplication(ctx, cell, "test_keyspace", "-10"); err != topo.ErrNoNode {
t.Errorf("GetShardReplication(not there): %v", err)
}
sr := &topodatapb.ShardReplication{
Nodes: []*topodatapb.ShardReplication_Node{
{
TabletAlias: &topodatapb.TabletAlias{
Cell: "c1",
Uid: 1,
},
},
},
}
if err := ts.UpdateShardReplicationFields(ctx, cell, "test_keyspace", "-10", func(oldSr *topodatapb.ShardReplication) error {
return topo.ErrNoUpdateNeeded
}); err != nil {
t.Fatalf("UpdateShardReplicationFields() failed: %v", err)
}
if err := ts.UpdateShardReplicationFields(ctx, cell, "test_keyspace", "-10", func(oldSr *topodatapb.ShardReplication) error {
*oldSr = *sr
return nil
}); err != nil {
t.Fatalf("UpdateShardReplicationFields() failed: %v", err)
}
if sri, err := ts.GetShardReplication(ctx, cell, "test_keyspace", "-10"); err != nil {
t.Errorf("GetShardReplication(new guy) failed: %v", err)
} else {
if len(sri.Nodes) != 1 ||
sri.Nodes[0].TabletAlias.Cell != "c1" ||
sri.Nodes[0].TabletAlias.Uid != 1 {
t.Errorf("GetShardReplication(new guy) returned wrong value: %v", *sri)
}
}
if err := ts.UpdateShardReplicationFields(ctx, cell, "test_keyspace", "-10", func(sr *topodatapb.ShardReplication) error {
sr.Nodes = append(sr.Nodes, &topodatapb.ShardReplication_Node{
TabletAlias: &topodatapb.TabletAlias{
Cell: "c3",
Uid: 3,
},
})
return nil
}); err != nil {
t.Errorf("UpdateShardReplicationFields() failed: %v", err)
}
if sri, err := ts.GetShardReplication(ctx, cell, "test_keyspace", "-10"); err != nil {
t.Errorf("GetShardReplication(after append) failed: %v", err)
} else {
if len(sri.Nodes) != 2 ||
sri.Nodes[0].TabletAlias.Cell != "c1" ||
sri.Nodes[0].TabletAlias.Uid != 1 ||
sri.Nodes[1].TabletAlias.Cell != "c3" ||
sri.Nodes[1].TabletAlias.Uid != 3 {
t.Errorf("GetShardReplication(new guy) returned wrong value: %v", *sri)
}
}
if err := ts.DeleteShardReplication(ctx, cell, "test_keyspace", "-10"); err != nil {
t.Errorf("DeleteShardReplication(existing) failed: %v", err)
}
if err := ts.DeleteShardReplication(ctx, cell, "test_keyspace", "-10"); err != topo.ErrNoNode {
t.Errorf("DeleteShardReplication(again) returned: %v", err)
}
if err := ts.DeleteKeyspaceReplication(ctx, cell, "test_keyspace"); err != nil {
t.Errorf("DeleteKeyspaceReplication(existing) failed: %v", err)
}
if err := ts.DeleteKeyspaceReplication(ctx, cell, "test_keyspace"); err != topo.ErrNoNode {
t.Errorf("DeleteKeyspaceReplication(again) returned: %v", err)
}
}
