// CopyKeyspaces will create the keyspaces in the destination topo func CopyKeyspaces(ctx context.Context, fromTS, toTS topo.Impl) { keyspaces, err := fromTS.GetKeyspaces(ctx) if err != nil { log.Fatalf("GetKeyspaces: %v", err) } wg := sync.WaitGroup{} rec := concurrency.AllErrorRecorder{} for _, keyspace := range keyspaces { wg.Add(1) go func(keyspace string) { defer wg.Done() k, _, err := fromTS.GetKeyspace(ctx, keyspace) if err != nil { rec.RecordError(fmt.Errorf("GetKeyspace(%v): %v", keyspace, err)) return } if err := toTS.CreateKeyspace(ctx, keyspace, k); err != nil { if err == topo.ErrNodeExists { log.Warningf("keyspace %v already exists", keyspace) } else { rec.RecordError(fmt.Errorf("CreateKeyspace(%v): %v", keyspace, err)) } } }(keyspace) } wg.Wait() if rec.HasErrors() { log.Fatalf("copyKeyspaces failed: %v", rec.Error()) } }
// CopyShardReplications will create the ShardReplication objects in // the destination topo func CopyShardReplications(ctx context.Context, fromTS, toTS topo.Impl) { keyspaces, err := fromTS.GetKeyspaces(ctx) if err != nil { log.Fatalf("fromTS.GetKeyspaces: %v", err) } tts := topo.Server{ Impl: toTS, } wg := sync.WaitGroup{} rec := concurrency.AllErrorRecorder{} for _, keyspace := range keyspaces { wg.Add(1) go func(keyspace string) { defer wg.Done() shards, err := fromTS.GetShardNames(ctx, keyspace) if err != nil { rec.RecordError(fmt.Errorf("GetShardNames(%v): %v", keyspace, err)) return } for _, shard := range shards { wg.Add(1) go func(keyspace, shard string) { defer wg.Done() // read the source shard to get the cells s, _, err := fromTS.GetShard(ctx, keyspace, shard) if err != nil { rec.RecordError(fmt.Errorf("GetShard(%v, %v): %v", keyspace, shard, err)) return } for _, cell := range s.Cells { sri, err := fromTS.GetShardReplication(ctx, cell, keyspace, shard) if err != nil { rec.RecordError(fmt.Errorf("GetShardReplication(%v, %v, %v): %v", cell, keyspace, shard, err)) continue } if err := tts.UpdateShardReplicationFields(ctx, cell, keyspace, shard, func(oldSR *topodatapb.ShardReplication) error { *oldSR = *sri.ShardReplication return nil }); err != nil { rec.RecordError(fmt.Errorf("UpdateShardReplicationFields(%v, %v, %v): %v", cell, keyspace, shard, err)) } } }(keyspace, shard) } }(keyspace) } wg.Wait() if rec.HasErrors() { log.Fatalf("copyShards failed: %v", rec.Error()) } }
// CopyShards will create the shards in the destination topo func CopyShards(ctx context.Context, fromTS, toTS topo.Impl) { keyspaces, err := fromTS.GetKeyspaces(ctx) if err != nil { log.Fatalf("fromTS.GetKeyspaces: %v", err) } wg := sync.WaitGroup{} rec := concurrency.AllErrorRecorder{} for _, keyspace := range keyspaces { wg.Add(1) go func(keyspace string) { defer wg.Done() shards, err := fromTS.GetShardNames(ctx, keyspace) if err != nil { rec.RecordError(fmt.Errorf("GetShardNames(%v): %v", keyspace, err)) return } for _, shard := range shards { wg.Add(1) go func(keyspace, shard string) { defer wg.Done() if err := toTS.CreateShard(ctx, keyspace, shard, &topodatapb.Shard{}); err != nil { if err == topo.ErrNodeExists { log.Warningf("shard %v/%v already exists", keyspace, shard) } else { rec.RecordError(fmt.Errorf("CreateShard(%v, %v): %v", keyspace, shard, err)) return } } s, _, err := fromTS.GetShard(ctx, keyspace, shard) if err != nil { rec.RecordError(fmt.Errorf("GetShard(%v, %v): %v", keyspace, shard, err)) return } _, toV, err := toTS.GetShard(ctx, keyspace, shard) if err != nil { rec.RecordError(fmt.Errorf("toTS.GetShard(%v, %v): %v", keyspace, shard, err)) return } if _, err := toTS.UpdateShard(ctx, keyspace, shard, s, toV); err != nil { rec.RecordError(fmt.Errorf("UpdateShard(%v, %v): %v", keyspace, shard, err)) } }(keyspace, shard) } }(keyspace) } wg.Wait() if rec.HasErrors() { log.Fatalf("copyShards failed: %v", rec.Error()) } }
// checkKeyspace tests the keyspace part of the API func checkKeyspace(t *testing.T, ts topo.Impl) { ctx := context.Background() keyspaces, err := ts.GetKeyspaces(ctx) if err != nil { t.Errorf("GetKeyspaces(empty): %v", err) } if len(keyspaces) != 0 { t.Errorf("len(GetKeyspaces()) != 0: %v", keyspaces) } if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil { t.Errorf("CreateKeyspace: %v", err) } if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != topo.ErrNodeExists { t.Errorf("CreateKeyspace(again) is not ErrNodeExists: %v", err) } // Delete and re-create. if err := ts.DeleteKeyspace(ctx, "test_keyspace"); err != nil { t.Errorf("DeleteKeyspace: %v", err) } if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil { t.Errorf("CreateKeyspace: %v", err) } keyspaces, err = ts.GetKeyspaces(ctx) if err != nil { t.Errorf("GetKeyspaces: %v", err) } if len(keyspaces) != 1 || keyspaces[0] != "test_keyspace" { t.Errorf("GetKeyspaces: want %v, got %v", []string{"test_keyspace"}, keyspaces) } k := &topodatapb.Keyspace{ ShardingColumnName: "user_id", ShardingColumnType: topodatapb.KeyspaceIdType_UINT64, ServedFroms: []*topodatapb.Keyspace_ServedFrom{ { TabletType: topodatapb.TabletType_REPLICA, Cells: []string{"c1", "c2"}, Keyspace: "test_keyspace3", }, { TabletType: topodatapb.TabletType_MASTER, Cells: nil, Keyspace: "test_keyspace3", }, }, } if err := ts.CreateKeyspace(ctx, "test_keyspace2", k); err != nil { t.Errorf("CreateKeyspace: %v", err) } keyspaces, err = ts.GetKeyspaces(ctx) if err != nil { t.Errorf("GetKeyspaces: %v", err) } if len(keyspaces) != 2 || keyspaces[0] != "test_keyspace" || keyspaces[1] != "test_keyspace2" { t.Errorf("GetKeyspaces: want %v, got %v", []string{"test_keyspace", "test_keyspace2"}, keyspaces) } // re-read and update. storedK, storedVersion, err := ts.GetKeyspace(ctx, "test_keyspace2") if err != nil { t.Fatalf("GetKeyspace: %v", err) } storedK.ShardingColumnName = "other_id" var newServedFroms []*topodatapb.Keyspace_ServedFrom for _, ksf := range storedK.ServedFroms { if ksf.TabletType == topodatapb.TabletType_MASTER { continue } if ksf.TabletType == topodatapb.TabletType_REPLICA { ksf.Keyspace = "test_keyspace4" } newServedFroms = append(newServedFroms, ksf) } storedK.ServedFroms = newServedFroms _, err = ts.UpdateKeyspace(ctx, "test_keyspace2", storedK, storedVersion) if err != nil { t.Fatalf("UpdateKeyspace: %v", err) } // unconditional update storedK.ShardingColumnType = topodatapb.KeyspaceIdType_BYTES _, err = ts.UpdateKeyspace(ctx, "test_keyspace2", storedK, -1) if err != nil { t.Fatalf("UpdateKeyspace(-1): %v", err) } storedK, storedVersion, err = ts.GetKeyspace(ctx, "test_keyspace2") if err != nil { t.Fatalf("GetKeyspace: %v", err) } if storedK.ShardingColumnName != "other_id" || storedK.ShardingColumnType != topodatapb.KeyspaceIdType_BYTES || len(storedK.ServedFroms) != 1 || storedK.ServedFroms[0].TabletType != topodatapb.TabletType_REPLICA || storedK.ServedFroms[0].Keyspace != "test_keyspace4" { t.Errorf("GetKeyspace: unexpected keyspace, got %v", *storedK) } }
// CheckKeyspace tests the keyspace part of the API func CheckKeyspace(ctx context.Context, t *testing.T, ts topo.Impl) { keyspaces, err := ts.GetKeyspaces(ctx) if err != nil { t.Errorf("GetKeyspaces(empty): %v", err) } if len(keyspaces) != 0 { t.Errorf("len(GetKeyspaces()) != 0: %v", keyspaces) } if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil { t.Errorf("CreateKeyspace: %v", err) } if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != topo.ErrNodeExists { t.Errorf("CreateKeyspace(again) is not ErrNodeExists: %v", err) } // Delete and re-create. if err := ts.DeleteKeyspace(ctx, "test_keyspace"); err != nil { t.Errorf("DeleteKeyspace: %v", err) } if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil { t.Errorf("CreateKeyspace: %v", err) } keyspaces, err = ts.GetKeyspaces(ctx) if err != nil { t.Errorf("GetKeyspaces: %v", err) } if len(keyspaces) != 1 || keyspaces[0] != "test_keyspace" { t.Errorf("GetKeyspaces: want %v, got %v", []string{"test_keyspace"}, keyspaces) } k := &pb.Keyspace{ ShardingColumnName: "user_id", ShardingColumnType: pb.KeyspaceIdType_UINT64, ServedFroms: []*pb.Keyspace_ServedFrom{ &pb.Keyspace_ServedFrom{ TabletType: pb.TabletType_REPLICA, Cells: []string{"c1", "c2"}, Keyspace: "test_keyspace3", }, &pb.Keyspace_ServedFrom{ TabletType: pb.TabletType_MASTER, Cells: nil, Keyspace: "test_keyspace3", }, }, SplitShardCount: 64, } if err := ts.CreateKeyspace(ctx, "test_keyspace2", k); err != nil { t.Errorf("CreateKeyspace: %v", err) } keyspaces, err = ts.GetKeyspaces(ctx) if err != nil { t.Errorf("GetKeyspaces: %v", err) } if len(keyspaces) != 2 || keyspaces[0] != "test_keyspace" || keyspaces[1] != "test_keyspace2" { t.Errorf("GetKeyspaces: want %v, got %v", []string{"test_keyspace", "test_keyspace2"}, keyspaces) } // Call delete shards and make sure the keyspace still exists. if err := ts.DeleteKeyspaceShards(ctx, "test_keyspace2"); err != nil { t.Errorf("DeleteKeyspaceShards: %v", err) } storedK, storedVersion, err := ts.GetKeyspace(ctx, "test_keyspace2") if err != nil { t.Fatalf("GetKeyspace: %v", err) } if !reflect.DeepEqual(storedK, k) { t.Fatalf("returned keyspace doesn't match: got %v expected %v", storedK, k) } storedK.ShardingColumnName = "other_id" storedK.ShardingColumnType = pb.KeyspaceIdType_BYTES var newServedFroms []*pb.Keyspace_ServedFrom for _, ksf := range storedK.ServedFroms { if ksf.TabletType == pb.TabletType_MASTER { continue } if ksf.TabletType == pb.TabletType_REPLICA { ksf.Keyspace = "test_keyspace4" } newServedFroms = append(newServedFroms, ksf) } storedK.ServedFroms = newServedFroms _, err = ts.UpdateKeyspace(ctx, "test_keyspace2", storedK, storedVersion) if err != nil { t.Fatalf("UpdateKeyspace: %v", err) } storedK, storedVersion, err = ts.GetKeyspace(ctx, "test_keyspace2") if err != nil { t.Fatalf("GetKeyspace: %v", err) } if storedK.ShardingColumnName != "other_id" || storedK.ShardingColumnType != pb.KeyspaceIdType_BYTES || len(storedK.ServedFroms) != 1 || storedK.ServedFroms[0].TabletType != pb.TabletType_REPLICA || storedK.ServedFroms[0].Keyspace != "test_keyspace4" { t.Errorf("GetKeyspace: unexpected keyspace, got %v", *storedK) } }