// finalizeTabletExternallyReparented performs slow, synchronized reconciliation
// tasks that ensure topology is self-consistent, and then marks the reparent as
// finished by updating the global shard record.
func (agent *ActionAgent) finalizeTabletExternallyReparented(ctx context.Context, si *topo.ShardInfo, ev *events.Reparent) (err error) {
var wg sync.WaitGroup
var errs concurrency.AllErrorRecorder
oldMasterAlias := si.MasterAlias
// Update the tablet records and serving graph for the old and new master concurrently.
event.DispatchUpdate(ev, "updating old and new master tablet records")
log.Infof("finalizeTabletExternallyReparented: updating tablet records")
wg.Add(1)
go func() {
defer wg.Done()
// Update our own record to master.
var updatedTablet *topo.Tablet
err := topo.UpdateTabletFields(ctx, agent.TopoServer, agent.TabletAlias,
func(tablet *topo.Tablet) error {
tablet.Type = topo.TYPE_MASTER
tablet.Health = nil
updatedTablet = tablet
return nil
})
if err != nil {
errs.RecordError(err)
return
}
// Update the serving graph for the tablet.
if updatedTablet != nil {
errs.RecordError(
topotools.UpdateTabletEndpoints(ctx, agent.TopoServer, updatedTablet))
}
}()
if !oldMasterAlias.IsZero() {
wg.Add(1)
go func() {
// Force the old master to spare.
var oldMasterTablet *topo.Tablet
err := topo.UpdateTabletFields(ctx, agent.TopoServer, oldMasterAlias,
func(tablet *topo.Tablet) error {
tablet.Type = topo.TYPE_SPARE
oldMasterTablet = tablet
return nil
})
if err != nil {
errs.RecordError(err)
wg.Done()
return
}
if oldMasterTablet != nil {
// Update the serving graph.
errs.RecordError(
topotools.UpdateTabletEndpoints(ctx, agent.TopoServer, oldMasterTablet))
wg.Done()
// Tell the old master to refresh its state. We don't need to wait for it.
tmc := tmclient.NewTabletManagerClient()
tmc.RefreshState(ctx, topo.NewTabletInfo(oldMasterTablet, -1))
}
}()
}
tablet := agent.Tablet()
// Wait for the tablet records to be updated. At that point, any rebuild will
// see the new master, so we're ready to mark the reparent as done in the
// global shard record.
wg.Wait()
if errs.HasErrors() {
return errs.Error()
}
// Update the master field in the global shard record. We don't use a lock
// here anymore. The lock was only to ensure that the global shard record
// didn't get modified between the time when we read it and the time when we
// write it back. Now we use an update loop pattern to do that instead.
event.DispatchUpdate(ev, "updating global shard record")
log.Infof("finalizeTabletExternallyReparented: updating global shard record")
si, err = topo.UpdateShardFields(ctx, agent.TopoServer, tablet.Keyspace, tablet.Shard, func(shard *topo.Shard) error {
shard.MasterAlias = tablet.Alias
return nil
})
if err != nil {
return err
}
// We already took care of updating the serving graph for the old and new masters.
// All that's left now is in case of a cross-cell reparent, we need to update the
// master cell setting in the SrvShard records of all cells.
if oldMasterAlias.Cell != tablet.Alias.Cell {
event.DispatchUpdate(ev, "rebuilding shard serving graph")
log.Infof("finalizeTabletExternallyReparented: updating SrvShard in all cells for cross-cell reparent")
if err := topotools.UpdateAllSrvShards(ctx, agent.TopoServer, si); err != nil {
return err
}
}
event.DispatchUpdate(ev, "finished")
return nil
}
// CheckShard verifies the Shard operations work correctly
func CheckShard(ctx context.Context, t *testing.T, ts topo.Server) {
if err := ts.CreateKeyspace(ctx, "test_keyspace", &topo.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != nil {
t.Fatalf("CreateShard: %v", err)
}
if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != topo.ErrNodeExists {
t.Errorf("CreateShard called second time, got: %v", err)
}
// Delete shard and see if we can re-create it.
if err := ts.DeleteShard(ctx, "test_keyspace", "b0-c0"); err != nil {
t.Fatalf("DeleteShard: %v", err)
}
if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != nil {
t.Fatalf("CreateShard: %v", err)
}
// Delete ALL shards.
if err := ts.DeleteKeyspaceShards(ctx, "test_keyspace"); err != nil {
t.Fatalf("DeleteKeyspaceShards: %v", err)
}
if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != nil {
t.Fatalf("CreateShard: %v", err)
}
if _, err := topo.GetShard(ctx, ts, "test_keyspace", "666"); err != topo.ErrNoNode {
t.Errorf("GetShard(666): %v", err)
}
shardInfo, err := topo.GetShard(ctx, ts, "test_keyspace", "b0-c0")
if err != nil {
t.Errorf("GetShard: %v", err)
}
if want := newKeyRange("b0-c0"); shardInfo.KeyRange != want {
t.Errorf("shardInfo.KeyRange: want %v, got %v", want, shardInfo.KeyRange)
}
master := topo.TabletAlias{Cell: "ny", Uid: 1}
shardInfo.MasterAlias = master
shardInfo.KeyRange = newKeyRange("b0-c0")
shardInfo.ServedTypesMap = map[topo.TabletType]*topo.ShardServedType{
topo.TYPE_MASTER: &topo.ShardServedType{},
topo.TYPE_REPLICA: &topo.ShardServedType{Cells: []string{"c1"}},
topo.TYPE_RDONLY: &topo.ShardServedType{},
}
shardInfo.SourceShards = []topo.SourceShard{
topo.SourceShard{
Uid: 1,
Keyspace: "source_ks",
Shard: "b8-c0",
KeyRange: newKeyRange("b8-c0"),
Tables: []string{"table1", "table2"},
},
}
shardInfo.TabletControlMap = map[topo.TabletType]*topo.TabletControl{
topo.TYPE_MASTER: &topo.TabletControl{
Cells: []string{"c1", "c2"},
BlacklistedTables: []string{"black1", "black2"},
},
topo.TYPE_REPLICA: &topo.TabletControl{
DisableQueryService: true,
},
}
if err := topo.UpdateShard(ctx, ts, shardInfo); err != nil {
t.Errorf("UpdateShard: %v", err)
}
other := topo.TabletAlias{Cell: "ny", Uid: 82873}
_, err = topo.UpdateShardFields(ctx, ts, "test_keyspace", "b0-c0", func(shard *topo.Shard) error {
shard.MasterAlias = other
return nil
})
if err != nil {
t.Fatalf("UpdateShardFields error: %v", err)
}
si, err := topo.GetShard(ctx, ts, "test_keyspace", "b0-c0")
if err != nil {
t.Fatalf("GetShard: %v", err)
}
if si.MasterAlias != other {
t.Fatalf("shard.MasterAlias = %v, want %v", si.MasterAlias, other)
}
_, err = topo.UpdateShardFields(ctx, ts, "test_keyspace", "b0-c0", func(shard *topo.Shard) error {
shard.MasterAlias = master
return nil
})
if err != nil {
t.Fatalf("UpdateShardFields error: %v", err)
}
updatedShardInfo, err := topo.GetShard(ctx, ts, "test_keyspace", "b0-c0")
if err != nil {
t.Fatalf("GetShard: %v", err)
}
if eq, err := shardEqual(shardInfo.Shard, updatedShardInfo.Shard); err != nil {
t.Errorf("cannot compare shards: %v", err)
} else if !eq {
t.Errorf("put and got shards are not identical:\n%#v\n%#v", shardInfo.Shard, updatedShardInfo.Shard)
}
// test GetShardNames
shards, err := ts.GetShardNames(ctx, "test_keyspace")
if err != nil {
t.Errorf("GetShardNames: %v", err)
}
if len(shards) != 1 || shards[0] != "b0-c0" {
t.Errorf(`GetShardNames: want [ "b0-c0" ], got %v`, shards)
}
if _, err := ts.GetShardNames(ctx, "test_keyspace666"); err != topo.ErrNoNode {
t.Errorf("GetShardNames(666): %v", err)
}
// test ValidateShard
if err := ts.ValidateShard(ctx, "test_keyspace", "b0-c0"); err != nil {
t.Errorf("ValidateShard(test_keyspace, b0-c0) failed: %v", err)
}
}