// rebuildShardIfNeeded will rebuild the serving graph if we need to
func (agent *ActionAgent) rebuildShardIfNeeded(tablet *topo.TabletInfo, targetTabletType topo.TabletType, lockTimeout time.Duration) error {
if topo.IsInServingGraph(targetTabletType) {
// TODO: interrupted may need to be a global one closed when we exit
interrupted := make(chan struct{})
if *topotools.UseSrvShardLocks {
// no need to take the shard lock in this case
if err := topotools.RebuildShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, topotools.RebuildShardOptions{Cells: []string{tablet.Alias.Cell}, IgnorePartialResult: true}, lockTimeout, interrupted); err != nil {
return fmt.Errorf("topotools.RebuildShard returned an error: %v", err)
}
} else {
actionNode := actionnode.RebuildShard()
lockPath, err := actionNode.LockShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, lockTimeout, interrupted)
if err != nil {
return fmt.Errorf("cannot lock shard for rebuild: %v", err)
}
err = topotools.RebuildShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, topotools.RebuildShardOptions{Cells: []string{tablet.Alias.Cell}, IgnorePartialResult: true}, lockTimeout, interrupted)
err = actionNode.UnlockShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, lockPath, err)
if err != nil {
return fmt.Errorf("UnlockShard returned an error: %v", err)
}
}
}
return nil
}
// Rebuild the serving and replication rollup data data while locking
// out other changes.
func (wr *Wrangler) RebuildShardGraph(keyspace, shard string, cells []string) error {
if *topotools.UseSrvShardLocks {
// no need to take the shard lock in this case
return topotools.RebuildShard(wr.ts, keyspace, shard, topotools.RebuildShardOptions{Cells: cells, IgnorePartialResult: false}, wr.lockTimeout, interrupted)
}
actionNode := actionnode.RebuildShard()
lockPath, err := wr.lockShard(keyspace, shard, actionNode)
if err != nil {
return err
}
err = topotools.RebuildShard(wr.ts, keyspace, shard, topotools.RebuildShardOptions{Cells: cells, IgnorePartialResult: false}, wr.lockTimeout, interrupted)
return wr.unlockShard(keyspace, shard, actionNode, lockPath, err)
}
// same as ChangeType, but assume we already have the shard lock,
// and do not have the option to force anything.
func (wr *Wrangler) changeTypeInternal(tabletAlias topo.TabletAlias, dbType topo.TabletType) error {
ti, err := wr.ts.GetTablet(tabletAlias)
if err != nil {
return err
}
rebuildRequired := ti.Tablet.IsInServingGraph()
// change the type
if wr.UseRPCs {
if err := wr.ai.RpcChangeType(ti, dbType, wr.ActionTimeout()); err != nil {
return err
}
} else {
actionPath, err := wr.ai.ChangeType(ti.Alias, dbType)
if err != nil {
return err
}
err = wr.WaitForCompletion(actionPath)
if err != nil {
return err
}
}
// rebuild if necessary
if rebuildRequired {
err = topotools.RebuildShard(wr.logger, wr.ts, ti.Keyspace, ti.Shard, []string{ti.Alias.Cell}, wr.lockTimeout, interrupted)
if err != nil {
return err
}
}
return nil
}
func (wr *Wrangler) finishReparent(si *topo.ShardInfo, masterElect *topo.TabletInfo, majorityRestart, leaveMasterReadOnly bool) error {
// If the majority of slaves restarted, move ahead.
if majorityRestart {
if leaveMasterReadOnly {
log.Warningf("leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias)
} else {
log.Infof("marking master-elect read-write %v", masterElect.Alias)
actionPath, err := wr.ai.SetReadWrite(masterElect.Alias)
if err == nil {
err = wr.ai.WaitForCompletion(actionPath, wr.actionTimeout())
}
if err != nil {
log.Warningf("master master-elect read-write failed, leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias)
}
}
} else {
log.Warningf("minority reparent, manual fixes are needed, leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias)
}
// save the new master in the shard info
si.MasterAlias = masterElect.Alias
if err := wr.ts.UpdateShard(si); err != nil {
log.Errorf("Failed to save new master into shard: %v", err)
return err
}
// We rebuild all the cells, as we may have taken tablets in and
// out of the graph.
log.Infof("rebuilding shard serving graph data")
return topotools.RebuildShard(wr.ts, masterElect.Keyspace, masterElect.Shard, topotools.RebuildShardOptions{IgnorePartialResult: false}, wr.lockTimeout, interrupted)
}
func (wr *Wrangler) finishReparent(si *topo.ShardInfo, masterElect *topo.TabletInfo, majorityRestart, leaveMasterReadOnly bool) error {
// If the majority of slaves restarted, move ahead.
if majorityRestart {
if leaveMasterReadOnly {
wr.logger.Warningf("leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias)
} else {
wr.logger.Infof("marking master-elect read-write %v", masterElect.Alias)
if err := wr.tmc.SetReadWrite(masterElect, wr.ActionTimeout()); err != nil {
wr.logger.Warningf("master master-elect read-write failed, leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias)
}
}
} else {
wr.logger.Warningf("minority reparent, manual fixes are needed, leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias)
}
// save the new master in the shard info
si.MasterAlias = masterElect.Alias
if err := topo.UpdateShard(wr.ts, si); err != nil {
wr.logger.Errorf("Failed to save new master into shard: %v", err)
return err
}
// We rebuild all the cells, as we may have taken tablets in and
// out of the graph.
wr.logger.Infof("rebuilding shard serving graph data")
_, err := topotools.RebuildShard(wr.logger, wr.ts, masterElect.Keyspace, masterElect.Shard, nil, wr.lockTimeout, interrupted)
return err
}
func (wr *Wrangler) shardExternallyReparentedLocked(keyspace, shard string, masterElectTabletAlias topo.TabletAlias) error {
// read the shard, make sure the master is not already good.
// critical read, we want up to date info (and the shard is locked).
shardInfo, err := wr.ts.GetShardCritical(keyspace, shard)
if err != nil {
return err
}
if shardInfo.MasterAlias == masterElectTabletAlias {
return fmt.Errorf("master-elect tablet %v is already master", masterElectTabletAlias)
}
// Read the tablets, make sure the master elect is known to us.
// Note we will keep going with a partial tablet map, which usually
// happens when a cell is not reachable. After these checks, the
// guarantees we'll have are:
// - global cell is reachable (we just locked and read the shard)
// - the local cell that contains the new master is reachable
// (as we're going to check the new master is in the list)
// That should be enough.
tabletMap, err := topo.GetTabletMapForShard(wr.ts, keyspace, shard)
switch err {
case nil:
// keep going
case topo.ErrPartialResult:
log.Warningf("Got topo.ErrPartialResult from GetTabletMapForShard, may need to re-init some tablets")
default:
return err
}
masterElectTablet, ok := tabletMap[masterElectTabletAlias]
if !ok {
return fmt.Errorf("master-elect tablet %v not found in replication graph %v/%v %v", masterElectTabletAlias, keyspace, shard, mapKeys(tabletMap))
}
// sort the tablets, and handle them
slaveTabletMap, masterTabletMap := sortedTabletMap(tabletMap)
err = wr.reparentShardExternal(slaveTabletMap, masterTabletMap, masterElectTablet)
if err != nil {
log.Infof("Skipping shard rebuild with failed reparent")
return err
}
// Compute the list of Cells we need to rebuild: old master and
// new master cells.
cells := []string{shardInfo.MasterAlias.Cell}
if shardInfo.MasterAlias.Cell != masterElectTabletAlias.Cell {
cells = append(cells, masterElectTabletAlias.Cell)
}
// now update the master record in the shard object
log.Infof("Updating Shard's MasterAlias record")
shardInfo.MasterAlias = masterElectTabletAlias
if err = wr.ts.UpdateShard(shardInfo); err != nil {
return err
}
// and rebuild the shard serving graph
log.Infof("Rebuilding shard serving graph data")
return topotools.RebuildShard(wr.ts, masterElectTablet.Keyspace, masterElectTablet.Shard, cells, wr.lockTimeout, interrupted)
}
// rebuildShardIfNeeded will rebuild the serving graph if we need to
func (agent *ActionAgent) rebuildShardIfNeeded(tablet *topo.TabletInfo, targetTabletType topo.TabletType) error {
if topo.IsInServingGraph(targetTabletType) {
// TODO: interrupted may need to be a global one closed when we exit
interrupted := make(chan struct{})
// no need to take the shard lock in this case
if err := topotools.RebuildShard(logutil.NewConsoleLogger(), agent.TopoServer, tablet.Keyspace, tablet.Shard, []string{tablet.Alias.Cell}, agent.LockTimeout, interrupted); err != nil {
return fmt.Errorf("topotools.RebuildShard returned an error: %v", err)
}
}
return nil
}
func tabletExternallyReparentedLocked(ts topo.Server, tablet *topo.TabletInfo, actionTimeout, lockTimeout time.Duration, interrupted chan struct{}) (err error) {
// read the shard, make sure again the master is not already good.
// critical read, we want up to date info (and the shard is locked).
shardInfo, err := ts.GetShardCritical(tablet.Keyspace, tablet.Shard)
if err != nil {
return err
}
if shardInfo.MasterAlias == tablet.Alias {
return fmt.Errorf("this tablet is already the master")
}
// Read the tablets, make sure the master elect is known to the shard
// (it's this tablet, so it better be!).
// Note we will keep going with a partial tablet map, which usually
// happens when a cell is not reachable. After these checks, the
// guarantees we'll have are:
// - global cell is reachable (we just locked and read the shard)
// - the local cell that contains the new master is reachable
// (as we're going to check the new master is in the list)
// That should be enough.
tabletMap, err := topo.GetTabletMapForShard(ts, tablet.Keyspace, tablet.Shard)
switch err {
case nil:
// keep going
case topo.ErrPartialResult:
log.Warningf("Got topo.ErrPartialResult from GetTabletMapForShard, may need to re-init some tablets")
default:
return err
}
masterElectTablet, ok := tabletMap[tablet.Alias]
if !ok {
return fmt.Errorf("this master-elect tablet %v not found in replication graph %v/%v %v", tablet.Alias, tablet.Keyspace, tablet.Shard, topotools.MapKeys(tabletMap))
}
// Create reusable Reparent event with available info
ev := &events.Reparent{
ShardInfo: *shardInfo,
NewMaster: *tablet.Tablet,
}
if oldMasterTablet, ok := tabletMap[shardInfo.MasterAlias]; ok {
ev.OldMaster = *oldMasterTablet.Tablet
}
defer func() {
if err != nil {
event.DispatchUpdate(ev, "failed: "+err.Error())
}
}()
// sort the tablets, and handle them
slaveTabletMap, masterTabletMap := topotools.SortedTabletMap(tabletMap)
event.DispatchUpdate(ev, "starting external from tablet")
// we fix the new master in the replication graph
event.DispatchUpdate(ev, "mark ourself as new master")
err = updateReplicationGraphForPromotedSlave(ts, tablet)
if err != nil {
// This suggests we can't talk to topo server. This is bad.
return fmt.Errorf("updateReplicationGraphForPromotedSlave failed: %v", err)
}
// Once this tablet is promoted, remove it from our maps
delete(slaveTabletMap, tablet.Alias)
delete(masterTabletMap, tablet.Alias)
// Then fix all the slaves, including the old master. This
// last step is very likely to time out for some tablets (one
// random guy is dead, the old master is dead, ...). We
// execute them all in parallel until we get to
// wr.ActionTimeout(). After this, no other action with a
// timeout is executed, so even if we got to the timeout,
// we're still good.
event.DispatchUpdate(ev, "restarting slaves")
logger := logutil.NewConsoleLogger()
ai := initiator.NewActionInitiator(ts)
topotools.RestartSlavesExternal(ts, logger, slaveTabletMap, masterTabletMap, masterElectTablet.Alias, func(ti *topo.TabletInfo, swrd *actionnode.SlaveWasRestartedArgs) error {
return ai.RpcSlaveWasRestarted(ti, swrd, actionTimeout)
})
// Compute the list of Cells we need to rebuild: old master and
// all other cells if reparenting to another cell.
cells := []string{shardInfo.MasterAlias.Cell}
if shardInfo.MasterAlias.Cell != tablet.Alias.Cell {
cells = nil
}
// now update the master record in the shard object
event.DispatchUpdate(ev, "updating shard record")
log.Infof("Updating Shard's MasterAlias record")
shardInfo.MasterAlias = tablet.Alias
if err = topo.UpdateShard(ts, shardInfo); err != nil {
return err
}
// and rebuild the shard serving graph
event.DispatchUpdate(ev, "rebuilding shard serving graph")
log.Infof("Rebuilding shard serving graph data")
if err = topotools.RebuildShard(logger, ts, tablet.Keyspace, tablet.Shard, cells, lockTimeout, interrupted); err != nil {
return err
}
event.DispatchUpdate(ev, "finished")
return nil
}
// RunHealthCheck takes the action mutex, runs the health check,
// and if we need to change our state, do it.
// If we are the master, we don't change our type, healthy or not.
// If we are not the master, we change to spare if not healthy,
// or to the passed in targetTabletType if healthy.
//
// Note we only update the topo record if we need to, that is if our type or
// health details changed.
func (agent *ActionAgent) RunHealthCheck(targetTabletType topo.TabletType, lockTimeout time.Duration) {
agent.actionMutex.Lock()
defer agent.actionMutex.Unlock()
// read the current tablet record
agent.mutex.Lock()
tablet := agent._tablet
agent.mutex.Unlock()
// run the health check
typeForHealthCheck := targetTabletType
if tablet.Type == topo.TYPE_MASTER {
typeForHealthCheck = topo.TYPE_MASTER
}
health, err := health.Run(typeForHealthCheck)
// start with no change
newTabletType := tablet.Type
if err != nil {
if tablet.Type != targetTabletType {
log.Infof("Tablet not healthy and in state %v, not changing it: %v", tablet.Type, err)
return
}
log.Infof("Tablet not healthy, converting it from %v to spare: %v", targetTabletType, err)
newTabletType = topo.TYPE_SPARE
} else {
// We are healthy, maybe with health, see if we need
// to update the record. We only change from spare to
// our target type.
if tablet.Type == topo.TYPE_SPARE {
newTabletType = targetTabletType
}
if tablet.Type == newTabletType && tablet.IsHealthEqual(health) {
// no change in health, not logging anything,
// and we're done
return
}
// we need to update our state
log.Infof("Updating tablet record as healthy type %v -> %v with health details %v -> %v", tablet.Type, newTabletType, tablet.Health, health)
}
// Change the Type, update the health. Note we pass in a map
// that's not nil, meaning if it's empty, we will clear it.
if err := topotools.ChangeType(agent.TopoServer, tablet.Alias, newTabletType, health, true /*runHooks*/); err != nil {
log.Infof("Error updating tablet record: %v", err)
return
}
// Rebuild the serving graph in our cell, only if we're dealing with
// a serving type
if topo.IsInServingGraph(targetTabletType) {
// TODO: interrupted may need to be a global one closed when we exit
interrupted := make(chan struct{})
if *topotools.UseSrvShardLocks {
// no need to take the shard lock in this case
if err := topotools.RebuildShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, topotools.RebuildShardOptions{Cells: []string{tablet.Alias.Cell}, IgnorePartialResult: true}, lockTimeout, interrupted); err != nil {
log.Warningf("topotools.RebuildShard returned an error: %v", err)
return
}
} else {
actionNode := actionnode.RebuildShard()
lockPath, err := actionNode.LockShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, lockTimeout, interrupted)
if err != nil {
log.Warningf("Cannot lock shard for rebuild: %v", err)
return
}
err = topotools.RebuildShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, topotools.RebuildShardOptions{Cells: []string{tablet.Alias.Cell}, IgnorePartialResult: true}, lockTimeout, interrupted)
err = actionNode.UnlockShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, lockPath, err)
if err != nil {
log.Warningf("UnlockShard returned an error: %v", err)
return
}
}
}
// run the post action callbacks
agent.afterAction("healthcheck", false /* reloadSchema */)
}
func (wr *Wrangler) shardExternallyReparentedLocked(keyspace, shard string, masterElectTabletAlias topo.TabletAlias) (err error) {
// read the shard, make sure the master is not already good.
shardInfo, err := wr.ts.GetShard(keyspace, shard)
if err != nil {
return err
}
if shardInfo.MasterAlias == masterElectTabletAlias {
return fmt.Errorf("master-elect tablet %v is already master", masterElectTabletAlias)
}
// Read the tablets, make sure the master elect is known to us.
// Note we will keep going with a partial tablet map, which usually
// happens when a cell is not reachable. After these checks, the
// guarantees we'll have are:
// - global cell is reachable (we just locked and read the shard)
// - the local cell that contains the new master is reachable
// (as we're going to check the new master is in the list)
// That should be enough.
tabletMap, err := topo.GetTabletMapForShard(wr.ts, keyspace, shard)
switch err {
case nil:
// keep going
case topo.ErrPartialResult:
wr.logger.Warningf("Got topo.ErrPartialResult from GetTabletMapForShard, may need to re-init some tablets")
default:
return err
}
masterElectTablet, ok := tabletMap[masterElectTabletAlias]
if !ok {
return fmt.Errorf("master-elect tablet %v not found in replication graph %v/%v %v", masterElectTabletAlias, keyspace, shard, topotools.MapKeys(tabletMap))
}
// Create reusable Reparent event with available info
ev := &events.Reparent{
ShardInfo: *shardInfo,
NewMaster: *masterElectTablet.Tablet,
}
if oldMasterTablet, ok := tabletMap[shardInfo.MasterAlias]; ok {
ev.OldMaster = *oldMasterTablet.Tablet
}
defer func() {
if err != nil {
event.DispatchUpdate(ev, "failed: "+err.Error())
}
}()
// sort the tablets, and handle them
slaveTabletMap, masterTabletMap := topotools.SortedTabletMap(tabletMap)
err = wr.reparentShardExternal(ev, slaveTabletMap, masterTabletMap, masterElectTablet)
if err != nil {
wr.logger.Infof("Skipping shard rebuild with failed reparent")
return err
}
// Compute the list of Cells we need to rebuild: old master and
// all other cells if reparenting to another cell.
cells := []string{shardInfo.MasterAlias.Cell}
if shardInfo.MasterAlias.Cell != masterElectTabletAlias.Cell {
cells = nil
}
// now update the master record in the shard object
event.DispatchUpdate(ev, "updating shard record")
wr.logger.Infof("Updating Shard's MasterAlias record")
shardInfo.MasterAlias = masterElectTabletAlias
if err = topo.UpdateShard(wr.ts, shardInfo); err != nil {
return err
}
// and rebuild the shard serving graph
event.DispatchUpdate(ev, "rebuilding shard serving graph")
wr.logger.Infof("Rebuilding shard serving graph data")
if _, err = topotools.RebuildShard(wr.logger, wr.ts, masterElectTablet.Keyspace, masterElectTablet.Shard, cells, wr.lockTimeout, interrupted); err != nil {
return err
}
event.DispatchUpdate(ev, "finished")
return nil
}
// Rebuild the serving and replication rollup data data while locking
// out other changes.
func (wr *Wrangler) RebuildShardGraph(keyspace, shard string, cells []string) error {
return topotools.RebuildShard(wr.logger, wr.ts, keyspace, shard, cells, wr.lockTimeout, interrupted)
}
func TestAPI(t *testing.T) {
ctx := context.Background()
cells := []string{"cell1", "cell2"}
ts := zktopo.NewTestServer(t, cells)
actionRepo := NewActionRepository(ts)
initAPI(ctx, ts, actionRepo)
server := httptest.NewServer(nil)
defer server.Close()
// Populate topo.
ts.CreateKeyspace(ctx, "ks1", &topodatapb.Keyspace{ShardingColumnName: "shardcol"})
ts.Impl.CreateShard(ctx, "ks1", "-80", &topodatapb.Shard{
Cells: cells,
KeyRange: &topodatapb.KeyRange{Start: nil, End: []byte{0x80}},
})
ts.Impl.CreateShard(ctx, "ks1", "80-", &topodatapb.Shard{
Cells: cells,
KeyRange: &topodatapb.KeyRange{Start: []byte{0x80}, End: nil},
})
ts.CreateTablet(ctx, &topodatapb.Tablet{
Alias: &topodatapb.TabletAlias{Cell: "cell1", Uid: 100},
Keyspace: "ks1",
Shard: "-80",
Type: topodatapb.TabletType_REPLICA,
KeyRange: &topodatapb.KeyRange{Start: nil, End: []byte{0x80}},
PortMap: map[string]int32{"vt": 100},
})
ts.CreateTablet(ctx, &topodatapb.Tablet{
Alias: &topodatapb.TabletAlias{Cell: "cell2", Uid: 200},
Keyspace: "ks1",
Shard: "-80",
Type: topodatapb.TabletType_REPLICA,
KeyRange: &topodatapb.KeyRange{Start: nil, End: []byte{0x80}},
PortMap: map[string]int32{"vt": 200},
})
topotools.RebuildShard(ctx, logutil.NewConsoleLogger(), ts, "ks1", "-80", cells)
// Populate fake actions.
actionRepo.RegisterKeyspaceAction("TestKeyspaceAction",
func(ctx context.Context, wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "TestKeyspaceAction Result", nil
})
actionRepo.RegisterShardAction("TestShardAction",
func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "TestShardAction Result", nil
})
actionRepo.RegisterTabletAction("TestTabletAction", "",
func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias, r *http.Request) (string, error) {
return "TestTabletAction Result", nil
})
// Test cases.
table := []struct {
method, path, want string
}{
// Cells
{"GET", "cells", `["cell1","cell2"]`},
// Keyspaces
{"GET", "keyspaces", `["ks1"]`},
{"GET", "keyspaces/ks1", `{
"sharding_column_name": "shardcol"
}`},
{"POST", "keyspaces/ks1?action=TestKeyspaceAction", `{
"Name": "TestKeyspaceAction",
"Parameters": "ks1",
"Output": "TestKeyspaceAction Result",
"Error": false
}`},
// Shards
{"GET", "shards/ks1/", `["-80","80-"]`},
{"GET", "shards/ks1/-80", `{
"key_range": {"end":"gA=="},
"cells": ["cell1", "cell2"]
}`},
{"POST", "shards/ks1/-80?action=TestShardAction", `{
"Name": "TestShardAction",
"Parameters": "ks1/-80",
"Output": "TestShardAction Result",
"Error": false
}`},
// Tablets
{"GET", "tablets/?shard=ks1%2F-80", `[
{"cell":"cell1","uid":100},
{"cell":"cell2","uid":200}
]`},
{"GET", "tablets/?cell=cell1", `[
{"cell":"cell1","uid":100}
]`},
{"GET", "tablets/?shard=ks1%2F-80&cell=cell2", `[
{"cell":"cell2","uid":200}
]`},
{"GET", "tablets/cell1-100", `{
"alias": {"cell": "cell1", "uid": 100},
"port_map": {"vt": 100},
"keyspace": "ks1",
"shard": "-80",
"key_range": {"end": "gA=="},
"type": 2
}`},
{"POST", "tablets/cell1-100?action=TestTabletAction", `{
"Name": "TestTabletAction",
"Parameters": "cell1-0000000100",
"Output": "TestTabletAction Result",
"Error": false
}`},
// EndPoints
{"GET", "endpoints/cell1/ks1/-80/", `[2]`},
{"GET", "endpoints/cell1/ks1/-80/replica", `{
"entries": [{
"uid": 100,
"port_map": {"vt": 100}
}]
}`},
}
for _, in := range table {
var resp *http.Response
var err error
switch in.method {
case "GET":
resp, err = http.Get(server.URL + apiPrefix + in.path)
case "POST":
resp, err = http.Post(server.URL+apiPrefix+in.path, "", nil)
default:
t.Errorf("[%v] unknown method: %v", in.path, in.method)
continue
}
if err != nil {
t.Errorf("[%v] http error: %v", in.path, err)
continue
}
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
t.Errorf("[%v] ioutil.ReadAll(resp.Body) error: %v", in.path, err)
continue
}
if got, want := compactJSON(body), compactJSON([]byte(in.want)); got != want {
t.Errorf("[%v] got %v, want %v", in.path, got, want)
}
}
}
// RebuildShardGraph rebuilds the serving and replication rollup data data while locking
// out other changes.
func (wr *Wrangler) RebuildShardGraph(ctx context.Context, keyspace, shard string, cells []string) (*topo.ShardInfo, error) {
return topotools.RebuildShard(ctx, wr.logger, wr.ts, keyspace, shard, cells, wr.lockTimeout)
}