// rebuildShardIfNeeded will rebuild the serving graph if we need to func (agent *ActionAgent) rebuildShardIfNeeded(tablet *topo.TabletInfo, targetTabletType topo.TabletType, lockTimeout time.Duration) error { if topo.IsInServingGraph(targetTabletType) { // TODO: interrupted may need to be a global one closed when we exit interrupted := make(chan struct{}) if *topotools.UseSrvShardLocks { // no need to take the shard lock in this case if err := topotools.RebuildShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, topotools.RebuildShardOptions{Cells: []string{tablet.Alias.Cell}, IgnorePartialResult: true}, lockTimeout, interrupted); err != nil { return fmt.Errorf("topotools.RebuildShard returned an error: %v", err) } } else { actionNode := actionnode.RebuildShard() lockPath, err := actionNode.LockShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, lockTimeout, interrupted) if err != nil { return fmt.Errorf("cannot lock shard for rebuild: %v", err) } err = topotools.RebuildShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, topotools.RebuildShardOptions{Cells: []string{tablet.Alias.Cell}, IgnorePartialResult: true}, lockTimeout, interrupted) err = actionNode.UnlockShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, lockPath, err) if err != nil { return fmt.Errorf("UnlockShard returned an error: %v", err) } } } return nil }
// Rebuild the serving and replication rollup data data while locking // out other changes. func (wr *Wrangler) RebuildShardGraph(keyspace, shard string, cells []string) error { if *topotools.UseSrvShardLocks { // no need to take the shard lock in this case return topotools.RebuildShard(wr.ts, keyspace, shard, topotools.RebuildShardOptions{Cells: cells, IgnorePartialResult: false}, wr.lockTimeout, interrupted) } actionNode := actionnode.RebuildShard() lockPath, err := wr.lockShard(keyspace, shard, actionNode) if err != nil { return err } err = topotools.RebuildShard(wr.ts, keyspace, shard, topotools.RebuildShardOptions{Cells: cells, IgnorePartialResult: false}, wr.lockTimeout, interrupted) return wr.unlockShard(keyspace, shard, actionNode, lockPath, err) }
// same as ChangeType, but assume we already have the shard lock, // and do not have the option to force anything. func (wr *Wrangler) changeTypeInternal(tabletAlias topo.TabletAlias, dbType topo.TabletType) error { ti, err := wr.ts.GetTablet(tabletAlias) if err != nil { return err } rebuildRequired := ti.Tablet.IsInServingGraph() // change the type if wr.UseRPCs { if err := wr.ai.RpcChangeType(ti, dbType, wr.ActionTimeout()); err != nil { return err } } else { actionPath, err := wr.ai.ChangeType(ti.Alias, dbType) if err != nil { return err } err = wr.WaitForCompletion(actionPath) if err != nil { return err } } // rebuild if necessary if rebuildRequired { err = topotools.RebuildShard(wr.logger, wr.ts, ti.Keyspace, ti.Shard, []string{ti.Alias.Cell}, wr.lockTimeout, interrupted) if err != nil { return err } } return nil }
func (wr *Wrangler) finishReparent(si *topo.ShardInfo, masterElect *topo.TabletInfo, majorityRestart, leaveMasterReadOnly bool) error { // If the majority of slaves restarted, move ahead. if majorityRestart { if leaveMasterReadOnly { log.Warningf("leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias) } else { log.Infof("marking master-elect read-write %v", masterElect.Alias) actionPath, err := wr.ai.SetReadWrite(masterElect.Alias) if err == nil { err = wr.ai.WaitForCompletion(actionPath, wr.actionTimeout()) } if err != nil { log.Warningf("master master-elect read-write failed, leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias) } } } else { log.Warningf("minority reparent, manual fixes are needed, leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias) } // save the new master in the shard info si.MasterAlias = masterElect.Alias if err := wr.ts.UpdateShard(si); err != nil { log.Errorf("Failed to save new master into shard: %v", err) return err } // We rebuild all the cells, as we may have taken tablets in and // out of the graph. log.Infof("rebuilding shard serving graph data") return topotools.RebuildShard(wr.ts, masterElect.Keyspace, masterElect.Shard, topotools.RebuildShardOptions{IgnorePartialResult: false}, wr.lockTimeout, interrupted) }
func (wr *Wrangler) finishReparent(si *topo.ShardInfo, masterElect *topo.TabletInfo, majorityRestart, leaveMasterReadOnly bool) error { // If the majority of slaves restarted, move ahead. if majorityRestart { if leaveMasterReadOnly { wr.logger.Warningf("leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias) } else { wr.logger.Infof("marking master-elect read-write %v", masterElect.Alias) if err := wr.tmc.SetReadWrite(masterElect, wr.ActionTimeout()); err != nil { wr.logger.Warningf("master master-elect read-write failed, leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias) } } } else { wr.logger.Warningf("minority reparent, manual fixes are needed, leaving master-elect read-only, change with: vtctl SetReadWrite %v", masterElect.Alias) } // save the new master in the shard info si.MasterAlias = masterElect.Alias if err := topo.UpdateShard(wr.ts, si); err != nil { wr.logger.Errorf("Failed to save new master into shard: %v", err) return err } // We rebuild all the cells, as we may have taken tablets in and // out of the graph. wr.logger.Infof("rebuilding shard serving graph data") _, err := topotools.RebuildShard(wr.logger, wr.ts, masterElect.Keyspace, masterElect.Shard, nil, wr.lockTimeout, interrupted) return err }
func (wr *Wrangler) shardExternallyReparentedLocked(keyspace, shard string, masterElectTabletAlias topo.TabletAlias) error { // read the shard, make sure the master is not already good. // critical read, we want up to date info (and the shard is locked). shardInfo, err := wr.ts.GetShardCritical(keyspace, shard) if err != nil { return err } if shardInfo.MasterAlias == masterElectTabletAlias { return fmt.Errorf("master-elect tablet %v is already master", masterElectTabletAlias) } // Read the tablets, make sure the master elect is known to us. // Note we will keep going with a partial tablet map, which usually // happens when a cell is not reachable. After these checks, the // guarantees we'll have are: // - global cell is reachable (we just locked and read the shard) // - the local cell that contains the new master is reachable // (as we're going to check the new master is in the list) // That should be enough. tabletMap, err := topo.GetTabletMapForShard(wr.ts, keyspace, shard) switch err { case nil: // keep going case topo.ErrPartialResult: log.Warningf("Got topo.ErrPartialResult from GetTabletMapForShard, may need to re-init some tablets") default: return err } masterElectTablet, ok := tabletMap[masterElectTabletAlias] if !ok { return fmt.Errorf("master-elect tablet %v not found in replication graph %v/%v %v", masterElectTabletAlias, keyspace, shard, mapKeys(tabletMap)) } // sort the tablets, and handle them slaveTabletMap, masterTabletMap := sortedTabletMap(tabletMap) err = wr.reparentShardExternal(slaveTabletMap, masterTabletMap, masterElectTablet) if err != nil { log.Infof("Skipping shard rebuild with failed reparent") return err } // Compute the list of Cells we need to rebuild: old master and // new master cells. cells := []string{shardInfo.MasterAlias.Cell} if shardInfo.MasterAlias.Cell != masterElectTabletAlias.Cell { cells = append(cells, masterElectTabletAlias.Cell) } // now update the master record in the shard object log.Infof("Updating Shard's MasterAlias record") shardInfo.MasterAlias = masterElectTabletAlias if err = wr.ts.UpdateShard(shardInfo); err != nil { return err } // and rebuild the shard serving graph log.Infof("Rebuilding shard serving graph data") return topotools.RebuildShard(wr.ts, masterElectTablet.Keyspace, masterElectTablet.Shard, cells, wr.lockTimeout, interrupted) }
// rebuildShardIfNeeded will rebuild the serving graph if we need to func (agent *ActionAgent) rebuildShardIfNeeded(tablet *topo.TabletInfo, targetTabletType topo.TabletType) error { if topo.IsInServingGraph(targetTabletType) { // TODO: interrupted may need to be a global one closed when we exit interrupted := make(chan struct{}) // no need to take the shard lock in this case if err := topotools.RebuildShard(logutil.NewConsoleLogger(), agent.TopoServer, tablet.Keyspace, tablet.Shard, []string{tablet.Alias.Cell}, agent.LockTimeout, interrupted); err != nil { return fmt.Errorf("topotools.RebuildShard returned an error: %v", err) } } return nil }
func tabletExternallyReparentedLocked(ts topo.Server, tablet *topo.TabletInfo, actionTimeout, lockTimeout time.Duration, interrupted chan struct{}) (err error) { // read the shard, make sure again the master is not already good. // critical read, we want up to date info (and the shard is locked). shardInfo, err := ts.GetShardCritical(tablet.Keyspace, tablet.Shard) if err != nil { return err } if shardInfo.MasterAlias == tablet.Alias { return fmt.Errorf("this tablet is already the master") } // Read the tablets, make sure the master elect is known to the shard // (it's this tablet, so it better be!). // Note we will keep going with a partial tablet map, which usually // happens when a cell is not reachable. After these checks, the // guarantees we'll have are: // - global cell is reachable (we just locked and read the shard) // - the local cell that contains the new master is reachable // (as we're going to check the new master is in the list) // That should be enough. tabletMap, err := topo.GetTabletMapForShard(ts, tablet.Keyspace, tablet.Shard) switch err { case nil: // keep going case topo.ErrPartialResult: log.Warningf("Got topo.ErrPartialResult from GetTabletMapForShard, may need to re-init some tablets") default: return err } masterElectTablet, ok := tabletMap[tablet.Alias] if !ok { return fmt.Errorf("this master-elect tablet %v not found in replication graph %v/%v %v", tablet.Alias, tablet.Keyspace, tablet.Shard, topotools.MapKeys(tabletMap)) } // Create reusable Reparent event with available info ev := &events.Reparent{ ShardInfo: *shardInfo, NewMaster: *tablet.Tablet, } if oldMasterTablet, ok := tabletMap[shardInfo.MasterAlias]; ok { ev.OldMaster = *oldMasterTablet.Tablet } defer func() { if err != nil { event.DispatchUpdate(ev, "failed: "+err.Error()) } }() // sort the tablets, and handle them slaveTabletMap, masterTabletMap := topotools.SortedTabletMap(tabletMap) event.DispatchUpdate(ev, "starting external from tablet") // we fix the new master in the replication graph event.DispatchUpdate(ev, "mark ourself as new master") err = updateReplicationGraphForPromotedSlave(ts, tablet) if err != nil { // This suggests we can't talk to topo server. This is bad. return fmt.Errorf("updateReplicationGraphForPromotedSlave failed: %v", err) } // Once this tablet is promoted, remove it from our maps delete(slaveTabletMap, tablet.Alias) delete(masterTabletMap, tablet.Alias) // Then fix all the slaves, including the old master. This // last step is very likely to time out for some tablets (one // random guy is dead, the old master is dead, ...). We // execute them all in parallel until we get to // wr.ActionTimeout(). After this, no other action with a // timeout is executed, so even if we got to the timeout, // we're still good. event.DispatchUpdate(ev, "restarting slaves") logger := logutil.NewConsoleLogger() ai := initiator.NewActionInitiator(ts) topotools.RestartSlavesExternal(ts, logger, slaveTabletMap, masterTabletMap, masterElectTablet.Alias, func(ti *topo.TabletInfo, swrd *actionnode.SlaveWasRestartedArgs) error { return ai.RpcSlaveWasRestarted(ti, swrd, actionTimeout) }) // Compute the list of Cells we need to rebuild: old master and // all other cells if reparenting to another cell. cells := []string{shardInfo.MasterAlias.Cell} if shardInfo.MasterAlias.Cell != tablet.Alias.Cell { cells = nil } // now update the master record in the shard object event.DispatchUpdate(ev, "updating shard record") log.Infof("Updating Shard's MasterAlias record") shardInfo.MasterAlias = tablet.Alias if err = topo.UpdateShard(ts, shardInfo); err != nil { return err } // and rebuild the shard serving graph event.DispatchUpdate(ev, "rebuilding shard serving graph") log.Infof("Rebuilding shard serving graph data") if err = topotools.RebuildShard(logger, ts, tablet.Keyspace, tablet.Shard, cells, lockTimeout, interrupted); err != nil { return err } event.DispatchUpdate(ev, "finished") return nil }
// RunHealthCheck takes the action mutex, runs the health check, // and if we need to change our state, do it. // If we are the master, we don't change our type, healthy or not. // If we are not the master, we change to spare if not healthy, // or to the passed in targetTabletType if healthy. // // Note we only update the topo record if we need to, that is if our type or // health details changed. func (agent *ActionAgent) RunHealthCheck(targetTabletType topo.TabletType, lockTimeout time.Duration) { agent.actionMutex.Lock() defer agent.actionMutex.Unlock() // read the current tablet record agent.mutex.Lock() tablet := agent._tablet agent.mutex.Unlock() // run the health check typeForHealthCheck := targetTabletType if tablet.Type == topo.TYPE_MASTER { typeForHealthCheck = topo.TYPE_MASTER } health, err := health.Run(typeForHealthCheck) // start with no change newTabletType := tablet.Type if err != nil { if tablet.Type != targetTabletType { log.Infof("Tablet not healthy and in state %v, not changing it: %v", tablet.Type, err) return } log.Infof("Tablet not healthy, converting it from %v to spare: %v", targetTabletType, err) newTabletType = topo.TYPE_SPARE } else { // We are healthy, maybe with health, see if we need // to update the record. We only change from spare to // our target type. if tablet.Type == topo.TYPE_SPARE { newTabletType = targetTabletType } if tablet.Type == newTabletType && tablet.IsHealthEqual(health) { // no change in health, not logging anything, // and we're done return } // we need to update our state log.Infof("Updating tablet record as healthy type %v -> %v with health details %v -> %v", tablet.Type, newTabletType, tablet.Health, health) } // Change the Type, update the health. Note we pass in a map // that's not nil, meaning if it's empty, we will clear it. if err := topotools.ChangeType(agent.TopoServer, tablet.Alias, newTabletType, health, true /*runHooks*/); err != nil { log.Infof("Error updating tablet record: %v", err) return } // Rebuild the serving graph in our cell, only if we're dealing with // a serving type if topo.IsInServingGraph(targetTabletType) { // TODO: interrupted may need to be a global one closed when we exit interrupted := make(chan struct{}) if *topotools.UseSrvShardLocks { // no need to take the shard lock in this case if err := topotools.RebuildShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, topotools.RebuildShardOptions{Cells: []string{tablet.Alias.Cell}, IgnorePartialResult: true}, lockTimeout, interrupted); err != nil { log.Warningf("topotools.RebuildShard returned an error: %v", err) return } } else { actionNode := actionnode.RebuildShard() lockPath, err := actionNode.LockShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, lockTimeout, interrupted) if err != nil { log.Warningf("Cannot lock shard for rebuild: %v", err) return } err = topotools.RebuildShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, topotools.RebuildShardOptions{Cells: []string{tablet.Alias.Cell}, IgnorePartialResult: true}, lockTimeout, interrupted) err = actionNode.UnlockShard(agent.TopoServer, tablet.Keyspace, tablet.Shard, lockPath, err) if err != nil { log.Warningf("UnlockShard returned an error: %v", err) return } } } // run the post action callbacks agent.afterAction("healthcheck", false /* reloadSchema */) }
func (wr *Wrangler) shardExternallyReparentedLocked(keyspace, shard string, masterElectTabletAlias topo.TabletAlias) (err error) { // read the shard, make sure the master is not already good. shardInfo, err := wr.ts.GetShard(keyspace, shard) if err != nil { return err } if shardInfo.MasterAlias == masterElectTabletAlias { return fmt.Errorf("master-elect tablet %v is already master", masterElectTabletAlias) } // Read the tablets, make sure the master elect is known to us. // Note we will keep going with a partial tablet map, which usually // happens when a cell is not reachable. After these checks, the // guarantees we'll have are: // - global cell is reachable (we just locked and read the shard) // - the local cell that contains the new master is reachable // (as we're going to check the new master is in the list) // That should be enough. tabletMap, err := topo.GetTabletMapForShard(wr.ts, keyspace, shard) switch err { case nil: // keep going case topo.ErrPartialResult: wr.logger.Warningf("Got topo.ErrPartialResult from GetTabletMapForShard, may need to re-init some tablets") default: return err } masterElectTablet, ok := tabletMap[masterElectTabletAlias] if !ok { return fmt.Errorf("master-elect tablet %v not found in replication graph %v/%v %v", masterElectTabletAlias, keyspace, shard, topotools.MapKeys(tabletMap)) } // Create reusable Reparent event with available info ev := &events.Reparent{ ShardInfo: *shardInfo, NewMaster: *masterElectTablet.Tablet, } if oldMasterTablet, ok := tabletMap[shardInfo.MasterAlias]; ok { ev.OldMaster = *oldMasterTablet.Tablet } defer func() { if err != nil { event.DispatchUpdate(ev, "failed: "+err.Error()) } }() // sort the tablets, and handle them slaveTabletMap, masterTabletMap := topotools.SortedTabletMap(tabletMap) err = wr.reparentShardExternal(ev, slaveTabletMap, masterTabletMap, masterElectTablet) if err != nil { wr.logger.Infof("Skipping shard rebuild with failed reparent") return err } // Compute the list of Cells we need to rebuild: old master and // all other cells if reparenting to another cell. cells := []string{shardInfo.MasterAlias.Cell} if shardInfo.MasterAlias.Cell != masterElectTabletAlias.Cell { cells = nil } // now update the master record in the shard object event.DispatchUpdate(ev, "updating shard record") wr.logger.Infof("Updating Shard's MasterAlias record") shardInfo.MasterAlias = masterElectTabletAlias if err = topo.UpdateShard(wr.ts, shardInfo); err != nil { return err } // and rebuild the shard serving graph event.DispatchUpdate(ev, "rebuilding shard serving graph") wr.logger.Infof("Rebuilding shard serving graph data") if _, err = topotools.RebuildShard(wr.logger, wr.ts, masterElectTablet.Keyspace, masterElectTablet.Shard, cells, wr.lockTimeout, interrupted); err != nil { return err } event.DispatchUpdate(ev, "finished") return nil }
// Rebuild the serving and replication rollup data data while locking // out other changes. func (wr *Wrangler) RebuildShardGraph(keyspace, shard string, cells []string) error { return topotools.RebuildShard(wr.logger, wr.ts, keyspace, shard, cells, wr.lockTimeout, interrupted) }
func TestAPI(t *testing.T) { ctx := context.Background() cells := []string{"cell1", "cell2"} ts := zktopo.NewTestServer(t, cells) actionRepo := NewActionRepository(ts) initAPI(ctx, ts, actionRepo) server := httptest.NewServer(nil) defer server.Close() // Populate topo. ts.CreateKeyspace(ctx, "ks1", &topodatapb.Keyspace{ShardingColumnName: "shardcol"}) ts.Impl.CreateShard(ctx, "ks1", "-80", &topodatapb.Shard{ Cells: cells, KeyRange: &topodatapb.KeyRange{Start: nil, End: []byte{0x80}}, }) ts.Impl.CreateShard(ctx, "ks1", "80-", &topodatapb.Shard{ Cells: cells, KeyRange: &topodatapb.KeyRange{Start: []byte{0x80}, End: nil}, }) ts.CreateTablet(ctx, &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{Cell: "cell1", Uid: 100}, Keyspace: "ks1", Shard: "-80", Type: topodatapb.TabletType_REPLICA, KeyRange: &topodatapb.KeyRange{Start: nil, End: []byte{0x80}}, PortMap: map[string]int32{"vt": 100}, }) ts.CreateTablet(ctx, &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{Cell: "cell2", Uid: 200}, Keyspace: "ks1", Shard: "-80", Type: topodatapb.TabletType_REPLICA, KeyRange: &topodatapb.KeyRange{Start: nil, End: []byte{0x80}}, PortMap: map[string]int32{"vt": 200}, }) topotools.RebuildShard(ctx, logutil.NewConsoleLogger(), ts, "ks1", "-80", cells) // Populate fake actions. actionRepo.RegisterKeyspaceAction("TestKeyspaceAction", func(ctx context.Context, wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "TestKeyspaceAction Result", nil }) actionRepo.RegisterShardAction("TestShardAction", func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "TestShardAction Result", nil }) actionRepo.RegisterTabletAction("TestTabletAction", "", func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias, r *http.Request) (string, error) { return "TestTabletAction Result", nil }) // Test cases. table := []struct { method, path, want string }{ // Cells {"GET", "cells", `["cell1","cell2"]`}, // Keyspaces {"GET", "keyspaces", `["ks1"]`}, {"GET", "keyspaces/ks1", `{ "sharding_column_name": "shardcol" }`}, {"POST", "keyspaces/ks1?action=TestKeyspaceAction", `{ "Name": "TestKeyspaceAction", "Parameters": "ks1", "Output": "TestKeyspaceAction Result", "Error": false }`}, // Shards {"GET", "shards/ks1/", `["-80","80-"]`}, {"GET", "shards/ks1/-80", `{ "key_range": {"end":"gA=="}, "cells": ["cell1", "cell2"] }`}, {"POST", "shards/ks1/-80?action=TestShardAction", `{ "Name": "TestShardAction", "Parameters": "ks1/-80", "Output": "TestShardAction Result", "Error": false }`}, // Tablets {"GET", "tablets/?shard=ks1%2F-80", `[ {"cell":"cell1","uid":100}, {"cell":"cell2","uid":200} ]`}, {"GET", "tablets/?cell=cell1", `[ {"cell":"cell1","uid":100} ]`}, {"GET", "tablets/?shard=ks1%2F-80&cell=cell2", `[ {"cell":"cell2","uid":200} ]`}, {"GET", "tablets/cell1-100", `{ "alias": {"cell": "cell1", "uid": 100}, "port_map": {"vt": 100}, "keyspace": "ks1", "shard": "-80", "key_range": {"end": "gA=="}, "type": 2 }`}, {"POST", "tablets/cell1-100?action=TestTabletAction", `{ "Name": "TestTabletAction", "Parameters": "cell1-0000000100", "Output": "TestTabletAction Result", "Error": false }`}, // EndPoints {"GET", "endpoints/cell1/ks1/-80/", `[2]`}, {"GET", "endpoints/cell1/ks1/-80/replica", `{ "entries": [{ "uid": 100, "port_map": {"vt": 100} }] }`}, } for _, in := range table { var resp *http.Response var err error switch in.method { case "GET": resp, err = http.Get(server.URL + apiPrefix + in.path) case "POST": resp, err = http.Post(server.URL+apiPrefix+in.path, "", nil) default: t.Errorf("[%v] unknown method: %v", in.path, in.method) continue } if err != nil { t.Errorf("[%v] http error: %v", in.path, err) continue } body, err := ioutil.ReadAll(resp.Body) resp.Body.Close() if err != nil { t.Errorf("[%v] ioutil.ReadAll(resp.Body) error: %v", in.path, err) continue } if got, want := compactJSON(body), compactJSON([]byte(in.want)); got != want { t.Errorf("[%v] got %v, want %v", in.path, got, want) } } }
// RebuildShardGraph rebuilds the serving and replication rollup data data while locking // out other changes. func (wr *Wrangler) RebuildShardGraph(ctx context.Context, keyspace, shard string, cells []string) (*topo.ShardInfo, error) { return topotools.RebuildShard(ctx, wr.logger, wr.ts, keyspace, shard, cells, wr.lockTimeout) }