// SetNodeRole sets role for node through control api.
func (c *testCluster) SetNodeRole(id string, role api.NodeRole) error {
node, ok := c.nodes[id]
if !ok {
return fmt.Errorf("set node role: node %s not found", id)
}
if node.IsManager() && role == api.NodeRoleManager {
return fmt.Errorf("node is already manager")
}
if !node.IsManager() && role == api.NodeRoleWorker {
return fmt.Errorf("node is already worker")
}
var initialTimeout time.Duration
// version might change between get and update, so retry
for i := 0; i < 5; i++ {
time.Sleep(initialTimeout)
initialTimeout += 500 * time.Millisecond
resp, err := c.api.GetNode(context.Background(), &api.GetNodeRequest{NodeID: id})
if err != nil {
return err
}
spec := resp.Node.Spec.Copy()
spec.DesiredRole = role
if _, err := c.api.UpdateNode(context.Background(), &api.UpdateNodeRequest{
NodeID: id,
Spec: spec,
NodeVersion: &resp.Node.Meta.Version,
}); err != nil {
// there possible problems on calling update node because redirecting
// node or leader might want to shut down
if grpc.ErrorDesc(err) == "update out of sequence" {
continue
}
return err
}
if role == api.NodeRoleManager {
// wait to become manager
return raftutils.PollFuncWithTimeout(nil, func() error {
if !node.IsManager() {
return fmt.Errorf("node is still not a manager")
}
return nil
}, opsTimeout)
}
// wait to become worker
return raftutils.PollFuncWithTimeout(nil, func() error {
if node.IsManager() {
return fmt.Errorf("node is still not a worker")
}
return nil
}, opsTimeout)
}
return fmt.Errorf("set role %s for node %s, got sequence error 5 times", role, id)
}
// RemoveNode removes node entirely. It tries to demote managers.
func (c *testCluster) RemoveNode(id string, graceful bool) error {
node, ok := c.nodes[id]
if !ok {
return fmt.Errorf("remove node: node %s not found", id)
}
// demote before removal
if node.IsManager() {
if err := c.SetNodeRole(id, api.NodeRoleWorker); err != nil {
return fmt.Errorf("demote manager: %v", err)
}
}
if err := node.Stop(); err != nil {
return err
}
delete(c.nodes, id)
if graceful {
if err := raftutils.PollFuncWithTimeout(nil, func() error {
resp, err := c.api.GetNode(context.Background(), &api.GetNodeRequest{NodeID: id})
if err != nil {
return fmt.Errorf("get node: %v", err)
}
if resp.Node.Status.State != api.NodeStatus_DOWN {
return fmt.Errorf("node %s is still not down", id)
}
return nil
}, opsTimeout); err != nil {
return err
}
}
if _, err := c.api.RemoveNode(context.Background(), &api.RemoveNodeRequest{NodeID: id, Force: !graceful}); err != nil {
return fmt.Errorf("remove node: %v", err)
}
return nil
}
