// RemoveNode updates a Node referenced by NodeID with the given NodeSpec.
// - Returns NotFound if the Node is not found.
// - Returns FailedPrecondition if the Node has manager role or not shut down.
// - Returns InvalidArgument if NodeID or NodeVersion is not valid.
// - Returns an error if the delete fails.
func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest) (*api.RemoveNodeResponse, error) {
if request.NodeID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if s.raft != nil {
memberlist := s.raft.GetMemberlist()
raftID, err := identity.ParseNodeID(request.NodeID)
if err == nil && memberlist[raftID] != nil {
return nil, grpc.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is part of the quorum. It must be demoted to worker before removal", request.NodeID)
}
}
err := s.store.Update(func(tx store.Tx) error {
node := store.GetNode(tx, request.NodeID)
if node == nil {
return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
if node.Spec.Role == api.NodeRoleManager {
return grpc.Errorf(codes.FailedPrecondition, "node %s role is set to manager. It should be demoted to worker for safe removal", request.NodeID)
}
if node.Status.State == api.NodeStatus_READY {
return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID)
}
return store.DeleteNode(tx, request.NodeID)
})
if err != nil {
return nil, err
}
return &api.RemoveNodeResponse{}, nil
}
// GetNode returns a Node given a NodeID.
// - Returns `InvalidArgument` if NodeID is not provided.
// - Returns `NotFound` if the Node is not found.
func (s *Server) GetNode(ctx context.Context, request *api.GetNodeRequest) (*api.GetNodeResponse, error) {
if request.NodeID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var node *api.Node
s.store.View(func(tx store.ReadTx) {
node = store.GetNode(tx, request.NodeID)
})
if node == nil {
return nil, grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
if s.raft != nil {
memberlist := s.raft.GetMemberlist()
raftID, err := identity.ParseNodeID(request.NodeID)
if err == nil && memberlist[raftID] != nil {
node.ManagerStatus = &api.ManagerStatus{Raft: *memberlist[raftID]}
}
}
return &api.GetNodeResponse{
Node: node,
}, nil
}
// Join asks to a member of the raft to propose
// a configuration change and add us as a member thus
// beginning the log replication process. This method
// is called from an aspiring member to an existing member
func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinResponse, error) {
nodeInfo, err := ca.RemoteNode(ctx)
if err != nil {
return nil, err
}
fields := logrus.Fields{
"node.id": nodeInfo.NodeID,
"method": "(*Node).Join",
}
if nodeInfo.ForwardedBy != nil {
fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID
}
log := log.G(ctx).WithFields(fields)
raftID, err := identity.ParseNodeID(nodeInfo.NodeID)
if err != nil {
return nil, err
}
// can't stop the raft node while an async RPC is in progress
n.stopMu.RLock()
defer n.stopMu.RUnlock()
if n.Node == nil {
log.WithError(ErrStopped).Errorf(ErrStopped.Error())
return nil, ErrStopped
}
// We submit a configuration change only if the node was not registered yet
if n.cluster.GetMember(raftID) == nil {
err = n.addMember(ctx, req.Addr, raftID)
if err != nil {
log.WithError(err).Errorf("failed to add member")
return nil, err
}
}
var nodes []*api.RaftMember
for _, node := range n.cluster.Members() {
nodes = append(nodes, &api.RaftMember{
RaftID: node.RaftID,
Addr: node.Addr,
})
}
log.Debugf("node joined")
return &api.JoinResponse{Members: nodes}, nil
}
// NewNode generates a new Raft node
func NewNode(ctx context.Context, opts NewNodeOptions) (*Node, error) {
cfg := opts.Config
if cfg == nil {
cfg = DefaultNodeConfig()
}
if opts.TickInterval == 0 {
opts.TickInterval = time.Second
}
raftID, err := identity.ParseNodeID(opts.ID)
if err != nil {
return nil, err
}
raftStore := raft.NewMemoryStorage()
ctx, cancel := context.WithCancel(ctx)
n := &Node{
Ctx: ctx,
cancel: cancel,
cluster: membership.NewCluster(),
tlsCredentials: opts.TLSCredentials,
raftStore: raftStore,
Address: opts.Addr,
Config: &raft.Config{
ElectionTick: cfg.ElectionTick,
HeartbeatTick: cfg.HeartbeatTick,
Storage: raftStore,
MaxSizePerMsg: cfg.MaxSizePerMsg,
MaxInflightMsgs: cfg.MaxInflightMsgs,
Logger: cfg.Logger,
ID: raftID,
},
forceNewCluster: opts.ForceNewCluster,
stopCh: make(chan struct{}),
doneCh: make(chan struct{}),
StateDir: opts.StateDir,
joinAddr: opts.JoinAddr,
sendTimeout: 2 * time.Second,
leadershipBroadcast: events.NewBroadcaster(),
}
n.memoryStore = store.NewMemoryStore(n)
if opts.ClockSource == nil {
n.ticker = clock.NewClock().NewTicker(opts.TickInterval)
} else {
n.ticker = opts.ClockSource.NewTicker(opts.TickInterval)
}
if opts.SendTimeout != 0 {
n.sendTimeout = opts.SendTimeout
}
if err := n.loadAndStart(ctx, opts.ForceNewCluster); err != nil {
n.ticker.Stop()
return nil, err
}
snapshot, err := raftStore.Snapshot()
// Snapshot never returns an error
if err != nil {
panic("could not get snapshot of raft store")
}
n.confState = snapshot.Metadata.ConfState
n.appliedIndex = snapshot.Metadata.Index
n.snapshotIndex = snapshot.Metadata.Index
n.reqIDGen = idutil.NewGenerator(uint16(n.Config.ID), time.Now())
n.wait = newWait()
if n.startNodePeers != nil {
if n.joinAddr != "" {
c, err := n.ConnectToMember(n.joinAddr, 10*time.Second)
if err != nil {
return nil, err
}
defer func() {
_ = c.Conn.Close()
}()
ctx, cancel := context.WithTimeout(n.Ctx, 10*time.Second)
defer cancel()
resp, err := c.Join(ctx, &api.JoinRequest{
Addr: n.Address,
})
if err != nil {
return nil, err
}
n.Node = raft.StartNode(n.Config, []raft.Peer{})
if err := n.registerNodes(resp.Members); err != nil {
return nil, err
}
} else {
n.Node = raft.StartNode(n.Config, n.startNodePeers)
if err := n.Campaign(n.Ctx); err != nil {
return nil, err
}
}
return n, nil
}
if n.joinAddr != "" {
n.Config.Logger.Warning("ignoring request to join cluster, because raft state already exists")
}
n.Node = raft.RestartNode(n.Config)
return n, nil
}
// ListNodes returns a list of all nodes.
func (s *Server) ListNodes(ctx context.Context, request *api.ListNodesRequest) (*api.ListNodesResponse, error) {
var (
nodes []*api.Node
err error
)
s.store.View(func(tx store.ReadTx) {
switch {
case request.Filters != nil && len(request.Filters.Names) > 0:
nodes, err = store.FindNodes(tx, buildFilters(store.ByName, request.Filters.Names))
case request.Filters != nil && len(request.Filters.IDPrefixes) > 0:
nodes, err = store.FindNodes(tx, buildFilters(store.ByIDPrefix, request.Filters.IDPrefixes))
case request.Filters != nil && len(request.Filters.Roles) > 0:
filters := make([]store.By, 0, len(request.Filters.Roles))
for _, v := range request.Filters.Roles {
filters = append(filters, store.ByRole(v))
}
nodes, err = store.FindNodes(tx, store.Or(filters...))
case request.Filters != nil && len(request.Filters.Memberships) > 0:
filters := make([]store.By, 0, len(request.Filters.Memberships))
for _, v := range request.Filters.Memberships {
filters = append(filters, store.ByMembership(v))
}
nodes, err = store.FindNodes(tx, store.Or(filters...))
default:
nodes, err = store.FindNodes(tx, store.All)
}
})
if err != nil {
return nil, err
}
if request.Filters != nil {
nodes = filterNodes(nodes,
func(e *api.Node) bool {
if len(request.Filters.Names) == 0 {
return true
}
if e.Description == nil {
return false
}
return filterContains(e.Description.Hostname, request.Filters.Names)
},
func(e *api.Node) bool {
return filterContainsPrefix(e.ID, request.Filters.IDPrefixes)
},
func(e *api.Node) bool {
if len(request.Filters.Labels) == 0 {
return true
}
if e.Description == nil {
return false
}
return filterMatchLabels(e.Description.Engine.Labels, request.Filters.Labels)
},
func(e *api.Node) bool {
if len(request.Filters.Roles) == 0 {
return true
}
for _, c := range request.Filters.Roles {
if c == e.Spec.Role {
return true
}
}
return false
},
func(e *api.Node) bool {
if len(request.Filters.Memberships) == 0 {
return true
}
for _, c := range request.Filters.Memberships {
if c == e.Spec.Membership {
return true
}
}
return false
},
)
}
// Add in manager information on nodes that are managers
if s.raft != nil {
memberlist := s.raft.GetMemberlist()
for _, n := range nodes {
raftID, err := identity.ParseNodeID(n.ID)
if err == nil && memberlist[raftID] != nil {
n.ManagerStatus = &api.ManagerStatus{Raft: *memberlist[raftID]}
}
}
}
return &api.ListNodesResponse{
Nodes: nodes,
}, nil
}