// GetNode returns a Node given a NodeID.
// - Returns `InvalidArgument` if NodeID is not provided.
// - Returns `NotFound` if the Node is not found.
func (s *Server) GetNode(ctx context.Context, request *api.GetNodeRequest) (*api.GetNodeResponse, error) {
if request.NodeID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var node *api.Node
s.store.View(func(tx store.ReadTx) {
node = store.GetNode(tx, request.NodeID)
})
if node == nil {
return nil, grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
if s.raft != nil {
memberlist := s.raft.GetMemberlist()
for _, member := range memberlist {
if member.NodeID == node.ID {
node.ManagerStatus = &api.ManagerStatus{
RaftID: member.RaftID,
Addr: member.Addr,
Leader: member.Status.Leader,
Reachability: member.Status.Reachability,
}
break
}
}
}
return &api.GetNodeResponse{
Node: node,
}, nil
}
// GetNode returns a Node given a NodeID.
// - Returns `InvalidArgument` if NodeID is not provided.
// - Returns `NotFound` if the Node is not found.
func (s *Server) GetNode(ctx context.Context, request *api.GetNodeRequest) (*api.GetNodeResponse, error) {
if request.NodeID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var node *api.Node
s.store.View(func(tx store.ReadTx) {
node = store.GetNode(tx, request.NodeID)
})
if node == nil {
return nil, grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
if s.raft != nil {
memberlist := s.raft.GetMemberlist()
raftID, err := identity.ParseNodeID(request.NodeID)
if err == nil && memberlist[raftID] != nil {
node.ManagerStatus = &api.ManagerStatus{Raft: *memberlist[raftID]}
}
}
return &api.GetNodeResponse{
Node: node,
}, nil
}
// nodesEqual returns true if the node states are functionaly equal, ignoring status,
// version and other superfluous fields.
//
// This used to decide whether or not to propagate a node update to executor.
func nodesEqual(a, b *api.Node) bool {
a, b = a.Copy(), b.Copy()
a.Status, b.Status = api.NodeStatus{}, api.NodeStatus{}
a.Meta, b.Meta = api.Meta{}, api.Meta{}
return reflect.DeepEqual(a, b)
}
// UpdateNode updates a Node referenced by NodeID with the given NodeSpec.
// - Returns `NotFound` if the Node is not found.
// - Returns `InvalidArgument` if the NodeSpec is malformed.
// - Returns an error if the update fails.
func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) (*api.UpdateNodeResponse, error) {
if request.NodeID == "" || request.NodeVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateNodeSpec(request.Spec); err != nil {
return nil, err
}
var (
node *api.Node
demote bool
)
err := s.store.Update(func(tx store.Tx) error {
node = store.GetNode(tx, request.NodeID)
if node == nil {
return nil
}
// Demotion sanity checks.
if node.Spec.Role == api.NodeRoleManager && request.Spec.Role == api.NodeRoleWorker {
demote = true
managers, err := store.FindNodes(tx, store.ByRole(api.NodeRoleManager))
if err != nil {
return grpc.Errorf(codes.Internal, "internal store error: %v", err)
}
if len(managers) == 1 && managers[0].ID == node.ID {
return grpc.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm")
}
}
node.Meta.Version = *request.NodeVersion
node.Spec = *request.Spec.Copy()
return store.UpdateNode(tx, node)
})
if err != nil {
return nil, err
}
if node == nil {
return nil, grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
if demote && s.raft != nil {
memberlist := s.raft.GetMemberlist()
for raftID, member := range memberlist {
if member.NodeID == request.NodeID {
if err := s.raft.RemoveMember(ctx, raftID); err != nil {
return nil, err
}
break
}
}
}
return &api.UpdateNodeResponse{
Node: node,
}, nil
}
// UpdateNode updates a Node referenced by NodeID with the given NodeSpec.
// - Returns `NotFound` if the Node is not found.
// - Returns `InvalidArgument` if the NodeSpec is malformed.
// - Returns an error if the update fails.
func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) (*api.UpdateNodeResponse, error) {
if request.NodeID == "" || request.NodeVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateNodeSpec(request.Spec); err != nil {
return nil, err
}
var (
node *api.Node
member *membership.Member
)
err := s.store.Update(func(tx store.Tx) error {
node = store.GetNode(tx, request.NodeID)
if node == nil {
return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
// Demotion sanity checks.
if node.Spec.DesiredRole == api.NodeRoleManager && request.Spec.DesiredRole == api.NodeRoleWorker {
// Check for manager entries in Store.
managers, err := store.FindNodes(tx, store.ByRole(api.NodeRoleManager))
if err != nil {
return grpc.Errorf(codes.Internal, "internal store error: %v", err)
}
if len(managers) == 1 && managers[0].ID == node.ID {
return grpc.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm")
}
// Check for node in memberlist
if member = s.raft.GetMemberByNodeID(request.NodeID); member == nil {
return grpc.Errorf(codes.NotFound, "can't find manager in raft memberlist")
}
// Quorum safeguard
if !s.raft.CanRemoveMember(member.RaftID) {
return grpc.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum")
}
}
node.Meta.Version = *request.NodeVersion
node.Spec = *request.Spec.Copy()
return store.UpdateNode(tx, node)
})
if err != nil {
return nil, err
}
return &api.UpdateNodeResponse{
Node: node,
}, nil
}
// register is used for registration of node with particular dispatcher.
func (d *Dispatcher) register(ctx context.Context, nodeID string, description *api.NodeDescription) (string, error) {
// prevent register until we're ready to accept it
if err := d.isRunningLocked(); err != nil {
return "", err
}
if err := d.nodes.CheckRateLimit(nodeID); err != nil {
return "", err
}
// create or update node in store
// TODO(stevvooe): Validate node specification.
var node *api.Node
err := d.store.Update(func(tx store.Tx) error {
node = store.GetNode(tx, nodeID)
if node == nil {
return ErrNodeNotFound
}
node.Description = description
node.Status = api.NodeStatus{
State: api.NodeStatus_READY,
}
return store.UpdateNode(tx, node)
})
if err != nil {
return "", err
}
expireFunc := func() {
nodeStatus := api.NodeStatus{State: api.NodeStatus_DOWN, Message: "heartbeat failure"}
log.G(ctx).Debugf("heartbeat expiration")
if err := d.nodeRemove(nodeID, nodeStatus); err != nil {
log.G(ctx).WithError(err).Errorf("failed deregistering node after heartbeat expiration")
}
}
rn := d.nodes.Add(node, expireFunc)
// NOTE(stevvooe): We need be a little careful with re-registration. The
// current implementation just matches the node id and then gives away the
// sessionID. If we ever want to use sessionID as a secret, which we may
// want to, this is giving away the keys to the kitchen.
//
// The right behavior is going to be informed by identity. Basically, each
// time a node registers, we invalidate the session and issue a new
// session, once identity is proven. This will cause misbehaved agents to
// be kicked when multiple connections are made.
return rn.SessionID, nil
}
// issueRenewCertificate receives a nodeID and a CSR and modifies the node's certificate entry with the new CSR
// and changes the state to RENEW, so it can be picked up and signed by the signing reconciliation loop
func (s *Server) issueRenewCertificate(ctx context.Context, nodeID string, csr []byte) (*api.IssueNodeCertificateResponse, error) {
var (
cert api.Certificate
node *api.Node
)
err := s.store.Update(func(tx store.Tx) error {
// Attempt to retrieve the node with nodeID
node = store.GetNode(tx, nodeID)
if node == nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": nodeID,
"method": "issueRenewCertificate",
}).Warnf("node does not exist")
// If this node doesn't exist, we shouldn't be renewing a certificate for it
return grpc.Errorf(codes.NotFound, "node %s not found when attempting to renew certificate", nodeID)
}
// Create a new Certificate entry for this node with the new CSR and a RENEW state
cert = api.Certificate{
CSR: csr,
CN: node.ID,
Role: node.Spec.Role,
Status: api.IssuanceStatus{
State: api.IssuanceStateRenew,
},
}
node.Certificate = cert
return store.UpdateNode(tx, node)
})
if err != nil {
return nil, err
}
log.G(ctx).WithFields(logrus.Fields{
"cert.cn": cert.CN,
"cert.role": cert.Role,
"method": "issueRenewCertificate",
}).Debugf("node certificate updated")
return &api.IssueNodeCertificateResponse{
NodeID: nodeID,
NodeMembership: node.Spec.Membership,
}, nil
}
func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) {
var (
isDelete bool
node *api.Node
)
switch v := ev.(type) {
case state.EventCreateNode:
node = v.Node.Copy()
case state.EventUpdateNode:
node = v.Node.Copy()
case state.EventDeleteNode:
isDelete = true
node = v.Node.Copy()
}
nc := a.netCtx
if isDelete {
if nc.nwkAllocator.IsNodeAllocated(node) {
if err := nc.nwkAllocator.DeallocateNode(node); err != nil {
log.G(ctx).WithError(err).Errorf("Failed freeing network resources for node %s", node.ID)
}
}
return
}
if !nc.nwkAllocator.IsNodeAllocated(node) {
if node.Attachment == nil {
node.Attachment = &api.NetworkAttachment{}
}
node.Attachment.Network = nc.ingressNetwork.Copy()
if err := a.allocateNode(ctx, node); err != nil {
log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s", node.ID)
return
}
if _, err := a.store.Batch(func(batch *store.Batch) error {
return a.commitAllocatedNode(ctx, batch, node)
}); err != nil {
log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s", node.ID)
}
}
}
func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev events.Event) {
var (
isDelete bool
node *api.Node
)
switch v := ev.(type) {
case state.EventCreateNode:
node = v.Node.Copy()
case state.EventUpdateNode:
node = v.Node.Copy()
case state.EventDeleteNode:
isDelete = true
node = v.Node.Copy()
}
if isDelete {
if nc.nwkAllocator.IsNodeAllocated(node) {
if err := nc.nwkAllocator.DeallocateNode(node); err != nil {
log.G(ctx).Errorf("Failed freeing network resources for node %s: %v", node.ID, err)
}
}
return
}
if !nc.nwkAllocator.IsNodeAllocated(node) {
if node.Attachment == nil {
node.Attachment = &api.NetworkAttachment{}
}
node.Attachment.Network = nc.ingressNetwork.Copy()
if err := a.allocateNode(ctx, nc, node); err != nil {
log.G(ctx).Errorf("Failed to allocate network resources for node %s: %v", node.ID, err)
}
}
}
// signNodeCert does the bulk of the work for signing a certificate
func (s *Server) signNodeCert(ctx context.Context, node *api.Node) {
rootCA := s.securityConfig.RootCA()
externalCA := s.securityConfig.externalCA
node = node.Copy()
nodeID := node.ID
// Convert the role from proto format
role, err := ParseRole(node.Certificate.Role)
if err != nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": node.ID,
"method": "(*Server).signNodeCert",
}).WithError(err).Errorf("failed to parse role")
return
}
// Attempt to sign the CSR
var (
rawCSR = node.Certificate.CSR
cn = node.Certificate.CN
ou = role
org = s.securityConfig.ClientTLSCreds.Organization()
)
// Try using the external CA first.
cert, err := externalCA.Sign(PrepareCSR(rawCSR, cn, ou, org))
if err == ErrNoExternalCAURLs {
// No external CA servers configured. Try using the local CA.
cert, err = rootCA.ParseValidateAndSignCSR(rawCSR, cn, ou, org)
}
if err != nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": node.ID,
"method": "(*Server).signNodeCert",
}).WithError(err).Errorf("failed to sign CSR")
// If this error is due the lack of signer, maybe some other
// manager in the future will pick it up. Return without
// changing the state of the certificate.
if err == ErrNoValidSigner {
return
}
// If the current state is already Failed, no need to change it
if node.Certificate.Status.State == api.IssuanceStateFailed {
return
}
// We failed to sign this CSR, change the state to FAILED
err = s.store.Update(func(tx store.Tx) error {
node := store.GetNode(tx, nodeID)
if node == nil {
return fmt.Errorf("node %s not found", nodeID)
}
node.Certificate.Status = api.IssuanceStatus{
State: api.IssuanceStateFailed,
Err: err.Error(),
}
return store.UpdateNode(tx, node)
})
if err != nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": nodeID,
"method": "(*Server).signNodeCert",
}).WithError(err).Errorf("transaction failed when setting state to FAILED")
}
return
}
// We were able to successfully sign the new CSR. Let's try to update the nodeStore
for {
err = s.store.Update(func(tx store.Tx) error {
node.Certificate.Certificate = cert
node.Certificate.Status = api.IssuanceStatus{
State: api.IssuanceStateIssued,
}
err := store.UpdateNode(tx, node)
if err != nil {
node = store.GetNode(tx, nodeID)
if node == nil {
err = fmt.Errorf("node %s does not exist", nodeID)
}
}
return err
})
if err == nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": node.ID,
"node.role": node.Certificate.Role,
"method": "(*Server).signNodeCert",
}).Debugf("certificate issued")
break
}
if err == store.ErrSequenceConflict {
continue
}
log.G(ctx).WithFields(logrus.Fields{
"node.id": nodeID,
"method": "(*Server).signNodeCert",
}).WithError(err).Errorf("transaction failed")
return
}
}
// signNodeCert does the bulk of the work for signing a certificate
func (s *Server) signNodeCert(ctx context.Context, node *api.Node) {
if !s.securityConfig.RootCA().CanSign() {
log.G(ctx).WithFields(logrus.Fields{
"node.id": node.ID,
"method": "(*Server).signNodeCert",
}).Errorf("no valid signer found")
return
}
node = node.Copy()
nodeID := node.ID
// Convert the role from proto format
role, err := ParseRole(node.Certificate.Role)
if err != nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": node.ID,
"method": "(*Server).signNodeCert",
}).WithError(err).Errorf("failed to parse role")
return
}
// Attempt to sign the CSR
cert, err := s.securityConfig.RootCA().ParseValidateAndSignCSR(node.Certificate.CSR, node.Certificate.CN, role, s.securityConfig.ClientTLSCreds.Organization())
if err != nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": node.ID,
"method": "(*Server).signNodeCert",
}).WithError(err).Errorf("failed to sign CSR")
// If this error is due the lack of signer, maybe some other
// manager in the future will pick it up. Return without
// changing the state of the certificate.
if err == ErrNoValidSigner {
return
}
// If the current state is already Failed, no need to change it
if node.Certificate.Status.State == api.IssuanceStateFailed {
return
}
// We failed to sign this CSR, change the state to FAILED
err = s.store.Update(func(tx store.Tx) error {
node := store.GetNode(tx, nodeID)
if node == nil {
return fmt.Errorf("node %s not found", nodeID)
}
node.Certificate.Status = api.IssuanceStatus{
State: api.IssuanceStateFailed,
Err: err.Error(),
}
return store.UpdateNode(tx, node)
})
if err != nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": nodeID,
"method": "(*Server).signNodeCert",
}).WithError(err).Errorf("transaction failed when setting state to FAILED")
}
return
}
// We were able to successfully sign the new CSR. Let's try to update the nodeStore
for {
err = s.store.Update(func(tx store.Tx) error {
// Remote nodes are expecting a full certificate chain, not just a signed certificate
node.Certificate.Certificate = append(cert, s.securityConfig.RootCA().Cert...)
node.Certificate.Status = api.IssuanceStatus{
State: api.IssuanceStateIssued,
}
err := store.UpdateNode(tx, node)
if err != nil {
node = store.GetNode(tx, nodeID)
if node == nil {
err = fmt.Errorf("node %s does not exist", nodeID)
}
}
return err
})
if err == nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": node.ID,
"node.role": node.Certificate.Role,
"method": "(*Server).signNodeCert",
}).Debugf("certificate issued")
break
}
if err == store.ErrSequenceConflict {
continue
}
log.G(ctx).WithFields(logrus.Fields{
"node.id": nodeID,
"method": "(*Server).signNodeCert",
}).WithError(err).Errorf("transaction failed")
return
}
}