// Join asks to a member of the raft to propose
// a configuration change and add us as a member thus
// beginning the log replication process. This method
// is called from an aspiring member to an existing member
func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinResponse, error) {
nodeInfo, err := ca.RemoteNode(ctx)
if err != nil {
return nil, err
}
fields := logrus.Fields{
"node.id": nodeInfo.NodeID,
"method": "(*Node).Join",
}
if nodeInfo.ForwardedBy != nil {
fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID
}
log := log.G(ctx).WithFields(fields)
// can't stop the raft node while an async RPC is in progress
n.stopMu.RLock()
defer n.stopMu.RUnlock()
n.membershipLock.Lock()
defer n.membershipLock.Unlock()
if n.Node == nil {
log.WithError(ErrStopped).Errorf(ErrStopped.Error())
return nil, ErrStopped
}
if !n.IsLeader() {
return nil, ErrLostLeadership
}
// Find a unique ID for the joining member.
var raftID uint64
for {
raftID = uint64(rand.Int63()) + 1
if n.cluster.GetMember(raftID) == nil && !n.cluster.IsIDRemoved(raftID) {
break
}
}
err = n.addMember(ctx, req.Addr, raftID, nodeInfo.NodeID)
if err != nil {
log.WithError(err).Errorf("failed to add member")
return nil, err
}
var nodes []*api.RaftMember
for _, node := range n.cluster.Members() {
nodes = append(nodes, &api.RaftMember{
RaftID: node.RaftID,
NodeID: node.NodeID,
Addr: node.Addr,
})
}
log.Debugf("node joined")
return &api.JoinResponse{Members: nodes, RaftID: raftID}, nil
}
func (d *Dispatcher) processTaskUpdates() {
d.taskUpdatesLock.Lock()
if len(d.taskUpdates) == 0 {
d.taskUpdatesLock.Unlock()
return
}
taskUpdates := d.taskUpdates
d.taskUpdates = make(map[string]*api.TaskStatus)
d.taskUpdatesLock.Unlock()
log := log.G(d.ctx).WithFields(logrus.Fields{
"method": "(*Dispatcher).processTaskUpdates",
})
_, err := d.store.Batch(func(batch *store.Batch) error {
for taskID, status := range taskUpdates {
err := batch.Update(func(tx store.Tx) error {
logger := log.WithField("task.id", taskID)
task := store.GetTask(tx, taskID)
if task == nil {
logger.Errorf("task unavailable")
return nil
}
logger = logger.WithField("state.transition", fmt.Sprintf("%v->%v", task.Status.State, status.State))
if task.Status == *status {
logger.Debug("task status identical, ignoring")
return nil
}
if task.Status.State > status.State {
logger.Debug("task status invalid transition")
return nil
}
task.Status = *status
if err := store.UpdateTask(tx, task); err != nil {
logger.WithError(err).Error("failed to update task status")
return nil
}
logger.Debug("task status updated")
return nil
})
if err != nil {
log.WithError(err).Error("dispatcher transaction failed")
}
}
return nil
})
if err != nil {
log.WithError(err).Error("dispatcher batch failed")
}
}
// RenewTLSConfigNow gets a new TLS cert and key, and updates the security config if provided. This is similar to
// RenewTLSConfig, except while that monitors for expiry, and periodically renews, this renews once and is blocking
func RenewTLSConfigNow(ctx context.Context, s *SecurityConfig, r remotes.Remotes) error {
s.renewalMu.Lock()
defer s.renewalMu.Unlock()
ctx = log.WithModule(ctx, "tls")
log := log.G(ctx).WithFields(logrus.Fields{
"node.id": s.ClientTLSCreds.NodeID(),
"node.role": s.ClientTLSCreds.Role(),
})
// Let's request new certs. Renewals don't require a token.
rootCA := s.RootCA()
tlsKeyPair, err := rootCA.RequestAndSaveNewCertificates(ctx,
s.KeyWriter(),
CertificateRequestConfig{
Remotes: r,
Credentials: s.ClientTLSCreds,
})
if err != nil {
log.WithError(err).Errorf("failed to renew the certificate")
return err
}
clientTLSConfig, err := NewClientTLSConfig(tlsKeyPair, rootCA.Pool, CARole)
if err != nil {
log.WithError(err).Errorf("failed to create a new client config")
return err
}
serverTLSConfig, err := NewServerTLSConfig(tlsKeyPair, rootCA.Pool)
if err != nil {
log.WithError(err).Errorf("failed to create a new server config")
return err
}
if err = s.ClientTLSCreds.LoadNewTLSConfig(clientTLSConfig); err != nil {
log.WithError(err).Errorf("failed to update the client credentials")
return err
}
// Update the external CA to use the new client TLS
// config using a copy without a serverName specified.
s.externalCA.UpdateTLSConfig(&tls.Config{
Certificates: clientTLSConfig.Certificates,
RootCAs: clientTLSConfig.RootCAs,
MinVersion: tls.VersionTLS12,
})
if err = s.ServerTLSCreds.LoadNewTLSConfig(serverTLSConfig); err != nil {
log.WithError(err).Errorf("failed to update the server TLS credentials")
return err
}
return nil
}
// Join asks to a member of the raft to propose
// a configuration change and add us as a member thus
// beginning the log replication process. This method
// is called from an aspiring member to an existing member
func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinResponse, error) {
nodeInfo, err := ca.RemoteNode(ctx)
if err != nil {
return nil, err
}
fields := logrus.Fields{
"node.id": nodeInfo.NodeID,
"method": "(*Node).Join",
}
if nodeInfo.ForwardedBy != nil {
fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID
}
log := log.G(ctx).WithFields(fields)
raftID, err := identity.ParseNodeID(nodeInfo.NodeID)
if err != nil {
return nil, err
}
// can't stop the raft node while an async RPC is in progress
n.stopMu.RLock()
defer n.stopMu.RUnlock()
if n.Node == nil {
log.WithError(ErrStopped).Errorf(ErrStopped.Error())
return nil, ErrStopped
}
// We submit a configuration change only if the node was not registered yet
if n.cluster.GetMember(raftID) == nil {
err = n.addMember(ctx, req.Addr, raftID)
if err != nil {
log.WithError(err).Errorf("failed to add member")
return nil, err
}
}
var nodes []*api.RaftMember
for _, node := range n.cluster.Members() {
nodes = append(nodes, &api.RaftMember{
RaftID: node.RaftID,
Addr: node.Addr,
})
}
log.Debugf("node joined")
return &api.JoinResponse{Members: nodes}, nil
}
// RenewTLSConfig will continuously monitor for the necessity of renewing the local certificates, either by
// issuing them locally if key-material is available, or requesting them from a remote CA.
func RenewTLSConfig(ctx context.Context, s *SecurityConfig, remotes remotes.Remotes, renew <-chan struct{}) <-chan CertificateUpdate {
updates := make(chan CertificateUpdate)
go func() {
var retry time.Duration
defer close(updates)
for {
ctx = log.WithModule(ctx, "tls")
log := log.G(ctx).WithFields(logrus.Fields{
"node.id": s.ClientTLSCreds.NodeID(),
"node.role": s.ClientTLSCreds.Role(),
})
// Our starting default will be 5 minutes
retry = 5 * time.Minute
// Since the expiration of the certificate is managed remotely we should update our
// retry timer on every iteration of this loop.
// Retrieve the current certificate expiration information.
validFrom, validUntil, err := readCertValidity(s.KeyReader())
if err != nil {
// We failed to read the expiration, let's stick with the starting default
log.Errorf("failed to read the expiration of the TLS certificate in: %s", s.KeyReader().Target())
updates <- CertificateUpdate{Err: errors.New("failed to read certificate expiration")}
} else {
// If we have an expired certificate, we let's stick with the starting default in
// the hope that this is a temporary clock skew.
if validUntil.Before(time.Now()) {
log.WithError(err).Errorf("failed to create a new client TLS config")
updates <- CertificateUpdate{Err: errors.New("TLS certificate is expired")}
} else {
// Random retry time between 50% and 80% of the total time to expiration
retry = calculateRandomExpiry(validFrom, validUntil)
}
}
log.WithFields(logrus.Fields{
"time": time.Now().Add(retry),
}).Debugf("next certificate renewal scheduled")
select {
case <-time.After(retry):
log.Infof("renewing certificate")
case <-renew:
log.Infof("forced certificate renewal")
case <-ctx.Done():
log.Infof("shuting down certificate renewal routine")
return
}
// ignore errors - it will just try again laster
if err := RenewTLSConfigNow(ctx, s, remotes); err != nil {
updates <- CertificateUpdate{Err: err}
} else {
updates <- CertificateUpdate{Role: s.ClientTLSCreds.Role()}
}
}
}()
return updates
}
func (d *Dispatcher) markNodesUnknown(ctx context.Context) error {
log := log.G(ctx).WithField("method", "(*Dispatcher).markNodesUnknown")
var nodes []*api.Node
var err error
d.store.View(func(tx store.ReadTx) {
nodes, err = store.FindNodes(tx, store.All)
})
if err != nil {
return fmt.Errorf("failed to get list of nodes: %v", err)
}
_, err = d.store.Batch(func(batch *store.Batch) error {
for _, n := range nodes {
err := batch.Update(func(tx store.Tx) error {
// check if node is still here
node := store.GetNode(tx, n.ID)
if node == nil {
return nil
}
// do not try to resurrect down nodes
if node.Status.State == api.NodeStatus_DOWN {
return nil
}
node.Status = api.NodeStatus{
State: api.NodeStatus_UNKNOWN,
Message: `Node moved to "unknown" state due to leadership change in cluster`,
}
nodeID := node.ID
expireFunc := func() {
log := log.WithField("node", nodeID)
nodeStatus := api.NodeStatus{State: api.NodeStatus_DOWN, Message: `heartbeat failure for node in "unknown" state`}
log.Debugf("heartbeat expiration for unknown node")
if err := d.nodeRemove(nodeID, nodeStatus); err != nil {
log.WithError(err).Errorf(`failed deregistering node after heartbeat expiration for node in "unknown" state`)
}
}
if err := d.nodes.AddUnknown(node, expireFunc); err != nil {
return fmt.Errorf(`adding node in "unknown" state to node store failed: %v`, err)
}
if err := store.UpdateNode(tx, node); err != nil {
return fmt.Errorf("update failed %v", err)
}
return nil
})
if err != nil {
log.WithField("node", n.ID).WithError(err).Errorf(`failed to move node to "unknown" state`)
}
}
return nil
})
return err
}
// RenewTLSConfig will continuously monitor for the necessity of renewing the local certificates, either by
// issuing them locally if key-material is available, or requesting them from a remote CA.
func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string, remotes remotes.Remotes, renew <-chan struct{}) <-chan CertificateUpdate {
paths := NewConfigPaths(baseCertDir)
updates := make(chan CertificateUpdate)
go func() {
var retry time.Duration
defer close(updates)
for {
ctx = log.WithModule(ctx, "tls")
log := log.G(ctx).WithFields(logrus.Fields{
"node.id": s.ClientTLSCreds.NodeID(),
"node.role": s.ClientTLSCreds.Role(),
})
// Our starting default will be 5 minutes
retry = 5 * time.Minute
// Since the expiration of the certificate is managed remotely we should update our
// retry timer on every iteration of this loop.
// Retrieve the time until the certificate expires.
expiresIn, err := readCertExpiration(paths.Node)
if err != nil {
// We failed to read the expiration, let's stick with the starting default
log.Errorf("failed to read the expiration of the TLS certificate in: %s", paths.Node.Cert)
updates <- CertificateUpdate{Err: fmt.Errorf("failed to read certificate expiration")}
} else {
// If we have an expired certificate, we let's stick with the starting default in
// the hope that this is a temporary clock skew.
if expiresIn.Minutes() < 0 {
log.WithError(err).Errorf("failed to create a new client TLS config")
updates <- CertificateUpdate{Err: fmt.Errorf("TLS certificate is expired")}
} else {
// Random retry time between 50% and 80% of the total time to expiration
retry = calculateRandomExpiry(expiresIn)
}
}
log.WithFields(logrus.Fields{
"time": time.Now().Add(retry),
}).Debugf("next certificate renewal scheduled")
select {
case <-time.After(retry):
log.Infof("renewing certificate")
case <-renew:
log.Infof("forced certificate renewal")
case <-ctx.Done():
log.Infof("shuting down certificate renewal routine")
return
}
// Let's request new certs. Renewals don't require a token.
rootCA := s.RootCA()
tlsKeyPair, err := rootCA.RequestAndSaveNewCertificates(ctx,
paths.Node,
"",
remotes,
s.ClientTLSCreds,
nil)
if err != nil {
log.WithError(err).Errorf("failed to renew the certificate")
updates <- CertificateUpdate{Err: err}
continue
}
clientTLSConfig, err := NewClientTLSConfig(tlsKeyPair, rootCA.Pool, CARole)
if err != nil {
log.WithError(err).Errorf("failed to create a new client config")
updates <- CertificateUpdate{Err: err}
}
serverTLSConfig, err := NewServerTLSConfig(tlsKeyPair, rootCA.Pool)
if err != nil {
log.WithError(err).Errorf("failed to create a new server config")
updates <- CertificateUpdate{Err: err}
}
err = s.ClientTLSCreds.LoadNewTLSConfig(clientTLSConfig)
if err != nil {
log.WithError(err).Errorf("failed to update the client credentials")
updates <- CertificateUpdate{Err: err}
}
// Update the external CA to use the new client TLS
// config using a copy without a serverName specified.
s.externalCA.UpdateTLSConfig(&tls.Config{
Certificates: clientTLSConfig.Certificates,
RootCAs: clientTLSConfig.RootCAs,
MinVersion: tls.VersionTLS12,
})
err = s.ServerTLSCreds.LoadNewTLSConfig(serverTLSConfig)
if err != nil {
log.WithError(err).Errorf("failed to update the server TLS credentials")
updates <- CertificateUpdate{Err: err}
}
updates <- CertificateUpdate{Role: s.ClientTLSCreds.Role()}
}
}()
return updates
}
// Join asks to a member of the raft to propose
// a configuration change and add us as a member thus
// beginning the log replication process. This method
// is called from an aspiring member to an existing member
func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinResponse, error) {
nodeInfo, err := ca.RemoteNode(ctx)
if err != nil {
return nil, err
}
fields := logrus.Fields{
"node.id": nodeInfo.NodeID,
"method": "(*Node).Join",
"raft_id": fmt.Sprintf("%x", n.Config.ID),
}
if nodeInfo.ForwardedBy != nil {
fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID
}
log := log.G(ctx).WithFields(fields)
log.Debug("")
// can't stop the raft node while an async RPC is in progress
n.stopMu.RLock()
defer n.stopMu.RUnlock()
n.membershipLock.Lock()
defer n.membershipLock.Unlock()
if !n.IsMember() {
return nil, ErrNoRaftMember
}
if !n.isLeader() {
return nil, ErrLostLeadership
}
// Find a unique ID for the joining member.
var raftID uint64
for {
raftID = uint64(rand.Int63()) + 1
if n.cluster.GetMember(raftID) == nil && !n.cluster.IsIDRemoved(raftID) {
break
}
}
remoteAddr := req.Addr
// If the joining node sent an address like 0.0.0.0:4242, automatically
// determine its actual address based on the GRPC connection. This
// avoids the need for a prospective member to know its own address.
requestHost, requestPort, err := net.SplitHostPort(remoteAddr)
if err != nil {
return nil, fmt.Errorf("invalid address %s in raft join request", remoteAddr)
}
requestIP := net.ParseIP(requestHost)
if requestIP != nil && requestIP.IsUnspecified() {
remoteHost, _, err := net.SplitHostPort(nodeInfo.RemoteAddr)
if err != nil {
return nil, err
}
remoteAddr = net.JoinHostPort(remoteHost, requestPort)
}
// We do not bother submitting a configuration change for the
// new member if we can't contact it back using its address
if err := n.checkHealth(ctx, remoteAddr, 5*time.Second); err != nil {
return nil, err
}
err = n.addMember(ctx, remoteAddr, raftID, nodeInfo.NodeID)
if err != nil {
log.WithError(err).Errorf("failed to add member %x", raftID)
return nil, err
}
var nodes []*api.RaftMember
for _, node := range n.cluster.Members() {
nodes = append(nodes, &api.RaftMember{
RaftID: node.RaftID,
NodeID: node.NodeID,
Addr: node.Addr,
})
}
log.Debugf("node joined")
return &api.JoinResponse{Members: nodes, RaftID: raftID}, nil
}
// Session is a stream which controls agent connection.
// Each message contains list of backup Managers with weights. Also there is
// a special boolean field Disconnect which if true indicates that node should
// reconnect to another Manager immediately.
func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_SessionServer) error {
ctx := stream.Context()
nodeInfo, err := ca.RemoteNode(ctx)
if err != nil {
return err
}
nodeID := nodeInfo.NodeID
if err := d.isRunningLocked(); err != nil {
return err
}
// register the node.
sessionID, err := d.register(stream.Context(), nodeID, r.Description)
if err != nil {
return err
}
fields := logrus.Fields{
"node.id": nodeID,
"node.session": sessionID,
"method": "(*Dispatcher).Session",
}
if nodeInfo.ForwardedBy != nil {
fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID
}
log := log.G(ctx).WithFields(fields)
var nodeObj *api.Node
nodeUpdates, cancel, err := store.ViewAndWatch(d.store, func(readTx store.ReadTx) error {
nodeObj = store.GetNode(readTx, nodeID)
return nil
}, state.EventUpdateNode{Node: &api.Node{ID: nodeID},
Checks: []state.NodeCheckFunc{state.NodeCheckID}},
)
if cancel != nil {
defer cancel()
}
if err != nil {
log.WithError(err).Error("ViewAndWatch Node failed")
}
if _, err = d.nodes.GetWithSession(nodeID, sessionID); err != nil {
return err
}
if err := stream.Send(&api.SessionMessage{
SessionID: sessionID,
Node: nodeObj,
Managers: d.getManagers(),
NetworkBootstrapKeys: d.networkBootstrapKeys,
}); err != nil {
return err
}
managerUpdates, mgrCancel := d.mgrQueue.Watch()
defer mgrCancel()
keyMgrUpdates, keyMgrCancel := d.keyMgrQueue.Watch()
defer keyMgrCancel()
// disconnectNode is a helper forcibly shutdown connection
disconnectNode := func() error {
// force disconnect by shutting down the stream.
transportStream, ok := transport.StreamFromContext(stream.Context())
if ok {
// if we have the transport stream, we can signal a disconnect
// in the client.
if err := transportStream.ServerTransport().Close(); err != nil {
log.WithError(err).Error("session end")
}
}
nodeStatus := api.NodeStatus{State: api.NodeStatus_DISCONNECTED, Message: "node is currently trying to find new manager"}
if err := d.nodeRemove(nodeID, nodeStatus); err != nil {
log.WithError(err).Error("failed to remove node")
}
// still return an abort if the transport closure was ineffective.
return grpc.Errorf(codes.Aborted, "node must disconnect")
}
for {
// After each message send, we need to check the nodes sessionID hasn't
// changed. If it has, we will the stream and make the node
// re-register.
node, err := d.nodes.GetWithSession(nodeID, sessionID)
if err != nil {
return err
}
var mgrs []*api.WeightedPeer
var disconnect bool
select {
case ev := <-managerUpdates:
mgrs = ev.([]*api.WeightedPeer)
case ev := <-nodeUpdates:
nodeObj = ev.(state.EventUpdateNode).Node
case <-stream.Context().Done():
return stream.Context().Err()
case <-node.Disconnect:
disconnect = true
case <-d.ctx.Done():
disconnect = true
case <-keyMgrUpdates:
}
if mgrs == nil {
mgrs = d.getManagers()
}
if err := stream.Send(&api.SessionMessage{
SessionID: sessionID,
Node: nodeObj,
Managers: mgrs,
NetworkBootstrapKeys: d.networkBootstrapKeys,
}); err != nil {
return err
}
if disconnect {
return disconnectNode()
}
}
}
// Assignments is a stream of assignments for a node. Each message contains
// either full list of tasks and secrets for the node, or an incremental update.
func (d *Dispatcher) Assignments(r *api.AssignmentsRequest, stream api.Dispatcher_AssignmentsServer) error {
nodeInfo, err := ca.RemoteNode(stream.Context())
if err != nil {
return err
}
nodeID := nodeInfo.NodeID
dctx, err := d.isRunningLocked()
if err != nil {
return err
}
fields := logrus.Fields{
"node.id": nodeID,
"node.session": r.SessionID,
"method": "(*Dispatcher).Assignments",
}
if nodeInfo.ForwardedBy != nil {
fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID
}
log := log.G(stream.Context()).WithFields(fields)
log.Debugf("")
if _, err = d.nodes.GetWithSession(nodeID, r.SessionID); err != nil {
return err
}
var (
sequence int64
appliesTo string
initial api.AssignmentsMessage
)
tasksMap := make(map[string]*api.Task)
tasksUsingSecret := make(map[string]map[string]struct{})
sendMessage := func(msg api.AssignmentsMessage, assignmentType api.AssignmentsMessage_Type) error {
sequence++
msg.AppliesTo = appliesTo
msg.ResultsIn = strconv.FormatInt(sequence, 10)
appliesTo = msg.ResultsIn
msg.Type = assignmentType
if err := stream.Send(&msg); err != nil {
return err
}
return nil
}
// returns a slice of new secrets to send down
addSecretsForTask := func(readTx store.ReadTx, t *api.Task) []*api.Secret {
container := t.Spec.GetContainer()
if container == nil {
return nil
}
var newSecrets []*api.Secret
for _, secretRef := range container.Secrets {
// Empty ID prefix will return all secrets. Bail if there is no SecretID
if secretRef.SecretID == "" {
log.Debugf("invalid secret reference")
continue
}
secretID := secretRef.SecretID
log := log.WithFields(logrus.Fields{
"secret.id": secretID,
"secret.name": secretRef.SecretName,
})
if len(tasksUsingSecret[secretID]) == 0 {
tasksUsingSecret[secretID] = make(map[string]struct{})
secrets, err := store.FindSecrets(readTx, store.ByIDPrefix(secretID))
if err != nil {
log.WithError(err).Errorf("error retrieving secret")
continue
}
if len(secrets) != 1 {
log.Debugf("secret not found")
continue
}
// If the secret was found and there was one result
// (there should never be more than one because of the
// uniqueness constraint), add this secret to our
// initial set that we send down.
newSecrets = append(newSecrets, secrets[0])
}
tasksUsingSecret[secretID][t.ID] = struct{}{}
}
return newSecrets
}
// TODO(aaronl): Also send node secrets that should be exposed to
// this node.
nodeTasks, cancel, err := store.ViewAndWatch(
d.store,
func(readTx store.ReadTx) error {
tasks, err := store.FindTasks(readTx, store.ByNodeID(nodeID))
if err != nil {
return err
}
for _, t := range tasks {
// We only care about tasks that are ASSIGNED or
// higher. If the state is below ASSIGNED, the
// task may not meet the constraints for this
// node, so we have to be careful about sending
// secrets associated with it.
if t.Status.State < api.TaskStateAssigned {
continue
}
tasksMap[t.ID] = t
taskChange := &api.AssignmentChange{
Assignment: &api.Assignment{
Item: &api.Assignment_Task{
Task: t,
},
},
Action: api.AssignmentChange_AssignmentActionUpdate,
}
initial.Changes = append(initial.Changes, taskChange)
// Only send secrets down if these tasks are in < RUNNING
if t.Status.State <= api.TaskStateRunning {
newSecrets := addSecretsForTask(readTx, t)
for _, secret := range newSecrets {
secretChange := &api.AssignmentChange{
Assignment: &api.Assignment{
Item: &api.Assignment_Secret{
Secret: secret,
},
},
Action: api.AssignmentChange_AssignmentActionUpdate,
}
initial.Changes = append(initial.Changes, secretChange)
}
}
}
return nil
},
state.EventUpdateTask{Task: &api.Task{NodeID: nodeID},
Checks: []state.TaskCheckFunc{state.TaskCheckNodeID}},
state.EventDeleteTask{Task: &api.Task{NodeID: nodeID},
Checks: []state.TaskCheckFunc{state.TaskCheckNodeID}},
state.EventUpdateSecret{},
state.EventDeleteSecret{},
)
if err != nil {
return err
}
defer cancel()
if err := sendMessage(initial, api.AssignmentsMessage_COMPLETE); err != nil {
return err
}
for {
// Check for session expiration
if _, err := d.nodes.GetWithSession(nodeID, r.SessionID); err != nil {
return err
}
// bursty events should be processed in batches and sent out together
var (
update api.AssignmentsMessage
modificationCnt int
batchingTimer *time.Timer
batchingTimeout <-chan time.Time
updateTasks = make(map[string]*api.Task)
updateSecrets = make(map[string]*api.Secret)
removeTasks = make(map[string]struct{})
removeSecrets = make(map[string]struct{})
)
oneModification := func() {
modificationCnt++
if batchingTimer != nil {
batchingTimer.Reset(batchingWaitTime)
} else {
batchingTimer = time.NewTimer(batchingWaitTime)
batchingTimeout = batchingTimer.C
}
}
// Release the secrets references from this task
releaseSecretsForTask := func(t *api.Task) bool {
var modified bool
container := t.Spec.GetContainer()
if container == nil {
return modified
}
for _, secretRef := range container.Secrets {
secretID := secretRef.SecretID
delete(tasksUsingSecret[secretID], t.ID)
if len(tasksUsingSecret[secretID]) == 0 {
// No tasks are using the secret anymore
delete(tasksUsingSecret, secretID)
removeSecrets[secretID] = struct{}{}
modified = true
}
}
return modified
}
// The batching loop waits for 50 ms after the most recent
// change, or until modificationBatchLimit is reached. The
// worst case latency is modificationBatchLimit * batchingWaitTime,
// which is 10 seconds.
batchingLoop:
for modificationCnt < modificationBatchLimit {
select {
case event := <-nodeTasks:
switch v := event.(type) {
// We don't monitor EventCreateTask because tasks are
// never created in the ASSIGNED state. First tasks are
// created by the orchestrator, then the scheduler moves
// them to ASSIGNED. If this ever changes, we will need
// to monitor task creations as well.
case state.EventUpdateTask:
// We only care about tasks that are ASSIGNED or
// higher.
if v.Task.Status.State < api.TaskStateAssigned {
continue
}
if oldTask, exists := tasksMap[v.Task.ID]; exists {
// States ASSIGNED and below are set by the orchestrator/scheduler,
// not the agent, so tasks in these states need to be sent to the
// agent even if nothing else has changed.
if equality.TasksEqualStable(oldTask, v.Task) && v.Task.Status.State > api.TaskStateAssigned {
// this update should not trigger a task change for the agent
tasksMap[v.Task.ID] = v.Task
// If this task got updated to a final state, let's release
// the secrets that are being used by the task
if v.Task.Status.State > api.TaskStateRunning {
// If releasing the secrets caused a secret to be
// removed from an agent, mark one modification
if releaseSecretsForTask(v.Task) {
oneModification()
}
}
continue
}
} else if v.Task.Status.State <= api.TaskStateRunning {
// If this task wasn't part of the assignment set before, and it's <= RUNNING
// add the secrets it references to the secrets assignment.
// Task states > RUNNING are worker reported only, are never created in
// a > RUNNING state.
var newSecrets []*api.Secret
d.store.View(func(readTx store.ReadTx) {
newSecrets = addSecretsForTask(readTx, v.Task)
})
for _, secret := range newSecrets {
updateSecrets[secret.ID] = secret
}
}
tasksMap[v.Task.ID] = v.Task
updateTasks[v.Task.ID] = v.Task
oneModification()
case state.EventDeleteTask:
if _, exists := tasksMap[v.Task.ID]; !exists {
continue
}
removeTasks[v.Task.ID] = struct{}{}
delete(tasksMap, v.Task.ID)
// Release the secrets being used by this task
// Ignoring the return here. We will always mark
// this as a modification, since a task is being
// removed.
releaseSecretsForTask(v.Task)
oneModification()
// TODO(aaronl): For node secrets, we'll need to handle
// EventCreateSecret.
case state.EventUpdateSecret:
if _, exists := tasksUsingSecret[v.Secret.ID]; !exists {
continue
}
log.Debugf("Secret %s (ID: %d) was updated though it was still referenced by one or more tasks",
v.Secret.Spec.Annotations.Name, v.Secret.ID)
case state.EventDeleteSecret:
if _, exists := tasksUsingSecret[v.Secret.ID]; !exists {
continue
}
log.Debugf("Secret %s (ID: %d) was deleted though it was still referenced by one or more tasks",
v.Secret.Spec.Annotations.Name, v.Secret.ID)
}
case <-batchingTimeout:
break batchingLoop
case <-stream.Context().Done():
return stream.Context().Err()
case <-dctx.Done():
return dctx.Err()
}
}
if batchingTimer != nil {
batchingTimer.Stop()
}
if modificationCnt > 0 {
for id, task := range updateTasks {
if _, ok := removeTasks[id]; !ok {
taskChange := &api.AssignmentChange{
Assignment: &api.Assignment{
Item: &api.Assignment_Task{
Task: task,
},
},
Action: api.AssignmentChange_AssignmentActionUpdate,
}
update.Changes = append(update.Changes, taskChange)
}
}
for id, secret := range updateSecrets {
// If, due to multiple updates, this secret is no longer in use,
// don't send it down.
if len(tasksUsingSecret[id]) == 0 {
// delete this secret for the secrets to be updated
// so that deleteSecrets knows the current list
delete(updateSecrets, id)
continue
}
secretChange := &api.AssignmentChange{
Assignment: &api.Assignment{
Item: &api.Assignment_Secret{
Secret: secret,
},
},
Action: api.AssignmentChange_AssignmentActionUpdate,
}
update.Changes = append(update.Changes, secretChange)
}
for id := range removeTasks {
taskChange := &api.AssignmentChange{
Assignment: &api.Assignment{
Item: &api.Assignment_Task{
Task: &api.Task{ID: id},
},
},
Action: api.AssignmentChange_AssignmentActionRemove,
}
update.Changes = append(update.Changes, taskChange)
}
for id := range removeSecrets {
// If this secret is also being sent on the updated set
// don't also add it to the removed set
if _, ok := updateSecrets[id]; ok {
continue
}
secretChange := &api.AssignmentChange{
Assignment: &api.Assignment{
Item: &api.Assignment_Secret{
Secret: &api.Secret{ID: id},
},
},
Action: api.AssignmentChange_AssignmentActionRemove,
}
update.Changes = append(update.Changes, secretChange)
}
if err := sendMessage(update, api.AssignmentsMessage_INCREMENTAL); err != nil {
return err
}
}
}
}
func (d *Dispatcher) processUpdates(ctx context.Context) {
var (
taskUpdates map[string]*api.TaskStatus
nodeUpdates map[string]nodeUpdate
)
d.taskUpdatesLock.Lock()
if len(d.taskUpdates) != 0 {
taskUpdates = d.taskUpdates
d.taskUpdates = make(map[string]*api.TaskStatus)
}
d.taskUpdatesLock.Unlock()
d.nodeUpdatesLock.Lock()
if len(d.nodeUpdates) != 0 {
nodeUpdates = d.nodeUpdates
d.nodeUpdates = make(map[string]nodeUpdate)
}
d.nodeUpdatesLock.Unlock()
if len(taskUpdates) == 0 && len(nodeUpdates) == 0 {
return
}
log := log.G(ctx).WithFields(logrus.Fields{
"method": "(*Dispatcher).processUpdates",
})
_, err := d.store.Batch(func(batch *store.Batch) error {
for taskID, status := range taskUpdates {
err := batch.Update(func(tx store.Tx) error {
logger := log.WithField("task.id", taskID)
task := store.GetTask(tx, taskID)
if task == nil {
logger.Errorf("task unavailable")
return nil
}
logger = logger.WithField("state.transition", fmt.Sprintf("%v->%v", task.Status.State, status.State))
if task.Status == *status {
logger.Debug("task status identical, ignoring")
return nil
}
if task.Status.State > status.State {
logger.Debug("task status invalid transition")
return nil
}
task.Status = *status
if err := store.UpdateTask(tx, task); err != nil {
logger.WithError(err).Error("failed to update task status")
return nil
}
logger.Debug("task status updated")
return nil
})
if err != nil {
log.WithError(err).Error("dispatcher task update transaction failed")
}
}
for nodeID, nodeUpdate := range nodeUpdates {
err := batch.Update(func(tx store.Tx) error {
logger := log.WithField("node.id", nodeID)
node := store.GetNode(tx, nodeID)
if node == nil {
logger.Errorf("node unavailable")
return nil
}
if nodeUpdate.status != nil {
node.Status.State = nodeUpdate.status.State
node.Status.Message = nodeUpdate.status.Message
if nodeUpdate.status.Addr != "" {
node.Status.Addr = nodeUpdate.status.Addr
}
}
if nodeUpdate.description != nil {
node.Description = nodeUpdate.description
}
if err := store.UpdateNode(tx, node); err != nil {
logger.WithError(err).Error("failed to update node status")
return nil
}
logger.Debug("node status updated")
return nil
})
if err != nil {
log.WithError(err).Error("dispatcher node update transaction failed")
}
}
return nil
})
if err != nil {
log.WithError(err).Error("dispatcher batch failed")
}
d.processUpdatesCond.Broadcast()
}
func (d *Dispatcher) markNodesUnknown(ctx context.Context) error {
log := log.G(ctx).WithField("method", "(*Dispatcher).markNodesUnknown")
var nodes []*api.Node
var err error
d.store.View(func(tx store.ReadTx) {
nodes, err = store.FindNodes(tx, store.All)
})
if err != nil {
return errors.Wrap(err, "failed to get list of nodes")
}
_, err = d.store.Batch(func(batch *store.Batch) error {
for _, n := range nodes {
err := batch.Update(func(tx store.Tx) error {
// check if node is still here
node := store.GetNode(tx, n.ID)
if node == nil {
return nil
}
// do not try to resurrect down nodes
if node.Status.State == api.NodeStatus_DOWN {
nodeCopy := node
expireFunc := func() {
if err := d.moveTasksToOrphaned(nodeCopy.ID); err != nil {
log.WithError(err).Error(`failed to move all tasks to "ORPHANED" state`)
}
d.downNodes.Delete(nodeCopy.ID)
}
d.downNodes.Add(nodeCopy, expireFunc)
return nil
}
node.Status.State = api.NodeStatus_UNKNOWN
node.Status.Message = `Node moved to "unknown" state due to leadership change in cluster`
nodeID := node.ID
expireFunc := func() {
log := log.WithField("node", nodeID)
log.Debugf("heartbeat expiration for unknown node")
if err := d.markNodeNotReady(nodeID, api.NodeStatus_DOWN, `heartbeat failure for node in "unknown" state`); err != nil {
log.WithError(err).Errorf(`failed deregistering node after heartbeat expiration for node in "unknown" state`)
}
}
if err := d.nodes.AddUnknown(node, expireFunc); err != nil {
return errors.Wrap(err, `adding node in "unknown" state to node store failed`)
}
if err := store.UpdateNode(tx, node); err != nil {
return errors.Wrap(err, "update failed")
}
return nil
})
if err != nil {
log.WithField("node", n.ID).WithError(err).Errorf(`failed to move node to "unknown" state`)
}
}
return nil
})
return err
}
func (s *session) watch(ctx context.Context) error {
log := log.G(ctx).WithFields(logrus.Fields{"method": "(*session).watch"})
log.Debugf("")
var (
resp *api.AssignmentsMessage
assignmentWatch api.Dispatcher_AssignmentsClient
tasksWatch api.Dispatcher_TasksClient
streamReference string
tasksFallback bool
err error
)
client := api.NewDispatcherClient(s.conn)
for {
// If this is the first time we're running the loop, or there was a reference mismatch
// attempt to get the assignmentWatch
if assignmentWatch == nil && !tasksFallback {
assignmentWatch, err = client.Assignments(ctx, &api.AssignmentsRequest{SessionID: s.sessionID})
if err != nil {
return err
}
}
// We have an assignmentWatch, let's try to receive an AssignmentMessage
if assignmentWatch != nil {
// If we get a code = 12 desc = unknown method Assignments, try to use tasks
resp, err = assignmentWatch.Recv()
if err != nil {
if grpc.Code(err) != codes.Unimplemented {
return err
}
tasksFallback = true
assignmentWatch = nil
log.WithError(err).Infof("falling back to Tasks")
}
}
// This code is here for backwards compatibility (so that newer clients can use the
// older method Tasks)
if tasksWatch == nil && tasksFallback {
tasksWatch, err = client.Tasks(ctx, &api.TasksRequest{SessionID: s.sessionID})
if err != nil {
return err
}
}
if tasksWatch != nil {
// When falling back to Tasks because of an old managers, we wrap the tasks in assignments.
var taskResp *api.TasksMessage
var assignmentChanges []*api.AssignmentChange
taskResp, err = tasksWatch.Recv()
if err != nil {
return err
}
for _, t := range taskResp.Tasks {
taskChange := &api.AssignmentChange{
Assignment: &api.Assignment{
Item: &api.Assignment_Task{
Task: t,
},
},
Action: api.AssignmentChange_AssignmentActionUpdate,
}
assignmentChanges = append(assignmentChanges, taskChange)
}
resp = &api.AssignmentsMessage{Type: api.AssignmentsMessage_COMPLETE, Changes: assignmentChanges}
}
// If there seems to be a gap in the stream, let's break out of the inner for and
// re-sync (by calling Assignments again).
if streamReference != "" && streamReference != resp.AppliesTo {
assignmentWatch = nil
} else {
streamReference = resp.ResultsIn
}
select {
case s.assignments <- resp:
case <-s.closed:
return errSessionClosed
case <-ctx.Done():
return ctx.Err()
}
}
}