func TestHeartbeatTimeout(t *testing.T) {
cfg := DefaultConfig()
cfg.HeartbeatPeriod = 100 * time.Millisecond
cfg.HeartbeatEpsilon = 0
gd, err := startDispatcher(cfg)
assert.NoError(t, err)
defer gd.Close()
var expectedSessionID string
{
stream, err := gd.Clients[0].Session(context.Background(), &api.SessionRequest{})
assert.NoError(t, err)
resp, err := stream.Recv()
assert.NoError(t, err)
assert.NotEmpty(t, resp.SessionID)
expectedSessionID = resp.SessionID
}
time.Sleep(500 * time.Millisecond)
gd.Store.View(func(readTx store.ReadTx) {
storeNodes, err := store.FindNodes(readTx, store.All)
assert.NoError(t, err)
assert.NotEmpty(t, storeNodes)
assert.Equal(t, api.NodeStatus_DOWN, storeNodes[0].Status.State)
})
// check that node is deregistered
resp, err := gd.Clients[0].Heartbeat(context.Background(), &api.HeartbeatRequest{SessionID: expectedSessionID})
assert.Nil(t, resp)
assert.Error(t, err)
assert.Equal(t, grpc.ErrorDesc(err), ErrNodeNotRegistered.Error())
}
// CheckValuesOnNodes checks that all the nodes in the cluster have the same
// replicated data, generally used to check if a node can catch up with the logs
// correctly
func CheckValuesOnNodes(t *testing.T, clockSource *fakeclock.FakeClock, checkNodes map[uint64]*TestNode, ids []string, values []*api.Node) {
iteration := 0
for checkNodeID, node := range checkNodes {
assert.NoError(t, PollFunc(clockSource, func() error {
var err error
node.MemoryStore().View(func(tx store.ReadTx) {
var allNodes []*api.Node
allNodes, err = store.FindNodes(tx, store.All)
if err != nil {
return
}
for i, id := range ids {
n := store.GetNode(tx, id)
if n == nil {
err = fmt.Errorf("node %s not found on %d (iteration %d)", id, checkNodeID, iteration)
return
}
if !reflect.DeepEqual(values[i], n) {
err = fmt.Errorf("node %s did not match expected value on %d (iteration %d)", id, checkNodeID, iteration)
return
}
}
if len(allNodes) != len(ids) {
err = fmt.Errorf("expected %d nodes, got %d (iteration %d)", len(ids), len(allNodes), iteration)
return
}
})
return err
}))
iteration++
}
}
func TestIssueNodeCertificateAgentFromDifferentOrgRenewal(t *testing.T) {
tc := testutils.NewTestCA(t, ca.DefaultAcceptancePolicy())
defer tc.Stop()
csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node)
assert.NoError(t, err)
// Since we're using a client that has a different Organization, this request will be treated
// as a new certificate request, not allowing auto-renewal
role := api.NodeRoleManager
issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Role: role}
issueResponse, err := tc.NodeCAClients[3].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
assert.NotNil(t, issueResponse.NodeID)
assert.Equal(t, api.NodeMembershipPending, issueResponse.NodeMembership)
tc.MemoryStore.View(func(readTx store.ReadTx) {
storeNodes, err := store.FindNodes(readTx, store.All)
assert.NoError(t, err)
assert.NotEmpty(t, storeNodes)
found := false
for _, node := range storeNodes {
if node.ID == issueResponse.NodeID {
found = true
assert.Equal(t, api.IssuanceStatePending, node.Certificate.Status.State)
}
}
assert.True(t, found)
})
}
// Run is the ConstraintEnforcer's main loop.
func (ce *ConstraintEnforcer) Run() {
defer close(ce.doneChan)
watcher, cancelWatch := state.Watch(ce.store.WatchQueue(), state.EventUpdateNode{})
defer cancelWatch()
var (
nodes []*api.Node
err error
)
ce.store.View(func(readTx store.ReadTx) {
nodes, err = store.FindNodes(readTx, store.All)
})
if err != nil {
log.L.WithError(err).Error("failed to check nodes for noncompliant tasks")
} else {
for _, node := range nodes {
ce.shutdownNoncompliantTasks(node)
}
}
for {
select {
case event := <-watcher:
node := event.(state.EventUpdateNode).Node
ce.shutdownNoncompliantTasks(node)
case <-ce.stopChan:
return
}
}
}
func testRaftRestartCluster(t *testing.T, stagger bool) {
nodes, clockSource := raftutils.NewRaftCluster(t, tc)
defer raftutils.TeardownCluster(t, nodes)
// Propose a value
values := make([]*api.Node, 2)
var err error
values[0], err = raftutils.ProposeValue(t, nodes[1], "id1")
assert.NoError(t, err, "failed to propose value")
// Stop all nodes
for _, node := range nodes {
node.Server.Stop()
node.Shutdown()
}
raftutils.AdvanceTicks(clockSource, 5)
// Restart all nodes
i := 0
for k, node := range nodes {
if stagger && i != 0 {
raftutils.AdvanceTicks(clockSource, 1)
}
nodes[k] = raftutils.RestartNode(t, clockSource, node, false)
i++
}
raftutils.WaitForCluster(t, clockSource, nodes)
// Propose another value
values[1], err = raftutils.ProposeValue(t, raftutils.Leader(nodes), "id2")
assert.NoError(t, err, "failed to propose value")
for _, node := range nodes {
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
var err error
node.MemoryStore().View(func(tx store.ReadTx) {
var allNodes []*api.Node
allNodes, err = store.FindNodes(tx, store.All)
if err != nil {
return
}
if len(allNodes) != 2 {
err = fmt.Errorf("expected 2 nodes, got %d", len(allNodes))
return
}
for i, nodeID := range []string{"id1", "id2"} {
n := store.GetNode(tx, nodeID)
if !reflect.DeepEqual(n, values[i]) {
err = fmt.Errorf("node %s did not match expected value", nodeID)
return
}
}
})
return err
}))
}
}
// Run is roleManager's main loop.
func (rm *roleManager) Run() {
defer close(rm.doneChan)
var (
nodes []*api.Node
ticker *time.Ticker
tickerCh <-chan time.Time
)
watcher, cancelWatch, err := store.ViewAndWatch(rm.store,
func(readTx store.ReadTx) error {
var err error
nodes, err = store.FindNodes(readTx, store.All)
return err
},
state.EventUpdateNode{})
defer cancelWatch()
if err != nil {
log.L.WithError(err).Error("failed to check nodes for role changes")
} else {
for _, node := range nodes {
rm.pending[node.ID] = node
rm.reconcileRole(node)
}
if len(rm.pending) != 0 {
ticker = time.NewTicker(roleReconcileInterval)
tickerCh = ticker.C
}
}
for {
select {
case event := <-watcher:
node := event.(state.EventUpdateNode).Node
rm.pending[node.ID] = node
rm.reconcileRole(node)
if len(rm.pending) != 0 && ticker == nil {
ticker = time.NewTicker(roleReconcileInterval)
tickerCh = ticker.C
}
case <-tickerCh:
for _, node := range rm.pending {
rm.reconcileRole(node)
}
if len(rm.pending) == 0 {
ticker.Stop()
ticker = nil
tickerCh = nil
}
case <-rm.ctx.Done():
if ticker != nil {
ticker.Stop()
}
return
}
}
}
// UpdateNode updates a Node referenced by NodeID with the given NodeSpec.
// - Returns `NotFound` if the Node is not found.
// - Returns `InvalidArgument` if the NodeSpec is malformed.
// - Returns an error if the update fails.
func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) (*api.UpdateNodeResponse, error) {
if request.NodeID == "" || request.NodeVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateNodeSpec(request.Spec); err != nil {
return nil, err
}
var (
node *api.Node
demote bool
)
err := s.store.Update(func(tx store.Tx) error {
node = store.GetNode(tx, request.NodeID)
if node == nil {
return nil
}
// Demotion sanity checks.
if node.Spec.Role == api.NodeRoleManager && request.Spec.Role == api.NodeRoleWorker {
demote = true
managers, err := store.FindNodes(tx, store.ByRole(api.NodeRoleManager))
if err != nil {
return grpc.Errorf(codes.Internal, "internal store error: %v", err)
}
if len(managers) == 1 && managers[0].ID == node.ID {
return grpc.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm")
}
}
node.Meta.Version = *request.NodeVersion
node.Spec = *request.Spec.Copy()
return store.UpdateNode(tx, node)
})
if err != nil {
return nil, err
}
if node == nil {
return nil, grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
if demote && s.raft != nil {
memberlist := s.raft.GetMemberlist()
for raftID, member := range memberlist {
if member.NodeID == request.NodeID {
if err := s.raft.RemoveMember(ctx, raftID); err != nil {
return nil, err
}
break
}
}
}
return &api.UpdateNodeResponse{
Node: node,
}, nil
}
// UpdateNode updates a Node referenced by NodeID with the given NodeSpec.
// - Returns `NotFound` if the Node is not found.
// - Returns `InvalidArgument` if the NodeSpec is malformed.
// - Returns an error if the update fails.
func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) (*api.UpdateNodeResponse, error) {
if request.NodeID == "" || request.NodeVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateNodeSpec(request.Spec); err != nil {
return nil, err
}
var (
node *api.Node
member *membership.Member
)
err := s.store.Update(func(tx store.Tx) error {
node = store.GetNode(tx, request.NodeID)
if node == nil {
return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
// Demotion sanity checks.
if node.Spec.DesiredRole == api.NodeRoleManager && request.Spec.DesiredRole == api.NodeRoleWorker {
// Check for manager entries in Store.
managers, err := store.FindNodes(tx, store.ByRole(api.NodeRoleManager))
if err != nil {
return grpc.Errorf(codes.Internal, "internal store error: %v", err)
}
if len(managers) == 1 && managers[0].ID == node.ID {
return grpc.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm")
}
// Check for node in memberlist
if member = s.raft.GetMemberByNodeID(request.NodeID); member == nil {
return grpc.Errorf(codes.NotFound, "can't find manager in raft memberlist")
}
// Quorum safeguard
if !s.raft.CanRemoveMember(member.RaftID) {
return grpc.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum")
}
}
node.Meta.Version = *request.NodeVersion
node.Spec = *request.Spec.Copy()
return store.UpdateNode(tx, node)
})
if err != nil {
return nil, err
}
return &api.UpdateNodeResponse{
Node: node,
}, nil
}
func (d *Dispatcher) markNodesUnknown(ctx context.Context) error {
log := log.G(ctx).WithField("method", "(*Dispatcher).markNodesUnknown")
var nodes []*api.Node
var err error
d.store.View(func(tx store.ReadTx) {
nodes, err = store.FindNodes(tx, store.All)
})
if err != nil {
return fmt.Errorf("failed to get list of nodes: %v", err)
}
_, err = d.store.Batch(func(batch *store.Batch) error {
for _, n := range nodes {
err := batch.Update(func(tx store.Tx) error {
// check if node is still here
node := store.GetNode(tx, n.ID)
if node == nil {
return nil
}
// do not try to resurrect down nodes
if node.Status.State == api.NodeStatus_DOWN {
return nil
}
node.Status = api.NodeStatus{
State: api.NodeStatus_UNKNOWN,
Message: `Node moved to "unknown" state due to leadership change in cluster`,
}
nodeID := node.ID
expireFunc := func() {
log := log.WithField("node", nodeID)
nodeStatus := api.NodeStatus{State: api.NodeStatus_DOWN, Message: `heartbeat failure for node in "unknown" state`}
log.Debugf("heartbeat expiration for unknown node")
if err := d.nodeRemove(nodeID, nodeStatus); err != nil {
log.WithError(err).Errorf(`failed deregistering node after heartbeat expiration for node in "unknown" state`)
}
}
if err := d.nodes.AddUnknown(node, expireFunc); err != nil {
return fmt.Errorf(`adding node in "unknown" state to node store failed: %v`, err)
}
if err := store.UpdateNode(tx, node); err != nil {
return fmt.Errorf("update failed %v", err)
}
return nil
})
if err != nil {
log.WithField("node", n.ID).WithError(err).Errorf(`failed to move node to "unknown" state`)
}
}
return nil
})
return err
}
func TestHeartbeat(t *testing.T) {
cfg := DefaultConfig()
cfg.HeartbeatPeriod = 500 * time.Millisecond
cfg.HeartbeatEpsilon = 0
gd, err := startDispatcher(DefaultConfig())
assert.NoError(t, err)
defer gd.Close()
var expectedSessionID string
{
stream, err := gd.Clients[0].Session(context.Background(), &api.SessionRequest{})
assert.NoError(t, err)
defer stream.CloseSend()
resp, err := stream.Recv()
assert.NoError(t, err)
assert.NotEmpty(t, resp.SessionID)
expectedSessionID = resp.SessionID
}
time.Sleep(250 * time.Millisecond)
{
// heartbeat without correct SessionID should fail
resp, err := gd.Clients[0].Heartbeat(context.Background(), &api.HeartbeatRequest{})
assert.Nil(t, resp)
assert.Error(t, err)
assert.Equal(t, grpc.Code(err), codes.InvalidArgument)
}
resp, err := gd.Clients[0].Heartbeat(context.Background(), &api.HeartbeatRequest{SessionID: expectedSessionID})
assert.NoError(t, err)
assert.NotZero(t, resp.Period)
time.Sleep(300 * time.Millisecond)
gd.Store.View(func(readTx store.ReadTx) {
storeNodes, err := store.FindNodes(readTx, store.All)
assert.NoError(t, err)
assert.NotEmpty(t, storeNodes)
found := false
for _, node := range storeNodes {
if node.ID == gd.SecurityConfigs[0].ClientTLSCreds.NodeID() {
found = true
assert.Equal(t, api.NodeStatus_READY, node.Status.State)
}
}
assert.True(t, found)
})
}
// CheckNoValue checks that there is no value replicated on nodes, generally
// used to test the absence of a leader
func CheckNoValue(t *testing.T, clockSource *fakeclock.FakeClock, raftNode *TestNode) {
assert.NoError(t, PollFunc(clockSource, func() error {
var err error
raftNode.MemoryStore().View(func(tx store.ReadTx) {
var allNodes []*api.Node
allNodes, err = store.FindNodes(tx, store.All)
if err != nil {
return
}
if len(allNodes) != 0 {
err = fmt.Errorf("expected no nodes, got %d", len(allNodes))
}
})
return err
}))
}
func (s *Scheduler) buildNodeSet(tx store.ReadTx, tasksByNode map[string]map[string]*api.Task) error {
nodes, err := store.FindNodes(tx, store.All)
if err != nil {
return err
}
s.nodeSet.alloc(len(nodes))
for _, n := range nodes {
var resources api.Resources
if n.Description != nil && n.Description.Resources != nil {
resources = *n.Description.Resources
}
s.nodeSet.addOrUpdateNode(newNodeInfo(n, tasksByNode[n.ID], resources))
}
return nil
}
// CheckValue checks that the value has been propagated between raft members
func CheckValue(t *testing.T, clockSource *fakeclock.FakeClock, raftNode *TestNode, createdNode *api.Node) {
assert.NoError(t, PollFunc(clockSource, func() error {
var err error
raftNode.MemoryStore().View(func(tx store.ReadTx) {
var allNodes []*api.Node
allNodes, err = store.FindNodes(tx, store.All)
if err != nil {
return
}
if len(allNodes) != 1 {
err = fmt.Errorf("expected 1 node, got %d nodes", len(allNodes))
return
}
if !reflect.DeepEqual(allNodes[0], createdNode) {
err = errors.New("node did not match expected value")
}
})
return err
}))
}
// GetAllValuesOnNode returns all values on this node
func GetAllValuesOnNode(t *testing.T, clockSource *fakeclock.FakeClock, raftNode *TestNode) ([]string, []*api.Node) {
ids := []string{}
values := []*api.Node{}
assert.NoError(t, PollFunc(clockSource, func() error {
var err error
raftNode.MemoryStore().View(func(tx store.ReadTx) {
var allNodes []*api.Node
allNodes, err = store.FindNodes(tx, store.All)
if err != nil {
return
}
for _, node := range allNodes {
ids = append(ids, node.ID)
values = append(values, node)
}
})
return err
}))
return ids, values
}
func TestClusterReelection(t *testing.T) {
if !*integrationTests {
t.Skip("integration test")
}
mCount, aCount := 5, 15
c := createManagersCluster(t, mCount, aCount)
require.NoError(t, testutils.PollFunc(nil, c.pollRegister))
require.NoError(t, c.destroyLeader())
// let's down some managers in the meantime
require.NoError(t, c.destroyAgents(5))
// ensure that cluster will converge to expected number of agents, we need big timeout because of heartbeat times
require.NoError(t, testutils.PollFuncWithTimeout(nil, c.pollRegister, 30*time.Second))
leader, err := c.leader()
assert.NoError(t, err)
// check nodes in store
var nodes []*api.Node
leader.m.RaftNode.MemoryStore().View(func(tx store.ReadTx) {
ns, err := store.FindNodes(tx, store.All)
assert.NoError(t, err)
for _, n := range ns {
if n.Spec.Role == api.NodeRoleWorker {
nodes = append(nodes, n)
}
}
})
assert.NoError(t, err)
assert.Len(t, nodes, aCount, "there should be all nodes in store")
var downAgentsCount int
for _, node := range nodes {
if node.Status.State == api.NodeStatus_DOWN {
downAgentsCount++
continue
}
assert.Equal(t, api.NodeStatus_READY, node.Status.State, "there should be only down and ready nodes at this point")
}
assert.Equal(t, 5, downAgentsCount, "unexpected number of down agents")
}
func (s *Scheduler) buildNodeHeap(tx store.ReadTx, tasksByNode map[string]map[string]*api.Task) error {
nodes, err := store.FindNodes(tx, store.All)
if err != nil {
return err
}
s.nodeHeap.alloc(len(nodes))
i := 0
for _, n := range nodes {
var resources api.Resources
if n.Description != nil && n.Description.Resources != nil {
resources = *n.Description.Resources
}
s.nodeHeap.heap = append(s.nodeHeap.heap, newNodeInfo(n, tasksByNode[n.ID], resources))
s.nodeHeap.index[n.ID] = i
i++
}
heap.Init(&s.nodeHeap)
return nil
}
// ListNodes returns a list of all nodes.
func (s *Server) ListNodes(ctx context.Context, request *api.ListNodesRequest) (*api.ListNodesResponse, error) {
var (
nodes []*api.Node
err error
)
s.store.View(func(tx store.ReadTx) {
switch {
case request.Filters != nil && len(request.Filters.Names) > 0:
nodes, err = store.FindNodes(tx, buildFilters(store.ByName, request.Filters.Names))
case request.Filters != nil && len(request.Filters.NamePrefixes) > 0:
nodes, err = store.FindNodes(tx, buildFilters(store.ByNamePrefix, request.Filters.NamePrefixes))
case request.Filters != nil && len(request.Filters.IDPrefixes) > 0:
nodes, err = store.FindNodes(tx, buildFilters(store.ByIDPrefix, request.Filters.IDPrefixes))
case request.Filters != nil && len(request.Filters.Roles) > 0:
filters := make([]store.By, 0, len(request.Filters.Roles))
for _, v := range request.Filters.Roles {
filters = append(filters, store.ByRole(v))
}
nodes, err = store.FindNodes(tx, store.Or(filters...))
case request.Filters != nil && len(request.Filters.Memberships) > 0:
filters := make([]store.By, 0, len(request.Filters.Memberships))
for _, v := range request.Filters.Memberships {
filters = append(filters, store.ByMembership(v))
}
nodes, err = store.FindNodes(tx, store.Or(filters...))
default:
nodes, err = store.FindNodes(tx, store.All)
}
})
if err != nil {
return nil, err
}
if request.Filters != nil {
nodes = filterNodes(nodes,
func(e *api.Node) bool {
if len(request.Filters.Names) == 0 {
return true
}
if e.Description == nil {
return false
}
return filterContains(e.Description.Hostname, request.Filters.Names)
},
func(e *api.Node) bool {
if len(request.Filters.NamePrefixes) == 0 {
return true
}
if e.Description == nil {
return false
}
return filterContainsPrefix(e.Description.Hostname, request.Filters.NamePrefixes)
},
func(e *api.Node) bool {
return filterContainsPrefix(e.ID, request.Filters.IDPrefixes)
},
func(e *api.Node) bool {
if len(request.Filters.Labels) == 0 {
return true
}
if e.Description == nil {
return false
}
return filterMatchLabels(e.Description.Engine.Labels, request.Filters.Labels)
},
func(e *api.Node) bool {
if len(request.Filters.Roles) == 0 {
return true
}
for _, c := range request.Filters.Roles {
if c == e.Spec.Role {
return true
}
}
return false
},
func(e *api.Node) bool {
if len(request.Filters.Memberships) == 0 {
return true
}
for _, c := range request.Filters.Memberships {
if c == e.Spec.Membership {
return true
}
}
return false
},
)
}
// Add in manager information on nodes that are managers
if s.raft != nil {
memberlist := s.raft.GetMemberlist()
for _, node := range nodes {
for _, member := range memberlist {
if member.NodeID == node.ID {
node.ManagerStatus = &api.ManagerStatus{
RaftID: member.RaftID,
Addr: member.Addr,
Leader: member.Status.Leader,
Reachability: member.Status.Reachability,
}
break
}
}
}
}
return &api.ListNodesResponse{
Nodes: nodes,
}, nil
}
func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
na, err := networkallocator.New()
if err != nil {
return err
}
nc := &networkContext{
nwkAllocator: na,
unallocatedTasks: make(map[string]*api.Task),
unallocatedServices: make(map[string]*api.Service),
unallocatedNetworks: make(map[string]*api.Network),
ingressNetwork: newIngressNetwork(),
}
a.netCtx = nc
defer func() {
// Clear a.netCtx if initialization was unsuccessful.
if err != nil {
a.netCtx = nil
}
}()
// Check if we have the ingress network. If not found create
// it before reading all network objects for allocation.
var networks []*api.Network
a.store.View(func(tx store.ReadTx) {
networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName))
if len(networks) > 0 {
nc.ingressNetwork = networks[0]
}
})
if err != nil {
return errors.Wrap(err, "failed to find ingress network during init")
}
// If ingress network is not found, create one right away
// using the predefined template.
if len(networks) == 0 {
if err := a.store.Update(func(tx store.Tx) error {
nc.ingressNetwork.ID = identity.NewID()
if err := store.CreateNetwork(tx, nc.ingressNetwork); err != nil {
return err
}
return nil
}); err != nil {
return errors.Wrap(err, "failed to create ingress network")
}
a.store.View(func(tx store.ReadTx) {
networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName))
if len(networks) > 0 {
nc.ingressNetwork = networks[0]
}
})
if err != nil {
return errors.Wrap(err, "failed to find ingress network after creating it")
}
}
// Try to complete ingress network allocation before anything else so
// that the we can get the preferred subnet for ingress
// network.
if !na.IsAllocated(nc.ingressNetwork) {
if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil {
log.G(ctx).WithError(err).Error("failed allocating ingress network during init")
} else if _, err := a.store.Batch(func(batch *store.Batch) error {
if err := a.commitAllocatedNetwork(ctx, batch, nc.ingressNetwork); err != nil {
log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
}
return nil
}); err != nil {
log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
}
}
// Allocate networks in the store so far before we started
// watching.
a.store.View(func(tx store.ReadTx) {
networks, err = store.FindNetworks(tx, store.All)
})
if err != nil {
return errors.Wrap(err, "error listing all networks in store while trying to allocate during init")
}
var allocatedNetworks []*api.Network
for _, n := range networks {
if na.IsAllocated(n) {
continue
}
if err := a.allocateNetwork(ctx, n); err != nil {
log.G(ctx).WithError(err).Errorf("failed allocating network %s during init", n.ID)
continue
}
allocatedNetworks = append(allocatedNetworks, n)
}
if _, err := a.store.Batch(func(batch *store.Batch) error {
for _, n := range allocatedNetworks {
if err := a.commitAllocatedNetwork(ctx, batch, n); err != nil {
log.G(ctx).WithError(err).Errorf("failed committing allocation of network %s during init", n.ID)
}
}
return nil
}); err != nil {
log.G(ctx).WithError(err).Error("failed committing allocation of networks during init")
}
// Allocate nodes in the store so far before we process watched events.
var nodes []*api.Node
a.store.View(func(tx store.ReadTx) {
nodes, err = store.FindNodes(tx, store.All)
})
if err != nil {
return errors.Wrap(err, "error listing all nodes in store while trying to allocate during init")
}
var allocatedNodes []*api.Node
for _, node := range nodes {
if na.IsNodeAllocated(node) {
continue
}
if node.Attachment == nil {
node.Attachment = &api.NetworkAttachment{}
}
node.Attachment.Network = nc.ingressNetwork.Copy()
if err := a.allocateNode(ctx, node); err != nil {
log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s during init", node.ID)
continue
}
allocatedNodes = append(allocatedNodes, node)
}
if _, err := a.store.Batch(func(batch *store.Batch) error {
for _, node := range allocatedNodes {
if err := a.commitAllocatedNode(ctx, batch, node); err != nil {
log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s during init", node.ID)
}
}
return nil
}); err != nil {
log.G(ctx).WithError(err).Error("Failed to commit allocation of network resources for nodes during init")
}
// Allocate services in the store so far before we process watched events.
var services []*api.Service
a.store.View(func(tx store.ReadTx) {
services, err = store.FindServices(tx, store.All)
})
if err != nil {
return errors.Wrap(err, "error listing all services in store while trying to allocate during init")
}
var allocatedServices []*api.Service
for _, s := range services {
if nc.nwkAllocator.IsServiceAllocated(s) {
continue
}
if err := a.allocateService(ctx, s); err != nil {
log.G(ctx).WithError(err).Errorf("failed allocating service %s during init", s.ID)
continue
}
allocatedServices = append(allocatedServices, s)
}
if _, err := a.store.Batch(func(batch *store.Batch) error {
for _, s := range allocatedServices {
if err := a.commitAllocatedService(ctx, batch, s); err != nil {
log.G(ctx).WithError(err).Errorf("failed committing allocation of service %s during init", s.ID)
}
}
return nil
}); err != nil {
log.G(ctx).WithError(err).Error("failed committing allocation of services during init")
}
// Allocate tasks in the store so far before we started watching.
var (
tasks []*api.Task
allocatedTasks []*api.Task
)
a.store.View(func(tx store.ReadTx) {
tasks, err = store.FindTasks(tx, store.All)
})
if err != nil {
return errors.Wrap(err, "error listing all tasks in store while trying to allocate during init")
}
for _, t := range tasks {
if taskDead(t) {
continue
}
var s *api.Service
if t.ServiceID != "" {
a.store.View(func(tx store.ReadTx) {
s = store.GetService(tx, t.ServiceID)
})
}
// Populate network attachments in the task
// based on service spec.
a.taskCreateNetworkAttachments(t, s)
if taskReadyForNetworkVote(t, s, nc) {
if t.Status.State >= api.TaskStatePending {
continue
}
if a.taskAllocateVote(networkVoter, t.ID) {
// If the task is not attached to any network, network
// allocators job is done. Immediately cast a vote so
// that the task can be moved to ALLOCATED state as
// soon as possible.
allocatedTasks = append(allocatedTasks, t)
}
continue
}
err := a.allocateTask(ctx, t)
if err == nil {
allocatedTasks = append(allocatedTasks, t)
} else if err != errNoChanges {
log.G(ctx).WithError(err).Errorf("failed allocating task %s during init", t.ID)
nc.unallocatedTasks[t.ID] = t
}
}
if _, err := a.store.Batch(func(batch *store.Batch) error {
for _, t := range allocatedTasks {
if err := a.commitAllocatedTask(ctx, batch, t); err != nil {
log.G(ctx).WithError(err).Errorf("failed committing allocation of task %s during init", t.ID)
}
}
return nil
}); err != nil {
log.G(ctx).WithError(err).Error("failed committing allocation of tasks during init")
}
return nil
}
func TestNodeCertificateAccept(t *testing.T) {
tc := testutils.NewTestCA(t, ca.DefaultAcceptancePolicy())
defer tc.Stop()
csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node)
assert.NoError(t, err)
testNode := &api.Node{
ID: "nodeID",
Spec: api.NodeSpec{
Membership: api.NodeMembershipAccepted,
Role: api.NodeRoleWorker,
},
Certificate: api.Certificate{
CN: "nodeID",
CSR: csr,
Status: api.IssuanceStatus{State: api.IssuanceStatePending},
},
}
err = tc.MemoryStore.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNode(tx, testNode))
return nil
})
assert.NoError(t, err)
statusRequest := &api.NodeCertificateStatusRequest{NodeID: "nodeID"}
resp, err := tc.NodeCAClients[1].NodeCertificateStatus(context.Background(), statusRequest)
assert.NoError(t, err)
assert.NotEmpty(t, resp.Certificate)
assert.NotEmpty(t, resp.Status)
assert.NotNil(t, resp.Certificate.Certificate)
assert.Equal(t, api.IssuanceStateIssued, resp.Status.State)
tc.MemoryStore.View(func(readTx store.ReadTx) {
storeNodes, err := store.FindNodes(readTx, store.All)
assert.NoError(t, err)
assert.NotEmpty(t, storeNodes)
var found bool
for _, node := range storeNodes {
if node.ID == "nodeID" {
assert.Equal(t, api.IssuanceStateIssued, node.Certificate.Status.State)
found = true
}
}
assert.True(t, found)
})
// Try it one more time for Worker, this time end-to-end
role := api.NodeRoleWorker
issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Role: role}
issueResponse, err := tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
require.NoError(t, err)
assert.NotNil(t, issueResponse.NodeID)
assert.Equal(t, api.NodeMembershipAccepted, issueResponse.NodeMembership)
// Try it one more time for Worker, this time end-to-end with manager
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role}
issueResponse, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
require.NoError(t, err)
assert.NotNil(t, issueResponse.NodeID)
assert.Equal(t, api.NodeMembershipPending, issueResponse.NodeMembership)
}
func TestRaftForceNewCluster(t *testing.T) {
t.Parallel()
nodes, clockSource := raftutils.NewRaftCluster(t, tc)
// Propose a value
values := make([]*api.Node, 2)
var err error
values[0], err = raftutils.ProposeValue(t, nodes[1], "id1")
assert.NoError(t, err, "failed to propose value")
// The memberlist should contain 3 members on each node
for i := 1; i <= 3; i++ {
assert.Equal(t, len(nodes[uint64(i)].GetMemberlist()), 3)
}
// Stop all nodes
for _, node := range nodes {
node.Server.Stop()
node.Shutdown()
}
raftutils.AdvanceTicks(clockSource, 5)
toClean := map[uint64]*raftutils.TestNode{
2: nodes[2],
3: nodes[3],
}
raftutils.TeardownCluster(t, toClean)
delete(nodes, 2)
delete(nodes, 3)
// Only restart the first node with force-new-cluster option
nodes[1] = raftutils.RestartNode(t, clockSource, nodes[1], true)
raftutils.WaitForCluster(t, clockSource, nodes)
// The memberlist should contain only one node (self)
assert.Equal(t, len(nodes[1].GetMemberlist()), 1)
// Add 2 more members
nodes[2] = raftutils.NewJoinNode(t, clockSource, nodes[1].Address, tc)
raftutils.WaitForCluster(t, clockSource, nodes)
nodes[3] = raftutils.NewJoinNode(t, clockSource, nodes[1].Address, tc)
raftutils.WaitForCluster(t, clockSource, nodes)
newCluster := map[uint64]*raftutils.TestNode{
1: nodes[1],
2: nodes[2],
3: nodes[3],
}
defer raftutils.TeardownCluster(t, newCluster)
// The memberlist should contain 3 members on each node
for i := 1; i <= 3; i++ {
assert.Equal(t, len(nodes[uint64(i)].GetMemberlist()), 3)
}
// Propose another value
values[1], err = raftutils.ProposeValue(t, raftutils.Leader(nodes), "id2")
assert.NoError(t, err, "failed to propose value")
for _, node := range nodes {
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
var err error
node.MemoryStore().View(func(tx store.ReadTx) {
var allNodes []*api.Node
allNodes, err = store.FindNodes(tx, store.All)
if err != nil {
return
}
if len(allNodes) != 2 {
err = fmt.Errorf("expected 2 nodes, got %d", len(allNodes))
return
}
for i, nodeID := range []string{"id1", "id2"} {
n := store.GetNode(tx, nodeID)
if !reflect.DeepEqual(n, values[i]) {
err = fmt.Errorf("node %s did not match expected value", nodeID)
return
}
}
})
return err
}))
}
}
func (d *Dispatcher) markNodesUnknown(ctx context.Context) error {
log := log.G(ctx).WithField("method", "(*Dispatcher).markNodesUnknown")
var nodes []*api.Node
var err error
d.store.View(func(tx store.ReadTx) {
nodes, err = store.FindNodes(tx, store.All)
})
if err != nil {
return errors.Wrap(err, "failed to get list of nodes")
}
_, err = d.store.Batch(func(batch *store.Batch) error {
for _, n := range nodes {
err := batch.Update(func(tx store.Tx) error {
// check if node is still here
node := store.GetNode(tx, n.ID)
if node == nil {
return nil
}
// do not try to resurrect down nodes
if node.Status.State == api.NodeStatus_DOWN {
nodeCopy := node
expireFunc := func() {
if err := d.moveTasksToOrphaned(nodeCopy.ID); err != nil {
log.WithError(err).Error(`failed to move all tasks to "ORPHANED" state`)
}
d.downNodes.Delete(nodeCopy.ID)
}
d.downNodes.Add(nodeCopy, expireFunc)
return nil
}
node.Status.State = api.NodeStatus_UNKNOWN
node.Status.Message = `Node moved to "unknown" state due to leadership change in cluster`
nodeID := node.ID
expireFunc := func() {
log := log.WithField("node", nodeID)
log.Debugf("heartbeat expiration for unknown node")
if err := d.markNodeNotReady(nodeID, api.NodeStatus_DOWN, `heartbeat failure for node in "unknown" state`); err != nil {
log.WithError(err).Errorf(`failed deregistering node after heartbeat expiration for node in "unknown" state`)
}
}
if err := d.nodes.AddUnknown(node, expireFunc); err != nil {
return errors.Wrap(err, `adding node in "unknown" state to node store failed`)
}
if err := store.UpdateNode(tx, node); err != nil {
return errors.Wrap(err, "update failed")
}
return nil
})
if err != nil {
log.WithField("node", n.ID).WithError(err).Errorf(`failed to move node to "unknown" state`)
}
}
return nil
})
return err
}
// Run contains the GlobalOrchestrator event loop
func (g *GlobalOrchestrator) Run(ctx context.Context) error {
defer close(g.doneChan)
// Watch changes to services and tasks
queue := g.store.WatchQueue()
watcher, cancel := queue.Watch()
defer cancel()
// Get list of nodes
var (
nodes []*api.Node
err error
)
g.store.View(func(readTx store.ReadTx) {
nodes, err = store.FindNodes(readTx, store.All)
})
if err != nil {
return err
}
for _, n := range nodes {
// if a node is in drain state, do not add it
if isValidNode(n) {
g.nodes[n.ID] = struct{}{}
}
}
// Lookup global services
var existingServices []*api.Service
g.store.View(func(readTx store.ReadTx) {
existingServices, err = store.FindServices(readTx, store.All)
})
if err != nil {
return err
}
for _, s := range existingServices {
if isGlobalService(s) {
g.globalServices[s.ID] = s
g.reconcileOneService(ctx, s)
}
}
for {
select {
case event := <-watcher:
// TODO(stevvooe): Use ctx to limit running time of operation.
switch v := event.(type) {
case state.EventCreateService:
if !isGlobalService(v.Service) {
continue
}
g.globalServices[v.Service.ID] = v.Service
g.reconcileOneService(ctx, v.Service)
case state.EventUpdateService:
if !isGlobalService(v.Service) {
continue
}
g.globalServices[v.Service.ID] = v.Service
g.reconcileOneService(ctx, v.Service)
case state.EventDeleteService:
if !isGlobalService(v.Service) {
continue
}
deleteServiceTasks(ctx, g.store, v.Service)
// delete the service from service map
delete(g.globalServices, v.Service.ID)
g.restarts.ClearServiceHistory(v.Service.ID)
case state.EventCreateNode:
g.reconcileOneNode(ctx, v.Node)
case state.EventUpdateNode:
switch v.Node.Status.State {
// NodeStatus_DISCONNECTED is a transient state, no need to make any change
case api.NodeStatus_DOWN:
g.removeTasksFromNode(ctx, v.Node)
case api.NodeStatus_READY:
// node could come back to READY from DOWN or DISCONNECT
g.reconcileOneNode(ctx, v.Node)
}
case state.EventDeleteNode:
g.removeTasksFromNode(ctx, v.Node)
delete(g.nodes, v.Node.ID)
case state.EventUpdateTask:
if _, exists := g.globalServices[v.Task.ServiceID]; !exists {
continue
}
// global orchestrator needs to inspect when a task has terminated
// it should ignore tasks whose DesiredState is past running, which
// means the task has been processed
if isTaskTerminated(v.Task) {
g.restartTask(ctx, v.Task.ID, v.Task.ServiceID)
}
case state.EventDeleteTask:
// CLI allows deleting task
if _, exists := g.globalServices[v.Task.ServiceID]; !exists {
continue
}
g.reconcileServiceOneNode(ctx, v.Task.ServiceID, v.Task.NodeID)
}
case <-g.stopChan:
return nil
}
}
}
// Run runs the CA signer main loop.
// The CA signer can be stopped with cancelling ctx or calling Stop().
func (s *Server) Run(ctx context.Context) error {
s.mu.Lock()
if s.isRunning() {
s.mu.Unlock()
return fmt.Errorf("CA signer is already running")
}
s.wg.Add(1)
s.mu.Unlock()
defer s.wg.Done()
logger := log.G(ctx).WithField("module", "ca")
ctx = log.WithLogger(ctx, logger)
// Run() should never be called twice, but just in case, we're
// attempting to close the started channel in a safe way
select {
case <-s.started:
return fmt.Errorf("CA server cannot be started more than once")
default:
close(s.started)
}
// Retrieve the channels to keep track of changes in the cluster
// Retrieve all the currently registered nodes
var nodes []*api.Node
updates, cancel, err := store.ViewAndWatch(
s.store,
func(readTx store.ReadTx) error {
clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
if err != nil {
return err
}
if len(clusters) != 1 {
return fmt.Errorf("could not find cluster object")
}
s.updateCluster(ctx, clusters[0])
nodes, err = store.FindNodes(readTx, store.All)
return err
},
state.EventCreateNode{},
state.EventUpdateNode{},
state.EventUpdateCluster{},
)
// Do this after updateCluster has been called, so isRunning never
// returns true without joinTokens being set correctly.
s.mu.Lock()
s.ctx, s.cancel = context.WithCancel(ctx)
s.mu.Unlock()
if err != nil {
log.G(ctx).WithFields(logrus.Fields{
"method": "(*Server).Run",
}).WithError(err).Errorf("snapshot store view failed")
return err
}
defer cancel()
// We might have missed some updates if there was a leader election,
// so let's pick up the slack.
if err := s.reconcileNodeCertificates(ctx, nodes); err != nil {
// We don't return here because that means the Run loop would
// never run. Log an error instead.
log.G(ctx).WithFields(logrus.Fields{
"method": "(*Server).Run",
}).WithError(err).Errorf("error attempting to reconcile certificates")
}
// Watch for new nodes being created, new nodes being updated, and changes
// to the cluster
for {
select {
case event := <-updates:
switch v := event.(type) {
case state.EventCreateNode:
s.evaluateAndSignNodeCert(ctx, v.Node)
case state.EventUpdateNode:
// If this certificate is already at a final state
// no need to evaluate and sign it.
if !isFinalState(v.Node.Certificate.Status) {
s.evaluateAndSignNodeCert(ctx, v.Node)
}
case state.EventUpdateCluster:
s.updateCluster(ctx, v.Cluster)
}
case <-ctx.Done():
return ctx.Err()
case <-s.ctx.Done():
return nil
}
}
}
func TestGCWAL(t *testing.T) {
t.Parallel()
// Additional log entries from cluster setup, leader election
extraLogEntries := 5
// Number of large entries to propose
proposals := 8
// Bring up a 3 node cluster
nodes, clockSource := raftutils.NewRaftCluster(t, tc, &api.RaftConfig{SnapshotInterval: uint64(proposals + extraLogEntries), LogEntriesForSlowFollowers: 0})
for i := 0; i != proposals; i++ {
_, err := proposeLargeValue(t, nodes[1], DefaultProposalTime, fmt.Sprintf("id%d", i))
assert.NoError(t, err, "failed to propose value")
}
time.Sleep(250 * time.Millisecond)
// Snapshot should have been triggered just as the WAL rotated, so
// both WAL files should be preserved
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
dirents, err := ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "snap"))
if err != nil {
return err
}
if len(dirents) != 1 {
return fmt.Errorf("expected 1 snapshot, found %d", len(dirents))
}
dirents, err = ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "wal"))
if err != nil {
return err
}
var walCount int
for _, f := range dirents {
if strings.HasSuffix(f.Name(), ".wal") {
walCount++
}
}
if walCount != 2 {
return fmt.Errorf("expected 2 WAL files, found %d", walCount)
}
return nil
}))
raftutils.TeardownCluster(t, nodes)
// Repeat this test, but trigger the snapshot after the WAL has rotated
proposals++
nodes, clockSource = raftutils.NewRaftCluster(t, tc, &api.RaftConfig{SnapshotInterval: uint64(proposals + extraLogEntries), LogEntriesForSlowFollowers: 0})
defer raftutils.TeardownCluster(t, nodes)
for i := 0; i != proposals; i++ {
_, err := proposeLargeValue(t, nodes[1], DefaultProposalTime, fmt.Sprintf("id%d", i))
assert.NoError(t, err, "failed to propose value")
}
time.Sleep(250 * time.Millisecond)
// This time only one WAL file should be saved.
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
dirents, err := ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "snap"))
if err != nil {
return err
}
if len(dirents) != 1 {
return fmt.Errorf("expected 1 snapshot, found %d", len(dirents))
}
dirents, err = ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "wal"))
if err != nil {
return err
}
var walCount int
for _, f := range dirents {
if strings.HasSuffix(f.Name(), ".wal") {
walCount++
}
}
if walCount != 1 {
return fmt.Errorf("expected 1 WAL file, found %d", walCount)
}
return nil
}))
// Restart the whole cluster
for _, node := range nodes {
node.Server.Stop()
node.Shutdown()
}
raftutils.AdvanceTicks(clockSource, 5)
i := 0
for k, node := range nodes {
nodes[k] = raftutils.RestartNode(t, clockSource, node, false)
i++
}
raftutils.WaitForCluster(t, clockSource, nodes)
// Is the data intact after restart?
for _, node := range nodes {
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
var err error
node.MemoryStore().View(func(tx store.ReadTx) {
var allNodes []*api.Node
allNodes, err = store.FindNodes(tx, store.All)
if err != nil {
return
}
if len(allNodes) != proposals {
err = fmt.Errorf("expected %d nodes, got %d", proposals, len(allNodes))
return
}
})
return err
}))
}
// It should still be possible to propose values
_, err := raftutils.ProposeValue(t, raftutils.Leader(nodes), DefaultProposalTime, "newnode")
assert.NoError(t, err, "failed to propose value")
for _, node := range nodes {
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
var err error
node.MemoryStore().View(func(tx store.ReadTx) {
var allNodes []*api.Node
allNodes, err = store.FindNodes(tx, store.All)
if err != nil {
return
}
if len(allNodes) != proposals+1 {
err = fmt.Errorf("expected %d nodes, got %d", proposals, len(allNodes))
return
}
})
return err
}))
}
}
func (a *Allocator) doNetworkInit(ctx context.Context) error {
na, err := networkallocator.New()
if err != nil {
return err
}
nc := &networkContext{
nwkAllocator: na,
unallocatedTasks: make(map[string]*api.Task),
unallocatedServices: make(map[string]*api.Service),
unallocatedNetworks: make(map[string]*api.Network),
}
// Check if we have the ingress network. If not found create
// it before reading all network objects for allocation.
var networks []*api.Network
a.store.View(func(tx store.ReadTx) {
networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName))
if len(networks) > 0 {
ingressNetwork = networks[0]
}
})
if err != nil {
return fmt.Errorf("failed to find ingress network during init: %v", err)
}
// If ingress network is not found, create one right away
// using the predefined template.
if len(networks) == 0 {
if err := a.store.Update(func(tx store.Tx) error {
ingressNetwork.ID = identity.NewID()
if err := store.CreateNetwork(tx, ingressNetwork); err != nil {
return err
}
return nil
}); err != nil {
return fmt.Errorf("failed to create ingress network: %v", err)
}
a.store.View(func(tx store.ReadTx) {
networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName))
if len(networks) > 0 {
ingressNetwork = networks[0]
}
})
if err != nil {
return fmt.Errorf("failed to find ingress network after creating it: %v", err)
}
}
// Try to complete ingress network allocation before anything else so
// that the we can get the preferred subnet for ingress
// network.
if !na.IsAllocated(ingressNetwork) {
if err := a.allocateNetwork(ctx, nc, ingressNetwork); err != nil {
log.G(ctx).Errorf("failed allocating ingress network during init: %v", err)
}
// Update store after allocation
if err := a.store.Update(func(tx store.Tx) error {
if err := store.UpdateNetwork(tx, ingressNetwork); err != nil {
return err
}
return nil
}); err != nil {
return fmt.Errorf("failed to create ingress network: %v", err)
}
}
// Allocate networks in the store so far before we started
// watching.
a.store.View(func(tx store.ReadTx) {
networks, err = store.FindNetworks(tx, store.All)
})
if err != nil {
return fmt.Errorf("error listing all networks in store while trying to allocate during init: %v", err)
}
for _, n := range networks {
if na.IsAllocated(n) {
continue
}
if err := a.allocateNetwork(ctx, nc, n); err != nil {
log.G(ctx).Errorf("failed allocating network %s during init: %v", n.ID, err)
}
}
// Allocate nodes in the store so far before we process watched events.
var nodes []*api.Node
a.store.View(func(tx store.ReadTx) {
nodes, err = store.FindNodes(tx, store.All)
})
if err != nil {
return fmt.Errorf("error listing all services in store while trying to allocate during init: %v", err)
}
for _, node := range nodes {
if na.IsNodeAllocated(node) {
continue
}
if node.Attachment == nil {
node.Attachment = &api.NetworkAttachment{}
}
node.Attachment.Network = ingressNetwork.Copy()
if err := a.allocateNode(ctx, nc, node); err != nil {
log.G(ctx).Errorf("Failed to allocate network resources for node %s during init: %v", node.ID, err)
}
}
// Allocate services in the store so far before we process watched events.
var services []*api.Service
a.store.View(func(tx store.ReadTx) {
services, err = store.FindServices(tx, store.All)
})
if err != nil {
return fmt.Errorf("error listing all services in store while trying to allocate during init: %v", err)
}
for _, s := range services {
if s.Spec.Endpoint == nil {
continue
}
if na.IsServiceAllocated(s) {
continue
}
if err := a.allocateService(ctx, nc, s); err != nil {
log.G(ctx).Errorf("failed allocating service %s during init: %v", s.ID, err)
}
}
// Allocate tasks in the store so far before we started watching.
var tasks []*api.Task
a.store.View(func(tx store.ReadTx) {
tasks, err = store.FindTasks(tx, store.All)
})
if err != nil {
return fmt.Errorf("error listing all tasks in store while trying to allocate during init: %v", err)
}
if _, err := a.store.Batch(func(batch *store.Batch) error {
for _, t := range tasks {
if taskDead(t) {
continue
}
var s *api.Service
if t.ServiceID != "" {
a.store.View(func(tx store.ReadTx) {
s = store.GetService(tx, t.ServiceID)
})
}
// Populate network attachments in the task
// based on service spec.
a.taskCreateNetworkAttachments(t, s)
if taskReadyForNetworkVote(t, s, nc) {
if t.Status.State >= api.TaskStateAllocated {
continue
}
if a.taskAllocateVote(networkVoter, t.ID) {
// If the task is not attached to any network, network
// allocators job is done. Immediately cast a vote so
// that the task can be moved to ALLOCATED state as
// soon as possible.
if err := batch.Update(func(tx store.Tx) error {
storeT := store.GetTask(tx, t.ID)
if storeT == nil {
return fmt.Errorf("task %s not found while trying to update state", t.ID)
}
updateTaskStatus(storeT, api.TaskStateAllocated, "allocated")
if err := store.UpdateTask(tx, storeT); err != nil {
return fmt.Errorf("failed updating state in store transaction for task %s: %v", storeT.ID, err)
}
return nil
}); err != nil {
log.G(ctx).WithError(err).Error("error updating task network")
}
}
continue
}
err := batch.Update(func(tx store.Tx) error {
_, err := a.allocateTask(ctx, nc, tx, t)
return err
})
if err != nil {
log.G(ctx).Errorf("failed allocating task %s during init: %v", t.ID, err)
nc.unallocatedTasks[t.ID] = t
}
}
return nil
}); err != nil {
return err
}
a.netCtx = nc
return nil
}
// Run contains the global orchestrator event loop
func (g *Orchestrator) Run(ctx context.Context) error {
defer close(g.doneChan)
// Watch changes to services and tasks
queue := g.store.WatchQueue()
watcher, cancel := queue.Watch()
defer cancel()
// lookup the cluster
var err error
g.store.View(func(readTx store.ReadTx) {
var clusters []*api.Cluster
clusters, err = store.FindClusters(readTx, store.ByName("default"))
if len(clusters) != 1 {
return // just pick up the cluster when it is created.
}
g.cluster = clusters[0]
})
if err != nil {
return err
}
// Get list of nodes
var nodes []*api.Node
g.store.View(func(readTx store.ReadTx) {
nodes, err = store.FindNodes(readTx, store.All)
})
if err != nil {
return err
}
for _, n := range nodes {
g.updateNode(n)
}
// Lookup global services
var existingServices []*api.Service
g.store.View(func(readTx store.ReadTx) {
existingServices, err = store.FindServices(readTx, store.All)
})
if err != nil {
return err
}
var reconcileServiceIDs []string
for _, s := range existingServices {
if orchestrator.IsGlobalService(s) {
g.updateService(s)
reconcileServiceIDs = append(reconcileServiceIDs, s.ID)
}
}
g.reconcileServices(ctx, reconcileServiceIDs)
for {
select {
case event := <-watcher:
// TODO(stevvooe): Use ctx to limit running time of operation.
switch v := event.(type) {
case state.EventUpdateCluster:
g.cluster = v.Cluster
case state.EventCreateService:
if !orchestrator.IsGlobalService(v.Service) {
continue
}
g.updateService(v.Service)
g.reconcileServices(ctx, []string{v.Service.ID})
case state.EventUpdateService:
if !orchestrator.IsGlobalService(v.Service) {
continue
}
g.updateService(v.Service)
g.reconcileServices(ctx, []string{v.Service.ID})
case state.EventDeleteService:
if !orchestrator.IsGlobalService(v.Service) {
continue
}
orchestrator.DeleteServiceTasks(ctx, g.store, v.Service)
// delete the service from service map
delete(g.globalServices, v.Service.ID)
g.restarts.ClearServiceHistory(v.Service.ID)
case state.EventCreateNode:
g.updateNode(v.Node)
g.reconcileOneNode(ctx, v.Node)
case state.EventUpdateNode:
g.updateNode(v.Node)
switch v.Node.Status.State {
// NodeStatus_DISCONNECTED is a transient state, no need to make any change
case api.NodeStatus_DOWN:
g.removeTasksFromNode(ctx, v.Node)
case api.NodeStatus_READY:
// node could come back to READY from DOWN or DISCONNECT
g.reconcileOneNode(ctx, v.Node)
}
case state.EventDeleteNode:
g.removeTasksFromNode(ctx, v.Node)
delete(g.nodes, v.Node.ID)
case state.EventUpdateTask:
if _, exists := g.globalServices[v.Task.ServiceID]; !exists {
continue
}
// global orchestrator needs to inspect when a task has terminated
// it should ignore tasks whose DesiredState is past running, which
// means the task has been processed
if isTaskTerminated(v.Task) {
g.restartTask(ctx, v.Task.ID, v.Task.ServiceID)
}
case state.EventDeleteTask:
// CLI allows deleting task
if _, exists := g.globalServices[v.Task.ServiceID]; !exists {
continue
}
g.reconcileServicesOneNode(ctx, []string{v.Task.ServiceID}, v.Task.NodeID)
}
case <-g.stopChan:
return nil
}
}
}
// Run runs the CA signer main loop.
// The CA signer can be stopped with cancelling ctx or calling Stop().
func (s *Server) Run(ctx context.Context) error {
s.mu.Lock()
if s.isRunning() {
s.mu.Unlock()
return errors.New("CA signer is already running")
}
s.wg.Add(1)
s.mu.Unlock()
defer s.wg.Done()
ctx = log.WithModule(ctx, "ca")
// Retrieve the channels to keep track of changes in the cluster
// Retrieve all the currently registered nodes
var nodes []*api.Node
updates, cancel, err := store.ViewAndWatch(
s.store,
func(readTx store.ReadTx) error {
clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
if err != nil {
return err
}
if len(clusters) != 1 {
return errors.New("could not find cluster object")
}
s.updateCluster(ctx, clusters[0])
nodes, err = store.FindNodes(readTx, store.All)
return err
},
state.EventCreateNode{},
state.EventUpdateNode{},
state.EventUpdateCluster{},
)
// Do this after updateCluster has been called, so isRunning never
// returns true without joinTokens being set correctly.
s.mu.Lock()
s.ctx, s.cancel = context.WithCancel(ctx)
s.mu.Unlock()
close(s.started)
if err != nil {
log.G(ctx).WithFields(logrus.Fields{
"method": "(*Server).Run",
}).WithError(err).Errorf("snapshot store view failed")
return err
}
defer cancel()
// We might have missed some updates if there was a leader election,
// so let's pick up the slack.
if err := s.reconcileNodeCertificates(ctx, nodes); err != nil {
// We don't return here because that means the Run loop would
// never run. Log an error instead.
log.G(ctx).WithFields(logrus.Fields{
"method": "(*Server).Run",
}).WithError(err).Errorf("error attempting to reconcile certificates")
}
ticker := time.NewTicker(s.reconciliationRetryInterval)
defer ticker.Stop()
// Watch for new nodes being created, new nodes being updated, and changes
// to the cluster
for {
select {
case event := <-updates:
switch v := event.(type) {
case state.EventCreateNode:
s.evaluateAndSignNodeCert(ctx, v.Node)
case state.EventUpdateNode:
// If this certificate is already at a final state
// no need to evaluate and sign it.
if !isFinalState(v.Node.Certificate.Status) {
s.evaluateAndSignNodeCert(ctx, v.Node)
}
case state.EventUpdateCluster:
s.updateCluster(ctx, v.Cluster)
}
case <-ticker.C:
for _, node := range s.pending {
if err := s.evaluateAndSignNodeCert(ctx, node); err != nil {
// If this sign operation did not succeed, the rest are
// unlikely to. Yield so that we don't hammer an external CA.
// Since the map iteration order is randomized, there is no
// risk of getting stuck on a problematic CSR.
break
}
}
case <-ctx.Done():
return ctx.Err()
case <-s.ctx.Done():
return nil
}
}
}