func TestIsConsulServer(t *testing.T) { m := serf.Member{ Name: "foo", Addr: net.IP([]byte{127, 0, 0, 1}), Tags: map[string]string{ "role": "consul", "dc": "east-aws", "port": "10000", "vsn": "1", }, } ok, parts := agent.IsConsulServer(m) if !ok || parts.Datacenter != "east-aws" || parts.Port != 10000 { t.Fatalf("bad: %v %v", ok, parts) } if parts.Name != "foo" { t.Fatalf("bad: %v", parts) } if parts.Bootstrap { t.Fatalf("unexpected bootstrap") } if parts.Expect != 0 { t.Fatalf("bad: %v", parts.Expect) } m.Tags["bootstrap"] = "1" m.Tags["disabled"] = "1" ok, parts = agent.IsConsulServer(m) if !ok { t.Fatalf("expected a valid consul server") } if !parts.Bootstrap { t.Fatalf("expected bootstrap") } if parts.Addr.String() != "127.0.0.1:10000" { t.Fatalf("bad addr: %v", parts.Addr) } if parts.Version != 1 { t.Fatalf("bad: %v", parts) } m.Tags["expect"] = "3" delete(m.Tags, "bootstrap") delete(m.Tags, "disabled") ok, parts = agent.IsConsulServer(m) if !ok || parts.Expect != 3 { t.Fatalf("bad: %v", parts.Expect) } if parts.Bootstrap { t.Fatalf("unexpected bootstrap") } delete(m.Tags, "role") ok, parts = agent.IsConsulServer(m) if ok { t.Fatalf("unexpected ok server") } }
// joinConsulServer is used to try to join another consul server func (s *Server) joinConsulServer(m serf.Member, parts *agent.Server) error { // Do not join ourself if m.Name == s.config.NodeName { return nil } // Check for possibility of multiple bootstrap nodes if parts.Bootstrap { members := s.serfLAN.Members() for _, member := range members { valid, p := agent.IsConsulServer(member) if valid && member.Name != m.Name && p.Bootstrap { s.logger.Printf("[ERR] consul: '%v' and '%v' are both in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.", m.Name, member.Name) return nil } } } // Attempt to add as a peer var addr net.Addr = &net.TCPAddr{IP: m.Addr, Port: parts.Port} future := s.raft.AddPeer(addr.String()) if err := future.Error(); err != nil && err != raft.ErrKnownPeer { s.logger.Printf("[ERR] consul: failed to add raft peer: %v", err) return err } return nil }
// wanNodeFailed is used to handle fail events on the WAN pool. func (s *Server) wanNodeFailed(me serf.MemberEvent) { for _, m := range me.Members { ok, parts := agent.IsConsulServer(m) if !ok { continue } s.logger.Printf("[INFO] consul: Removing WAN server %s", parts) // Remove the server if known s.remoteLock.Lock() existing := s.remoteConsuls[parts.Datacenter] n := len(existing) for i := 0; i < n; i++ { if existing[i].Name == parts.Name { existing[i], existing[n-1] = existing[n-1], nil existing = existing[:n-1] n-- break } } // Trim the list if all known consuls are dead if n == 0 { delete(s.remoteConsuls, parts.Datacenter) } else { s.remoteConsuls[parts.Datacenter] = existing } s.remoteLock.Unlock() } }
// handleDeregisterMember is used to deregister a member of a given reason func (s *Server) handleDeregisterMember(reason string, member serf.Member) error { // Do not deregister ourself. This can only happen if the current leader // is leaving. Instead, we should allow a follower to take-over and // deregister us later. if member.Name == s.config.NodeName { s.logger.Printf("[WARN] consul: deregistering self (%s) should be done by follower", s.config.NodeName) return nil } // Remove from Raft peers if this was a server if valid, parts := agent.IsConsulServer(member); valid { if err := s.removeConsulServer(member, parts.Port); err != nil { return err } } // Check if the node does not exist state := s.fsm.State() _, node, err := state.GetNode(member.Name) if err != nil { return err } if node == nil { return nil } // Deregister the node s.logger.Printf("[INFO] consul: member '%s' %s, deregistering", member.Name, reason) req := structs.DeregisterRequest{ Datacenter: s.config.Datacenter, Node: member.Name, } var out struct{} return s.endpoints.Catalog.Deregister(&req, &out) }
// wanNodeJoin is used to handle join events on the WAN pool. func (s *Server) wanNodeJoin(me serf.MemberEvent) { for _, m := range me.Members { ok, parts := agent.IsConsulServer(m) if !ok { s.logger.Printf("[WARN] consul: Non-server in WAN pool: %s", m.Name) continue } s.logger.Printf("[INFO] consul: Adding WAN server %s", parts) // Search for this node in our existing remotes. found := false s.remoteLock.Lock() existing := s.remoteConsuls[parts.Datacenter] for idx, e := range existing { if e.Name == parts.Name { existing[idx] = parts found = true break } } // Add to the list if not known. if !found { s.remoteConsuls[parts.Datacenter] = append(existing, parts) } s.remoteLock.Unlock() } }
// shouldHandleMember checks if this is a Consul pool member func (s *Server) shouldHandleMember(member serf.Member) bool { if valid, dc := isConsulNode(member); valid && dc == s.config.Datacenter { return true } if valid, parts := agent.IsConsulServer(member); valid && parts.Datacenter == s.config.Datacenter { return true } return false }
func (md *wanMergeDelegate) NotifyMerge(members []*serf.Member) error { for _, m := range members { ok, _ := agent.IsConsulServer(*m) if !ok { return fmt.Errorf("Member '%s' is not a server", m.Name) } } return nil }
// nodeFail is used to handle fail events on the serf cluster func (c *Client) nodeFail(me serf.MemberEvent) { for _, m := range me.Members { ok, parts := agent.IsConsulServer(m) if !ok { continue } c.logger.Printf("[INFO] consul: removing server %s", parts) c.servers.RemoveServer(parts) } }
// RaftGetConfiguration is used to retrieve the current Raft configuration. func (op *Operator) RaftGetConfiguration(args *structs.DCSpecificRequest, reply *structs.RaftConfigurationResponse) error { if done, err := op.srv.forward("Operator.RaftGetConfiguration", args, args, reply); done { return err } // This action requires operator read access. acl, err := op.srv.resolveToken(args.Token) if err != nil { return err } if acl != nil && !acl.OperatorRead() { return permissionDeniedErr } // We can't fetch the leader and the configuration atomically with // the current Raft API. future := op.srv.raft.GetConfiguration() if err := future.Error(); err != nil { return err } // Index the Consul information about the servers. serverMap := make(map[raft.ServerAddress]serf.Member) for _, member := range op.srv.serfLAN.Members() { valid, parts := agent.IsConsulServer(member) if !valid { continue } addr := (&net.TCPAddr{IP: member.Addr, Port: parts.Port}).String() serverMap[raft.ServerAddress(addr)] = member } // Fill out the reply. leader := op.srv.raft.Leader() reply.Index = future.Index() for _, server := range future.Configuration().Servers { node := "(unknown)" if member, ok := serverMap[server.Address]; ok { node = member.Name } entry := &structs.RaftServer{ ID: server.ID, Node: node, Address: server.Address, Leader: server.Address == leader, Voter: server.Suffrage == raft.Voter, } reply.Servers = append(reply.Servers, entry) } return nil }
// maybeBootsrap is used to handle bootstrapping when a new consul server joins func (s *Server) maybeBootstrap() { index, err := s.raftStore.LastIndex() if err != nil { s.logger.Printf("[ERR] consul: failed to read last raft index: %v", err) return } // Bootstrap can only be done if there are no committed logs, // remove our expectations of bootstrapping if index != 0 { s.config.BootstrapExpect = 0 return } // Scan for all the known servers members := s.serfLAN.Members() addrs := make([]string, 0) for _, member := range members { valid, p := agent.IsConsulServer(member) if !valid { continue } if p.Datacenter != s.config.Datacenter { s.logger.Printf("[ERR] consul: Member %v has a conflicting datacenter, ignoring", member) continue } if p.Expect != 0 && p.Expect != s.config.BootstrapExpect { s.logger.Printf("[ERR] consul: Member %v has a conflicting expect value. All nodes should expect the same number.", member) return } if p.Bootstrap { s.logger.Printf("[ERR] consul: Member %v has bootstrap mode. Expect disabled.", member) return } addr := &net.TCPAddr{IP: member.Addr, Port: p.Port} addrs = append(addrs, addr.String()) } // Skip if we haven't met the minimum expect count if len(addrs) < s.config.BootstrapExpect { return } // Update the peer set s.logger.Printf("[INFO] consul: Attempting bootstrap with nodes: %v", addrs) if err := s.raft.SetPeers(addrs).Error(); err != nil { s.logger.Printf("[ERR] consul: failed to bootstrap peers: %v", err) } // Bootstrapping complete, don't enter this again s.config.BootstrapExpect = 0 }
// lanNodeFailed is used to handle fail events on the LAN pool. func (s *Server) lanNodeFailed(me serf.MemberEvent) { for _, m := range me.Members { ok, parts := agent.IsConsulServer(m) if !ok { continue } s.logger.Printf("[INFO] consul: Removing LAN server %s", parts) s.localLock.Lock() delete(s.localConsuls, raft.ServerAddress(parts.Addr.String())) s.localLock.Unlock() } }
func (md *lanMergeDelegate) NotifyMerge(members []*serf.Member) error { for _, m := range members { ok, dc := isConsulNode(*m) if ok { if dc != md.dc { return fmt.Errorf("Member '%s' part of wrong datacenter '%s'", m.Name, dc) } continue } ok, parts := agent.IsConsulServer(*m) if ok && parts.Datacenter != md.dc { return fmt.Errorf("Member '%s' part of wrong datacenter '%s'", m.Name, parts.Datacenter) } } return nil }
// joinConsulServer is used to try to join another consul server func (s *Server) joinConsulServer(m serf.Member, parts *agent.Server) error { // Do not join ourself if m.Name == s.config.NodeName { return nil } // Check for possibility of multiple bootstrap nodes if parts.Bootstrap { members := s.serfLAN.Members() for _, member := range members { valid, p := agent.IsConsulServer(member) if valid && member.Name != m.Name && p.Bootstrap { s.logger.Printf("[ERR] consul: '%v' and '%v' are both in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.", m.Name, member.Name) return nil } } } // TODO (slackpad) - This will need to be changed once we support node IDs. addr := (&net.TCPAddr{IP: m.Addr, Port: parts.Port}).String() // See if it's already in the configuration. It's harmless to re-add it // but we want to avoid doing that if possible to prevent useless Raft // log entries. configFuture := s.raft.GetConfiguration() if err := configFuture.Error(); err != nil { s.logger.Printf("[ERR] consul: failed to get raft configuration: %v", err) return err } for _, server := range configFuture.Configuration().Servers { if server.Address == raft.ServerAddress(addr) { return nil } } // Attempt to add as a peer addFuture := s.raft.AddPeer(raft.ServerAddress(addr)) if err := addFuture.Error(); err != nil { s.logger.Printf("[ERR] consul: failed to add raft peer: %v", err) return err } return nil }
// nodeJoin is used to handle join events on the serf cluster func (c *Client) nodeJoin(me serf.MemberEvent) { for _, m := range me.Members { ok, parts := agent.IsConsulServer(m) if !ok { continue } if parts.Datacenter != c.config.Datacenter { c.logger.Printf("[WARN] consul: server %s for datacenter %s has joined wrong cluster", m.Name, parts.Datacenter) continue } c.logger.Printf("[INFO] consul: adding server %s", parts) c.servers.AddServer(parts) // Trigger the callback if c.config.ServerUp != nil { c.config.ServerUp() } } }
// lanNodeJoin is used to handle join events on the LAN pool. func (s *Server) lanNodeJoin(me serf.MemberEvent) { for _, m := range me.Members { ok, parts := agent.IsConsulServer(m) if !ok { continue } s.logger.Printf("[INFO] consul: Adding LAN server %s", parts) // See if it's configured as part of our DC. if parts.Datacenter == s.config.Datacenter { s.localLock.Lock() s.localConsuls[raft.ServerAddress(parts.Addr.String())] = parts s.localLock.Unlock() } // If we still expecting to bootstrap, may need to handle this. if s.config.BootstrapExpect != 0 { s.maybeBootstrap() } } }
// handleAliveMember is used to ensure the node // is registered, with a passing health check. func (s *Server) handleAliveMember(member serf.Member) error { // Register consul service if a server var service *structs.NodeService if valid, parts := agent.IsConsulServer(member); valid { service = &structs.NodeService{ ID: ConsulServiceID, Service: ConsulServiceName, Port: parts.Port, } // Attempt to join the consul server if err := s.joinConsulServer(member, parts); err != nil { return err } } // Check if the node exists state := s.fsm.State() _, node, err := state.GetNode(member.Name) if err != nil { return err } if node != nil && node.Address == member.Addr.String() { // Check if the associated service is available if service != nil { match := false _, services, err := state.NodeServices(member.Name) if err != nil { return err } if services != nil { for id, _ := range services.Services { if id == service.ID { match = true } } } if !match { goto AFTER_CHECK } } // Check if the serfCheck is in the passing state _, checks, err := state.NodeChecks(member.Name) if err != nil { return err } for _, check := range checks { if check.CheckID == SerfCheckID && check.Status == structs.HealthPassing { return nil } } } AFTER_CHECK: s.logger.Printf("[INFO] consul: member '%s' joined, marking health alive", member.Name) // Register with the catalog req := structs.RegisterRequest{ Datacenter: s.config.Datacenter, Node: member.Name, Address: member.Addr.String(), Service: service, Check: &structs.HealthCheck{ Node: member.Name, CheckID: SerfCheckID, Name: SerfCheckName, Status: structs.HealthPassing, Output: SerfCheckAliveOutput, }, WriteRequest: structs.WriteRequest{Token: s.config.ACLToken}, } var out struct{} return s.endpoints.Catalog.Register(&req, &out) }
// maybeBootsrap is used to handle bootstrapping when a new consul server joins func (s *Server) maybeBootstrap() { // Bootstrap can only be done if there are no committed logs, remove our // expectations of bootstrapping. This is slightly cheaper than the full // check that BootstrapCluster will do, so this is a good pre-filter. index, err := s.raftStore.LastIndex() if err != nil { s.logger.Printf("[ERR] consul: Failed to read last raft index: %v", err) return } if index != 0 { s.config.BootstrapExpect = 0 return } // Scan for all the known servers. members := s.serfLAN.Members() addrs := make([]string, 0) for _, member := range members { valid, p := agent.IsConsulServer(member) if !valid { continue } if p.Datacenter != s.config.Datacenter { s.logger.Printf("[ERR] consul: Member %v has a conflicting datacenter, ignoring", member) continue } if p.Expect != 0 && p.Expect != s.config.BootstrapExpect { s.logger.Printf("[ERR] consul: Member %v has a conflicting expect value. All nodes should expect the same number.", member) return } if p.Bootstrap { s.logger.Printf("[ERR] consul: Member %v has bootstrap mode. Expect disabled.", member) return } addr := &net.TCPAddr{IP: member.Addr, Port: p.Port} addrs = append(addrs, addr.String()) } // Skip if we haven't met the minimum expect count. if len(addrs) < s.config.BootstrapExpect { return } // Attempt a live bootstrap! var configuration raft.Configuration for _, addr := range addrs { // TODO (slackpad) - This will need to be updated once we support // node IDs. server := raft.Server{ ID: raft.ServerID(addr), Address: raft.ServerAddress(addr), } configuration.Servers = append(configuration.Servers, server) } s.logger.Printf("[INFO] consul: Found expected number of peers (%s), attempting to bootstrap cluster...", strings.Join(addrs, ",")) future := s.raft.BootstrapCluster(configuration) if err := future.Error(); err != nil { s.logger.Printf("[ERR] consul: Failed to bootstrap cluster: %v", err) } // Bootstrapping complete, don't enter this again. s.config.BootstrapExpect = 0 }
// maybeBootstrap is used to handle bootstrapping when a new consul server joins. func (s *Server) maybeBootstrap() { // Bootstrap can only be done if there are no committed logs, remove our // expectations of bootstrapping. This is slightly cheaper than the full // check that BootstrapCluster will do, so this is a good pre-filter. index, err := s.raftStore.LastIndex() if err != nil { s.logger.Printf("[ERR] consul: Failed to read last raft index: %v", err) return } if index != 0 { s.logger.Printf("[INFO] consul: Raft data found, disabling bootstrap mode") s.config.BootstrapExpect = 0 return } // Scan for all the known servers. members := s.serfLAN.Members() var servers []agent.Server for _, member := range members { valid, p := agent.IsConsulServer(member) if !valid { continue } if p.Datacenter != s.config.Datacenter { s.logger.Printf("[ERR] consul: Member %v has a conflicting datacenter, ignoring", member) continue } if p.Expect != 0 && p.Expect != s.config.BootstrapExpect { s.logger.Printf("[ERR] consul: Member %v has a conflicting expect value. All nodes should expect the same number.", member) return } if p.Bootstrap { s.logger.Printf("[ERR] consul: Member %v has bootstrap mode. Expect disabled.", member) return } servers = append(servers, *p) } // Skip if we haven't met the minimum expect count. if len(servers) < s.config.BootstrapExpect { return } // Query each of the servers and make sure they report no Raft peers. for _, server := range servers { var peers []string // Retry with exponential backoff to get peer status from this server for attempt := uint(0); attempt < maxPeerRetries; attempt++ { if err := s.connPool.RPC(s.config.Datacenter, server.Addr, server.Version, "Status.Peers", &struct{}{}, &peers); err != nil { nextRetry := time.Duration((1 << attempt) * peerRetryBase) s.logger.Printf("[ERR] consul: Failed to confirm peer status for %s: %v. Retrying in "+ "%v...", server.Name, err, nextRetry.String()) time.Sleep(nextRetry) } else { break } } // Found a node with some Raft peers, stop bootstrap since there's // evidence of an existing cluster. We should get folded in by the // existing servers if that's the case, so it's cleaner to sit as a // candidate with no peers so we don't cause spurious elections. // It's OK this is racy, because even with an initial bootstrap // as long as one peer runs bootstrap things will work, and if we // have multiple peers bootstrap in the same way, that's OK. We // just don't want a server added much later to do a live bootstrap // and interfere with the cluster. This isn't required for Raft's // correctness because no server in the existing cluster will vote // for this server, but it makes things much more stable. if len(peers) > 0 { s.logger.Printf("[INFO] consul: Existing Raft peers reported by %s, disabling bootstrap mode", server.Name) s.config.BootstrapExpect = 0 return } } // Attempt a live bootstrap! var configuration raft.Configuration var addrs []string for _, server := range servers { addr := server.Addr.String() addrs = append(addrs, addr) peer := raft.Server{ ID: raft.ServerID(addr), Address: raft.ServerAddress(addr), } configuration.Servers = append(configuration.Servers, peer) } s.logger.Printf("[INFO] consul: Found expected number of peers, attempting bootstrap: %s", strings.Join(addrs, ",")) future := s.raft.BootstrapCluster(configuration) if err := future.Error(); err != nil { s.logger.Printf("[ERR] consul: Failed to bootstrap cluster: %v", err) } // Bootstrapping complete, or failed for some reason, don't enter this // again. s.config.BootstrapExpect = 0 }