// Try all possible ways to find clusters to join // Include -discovery, -peers and log data in -data-dir // // Peer discovery follows this order: // 1. -discovery // 2. -peers // 3. previous peers in -data-dir func (s *PeerServer) findCluster(discoverURL string, peers []string) { // Attempt cluster discovery toDiscover := discoverURL != "" if toDiscover { discoverPeers, discoverErr := s.handleDiscovery(discoverURL) // It is registered in discover url if discoverErr == nil { // start as a leader in a new cluster if len(discoverPeers) == 0 { log.Debug("This peer is starting a brand new cluster based on discover URL.") s.startAsLeader() } else { s.startAsFollower(discoverPeers) } return } } hasPeerList := len(peers) > 0 // if there is log in data dir, append previous peers to peers in config // to find cluster prevPeers := s.registry.PeerURLs(s.raftServer.Leader(), s.Config.Name) for i := 0; i < len(prevPeers); i++ { u, err := url.Parse(prevPeers[i]) if err != nil { log.Debug("rejoin cannot parse url: ", err) } prevPeers[i] = u.Host } peers = append(peers, prevPeers...) // if there is backup peer lists, use it to find cluster if len(peers) > 0 { ok := s.joinCluster(peers) if !ok { log.Warn("No living peers are found!") } else { log.Debugf("%s restart as a follower based on peers[%v]", s.Config.Name) return } } if !s.raftServer.IsLogEmpty() { log.Debug("Entire cluster is down! %v will restart the cluster.", s.Config.Name) return } if toDiscover { log.Fatalf("Discovery failed, no available peers in backup list, and no log data") } if hasPeerList { log.Fatalf("No available peers in backup list, and no log data") } log.Infof("This peer is starting a brand new cluster now.") s.startAsLeader() }
// Start the raft server func (s *PeerServer) ListenAndServe(snapshot bool, cluster []string) error { // LoadSnapshot if snapshot { err := s.raftServer.LoadSnapshot() if err == nil { log.Debugf("%s finished load snapshot", s.name) } else { log.Debug(err) } } s.raftServer.SetElectionTimeout(s.ElectionTimeout) s.raftServer.SetHeartbeatTimeout(s.HeartbeatTimeout) s.raftServer.Start() if s.raftServer.IsLogEmpty() { // start as a leader in a new cluster if len(cluster) == 0 { s.startAsLeader() } else { s.startAsFollower(cluster) } } else { // Rejoin the previous cluster cluster = s.registry.PeerURLs(s.raftServer.Leader(), s.name) for i := 0; i < len(cluster); i++ { u, err := url.Parse(cluster[i]) if err != nil { log.Debug("rejoin cannot parse url: ", err) } cluster[i] = u.Host } ok := s.joinCluster(cluster) if !ok { log.Warn("the entire cluster is down! this peer will restart the cluster.") } log.Debugf("%s restart as a follower", s.name) } s.closeChan = make(chan bool) go s.monitorSync() go s.monitorTimeoutThreshold(s.closeChan) // open the snapshot if snapshot { go s.monitorSnapshot() } // start to response to raft requests return s.startTransport(s.tlsConf.Scheme, s.tlsConf.Server) }
// Response to the join request func (ps *PeerServer) JoinHttpHandler(w http.ResponseWriter, req *http.Request) { command := &JoinCommand{} // Write CORS header. if ps.server.OriginAllowed("*") { w.Header().Add("Access-Control-Allow-Origin", "*") } else if ps.server.OriginAllowed(req.Header.Get("Origin")) { w.Header().Add("Access-Control-Allow-Origin", req.Header.Get("Origin")) } err := decodeJsonRequest(req, command) if err != nil { w.WriteHeader(http.StatusInternalServerError) return } log.Debugf("Receive Join Request from %s", command.Name) err = ps.server.Dispatch(command, w, req) // Return status. if err != nil { if etcdErr, ok := err.(*etcdErr.Error); ok { log.Debug("Return error: ", (*etcdErr).Error()) etcdErr.Write(w) } else { http.Error(w, err.Error(), http.StatusInternalServerError) } } }
// Adds a server handler to the router. func (s *Server) handleFunc(r *mux.Router, path string, f func(http.ResponseWriter, *http.Request) error) *mux.Route { // Wrap the standard HandleFunc interface to pass in the server reference. return r.HandleFunc(path, func(w http.ResponseWriter, req *http.Request) { if req.Method == "HEAD" { w = &HEADResponseWriter{w} } // Log request. log.Debugf("[recv] %s %s %s [%s]", req.Method, s.URL(), req.URL.Path, req.RemoteAddr) // Forward request along if the server is a standby. if s.peerServer.Mode() == StandbyMode { if s.peerServer.standbyClientURL == "" { w.Header().Set("Content-Type", "application/json") etcdErr.NewError(402, "", 0).Write(w) return } uhttp.Redirect(s.peerServer.standbyClientURL, w, req) return } // Execute handler function and return error if necessary. if err := f(w, req); err != nil { if etcdErr, ok := err.(*etcdErr.Error); ok { log.Debug("Return error: ", (*etcdErr).Error()) w.Header().Set("Content-Type", "application/json") etcdErr.Write(w) } else { http.Error(w, err.Error(), http.StatusInternalServerError) } } }) }
// Start the raft server func (s *PeerServer) Start(snapshot bool, discoverURL string, peers []string) error { // LoadSnapshot if snapshot { err := s.raftServer.LoadSnapshot() if err == nil { log.Debugf("%s finished load snapshot", s.Config.Name) } else { log.Debug(err) } } s.raftServer.Start() s.findCluster(discoverURL, peers) s.closeChan = make(chan bool) go s.monitorSync() go s.monitorTimeoutThreshold(s.closeChan) // open the snapshot if snapshot { go s.monitorSnapshot() } return nil }
// Adds a server handler to the router. func (s *Server) handleFunc(path string, f func(http.ResponseWriter, *http.Request) error) *mux.Route { r := s.Handler.(*mux.Router) // Wrap the standard HandleFunc interface to pass in the server reference. return r.HandleFunc(path, func(w http.ResponseWriter, req *http.Request) { // Log request. log.Debugf("[recv] %s %s %s [%s]", req.Method, s.url, req.URL.Path, req.RemoteAddr) // Write CORS header. if s.OriginAllowed("*") { w.Header().Add("Access-Control-Allow-Origin", "*") } else if origin := req.Header.Get("Origin"); s.OriginAllowed(origin) { w.Header().Add("Access-Control-Allow-Origin", origin) } // Execute handler function and return error if necessary. if err := f(w, req); err != nil { if etcdErr, ok := err.(*etcdErr.Error); ok { log.Debug("Return error: ", (*etcdErr).Error()) w.Header().Set("Content-Type", "application/json") etcdErr.Write(w) } else { http.Error(w, err.Error(), http.StatusInternalServerError) } } }) }
// Join a server to the cluster func (c *JoinCommand) Apply(server raft.Server) (interface{}, error) { ps, _ := server.Context().(*PeerServer) b := make([]byte, 8) binary.PutUvarint(b, server.CommitIndex()) // Make sure we're not getting a cached value from the registry. ps.registry.Invalidate(c.Name) // Check if the join command is from a previous peer, who lost all its previous log. if _, ok := ps.registry.ClientURL(c.Name); ok { return b, nil } // Check peer number in the cluster if ps.registry.Count() == ps.MaxClusterSize { log.Debug("Reject join request from ", c.Name) return []byte{0}, etcdErr.NewError(etcdErr.EcodeNoMorePeer, "", server.CommitIndex()) } // Add to shared peer registry. ps.registry.Register(c.Name, c.RaftURL, c.EtcdURL) // Add peer in raft err := server.AddPeer(c.Name, "") // Add peer stats if c.Name != ps.RaftServer().Name() { ps.followersStats.Followers[c.Name] = &raftFollowerStats{} ps.followersStats.Followers[c.Name].Latency.Minimum = 1 << 63 } return b, err }
// applyJoin attempts to join a machine to the cluster. func applyJoin(c *JoinCommand, context raft.Context) (uint64, error) { ps, _ := context.Server().Context().(*PeerServer) commitIndex := context.CommitIndex() // Make sure we're not getting a cached value from the registry. ps.registry.Invalidate(c.Name) // Check if the join command is from a previous peer, who lost all its previous log. if peerURL, ok := ps.registry.PeerURL(c.Name); ok { // If previous node restarts with different peer URL, // update its information. if peerURL != c.RaftURL { log.Infof("Rejoin with %v instead of %v from %v", c.RaftURL, peerURL, c.Name) if err := updatePeerURL(c, ps); err != nil { return 0, err } } if c.Name == context.Server().Name() { ps.removedInLog = false } return commitIndex, nil } // Check if the join command adds an instance that collides with existing one on peer URL. peerURLs := ps.registry.PeerURLs(ps.raftServer.Leader(), c.Name) for _, peerURL := range peerURLs { if peerURL == c.RaftURL { log.Warnf("%v tries to join the cluster with existing URL %v", c.Name, c.EtcdURL) return 0, etcdErr.NewError(etcdErr.EcodeExistingPeerAddr, c.EtcdURL, context.CommitIndex()) } } // Check peer number in the cluster count := ps.registry.Count() // ClusterConfig doesn't init until first machine is added if count > 0 && count >= ps.ClusterConfig().ActiveSize { log.Debug("Reject join request from ", c.Name) return 0, etcdErr.NewError(etcdErr.EcodeNoMorePeer, "", context.CommitIndex()) } // Add to shared peer registry. ps.registry.Register(c.Name, c.RaftURL, c.EtcdURL) // Add peer in raft if err := context.Server().AddPeer(c.Name, ""); err != nil { return 0, err } // Add peer stats if c.Name != ps.RaftServer().Name() { ps.followersStats.Followers[c.Name] = &raftFollowerStats{} ps.followersStats.Followers[c.Name].Latency.Minimum = 1 << 63 } if c.Name == context.Server().Name() { ps.removedInLog = false } return commitIndex, nil }
// Send join requests to peer. func (s *PeerServer) joinByPeer(server raft.Server, peer string, scheme string) error { var b bytes.Buffer // t must be ok t, _ := server.Transporter().(*transporter) // Our version must match the leaders version versionURL := url.URL{Host: peer, Scheme: scheme, Path: "/version"} version, err := getVersion(t, versionURL) if err != nil { return fmt.Errorf("Error during join version check: %v", err) } if version < store.MinVersion() || version > store.MaxVersion() { return fmt.Errorf("Unable to join: cluster version is %d; version compatibility is %d - %d", version, store.MinVersion(), store.MaxVersion()) } json.NewEncoder(&b).Encode(NewJoinCommand(store.MinVersion(), store.MaxVersion(), server.Name(), s.Config.URL, s.server.URL())) joinURL := url.URL{Host: peer, Scheme: scheme, Path: "/join"} log.Debugf("Send Join Request to %s", joinURL.String()) resp, req, err := t.Post(joinURL.String(), &b) for { if err != nil { return fmt.Errorf("Unable to join: %v", err) } if resp != nil { defer resp.Body.Close() t.CancelWhenTimeout(req) if resp.StatusCode == http.StatusOK { b, _ := ioutil.ReadAll(resp.Body) s.joinIndex, _ = binary.Uvarint(b) return nil } if resp.StatusCode == http.StatusTemporaryRedirect { address := resp.Header.Get("Location") log.Debugf("Send Join Request to %s", address) json.NewEncoder(&b).Encode(NewJoinCommand(store.MinVersion(), store.MaxVersion(), server.Name(), s.Config.URL, s.server.URL())) resp, req, err = t.Post(address, &b) } else if resp.StatusCode == http.StatusBadRequest { log.Debug("Reach max number peers in the cluster") decoder := json.NewDecoder(resp.Body) err := &etcdErr.Error{} decoder.Decode(err) return *err } else { return fmt.Errorf("Unable to join") } } } }
// Get the key func (c *GetCommand) Apply(context raft.Context) (interface{}, error) { s, _ := context.Server().StateMachine().(store.Store) e, err := s.Get(c.Key, c.Recursive, c.Sorted) if err != nil { log.Debug(err) return nil, err } return e, nil }
// Delete the key func (c *DeleteCommand) Apply(server raft.Server) (interface{}, error) { s, _ := server.StateMachine().(store.Store) e, err := s.Delete(c.Key, c.Recursive) if err != nil { log.Debug(err) return nil, err } return e, nil }
// Create node func (c *UpdateCommand) Apply(server raft.Server) (interface{}, error) { s, _ := server.StateMachine().(store.Store) e, err := s.Update(c.Key, c.Value, c.ExpireTime) if err != nil { log.Debug(err) return nil, err } return e, nil }
// Set the key-value pair if the current value of the key equals to the given prevValue func (c *CompareAndDeleteCommand) Apply(server raft.Server) (interface{}, error) { s, _ := server.StateMachine().(store.Store) e, err := s.CompareAndDelete(c.Key, c.PrevValue, c.PrevIndex) if err != nil { log.Debug(err) return nil, err } return e, nil }
// Set the key-value pair if the current value of the key equals to the given prevValue func (c *CompareAndSwapCommand) Apply(context raft.Context) (interface{}, error) { s, _ := context.Server().StateMachine().(store.Store) e, err := s.CompareAndSwap(c.Key, c.PrevValue, c.PrevIndex, c.Value, c.ExpireTime) if err != nil { log.Debug(err) return nil, err } return e, nil }
// Create node func (c *CreateCommand) Apply(context raft.Context) (interface{}, error) { s, _ := context.Server().StateMachine().(store.Store) e, err := s.Create(c.Key, c.Dir, c.Value, c.Unique, c.ExpireTime) if err != nil { log.Debug(err) return nil, err } return e, nil }
// getKnownPeers gets the previous peers from log func (s *PeerServer) getKnownPeers() []string { peers := s.registry.PeerURLs(s.raftServer.Leader(), s.Config.Name) log.Infof("Peer URLs in log: %s / %s (%s)", s.raftServer.Leader(), s.Config.Name, strings.Join(peers, ",")) for i := range peers { u, err := url.Parse(peers[i]) if err != nil { log.Debug("getPrevPeers cannot parse url %v", peers[i]) } peers[i] = u.Host } return peers }
// Create node func (c *SetCommand) Apply(server raft.Server) (interface{}, error) { s, _ := server.StateMachine().(store.Store) // create a new node or replace the old node. e, err := s.Set(c.Key, c.Value, c.ExpireTime) if err != nil { log.Debug(err) return nil, err } return e, nil }
// Join a server to the cluster func (c *JoinCommandV1) Apply(context raft.Context) (interface{}, error) { ps, _ := context.Server().Context().(*PeerServer) b := make([]byte, 8) binary.PutUvarint(b, context.CommitIndex()) // Make sure we're not getting a cached value from the registry. ps.registry.Invalidate(c.Name) // Check if the join command is from a previous peer, who lost all its previous log. if peerURL, ok := ps.registry.PeerURL(c.Name); ok { // If previous node restarts with different peer URL, // update its information. if peerURL != c.RaftURL { log.Infof("Rejoin with %v instead of %v from %v", c.RaftURL, peerURL, c.Name) if err := c.updatePeerURL(ps); err != nil { return []byte{0}, err } } return b, nil } // Check if the join command adds an instance that collides with existing one on peer URL. peerURLs := ps.registry.PeerURLs(ps.raftServer.Leader(), c.Name) for _, peerURL := range peerURLs { if peerURL == c.RaftURL { log.Warnf("%v tries to join the cluster with existing URL %v", c.Name, c.EtcdURL) return []byte{0}, etcdErr.NewError(etcdErr.EcodeExistingPeerAddr, c.EtcdURL, context.CommitIndex()) } } // Check peer number in the cluster if ps.registry.PeerCount() >= ps.ClusterConfig().ActiveSize { log.Debug("Reject join request from ", c.Name) return []byte{0}, etcdErr.NewError(etcdErr.EcodeNoMorePeer, "", context.CommitIndex()) } // Add to shared peer registry. ps.registry.RegisterPeer(c.Name, c.RaftURL, c.EtcdURL) // Add peer in raft err := context.Server().AddPeer(c.Name, "") // Add peer stats if c.Name != ps.RaftServer().Name() { ps.followersStats.Followers[c.Name] = &raftFollowerStats{} ps.followersStats.Followers[c.Name].Latency.Minimum = 1 << 63 } return b, err }
// Delete the key func (c *DeleteCommand) Apply(context raft.Context) (interface{}, error) { s, _ := context.Server().StateMachine().(store.Store) if c.Recursive { // recursive implies dir c.Dir = true } e, err := s.Delete(c.Key, c.Dir, c.Recursive) if err != nil { log.Debug(err) return nil, err } return e, nil }
// Start starts the raft server. // The function assumes that join has been accepted successfully. func (s *PeerServer) Start(snapshot bool, discoverURL string, peers []string) error { s.Lock() defer s.Unlock() if s.started { return nil } s.started = true // LoadSnapshot if snapshot { err := s.raftServer.LoadSnapshot() if err == nil { log.Debugf("%s finished load snapshot", s.Config.Name) } else { log.Debug(err) } } s.raftServer.Init() // Set NOCOW for data directory in btrfs if btrfs.IsBtrfs(s.raftServer.LogPath()) { if err := btrfs.SetNOCOWFile(s.raftServer.LogPath()); err != nil { log.Warnf("Failed setting NOCOW: %v", err) } } s.findCluster(discoverURL, peers) s.stopNotify = make(chan bool) s.removeNotify = make(chan bool) s.closeChan = make(chan bool) s.startRoutine(s.monitorSync) s.startRoutine(s.monitorTimeoutThreshold) s.startRoutine(s.monitorActiveSize) s.startRoutine(s.monitorPeerActivity) // open the snapshot if snapshot { s.startRoutine(s.monitorSnapshot) } return nil }
// Adds a machine to the cluster. func (ps *PeerServer) addMachineHttpHandler(w http.ResponseWriter, req *http.Request) { c := &JoinCommandV2{} if err := uhttp.DecodeJsonRequest(req, c); err != nil { w.WriteHeader(http.StatusInternalServerError) return } log.Debugf("Receive Join Request (v2) from %s", c.Name) if err := ps.server.Dispatch(c, w, req); err != nil { if etcdErr, ok := err.(*etcdErr.Error); ok { log.Debug("Return error: ", (*etcdErr).Error()) etcdErr.Write(w) } else { http.Error(w, err.Error(), http.StatusInternalServerError) } } }
// Apply attempts to join a machine to the cluster. func (c *JoinCommandV2) Apply(context raft.Context) (interface{}, error) { ps, _ := context.Server().Context().(*PeerServer) var msg = joinMessageV2{ Mode: PeerMode, CommitIndex: context.CommitIndex(), } // Make sure we're not getting a cached value from the registry. ps.registry.Invalidate(c.Name) // Check if the join command is from a previous peer, who lost all its previous log. if _, ok := ps.registry.ClientURL(c.Name); ok { return json.Marshal(msg) } // Check peer number in the cluster. if ps.registry.PeerCount() >= ps.ClusterConfig().ActiveSize { log.Debug("Join as proxy ", c.Name) ps.registry.RegisterProxy(c.Name, c.PeerURL, c.ClientURL) msg.Mode = ProxyMode return json.Marshal(msg) } // Remove it as a proxy if it is one. if ps.registry.ProxyExists(c.Name) { ps.registry.UnregisterProxy(c.Name) } // Add to shared peer registry. ps.registry.RegisterPeer(c.Name, c.PeerURL, c.ClientURL) // Add peer in raft if err := context.Server().AddPeer(c.Name, ""); err != nil { b, _ := json.Marshal(msg) return b, err } // Add peer stats if c.Name != ps.RaftServer().Name() { ps.followersStats.Followers[c.Name] = &raftFollowerStats{} ps.followersStats.Followers[c.Name].Latency.Minimum = 1 << 63 } return json.Marshal(msg) }
// Adds a server handler to the router. func (s *Server) handleFunc(r *mux.Router, path string, f func(http.ResponseWriter, *http.Request) error) *mux.Route { // Wrap the standard HandleFunc interface to pass in the server reference. return r.HandleFunc(path, func(w http.ResponseWriter, req *http.Request) { // Log request. log.Debugf("[recv] %s %s %s [%s]", req.Method, s.URL(), req.URL.Path, req.RemoteAddr) // Execute handler function and return error if necessary. if err := f(w, req); err != nil { if etcdErr, ok := err.(*etcdErr.Error); ok { log.Debug("Return error: ", (*etcdErr).Error()) w.Header().Set("Content-Type", "application/json") etcdErr.Write(w) } else { http.Error(w, err.Error(), http.StatusInternalServerError) } } }) }
func (s *PeerServer) SetRaftServer(raftServer raft.Server, snapshot bool) { s.snapConf = &snapshotConf{ checkingInterval: time.Second * 3, // this is not accurate, we will update raft to provide an api lastIndex: raftServer.CommitIndex(), snapshotThr: uint64(s.Config.SnapshotCount), } raftServer.AddEventListener(raft.StateChangeEventType, s.raftEventLogger) raftServer.AddEventListener(raft.LeaderChangeEventType, s.raftEventLogger) raftServer.AddEventListener(raft.TermChangeEventType, s.raftEventLogger) raftServer.AddEventListener(raft.AddPeerEventType, s.raftEventLogger) raftServer.AddEventListener(raft.RemovePeerEventType, s.raftEventLogger) raftServer.AddEventListener(raft.HeartbeatIntervalEventType, s.raftEventLogger) raftServer.AddEventListener(raft.ElectionTimeoutThresholdEventType, s.raftEventLogger) raftServer.AddEventListener(raft.HeartbeatEventType, s.recordMetricEvent) raftServer.AddEventListener(raft.RemovedEventType, s.removedEvent) s.raftServer = raftServer s.removedInLog = false // LoadSnapshot if snapshot { err := s.raftServer.LoadSnapshot() if err == nil { log.Debugf("%s finished load snapshot", s.Config.Name) } else { log.Debug(err) } } s.raftServer.Init() // Set NOCOW for data directory in btrfs if btrfs.IsBtrfs(s.raftServer.LogPath()) { if err := btrfs.SetNOCOWFile(s.raftServer.LogPath()); err != nil { log.Warnf("Failed setting NOCOW: %v", err) } } }
// Response to the join request func (ps *PeerServer) JoinHttpHandler(w http.ResponseWriter, req *http.Request) { command := &JoinCommand{} err := decodeJsonRequest(req, command) if err != nil { w.WriteHeader(http.StatusInternalServerError) return } log.Debugf("Receive Join Request from %s", command.Name) err = ps.server.Dispatch(command, w, req) // Return status. if err != nil { if etcdErr, ok := err.(*etcdErr.Error); ok { log.Debug("Return error: ", (*etcdErr).Error()) etcdErr.Write(w) } else { http.Error(w, err.Error(), http.StatusInternalServerError) } } }
// Send join requests to peer. func (s *PeerServer) joinByPeer(server raft.Server, peer string, scheme string) error { // t must be ok t, _ := server.Transporter().(*transporter) // Our version must match the leaders version versionURL := url.URL{Host: peer, Scheme: scheme, Path: "/version"} version, err := getVersion(t, versionURL) if err != nil { return fmt.Errorf("Error during join version check: %v", err) } if version < store.MinVersion() || version > store.MaxVersion() { return fmt.Errorf("Unable to join: cluster version is %d; version compatibility is %d - %d", version, store.MinVersion(), store.MaxVersion()) } var b bytes.Buffer c := &JoinCommandV2{ MinVersion: store.MinVersion(), MaxVersion: store.MaxVersion(), Name: server.Name(), PeerURL: s.Config.URL, ClientURL: s.server.URL(), } json.NewEncoder(&b).Encode(c) joinURL := url.URL{Host: peer, Scheme: scheme, Path: "/v2/admin/machines/" + server.Name()} log.Infof("Send Join Request to %s", joinURL.String()) req, _ := http.NewRequest("PUT", joinURL.String(), &b) resp, err := t.client.Do(req) for { if err != nil { return fmt.Errorf("Unable to join: %v", err) } if resp != nil { defer resp.Body.Close() log.Infof("»»»» %d", resp.StatusCode) if resp.StatusCode == http.StatusOK { var msg joinMessageV2 if err := json.NewDecoder(resp.Body).Decode(&msg); err != nil { log.Debugf("Error reading join response: %v", err) return err } s.joinIndex = msg.CommitIndex s.setMode(msg.Mode) if msg.Mode == ProxyMode { s.proxyClientURL = resp.Header.Get("X-Leader-Client-URL") s.proxyPeerURL = resp.Header.Get("X-Leader-Peer-URL") } return nil } if resp.StatusCode == http.StatusTemporaryRedirect { address := resp.Header.Get("Location") log.Debugf("Send Join Request to %s", address) c := &JoinCommandV1{ MinVersion: store.MinVersion(), MaxVersion: store.MaxVersion(), Name: server.Name(), RaftURL: s.Config.URL, EtcdURL: s.server.URL(), } json.NewEncoder(&b).Encode(c) resp, _, err = t.Post(address, &b) } else if resp.StatusCode == http.StatusBadRequest { log.Debug("Reach max number peers in the cluster") decoder := json.NewDecoder(resp.Body) err := &etcdErr.Error{} decoder.Decode(err) return *err } else { return fmt.Errorf("Unable to join") } } } }
// Apply attempts to join a machine to the cluster. func (c *JoinCommandV2) Apply(context raft.Context) (interface{}, error) { ps, _ := context.Server().Context().(*PeerServer) var msg = joinMessageV2{ Mode: PeerMode, CommitIndex: context.CommitIndex(), } // Make sure we're not getting a cached value from the registry. ps.registry.Invalidate(c.Name) // Check if the join command is from a previous peer, who lost all its previous log. if peerURL, ok := ps.registry.PeerURL(c.Name); ok { // If previous node restarts with different peer URL, // update its information. if peerURL != c.PeerURL { log.Infof("Rejoin with %v instead of %v from %v", c.PeerURL, peerURL, c.Name) if err := c.updatePeerURL(ps); err != nil { return []byte{0}, err } } return json.Marshal(msg) } // Check if the join command adds an instance that collides with existing one on peer URL. peerURLs := ps.registry.PeerURLs(ps.raftServer.Leader(), c.Name) for _, peerURL := range peerURLs { if peerURL == c.PeerURL { log.Warnf("%v tries to join the cluster with existing URL %v", c.Name, c.PeerURL) return []byte{0}, etcdErr.NewError(etcdErr.EcodeExistingPeerAddr, c.PeerURL, context.CommitIndex()) } } // Check peer number in the cluster. if ps.registry.PeerCount() >= ps.ClusterConfig().ActiveSize { log.Debug("Join as standby ", c.Name) ps.registry.RegisterStandby(c.Name, c.PeerURL, c.ClientURL) msg.Mode = StandbyMode return json.Marshal(msg) } // Remove it as a standby if it is one. if ps.registry.StandbyExists(c.Name) { ps.registry.UnregisterStandby(c.Name) } // Add to shared peer registry. ps.registry.RegisterPeer(c.Name, c.PeerURL, c.ClientURL) // Add peer in raft if err := context.Server().AddPeer(c.Name, ""); err != nil { b, _ := json.Marshal(msg) return b, err } // Add peer stats if c.Name != ps.RaftServer().Name() { ps.followersStats.Followers[c.Name] = &raftFollowerStats{} ps.followersStats.Followers[c.Name].Latency.Minimum = 1 << 63 } return json.Marshal(msg) }