// Response to recover from snapshot request func (ps *PeerServer) SnapshotHttpHandler(w http.ResponseWriter, req *http.Request) { ssreq := &raft.SnapshotRequest{} if _, err := ssreq.Decode(req.Body); err != nil { http.Error(w, "", http.StatusBadRequest) log.Warnf("[recv] BADREQUEST %s/snapshot [%v]", ps.Config.URL, err) return } log.Debugf("[recv] POST %s/snapshot", ps.Config.URL) resp := ps.raftServer.RequestSnapshot(ssreq) if resp == nil { log.Warn("[ss] Error: nil response") http.Error(w, "", http.StatusInternalServerError) return } if _, err := resp.Encode(w); err != nil { log.Warn("[ss] Error: %v", err) http.Error(w, "", http.StatusInternalServerError) return } }
// Response to vote request func (ps *PeerServer) VoteHttpHandler(w http.ResponseWriter, req *http.Request) { rvreq := &raft.RequestVoteRequest{} if _, err := rvreq.Decode(req.Body); err != nil { http.Error(w, "", http.StatusBadRequest) log.Warnf("[recv] BADREQUEST %s/vote [%v]", ps.Config.URL, err) return } log.Debugf("[recv] POST %s/vote [%s]", ps.Config.URL, rvreq.CandidateName) resp := ps.raftServer.RequestVote(rvreq) if resp == nil { log.Warn("[vote] Error: nil response") http.Error(w, "", http.StatusInternalServerError) return } if _, err := resp.Encode(w); err != nil { log.Warn("[vote] Error: %v", err) http.Error(w, "", http.StatusInternalServerError) return } }
// Response to append entries request func (ps *PeerServer) AppendEntriesHttpHandler(w http.ResponseWriter, req *http.Request) { start := time.Now() aereq := &raft.AppendEntriesRequest{} if _, err := aereq.Decode(req.Body); err != nil { http.Error(w, "", http.StatusBadRequest) log.Warnf("[recv] BADREQUEST %s/log/append [%v]", ps.Config.URL, err) return } log.Debugf("[recv] POST %s/log/append [%d]", ps.Config.URL, len(aereq.Entries)) ps.serverStats.RecvAppendReq(aereq.LeaderName, int(req.ContentLength)) resp := ps.raftServer.AppendEntries(aereq) if resp == nil { log.Warn("[ae] Error: nil response") http.Error(w, "", http.StatusInternalServerError) return } if !resp.Success { log.Debugf("[Append Entry] Step back") } if _, err := resp.Encode(w); err != nil { log.Warn("[ae] Error: %v", err) http.Error(w, "", http.StatusInternalServerError) return } (*ps.metrics).Timer("timer.appendentries.handle").UpdateSince(start) }
// Sends SnapshotRecoveryRequest RPCs to a peer when the server is the candidate. func (t *transporter) SendSnapshotRecoveryRequest(server raft.Server, peer *raft.Peer, req *raft.SnapshotRecoveryRequest) *raft.SnapshotRecoveryResponse { var b bytes.Buffer if _, err := req.Encode(&b); err != nil { log.Warn("transporter.ss.encoding.error:", err) return nil } u, _ := t.peerServer.registry.PeerURL(peer.Name) log.Debugf("Send Snapshot Recovery from %s to %s", server.Name(), u) resp, httpRequest, err := t.Post(fmt.Sprintf("%s/snapshotRecovery", u), &b) if err != nil { log.Debugf("Cannot send Snapshot Recovery to %s : %s", u, err) } if resp != nil { defer resp.Body.Close() t.CancelWhenTimeout(httpRequest) ssrrsp := &raft.SnapshotRecoveryResponse{} if _, err = ssrrsp.Decode(resp.Body); err != nil && err != io.EOF { log.Warn("transporter.ssr.decoding.error:", err) return nil } return ssrrsp } return nil }
// Sends RequestVote RPCs to a peer when the server is the candidate. func (t *transporter) SendVoteRequest(server raft.Server, peer *raft.Peer, req *raft.RequestVoteRequest) *raft.RequestVoteResponse { var b bytes.Buffer if _, err := req.Encode(&b); err != nil { log.Warn("transporter.vr.encoding.error:", err) return nil } u, _ := t.registry.PeerURL(peer.Name) log.Debugf("Send Vote from %s to %s", server.Name(), u) resp, _, err := t.Post(fmt.Sprintf("%s/vote", u), &b) if err != nil { log.Debugf("Cannot send VoteRequest to %s : %s", u, err) } if resp != nil { defer resp.Body.Close() rvrsp := &raft.RequestVoteResponse{} if _, err = rvrsp.Decode(resp.Body); err != nil && err != io.EOF { log.Warn("transporter.vr.decoding.error:", err) return nil } return rvrsp } return nil }
// Sends AppendEntries RPCs to a peer when the server is the leader. func (t *transporter) SendAppendEntriesRequest(server raft.Server, peer *raft.Peer, req *raft.AppendEntriesRequest) *raft.AppendEntriesResponse { var b bytes.Buffer if _, err := req.Encode(&b); err != nil { log.Warn("transporter.ae.encoding.error:", err) return nil } size := b.Len() t.peerServer.serverStats.SendAppendReq(size) u, _ := t.peerServer.registry.PeerURL(peer.Name) log.Debugf("Send LogEntries to %s ", u) thisFollowerStats, ok := t.peerServer.followersStats.Followers[peer.Name] if !ok { //this is the first time this follower has been seen thisFollowerStats = &raftFollowerStats{} thisFollowerStats.Latency.Minimum = 1 << 63 t.peerServer.followersStats.Followers[peer.Name] = thisFollowerStats } start := time.Now() resp, httpRequest, err := t.Post(fmt.Sprintf("%s/log/append", u), &b) end := time.Now() if err != nil { log.Debugf("Cannot send AppendEntriesRequest to %s: %s", u, err) if ok { thisFollowerStats.Fail() } return nil } else { if ok { thisFollowerStats.Succ(end.Sub(start)) } } if resp != nil { defer resp.Body.Close() t.CancelWhenTimeout(httpRequest) aeresp := &raft.AppendEntriesResponse{} if _, err = aeresp.Decode(resp.Body); err != nil && err != io.EOF { log.Warn("transporter.ae.decoding.error:", err) return nil } return aeresp } return nil }
func GetHandler(w http.ResponseWriter, req *http.Request, s Server) error { vars := mux.Vars(req) key := "/" + vars["key"] // Help client to redirect the request to the current leader if req.FormValue("consistent") == "true" && s.State() != raft.Leader { leader := s.Leader() hostname, _ := s.ClientURL(leader) url, err := url.Parse(hostname) if err != nil { log.Warn("Redirect cannot parse hostName ", hostname) return err } url.RawQuery = req.URL.RawQuery url.Path = req.URL.Path log.Debugf("Redirect consistent get to %s", url.String()) http.Redirect(w, req, url.String(), http.StatusTemporaryRedirect) return nil } recursive := (req.FormValue("recursive") == "true") sort := (req.FormValue("sorted") == "true") waitIndex := req.FormValue("waitIndex") stream := (req.FormValue("stream") == "true") if req.FormValue("wait") == "true" { return handleWatch(key, recursive, stream, waitIndex, w, s) } return handleGet(key, recursive, sort, w, s) }
// Try all possible ways to find clusters to join // Include -discovery, -peers and log data in -data-dir // // Peer discovery follows this order: // 1. -discovery // 2. -peers // 3. previous peers in -data-dir func (s *PeerServer) findCluster(discoverURL string, peers []string) { // Attempt cluster discovery toDiscover := discoverURL != "" if toDiscover { discoverPeers, discoverErr := s.handleDiscovery(discoverURL) // It is registered in discover url if discoverErr == nil { // start as a leader in a new cluster if len(discoverPeers) == 0 { log.Debug("This peer is starting a brand new cluster based on discover URL.") s.startAsLeader() } else { s.startAsFollower(discoverPeers) } return } } hasPeerList := len(peers) > 0 // if there is log in data dir, append previous peers to peers in config // to find cluster prevPeers := s.registry.PeerURLs(s.raftServer.Leader(), s.Config.Name) for i := 0; i < len(prevPeers); i++ { u, err := url.Parse(prevPeers[i]) if err != nil { log.Debug("rejoin cannot parse url: ", err) } prevPeers[i] = u.Host } peers = append(peers, prevPeers...) // if there is backup peer lists, use it to find cluster if len(peers) > 0 { ok := s.joinCluster(peers) if !ok { log.Warn("No living peers are found!") } else { log.Debugf("%s restart as a follower based on peers[%v]", s.Config.Name) return } } if !s.raftServer.IsLogEmpty() { log.Debug("Entire cluster is down! %v will restart the cluster.", s.Config.Name) return } if toDiscover { log.Fatalf("Discovery failed, no available peers in backup list, and no log data") } if hasPeerList { log.Fatalf("No available peers in backup list, and no log data") } log.Infof("This peer is starting a brand new cluster now.") s.startAsLeader() }
// Start the raft server func (s *PeerServer) ListenAndServe(snapshot bool, cluster []string) error { // LoadSnapshot if snapshot { err := s.raftServer.LoadSnapshot() if err == nil { log.Debugf("%s finished load snapshot", s.name) } else { log.Debug(err) } } s.raftServer.SetElectionTimeout(s.ElectionTimeout) s.raftServer.SetHeartbeatTimeout(s.HeartbeatTimeout) s.raftServer.Start() if s.raftServer.IsLogEmpty() { // start as a leader in a new cluster if len(cluster) == 0 { s.startAsLeader() } else { s.startAsFollower(cluster) } } else { // Rejoin the previous cluster cluster = s.registry.PeerURLs(s.raftServer.Leader(), s.name) for i := 0; i < len(cluster); i++ { u, err := url.Parse(cluster[i]) if err != nil { log.Debug("rejoin cannot parse url: ", err) } cluster[i] = u.Host } ok := s.joinCluster(cluster) if !ok { log.Warn("the entire cluster is down! this peer will restart the cluster.") } log.Debugf("%s restart as a follower", s.name) } s.closeChan = make(chan bool) go s.monitorSync() go s.monitorTimeoutThreshold(s.closeChan) // open the snapshot if snapshot { go s.monitorSnapshot() } // start to response to raft requests return s.startTransport(s.tlsConf.Scheme, s.tlsConf.Server) }
func GetHandler(w http.ResponseWriter, req *http.Request, s Server) error { var err error var event *store.Event vars := mux.Vars(req) key := "/" + vars["key"] // Help client to redirect the request to the current leader if req.FormValue("consistent") == "true" && s.State() != raft.Leader { leader := s.Leader() hostname, _ := s.ClientURL(leader) url, err := url.Parse(hostname) if err != nil { log.Warn("Redirect cannot parse hostName ", hostname) return err } url.RawQuery = req.URL.RawQuery url.Path = req.URL.Path log.Debugf("Redirect consistent get to %s", url.String()) http.Redirect(w, req, url.String(), http.StatusTemporaryRedirect) return nil } recursive := (req.FormValue("recursive") == "true") sorted := (req.FormValue("sorted") == "true") if req.FormValue("wait") == "true" { // watch // Create a command to watch from a given index (default 0). var sinceIndex uint64 = 0 waitIndex := req.FormValue("waitIndex") if waitIndex != "" { sinceIndex, err = strconv.ParseUint(string(req.FormValue("waitIndex")), 10, 64) if err != nil { return etcdErr.NewError(etcdErr.EcodeIndexNaN, "Watch From Index", s.Store().Index()) } } // Start the watcher on the store. eventChan, err := s.Store().Watch(key, recursive, sinceIndex) if err != nil { return etcdErr.NewError(500, key, s.Store().Index()) } cn, _ := w.(http.CloseNotifier) closeChan := cn.CloseNotify() select { case <-closeChan: return nil case event = <-eventChan: } } else { //get // Retrieve the key from the store. event, err = s.Store().Get(key, recursive, sorted) if err != nil { return err } } w.Header().Set("Content-Type", "application/json") w.Header().Add("X-Etcd-Index", fmt.Sprint(s.Store().Index())) w.Header().Add("X-Raft-Index", fmt.Sprint(s.CommitIndex())) w.Header().Add("X-Raft-Term", fmt.Sprint(s.Term())) w.WriteHeader(http.StatusOK) b, _ := json.Marshal(event) w.Write(b) return nil }