// Response to append entries request func (ps *PeerServer) AppendEntriesHttpHandler(w http.ResponseWriter, req *http.Request) { start := time.Now() aereq := &raft.AppendEntriesRequest{} if _, err := aereq.Decode(req.Body); err != nil { http.Error(w, "", http.StatusBadRequest) log.Warnf("[recv] BADREQUEST %s/log/append [%v]", ps.Config.URL, err) return } log.Debugf("[recv] POST %s/log/append [%d]", ps.Config.URL, len(aereq.Entries)) ps.serverStats.RecvAppendReq(aereq.LeaderName, int(req.ContentLength)) resp := ps.raftServer.AppendEntries(aereq) if resp == nil { log.Warn("[ae] Error: nil response") http.Error(w, "", http.StatusInternalServerError) return } if !resp.Success() { log.Debugf("[Append Entry] Step back") } if _, err := resp.Encode(w); err != nil { log.Warn("[ae] Error: %v", err) http.Error(w, "", http.StatusInternalServerError) return } (*ps.metrics).Timer("timer.appendentries.handle").UpdateSince(start) }
// Sends SnapshotRecoveryRequest RPCs to a peer when the server is the candidate. func (t *transporter) SendSnapshotRecoveryRequest(server raft.Server, peer *raft.Peer, req *raft.SnapshotRecoveryRequest) *raft.SnapshotRecoveryResponse { var b bytes.Buffer if _, err := req.Encode(&b); err != nil { log.Warn("transporter.ss.encoding.error:", err) return nil } u, _ := t.registry.PeerURL(peer.Name) log.Debugf("Send Snapshot Recovery from %s to %s", server.Name(), u) resp, httpRequest, err := t.Post(fmt.Sprintf("%s/snapshotRecovery", u), &b) if err != nil { log.Debugf("Cannot send Snapshot Recovery to %s : %s", u, err) } if resp != nil { defer resp.Body.Close() t.CancelWhenTimeout(httpRequest) ssrrsp := &raft.SnapshotRecoveryResponse{} if _, err = ssrrsp.Decode(resp.Body); err != nil && err != io.EOF { log.Warn("transporter.ssr.decoding.error:", err) return nil } return ssrrsp } return nil }
// Sends AppendEntries RPCs to a peer when the server is the leader. func (t *transporter) SendAppendEntriesRequest(server raft.Server, peer *raft.Peer, req *raft.AppendEntriesRequest) *raft.AppendEntriesResponse { var b bytes.Buffer if _, err := req.Encode(&b); err != nil { log.Warn("transporter.ae.encoding.error:", err) return nil } size := b.Len() t.serverStats.SendAppendReq(size) u, _ := t.registry.PeerURL(peer.Name) log.Debugf("Send LogEntries to %s ", u) thisFollowerStats, ok := t.followersStats.Followers[peer.Name] if !ok { //this is the first time this follower has been seen thisFollowerStats = &raftFollowerStats{} thisFollowerStats.Latency.Minimum = 1 << 63 t.followersStats.Followers[peer.Name] = thisFollowerStats } start := time.Now() resp, httpRequest, err := t.Post(fmt.Sprintf("%s/log/append", u), &b) end := time.Now() if err != nil { log.Debugf("Cannot send AppendEntriesRequest to %s: %s", u, err) if ok { thisFollowerStats.Fail() } return nil } else { if ok { thisFollowerStats.Succ(end.Sub(start)) } } if resp != nil { defer resp.Body.Close() t.CancelWhenTimeout(httpRequest) aeresp := &raft.AppendEntriesResponse{} if _, err = aeresp.Decode(resp.Body); err != nil && err != io.EOF { log.Warn("transporter.ae.decoding.error:", err) return nil } return aeresp } return nil }
// Send join requests to peer. func (s *PeerServer) joinByPeer(server raft.Server, peer string, scheme string) error { var b bytes.Buffer // t must be ok t, _ := server.Transporter().(*transporter) // Our version must match the leaders version versionURL := url.URL{Host: peer, Scheme: scheme, Path: "/version"} version, err := getVersion(t, versionURL) if err != nil { return fmt.Errorf("Error during join version check: %v", err) } if version < store.MinVersion() || version > store.MaxVersion() { return fmt.Errorf("Unable to join: cluster version is %d; version compatibility is %d - %d", version, store.MinVersion(), store.MaxVersion()) } json.NewEncoder(&b).Encode(NewJoinCommand(store.MinVersion(), store.MaxVersion(), server.Name(), s.Config.URL, s.server.URL())) joinURL := url.URL{Host: peer, Scheme: scheme, Path: "/join"} log.Debugf("Send Join Request to %s", joinURL.String()) resp, req, err := t.Post(joinURL.String(), &b) for { if err != nil { return fmt.Errorf("Unable to join: %v", err) } if resp != nil { defer resp.Body.Close() t.CancelWhenTimeout(req) if resp.StatusCode == http.StatusOK { b, _ := ioutil.ReadAll(resp.Body) s.joinIndex, _ = binary.Uvarint(b) return nil } if resp.StatusCode == http.StatusTemporaryRedirect { address := resp.Header.Get("Location") log.Debugf("Send Join Request to %s", address) json.NewEncoder(&b).Encode(NewJoinCommand(store.MinVersion(), store.MaxVersion(), server.Name(), s.Config.URL, s.server.URL())) resp, req, err = t.Post(address, &b) } else if resp.StatusCode == http.StatusBadRequest { log.Debug("Reach max number peers in the cluster") decoder := json.NewDecoder(resp.Body) err := &etcdErr.Error{} decoder.Decode(err) return *err } else { return fmt.Errorf("Unable to join") } } } }
// Start the raft server func (s *PeerServer) Start(snapshot bool, cluster []string) error { // LoadSnapshot if snapshot { err := s.raftServer.LoadSnapshot() if err == nil { log.Debugf("%s finished load snapshot", s.Config.Name) } else { log.Debug(err) } } s.raftServer.Start() if s.raftServer.IsLogEmpty() { // start as a leader in a new cluster if len(cluster) == 0 { s.startAsLeader() } else { s.startAsFollower(cluster) } } else { // Rejoin the previous cluster cluster = s.registry.PeerURLs(s.raftServer.Leader(), s.Config.Name) for i := 0; i < len(cluster); i++ { u, err := url.Parse(cluster[i]) if err != nil { log.Debug("rejoin cannot parse url: ", err) } cluster[i] = u.Host } ok := s.joinCluster(cluster) if !ok { log.Warn("the entire cluster is down! this peer will restart the cluster.") } log.Debugf("%s restart as a follower", s.Config.Name) } s.closeChan = make(chan bool) go s.monitorSync() go s.monitorTimeoutThreshold(s.closeChan) // open the snapshot if snapshot { go s.monitorSnapshot() } return nil }
// Response to vote request func (ps *PeerServer) VoteHttpHandler(w http.ResponseWriter, req *http.Request) { rvreq := &raft.RequestVoteRequest{} if _, err := rvreq.Decode(req.Body); err != nil { http.Error(w, "", http.StatusBadRequest) log.Warnf("[recv] BADREQUEST %s/vote [%v]", ps.Config.URL, err) return } log.Debugf("[recv] POST %s/vote [%s]", ps.Config.URL, rvreq.CandidateName) resp := ps.raftServer.RequestVote(rvreq) if resp == nil { log.Warn("[vote] Error: nil response") http.Error(w, "", http.StatusInternalServerError) return } if _, err := resp.Encode(w); err != nil { log.Warn("[vote] Error: %v", err) http.Error(w, "", http.StatusInternalServerError) return } }
// Response to recover from snapshot request func (ps *PeerServer) SnapshotHttpHandler(w http.ResponseWriter, req *http.Request) { ssreq := &raft.SnapshotRequest{} if _, err := ssreq.Decode(req.Body); err != nil { http.Error(w, "", http.StatusBadRequest) log.Warnf("[recv] BADREQUEST %s/snapshot [%v]", ps.Config.URL, err) return } log.Debugf("[recv] POST %s/snapshot", ps.Config.URL) resp := ps.raftServer.RequestSnapshot(ssreq) if resp == nil { log.Warn("[ss] Error: nil response") http.Error(w, "", http.StatusInternalServerError) return } if _, err := resp.Encode(w); err != nil { log.Warn("[ss] Error: %v", err) http.Error(w, "", http.StatusInternalServerError) return } }
func GetHandler(w http.ResponseWriter, req *http.Request, s Server) error { vars := mux.Vars(req) key := "/" + vars["key"] // Help client to redirect the request to the current leader if req.FormValue("consistent") == "true" && s.State() != raft.Leader { leader := s.Leader() hostname, _ := s.ClientURL(leader) url, err := url.Parse(hostname) if err != nil { log.Warn("Redirect cannot parse hostName ", hostname) return err } url.RawQuery = req.URL.RawQuery url.Path = req.URL.Path log.Debugf("Redirect consistent get to %s", url.String()) http.Redirect(w, req, url.String(), http.StatusTemporaryRedirect) return nil } recursive := (req.FormValue("recursive") == "true") sort := (req.FormValue("sorted") == "true") waitIndex := req.FormValue("waitIndex") stream := (req.FormValue("stream") == "true") if req.FormValue("wait") == "true" { return handleWatch(key, recursive, stream, waitIndex, w, s) } return handleGet(key, recursive, sort, w, s) }
// Checks whether a given version is supported. func (ps *PeerServer) VersionCheckHttpHandler(w http.ResponseWriter, req *http.Request) { log.Debugf("[recv] Get %s%s ", ps.Config.URL, req.URL.Path) vars := mux.Vars(req) version, _ := strconv.Atoi(vars["version"]) if version >= store.MinVersion() && version <= store.MaxVersion() { w.WriteHeader(http.StatusOK) } else { w.WriteHeader(http.StatusForbidden) } }
func (s *PeerServer) startAsLeader() { // leader need to join self as a peer for { _, err := s.raftServer.Do(NewJoinCommand(store.MinVersion(), store.MaxVersion(), s.raftServer.Name(), s.Config.URL, s.server.URL())) if err == nil { break } } log.Debugf("%s start as a leader", s.Config.Name) }
// Remove a server from the cluster func (c *RemoveCommand) Apply(context raft.Context) (interface{}, error) { ps, _ := context.Server().Context().(*PeerServer) // Remove node from the shared registry. err := ps.registry.Unregister(c.Name) // Delete from stats delete(ps.followersStats.Followers, c.Name) if err != nil { log.Debugf("Error while unregistering: %s (%v)", c.Name, err) return []byte{0}, err } // Remove peer in raft err = context.Server().RemovePeer(c.Name) if err != nil { log.Debugf("Unable to remove peer: %s (%v)", c.Name, err) return []byte{0}, err } if c.Name == context.Server().Name() { // the removed node is this node // if the node is not replaying the previous logs // and the node has sent out a join request in this // start. It is sure that this node received a new remove // command and need to be removed if context.CommitIndex() > ps.joinIndex && ps.joinIndex != 0 { log.Debugf("server [%s] is removed", context.Server().Name()) os.Exit(0) } else { // else ignore remove log.Debugf("ignore previous remove command.") } } b := make([]byte, 8) binary.PutUvarint(b, context.CommitIndex()) return b, err }
// DashboardHttpHandler either uses the compiled in virtual filesystem for the // dashboard assets or if ETCD_DASHBOARD_DIR is set uses that as the source of // assets. func HttpHandler() (handler http.Handler) { handler = http.HandlerFunc(memoryFileServer) // Serve the dashboard from a filesystem if the magic env variable is enabled dashDir := os.Getenv("ETCD_DASHBOARD_DIR") if len(dashDir) != 0 { log.Debugf("Using dashboard directory %s", dashDir) handler = http.FileServer(http.Dir(dashDir)) } return handler }
// Removes a node from the registry. func (r *Registry) Unregister(name string) error { r.Lock() defer r.Unlock() // Remove from cache. // delete(r.nodes, name) // Remove the key from the store. _, err := r.store.Delete(path.Join(RegistryKey, name), false, false) log.Debugf("Unregister: %s", name) return err }
// Adds a node to the registry. func (r *Registry) Register(name string, peerURL string, machURL string) error { r.Lock() defer r.Unlock() // Write data to store. key := path.Join(RegistryKey, name) v := url.Values{} v.Set("raft", peerURL) v.Set("etcd", machURL) _, err := r.store.Create(key, false, v.Encode(), false, store.Permanent) log.Debugf("Register: %s", name) return err }
// Response to remove request func (ps *PeerServer) RemoveHttpHandler(w http.ResponseWriter, req *http.Request) { if req.Method != "DELETE" { w.WriteHeader(http.StatusMethodNotAllowed) return } vars := mux.Vars(req) command := &RemoveCommand{ Name: vars["name"], } log.Debugf("[recv] Remove Request [%s]", command.Name) ps.server.Dispatch(command, w, req) }
// Upgrades the current store version to the next version. func (ps *PeerServer) UpgradeHttpHandler(w http.ResponseWriter, req *http.Request) { log.Debugf("[recv] Get %s/version", ps.Config.URL) // Check if upgrade is possible for all nodes. if err := ps.Upgradable(); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } // Create an upgrade command from the current version. c := ps.store.CommandFactory().CreateUpgradeCommand() if err := ps.server.Dispatch(c, w, req); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } w.WriteHeader(http.StatusOK) }
// Adds a server handler to the router. func (s *Server) handleFunc(r *mux.Router, path string, f func(http.ResponseWriter, *http.Request) error) *mux.Route { // Wrap the standard HandleFunc interface to pass in the server reference. return r.HandleFunc(path, func(w http.ResponseWriter, req *http.Request) { // Log request. log.Debugf("[recv] %s %s %s [%s]", req.Method, s.URL(), req.URL.Path, req.RemoteAddr) // Execute handler function and return error if necessary. if err := f(w, req); err != nil { if etcdErr, ok := err.(*etcdErr.Error); ok { log.Debug("Return error: ", (*etcdErr).Error()) w.Header().Set("Content-Type", "application/json") etcdErr.Write(w) } else { http.Error(w, err.Error(), http.StatusInternalServerError) } } }) }
func (s *PeerServer) joinCluster(cluster []string) bool { for _, peer := range cluster { if len(peer) == 0 { continue } err := s.joinByPeer(s.raftServer, peer, s.Config.Scheme) if err == nil { log.Debugf("%s joined the cluster via peer %s", s.Config.Name, peer) return true } if _, ok := err.(etcdErr.Error); ok { log.Fatal(err) } log.Warnf("Attempt to join via %s failed: %s", peer, err) } return false }
// Response to the join request func (ps *PeerServer) JoinHttpHandler(w http.ResponseWriter, req *http.Request) { command := &JoinCommand{} err := uhttp.DecodeJsonRequest(req, command) if err != nil { w.WriteHeader(http.StatusInternalServerError) return } log.Debugf("Receive Join Request from %s", command.Name) err = ps.server.Dispatch(command, w, req) // Return status. if err != nil { if etcdErr, ok := err.(*etcdErr.Error); ok { log.Debug("Return error: ", (*etcdErr).Error()) etcdErr.Write(w) } else { http.Error(w, err.Error(), http.StatusInternalServerError) } } }
func memoryFileServer(w http.ResponseWriter, req *http.Request) { log.Debugf("[recv] %s %s [%s]", req.Method, req.URL.Path, req.RemoteAddr) upath := req.URL.Path if len(upath) == 0 { upath = "index.html" } // TODO: use the new mux to do this work dir, file := path.Split(upath) if file == "browser" || file == "stats" { file = file + ".html" } upath = path.Join(dir, file) b, err := resources.Asset(upath) if err != nil { http.Error(w, upath+": File not found", http.StatusNotFound) return } http.ServeContent(w, req, upath, time.Time{}, bytes.NewReader(b)) return }
// Response to the name request func (ps *PeerServer) VersionHttpHandler(w http.ResponseWriter, req *http.Request) { log.Debugf("[recv] Get %s/version/ ", ps.Config.URL) w.WriteHeader(http.StatusOK) w.Write([]byte(strconv.Itoa(ps.store.Version()))) }
// Get the port that listening for etcd connecting of the server func (ps *PeerServer) EtcdURLHttpHandler(w http.ResponseWriter, req *http.Request) { log.Debugf("[recv] Get %s/etcdURL/ ", ps.Config.URL) w.WriteHeader(http.StatusOK) w.Write([]byte(ps.server.URL())) }
// Get all the current logs func (ps *PeerServer) GetLogHttpHandler(w http.ResponseWriter, req *http.Request) { log.Debugf("[recv] GET %s/log", ps.Config.URL) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(ps.raftServer.LogEntries()) }
// Response to the name request func (ps *PeerServer) NameHttpHandler(w http.ResponseWriter, req *http.Request) { log.Debugf("[recv] Get %s/name/ ", ps.Config.URL) w.WriteHeader(http.StatusOK) w.Write([]byte(ps.Config.Name)) }