// Response to recover from snapshot request
func (ps *PeerServer) SnapshotHttpHandler(w http.ResponseWriter, req *http.Request) {
ssreq := &raft.SnapshotRequest{}
if _, err := ssreq.Decode(req.Body); err != nil {
http.Error(w, "", http.StatusBadRequest)
log.Warnf("[recv] BADREQUEST %s/snapshot [%v]", ps.Config.URL, err)
return
}
log.Debugf("[recv] POST %s/snapshot", ps.Config.URL)
resp := ps.raftServer.RequestSnapshot(ssreq)
if resp == nil {
log.Warn("[ss] Error: nil response")
http.Error(w, "", http.StatusInternalServerError)
return
}
if _, err := resp.Encode(w); err != nil {
log.Warn("[ss] Error: %v", err)
http.Error(w, "", http.StatusInternalServerError)
return
}
}
// Response to vote request
func (ps *PeerServer) VoteHttpHandler(w http.ResponseWriter, req *http.Request) {
rvreq := &raft.RequestVoteRequest{}
if _, err := rvreq.Decode(req.Body); err != nil {
http.Error(w, "", http.StatusBadRequest)
log.Warnf("[recv] BADREQUEST %s/vote [%v]", ps.Config.URL, err)
return
}
log.Debugf("[recv] POST %s/vote [%s]", ps.Config.URL, rvreq.CandidateName)
resp := ps.raftServer.RequestVote(rvreq)
if resp == nil {
log.Warn("[vote] Error: nil response")
http.Error(w, "", http.StatusInternalServerError)
return
}
if _, err := resp.Encode(w); err != nil {
log.Warn("[vote] Error: %v", err)
http.Error(w, "", http.StatusInternalServerError)
return
}
}
// Response to append entries request
func (ps *PeerServer) AppendEntriesHttpHandler(w http.ResponseWriter, req *http.Request) {
start := time.Now()
aereq := &raft.AppendEntriesRequest{}
if _, err := aereq.Decode(req.Body); err != nil {
http.Error(w, "", http.StatusBadRequest)
log.Warnf("[recv] BADREQUEST %s/log/append [%v]", ps.Config.URL, err)
return
}
log.Debugf("[recv] POST %s/log/append [%d]", ps.Config.URL, len(aereq.Entries))
ps.serverStats.RecvAppendReq(aereq.LeaderName, int(req.ContentLength))
resp := ps.raftServer.AppendEntries(aereq)
if resp == nil {
log.Warn("[ae] Error: nil response")
http.Error(w, "", http.StatusInternalServerError)
return
}
if !resp.Success() {
log.Debugf("[Append Entry] Step back")
}
if _, err := resp.Encode(w); err != nil {
log.Warn("[ae] Error: %v", err)
http.Error(w, "", http.StatusInternalServerError)
return
}
(*ps.metrics).Timer("timer.appendentries.handle").UpdateSince(start)
}
func (c *Config) handleDiscovery() error {
p, err := discovery.Do(c.Discovery, c.Name, c.Peer.Addr)
// This is fatal, discovery encountered an unexpected error
// and we have no peer list.
if err != nil && len(c.Peers) == 0 {
log.Fatalf("Discovery failed and a backup peer list wasn't provided: %v", err)
return err
}
// Warn about errors coming from discovery, this isn't fatal
// since the user might have provided a peer list elsewhere.
if err != nil {
log.Warnf("Discovery encountered an error but a backup peer list (%v) was provided: %v", c.Peers, err)
}
for i := range p {
// Strip the scheme off of the peer if it has one
// TODO(bp): clean this up!
purl, err := url.Parse(p[i])
if err == nil {
p[i] = purl.Host
}
}
c.Peers = p
return nil
}
func (s *PeerServer) startAsFollower(cluster []string) {
// start as a follower in a existing cluster
for i := 0; i < s.Config.RetryTimes; i++ {
ok := s.joinCluster(cluster)
if ok {
return
}
log.Warnf("Unable to join the cluster using any of the peers %v. Retrying in %.1f seconds", cluster, s.Config.RetryInterval)
time.Sleep(time.Second * time.Duration(s.Config.RetryInterval))
}
log.Fatalf("Cannot join the cluster via given peers after %x retries", s.Config.RetryTimes)
}
func (d *Discoverer) startHeartbeat() {
// In case of errors we should attempt to heartbeat fairly frequently
heartbeatInterval := defaultTTL / 8
ticker := time.Tick(time.Second * time.Duration(heartbeatInterval))
for {
select {
case <-ticker:
err := d.heartbeat()
if err != nil {
log.Warnf("Discovery heartbeat failed: %v", err)
}
}
}
}
func (s *PeerServer) joinCluster(cluster []string) bool {
for _, peer := range cluster {
if len(peer) == 0 {
continue
}
err := s.joinByPeer(s.raftServer, peer, s.Config.Scheme)
if err == nil {
log.Debugf("%s joined the cluster via peer %s", s.Config.Name, peer)
return true
}
if _, ok := err.(etcdErr.Error); ok {
log.Fatal(err)
}
log.Warnf("Attempt to join via %s failed: %s", peer, err)
}
return false
}
// DataDirFromName sets the data dir from a machine name and issue a warning
// that etcd is guessing.
func (c *Config) DataDirFromName() {
c.DataDir = c.Name + ".etcd"
log.Warnf("Using the directory %s as the etcd curation directory because a directory was not specified. ", c.DataDir)
return
}
func main() {
// Load configuration.
var config = config.New()
if err := config.Load(os.Args[1:]); err != nil {
fmt.Println(server.Usage() + "\n")
fmt.Println(err.Error() + "\n")
os.Exit(1)
} else if config.ShowVersion {
fmt.Println(server.ReleaseVersion)
os.Exit(0)
} else if config.ShowHelp {
fmt.Println(server.Usage() + "\n")
os.Exit(0)
}
// Enable options.
if config.VeryVeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Trace)
} else if config.VeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Debug)
} else if config.Verbose {
log.Verbose = true
}
if config.CPUProfileFile != "" {
profile(config.CPUProfileFile)
}
if config.DataDir == "" {
log.Fatal("The data dir was not set and could not be guessed from machine name")
}
// Create data directory if it doesn't already exist.
if err := os.MkdirAll(config.DataDir, 0744); err != nil {
log.Fatalf("Unable to create path: %s", err)
}
// Warn people if they have an info file
info := filepath.Join(config.DataDir, "info")
if _, err := os.Stat(info); err == nil {
log.Warnf("All cached configuration is now ignored. The file %s can be removed.", info)
}
var mbName string
if config.Trace() {
mbName = config.MetricsBucketName()
runtime.SetBlockProfileRate(1)
}
mb := metrics.NewBucket(mbName)
if config.GraphiteHost != "" {
err := mb.Publish(config.GraphiteHost)
if err != nil {
panic(err)
}
}
// Retrieve CORS configuration
corsInfo, err := ehttp.NewCORSInfo(config.CorsOrigins)
if err != nil {
log.Fatal("CORS:", err)
}
// Create etcd key-value store and registry.
store := store.New()
registry := server.NewRegistry(store)
// Create stats objects
followersStats := server.NewRaftFollowersStats(config.Name)
serverStats := server.NewRaftServerStats(config.Name)
// Calculate all of our timeouts
heartbeatTimeout := time.Duration(config.Peer.HeartbeatTimeout) * time.Millisecond
electionTimeout := time.Duration(config.Peer.ElectionTimeout) * time.Millisecond
dialTimeout := (3 * heartbeatTimeout) + electionTimeout
responseHeaderTimeout := (3 * heartbeatTimeout) + electionTimeout
// Create peer server
psConfig := server.PeerServerConfig{
Name: config.Name,
Scheme: config.PeerTLSInfo().Scheme(),
URL: config.Peer.Addr,
SnapshotCount: config.SnapshotCount,
MaxClusterSize: config.MaxClusterSize,
RetryTimes: config.MaxRetryAttempts,
RetryInterval: config.RetryInterval,
}
ps := server.NewPeerServer(psConfig, registry, store, &mb, followersStats, serverStats)
var psListener net.Listener
if psConfig.Scheme == "https" {
peerServerTLSConfig, err := config.PeerTLSInfo().ServerConfig()
if err != nil {
log.Fatal("peer server TLS error: ", err)
}
psListener, err = server.NewTLSListener(config.Peer.BindAddr, peerServerTLSConfig)
if err != nil {
log.Fatal("Failed to create peer listener: ", err)
}
} else {
psListener, err = server.NewListener(config.Peer.BindAddr)
if err != nil {
log.Fatal("Failed to create peer listener: ", err)
}
}
// Create raft transporter and server
raftTransporter := server.NewTransporter(followersStats, serverStats, registry, heartbeatTimeout, dialTimeout, responseHeaderTimeout)
if psConfig.Scheme == "https" {
raftClientTLSConfig, err := config.PeerTLSInfo().ClientConfig()
if err != nil {
log.Fatal("raft client TLS error: ", err)
}
raftTransporter.SetTLSConfig(*raftClientTLSConfig)
}
raftServer, err := raft.NewServer(config.Name, config.DataDir, raftTransporter, store, ps, "")
if err != nil {
log.Fatal(err)
}
raftServer.SetElectionTimeout(electionTimeout)
raftServer.SetHeartbeatInterval(heartbeatTimeout)
ps.SetRaftServer(raftServer)
// Create etcd server
s := server.New(config.Name, config.Addr, ps, registry, store, &mb)
if config.Trace() {
s.EnableTracing()
}
var sListener net.Listener
if config.EtcdTLSInfo().Scheme() == "https" {
etcdServerTLSConfig, err := config.EtcdTLSInfo().ServerConfig()
if err != nil {
log.Fatal("etcd TLS error: ", err)
}
sListener, err = server.NewTLSListener(config.BindAddr, etcdServerTLSConfig)
if err != nil {
log.Fatal("Failed to create TLS etcd listener: ", err)
}
} else {
sListener, err = server.NewListener(config.BindAddr)
if err != nil {
log.Fatal("Failed to create etcd listener: ", err)
}
}
ps.SetServer(s)
ps.Start(config.Snapshot, config.Peers)
go func() {
log.Infof("peer server [name %s, listen on %s, advertised url %s]", ps.Config.Name, psListener.Addr(), ps.Config.URL)
sHTTP := &ehttp.CORSHandler{ps.HTTPHandler(), corsInfo}
log.Fatal(http.Serve(psListener, sHTTP))
}()
log.Infof("etcd server [name %s, listen on %s, advertised url %s]", s.Name, sListener.Addr(), s.URL())
sHTTP := &ehttp.CORSHandler{s.HTTPHandler(), corsInfo}
log.Fatal(http.Serve(sListener, sHTTP))
}
