func (d *Discoverer) Do(discoveryURL string, name string, peer string) (peers []string, err error) { d.name = name d.peer = peer d.discoveryURL = discoveryURL u, err := url.Parse(discoveryURL) if err != nil { return } // prefix is prepended to all keys for this discovery d.prefix = strings.TrimPrefix(u.Path, "/v2/keys/") // keep the old path in case we need to set the KeyPrefix below oldPath := u.Path u.Path = "" // Connect to a scheme://host not a full URL with path log.Infof("Discovery via %s using prefix %s.", u.String(), d.prefix) d.client = etcd.NewClient([]string{u.String()}) if !strings.HasPrefix(oldPath, "/v2/keys") { d.client.SetKeyPrefix("") } // Register this machine first and announce that we are a member of // this cluster err = d.heartbeat() if err != nil { return } // Start the very slow heartbeat to the cluster now in anticipation // that everything is going to go alright now go d.startHeartbeat() // Attempt to take the leadership role, if there is no error we are it! resp, err := d.client.Create(path.Join(d.prefix, stateKey), startedState, 0) // Bail out on unexpected errors if err != nil { if clientErr, ok := err.(*etcd.EtcdError); !ok || clientErr.ErrorCode != etcdErr.EcodeNodeExist { return nil, err } } // If we got a response then the CAS was successful, we are leader if resp != nil && resp.Node.Value == startedState { // We are the leader, we have no peers log.Infof("Discovery _state was empty, so this machine is the initial leader.") return nil, nil } // Fall through to finding the other discovery peers return d.findPeers() }
// logSnapshot logs about the snapshot that was taken. func (s *PeerServer) logSnapshot(err error, currentIndex, count uint64) { info := fmt.Sprintf("%s: snapshot of %d events at index %d", s.Config.Name, count, currentIndex) if err != nil { log.Infof("%s attempted and failed: %v", info, err) } else { log.Infof("%s completed", info) } }
// Retrieves the URLs for all nodes using url function. func (r *Registry) urls(leaderName, selfName string, url func(name string) (string, bool)) []string { r.Lock() defer r.Unlock() // Build list including the leader and self. urls := make([]string, 0) if url, _ := url(leaderName); len(url) > 0 { urls = append(urls, url) } // Retrieve a list of all nodes. if e, _ := r.store.Get(RegistryKey, false, false); e != nil { // Lookup the URL for each one. for _, pair := range e.Node.Nodes { _, name := filepath.Split(pair.Key) if url, _ := url(name); len(url) > 0 && name != leaderName { urls = append(urls, url) } } } log.Infof("URLs: %s / %s (%s)", leaderName, selfName, strings.Join(urls, ",")) return urls }
func (d *Discoverer) findPeers() (peers []string, err error) { resp, err := d.client.Get(path.Join(d.prefix), false, true) if err != nil { return nil, err } node := resp.Node if node == nil { return nil, fmt.Errorf("%s key doesn't exist.", d.prefix) } for _, n := range node.Nodes { // Skip our own entry in the list, there is no point if strings.HasSuffix(n.Key, "/"+d.name) { continue } peers = append(peers, n.Value) } if len(peers) == 0 { return nil, errors.New("Discovery found an initialized cluster but no peers are registered.") } log.Infof("Discovery found peers %v", peers) return }
// monitorTimeoutThreshold groups timeout threshold events together and prints // them as a single log line. func (s *PeerServer) monitorTimeoutThreshold(closeChan chan bool) { for { select { case value := <-s.timeoutThresholdChan: log.Infof("%s: warning: heartbeat near election timeout: %v", s.Config.Name, value) case <-closeChan: return } time.Sleep(ThresholdMonitorTimeout) } }
// raftEventLogger converts events from the Raft server into log messages. func (s *PeerServer) raftEventLogger(event raft.Event) { value := event.Value() prevValue := event.PrevValue() if value == nil { value = "<nil>" } if prevValue == nil { prevValue = "<nil>" } switch event.Type() { case raft.StateChangeEventType: log.Infof("%s: state changed from '%v' to '%v'.", s.Config.Name, prevValue, value) case raft.TermChangeEventType: log.Infof("%s: term #%v started.", s.Config.Name, value) case raft.LeaderChangeEventType: log.Infof("%s: leader changed from '%v' to '%v'.", s.Config.Name, prevValue, value) case raft.AddPeerEventType: log.Infof("%s: peer added: '%v'", s.Config.Name, value) case raft.RemovePeerEventType: log.Infof("%s: peer removed: '%v'", s.Config.Name, value) case raft.HeartbeatIntervalEventType: var name = "<unknown>" if peer, ok := value.(*raft.Peer); ok { name = peer.Name } log.Infof("%s: warning: heartbeat timed out: '%v'", s.Config.Name, name) case raft.ElectionTimeoutThresholdEventType: select { case s.timeoutThresholdChan <- value: default: } } }
func (e *Etcd) Start(withEtcdServer bool) { e.PeerServer.Start(e.Config.Snapshot, e.Config.Peers) if withEtcdServer { sListener := e.configEtcdListener() go func() { log.Infof("etcd server [name %s, listen on %s, advertised url %s]", e.EtcdServer.Name, sListener.Addr(), e.EtcdServer.URL()) corsInfo, err := ehttp.NewCORSInfo(e.Config.CorsOrigins) if err != nil { log.Fatal("CORS:", err) } sHTTP := &ehttp.CORSHandler{e.EtcdServer.HTTPHandler(), corsInfo} log.Fatal(http.Serve(sListener, sHTTP)) }() } log.Infof("peer server [name %s, listen on %s, advertised url %s]", e.PeerServer.Config.Name, e.PeerServerListener.Addr(), e.PeerServer.Config.URL) // Retrieve CORS configuration corsInfo, err := ehttp.NewCORSInfo(e.Config.CorsOrigins) if err != nil { log.Fatal("CORS:", err) } sHTTP := &ehttp.CORSHandler{e.PeerServer.HTTPHandler(), corsInfo} log.Fatal(http.Serve(e.PeerServerListener, sHTTP)) }
// profile starts CPU profiling. func profile(path string) { f, err := os.Create(path) if err != nil { log.Fatal(err) } pprof.StartCPUProfile(f) c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) go func() { sig := <-c log.Infof("captured %v, stopping profiler and exiting..", sig) pprof.StopCPUProfile() os.Exit(1) }() }
func main() { // Load configuration. var config = config.New() if err := config.Load(os.Args[1:]); err != nil { fmt.Println(server.Usage() + "\n") fmt.Println(err.Error() + "\n") os.Exit(1) } else if config.ShowVersion { fmt.Println(server.ReleaseVersion) os.Exit(0) } else if config.ShowHelp { fmt.Println(server.Usage() + "\n") os.Exit(0) } // Enable options. if config.VeryVeryVerbose { log.Verbose = true raft.SetLogLevel(raft.Trace) } else if config.VeryVerbose { log.Verbose = true raft.SetLogLevel(raft.Debug) } else if config.Verbose { log.Verbose = true } if config.CPUProfileFile != "" { profile(config.CPUProfileFile) } if config.DataDir == "" { log.Fatal("The data dir was not set and could not be guessed from machine name") } // Create data directory if it doesn't already exist. if err := os.MkdirAll(config.DataDir, 0744); err != nil { log.Fatalf("Unable to create path: %s", err) } // Warn people if they have an info file info := filepath.Join(config.DataDir, "info") if _, err := os.Stat(info); err == nil { log.Warnf("All cached configuration is now ignored. The file %s can be removed.", info) } var mbName string if config.Trace() { mbName = config.MetricsBucketName() runtime.SetBlockProfileRate(1) } mb := metrics.NewBucket(mbName) if config.GraphiteHost != "" { err := mb.Publish(config.GraphiteHost) if err != nil { panic(err) } } // Retrieve CORS configuration corsInfo, err := ehttp.NewCORSInfo(config.CorsOrigins) if err != nil { log.Fatal("CORS:", err) } // Create etcd key-value store and registry. store := store.New() registry := server.NewRegistry(store) // Create stats objects followersStats := server.NewRaftFollowersStats(config.Name) serverStats := server.NewRaftServerStats(config.Name) // Calculate all of our timeouts heartbeatTimeout := time.Duration(config.Peer.HeartbeatTimeout) * time.Millisecond electionTimeout := time.Duration(config.Peer.ElectionTimeout) * time.Millisecond dialTimeout := (3 * heartbeatTimeout) + electionTimeout responseHeaderTimeout := (3 * heartbeatTimeout) + electionTimeout // Create peer server psConfig := server.PeerServerConfig{ Name: config.Name, Scheme: config.PeerTLSInfo().Scheme(), URL: config.Peer.Addr, SnapshotCount: config.SnapshotCount, MaxClusterSize: config.MaxClusterSize, RetryTimes: config.MaxRetryAttempts, RetryInterval: config.RetryInterval, } ps := server.NewPeerServer(psConfig, registry, store, &mb, followersStats, serverStats) var psListener net.Listener if psConfig.Scheme == "https" { peerServerTLSConfig, err := config.PeerTLSInfo().ServerConfig() if err != nil { log.Fatal("peer server TLS error: ", err) } psListener, err = server.NewTLSListener(config.Peer.BindAddr, peerServerTLSConfig) if err != nil { log.Fatal("Failed to create peer listener: ", err) } } else { psListener, err = server.NewListener(config.Peer.BindAddr) if err != nil { log.Fatal("Failed to create peer listener: ", err) } } // Create raft transporter and server raftTransporter := server.NewTransporter(followersStats, serverStats, registry, heartbeatTimeout, dialTimeout, responseHeaderTimeout) if psConfig.Scheme == "https" { raftClientTLSConfig, err := config.PeerTLSInfo().ClientConfig() if err != nil { log.Fatal("raft client TLS error: ", err) } raftTransporter.SetTLSConfig(*raftClientTLSConfig) } raftServer, err := raft.NewServer(config.Name, config.DataDir, raftTransporter, store, ps, "") if err != nil { log.Fatal(err) } raftServer.SetElectionTimeout(electionTimeout) raftServer.SetHeartbeatInterval(heartbeatTimeout) ps.SetRaftServer(raftServer) // Create etcd server s := server.New(config.Name, config.Addr, ps, registry, store, &mb) if config.Trace() { s.EnableTracing() } var sListener net.Listener if config.EtcdTLSInfo().Scheme() == "https" { etcdServerTLSConfig, err := config.EtcdTLSInfo().ServerConfig() if err != nil { log.Fatal("etcd TLS error: ", err) } sListener, err = server.NewTLSListener(config.BindAddr, etcdServerTLSConfig) if err != nil { log.Fatal("Failed to create TLS etcd listener: ", err) } } else { sListener, err = server.NewListener(config.BindAddr) if err != nil { log.Fatal("Failed to create etcd listener: ", err) } } ps.SetServer(s) ps.Start(config.Snapshot, config.Peers) go func() { log.Infof("peer server [name %s, listen on %s, advertised url %s]", ps.Config.Name, psListener.Addr(), ps.Config.URL) sHTTP := &ehttp.CORSHandler{ps.HTTPHandler(), corsInfo} log.Fatal(http.Serve(psListener, sHTTP)) }() log.Infof("etcd server [name %s, listen on %s, advertised url %s]", s.Name, sListener.Addr(), s.URL()) sHTTP := &ehttp.CORSHandler{s.HTTPHandler(), corsInfo} log.Fatal(http.Serve(sListener, sHTTP)) }