func (d *Discoverer) Do(discoveryURL string, name string, peer string) (peers []string, err error) {
d.name = name
d.peer = peer
d.discoveryURL = discoveryURL
u, err := url.Parse(discoveryURL)
if err != nil {
return
}
// prefix is prepended to all keys for this discovery
d.prefix = strings.TrimPrefix(u.Path, "/v2/keys/")
// keep the old path in case we need to set the KeyPrefix below
oldPath := u.Path
u.Path = ""
// Connect to a scheme://host not a full URL with path
log.Infof("Discovery via %s using prefix %s.", u.String(), d.prefix)
d.client = etcd.NewClient([]string{u.String()})
if !strings.HasPrefix(oldPath, "/v2/keys") {
d.client.SetKeyPrefix("")
}
// Register this machine first and announce that we are a member of
// this cluster
err = d.heartbeat()
if err != nil {
return
}
// Start the very slow heartbeat to the cluster now in anticipation
// that everything is going to go alright now
go d.startHeartbeat()
// Attempt to take the leadership role, if there is no error we are it!
resp, err := d.client.Create(path.Join(d.prefix, stateKey), startedState, 0)
// Bail out on unexpected errors
if err != nil {
if clientErr, ok := err.(*etcd.EtcdError); !ok || clientErr.ErrorCode != etcdErr.EcodeNodeExist {
return nil, err
}
}
// If we got a response then the CAS was successful, we are leader
if resp != nil && resp.Node.Value == startedState {
// We are the leader, we have no peers
log.Infof("Discovery _state was empty, so this machine is the initial leader.")
return nil, nil
}
// Fall through to finding the other discovery peers
return d.findPeers()
}
// logSnapshot logs about the snapshot that was taken.
func (s *PeerServer) logSnapshot(err error, currentIndex, count uint64) {
info := fmt.Sprintf("%s: snapshot of %d events at index %d", s.Config.Name, count, currentIndex)
if err != nil {
log.Infof("%s attempted and failed: %v", info, err)
} else {
log.Infof("%s completed", info)
}
}
// Retrieves the URLs for all nodes using url function.
func (r *Registry) urls(leaderName, selfName string, url func(name string) (string, bool)) []string {
r.Lock()
defer r.Unlock()
// Build list including the leader and self.
urls := make([]string, 0)
if url, _ := url(leaderName); len(url) > 0 {
urls = append(urls, url)
}
// Retrieve a list of all nodes.
if e, _ := r.store.Get(RegistryKey, false, false); e != nil {
// Lookup the URL for each one.
for _, pair := range e.Node.Nodes {
_, name := filepath.Split(pair.Key)
if url, _ := url(name); len(url) > 0 && name != leaderName {
urls = append(urls, url)
}
}
}
log.Infof("URLs: %s / %s (%s)", leaderName, selfName, strings.Join(urls, ","))
return urls
}
func (d *Discoverer) findPeers() (peers []string, err error) {
resp, err := d.client.Get(path.Join(d.prefix), false, true)
if err != nil {
return nil, err
}
node := resp.Node
if node == nil {
return nil, fmt.Errorf("%s key doesn't exist.", d.prefix)
}
for _, n := range node.Nodes {
// Skip our own entry in the list, there is no point
if strings.HasSuffix(n.Key, "/"+d.name) {
continue
}
peers = append(peers, n.Value)
}
if len(peers) == 0 {
return nil, errors.New("Discovery found an initialized cluster but no peers are registered.")
}
log.Infof("Discovery found peers %v", peers)
return
}
// monitorTimeoutThreshold groups timeout threshold events together and prints
// them as a single log line.
func (s *PeerServer) monitorTimeoutThreshold(closeChan chan bool) {
for {
select {
case value := <-s.timeoutThresholdChan:
log.Infof("%s: warning: heartbeat near election timeout: %v", s.Config.Name, value)
case <-closeChan:
return
}
time.Sleep(ThresholdMonitorTimeout)
}
}
// raftEventLogger converts events from the Raft server into log messages.
func (s *PeerServer) raftEventLogger(event raft.Event) {
value := event.Value()
prevValue := event.PrevValue()
if value == nil {
value = "<nil>"
}
if prevValue == nil {
prevValue = "<nil>"
}
switch event.Type() {
case raft.StateChangeEventType:
log.Infof("%s: state changed from '%v' to '%v'.", s.Config.Name, prevValue, value)
case raft.TermChangeEventType:
log.Infof("%s: term #%v started.", s.Config.Name, value)
case raft.LeaderChangeEventType:
log.Infof("%s: leader changed from '%v' to '%v'.", s.Config.Name, prevValue, value)
case raft.AddPeerEventType:
log.Infof("%s: peer added: '%v'", s.Config.Name, value)
case raft.RemovePeerEventType:
log.Infof("%s: peer removed: '%v'", s.Config.Name, value)
case raft.HeartbeatIntervalEventType:
var name = "<unknown>"
if peer, ok := value.(*raft.Peer); ok {
name = peer.Name
}
log.Infof("%s: warning: heartbeat timed out: '%v'", s.Config.Name, name)
case raft.ElectionTimeoutThresholdEventType:
select {
case s.timeoutThresholdChan <- value:
default:
}
}
}
func (e *Etcd) Start(withEtcdServer bool) {
e.PeerServer.Start(e.Config.Snapshot, e.Config.Peers)
if withEtcdServer {
sListener := e.configEtcdListener()
go func() {
log.Infof("etcd server [name %s, listen on %s, advertised url %s]", e.EtcdServer.Name, sListener.Addr(), e.EtcdServer.URL())
corsInfo, err := ehttp.NewCORSInfo(e.Config.CorsOrigins)
if err != nil {
log.Fatal("CORS:", err)
}
sHTTP := &ehttp.CORSHandler{e.EtcdServer.HTTPHandler(), corsInfo}
log.Fatal(http.Serve(sListener, sHTTP))
}()
}
log.Infof("peer server [name %s, listen on %s, advertised url %s]", e.PeerServer.Config.Name, e.PeerServerListener.Addr(), e.PeerServer.Config.URL)
// Retrieve CORS configuration
corsInfo, err := ehttp.NewCORSInfo(e.Config.CorsOrigins)
if err != nil {
log.Fatal("CORS:", err)
}
sHTTP := &ehttp.CORSHandler{e.PeerServer.HTTPHandler(), corsInfo}
log.Fatal(http.Serve(e.PeerServerListener, sHTTP))
}
// profile starts CPU profiling.
func profile(path string) {
f, err := os.Create(path)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
sig := <-c
log.Infof("captured %v, stopping profiler and exiting..", sig)
pprof.StopCPUProfile()
os.Exit(1)
}()
}
func main() {
// Load configuration.
var config = config.New()
if err := config.Load(os.Args[1:]); err != nil {
fmt.Println(server.Usage() + "\n")
fmt.Println(err.Error() + "\n")
os.Exit(1)
} else if config.ShowVersion {
fmt.Println(server.ReleaseVersion)
os.Exit(0)
} else if config.ShowHelp {
fmt.Println(server.Usage() + "\n")
os.Exit(0)
}
// Enable options.
if config.VeryVeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Trace)
} else if config.VeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Debug)
} else if config.Verbose {
log.Verbose = true
}
if config.CPUProfileFile != "" {
profile(config.CPUProfileFile)
}
if config.DataDir == "" {
log.Fatal("The data dir was not set and could not be guessed from machine name")
}
// Create data directory if it doesn't already exist.
if err := os.MkdirAll(config.DataDir, 0744); err != nil {
log.Fatalf("Unable to create path: %s", err)
}
// Warn people if they have an info file
info := filepath.Join(config.DataDir, "info")
if _, err := os.Stat(info); err == nil {
log.Warnf("All cached configuration is now ignored. The file %s can be removed.", info)
}
var mbName string
if config.Trace() {
mbName = config.MetricsBucketName()
runtime.SetBlockProfileRate(1)
}
mb := metrics.NewBucket(mbName)
if config.GraphiteHost != "" {
err := mb.Publish(config.GraphiteHost)
if err != nil {
panic(err)
}
}
// Retrieve CORS configuration
corsInfo, err := ehttp.NewCORSInfo(config.CorsOrigins)
if err != nil {
log.Fatal("CORS:", err)
}
// Create etcd key-value store and registry.
store := store.New()
registry := server.NewRegistry(store)
// Create stats objects
followersStats := server.NewRaftFollowersStats(config.Name)
serverStats := server.NewRaftServerStats(config.Name)
// Calculate all of our timeouts
heartbeatTimeout := time.Duration(config.Peer.HeartbeatTimeout) * time.Millisecond
electionTimeout := time.Duration(config.Peer.ElectionTimeout) * time.Millisecond
dialTimeout := (3 * heartbeatTimeout) + electionTimeout
responseHeaderTimeout := (3 * heartbeatTimeout) + electionTimeout
// Create peer server
psConfig := server.PeerServerConfig{
Name: config.Name,
Scheme: config.PeerTLSInfo().Scheme(),
URL: config.Peer.Addr,
SnapshotCount: config.SnapshotCount,
MaxClusterSize: config.MaxClusterSize,
RetryTimes: config.MaxRetryAttempts,
RetryInterval: config.RetryInterval,
}
ps := server.NewPeerServer(psConfig, registry, store, &mb, followersStats, serverStats)
var psListener net.Listener
if psConfig.Scheme == "https" {
peerServerTLSConfig, err := config.PeerTLSInfo().ServerConfig()
if err != nil {
log.Fatal("peer server TLS error: ", err)
}
psListener, err = server.NewTLSListener(config.Peer.BindAddr, peerServerTLSConfig)
if err != nil {
log.Fatal("Failed to create peer listener: ", err)
}
} else {
psListener, err = server.NewListener(config.Peer.BindAddr)
if err != nil {
log.Fatal("Failed to create peer listener: ", err)
}
}
// Create raft transporter and server
raftTransporter := server.NewTransporter(followersStats, serverStats, registry, heartbeatTimeout, dialTimeout, responseHeaderTimeout)
if psConfig.Scheme == "https" {
raftClientTLSConfig, err := config.PeerTLSInfo().ClientConfig()
if err != nil {
log.Fatal("raft client TLS error: ", err)
}
raftTransporter.SetTLSConfig(*raftClientTLSConfig)
}
raftServer, err := raft.NewServer(config.Name, config.DataDir, raftTransporter, store, ps, "")
if err != nil {
log.Fatal(err)
}
raftServer.SetElectionTimeout(electionTimeout)
raftServer.SetHeartbeatInterval(heartbeatTimeout)
ps.SetRaftServer(raftServer)
// Create etcd server
s := server.New(config.Name, config.Addr, ps, registry, store, &mb)
if config.Trace() {
s.EnableTracing()
}
var sListener net.Listener
if config.EtcdTLSInfo().Scheme() == "https" {
etcdServerTLSConfig, err := config.EtcdTLSInfo().ServerConfig()
if err != nil {
log.Fatal("etcd TLS error: ", err)
}
sListener, err = server.NewTLSListener(config.BindAddr, etcdServerTLSConfig)
if err != nil {
log.Fatal("Failed to create TLS etcd listener: ", err)
}
} else {
sListener, err = server.NewListener(config.BindAddr)
if err != nil {
log.Fatal("Failed to create etcd listener: ", err)
}
}
ps.SetServer(s)
ps.Start(config.Snapshot, config.Peers)
go func() {
log.Infof("peer server [name %s, listen on %s, advertised url %s]", ps.Config.Name, psListener.Addr(), ps.Config.URL)
sHTTP := &ehttp.CORSHandler{ps.HTTPHandler(), corsInfo}
log.Fatal(http.Serve(psListener, sHTTP))
}()
log.Infof("etcd server [name %s, listen on %s, advertised url %s]", s.Name, sListener.Addr(), s.URL())
sHTTP := &ehttp.CORSHandler{s.HTTPHandler(), corsInfo}
log.Fatal(http.Serve(sListener, sHTTP))
}