func NewPeerServer(name string, path string, url string, bindAddr string, tlsConf *TLSConfig, tlsInfo *TLSInfo, registry *Registry, store store.Store, snapshotCount int, heartbeatTimeout, electionTimeout time.Duration) *PeerServer {
s := &PeerServer{
name: name,
url: url,
bindAddr: bindAddr,
tlsConf: tlsConf,
tlsInfo: tlsInfo,
registry: registry,
store: store,
followersStats: &raftFollowersStats{
Leader: name,
Followers: make(map[string]*raftFollowerStats),
},
serverStats: &raftServerStats{
Name: name,
StartTime: time.Now(),
sendRateQueue: &statsQueue{
back: -1,
},
recvRateQueue: &statsQueue{
back: -1,
},
},
HeartbeatTimeout: heartbeatTimeout,
ElectionTimeout: electionTimeout,
timeoutThresholdChan: make(chan interface{}, 1),
}
// Create transporter for raft
raftTransporter := newTransporter(tlsConf.Scheme, tlsConf.Client, s)
// Create raft server
raftServer, err := raft.NewServer(name, path, raftTransporter, s.store, s, "")
if err != nil {
log.Fatal(err)
}
s.snapConf = &snapshotConf{
checkingInterval: time.Second * 3,
// this is not accurate, we will update raft to provide an api
lastIndex: raftServer.CommitIndex(),
snapshotThr: uint64(snapshotCount),
}
s.raftServer = raftServer
s.raftServer.AddEventListener(raft.StateChangeEventType, s.raftEventLogger)
s.raftServer.AddEventListener(raft.LeaderChangeEventType, s.raftEventLogger)
s.raftServer.AddEventListener(raft.TermChangeEventType, s.raftEventLogger)
s.raftServer.AddEventListener(raft.AddPeerEventType, s.raftEventLogger)
s.raftServer.AddEventListener(raft.RemovePeerEventType, s.raftEventLogger)
s.raftServer.AddEventListener(raft.HeartbeatTimeoutEventType, s.raftEventLogger)
s.raftServer.AddEventListener(raft.ElectionTimeoutThresholdEventType, s.raftEventLogger)
return s
}
func NewPeerServer(name string, path string, url string, bindAddr string, tlsConf *TLSConfig, tlsInfo *TLSInfo, registry *Registry, store store.Store, snapshotCount int) *PeerServer {
s := &PeerServer{
name: name,
url: url,
bindAddr: bindAddr,
tlsConf: tlsConf,
tlsInfo: tlsInfo,
registry: registry,
store: store,
snapConf: &snapshotConf{time.Second * 3, 0, uint64(snapshotCount)},
followersStats: &raftFollowersStats{
Leader: name,
Followers: make(map[string]*raftFollowerStats),
},
serverStats: &raftServerStats{
StartTime: time.Now(),
sendRateQueue: &statsQueue{
back: -1,
},
recvRateQueue: &statsQueue{
back: -1,
},
},
HeartbeatTimeout: defaultHeartbeatTimeout,
ElectionTimeout: defaultElectionTimeout,
}
// Create transporter for raft
raftTransporter := newTransporter(tlsConf.Scheme, tlsConf.Client, s)
// Create raft server
raftServer, err := raft.NewServer(name, path, raftTransporter, s.store, s, "")
if err != nil {
log.Fatal(err)
}
s.raftServer = raftServer
return s
}
func main() {
// Load configuration.
var config = server.NewConfig()
if err := config.Load(os.Args[1:]); err != nil {
fmt.Println(server.Usage() + "\n")
fmt.Println(err.Error() + "\n")
os.Exit(1)
} else if config.ShowVersion {
fmt.Println(server.ReleaseVersion)
os.Exit(0)
} else if config.ShowHelp {
fmt.Println(server.Usage() + "\n")
os.Exit(0)
}
// Enable options.
if config.VeryVeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Trace)
} else if config.VeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Debug)
} else if config.Verbose {
log.Verbose = true
}
if config.CPUProfileFile != "" {
profile(config.CPUProfileFile)
}
if config.DataDir == "" {
log.Fatal("The data dir was not set and could not be guessed from machine name")
}
// Create data directory if it doesn't already exist.
if err := os.MkdirAll(config.DataDir, 0744); err != nil {
log.Fatalf("Unable to create path: %s", err)
}
// Load info object.
info, err := config.Info()
if err != nil {
log.Fatal("info:", err)
}
// Retrieve TLS configuration.
tlsConfig, err := info.EtcdTLS.Config()
if err != nil {
log.Fatal("Client TLS:", err)
}
peerTLSConfig, err := info.RaftTLS.Config()
if err != nil {
log.Fatal("Peer TLS:", err)
}
var mbName string
if config.Trace() {
mbName = config.MetricsBucketName()
runtime.SetBlockProfileRate(1)
}
mb := metrics.NewBucket(mbName)
if config.GraphiteHost != "" {
err := mb.Publish(config.GraphiteHost)
if err != nil {
panic(err)
}
}
// Retrieve CORS configuration
corsInfo, err := ehttp.NewCORSInfo(config.CorsOrigins)
if err != nil {
log.Fatal("CORS:", err)
}
// Create etcd key-value store and registry.
store := store.New()
registry := server.NewRegistry(store)
// Create stats objects
followersStats := server.NewRaftFollowersStats(info.Name)
serverStats := server.NewRaftServerStats(info.Name)
// Calculate all of our timeouts
heartbeatTimeout := time.Duration(config.Peer.HeartbeatTimeout) * time.Millisecond
electionTimeout := time.Duration(config.Peer.ElectionTimeout) * time.Millisecond
dialTimeout := (3 * heartbeatTimeout) + electionTimeout
responseHeaderTimeout := (3 * heartbeatTimeout) + electionTimeout
// Create peer server.
psConfig := server.PeerServerConfig{
Name: info.Name,
Scheme: peerTLSConfig.Scheme,
URL: info.RaftURL,
SnapshotCount: config.SnapshotCount,
MaxClusterSize: config.MaxClusterSize,
RetryTimes: config.MaxRetryAttempts,
}
ps := server.NewPeerServer(psConfig, registry, store, &mb, followersStats, serverStats)
var psListener net.Listener
if psConfig.Scheme == "https" {
psListener, err = server.NewTLSListener(info.RaftListenHost, info.RaftTLS.CertFile, info.RaftTLS.KeyFile)
} else {
psListener, err = server.NewListener(info.RaftListenHost)
}
if err != nil {
panic(err)
}
// Create Raft transporter and server
raftTransporter := server.NewTransporter(followersStats, serverStats, registry, heartbeatTimeout, dialTimeout, responseHeaderTimeout)
if psConfig.Scheme == "https" {
raftTransporter.SetTLSConfig(peerTLSConfig.Client)
}
raftServer, err := raft.NewServer(info.Name, config.DataDir, raftTransporter, store, ps, "")
if err != nil {
log.Fatal(err)
}
raftServer.SetElectionTimeout(electionTimeout)
raftServer.SetHeartbeatTimeout(heartbeatTimeout)
ps.SetRaftServer(raftServer)
// Create client server.
s := server.New(info.Name, info.EtcdURL, ps, registry, store, &mb)
if config.Trace() {
s.EnableTracing()
}
var sListener net.Listener
if tlsConfig.Scheme == "https" {
sListener, err = server.NewTLSListener(info.EtcdListenHost, info.EtcdTLS.CertFile, info.EtcdTLS.KeyFile)
} else {
sListener, err = server.NewListener(info.EtcdListenHost)
}
if err != nil {
panic(err)
}
ps.SetServer(s)
ps.Start(config.Snapshot, config.Peers)
// Run peer server in separate thread while the client server blocks.
go func() {
log.Infof("raft server [name %s, listen on %s, advertised url %s]", ps.Config.Name, psListener.Addr(), ps.Config.URL)
sHTTP := &ehttp.CORSHandler{ps.HTTPHandler(), corsInfo}
log.Fatal(http.Serve(psListener, sHTTP))
}()
log.Infof("etcd server [name %s, listen on %s, advertised url %s]", s.Name, sListener.Addr(), s.URL())
sHTTP := &ehttp.CORSHandler{s.HTTPHandler(), corsInfo}
log.Fatal(http.Serve(sListener, sHTTP))
}
// Starts a server in a temporary directory.
func RunServer(f func(*server.Server)) {
path, _ := ioutil.TempDir("", "etcd-")
defer os.RemoveAll(path)
store := store.New()
registry := server.NewRegistry(store)
serverStats := server.NewRaftServerStats(testName)
followersStats := server.NewRaftFollowersStats(testName)
psConfig := server.PeerServerConfig{
Name: testName,
URL: "http://" + testRaftURL,
Scheme: "http",
SnapshotCount: testSnapshotCount,
MaxClusterSize: 9,
}
ps := server.NewPeerServer(psConfig, registry, store, nil, followersStats, serverStats)
psListener, err := server.NewListener(testRaftURL)
if err != nil {
panic(err)
}
// Create Raft transporter and server
dialTimeout := (3 * testHeartbeatTimeout) + testElectionTimeout
responseHeaderTimeout := (3 * testHeartbeatTimeout) + testElectionTimeout
raftTransporter := server.NewTransporter(followersStats, serverStats, registry, testHeartbeatTimeout, dialTimeout, responseHeaderTimeout)
raftServer, err := raft.NewServer(testName, path, raftTransporter, store, ps, "")
if err != nil {
panic(err)
}
raftServer.SetElectionTimeout(testElectionTimeout)
raftServer.SetHeartbeatTimeout(testHeartbeatTimeout)
ps.SetRaftServer(raftServer)
s := server.New(testName, "http://"+testClientURL, ps, registry, store, nil)
sListener, err := server.NewListener(testClientURL)
if err != nil {
panic(err)
}
ps.SetServer(s)
// Start up peer server.
c := make(chan bool)
go func() {
c <- true
ps.Start(false, []string{})
http.Serve(psListener, ps.HTTPHandler())
}()
<-c
// Start up etcd server.
go func() {
c <- true
http.Serve(sListener, s.HTTPHandler())
}()
<-c
// Wait to make sure servers have started.
time.Sleep(50 * time.Millisecond)
// Execute the function passed in.
f(s)
// Clean up servers.
ps.Stop()
psListener.Close()
sListener.Close()
}