func TestStandbyJoinMiss(t *testing.T) {
clusterSize := 2
_, etcds, err := CreateCluster(clusterSize, &os.ProcAttr{Files: []*os.File{nil, os.Stdout, os.Stderr}}, false)
if err != nil {
t.Fatal("cannot create cluster")
}
defer DestroyCluster(etcds)
c := etcd.NewClient(nil)
c.SyncCluster()
time.Sleep(1 * time.Second)
// Verify that we have two machines.
result, err := c.Get("_etcd/machines", false, true)
assert.NoError(t, err)
assert.Equal(t, len(result.Node.Nodes), clusterSize)
resp, _ := tests.Put("http://localhost:7001/v2/admin/config", "application/json", bytes.NewBufferString(`{"removeDelay":4, "syncInterval":4}`))
if !assert.Equal(t, resp.StatusCode, 200) {
t.FailNow()
}
time.Sleep(time.Second)
resp, _ = tests.Delete("http://localhost:7001/v2/admin/machines/node2", "application/json", nil)
if !assert.Equal(t, resp.StatusCode, 200) {
t.FailNow()
}
// Wait for a monitor cycle before checking for removal.
time.Sleep(server.ActiveMonitorTimeout + (1 * time.Second))
// Verify that we now have one peer.
result, err = c.Get("_etcd/machines", false, true)
assert.NoError(t, err)
assert.Equal(t, len(result.Node.Nodes), 1)
// Simulate the join failure
_, err = server.NewClient(nil).AddMachine("http://localhost:7001",
&server.JoinCommand{
MinVersion: store.MinVersion(),
MaxVersion: store.MaxVersion(),
Name: "node2",
RaftURL: "http://127.0.0.1:7002",
EtcdURL: "http://127.0.0.1:4002",
})
assert.NoError(t, err)
time.Sleep(6 * time.Second)
go tests.Delete("http://localhost:7001/v2/admin/machines/node2", "application/json", nil)
time.Sleep(time.Second)
result, err = c.Get("_etcd/machines", false, true)
assert.NoError(t, err)
assert.Equal(t, len(result.Node.Nodes), 1)
}
// Run the etcd instance.
func (e *Etcd) Run() {
// Sanitize all the input fields.
if err := e.Config.Sanitize(); err != nil {
log.Fatalf("failed sanitizing configuration: %v", err)
}
// Force remove server configuration if specified.
if e.Config.Force {
e.Config.Reset()
}
// Enable options.
if e.Config.VeryVeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Trace)
goetcd.SetLogger(
golog.New(
"go-etcd",
false,
golog.CombinedSink(
os.Stdout,
"[%s] %s %-9s | %s\n",
[]string{"prefix", "time", "priority", "message"},
),
),
)
} else if e.Config.VeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Debug)
} else if e.Config.Verbose {
log.Verbose = true
}
if e.Config.CPUProfileFile != "" {
profile(e.Config.CPUProfileFile)
}
if e.Config.DataDir == "" {
log.Fatal("The data dir was not set and could not be guessed from machine name")
}
// Create data directory if it doesn't already exist.
if err := os.MkdirAll(e.Config.DataDir, 0744); err != nil {
log.Fatalf("Unable to create path: %s", err)
}
// Warn people if they have an info file
info := filepath.Join(e.Config.DataDir, "info")
if _, err := os.Stat(info); err == nil {
log.Warnf("All cached configuration is now ignored. The file %s can be removed.", info)
}
var mbName string
if e.Config.Trace() {
mbName = e.Config.MetricsBucketName()
runtime.SetBlockProfileRate(1)
}
mb := metrics.NewBucket(mbName)
if e.Config.GraphiteHost != "" {
err := mb.Publish(e.Config.GraphiteHost)
if err != nil {
panic(err)
}
}
// Retrieve CORS configuration
corsInfo, err := ehttp.NewCORSInfo(e.Config.CorsOrigins)
if err != nil {
log.Fatal("CORS:", err)
}
// Create etcd key-value store and registry.
e.Store = store.New()
e.Registry = server.NewRegistry(e.Store)
// Create stats objects
followersStats := server.NewRaftFollowersStats(e.Config.Name)
serverStats := server.NewRaftServerStats(e.Config.Name)
// Calculate all of our timeouts
heartbeatInterval := time.Duration(e.Config.Peer.HeartbeatInterval) * time.Millisecond
electionTimeout := time.Duration(e.Config.Peer.ElectionTimeout) * time.Millisecond
dialTimeout := (3 * heartbeatInterval) + electionTimeout
responseHeaderTimeout := (3 * heartbeatInterval) + electionTimeout
clientTransporter := &httpclient.Transport{
ResponseHeaderTimeout: responseHeaderTimeout + extraTimeout,
// This is a workaround for Transport.CancelRequest doesn't work on
// HTTPS connections blocked. The patch for it is in progress,
// and would be available in Go1.3
// More: https://codereview.appspot.com/69280043/
ConnectTimeout: dialTimeout + extraTimeout,
RequestTimeout: responseHeaderTimeout + dialTimeout + 2*extraTimeout,
}
if e.Config.PeerTLSInfo().Scheme() == "https" {
clientTLSConfig, err := e.Config.PeerTLSInfo().ClientConfig()
if err != nil {
log.Fatal("client TLS error: ", err)
}
clientTransporter.TLSClientConfig = clientTLSConfig
clientTransporter.DisableCompression = true
}
client := server.NewClient(clientTransporter)
// Create peer server
psConfig := server.PeerServerConfig{
Name: e.Config.Name,
Scheme: e.Config.PeerTLSInfo().Scheme(),
URL: e.Config.Peer.Addr,
SnapshotCount: e.Config.SnapshotCount,
RetryTimes: e.Config.MaxRetryAttempts,
RetryInterval: e.Config.RetryInterval,
}
e.PeerServer = server.NewPeerServer(psConfig, client, e.Registry, e.Store, &mb, followersStats, serverStats)
// Create raft transporter and server
raftTransporter := server.NewTransporter(followersStats, serverStats, e.Registry, heartbeatInterval, dialTimeout, responseHeaderTimeout)
if e.Config.PeerTLSInfo().Scheme() == "https" {
raftClientTLSConfig, err := e.Config.PeerTLSInfo().ClientConfig()
if err != nil {
log.Fatal("raft client TLS error: ", err)
}
raftTransporter.SetTLSConfig(*raftClientTLSConfig)
}
raftServer, err := raft.NewServer(e.Config.Name, e.Config.DataDir, raftTransporter, e.Store, e.PeerServer, "")
if err != nil {
log.Fatal(err)
}
raftServer.SetElectionTimeout(electionTimeout)
raftServer.SetHeartbeatInterval(heartbeatInterval)
e.PeerServer.SetRaftServer(raftServer, e.Config.Snapshot)
// Create etcd server
e.Server = server.New(e.Config.Name, e.Config.Addr, e.PeerServer, e.Registry, e.Store, &mb)
if e.Config.Trace() {
e.Server.EnableTracing()
}
e.PeerServer.SetServer(e.Server)
// Create standby server
ssConfig := server.StandbyServerConfig{
Name: e.Config.Name,
PeerScheme: e.Config.PeerTLSInfo().Scheme(),
PeerURL: e.Config.Peer.Addr,
ClientURL: e.Config.Addr,
DataDir: e.Config.DataDir,
}
e.StandbyServer = server.NewStandbyServer(ssConfig, client)
e.StandbyServer.SetRaftServer(raftServer)
// Generating config could be slow.
// Put it here to make listen happen immediately after peer-server starting.
peerTLSConfig := server.TLSServerConfig(e.Config.PeerTLSInfo())
etcdTLSConfig := server.TLSServerConfig(e.Config.EtcdTLSInfo())
if !e.StandbyServer.IsRunning() {
startPeerServer, possiblePeers, err := e.PeerServer.FindCluster(e.Config.Discovery, e.Config.Peers)
if err != nil {
log.Fatal(err)
}
if startPeerServer {
e.setMode(PeerMode)
} else {
e.StandbyServer.SyncCluster(possiblePeers)
e.setMode(StandbyMode)
}
} else {
e.setMode(StandbyMode)
}
serverHTTPHandler := &ehttp.CORSHandler{e.Server.HTTPHandler(), corsInfo}
peerServerHTTPHandler := &ehttp.CORSHandler{e.PeerServer.HTTPHandler(), corsInfo}
standbyServerHTTPHandler := &ehttp.CORSHandler{e.StandbyServer.ClientHTTPHandler(), corsInfo}
log.Infof("etcd server [name %s, listen on %s, advertised url %s]", e.Server.Name, e.Config.BindAddr, e.Server.URL())
listener := server.NewListener(e.Config.EtcdTLSInfo().Scheme(), e.Config.BindAddr, etcdTLSConfig)
e.server = &http.Server{Handler: &ModeHandler{e, serverHTTPHandler, standbyServerHTTPHandler},
ReadTimeout: time.Duration(e.Config.HTTPReadTimeout) * time.Second,
WriteTimeout: time.Duration(e.Config.HTTPWriteTimeout) * time.Second,
}
log.Infof("peer server [name %s, listen on %s, advertised url %s]", e.PeerServer.Config.Name, e.Config.Peer.BindAddr, e.PeerServer.Config.URL)
peerListener := server.NewListener(e.Config.PeerTLSInfo().Scheme(), e.Config.Peer.BindAddr, peerTLSConfig)
e.peerServer = &http.Server{Handler: &ModeHandler{e, peerServerHTTPHandler, http.NotFoundHandler()},
ReadTimeout: time.Duration(server.DefaultReadTimeout) * time.Second,
WriteTimeout: time.Duration(server.DefaultWriteTimeout) * time.Second,
}
wg := sync.WaitGroup{}
wg.Add(2)
go func() {
<-e.readyNotify
defer wg.Done()
if err := e.server.Serve(listener); err != nil {
if !isListenerClosing(err) {
log.Fatal(err)
}
}
}()
go func() {
<-e.readyNotify
defer wg.Done()
if err := e.peerServer.Serve(peerListener); err != nil {
if !isListenerClosing(err) {
log.Fatal(err)
}
}
}()
e.runServer()
listener.Close()
peerListener.Close()
wg.Wait()
log.Infof("etcd instance is stopped [name %s]", e.Config.Name)
close(e.stopNotify)
}
// Run the etcd instance.
func (e *Etcd) Run() {
// Sanitize all the input fields.
if err := e.Config.Sanitize(); err != nil {
log.Fatalf("failed sanitizing configuration: %v", err)
}
// Force remove server configuration if specified.
if e.Config.Force {
e.Config.Reset()
}
// Enable options.
if e.Config.VeryVeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Trace)
goetcd.SetLogger(
golog.New(
"go-etcd",
false,
golog.CombinedSink(
os.Stdout,
"[%s] %s %-9s | %s\n",
[]string{"prefix", "time", "priority", "message"},
),
),
)
} else if e.Config.VeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Debug)
} else if e.Config.Verbose {
log.Verbose = true
}
if e.Config.CPUProfileFile != "" {
profile(e.Config.CPUProfileFile)
}
if e.Config.DataDir == "" {
log.Fatal("The data dir was not set and could not be guessed from machine name")
}
// Create data directory if it doesn't already exist.
if err := os.MkdirAll(e.Config.DataDir, 0744); err != nil {
log.Fatalf("Unable to create path: %s", err)
}
// Warn people if they have an info file
info := filepath.Join(e.Config.DataDir, "info")
if _, err := os.Stat(info); err == nil {
log.Warnf("All cached configuration is now ignored. The file %s can be removed.", info)
}
var mbName string
if e.Config.Trace() {
mbName = e.Config.MetricsBucketName()
runtime.SetBlockProfileRate(1)
}
mb := metrics.NewBucket(mbName)
if e.Config.GraphiteHost != "" {
err := mb.Publish(e.Config.GraphiteHost)
if err != nil {
panic(err)
}
}
// Retrieve CORS configuration
corsInfo, err := ehttp.NewCORSInfo(e.Config.CorsOrigins)
if err != nil {
log.Fatal("CORS:", err)
}
// Create etcd key-value store and registry.
e.Store = store.New()
e.Registry = server.NewRegistry(e.Store)
// Create stats objects
followersStats := server.NewRaftFollowersStats(e.Config.Name)
serverStats := server.NewRaftServerStats(e.Config.Name)
// Calculate all of our timeouts
heartbeatInterval := time.Duration(e.Config.Peer.HeartbeatInterval) * time.Millisecond
electionTimeout := time.Duration(e.Config.Peer.ElectionTimeout) * time.Millisecond
dialTimeout := (3 * heartbeatInterval) + electionTimeout
responseHeaderTimeout := (3 * heartbeatInterval) + electionTimeout
clientTransporter := &httpclient.Transport{
ResponseHeaderTimeout: responseHeaderTimeout + extraTimeout,
// This is a workaround for Transport.CancelRequest doesn't work on
// HTTPS connections blocked. The patch for it is in progress,
// and would be available in Go1.3
// More: https://codereview.appspot.com/69280043/
ConnectTimeout: dialTimeout + extraTimeout,
RequestTimeout: responseHeaderTimeout + dialTimeout + 2*extraTimeout,
}
if e.Config.PeerTLSInfo().Scheme() == "https" {
clientTLSConfig, err := e.Config.PeerTLSInfo().ClientConfig()
if err != nil {
log.Fatal("client TLS error: ", err)
}
clientTransporter.TLSClientConfig = clientTLSConfig
clientTransporter.DisableCompression = true
}
client := server.NewClient(clientTransporter)
// Create peer server
psConfig := server.PeerServerConfig{
Name: e.Config.Name,
Scheme: e.Config.PeerTLSInfo().Scheme(),
URL: e.Config.Peer.Addr,
SnapshotCount: e.Config.SnapshotCount,
RetryTimes: e.Config.MaxRetryAttempts,
RetryInterval: e.Config.RetryInterval,
}
e.PeerServer = server.NewPeerServer(psConfig, client, e.Registry, e.Store, &mb, followersStats, serverStats)
// Create raft transporter and server
raftTransporter := server.NewTransporter(followersStats, serverStats, e.Registry, heartbeatInterval, dialTimeout, responseHeaderTimeout)
if psConfig.Scheme == "https" {
raftClientTLSConfig, err := e.Config.PeerTLSInfo().ClientConfig()
if err != nil {
log.Fatal("raft client TLS error: ", err)
}
raftTransporter.SetTLSConfig(*raftClientTLSConfig)
}
raftServer, err := raft.NewServer(e.Config.Name, e.Config.DataDir, raftTransporter, e.Store, e.PeerServer, "")
if err != nil {
log.Fatal(err)
}
raftServer.SetElectionTimeout(electionTimeout)
raftServer.SetHeartbeatInterval(heartbeatInterval)
e.PeerServer.SetRaftServer(raftServer)
// Create etcd server
e.Server = server.New(e.Config.Name, e.Config.Addr, e.PeerServer, e.Registry, e.Store, &mb)
if e.Config.Trace() {
e.Server.EnableTracing()
}
e.PeerServer.SetServer(e.Server)
// Generating config could be slow.
// Put it here to make listen happen immediately after peer-server starting.
peerTLSConfig := server.TLSServerConfig(e.Config.PeerTLSInfo())
etcdTLSConfig := server.TLSServerConfig(e.Config.EtcdTLSInfo())
log.Infof("etcd server [name %s, listen on %s, advertised url %s]", e.Server.Name, e.Config.BindAddr, e.Server.URL())
e.listener = server.NewListener(e.Config.EtcdTLSInfo().Scheme(), e.Config.BindAddr, etcdTLSConfig)
// An error string equivalent to net.errClosing for using with
// http.Serve() during server shutdown. Need to re-declare
// here because it is not exported by "net" package.
const errClosing = "use of closed network connection"
peerServerClosed := make(chan bool)
go func() {
// Starting peer server should be followed close by listening on its port
// If not, it may leave many requests unaccepted, or cannot receive heartbeat from the cluster.
// One severe problem caused if failing receiving heartbeats is when the second node joins one-node cluster,
// the cluster could be out of work as long as the two nodes cannot transfer messages.
e.PeerServer.Start(e.Config.Snapshot, e.Config.Discovery, e.Config.Peers)
go func() {
select {
case <-e.PeerServer.StopNotify():
case <-e.PeerServer.RemoveNotify():
log.Infof("peer server is removed")
os.Exit(0)
}
}()
log.Infof("peer server [name %s, listen on %s, advertised url %s]", e.PeerServer.Config.Name, e.Config.Peer.BindAddr, e.PeerServer.Config.URL)
e.peerListener = server.NewListener(psConfig.Scheme, e.Config.Peer.BindAddr, peerTLSConfig)
close(e.readyC) // etcd server is ready to accept connections, notify waiters.
sHTTP := &ehttp.CORSHandler{e.PeerServer.HTTPHandler(), corsInfo}
if err := http.Serve(e.peerListener, sHTTP); err != nil {
if !strings.Contains(err.Error(), errClosing) {
log.Fatal(err)
}
}
close(peerServerClosed)
}()
sHTTP := &ehttp.CORSHandler{e.Server.HTTPHandler(), corsInfo}
if err := http.Serve(e.listener, sHTTP); err != nil {
if !strings.Contains(err.Error(), errClosing) {
log.Fatal(err)
}
}
<-peerServerClosed
log.Infof("etcd instance is stopped [name %s]", e.Config.Name)
}