func (e *Etcd) serve() (err error) {
var ctlscfg *tls.Config
if !e.cfg.ClientTLSInfo.Empty() {
plog.Infof("ClientTLS: %s", e.cfg.ClientTLSInfo)
if ctlscfg, err = e.cfg.ClientTLSInfo.ServerConfig(); err != nil {
return err
}
}
if e.cfg.CorsInfo.String() != "" {
plog.Infof("cors = %s", e.cfg.CorsInfo)
}
// Start the peer server in a goroutine
ph := v2http.NewPeerHandler(e.Server)
for _, l := range e.Peers {
go func(l net.Listener) {
e.errc <- servePeerHTTP(l, ph)
}(l)
}
// Start a client server goroutine for each listen address
ch := http.Handler(&cors.CORSHandler{
Handler: v2http.NewClientHandler(e.Server, e.Server.Cfg.ReqTimeout()),
Info: e.cfg.CorsInfo,
})
for _, sctx := range e.sctxs {
// read timeout does not work with http close notify
// TODO: https://github.com/golang/go/issues/9524
go func(s *serveCtx) {
e.errc <- s.serve(e.Server, ctlscfg, ch, e.errc)
}(sctx)
}
return nil
}
// launch will attempt to start the etcd server
func (m *EtcdTestServer) launch(t *testing.T) error {
var err error
if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
return fmt.Errorf("failed to initialize the etcd server: %v", err)
}
m.s.SyncTicker = time.Tick(500 * time.Millisecond)
m.s.Start()
m.raftHandler = &testutil.PauseableHandler{Next: v2http.NewPeerHandler(m.s)}
for _, ln := range m.PeerListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: m.raftHandler},
}
hs.Start()
m.hss = append(m.hss, hs)
}
for _, ln := range m.ClientListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: v2http.NewClientHandler(m.s, m.ServerConfig.ReqTimeout())},
}
hs.Start()
m.hss = append(m.hss, hs)
}
return nil
}
// Launch starts a member based on ServerConfig, PeerListeners
// and ClientListeners.
func (m *member) Launch() error {
plog.Printf("launching %s (%s)", m.Name, m.grpcAddr)
var err error
if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
return fmt.Errorf("failed to initialize the etcd server: %v", err)
}
m.s.SyncTicker = time.Tick(500 * time.Millisecond)
m.s.Start()
m.raftHandler = &testutil.PauseableHandler{Next: v2http.NewPeerHandler(m.s)}
for _, ln := range m.PeerListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: m.raftHandler},
}
if m.PeerTLSInfo == nil {
hs.Start()
} else {
hs.TLS, err = m.PeerTLSInfo.ServerConfig()
if err != nil {
return err
}
hs.StartTLS()
}
m.hss = append(m.hss, hs)
}
for _, ln := range m.ClientListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: v2http.NewClientHandler(m.s, m.ServerConfig.ReqTimeout())},
}
if m.ClientTLSInfo == nil {
hs.Start()
} else {
hs.TLS, err = m.ClientTLSInfo.ServerConfig()
if err != nil {
return err
}
hs.StartTLS()
}
m.hss = append(m.hss, hs)
}
if m.grpcListener != nil {
var (
tlscfg *tls.Config
)
if m.ClientTLSInfo != nil && !m.ClientTLSInfo.Empty() {
tlscfg, err = m.ClientTLSInfo.ServerConfig()
if err != nil {
return err
}
}
m.grpcServer = v3rpc.Server(m.s, tlscfg)
go m.grpcServer.Serve(m.grpcListener)
}
plog.Printf("launched %s (%s)", m.Name, m.grpcAddr)
return nil
}
// Starts starts the etcd server and listening for client connections
func (e *EtcdServer) Start() {
var err error
e.EtcdServer, err = etcdserver.NewServer(e.config)
if err != nil {
msg := fmt.Sprintf("Etcd config error: %v", err)
panic(msg)
}
// create client listeners
clientListeners := createListenersOrPanic(e.config.ClientURLs)
// start etcd
e.EtcdServer.Start()
// setup client listeners
ch := v2http.NewClientHandler(e.EtcdServer, e.requestTimeout())
for _, l := range clientListeners {
go func(l net.Listener) {
srv := &http.Server{
Handler: ch,
ReadTimeout: 5 * time.Minute,
}
panic(srv.Serve(l))
}(l)
}
}
// Start starts the etcd server and listening for client connections
func (e *EtcdServer) Start() error {
var err error
e.EtcdServer, err = etcdserver.NewServer(e.config)
if err != nil {
return err
}
// create client listener, there should be only one url
e.clientListen, err = createListener(e.config.ClientURLs[0])
if err != nil {
return err
}
// start etcd
e.EtcdServer.Start()
// setup client listener
ch := v2http.NewClientHandler(e.EtcdServer, e.config.ReqTimeout())
errCh := make(chan error)
go func(l net.Listener) {
defer close(errCh)
srv := &http.Server{
Handler: ch,
ReadTimeout: 5 * time.Minute,
}
// Serve always returns a non-nil error.
errCh <- srv.Serve(l)
}(e.clientListen)
err = readinessCheck([]string{etcdHealthCheckURL}, errCh)
if err != nil {
return err
}
return nil
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd(cfg *config) (<-chan struct{}, error) {
urlsmap, token, err := getPeerURLsMapAndToken(cfg, "etcd")
if err != nil {
return nil, fmt.Errorf("error setting up initial cluster: %v", err)
}
if cfg.peerAutoTLS && cfg.peerTLSInfo.Empty() {
phosts := make([]string, 0)
for _, u := range cfg.lpurls {
phosts = append(phosts, u.Host)
}
cfg.peerTLSInfo, err = transport.SelfCert(cfg.dir, phosts)
if err != nil {
plog.Fatalf("could not get certs (%v)", err)
}
} else if cfg.peerAutoTLS {
plog.Warningf("ignoring peer auto TLS since certs given")
}
if !cfg.peerTLSInfo.Empty() {
plog.Infof("peerTLS: %s", cfg.peerTLSInfo)
}
plns := make([]net.Listener, 0)
for _, u := range cfg.lpurls {
if u.Scheme == "http" {
if !cfg.peerTLSInfo.Empty() {
plog.Warningf("The scheme of peer url %s is HTTP while peer key/cert files are presented. Ignored peer key/cert files.", u.String())
}
if cfg.peerTLSInfo.ClientCertAuth {
plog.Warningf("The scheme of peer url %s is HTTP while client cert auth (--peer-client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
}
}
var (
l net.Listener
tlscfg *tls.Config
)
if !cfg.peerTLSInfo.Empty() {
tlscfg, err = cfg.peerTLSInfo.ServerConfig()
if err != nil {
return nil, err
}
}
l, err = rafthttp.NewListener(u, tlscfg)
if err != nil {
return nil, err
}
urlStr := u.String()
plog.Info("listening for peers on ", urlStr)
defer func() {
if err != nil {
l.Close()
plog.Info("stopping listening for peers on ", urlStr)
}
}()
plns = append(plns, l)
}
var ctlscfg *tls.Config
if !cfg.clientTLSInfo.Empty() {
plog.Infof("clientTLS: %s", cfg.clientTLSInfo)
ctlscfg, err = cfg.clientTLSInfo.ServerConfig()
if err != nil {
return nil, err
}
}
sctxs := make(map[string]*serveCtx)
for _, u := range cfg.lcurls {
if u.Scheme == "http" {
if !cfg.clientTLSInfo.Empty() {
plog.Warningf("The scheme of client url %s is HTTP while peer key/cert files are presented. Ignored key/cert files.", u.String())
}
if cfg.clientTLSInfo.ClientCertAuth {
plog.Warningf("The scheme of client url %s is HTTP while client cert auth (--client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
}
}
if u.Scheme == "https" && ctlscfg == nil {
return nil, fmt.Errorf("TLS key/cert (--cert-file, --key-file) must be provided for client url %s with HTTPs scheme", u.String())
}
ctx := &serveCtx{host: u.Host}
if u.Scheme == "https" {
ctx.secure = true
} else {
ctx.insecure = true
}
if sctxs[u.Host] != nil {
if ctx.secure {
sctxs[u.Host].secure = true
}
if ctx.insecure {
sctxs[u.Host].insecure = true
}
continue
}
var l net.Listener
l, err = net.Listen("tcp", u.Host)
if err != nil {
return nil, err
}
var fdLimit uint64
if fdLimit, err = runtimeutil.FDLimit(); err == nil {
if fdLimit <= reservedInternalFDNum {
plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum)
}
l = transport.LimitListener(l, int(fdLimit-reservedInternalFDNum))
}
l, err = transport.NewKeepAliveListener(l, "tcp", nil)
ctx.l = l
if err != nil {
return nil, err
}
plog.Info("listening for client requests on ", u.Host)
defer func() {
if err != nil {
l.Close()
plog.Info("stopping listening for client requests on ", u.Host)
}
}()
sctxs[u.Host] = ctx
}
srvcfg := &etcdserver.ServerConfig{
Name: cfg.name,
ClientURLs: cfg.acurls,
PeerURLs: cfg.apurls,
DataDir: cfg.dir,
DedicatedWALDir: cfg.walDir,
SnapCount: cfg.snapCount,
MaxSnapFiles: cfg.maxSnapFiles,
MaxWALFiles: cfg.maxWalFiles,
InitialPeerURLsMap: urlsmap,
InitialClusterToken: token,
DiscoveryURL: cfg.durl,
DiscoveryProxy: cfg.dproxy,
NewCluster: cfg.isNewCluster(),
ForceNewCluster: cfg.forceNewCluster,
PeerTLSInfo: cfg.peerTLSInfo,
TickMs: cfg.TickMs,
ElectionTicks: cfg.electionTicks(),
AutoCompactionRetention: cfg.autoCompactionRetention,
QuotaBackendBytes: cfg.quotaBackendBytes,
StrictReconfigCheck: cfg.strictReconfigCheck,
EnablePprof: cfg.enablePprof,
}
var s *etcdserver.EtcdServer
s, err = etcdserver.NewServer(srvcfg)
if err != nil {
return nil, err
}
s.Start()
osutil.RegisterInterruptHandler(s.Stop)
if cfg.corsInfo.String() != "" {
plog.Infof("cors = %s", cfg.corsInfo)
}
ch := http.Handler(&cors.CORSHandler{
Handler: v2http.NewClientHandler(s, srvcfg.ReqTimeout()),
Info: cfg.corsInfo,
})
ph := v2http.NewPeerHandler(s)
// Start the peer server in a goroutine
for _, l := range plns {
go func(l net.Listener) {
plog.Fatal(servePeerHTTP(l, ph))
}(l)
}
// Start a client server goroutine for each listen address
for _, sctx := range sctxs {
go func(sctx *serveCtx) {
// read timeout does not work with http close notify
// TODO: https://github.com/golang/go/issues/9524
plog.Fatal(serve(sctx, s, ctlscfg, ch))
}(sctx)
}
return s.StopNotify(), nil
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd(cfg *config) (<-chan struct{}, error) {
initialPeers, token, err := setupCluster(cfg)
if err != nil {
return nil, fmt.Errorf("error setting up initial cluster: %v", err)
}
if !cfg.peerTLSInfo.Empty() {
glog.V(2).Infof("etcd: peerTLS: %s", cfg.peerTLSInfo)
}
plns := make([]net.Listener, 0)
for _, u := range cfg.lpurls {
var l net.Listener
peerTLSConfig, err := cfg.peerTLSInfo.ServerConfig()
if err != nil {
return nil, err
}
l, err = transport.NewTimeoutListener(u.Host, u.Scheme, peerTLSConfig, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
urlStr := u.String()
glog.V(2).Info("etcd: listening for peers on ", urlStr)
defer func() {
if err != nil {
l.Close()
glog.V(2).Info("etcd: stopping listening for peers on ", urlStr)
}
}()
plns = append(plns, l)
}
if !cfg.clientTLSInfo.Empty() {
glog.V(2).Infof("etcd: clientTLS: %s", cfg.clientTLSInfo)
}
clns := make([]net.Listener, 0)
for _, u := range cfg.lcurls {
l, err := net.Listen("tcp", u.Host)
if err != nil {
return nil, err
}
clientTLSConfig, err := cfg.clientTLSInfo.ServerConfig()
if err != nil {
return nil, err
}
l, err = transport.NewKeepAliveListener(l, u.Scheme, clientTLSConfig)
if err != nil {
return nil, err
}
urlStr := u.String()
glog.V(2).Info("etcd: listening for client requests on ", urlStr)
defer func() {
if err != nil {
l.Close()
glog.V(2).Info("etcd: stopping listening for client requests on ", urlStr)
}
}()
clns = append(clns, l)
}
srvcfg := &etcdserver.ServerConfig{
Name: cfg.name,
ClientURLs: cfg.acurls,
PeerURLs: cfg.apurls,
DataDir: cfg.dir,
SnapCount: cfg.snapCount,
MaxSnapFiles: cfg.maxSnapFiles,
InitialPeerURLsMap: initialPeers,
InitialClusterToken: token,
MaxWALFiles: cfg.maxWalFiles,
NewCluster: true,
ForceNewCluster: false,
TickMs: cfg.TickMs,
ElectionTicks: cfg.electionTicks(),
PeerTLSInfo: cfg.peerTLSInfo,
}
var s *etcdserver.EtcdServer
s, err = etcdserver.NewServer(srvcfg)
if err != nil {
return nil, err
}
osutil.HandleInterrupts()
s.Start()
osutil.RegisterInterruptHandler(s.Stop)
ch := etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout())
ph := etcdhttp.NewPeerHandler(s)
// Start the peer server in a goroutine
for _, l := range plns {
go func(l net.Listener) {
glog.Fatal(serveHTTP(l, ph, 5*time.Minute))
}(l)
}
// Start a client server goroutine for each listen address
for _, l := range clns {
go func(l net.Listener) {
// read timeout does not work with http close notify
// TODO: https://github.com/golang/go/issues/9524
glog.Fatal(serveHTTP(l, ch, 0))
}(l)
}
return s.StopNotify(), nil
}
func NewEmbeddedEtcd(port int, dataDir string) (*EmbeddedEtcd, error) {
var err error
se := &EmbeddedEtcd{Port: port}
se.listener, err = net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil {
return nil, err
}
se.Port = se.listener.Addr().(*net.TCPAddr).Port
clientURLs, err := interfaceURLs(se.Port)
if err != nil {
se.Stop()
return nil, err
}
peerURLs, err := types.NewURLs([]string{peerURL})
if err != nil {
se.Stop()
return nil, err
}
cfg := &etcdserver.ServerConfig{
Name: memberName,
ClientURLs: clientURLs,
PeerURLs: peerURLs,
DataDir: dataDir,
InitialPeerURLsMap: types.URLsMap{
memberName: peerURLs,
},
NewCluster: true,
TickMs: 100,
ElectionTicks: 10,
}
se.server, err = etcdserver.NewServer(cfg)
if err != nil {
return nil, err
}
se.server.Start()
osutil.RegisterInterruptHandler(se.server.Stop)
go http.Serve(se.listener,
v2http.NewClientHandler(se.server, cfg.ReqTimeout()))
// Wait for etcd server to be ready
t := time.Now().Add(startTimeout)
etcdClient, err := etcd.New(etcd.Config{
Endpoints: []string{fmt.Sprintf("http://localhost:%d", port)},
Transport: etcd.DefaultTransport,
HeaderTimeoutPerRequest: time.Second,
})
if err != nil {
return nil, err
}
kapi := etcd.NewKeysAPI(etcdClient)
for {
if time.Now().After(t) {
return nil, errors.New("Failed to start etcd")
}
if _, err := kapi.Set(context.Background(), "/skydive", "", nil); err == nil {
break
}
time.Sleep(time.Second)
}
return se, nil
}