func mustNewTransport(t *testing.T) *http.Transport { tr, err := transport.NewTimeoutTransport(transport.TLSInfo{}, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { t.Fatal(err) } return tr }
func mustNewTransport(t *testing.T, tlsInfo transport.TLSInfo) *http.Transport { // tick in integration test is short, so 1s dial timeout could play well. tr, err := transport.NewTimeoutTransport(tlsInfo, time.Second, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { t.Fatal(err) } return tr }
// configureTestCluster will set the params to start an etcd server func configureTestCluster(t *testing.T, name string) *EtcdTestServer { var err error m := &EtcdTestServer{} pln := newLocalListener(t) m.PeerListeners = []net.Listener{pln} m.PeerURLs, err = types.NewURLs([]string{"http://" + pln.Addr().String()}) if err != nil { t.Fatal(err) } m.CertificatesDir, err = ioutil.TempDir(os.TempDir(), "etcd_certificates") if err != nil { t.Fatal(err) } m.CertFile = path.Join(m.CertificatesDir, "etcdcert.pem") if err = ioutil.WriteFile(m.CertFile, []byte(CertFileContent), 0644); err != nil { t.Fatal(err) } m.KeyFile = path.Join(m.CertificatesDir, "etcdkey.pem") if err = ioutil.WriteFile(m.KeyFile, []byte(KeyFileContent), 0644); err != nil { t.Fatal(err) } m.CAFile = path.Join(m.CertificatesDir, "ca.pem") if err = ioutil.WriteFile(m.CAFile, []byte(CAFileContent), 0644); err != nil { t.Fatal(err) } cln := newSecuredLocalListener(t, m.CertFile, m.KeyFile, m.CAFile) m.ClientListeners = []net.Listener{cln} m.ClientURLs, err = types.NewURLs([]string{"https://" + cln.Addr().String()}) if err != nil { t.Fatal(err) } m.Name = name m.DataDir, err = ioutil.TempDir(os.TempDir(), "etcd") if err != nil { t.Fatal(err) } clusterStr := fmt.Sprintf("%s=http://%s", name, pln.Addr().String()) m.InitialPeerURLsMap, err = types.NewURLsMap(clusterStr) if err != nil { t.Fatal(err) } m.Transport, err = transport.NewTimeoutTransport(transport.TLSInfo{}, time.Second, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { t.Fatal(err) } m.NewCluster = true m.ForceNewCluster = false m.ElectionTicks = 10 m.TickMs = uint(10) return m }
func (t *Transport) Start() error { var err error // Read/write timeout is set for stream roundTripper to promptly // find out broken status, which minimizes the number of messages // sent on broken connection. t.streamRt, err = transport.NewTimeoutTransport(t.TLSInfo, t.DialTimeout, ConnReadTimeout, ConnWriteTimeout) if err != nil { return err } t.pipelineRt, err = transport.NewTransport(t.TLSInfo, t.DialTimeout) if err != nil { return err } t.remotes = make(map[types.ID]*remote) t.peers = make(map[types.ID]Peer) t.prober = probing.NewProber(t.pipelineRt) return nil }
// configureTestCluster will set the params to start an etcd server func configureTestCluster(t *testing.T, name string) *EtcdTestServer { var err error m := &EtcdTestServer{} pln := newLocalListener(t) m.PeerListeners = []net.Listener{pln} m.PeerURLs, err = types.NewURLs([]string{"http://" + pln.Addr().String()}) if err != nil { t.Fatal(err) } cln := newLocalListener(t) m.ClientListeners = []net.Listener{cln} m.ClientURLs, err = types.NewURLs([]string{"http://" + cln.Addr().String()}) if err != nil { t.Fatal(err) } m.Name = name m.DataDir, err = ioutil.TempDir(os.TempDir(), "etcd") if err != nil { t.Fatal(err) } clusterStr := fmt.Sprintf("%s=http://%s", name, pln.Addr().String()) m.InitialPeerURLsMap, err = types.NewURLsMap(clusterStr) if err != nil { t.Fatal(err) } m.Transport, err = transport.NewTimeoutTransport(transport.TLSInfo{}, time.Second, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { t.Fatal(err) } m.NewCluster = true m.ForceNewCluster = false m.ElectionTicks = 10 m.TickMs = uint(10) return m }
// RenewHTTP renews a lease at a given primary server. func RenewHTTP(id LeaseID, url string, tlsInfo transport.TLSInfo, timeout time.Duration) (int64, error) { // will post lreq protobuf to leader lreq, err := (&pb.LeaseKeepAliveRequest{ID: int64(id)}).Marshal() if err != nil { return -1, err } // TODO creating a new transporter for each forward request // can be expensive; in the future reuse transports and batch requests rt, err := transport.NewTimeoutTransport(tlsInfo, timeout, 0, 0) if err != nil { return -1, err } cc := &http.Client{Transport: rt, Timeout: timeout} resp, err := cc.Post(url, "application/protobuf", bytes.NewReader(lreq)) if err != nil { // TODO detect if leader failed and retry? return -1, err } b, err := ioutil.ReadAll(resp.Body) if err != nil { return -1, err } if resp.StatusCode != http.StatusOK { return -1, fmt.Errorf("lease: %s", string(b)) } lresp := &pb.LeaseKeepAliveResponse{} if err := lresp.Unmarshal(b); err != nil { return -1, fmt.Errorf(`lease: %v. data = "%s"`, err, string(b)) } if lresp.ID != int64(id) { return -1, fmt.Errorf("lease: renew id mismatch") } return lresp.TTL, err }
// Metric gets the metric value for a member func (m *member) Metric(metricName string) (string, error) { cfgtls := transport.TLSInfo{} tr, err := transport.NewTimeoutTransport(cfgtls, time.Second, time.Second, time.Second) if err != nil { return "", err } cli := &http.Client{Transport: tr} resp, err := cli.Get(m.ClientURLs[0].String() + "/metrics") if err != nil { return "", err } defer resp.Body.Close() b, rerr := ioutil.ReadAll(resp.Body) if rerr != nil { return "", rerr } lines := strings.Split(string(b), "\n") for _, l := range lines { if strings.HasPrefix(l, metricName) { return strings.Split(l, " ")[1], nil } } return "", nil }
// startEtcd launches the etcd server and HTTP handlers for client/server communication. func startEtcd(cfg *config) (<-chan struct{}, error) { initialPeers, token, err := setupCluster(cfg) if err != nil { return nil, fmt.Errorf("error setting up initial cluster: %v", err) } pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, time.Second, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } if !cfg.peerTLSInfo.Empty() { glog.V(2).Infof("etcd: peerTLS: %s", cfg.peerTLSInfo) } plns := make([]net.Listener, 0) for _, u := range cfg.lpurls { var l net.Listener l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } urlStr := u.String() glog.V(2).Info("etcd: listening for peers on ", urlStr) defer func() { if err != nil { l.Close() glog.V(2).Info("etcd: stopping listening for peers on ", urlStr) } }() plns = append(plns, l) } if !cfg.clientTLSInfo.Empty() { glog.V(2).Infof("etcd: clientTLS: %s", cfg.clientTLSInfo) } clns := make([]net.Listener, 0) for _, u := range cfg.lcurls { l, err := net.Listen("tcp", u.Host) if err != nil { return nil, err } l, err = transport.NewKeepAliveListener(l, u.Scheme, cfg.clientTLSInfo) if err != nil { return nil, err } urlStr := u.String() glog.V(2).Info("etcd: listening for client requests on ", urlStr) defer func() { if err != nil { l.Close() glog.V(2).Info("etcd: stopping listening for client requests on ", urlStr) } }() clns = append(clns, l) } srvcfg := &etcdserver.ServerConfig{ Name: cfg.name, ClientURLs: cfg.acurls, PeerURLs: cfg.apurls, DataDir: cfg.dir, SnapCount: cfg.snapCount, MaxSnapFiles: cfg.maxSnapFiles, InitialPeerURLsMap: initialPeers, InitialClusterToken: token, MaxWALFiles: cfg.maxWalFiles, NewCluster: true, ForceNewCluster: false, Transport: pt, TickMs: cfg.TickMs, ElectionTicks: cfg.electionTicks(), } var s *etcdserver.EtcdServer s, err = etcdserver.NewServer(srvcfg) if err != nil { return nil, err } osutil.HandleInterrupts() s.Start() osutil.RegisterInterruptHandler(s.Stop) ch := etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout()) ph := etcdhttp.NewPeerHandler(s.Cluster(), s.RaftHandler()) // Start the peer server in a goroutine for _, l := range plns { go func(l net.Listener) { glog.Fatal(serveHTTP(l, ph, 5*time.Minute)) }(l) } // Start a client server goroutine for each listen address for _, l := range clns { go func(l net.Listener) { // read timeout does not work with http close notify // TODO: https://github.com/golang/go/issues/9524 glog.Fatal(serveHTTP(l, ch, 0)) }(l) } return s.StopNotify(), nil }
// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes. func startProxy(cfg *config) error { urlsmap, _, err := getPeerURLsMapAndToken(cfg, "proxy") if err != nil { return fmt.Errorf("error setting up initial cluster: %v", err) } pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, time.Duration(cfg.proxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.proxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.proxyWriteTimeoutMs)*time.Millisecond) if err != nil { return err } pt.MaxIdleConnsPerHost = proxy.DefaultMaxIdleConnsPerHost tr, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, time.Duration(cfg.proxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.proxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.proxyWriteTimeoutMs)*time.Millisecond) if err != nil { return err } cfg.dir = path.Join(cfg.dir, "proxy") err = os.MkdirAll(cfg.dir, 0700) if err != nil { return err } var peerURLs []string clusterfile := path.Join(cfg.dir, "cluster") b, err := ioutil.ReadFile(clusterfile) switch { case err == nil: if cfg.durl != "" { plog.Warningf("discovery token ignored since the proxy has already been initialized. Valid cluster file found at %q", clusterfile) } urls := struct{ PeerURLs []string }{} err := json.Unmarshal(b, &urls) if err != nil { return err } peerURLs = urls.PeerURLs plog.Infof("proxy: using peer urls %v from cluster file %q", peerURLs, clusterfile) case os.IsNotExist(err): if cfg.durl != "" { s, err := discovery.GetCluster(cfg.durl, cfg.dproxy) if err != nil { return err } if urlsmap, err = types.NewURLsMap(s); err != nil { return err } } peerURLs = urlsmap.URLs() plog.Infof("proxy: using peer urls %v ", peerURLs) default: return err } clientURLs := []string{} uf := func() []string { gcls, err := etcdserver.GetClusterFromRemotePeers(peerURLs, tr) // TODO: remove the 2nd check when we fix GetClusterFromPeers // GetClusterFromPeers should not return nil error with an invalid empty cluster if err != nil { plog.Warningf("proxy: %v", err) return []string{} } if len(gcls.Members()) == 0 { return clientURLs } clientURLs = gcls.ClientURLs() urls := struct{ PeerURLs []string }{gcls.PeerURLs()} b, err := json.Marshal(urls) if err != nil { plog.Warningf("proxy: error on marshal peer urls %s", err) return clientURLs } err = ioutil.WriteFile(clusterfile+".bak", b, 0600) if err != nil { plog.Warningf("proxy: error on writing urls %s", err) return clientURLs } err = os.Rename(clusterfile+".bak", clusterfile) if err != nil { plog.Warningf("proxy: error on updating clusterfile %s", err) return clientURLs } if !reflect.DeepEqual(gcls.PeerURLs(), peerURLs) { plog.Noticef("proxy: updated peer urls in cluster file from %v to %v", peerURLs, gcls.PeerURLs()) } peerURLs = gcls.PeerURLs() return clientURLs } ph := proxy.NewHandler(pt, uf, time.Duration(cfg.proxyFailureWaitMs)*time.Millisecond, time.Duration(cfg.proxyRefreshIntervalMs)*time.Millisecond) ph = &cors.CORSHandler{ Handler: ph, Info: cfg.corsInfo, } if cfg.isReadonlyProxy() { ph = proxy.NewReadonlyHandler(ph) } // Start a proxy server goroutine for each listen address for _, u := range cfg.lcurls { l, err := transport.NewListener(u.Host, u.Scheme, cfg.clientTLSInfo) if err != nil { return err } host := u.String() go func() { plog.Info("proxy: listening for client requests on ", host) mux := http.NewServeMux() mux.Handle("/metrics", prometheus.Handler()) mux.Handle("/", ph) plog.Fatal(http.Serve(l, mux)) }() } return nil }
// startEtcd launches the etcd server and HTTP handlers for client/server communication. func startEtcd(cfg *config) (<-chan struct{}, error) { urlsmap, token, err := getPeerURLsMapAndToken(cfg, "etcd") if err != nil { return nil, fmt.Errorf("error setting up initial cluster: %v", err) } pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, peerDialTimeout(cfg.ElectionMs), rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } if !cfg.peerTLSInfo.Empty() { plog.Infof("peerTLS: %s", cfg.peerTLSInfo) } plns := make([]net.Listener, 0) for _, u := range cfg.lpurls { if u.Scheme == "http" && !cfg.peerTLSInfo.Empty() { plog.Warningf("The scheme of peer url %s is http while peer key/cert files are presented. Ignored peer key/cert files.", u.String()) } var l net.Listener l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } urlStr := u.String() plog.Info("listening for peers on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for peers on ", urlStr) } }() plns = append(plns, l) } if !cfg.clientTLSInfo.Empty() { plog.Infof("clientTLS: %s", cfg.clientTLSInfo) } clns := make([]net.Listener, 0) for _, u := range cfg.lcurls { if u.Scheme == "http" && !cfg.clientTLSInfo.Empty() { plog.Warningf("The scheme of client url %s is http while client key/cert files are presented. Ignored client key/cert files.", u.String()) } var l net.Listener l, err = transport.NewKeepAliveListener(u.Host, u.Scheme, cfg.clientTLSInfo) if err != nil { return nil, err } if fdLimit, err := runtimeutil.FDLimit(); err == nil { if fdLimit <= reservedInternalFDNum { plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum) } l = netutil.LimitListener(l, int(fdLimit-reservedInternalFDNum)) } urlStr := u.String() plog.Info("listening for client requests on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for client requests on ", urlStr) } }() clns = append(clns, l) } var v3l net.Listener if cfg.v3demo { v3l, err = net.Listen("tcp", cfg.gRPCAddr) if err != nil { plog.Fatal(err) } plog.Infof("listening for client rpc on %s", cfg.gRPCAddr) } srvcfg := &etcdserver.ServerConfig{ Name: cfg.name, ClientURLs: cfg.acurls, PeerURLs: cfg.apurls, DataDir: cfg.dir, DedicatedWALDir: cfg.walDir, SnapCount: cfg.snapCount, MaxSnapFiles: cfg.maxSnapFiles, MaxWALFiles: cfg.maxWalFiles, InitialPeerURLsMap: urlsmap, InitialClusterToken: token, DiscoveryURL: cfg.durl, DiscoveryProxy: cfg.dproxy, NewCluster: cfg.isNewCluster(), ForceNewCluster: cfg.forceNewCluster, Transport: pt, TickMs: cfg.TickMs, ElectionTicks: cfg.electionTicks(), V3demo: cfg.v3demo, StrictReconfigCheck: cfg.strictReconfigCheck, } var s *etcdserver.EtcdServer s, err = etcdserver.NewServer(srvcfg) if err != nil { return nil, err } s.Start() osutil.RegisterInterruptHandler(s.Stop) if cfg.corsInfo.String() != "" { plog.Infof("cors = %s", cfg.corsInfo) } ch := &cors.CORSHandler{ Handler: etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout()), Info: cfg.corsInfo, } ph := etcdhttp.NewPeerHandler(s.Cluster(), s.RaftHandler()) // Start the peer server in a goroutine for _, l := range plns { go func(l net.Listener) { plog.Fatal(serveHTTP(l, ph, 5*time.Minute)) }(l) } // Start a client server goroutine for each listen address for _, l := range clns { go func(l net.Listener) { // read timeout does not work with http close notify // TODO: https://github.com/golang/go/issues/9524 plog.Fatal(serveHTTP(l, ch, 0)) }(l) } if cfg.v3demo { // set up v3 demo rpc grpcServer := grpc.NewServer() etcdserverpb.RegisterEtcdServer(grpcServer, v3rpc.New(s)) go plog.Fatal(grpcServer.Serve(v3l)) } return s.StopNotify(), nil }
// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes. func startProxy(cfg *config) error { plog.Notice("proxy: this proxy supports v2 API only!") pt, err := transport.NewTimeoutTransport(cfg.PeerTLSInfo, time.Duration(cfg.ProxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyWriteTimeoutMs)*time.Millisecond) if err != nil { return err } pt.MaxIdleConnsPerHost = httpproxy.DefaultMaxIdleConnsPerHost tr, err := transport.NewTimeoutTransport(cfg.PeerTLSInfo, time.Duration(cfg.ProxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyWriteTimeoutMs)*time.Millisecond) if err != nil { return err } cfg.Dir = path.Join(cfg.Dir, "proxy") err = os.MkdirAll(cfg.Dir, fileutil.PrivateDirMode) if err != nil { return err } var peerURLs []string clusterfile := path.Join(cfg.Dir, "cluster") b, err := ioutil.ReadFile(clusterfile) switch { case err == nil: if cfg.Durl != "" { plog.Warningf("discovery token ignored since the proxy has already been initialized. Valid cluster file found at %q", clusterfile) } if cfg.DNSCluster != "" { plog.Warningf("DNS SRV discovery ignored since the proxy has already been initialized. Valid cluster file found at %q", clusterfile) } urls := struct{ PeerURLs []string }{} err = json.Unmarshal(b, &urls) if err != nil { return err } peerURLs = urls.PeerURLs plog.Infof("proxy: using peer urls %v from cluster file %q", peerURLs, clusterfile) case os.IsNotExist(err): var urlsmap types.URLsMap urlsmap, _, err = cfg.PeerURLsMapAndToken("proxy") if err != nil { return fmt.Errorf("error setting up initial cluster: %v", err) } if cfg.Durl != "" { var s string s, err = discovery.GetCluster(cfg.Durl, cfg.Dproxy) if err != nil { return err } if urlsmap, err = types.NewURLsMap(s); err != nil { return err } } peerURLs = urlsmap.URLs() plog.Infof("proxy: using peer urls %v ", peerURLs) default: return err } clientURLs := []string{} uf := func() []string { gcls, gerr := etcdserver.GetClusterFromRemotePeers(peerURLs, tr) if gerr != nil { plog.Warningf("proxy: %v", gerr) return []string{} } clientURLs = gcls.ClientURLs() urls := struct{ PeerURLs []string }{gcls.PeerURLs()} b, jerr := json.Marshal(urls) if jerr != nil { plog.Warningf("proxy: error on marshal peer urls %s", jerr) return clientURLs } err = pkgioutil.WriteAndSyncFile(clusterfile+".bak", b, 0600) if err != nil { plog.Warningf("proxy: error on writing urls %s", err) return clientURLs } err = os.Rename(clusterfile+".bak", clusterfile) if err != nil { plog.Warningf("proxy: error on updating clusterfile %s", err) return clientURLs } if !reflect.DeepEqual(gcls.PeerURLs(), peerURLs) { plog.Noticef("proxy: updated peer urls in cluster file from %v to %v", peerURLs, gcls.PeerURLs()) } peerURLs = gcls.PeerURLs() return clientURLs } ph := httpproxy.NewHandler(pt, uf, time.Duration(cfg.ProxyFailureWaitMs)*time.Millisecond, time.Duration(cfg.ProxyRefreshIntervalMs)*time.Millisecond) ph = &cors.CORSHandler{ Handler: ph, Info: cfg.CorsInfo, } if cfg.isReadonlyProxy() { ph = httpproxy.NewReadonlyHandler(ph) } // Start a proxy server goroutine for each listen address for _, u := range cfg.LCUrls { var ( l net.Listener tlscfg *tls.Config ) if !cfg.ClientTLSInfo.Empty() { tlscfg, err = cfg.ClientTLSInfo.ServerConfig() if err != nil { return err } } l, err := transport.NewListener(u.Host, u.Scheme, tlscfg) if err != nil { return err } host := u.String() go func() { plog.Info("proxy: listening for client requests on ", host) mux := http.NewServeMux() mux.Handle("/metrics", prometheus.Handler()) mux.Handle("/", ph) plog.Fatal(http.Serve(l, mux)) }() } return nil }
// newStreamRoundTripper returns a roundTripper used to send stream requests // to rafthttp listener of remote peers. // Read/write timeout is set for stream roundTripper to promptly // find out broken status, which minimizes the number of messages // sent on broken connection. func newStreamRoundTripper(tlsInfo transport.TLSInfo, dialTimeout time.Duration) (http.RoundTripper, error) { return transport.NewTimeoutTransport(tlsInfo, dialTimeout, ConnReadTimeout, ConnWriteTimeout) }
// NewRoundTripper returns a roundTripper used to send requests // to rafthttp listener of remote peers. func NewRoundTripper(tlsInfo transport.TLSInfo, dialTimeout time.Duration) (http.RoundTripper, error) { // It uses timeout transport to pair with remote timeout listeners. // It sets no read/write timeout, because message in requests may // take long time to write out before reading out the response. return transport.NewTimeoutTransport(tlsInfo, dialTimeout, 0, 0) }
// startEtcd launches the etcd server and HTTP handlers for client/server communication. func startEtcd() (<-chan struct{}, error) { apurls, err := flags.URLsFromFlags(fs, "initial-advertise-peer-urls", "addr", peerTLSInfo) if err != nil { return nil, err } cls, err := setupCluster(apurls) if err != nil { return nil, fmt.Errorf("error setting up initial cluster: %v", err) } if *dir == "" { *dir = fmt.Sprintf("%v.etcd", *name) log.Printf("no data-dir provided, using default data-dir ./%s", *dir) } if err := os.MkdirAll(*dir, privateDirMode); err != nil { return nil, fmt.Errorf("cannot create data directory: %v", err) } if err := fileutil.IsDirWriteable(*dir); err != nil { return nil, fmt.Errorf("cannot write to data directory: %v", err) } pt, err := transport.NewTimeoutTransport(peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } acurls, err := flags.URLsFromFlags(fs, "advertise-client-urls", "addr", clientTLSInfo) if err != nil { return nil, err } lpurls, err := flags.URLsFromFlags(fs, "listen-peer-urls", "peer-bind-addr", peerTLSInfo) if err != nil { return nil, err } if !peerTLSInfo.Empty() { log.Printf("etcd: peerTLS: %s", peerTLSInfo) } plns := make([]net.Listener, 0) for _, u := range lpurls { var l net.Listener l, err = transport.NewTimeoutListener(u.Host, u.Scheme, peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } urlStr := u.String() log.Print("etcd: listening for peers on ", urlStr) defer func() { if err != nil { l.Close() log.Print("etcd: stopping listening for peers on ", urlStr) } }() plns = append(plns, l) } lcurls, err := flags.URLsFromFlags(fs, "listen-client-urls", "bind-addr", clientTLSInfo) if err != nil { return nil, err } if !clientTLSInfo.Empty() { log.Printf("etcd: clientTLS: %s", clientTLSInfo) } clns := make([]net.Listener, 0) for _, u := range lcurls { var l net.Listener l, err = transport.NewListener(u.Host, u.Scheme, clientTLSInfo) if err != nil { return nil, err } urlStr := u.String() log.Print("etcd: listening for client requests on ", urlStr) defer func() { if err != nil { l.Close() log.Print("etcd: stopping listening for client requests on ", urlStr) } }() clns = append(clns, l) } cfg := &etcdserver.ServerConfig{ Name: *name, ClientURLs: acurls, PeerURLs: apurls, DataDir: *dir, SnapCount: *snapCount, MaxSnapFiles: *maxSnapFiles, MaxWALFiles: *maxWalFiles, Cluster: cls, DiscoveryURL: *durl, DiscoveryProxy: *dproxy, NewCluster: clusterStateFlag.String() == clusterStateFlagNew, ForceNewCluster: *forceNewCluster, Transport: pt, } var s *etcdserver.EtcdServer s, err = etcdserver.NewServer(cfg) if err != nil { return nil, err } s.Start() if corsInfo.String() != "" { log.Printf("etcd: cors = %s", corsInfo) } ch := &cors.CORSHandler{ Handler: etcdhttp.NewClientHandler(s), Info: corsInfo, } ph := etcdhttp.NewPeerHandler(s) // Start the peer server in a goroutine for _, l := range plns { go func(l net.Listener) { log.Fatal(http.Serve(l, ph)) }(l) } // Start a client server goroutine for each listen address for _, l := range clns { go func(l net.Listener) { log.Fatal(http.Serve(l, ch)) }(l) } return s.StopNotify(), nil }
// startEtcd launches the etcd server and HTTP handlers for client/server communication. func startEtcd(cfg *config) (<-chan struct{}, error) { urlsmap, token, err := getPeerURLsMapAndToken(cfg) if err != nil { return nil, fmt.Errorf("error setting up initial cluster: %v", err) } pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, rafthttp.DialTimeout, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } if !cfg.peerTLSInfo.Empty() { plog.Infof("peerTLS: %s", cfg.peerTLSInfo) } plns := make([]net.Listener, 0) for _, u := range cfg.lpurls { var l net.Listener l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } urlStr := u.String() plog.Info("listening for peers on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for peers on ", urlStr) } }() plns = append(plns, l) } if !cfg.clientTLSInfo.Empty() { plog.Infof("clientTLS: %s", cfg.clientTLSInfo) } clns := make([]net.Listener, 0) for _, u := range cfg.lcurls { var l net.Listener l, err = transport.NewKeepAliveListener(u.Host, u.Scheme, cfg.clientTLSInfo) if err != nil { return nil, err } urlStr := u.String() plog.Info("listening for client requests on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for client requests on ", urlStr) } }() clns = append(clns, l) } srvcfg := &etcdserver.ServerConfig{ Name: cfg.name, ClientURLs: cfg.acurls, PeerURLs: cfg.apurls, DataDir: cfg.dir, SnapCount: cfg.snapCount, MaxSnapFiles: cfg.maxSnapFiles, MaxWALFiles: cfg.maxWalFiles, InitialPeerURLsMap: urlsmap, InitialClusterToken: token, DiscoveryURL: cfg.durl, DiscoveryProxy: cfg.dproxy, NewCluster: cfg.isNewCluster(), ForceNewCluster: cfg.forceNewCluster, Transport: pt, TickMs: cfg.TickMs, ElectionTicks: cfg.electionTicks(), } var s *etcdserver.EtcdServer s, err = etcdserver.NewServer(srvcfg) if err != nil { return nil, err } s.Start() osutil.RegisterInterruptHandler(s.Stop) if cfg.corsInfo.String() != "" { plog.Infof("cors = %s", cfg.corsInfo) } ch := &cors.CORSHandler{ Handler: etcdhttp.NewClientHandler(s), Info: cfg.corsInfo, } ph := etcdhttp.NewPeerHandler(s.Cluster(), s.RaftHandler()) // Start the peer server in a goroutine for _, l := range plns { go func(l net.Listener) { plog.Fatal(serveHTTP(l, ph, 5*time.Minute)) }(l) } // Start a client server goroutine for each listen address for _, l := range clns { go func(l net.Listener) { // read timeout does not work with http close notify // TODO: https://github.com/golang/go/issues/9524 plog.Fatal(serveHTTP(l, ch, 0)) }(l) } return s.StopNotify(), nil }
// startEtcd launches the etcd server and HTTP handlers for client/server communication. func startEtcd(cfg *config) (<-chan struct{}, error) { cls, err := setupCluster(cfg) if err != nil { return nil, fmt.Errorf("error setting up initial cluster: %v", err) } if cfg.dir == "" { cfg.dir = fmt.Sprintf("%v.etcd", cfg.name) log.Printf("no data-dir provided, using default data-dir ./%s", cfg.dir) } if err := makeMemberDir(cfg.dir); err != nil { return nil, fmt.Errorf("cannot use /member sub-directory: %v", err) } membdir := path.Join(cfg.dir, "member") if err := fileutil.IsDirWriteable(membdir); err != nil { return nil, fmt.Errorf("cannot write to data directory: %v", err) } pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } if !cfg.peerTLSInfo.Empty() { log.Printf("etcd: peerTLS: %s", cfg.peerTLSInfo) } plns := make([]net.Listener, 0) for _, u := range cfg.lpurls { var l net.Listener l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } urlStr := u.String() log.Print("etcd: listening for peers on ", urlStr) defer func() { if err != nil { l.Close() log.Print("etcd: stopping listening for peers on ", urlStr) } }() plns = append(plns, l) } if !cfg.clientTLSInfo.Empty() { log.Printf("etcd: clientTLS: %s", cfg.clientTLSInfo) } clns := make([]net.Listener, 0) for _, u := range cfg.lcurls { var l net.Listener l, err = transport.NewKeepAliveListener(u.Host, u.Scheme, cfg.clientTLSInfo) if err != nil { return nil, err } urlStr := u.String() log.Print("etcd: listening for client requests on ", urlStr) defer func() { if err != nil { l.Close() log.Print("etcd: stopping listening for client requests on ", urlStr) } }() clns = append(clns, l) } srvcfg := &etcdserver.ServerConfig{ Name: cfg.name, ClientURLs: cfg.acurls, PeerURLs: cfg.apurls, DataDir: membdir, SnapCount: cfg.snapCount, MaxSnapFiles: cfg.maxSnapFiles, MaxWALFiles: cfg.maxWalFiles, Cluster: cls, DiscoveryURL: cfg.durl, DiscoveryProxy: cfg.dproxy, NewCluster: cfg.isNewCluster(), ForceNewCluster: cfg.forceNewCluster, Transport: pt, TickMs: cfg.TickMs, ElectionTicks: cfg.electionTicks(), } var s *etcdserver.EtcdServer s, err = etcdserver.NewServer(srvcfg) if err != nil { return nil, err } s.Start() if cfg.corsInfo.String() != "" { log.Printf("etcd: cors = %s", cfg.corsInfo) } ch := &cors.CORSHandler{ Handler: etcdhttp.NewClientHandler(s), Info: cfg.corsInfo, } ph := etcdhttp.NewPeerHandler(s.Cluster, s.RaftHandler()) // Start the peer server in a goroutine for _, l := range plns { go func(l net.Listener) { log.Fatal(serveHTTP(l, ph, 5*time.Minute)) }(l) } // Start a client server goroutine for each listen address for _, l := range clns { go func(l net.Listener) { // read timeout does not work with http close notify // TODO: https://github.com/golang/go/issues/9524 log.Fatal(serveHTTP(l, ch, 0)) }(l) } return s.StopNotify(), nil }