// startEtcd runs StartEtcd in addition to hooks needed for standalone etcd. func startEtcd(cfg *embed.Config) (<-chan struct{}, <-chan error, error) { e, err := embed.StartEtcd(cfg) if err != nil { return nil, nil, err } osutil.RegisterInterruptHandler(e.Server.Stop) return e.Server.StopNotify(), e.Err(), nil }
// startEtcd runs StartEtcd in addition to hooks needed for standalone etcd. func startEtcd(cfg *embed.Config) (<-chan struct{}, <-chan error, error) { defaultHost, dhErr := cfg.IsDefaultHost() if defaultHost != "" { if dhErr == nil { plog.Infof("advertising using detected default host %q", defaultHost) } else { plog.Noticef("failed to detect default host, advertise falling back to %q (%v)", defaultHost, dhErr) } } e, err := embed.StartEtcd(cfg) if err != nil { return nil, nil, err } osutil.RegisterInterruptHandler(e.Server.Stop) <-e.Server.ReadyNotify() // wait for e.Server to join the cluster return e.Server.StopNotify(), e.Err(), nil }
// startEtcd launches the etcd server and HTTP handlers for client/server communication. func startEtcd(cfg *config) (<-chan struct{}, error) { initialPeers, token, err := setupCluster(cfg) if err != nil { return nil, fmt.Errorf("error setting up initial cluster: %v", err) } pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, time.Second, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } if !cfg.peerTLSInfo.Empty() { glog.V(2).Infof("etcd: peerTLS: %s", cfg.peerTLSInfo) } plns := make([]net.Listener, 0) for _, u := range cfg.lpurls { var l net.Listener l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } urlStr := u.String() glog.V(2).Info("etcd: listening for peers on ", urlStr) defer func() { if err != nil { l.Close() glog.V(2).Info("etcd: stopping listening for peers on ", urlStr) } }() plns = append(plns, l) } if !cfg.clientTLSInfo.Empty() { glog.V(2).Infof("etcd: clientTLS: %s", cfg.clientTLSInfo) } clns := make([]net.Listener, 0) for _, u := range cfg.lcurls { l, err := net.Listen("tcp", u.Host) if err != nil { return nil, err } l, err = transport.NewKeepAliveListener(l, u.Scheme, cfg.clientTLSInfo) if err != nil { return nil, err } urlStr := u.String() glog.V(2).Info("etcd: listening for client requests on ", urlStr) defer func() { if err != nil { l.Close() glog.V(2).Info("etcd: stopping listening for client requests on ", urlStr) } }() clns = append(clns, l) } srvcfg := &etcdserver.ServerConfig{ Name: cfg.name, ClientURLs: cfg.acurls, PeerURLs: cfg.apurls, DataDir: cfg.dir, SnapCount: cfg.snapCount, MaxSnapFiles: cfg.maxSnapFiles, InitialPeerURLsMap: initialPeers, InitialClusterToken: token, MaxWALFiles: cfg.maxWalFiles, NewCluster: true, ForceNewCluster: false, Transport: pt, TickMs: cfg.TickMs, ElectionTicks: cfg.electionTicks(), } var s *etcdserver.EtcdServer s, err = etcdserver.NewServer(srvcfg) if err != nil { return nil, err } osutil.HandleInterrupts() s.Start() osutil.RegisterInterruptHandler(s.Stop) ch := etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout()) ph := etcdhttp.NewPeerHandler(s.Cluster(), s.RaftHandler()) // Start the peer server in a goroutine for _, l := range plns { go func(l net.Listener) { glog.Fatal(serveHTTP(l, ph, 5*time.Minute)) }(l) } // Start a client server goroutine for each listen address for _, l := range clns { go func(l net.Listener) { // read timeout does not work with http close notify // TODO: https://github.com/golang/go/issues/9524 glog.Fatal(serveHTTP(l, ch, 0)) }(l) } return s.StopNotify(), nil }
// startEtcd launches the etcd server and HTTP handlers for client/server communication. func startEtcd(cfg *config) (<-chan struct{}, error) { urlsmap, token, err := getPeerURLsMapAndToken(cfg, "etcd") if err != nil { return nil, fmt.Errorf("error setting up initial cluster: %v", err) } pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, peerDialTimeout(cfg.ElectionMs), rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } if !cfg.peerTLSInfo.Empty() { plog.Infof("peerTLS: %s", cfg.peerTLSInfo) } plns := make([]net.Listener, 0) for _, u := range cfg.lpurls { if u.Scheme == "http" && !cfg.peerTLSInfo.Empty() { plog.Warningf("The scheme of peer url %s is http while peer key/cert files are presented. Ignored peer key/cert files.", u.String()) } var l net.Listener l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } urlStr := u.String() plog.Info("listening for peers on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for peers on ", urlStr) } }() plns = append(plns, l) } if !cfg.clientTLSInfo.Empty() { plog.Infof("clientTLS: %s", cfg.clientTLSInfo) } clns := make([]net.Listener, 0) for _, u := range cfg.lcurls { if u.Scheme == "http" && !cfg.clientTLSInfo.Empty() { plog.Warningf("The scheme of client url %s is http while client key/cert files are presented. Ignored client key/cert files.", u.String()) } var l net.Listener l, err = transport.NewKeepAliveListener(u.Host, u.Scheme, cfg.clientTLSInfo) if err != nil { return nil, err } if fdLimit, err := runtimeutil.FDLimit(); err == nil { if fdLimit <= reservedInternalFDNum { plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum) } l = netutil.LimitListener(l, int(fdLimit-reservedInternalFDNum)) } urlStr := u.String() plog.Info("listening for client requests on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for client requests on ", urlStr) } }() clns = append(clns, l) } var v3l net.Listener if cfg.v3demo { v3l, err = net.Listen("tcp", cfg.gRPCAddr) if err != nil { plog.Fatal(err) } plog.Infof("listening for client rpc on %s", cfg.gRPCAddr) } srvcfg := &etcdserver.ServerConfig{ Name: cfg.name, ClientURLs: cfg.acurls, PeerURLs: cfg.apurls, DataDir: cfg.dir, DedicatedWALDir: cfg.walDir, SnapCount: cfg.snapCount, MaxSnapFiles: cfg.maxSnapFiles, MaxWALFiles: cfg.maxWalFiles, InitialPeerURLsMap: urlsmap, InitialClusterToken: token, DiscoveryURL: cfg.durl, DiscoveryProxy: cfg.dproxy, NewCluster: cfg.isNewCluster(), ForceNewCluster: cfg.forceNewCluster, Transport: pt, TickMs: cfg.TickMs, ElectionTicks: cfg.electionTicks(), V3demo: cfg.v3demo, StrictReconfigCheck: cfg.strictReconfigCheck, } var s *etcdserver.EtcdServer s, err = etcdserver.NewServer(srvcfg) if err != nil { return nil, err } s.Start() osutil.RegisterInterruptHandler(s.Stop) if cfg.corsInfo.String() != "" { plog.Infof("cors = %s", cfg.corsInfo) } ch := &cors.CORSHandler{ Handler: etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout()), Info: cfg.corsInfo, } ph := etcdhttp.NewPeerHandler(s.Cluster(), s.RaftHandler()) // Start the peer server in a goroutine for _, l := range plns { go func(l net.Listener) { plog.Fatal(serveHTTP(l, ph, 5*time.Minute)) }(l) } // Start a client server goroutine for each listen address for _, l := range clns { go func(l net.Listener) { // read timeout does not work with http close notify // TODO: https://github.com/golang/go/issues/9524 plog.Fatal(serveHTTP(l, ch, 0)) }(l) } if cfg.v3demo { // set up v3 demo rpc grpcServer := grpc.NewServer() etcdserverpb.RegisterEtcdServer(grpcServer, v3rpc.New(s)) go plog.Fatal(grpcServer.Serve(v3l)) } return s.StopNotify(), nil }
// startEtcd launches the etcd server and HTTP handlers for client/server communication. func startEtcd(cfg *config) (<-chan struct{}, error) { urlsmap, token, err := getPeerURLsMapAndToken(cfg, "etcd") if err != nil { return nil, fmt.Errorf("error setting up initial cluster: %v", err) } if cfg.peerAutoTLS && cfg.peerTLSInfo.Empty() { phosts := make([]string, 0) for _, u := range cfg.lpurls { phosts = append(phosts, u.Host) } cfg.peerTLSInfo, err = transport.SelfCert(cfg.dir, phosts) if err != nil { plog.Fatalf("could not get certs (%v)", err) } } else if cfg.peerAutoTLS { plog.Warningf("ignoring peer auto TLS since certs given") } if !cfg.peerTLSInfo.Empty() { plog.Infof("peerTLS: %s", cfg.peerTLSInfo) } plns := make([]net.Listener, 0) for _, u := range cfg.lpurls { if u.Scheme == "http" { if !cfg.peerTLSInfo.Empty() { plog.Warningf("The scheme of peer url %s is HTTP while peer key/cert files are presented. Ignored peer key/cert files.", u.String()) } if cfg.peerTLSInfo.ClientCertAuth { plog.Warningf("The scheme of peer url %s is HTTP while client cert auth (--peer-client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String()) } } var ( l net.Listener tlscfg *tls.Config ) if !cfg.peerTLSInfo.Empty() { tlscfg, err = cfg.peerTLSInfo.ServerConfig() if err != nil { return nil, err } } l, err = rafthttp.NewListener(u, tlscfg) if err != nil { return nil, err } urlStr := u.String() plog.Info("listening for peers on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for peers on ", urlStr) } }() plns = append(plns, l) } var ctlscfg *tls.Config if !cfg.clientTLSInfo.Empty() { plog.Infof("clientTLS: %s", cfg.clientTLSInfo) ctlscfg, err = cfg.clientTLSInfo.ServerConfig() if err != nil { return nil, err } } sctxs := make(map[string]*serveCtx) for _, u := range cfg.lcurls { if u.Scheme == "http" { if !cfg.clientTLSInfo.Empty() { plog.Warningf("The scheme of client url %s is HTTP while peer key/cert files are presented. Ignored key/cert files.", u.String()) } if cfg.clientTLSInfo.ClientCertAuth { plog.Warningf("The scheme of client url %s is HTTP while client cert auth (--client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String()) } } if u.Scheme == "https" && ctlscfg == nil { return nil, fmt.Errorf("TLS key/cert (--cert-file, --key-file) must be provided for client url %s with HTTPs scheme", u.String()) } ctx := &serveCtx{host: u.Host} if u.Scheme == "https" { ctx.secure = true } else { ctx.insecure = true } if sctxs[u.Host] != nil { if ctx.secure { sctxs[u.Host].secure = true } if ctx.insecure { sctxs[u.Host].insecure = true } continue } var l net.Listener l, err = net.Listen("tcp", u.Host) if err != nil { return nil, err } var fdLimit uint64 if fdLimit, err = runtimeutil.FDLimit(); err == nil { if fdLimit <= reservedInternalFDNum { plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum) } l = transport.LimitListener(l, int(fdLimit-reservedInternalFDNum)) } l, err = transport.NewKeepAliveListener(l, "tcp", nil) ctx.l = l if err != nil { return nil, err } plog.Info("listening for client requests on ", u.Host) defer func() { if err != nil { l.Close() plog.Info("stopping listening for client requests on ", u.Host) } }() sctxs[u.Host] = ctx } srvcfg := &etcdserver.ServerConfig{ Name: cfg.name, ClientURLs: cfg.acurls, PeerURLs: cfg.apurls, DataDir: cfg.dir, DedicatedWALDir: cfg.walDir, SnapCount: cfg.snapCount, MaxSnapFiles: cfg.maxSnapFiles, MaxWALFiles: cfg.maxWalFiles, InitialPeerURLsMap: urlsmap, InitialClusterToken: token, DiscoveryURL: cfg.durl, DiscoveryProxy: cfg.dproxy, NewCluster: cfg.isNewCluster(), ForceNewCluster: cfg.forceNewCluster, PeerTLSInfo: cfg.peerTLSInfo, TickMs: cfg.TickMs, ElectionTicks: cfg.electionTicks(), AutoCompactionRetention: cfg.autoCompactionRetention, QuotaBackendBytes: cfg.quotaBackendBytes, StrictReconfigCheck: cfg.strictReconfigCheck, EnablePprof: cfg.enablePprof, } var s *etcdserver.EtcdServer s, err = etcdserver.NewServer(srvcfg) if err != nil { return nil, err } s.Start() osutil.RegisterInterruptHandler(s.Stop) if cfg.corsInfo.String() != "" { plog.Infof("cors = %s", cfg.corsInfo) } ch := http.Handler(&cors.CORSHandler{ Handler: v2http.NewClientHandler(s, srvcfg.ReqTimeout()), Info: cfg.corsInfo, }) ph := v2http.NewPeerHandler(s) // Start the peer server in a goroutine for _, l := range plns { go func(l net.Listener) { plog.Fatal(servePeerHTTP(l, ph)) }(l) } // Start a client server goroutine for each listen address for _, sctx := range sctxs { go func(sctx *serveCtx) { // read timeout does not work with http close notify // TODO: https://github.com/golang/go/issues/9524 plog.Fatal(serve(sctx, s, ctlscfg, ch)) }(sctx) } return s.StopNotify(), nil }
// startEtcd launches the etcd server and HTTP handlers for client/server communication. func startEtcd(cfg *config) (<-chan struct{}, error) { urlsmap, token, err := getPeerURLsMapAndToken(cfg) if err != nil { return nil, fmt.Errorf("error setting up initial cluster: %v", err) } pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, rafthttp.DialTimeout, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } if !cfg.peerTLSInfo.Empty() { plog.Infof("peerTLS: %s", cfg.peerTLSInfo) } plns := make([]net.Listener, 0) for _, u := range cfg.lpurls { var l net.Listener l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { return nil, err } urlStr := u.String() plog.Info("listening for peers on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for peers on ", urlStr) } }() plns = append(plns, l) } if !cfg.clientTLSInfo.Empty() { plog.Infof("clientTLS: %s", cfg.clientTLSInfo) } clns := make([]net.Listener, 0) for _, u := range cfg.lcurls { var l net.Listener l, err = transport.NewKeepAliveListener(u.Host, u.Scheme, cfg.clientTLSInfo) if err != nil { return nil, err } urlStr := u.String() plog.Info("listening for client requests on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for client requests on ", urlStr) } }() clns = append(clns, l) } srvcfg := &etcdserver.ServerConfig{ Name: cfg.name, ClientURLs: cfg.acurls, PeerURLs: cfg.apurls, DataDir: cfg.dir, SnapCount: cfg.snapCount, MaxSnapFiles: cfg.maxSnapFiles, MaxWALFiles: cfg.maxWalFiles, InitialPeerURLsMap: urlsmap, InitialClusterToken: token, DiscoveryURL: cfg.durl, DiscoveryProxy: cfg.dproxy, NewCluster: cfg.isNewCluster(), ForceNewCluster: cfg.forceNewCluster, Transport: pt, TickMs: cfg.TickMs, ElectionTicks: cfg.electionTicks(), } var s *etcdserver.EtcdServer s, err = etcdserver.NewServer(srvcfg) if err != nil { return nil, err } s.Start() osutil.RegisterInterruptHandler(s.Stop) if cfg.corsInfo.String() != "" { plog.Infof("cors = %s", cfg.corsInfo) } ch := &cors.CORSHandler{ Handler: etcdhttp.NewClientHandler(s), Info: cfg.corsInfo, } ph := etcdhttp.NewPeerHandler(s.Cluster(), s.RaftHandler()) // Start the peer server in a goroutine for _, l := range plns { go func(l net.Listener) { plog.Fatal(serveHTTP(l, ph, 5*time.Minute)) }(l) } // Start a client server goroutine for each listen address for _, l := range clns { go func(l net.Listener) { // read timeout does not work with http close notify // TODO: https://github.com/golang/go/issues/9524 plog.Fatal(serveHTTP(l, ch, 0)) }(l) } return s.StopNotify(), nil }
// startEtcd launches the etcd server and HTTP handlers for client/server communication. //开启ETCD func startEtcd(cfg *config) (<-chan struct{}, error) { //获取ETCD集群信息(静态模式时获取与acurls相同, 动态模式时将从ETCD Discovery或DNS Discovery获取) 与 token信息 urlsmap, token, err := getPeerURLsMapAndToken(cfg, "etcd") if err != nil { return nil, fmt.Errorf("error setting up initial cluster: %v", err) } //加密文本传输协议 if cfg.peerAutoTLS && cfg.peerTLSInfo.Empty() { phosts := make([]string, 0) for _, u := range cfg.lpurls { phosts = append(phosts, u.Host) } cfg.peerTLSInfo, err = transport.SelfCert(cfg.dir, phosts) if err != nil { plog.Fatalf("could not get certs (%v)", err) } } else if cfg.peerAutoTLS { plog.Warningf("ignoring peer auto TLS since certs given") } if !cfg.peerTLSInfo.Empty() { plog.Infof("peerTLS: %s", cfg.peerTLSInfo) } plns := make([]net.Listener, 0) //监听etcd初期准备 for _, u := range cfg.lpurls { if u.Scheme == "http" { if !cfg.peerTLSInfo.Empty() { plog.Warningf("The scheme of peer url %s is HTTP while peer key/cert files are presented. Ignored peer key/cert files.", u.String()) } if cfg.peerTLSInfo.ClientCertAuth { plog.Warningf("The scheme of peer url %s is HTTP while client cert auth (--peer-client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String()) } } var ( l net.Listener tlscfg *tls.Config ) if !cfg.peerTLSInfo.Empty() { tlscfg, err = cfg.peerTLSInfo.ServerConfig() if err != nil { return nil, err } } l, err = rafthttp.NewListener(u, tlscfg) if err != nil { return nil, err } urlStr := u.String() plog.Info("listening for peers on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for peers on ", urlStr) } }() plns = append(plns, l) } var ctlscfg *tls.Config if !cfg.clientTLSInfo.Empty() { plog.Infof("clientTLS: %s", cfg.clientTLSInfo) ctlscfg, err = cfg.clientTLSInfo.ServerConfig() if err != nil { return nil, err } } sctxs := make(map[string]*serveCtx) //监听客户端初期准备 for _, u := range cfg.lcurls { if u.Scheme == "http" { if !cfg.clientTLSInfo.Empty() { plog.Warningf("The scheme of client url %s is HTTP while peer key/cert files are presented. Ignored key/cert files.", u.String()) } if cfg.clientTLSInfo.ClientCertAuth { plog.Warningf("The scheme of client url %s is HTTP while client cert auth (--client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String()) } } if u.Scheme == "https" && ctlscfg == nil { return nil, fmt.Errorf("TLS key/cert (--cert-file, --key-file) must be provided for client url %s with HTTPs scheme", u.String()) } ctx := &serveCtx{host: u.Host} if u.Scheme == "https" { ctx.secure = true } else { ctx.insecure = true } if sctxs[u.Host] != nil { if ctx.secure { sctxs[u.Host].secure = true } if ctx.insecure { sctxs[u.Host].insecure = true } continue } var l net.Listener l, err = net.Listen("tcp", u.Host) if err != nil { return nil, err } var fdLimit uint64 if fdLimit, err = runtimeutil.FDLimit(); err == nil { if fdLimit <= reservedInternalFDNum { plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum) } l = transport.LimitListener(l, int(fdLimit-reservedInternalFDNum)) } l, err = transport.NewKeepAliveListener(l, "tcp", nil) ctx.l = l if err != nil { return nil, err } plog.Info("listening for client requests on ", u.Host) defer func() { if err != nil { l.Close() plog.Info("stopping listening for client requests on ", u.Host) } }() sctxs[u.Host] = ctx } //载入服务器相关配置 srvcfg := &etcdserver.ServerConfig{ Name: cfg.name, //etcd服务名 ClientURLs: cfg.acurls, //其他etcd监听型号地址(一般为其他主机或其他端口) (同步与交互etcd信息) PeerURLs: cfg.apurls, //监听信号地址(本机端口) (同步与交互etcd信息) DataDir: cfg.dir, //数据通信保存位置 (默认产生的文件夹 member/snap member/wal) DedicatedWALDir: cfg.walDir, //ETCD将写WAL文件到walDir而不是DATADIR。这允许使用一个专用磁盘 并有助于避免记录和其它IO操作之间io的竞争。 默认为空 写入DataDir SnapCount: cfg.snapCount, //提交的事务数来触发快照到磁盘 默认为10000 MaxSnapFiles: cfg.maxSnapFiles, //快照文件保留最大数量(0为无限制)默认值:5 windows为:0 MaxWALFiles: cfg.maxWalFiles, //wal文件保留最大数量(0为无限制)默认值:5 windows为:0 InitialPeerURLsMap: urlsmap, //其他etcd监听型号地址 InitialClusterToken: token, //通信认证用 DiscoveryURL: cfg.durl, //动态发现协议的url DiscoveryProxy: cfg.dproxy, //动态发现协议(HTTP代理) NewCluster: cfg.isNewCluster(), //是否为新建集群 ForceNewCluster: cfg.forceNewCluster, //--???-- PeerTLSInfo: cfg.peerTLSInfo, //加密文本传输协议基本信息 TickMs: cfg.TickMs, //心跳间隔的时间(毫秒)。 ElectionTicks: cfg.electionTicks(), //时间(毫秒)选举超时 AutoCompactionRetention: cfg.autoCompactionRetention, StrictReconfigCheck: cfg.strictReconfigCheck, //严格配置检查 默认为false EnablePprof: cfg.enablePprof, //启用通过HTTP服务器运行时分析数据。地址是在客户端的URL+“/debug/ pprof” } //启动服务 var s *etcdserver.EtcdServer s, err = etcdserver.NewServer(srvcfg) if err != nil { return nil, err } s.Start() osutil.RegisterInterruptHandler(s.Stop) if cfg.corsInfo.String() != "" { plog.Infof("cors = %s", cfg.corsInfo) } ch := http.Handler(&cors.CORSHandler{ Handler: etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout()), Info: cfg.corsInfo, }) ph := etcdhttp.NewPeerHandler(s) // Start the peer server in a goroutine for _, l := range plns { go func(l net.Listener) { plog.Fatal(servePeerHTTP(l, ph)) }(l) } // Start a client server goroutine for each listen address for _, sctx := range sctxs { go func(sctx *serveCtx) { // read timeout does not work with http close notify // TODO: https://github.com/golang/go/issues/9524 plog.Fatal(serve(sctx, s, ctlscfg, ch)) }(sctx) } return s.StopNotify(), nil }
func NewEmbeddedEtcd(port int, dataDir string) (*EmbeddedEtcd, error) { var err error se := &EmbeddedEtcd{Port: port} se.listener, err = net.Listen("tcp", fmt.Sprintf(":%d", port)) if err != nil { return nil, err } se.Port = se.listener.Addr().(*net.TCPAddr).Port clientURLs, err := interfaceURLs(se.Port) if err != nil { se.Stop() return nil, err } peerURLs, err := types.NewURLs([]string{peerURL}) if err != nil { se.Stop() return nil, err } cfg := &etcdserver.ServerConfig{ Name: memberName, ClientURLs: clientURLs, PeerURLs: peerURLs, DataDir: dataDir, InitialPeerURLsMap: types.URLsMap{ memberName: peerURLs, }, NewCluster: true, TickMs: 100, ElectionTicks: 10, } se.server, err = etcdserver.NewServer(cfg) if err != nil { return nil, err } se.server.Start() osutil.RegisterInterruptHandler(se.server.Stop) go http.Serve(se.listener, v2http.NewClientHandler(se.server, cfg.ReqTimeout())) // Wait for etcd server to be ready t := time.Now().Add(startTimeout) etcdClient, err := etcd.New(etcd.Config{ Endpoints: []string{fmt.Sprintf("http://localhost:%d", port)}, Transport: etcd.DefaultTransport, HeaderTimeoutPerRequest: time.Second, }) if err != nil { return nil, err } kapi := etcd.NewKeysAPI(etcdClient) for { if time.Now().After(t) { return nil, errors.New("Failed to start etcd") } if _, err := kapi.Set(context.Background(), "/skydive", "", nil); err == nil { break } time.Sleep(time.Second) } return se, nil }
// startEtcd launches the etcd server and HTTP handlers for client/server communication. func startEtcd(cfg *config) (<-chan struct{}, error) { urlsmap, token, err := getPeerURLsMapAndToken(cfg, "etcd") if err != nil { return nil, fmt.Errorf("error setting up initial cluster: %v", err) } if cfg.peerAutoTLS && cfg.peerTLSInfo.Empty() { phosts := make([]string, 0) for _, u := range cfg.lpurls { phosts = append(phosts, u.Host) } cfg.peerTLSInfo, err = transport.SelfCert(cfg.dir, phosts) if err != nil { plog.Fatalf("could not get certs (%v)", err) } } else if cfg.peerAutoTLS { plog.Warningf("ignoring peer auto TLS since certs given") } if !cfg.peerTLSInfo.Empty() { plog.Infof("peerTLS: %s", cfg.peerTLSInfo) } plns := make([]net.Listener, 0) for _, u := range cfg.lpurls { if u.Scheme == "http" && !cfg.peerTLSInfo.Empty() { plog.Warningf("The scheme of peer url %s is http while peer key/cert files are presented. Ignored peer key/cert files.", u.String()) } var l net.Listener l, err = rafthttp.NewListener(u, cfg.peerTLSInfo) if err != nil { return nil, err } urlStr := u.String() plog.Info("listening for peers on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for peers on ", urlStr) } }() plns = append(plns, l) } if !cfg.clientTLSInfo.Empty() { plog.Infof("clientTLS: %s", cfg.clientTLSInfo) } clns := make([]net.Listener, 0) for _, u := range cfg.lcurls { if u.Scheme == "http" && !cfg.clientTLSInfo.Empty() { plog.Warningf("The scheme of client url %s is http while client key/cert files are presented. Ignored client key/cert files.", u.String()) } var l net.Listener l, err = net.Listen("tcp", u.Host) if err != nil { return nil, err } if fdLimit, err := runtimeutil.FDLimit(); err == nil { if fdLimit <= reservedInternalFDNum { plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum) } l = transport.LimitListener(l, int(fdLimit-reservedInternalFDNum)) } // Do not wrap around this listener if TLS Info is set. // HTTPS server expects TLS Conn created by TLSListener. l, err = transport.NewKeepAliveListener(l, u.Scheme, cfg.clientTLSInfo) if err != nil { return nil, err } urlStr := u.String() plog.Info("listening for client requests on ", urlStr) defer func() { if err != nil { l.Close() plog.Info("stopping listening for client requests on ", urlStr) } }() clns = append(clns, l) } srvcfg := &etcdserver.ServerConfig{ Name: cfg.name, ClientURLs: cfg.acurls, PeerURLs: cfg.apurls, DataDir: cfg.dir, DedicatedWALDir: cfg.walDir, SnapCount: cfg.snapCount, MaxSnapFiles: cfg.maxSnapFiles, MaxWALFiles: cfg.maxWalFiles, InitialPeerURLsMap: urlsmap, InitialClusterToken: token, DiscoveryURL: cfg.durl, DiscoveryProxy: cfg.dproxy, NewCluster: cfg.isNewCluster(), ForceNewCluster: cfg.forceNewCluster, PeerTLSInfo: cfg.peerTLSInfo, TickMs: cfg.TickMs, ElectionTicks: cfg.electionTicks(), V3demo: cfg.v3demo, AutoCompactionRetention: cfg.autoCompactionRetention, StrictReconfigCheck: cfg.strictReconfigCheck, EnablePprof: cfg.enablePprof, } var s *etcdserver.EtcdServer s, err = etcdserver.NewServer(srvcfg) if err != nil { return nil, err } s.Start() osutil.RegisterInterruptHandler(s.Stop) if cfg.corsInfo.String() != "" { plog.Infof("cors = %s", cfg.corsInfo) } ch := &cors.CORSHandler{ Handler: etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout()), Info: cfg.corsInfo, } ph := etcdhttp.NewPeerHandler(s) var grpcS *grpc.Server if cfg.v3demo { // set up v3 demo rpc tls := &cfg.clientTLSInfo if cfg.clientTLSInfo.Empty() { tls = nil } grpcS, err = v3rpc.Server(s, tls) if err != nil { s.Stop() <-s.StopNotify() return nil, err } } // Start the peer server in a goroutine for _, l := range plns { go func(l net.Listener) { plog.Fatal(serve(l, nil, ph, 5*time.Minute)) }(l) } // Start a client server goroutine for each listen address for _, l := range clns { go func(l net.Listener) { // read timeout does not work with http close notify // TODO: https://github.com/golang/go/issues/9524 plog.Fatal(serve(l, grpcS, ch, 0)) }(l) } return s.StopNotify(), nil }