// Launch starts a member based on ServerConfig, PeerListeners
// and ClientListeners.
func (m *member) Launch() error {
var err error
if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
return fmt.Errorf("failed to initialize the etcd server: %v", err)
}
m.s.SyncTicker = time.Tick(500 * time.Millisecond)
m.s.Start()
m.raftHandler = &testutil.PauseableHandler{Next: etcdhttp.NewPeerHandler(m.s.Cluster, m.s.RaftHandler())}
for _, ln := range m.PeerListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: m.raftHandler},
}
hs.Start()
m.hss = append(m.hss, hs)
}
for _, ln := range m.ClientListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: etcdhttp.NewClientHandler(m.s)},
}
hs.Start()
m.hss = append(m.hss, hs)
}
return nil
}
// launch will attempt to start the etcd server
func (m *EtcdTestServer) launch(t *testing.T) error {
var err error
if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
return fmt.Errorf("failed to initialize the etcd server: %v", err)
}
m.s.SyncTicker = time.Tick(500 * time.Millisecond)
m.s.Start()
m.raftHandler = etcdhttp.NewPeerHandler(m.s.Cluster(), m.s.RaftHandler())
for _, ln := range m.PeerListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: m.raftHandler},
}
hs.Start()
m.hss = append(m.hss, hs)
}
for _, ln := range m.ClientListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: etcdhttp.NewClientHandler(m.s, m.ServerConfig.ReqTimeout())},
}
hs.Start()
m.hss = append(m.hss, hs)
}
return nil
}
// Launch starts a member based on ServerConfig, PeerListeners
// and ClientListeners.
func (m *member) Launch() error {
var err error
if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
return fmt.Errorf("failed to initialize the etcd server: %v", err)
}
m.s.SyncTicker = time.Tick(500 * time.Millisecond)
m.s.Start()
m.raftHandler = &testutil.PauseableHandler{Next: etcdhttp.NewPeerHandler(m.s)}
for _, ln := range m.PeerListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: m.raftHandler},
}
if m.PeerTLSInfo == nil {
hs.Start()
} else {
hs.TLS, err = m.PeerTLSInfo.ServerConfig()
if err != nil {
return err
}
hs.StartTLS()
}
m.hss = append(m.hss, hs)
}
for _, ln := range m.ClientListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: etcdhttp.NewClientHandler(m.s, m.ServerConfig.ReqTimeout())},
}
if m.ClientTLSInfo == nil {
hs.Start()
} else {
hs.TLS, err = m.ClientTLSInfo.ServerConfig()
if err != nil {
return err
}
hs.StartTLS()
}
m.hss = append(m.hss, hs)
}
if m.grpcListener != nil {
var (
tlscfg *tls.Config
)
if m.ClientTLSInfo != nil && !m.ClientTLSInfo.Empty() {
tlscfg, err = m.ClientTLSInfo.ServerConfig()
if err != nil {
return err
}
}
m.grpcServer = v3rpc.Server(m.s, tlscfg)
go m.grpcServer.Serve(m.grpcListener)
}
return nil
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd() {
id, err := strconv.ParseInt(*fid, 0, 64)
if err != nil {
log.Fatal(err)
}
if id == raft.None {
log.Fatalf("etcd: cannot use None(%d) as etcdserver id", raft.None)
}
if peers.Pick(id) == "" {
log.Fatalf("%#x=<addr> must be specified in peers", id)
}
if *dir == "" {
*dir = fmt.Sprintf("%v_etcd_data", *fid)
log.Printf("main: no data-dir is given, using default data-dir ./%s", *dir)
}
if err := os.MkdirAll(*dir, privateDirMode); err != nil {
log.Fatalf("main: cannot create data directory: %v", err)
}
n, w := startRaft(id, peers.IDs(), path.Join(*dir, "wal"))
s := &etcdserver.EtcdServer{
Store: store.New(),
Node: n,
Save: w.Save,
Send: etcdhttp.Sender(*peers),
Ticker: time.Tick(100 * time.Millisecond),
SyncTicker: time.Tick(500 * time.Millisecond),
}
s.Start()
ch := etcdhttp.NewClientHandler(s, *peers, *timeout)
ph := etcdhttp.NewPeerHandler(s)
// Start the peer server in a goroutine
go func() {
log.Print("Listening for peers on ", *paddr)
log.Fatal(http.ListenAndServe(*paddr, ph))
}()
// Start a client server goroutine for each listen address
for _, addr := range *addrs {
addr := addr
go func() {
log.Print("Listening for client requests on ", addr)
log.Fatal(http.ListenAndServe(addr, ch))
}()
}
}
func NewSimpleEtcd() (*SimpleEtcd, error) {
var err error
se := &SimpleEtcd{}
se.listener, err = net.Listen("tcp", ":0")
if err != nil {
return nil, err
}
se.Port = se.listener.Addr().(*net.TCPAddr).Port
clientURLs, err := interfaceURLs(se.Port)
if err != nil {
se.Destroy()
return nil, err
}
se.dataDir, err = ioutil.TempDir("", tempPrefix)
if err != nil {
se.Destroy()
return nil, err
}
peerURLs, err := types.NewURLs([]string{peerURL})
if err != nil {
se.Destroy()
return nil, err
}
cfg := &etcdserver.ServerConfig{
Name: memberName,
ClientURLs: clientURLs,
PeerURLs: peerURLs,
DataDir: se.dataDir,
InitialPeerURLsMap: types.URLsMap{
memberName: peerURLs,
},
NewCluster: true,
TickMs: 100,
ElectionTicks: 10,
}
se.server, err = etcdserver.NewServer(cfg)
if err != nil {
return nil, err
}
se.server.Start()
go http.Serve(se.listener,
etcdhttp.NewClientHandler(se.server, cfg.ReqTimeout()))
return se, nil
}
func TestSet(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
n := raft.Start(1, []int64{1}, 0, 0)
n.Campaign(ctx)
srv := &etcdserver.EtcdServer{
Store: store.New(),
Node: n,
Save: func(st raftpb.HardState, ents []raftpb.Entry) {},
Send: etcdserver.SendFunc(nopSend),
}
srv.Start()
defer srv.Stop()
h := etcdhttp.NewClientHandler(srv, nil, time.Hour)
s := httptest.NewServer(h)
defer s.Close()
resp, err := http.PostForm(s.URL+"/v2/keys/foo", url.Values{"value": {"bar"}})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != 201 {
t.Errorf("StatusCode = %d, expected %d", resp.StatusCode, 201)
}
g := new(store.Event)
if err := json.NewDecoder(resp.Body).Decode(&g); err != nil {
t.Fatal(err)
}
w := &store.NodeExtern{
Key: "/foo/1",
Value: stringp("bar"),
ModifiedIndex: 1,
CreatedIndex: 1,
}
if !reflect.DeepEqual(g.Node, w) {
t.Errorf("g = %+v, want %+v", g.Node, w)
}
}
// Launch starts a member based on ServerConfig, PeerListeners
// and ClientListeners.
func (m *member) Launch(t *testing.T) {
m.s = etcdserver.NewServer(&m.ServerConfig)
m.s.Ticker = time.Tick(tickDuration)
m.s.SyncTicker = nil
m.s.Start()
for _, ln := range m.PeerListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: etcdhttp.NewPeerHandler(m.s)},
}
hs.Start()
m.hss = append(m.hss, hs)
}
for _, ln := range m.ClientListeners {
hs := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: etcdhttp.NewClientHandler(m.s)},
}
hs.Start()
m.hss = append(m.hss, hs)
}
}
func NewEmbeddedEtcd(port int, dataDir string) (*EmbeddedEtcd, error) {
var err error
se := &EmbeddedEtcd{Port: port}
se.listener, err = net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil {
return nil, err
}
se.Port = se.listener.Addr().(*net.TCPAddr).Port
clientURLs, err := interfaceURLs(se.Port)
if err != nil {
se.Stop()
return nil, err
}
peerURLs, err := types.NewURLs([]string{peerURL})
if err != nil {
se.Stop()
return nil, err
}
cfg := &etcdserver.ServerConfig{
Name: memberName,
ClientURLs: clientURLs,
PeerURLs: peerURLs,
DataDir: dataDir,
InitialPeerURLsMap: types.URLsMap{
memberName: peerURLs,
},
NewCluster: true,
TickMs: 100,
ElectionTicks: 10,
}
se.server, err = etcdserver.NewServer(cfg)
if err != nil {
return nil, err
}
se.server.Start()
go http.Serve(se.listener,
etcdhttp.NewClientHandler(se.server, cfg.ReqTimeout()))
// Wait for etcd server to be ready
t := time.Now().Add(startTimeout)
etcdClient, err := etcd.New(etcd.Config{
Endpoints: []string{fmt.Sprintf("http://localhost:%d", port)},
Transport: etcd.DefaultTransport,
HeaderTimeoutPerRequest: time.Second,
})
if err != nil {
return nil, err
}
kapi := etcd.NewKeysAPI(etcdClient)
for {
if time.Now().After(t) {
return nil, errors.New("Failed to start etcd")
}
if _, err := kapi.Get(context.Background(), "/", nil); err == nil {
break
}
time.Sleep(time.Second)
}
return se, nil
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd() {
cls, err := setupCluster()
if err != nil {
log.Fatalf("etcd: error setting up initial cluster: %v", err)
}
if *dir == "" {
*dir = fmt.Sprintf("%v.etcd", *name)
log.Printf("etcd: no data-dir provided, using default data-dir ./%s", *dir)
}
if err := os.MkdirAll(*dir, privateDirMode); err != nil {
log.Fatalf("etcd: cannot create data directory: %v", err)
}
if err := fileutil.IsDirWriteable(*dir); err != nil {
log.Fatalf("etcd: cannot write to data directory: %v", err)
}
pt, err := transport.NewTransport(peerTLSInfo)
if err != nil {
log.Fatal(err)
}
acurls, err := flags.URLsFromFlags(fs, "advertise-client-urls", "addr", clientTLSInfo)
if err != nil {
log.Fatal(err.Error())
}
cfg := &etcdserver.ServerConfig{
Name: *name,
ClientURLs: acurls,
DataDir: *dir,
SnapCount: *snapCount,
Cluster: cls,
DiscoveryURL: *durl,
ClusterState: *clusterState,
Transport: pt,
}
s := etcdserver.NewServer(cfg)
s.Start()
ch := &cors.CORSHandler{
Handler: etcdhttp.NewClientHandler(s),
Info: corsInfo,
}
ph := etcdhttp.NewPeerHandler(s)
lpurls, err := flags.URLsFromFlags(fs, "listen-peer-urls", "peer-bind-addr", peerTLSInfo)
if err != nil {
log.Fatal(err.Error())
}
for _, u := range lpurls {
l, err := transport.NewListener(u.Host, peerTLSInfo)
if err != nil {
log.Fatal(err)
}
// Start the peer server in a goroutine
urlStr := u.String()
go func() {
log.Print("etcd: listening for peers on ", urlStr)
log.Fatal(http.Serve(l, ph))
}()
}
lcurls, err := flags.URLsFromFlags(fs, "listen-client-urls", "bind-addr", clientTLSInfo)
if err != nil {
log.Fatal(err.Error())
}
// Start a client server goroutine for each listen address
for _, u := range lcurls {
l, err := transport.NewListener(u.Host, clientTLSInfo)
if err != nil {
log.Fatal(err)
}
urlStr := u.String()
go func() {
log.Print("etcd: listening for client requests on ", urlStr)
log.Fatal(http.Serve(l, ch))
}()
}
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd() (<-chan struct{}, error) {
apurls, err := flags.URLsFromFlags(fs, "initial-advertise-peer-urls", "addr", peerTLSInfo)
if err != nil {
return nil, err
}
cls, err := setupCluster(apurls)
if err != nil {
return nil, fmt.Errorf("error setting up initial cluster: %v", err)
}
if *dir == "" {
*dir = fmt.Sprintf("%v.etcd", *name)
log.Printf("no data-dir provided, using default data-dir ./%s", *dir)
}
if err := os.MkdirAll(*dir, privateDirMode); err != nil {
return nil, fmt.Errorf("cannot create data directory: %v", err)
}
if err := fileutil.IsDirWriteable(*dir); err != nil {
return nil, fmt.Errorf("cannot write to data directory: %v", err)
}
pt, err := transport.NewTimeoutTransport(peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
acurls, err := flags.URLsFromFlags(fs, "advertise-client-urls", "addr", clientTLSInfo)
if err != nil {
return nil, err
}
lpurls, err := flags.URLsFromFlags(fs, "listen-peer-urls", "peer-bind-addr", peerTLSInfo)
if err != nil {
return nil, err
}
if !peerTLSInfo.Empty() {
log.Printf("etcd: peerTLS: %s", peerTLSInfo)
}
plns := make([]net.Listener, 0)
for _, u := range lpurls {
var l net.Listener
l, err = transport.NewTimeoutListener(u.Host, u.Scheme, peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
urlStr := u.String()
log.Print("etcd: listening for peers on ", urlStr)
defer func() {
if err != nil {
l.Close()
log.Print("etcd: stopping listening for peers on ", urlStr)
}
}()
plns = append(plns, l)
}
lcurls, err := flags.URLsFromFlags(fs, "listen-client-urls", "bind-addr", clientTLSInfo)
if err != nil {
return nil, err
}
if !clientTLSInfo.Empty() {
log.Printf("etcd: clientTLS: %s", clientTLSInfo)
}
clns := make([]net.Listener, 0)
for _, u := range lcurls {
var l net.Listener
l, err = transport.NewListener(u.Host, u.Scheme, clientTLSInfo)
if err != nil {
return nil, err
}
urlStr := u.String()
log.Print("etcd: listening for client requests on ", urlStr)
defer func() {
if err != nil {
l.Close()
log.Print("etcd: stopping listening for client requests on ", urlStr)
}
}()
clns = append(clns, l)
}
cfg := &etcdserver.ServerConfig{
Name: *name,
ClientURLs: acurls,
PeerURLs: apurls,
DataDir: *dir,
SnapCount: *snapCount,
MaxSnapFiles: *maxSnapFiles,
MaxWALFiles: *maxWalFiles,
Cluster: cls,
DiscoveryURL: *durl,
DiscoveryProxy: *dproxy,
NewCluster: clusterStateFlag.String() == clusterStateFlagNew,
ForceNewCluster: *forceNewCluster,
Transport: pt,
}
var s *etcdserver.EtcdServer
s, err = etcdserver.NewServer(cfg)
if err != nil {
return nil, err
}
s.Start()
if corsInfo.String() != "" {
log.Printf("etcd: cors = %s", corsInfo)
}
ch := &cors.CORSHandler{
Handler: etcdhttp.NewClientHandler(s),
Info: corsInfo,
}
ph := etcdhttp.NewPeerHandler(s)
// Start the peer server in a goroutine
for _, l := range plns {
go func(l net.Listener) {
log.Fatal(http.Serve(l, ph))
}(l)
}
// Start a client server goroutine for each listen address
for _, l := range clns {
go func(l net.Listener) {
log.Fatal(http.Serve(l, ch))
}(l)
}
return s.StopNotify(), nil
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd(cfg *config) (<-chan struct{}, error) {
urlsmap, token, err := getPeerURLsMapAndToken(cfg)
if err != nil {
return nil, fmt.Errorf("error setting up initial cluster: %v", err)
}
pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, rafthttp.DialTimeout, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
if !cfg.peerTLSInfo.Empty() {
plog.Infof("peerTLS: %s", cfg.peerTLSInfo)
}
plns := make([]net.Listener, 0)
for _, u := range cfg.lpurls {
var l net.Listener
l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
urlStr := u.String()
plog.Info("listening for peers on ", urlStr)
defer func() {
if err != nil {
l.Close()
plog.Info("stopping listening for peers on ", urlStr)
}
}()
plns = append(plns, l)
}
if !cfg.clientTLSInfo.Empty() {
plog.Infof("clientTLS: %s", cfg.clientTLSInfo)
}
clns := make([]net.Listener, 0)
for _, u := range cfg.lcurls {
var l net.Listener
l, err = transport.NewKeepAliveListener(u.Host, u.Scheme, cfg.clientTLSInfo)
if err != nil {
return nil, err
}
urlStr := u.String()
plog.Info("listening for client requests on ", urlStr)
defer func() {
if err != nil {
l.Close()
plog.Info("stopping listening for client requests on ", urlStr)
}
}()
clns = append(clns, l)
}
srvcfg := &etcdserver.ServerConfig{
Name: cfg.name,
ClientURLs: cfg.acurls,
PeerURLs: cfg.apurls,
DataDir: cfg.dir,
SnapCount: cfg.snapCount,
MaxSnapFiles: cfg.maxSnapFiles,
MaxWALFiles: cfg.maxWalFiles,
InitialPeerURLsMap: urlsmap,
InitialClusterToken: token,
DiscoveryURL: cfg.durl,
DiscoveryProxy: cfg.dproxy,
NewCluster: cfg.isNewCluster(),
ForceNewCluster: cfg.forceNewCluster,
Transport: pt,
TickMs: cfg.TickMs,
ElectionTicks: cfg.electionTicks(),
}
var s *etcdserver.EtcdServer
s, err = etcdserver.NewServer(srvcfg)
if err != nil {
return nil, err
}
s.Start()
osutil.RegisterInterruptHandler(s.Stop)
if cfg.corsInfo.String() != "" {
plog.Infof("cors = %s", cfg.corsInfo)
}
ch := &cors.CORSHandler{
Handler: etcdhttp.NewClientHandler(s),
Info: cfg.corsInfo,
}
ph := etcdhttp.NewPeerHandler(s.Cluster(), s.RaftHandler())
// Start the peer server in a goroutine
for _, l := range plns {
go func(l net.Listener) {
plog.Fatal(serveHTTP(l, ph, 5*time.Minute))
}(l)
}
// Start a client server goroutine for each listen address
for _, l := range clns {
go func(l net.Listener) {
// read timeout does not work with http close notify
// TODO: https://github.com/golang/go/issues/9524
plog.Fatal(serveHTTP(l, ch, 0))
}(l)
}
return s.StopNotify(), nil
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd() {
self := cluster.FindName(*name)
if self == nil {
log.Fatalf("etcd: no member with name=%q exists", *name)
}
if self.ID == raft.None {
log.Fatalf("etcd: cannot use None(%d) as member id", raft.None)
}
if *dir == "" {
*dir = fmt.Sprintf("%v_etcd_data", self.ID)
log.Printf("main: no data-dir provided, using default data-dir ./%s", *dir)
}
if err := os.MkdirAll(*dir, privateDirMode); err != nil {
log.Fatalf("main: cannot create data directory: %v", err)
}
pt, err := transport.NewTransport(peerTLSInfo)
if err != nil {
log.Fatal(err)
}
acurls, err := pkg.URLsFromFlags(flag.CommandLine, "advertise-client-urls", "addr", clientTLSInfo)
if err != nil {
log.Fatal(err.Error())
}
cfg := &etcdserver.ServerConfig{
Name: *name,
ClientURLs: acurls,
DataDir: *dir,
SnapCount: int64(*snapCount),
Cluster: cluster,
Transport: pt,
}
s := etcdserver.NewServer(cfg)
s.Start()
ch := &pkg.CORSHandler{
Handler: etcdhttp.NewClientHandler(s),
Info: cors,
}
ph := etcdhttp.NewPeerHandler(s)
lpurls, err := pkg.URLsFromFlags(flag.CommandLine, "listen-peer-urls", "peer-bind-addr", peerTLSInfo)
if err != nil {
log.Fatal(err.Error())
}
for _, u := range lpurls {
l, err := transport.NewListener(u.Host, peerTLSInfo)
if err != nil {
log.Fatal(err)
}
// Start the peer server in a goroutine
urlStr := u.String()
go func() {
log.Print("Listening for peers on ", urlStr)
log.Fatal(http.Serve(l, ph))
}()
}
lcurls, err := pkg.URLsFromFlags(flag.CommandLine, "listen-client-urls", "bind-addr", clientTLSInfo)
if err != nil {
log.Fatal(err.Error())
}
// Start a client server goroutine for each listen address
for _, u := range lcurls {
l, err := transport.NewListener(u.Host, clientTLSInfo)
if err != nil {
log.Fatal(err)
}
urlStr := u.String()
go func() {
log.Print("Listening for client requests on ", urlStr)
log.Fatal(http.Serve(l, ch))
}()
}
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd(cfg *config) (<-chan struct{}, error) {
cls, err := setupCluster(cfg)
if err != nil {
return nil, fmt.Errorf("error setting up initial cluster: %v", err)
}
if cfg.dir == "" {
cfg.dir = fmt.Sprintf("%v.etcd", cfg.name)
log.Printf("no data-dir provided, using default data-dir ./%s", cfg.dir)
}
if err := makeMemberDir(cfg.dir); err != nil {
return nil, fmt.Errorf("cannot use /member sub-directory: %v", err)
}
membdir := path.Join(cfg.dir, "member")
if err := fileutil.IsDirWriteable(membdir); err != nil {
return nil, fmt.Errorf("cannot write to data directory: %v", err)
}
pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
if !cfg.peerTLSInfo.Empty() {
log.Printf("etcd: peerTLS: %s", cfg.peerTLSInfo)
}
plns := make([]net.Listener, 0)
for _, u := range cfg.lpurls {
var l net.Listener
l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
urlStr := u.String()
log.Print("etcd: listening for peers on ", urlStr)
defer func() {
if err != nil {
l.Close()
log.Print("etcd: stopping listening for peers on ", urlStr)
}
}()
plns = append(plns, l)
}
if !cfg.clientTLSInfo.Empty() {
log.Printf("etcd: clientTLS: %s", cfg.clientTLSInfo)
}
clns := make([]net.Listener, 0)
for _, u := range cfg.lcurls {
var l net.Listener
l, err = transport.NewKeepAliveListener(u.Host, u.Scheme, cfg.clientTLSInfo)
if err != nil {
return nil, err
}
urlStr := u.String()
log.Print("etcd: listening for client requests on ", urlStr)
defer func() {
if err != nil {
l.Close()
log.Print("etcd: stopping listening for client requests on ", urlStr)
}
}()
clns = append(clns, l)
}
srvcfg := &etcdserver.ServerConfig{
Name: cfg.name,
ClientURLs: cfg.acurls,
PeerURLs: cfg.apurls,
DataDir: membdir,
SnapCount: cfg.snapCount,
MaxSnapFiles: cfg.maxSnapFiles,
MaxWALFiles: cfg.maxWalFiles,
Cluster: cls,
DiscoveryURL: cfg.durl,
DiscoveryProxy: cfg.dproxy,
NewCluster: cfg.isNewCluster(),
ForceNewCluster: cfg.forceNewCluster,
Transport: pt,
TickMs: cfg.TickMs,
ElectionTicks: cfg.electionTicks(),
}
var s *etcdserver.EtcdServer
s, err = etcdserver.NewServer(srvcfg)
if err != nil {
return nil, err
}
s.Start()
if cfg.corsInfo.String() != "" {
log.Printf("etcd: cors = %s", cfg.corsInfo)
}
ch := &cors.CORSHandler{
Handler: etcdhttp.NewClientHandler(s),
Info: cfg.corsInfo,
}
ph := etcdhttp.NewPeerHandler(s.Cluster, s.RaftHandler())
// Start the peer server in a goroutine
for _, l := range plns {
go func(l net.Listener) {
log.Fatal(serveHTTP(l, ph, 5*time.Minute))
}(l)
}
// Start a client server goroutine for each listen address
for _, l := range clns {
go func(l net.Listener) {
// read timeout does not work with http close notify
// TODO: https://github.com/golang/go/issues/9524
log.Fatal(serveHTTP(l, ch, 0))
}(l)
}
return s.StopNotify(), nil
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd(cfg *config) (<-chan struct{}, error) {
urlsmap, token, err := getPeerURLsMapAndToken(cfg, "etcd")
if err != nil {
return nil, fmt.Errorf("error setting up initial cluster: %v", err)
}
pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, peerDialTimeout(cfg.ElectionMs), rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
if !cfg.peerTLSInfo.Empty() {
plog.Infof("peerTLS: %s", cfg.peerTLSInfo)
}
plns := make([]net.Listener, 0)
for _, u := range cfg.lpurls {
if u.Scheme == "http" && !cfg.peerTLSInfo.Empty() {
plog.Warningf("The scheme of peer url %s is http while peer key/cert files are presented. Ignored peer key/cert files.", u.String())
}
var l net.Listener
l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
urlStr := u.String()
plog.Info("listening for peers on ", urlStr)
defer func() {
if err != nil {
l.Close()
plog.Info("stopping listening for peers on ", urlStr)
}
}()
plns = append(plns, l)
}
if !cfg.clientTLSInfo.Empty() {
plog.Infof("clientTLS: %s", cfg.clientTLSInfo)
}
clns := make([]net.Listener, 0)
for _, u := range cfg.lcurls {
if u.Scheme == "http" && !cfg.clientTLSInfo.Empty() {
plog.Warningf("The scheme of client url %s is http while client key/cert files are presented. Ignored client key/cert files.", u.String())
}
var l net.Listener
l, err = transport.NewKeepAliveListener(u.Host, u.Scheme, cfg.clientTLSInfo)
if err != nil {
return nil, err
}
if fdLimit, err := runtimeutil.FDLimit(); err == nil {
if fdLimit <= reservedInternalFDNum {
plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum)
}
l = netutil.LimitListener(l, int(fdLimit-reservedInternalFDNum))
}
urlStr := u.String()
plog.Info("listening for client requests on ", urlStr)
defer func() {
if err != nil {
l.Close()
plog.Info("stopping listening for client requests on ", urlStr)
}
}()
clns = append(clns, l)
}
var v3l net.Listener
if cfg.v3demo {
v3l, err = net.Listen("tcp", cfg.gRPCAddr)
if err != nil {
plog.Fatal(err)
}
plog.Infof("listening for client rpc on %s", cfg.gRPCAddr)
}
srvcfg := &etcdserver.ServerConfig{
Name: cfg.name,
ClientURLs: cfg.acurls,
PeerURLs: cfg.apurls,
DataDir: cfg.dir,
DedicatedWALDir: cfg.walDir,
SnapCount: cfg.snapCount,
MaxSnapFiles: cfg.maxSnapFiles,
MaxWALFiles: cfg.maxWalFiles,
InitialPeerURLsMap: urlsmap,
InitialClusterToken: token,
DiscoveryURL: cfg.durl,
DiscoveryProxy: cfg.dproxy,
NewCluster: cfg.isNewCluster(),
ForceNewCluster: cfg.forceNewCluster,
Transport: pt,
TickMs: cfg.TickMs,
ElectionTicks: cfg.electionTicks(),
V3demo: cfg.v3demo,
StrictReconfigCheck: cfg.strictReconfigCheck,
}
var s *etcdserver.EtcdServer
s, err = etcdserver.NewServer(srvcfg)
if err != nil {
return nil, err
}
s.Start()
osutil.RegisterInterruptHandler(s.Stop)
if cfg.corsInfo.String() != "" {
plog.Infof("cors = %s", cfg.corsInfo)
}
ch := &cors.CORSHandler{
Handler: etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout()),
Info: cfg.corsInfo,
}
ph := etcdhttp.NewPeerHandler(s.Cluster(), s.RaftHandler())
// Start the peer server in a goroutine
for _, l := range plns {
go func(l net.Listener) {
plog.Fatal(serveHTTP(l, ph, 5*time.Minute))
}(l)
}
// Start a client server goroutine for each listen address
for _, l := range clns {
go func(l net.Listener) {
// read timeout does not work with http close notify
// TODO: https://github.com/golang/go/issues/9524
plog.Fatal(serveHTTP(l, ch, 0))
}(l)
}
if cfg.v3demo {
// set up v3 demo rpc
grpcServer := grpc.NewServer()
etcdserverpb.RegisterEtcdServer(grpcServer, v3rpc.New(s))
go plog.Fatal(grpcServer.Serve(v3l))
}
return s.StopNotify(), nil
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
//开启ETCD
func startEtcd(cfg *config) (<-chan struct{}, error) {
//获取ETCD集群信息(静态模式时获取与acurls相同, 动态模式时将从ETCD Discovery或DNS Discovery获取) 与 token信息
urlsmap, token, err := getPeerURLsMapAndToken(cfg, "etcd")
if err != nil {
return nil, fmt.Errorf("error setting up initial cluster: %v", err)
}
//加密文本传输协议
if cfg.peerAutoTLS && cfg.peerTLSInfo.Empty() {
phosts := make([]string, 0)
for _, u := range cfg.lpurls {
phosts = append(phosts, u.Host)
}
cfg.peerTLSInfo, err = transport.SelfCert(cfg.dir, phosts)
if err != nil {
plog.Fatalf("could not get certs (%v)", err)
}
} else if cfg.peerAutoTLS {
plog.Warningf("ignoring peer auto TLS since certs given")
}
if !cfg.peerTLSInfo.Empty() {
plog.Infof("peerTLS: %s", cfg.peerTLSInfo)
}
plns := make([]net.Listener, 0)
//监听etcd初期准备
for _, u := range cfg.lpurls {
if u.Scheme == "http" {
if !cfg.peerTLSInfo.Empty() {
plog.Warningf("The scheme of peer url %s is HTTP while peer key/cert files are presented. Ignored peer key/cert files.", u.String())
}
if cfg.peerTLSInfo.ClientCertAuth {
plog.Warningf("The scheme of peer url %s is HTTP while client cert auth (--peer-client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
}
}
var (
l net.Listener
tlscfg *tls.Config
)
if !cfg.peerTLSInfo.Empty() {
tlscfg, err = cfg.peerTLSInfo.ServerConfig()
if err != nil {
return nil, err
}
}
l, err = rafthttp.NewListener(u, tlscfg)
if err != nil {
return nil, err
}
urlStr := u.String()
plog.Info("listening for peers on ", urlStr)
defer func() {
if err != nil {
l.Close()
plog.Info("stopping listening for peers on ", urlStr)
}
}()
plns = append(plns, l)
}
var ctlscfg *tls.Config
if !cfg.clientTLSInfo.Empty() {
plog.Infof("clientTLS: %s", cfg.clientTLSInfo)
ctlscfg, err = cfg.clientTLSInfo.ServerConfig()
if err != nil {
return nil, err
}
}
sctxs := make(map[string]*serveCtx)
//监听客户端初期准备
for _, u := range cfg.lcurls {
if u.Scheme == "http" {
if !cfg.clientTLSInfo.Empty() {
plog.Warningf("The scheme of client url %s is HTTP while peer key/cert files are presented. Ignored key/cert files.", u.String())
}
if cfg.clientTLSInfo.ClientCertAuth {
plog.Warningf("The scheme of client url %s is HTTP while client cert auth (--client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
}
}
if u.Scheme == "https" && ctlscfg == nil {
return nil, fmt.Errorf("TLS key/cert (--cert-file, --key-file) must be provided for client url %s with HTTPs scheme", u.String())
}
ctx := &serveCtx{host: u.Host}
if u.Scheme == "https" {
ctx.secure = true
} else {
ctx.insecure = true
}
if sctxs[u.Host] != nil {
if ctx.secure {
sctxs[u.Host].secure = true
}
if ctx.insecure {
sctxs[u.Host].insecure = true
}
continue
}
var l net.Listener
l, err = net.Listen("tcp", u.Host)
if err != nil {
return nil, err
}
var fdLimit uint64
if fdLimit, err = runtimeutil.FDLimit(); err == nil {
if fdLimit <= reservedInternalFDNum {
plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum)
}
l = transport.LimitListener(l, int(fdLimit-reservedInternalFDNum))
}
l, err = transport.NewKeepAliveListener(l, "tcp", nil)
ctx.l = l
if err != nil {
return nil, err
}
plog.Info("listening for client requests on ", u.Host)
defer func() {
if err != nil {
l.Close()
plog.Info("stopping listening for client requests on ", u.Host)
}
}()
sctxs[u.Host] = ctx
}
//载入服务器相关配置
srvcfg := &etcdserver.ServerConfig{
Name: cfg.name, //etcd服务名
ClientURLs: cfg.acurls, //其他etcd监听型号地址(一般为其他主机或其他端口) (同步与交互etcd信息)
PeerURLs: cfg.apurls, //监听信号地址(本机端口) (同步与交互etcd信息)
DataDir: cfg.dir, //数据通信保存位置 (默认产生的文件夹 member/snap member/wal)
DedicatedWALDir: cfg.walDir, //ETCD将写WAL文件到walDir而不是DATADIR。这允许使用一个专用磁盘 并有助于避免记录和其它IO操作之间io的竞争。 默认为空 写入DataDir
SnapCount: cfg.snapCount, //提交的事务数来触发快照到磁盘 默认为10000
MaxSnapFiles: cfg.maxSnapFiles, //快照文件保留最大数量（0为无限制）默认值：5 windows为:0
MaxWALFiles: cfg.maxWalFiles, //wal文件保留最大数量（0为无限制）默认值：5 windows为:0
InitialPeerURLsMap: urlsmap, //其他etcd监听型号地址
InitialClusterToken: token, //通信认证用
DiscoveryURL: cfg.durl, //动态发现协议的url
DiscoveryProxy: cfg.dproxy, //动态发现协议(HTTP代理)
NewCluster: cfg.isNewCluster(), //是否为新建集群
ForceNewCluster: cfg.forceNewCluster, //--???--
PeerTLSInfo: cfg.peerTLSInfo, //加密文本传输协议基本信息
TickMs: cfg.TickMs, //心跳间隔的时间（毫秒）。
ElectionTicks: cfg.electionTicks(), //时间（毫秒）选举超时
AutoCompactionRetention: cfg.autoCompactionRetention,
StrictReconfigCheck: cfg.strictReconfigCheck, //严格配置检查 默认为false
EnablePprof: cfg.enablePprof, //启用通过HTTP服务器运行时分析数据。地址是在客户端的URL+“/debug/ pprof”
}
//启动服务
var s *etcdserver.EtcdServer
s, err = etcdserver.NewServer(srvcfg)
if err != nil {
return nil, err
}
s.Start()
osutil.RegisterInterruptHandler(s.Stop)
if cfg.corsInfo.String() != "" {
plog.Infof("cors = %s", cfg.corsInfo)
}
ch := http.Handler(&cors.CORSHandler{
Handler: etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout()),
Info: cfg.corsInfo,
})
ph := etcdhttp.NewPeerHandler(s)
// Start the peer server in a goroutine
for _, l := range plns {
go func(l net.Listener) {
plog.Fatal(servePeerHTTP(l, ph))
}(l)
}
// Start a client server goroutine for each listen address
for _, sctx := range sctxs {
go func(sctx *serveCtx) {
// read timeout does not work with http close notify
// TODO: https://github.com/golang/go/issues/9524
plog.Fatal(serve(sctx, s, ctlscfg, ch))
}(sctx)
}
return s.StopNotify(), nil
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd(cfg *config) (<-chan struct{}, error) {
initialPeers, token, err := setupCluster(cfg)
if err != nil {
return nil, fmt.Errorf("error setting up initial cluster: %v", err)
}
pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, time.Second, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
if !cfg.peerTLSInfo.Empty() {
glog.V(2).Infof("etcd: peerTLS: %s", cfg.peerTLSInfo)
}
plns := make([]net.Listener, 0)
for _, u := range cfg.lpurls {
var l net.Listener
l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
urlStr := u.String()
glog.V(2).Info("etcd: listening for peers on ", urlStr)
defer func() {
if err != nil {
l.Close()
glog.V(2).Info("etcd: stopping listening for peers on ", urlStr)
}
}()
plns = append(plns, l)
}
if !cfg.clientTLSInfo.Empty() {
glog.V(2).Infof("etcd: clientTLS: %s", cfg.clientTLSInfo)
}
clns := make([]net.Listener, 0)
for _, u := range cfg.lcurls {
l, err := net.Listen("tcp", u.Host)
if err != nil {
return nil, err
}
l, err = transport.NewKeepAliveListener(l, u.Scheme, cfg.clientTLSInfo)
if err != nil {
return nil, err
}
urlStr := u.String()
glog.V(2).Info("etcd: listening for client requests on ", urlStr)
defer func() {
if err != nil {
l.Close()
glog.V(2).Info("etcd: stopping listening for client requests on ", urlStr)
}
}()
clns = append(clns, l)
}
srvcfg := &etcdserver.ServerConfig{
Name: cfg.name,
ClientURLs: cfg.acurls,
PeerURLs: cfg.apurls,
DataDir: cfg.dir,
SnapCount: cfg.snapCount,
MaxSnapFiles: cfg.maxSnapFiles,
InitialPeerURLsMap: initialPeers,
InitialClusterToken: token,
MaxWALFiles: cfg.maxWalFiles,
NewCluster: true,
ForceNewCluster: false,
Transport: pt,
TickMs: cfg.TickMs,
ElectionTicks: cfg.electionTicks(),
}
var s *etcdserver.EtcdServer
s, err = etcdserver.NewServer(srvcfg)
if err != nil {
return nil, err
}
osutil.HandleInterrupts()
s.Start()
osutil.RegisterInterruptHandler(s.Stop)
ch := etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout())
ph := etcdhttp.NewPeerHandler(s.Cluster(), s.RaftHandler())
// Start the peer server in a goroutine
for _, l := range plns {
go func(l net.Listener) {
glog.Fatal(serveHTTP(l, ph, 5*time.Minute))
}(l)
}
// Start a client server goroutine for each listen address
for _, l := range clns {
go func(l net.Listener) {
// read timeout does not work with http close notify
// TODO: https://github.com/golang/go/issues/9524
glog.Fatal(serveHTTP(l, ch, 0))
}(l)
}
return s.StopNotify(), nil
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd(cfg *config) (<-chan struct{}, error) {
urlsmap, token, err := getPeerURLsMapAndToken(cfg, "etcd")
if err != nil {
return nil, fmt.Errorf("error setting up initial cluster: %v", err)
}
if cfg.peerAutoTLS && cfg.peerTLSInfo.Empty() {
phosts := make([]string, 0)
for _, u := range cfg.lpurls {
phosts = append(phosts, u.Host)
}
cfg.peerTLSInfo, err = transport.SelfCert(cfg.dir, phosts)
if err != nil {
plog.Fatalf("could not get certs (%v)", err)
}
} else if cfg.peerAutoTLS {
plog.Warningf("ignoring peer auto TLS since certs given")
}
if !cfg.peerTLSInfo.Empty() {
plog.Infof("peerTLS: %s", cfg.peerTLSInfo)
}
plns := make([]net.Listener, 0)
for _, u := range cfg.lpurls {
if u.Scheme == "http" && !cfg.peerTLSInfo.Empty() {
plog.Warningf("The scheme of peer url %s is http while peer key/cert files are presented. Ignored peer key/cert files.", u.String())
}
var l net.Listener
l, err = rafthttp.NewListener(u, cfg.peerTLSInfo)
if err != nil {
return nil, err
}
urlStr := u.String()
plog.Info("listening for peers on ", urlStr)
defer func() {
if err != nil {
l.Close()
plog.Info("stopping listening for peers on ", urlStr)
}
}()
plns = append(plns, l)
}
if !cfg.clientTLSInfo.Empty() {
plog.Infof("clientTLS: %s", cfg.clientTLSInfo)
}
clns := make([]net.Listener, 0)
for _, u := range cfg.lcurls {
if u.Scheme == "http" && !cfg.clientTLSInfo.Empty() {
plog.Warningf("The scheme of client url %s is http while client key/cert files are presented. Ignored client key/cert files.", u.String())
}
var l net.Listener
l, err = net.Listen("tcp", u.Host)
if err != nil {
return nil, err
}
if fdLimit, err := runtimeutil.FDLimit(); err == nil {
if fdLimit <= reservedInternalFDNum {
plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum)
}
l = transport.LimitListener(l, int(fdLimit-reservedInternalFDNum))
}
// Do not wrap around this listener if TLS Info is set.
// HTTPS server expects TLS Conn created by TLSListener.
l, err = transport.NewKeepAliveListener(l, u.Scheme, cfg.clientTLSInfo)
if err != nil {
return nil, err
}
urlStr := u.String()
plog.Info("listening for client requests on ", urlStr)
defer func() {
if err != nil {
l.Close()
plog.Info("stopping listening for client requests on ", urlStr)
}
}()
clns = append(clns, l)
}
srvcfg := &etcdserver.ServerConfig{
Name: cfg.name,
ClientURLs: cfg.acurls,
PeerURLs: cfg.apurls,
DataDir: cfg.dir,
DedicatedWALDir: cfg.walDir,
SnapCount: cfg.snapCount,
MaxSnapFiles: cfg.maxSnapFiles,
MaxWALFiles: cfg.maxWalFiles,
InitialPeerURLsMap: urlsmap,
InitialClusterToken: token,
DiscoveryURL: cfg.durl,
DiscoveryProxy: cfg.dproxy,
NewCluster: cfg.isNewCluster(),
ForceNewCluster: cfg.forceNewCluster,
PeerTLSInfo: cfg.peerTLSInfo,
TickMs: cfg.TickMs,
ElectionTicks: cfg.electionTicks(),
V3demo: cfg.v3demo,
AutoCompactionRetention: cfg.autoCompactionRetention,
StrictReconfigCheck: cfg.strictReconfigCheck,
EnablePprof: cfg.enablePprof,
}
var s *etcdserver.EtcdServer
s, err = etcdserver.NewServer(srvcfg)
if err != nil {
return nil, err
}
s.Start()
osutil.RegisterInterruptHandler(s.Stop)
if cfg.corsInfo.String() != "" {
plog.Infof("cors = %s", cfg.corsInfo)
}
ch := &cors.CORSHandler{
Handler: etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout()),
Info: cfg.corsInfo,
}
ph := etcdhttp.NewPeerHandler(s)
var grpcS *grpc.Server
if cfg.v3demo {
// set up v3 demo rpc
tls := &cfg.clientTLSInfo
if cfg.clientTLSInfo.Empty() {
tls = nil
}
grpcS, err = v3rpc.Server(s, tls)
if err != nil {
s.Stop()
<-s.StopNotify()
return nil, err
}
}
// Start the peer server in a goroutine
for _, l := range plns {
go func(l net.Listener) {
plog.Fatal(serve(l, nil, ph, 5*time.Minute))
}(l)
}
// Start a client server goroutine for each listen address
for _, l := range clns {
go func(l net.Listener) {
// read timeout does not work with http close notify
// TODO: https://github.com/golang/go/issues/9524
plog.Fatal(serve(l, grpcS, ch, 0))
}(l)
}
return s.StopNotify(), nil
}