// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd(cfg *config) (<-chan struct{}, error) {
initialPeers, token, err := setupCluster(cfg)
if err != nil {
return nil, fmt.Errorf("error setting up initial cluster: %v", err)
}
pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, time.Second, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
if !cfg.peerTLSInfo.Empty() {
glog.V(2).Infof("etcd: peerTLS: %s", cfg.peerTLSInfo)
}
plns := make([]net.Listener, 0)
for _, u := range cfg.lpurls {
var l net.Listener
l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
if err != nil {
return nil, err
}
urlStr := u.String()
glog.V(2).Info("etcd: listening for peers on ", urlStr)
defer func() {
if err != nil {
l.Close()
glog.V(2).Info("etcd: stopping listening for peers on ", urlStr)
}
}()
plns = append(plns, l)
}
if !cfg.clientTLSInfo.Empty() {
glog.V(2).Infof("etcd: clientTLS: %s", cfg.clientTLSInfo)
}
clns := make([]net.Listener, 0)
for _, u := range cfg.lcurls {
l, err := net.Listen("tcp", u.Host)
if err != nil {
return nil, err
}
l, err = transport.NewKeepAliveListener(l, u.Scheme, cfg.clientTLSInfo)
if err != nil {
return nil, err
}
urlStr := u.String()
glog.V(2).Info("etcd: listening for client requests on ", urlStr)
defer func() {
if err != nil {
l.Close()
glog.V(2).Info("etcd: stopping listening for client requests on ", urlStr)
}
}()
clns = append(clns, l)
}
srvcfg := &etcdserver.ServerConfig{
Name: cfg.name,
ClientURLs: cfg.acurls,
PeerURLs: cfg.apurls,
DataDir: cfg.dir,
SnapCount: cfg.snapCount,
MaxSnapFiles: cfg.maxSnapFiles,
InitialPeerURLsMap: initialPeers,
InitialClusterToken: token,
MaxWALFiles: cfg.maxWalFiles,
NewCluster: true,
ForceNewCluster: false,
Transport: pt,
TickMs: cfg.TickMs,
ElectionTicks: cfg.electionTicks(),
}
var s *etcdserver.EtcdServer
s, err = etcdserver.NewServer(srvcfg)
if err != nil {
return nil, err
}
osutil.HandleInterrupts()
s.Start()
osutil.RegisterInterruptHandler(s.Stop)
ch := etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout())
ph := etcdhttp.NewPeerHandler(s.Cluster(), s.RaftHandler())
// Start the peer server in a goroutine
for _, l := range plns {
go func(l net.Listener) {
glog.Fatal(serveHTTP(l, ph, 5*time.Minute))
}(l)
}
// Start a client server goroutine for each listen address
for _, l := range clns {
go func(l net.Listener) {
// read timeout does not work with http close notify
// TODO: https://github.com/golang/go/issues/9524
glog.Fatal(serveHTTP(l, ch, 0))
}(l)
}
return s.StopNotify(), nil
}
func Main() {
capnslog.SetFormatter(capnslog.NewStringFormatter(os.Stderr))
cfg := NewConfig()
err := cfg.Parse(os.Args[1:])
if err != nil {
plog.Errorf("error verifying flags, %v. See 'etcd --help'.", err)
switch err {
case errUnsetAdvertiseClientURLsFlag:
plog.Errorf("When listening on specific address(es), this etcd process must advertise accessible url(s) to each connected client.")
}
os.Exit(1)
}
setupLogging(cfg)
var stopped <-chan struct{}
GoMaxProcs := 1
if envMaxProcs, err := strconv.Atoi(os.Getenv("GOMAXPROCS")); err == nil {
GoMaxProcs = envMaxProcs
}
plog.Infof("setting maximum number of CPUs to %d, total number of available CPUs is %d", GoMaxProcs, runtime.NumCPU())
runtime.GOMAXPROCS(GoMaxProcs)
// TODO: check whether fields are set instead of whether fields have default value
if cfg.name != defaultName && cfg.initialCluster == initialClusterFromName(defaultName) {
cfg.initialCluster = initialClusterFromName(cfg.name)
}
if cfg.dir == "" {
cfg.dir = fmt.Sprintf("%v.etcd", cfg.name)
plog.Warningf("no data-dir provided, using default data-dir ./%s", cfg.dir)
}
which := identifyDataDirOrDie(cfg.dir)
if which != dirEmpty {
plog.Noticef("the server is already initialized as %v before, starting as etcd %v...", which, which)
}
shouldProxy := cfg.isProxy() || which == dirProxy
if !shouldProxy {
stopped, err = startEtcd(cfg)
if err == discovery.ErrFullCluster && cfg.shouldFallbackToProxy() {
plog.Noticef("discovery cluster full, falling back to %s", fallbackFlagProxy)
shouldProxy = true
}
}
if shouldProxy {
err = startProxy(cfg)
}
if err != nil {
switch err {
case discovery.ErrDuplicateID:
plog.Errorf("member %q has previously registered with discovery service token (%s).", cfg.name, cfg.durl)
plog.Errorf("But etcd could not find vaild cluster configuration in the given data dir (%s).", cfg.dir)
plog.Infof("Please check the given data dir path if the previous bootstrap succeeded")
plog.Infof("or use a new discovery token if the previous bootstrap failed.")
os.Exit(1)
default:
plog.Fatalf("%v", err)
}
}
osutil.HandleInterrupts()
<-stopped
osutil.Exit(0)
}
func Main() {
cfg := NewConfig()
err := cfg.Parse(os.Args[1:])
if err != nil {
plog.Errorf("error verifying flags, %v. See 'etcd --help'.", err)
switch err {
case errUnsetAdvertiseClientURLsFlag:
plog.Errorf("When listening on specific address(es), this etcd process must advertise accessible url(s) to each connected client.")
}
os.Exit(1)
}
setupLogging(cfg)
var stopped <-chan struct{}
plog.Infof("etcd Version: %s\n", version.Version)
plog.Infof("Git SHA: %s\n", version.GitSHA)
plog.Infof("Go Version: %s\n", runtime.Version())
plog.Infof("Go OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
GoMaxProcs := runtime.GOMAXPROCS(0)
plog.Infof("setting maximum number of CPUs to %d, total number of available CPUs is %d", GoMaxProcs, runtime.NumCPU())
// TODO: check whether fields are set instead of whether fields have default value
if cfg.name != defaultName && cfg.initialCluster == initialClusterFromName(defaultName) {
cfg.initialCluster = initialClusterFromName(cfg.name)
}
if cfg.dir == "" {
cfg.dir = fmt.Sprintf("%v.etcd", cfg.name)
plog.Warningf("no data-dir provided, using default data-dir ./%s", cfg.dir)
}
which := identifyDataDirOrDie(cfg.dir)
if which != dirEmpty {
plog.Noticef("the server is already initialized as %v before, starting as etcd %v...", which, which)
switch which {
case dirMember:
stopped, err = startEtcd(cfg)
case dirProxy:
err = startProxy(cfg)
default:
plog.Panicf("unhandled dir type %v", which)
}
} else {
shouldProxy := cfg.isProxy()
if !shouldProxy {
stopped, err = startEtcd(cfg)
if err == discovery.ErrFullCluster && cfg.shouldFallbackToProxy() {
plog.Noticef("discovery cluster full, falling back to %s", fallbackFlagProxy)
shouldProxy = true
}
}
if shouldProxy {
err = startProxy(cfg)
}
}
if err != nil {
switch err {
case discovery.ErrDuplicateID:
plog.Errorf("member %q has previously registered with discovery service token (%s).", cfg.name, cfg.durl)
plog.Errorf("But etcd could not find valid cluster configuration in the given data dir (%s).", cfg.dir)
plog.Infof("Please check the given data dir path if the previous bootstrap succeeded")
plog.Infof("or use a new discovery token if the previous bootstrap failed.")
case discovery.ErrDuplicateName:
plog.Errorf("member with duplicated name has registered with discovery service token(%s).", cfg.durl)
plog.Errorf("please check (cURL) the discovery token for more information.")
plog.Errorf("please do not reuse the discovery token and generate a new one to bootstrap the cluster.")
default:
if strings.Contains(err.Error(), "include") && strings.Contains(err.Error(), "--initial-cluster") {
plog.Infof("%v", err)
if cfg.initialCluster == initialClusterFromName(cfg.name) {
plog.Infof("forgot to set --initial-cluster flag?")
}
if types.URLs(cfg.apurls).String() == defaultInitialAdvertisePeerURLs {
plog.Infof("forgot to set --initial-advertise-peer-urls flag?")
}
if cfg.initialCluster == initialClusterFromName(cfg.name) && len(cfg.durl) == 0 {
plog.Infof("if you want to use discovery service, please set --discovery flag.")
}
os.Exit(1)
}
if etcdserver.IsDiscoveryError(err) {
plog.Errorf("%v", err)
plog.Infof("discovery token %s was used, but failed to bootstrap the cluster.", cfg.durl)
plog.Infof("please generate a new discovery token and try to bootstrap again.")
os.Exit(1)
}
plog.Fatalf("%v", err)
}
os.Exit(1)
}
osutil.HandleInterrupts()
if systemdutil.IsRunningSystemd() {
// At this point, the initialization of etcd is done.
// The listeners are listening on the TCP ports and ready
// for accepting connections.
// The http server is probably ready for serving incoming
// connections. If it is not, the connection might be pending
// for less than one second.
err := daemon.SdNotify("READY=1")
if err != nil {
plog.Errorf("failed to notify systemd for readiness")
}
}
<-stopped
osutil.Exit(0)
}
// RunEtcd starts an etcd server and runs it forever
func RunEtcd(etcdServerConfig *configapi.EtcdConfig) {
cfg := embed.NewConfig()
cfg.Debug = true
cfg.Name = defaultName
cfg.Dir = etcdServerConfig.StorageDir
clientTLS := configapi.UseTLS(etcdServerConfig.ServingInfo)
if clientTLS {
cfg.ClientTLSInfo.CAFile = etcdServerConfig.ServingInfo.ClientCA
cfg.ClientTLSInfo.CertFile = etcdServerConfig.ServingInfo.ServerCert.CertFile
cfg.ClientTLSInfo.KeyFile = etcdServerConfig.ServingInfo.ServerCert.KeyFile
cfg.ClientTLSInfo.ClientCertAuth = len(cfg.ClientTLSInfo.CAFile) > 0
}
u, err := types.NewURLs(addressToURLs(etcdServerConfig.ServingInfo.BindAddress, clientTLS))
if err != nil {
glog.Fatalf("Unable to build etcd peer URLs: %v", err)
}
cfg.LCUrls = []url.URL(u)
peerTLS := configapi.UseTLS(etcdServerConfig.PeerServingInfo)
if peerTLS {
cfg.PeerTLSInfo.CAFile = etcdServerConfig.PeerServingInfo.ClientCA
cfg.PeerTLSInfo.CertFile = etcdServerConfig.PeerServingInfo.ServerCert.CertFile
cfg.PeerTLSInfo.KeyFile = etcdServerConfig.PeerServingInfo.ServerCert.KeyFile
cfg.PeerTLSInfo.ClientCertAuth = len(cfg.PeerTLSInfo.CAFile) > 0
}
u, err = types.NewURLs(addressToURLs(etcdServerConfig.PeerServingInfo.BindAddress, peerTLS))
if err != nil {
glog.Fatalf("Unable to build etcd peer URLs: %v", err)
}
cfg.LPUrls = []url.URL(u)
u, err = types.NewURLs(addressToURLs(etcdServerConfig.Address, clientTLS))
if err != nil {
glog.Fatalf("Unable to build etcd announce client URLs: %v", err)
}
cfg.ACUrls = []url.URL(u)
u, err = types.NewURLs(addressToURLs(etcdServerConfig.PeerAddress, peerTLS))
if err != nil {
glog.Fatalf("Unable to build etcd announce peer URLs: %v", err)
}
cfg.APUrls = []url.URL(u)
cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
osutil.HandleInterrupts()
e, err := embed.StartEtcd(cfg)
if err != nil {
glog.Fatalf("Unable to start etcd: %v", err)
}
go func() {
defer e.Close()
select {
case <-e.Server.ReadyNotify():
glog.Infof("Started etcd at %s", etcdServerConfig.Address)
case <-time.After(60 * time.Second):
glog.Warning("etcd took too long to start, stopped")
e.Server.Stop() // trigger a shutdown
}
glog.Fatalf("etcd has returned an error: %v", <-e.Err())
}()
}
