// setupCluster sets up the cluster definition for bootstrap or discovery.
func setupCluster() (*etcdserver.Cluster, error) {
set := make(map[string]bool)
fs.Visit(func(f *flag.Flag) {
set[f.Name] = true
})
if set["discovery"] && set["initial-cluster"] {
return nil, fmt.Errorf("both discovery and bootstrap-config are set")
}
apurls, err := flags.URLsFromFlags(fs, "initial-advertise-peer-urls", "addr", peerTLSInfo)
if err != nil {
return nil, err
}
var cls *etcdserver.Cluster
switch {
case set["discovery"]:
clusterStr := genClusterString(*name, apurls)
cls, err = etcdserver.NewClusterFromString(*durl, clusterStr)
case set["initial-cluster"]:
fallthrough
default:
// We're statically configured, and cluster has appropriately been set.
// Try to configure by indexing the static cluster by name.
cls, err = etcdserver.NewClusterFromString(*initialClusterName, *initialCluster)
}
return cls, err
}
func mustNewMember(t *testing.T, name string) *member {
var err error
m := &member{}
pln := newLocalListener(t)
m.PeerListeners = []net.Listener{pln}
m.PeerURLs, err = types.NewURLs([]string{"http://" + pln.Addr().String()})
if err != nil {
t.Fatal(err)
}
cln := newLocalListener(t)
m.ClientListeners = []net.Listener{cln}
m.ClientURLs, err = types.NewURLs([]string{"http://" + cln.Addr().String()})
if err != nil {
t.Fatal(err)
}
m.Name = name
m.DataDir, err = ioutil.TempDir(os.TempDir(), "etcd")
if err != nil {
t.Fatal(err)
}
clusterStr := fmt.Sprintf("%s=http://%s", name, pln.Addr().String())
m.Cluster, err = etcdserver.NewClusterFromString(clusterName, clusterStr)
if err != nil {
t.Fatal(err)
}
m.NewCluster = true
m.Transport = mustNewTransport(t)
m.ElectionTicks = electionTicks
m.TickMs = uint(tickDuration / time.Millisecond)
return m
}
// Clone returns a member with the same server configuration. The returned
// member will not set PeerListeners and ClientListeners.
func (m *member) Clone(t *testing.T) *member {
mm := &member{}
mm.ServerConfig = m.ServerConfig
var err error
clientURLStrs := m.ClientURLs.StringSlice()
mm.ClientURLs, err = types.NewURLs(clientURLStrs)
if err != nil {
// this should never fail
panic(err)
}
peerURLStrs := m.PeerURLs.StringSlice()
mm.PeerURLs, err = types.NewURLs(peerURLStrs)
if err != nil {
// this should never fail
panic(err)
}
clusterStr := m.Cluster.String()
mm.Cluster, err = etcdserver.NewClusterFromString(clusterName, clusterStr)
if err != nil {
// this should never fail
panic(err)
}
mm.Transport = mustNewTransport(t)
mm.ElectionTicks = m.ElectionTicks
return mm
}
func (c *cluster) AddMember(t *testing.T) {
clusterStr := c.Members[0].Cluster.String()
idx := len(c.Members)
m := mustNewMember(t, c.name(idx))
// send add request to the cluster
cc := mustNewHTTPClient(t, []string{c.URL(0)})
ma := client.NewMembersAPI(cc)
ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
peerURL := "http://" + m.PeerListeners[0].Addr().String()
if _, err := ma.Add(ctx, peerURL); err != nil {
t.Fatalf("add member on %s error: %v", c.URL(0), err)
}
cancel()
// wait for the add node entry applied in the cluster
members := append(c.HTTPMembers(), httptypes.Member{PeerURLs: []string{peerURL}, ClientURLs: []string{}})
c.waitMembersMatch(t, members)
for _, ln := range m.PeerListeners {
clusterStr += fmt.Sprintf(",%s=http://%s", m.Name, ln.Addr().String())
}
var err error
m.Cluster, err = etcdserver.NewClusterFromString(clusterName, clusterStr)
if err != nil {
t.Fatal(err)
}
m.NewCluster = false
if err := m.Launch(); err != nil {
t.Fatal(err)
}
c.Members = append(c.Members, m)
// wait cluster to be stable to receive future client requests
c.waitMembersMatch(t, c.HTTPMembers())
}
// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes.
func startProxy(cfg *config) error {
cls, err := setupCluster(cfg)
if err != nil {
return fmt.Errorf("error setting up initial cluster: %v", err)
}
if cfg.durl != "" {
s, err := discovery.GetCluster(cfg.durl, cfg.dproxy)
if err != nil {
return err
}
if cls, err = etcdserver.NewClusterFromString(cfg.durl, s); err != nil {
return err
}
}
pt, err := transport.NewTransport(cfg.clientTLSInfo)
if err != nil {
return err
}
// TODO(jonboulle): update peerURLs dynamically (i.e. when updating
// clientURLs) instead of just using the initial fixed list here
peerURLs := cls.PeerURLs()
uf := func() []string {
cls, err := etcdserver.GetClusterFromPeers(peerURLs)
if err != nil {
log.Printf("proxy: %v", err)
return []string{}
}
return cls.ClientURLs()
}
ph := proxy.NewHandler(pt, uf)
ph = &cors.CORSHandler{
Handler: ph,
Info: cfg.corsInfo,
}
if cfg.isReadonlyProxy() {
ph = proxy.NewReadonlyHandler(ph)
}
// Start a proxy server goroutine for each listen address
for _, u := range cfg.lcurls {
l, err := transport.NewListener(u.Host, u.Scheme, cfg.clientTLSInfo)
if err != nil {
return err
}
host := u.Host
go func() {
log.Print("proxy: listening for client requests on ", host)
log.Fatal(http.Serve(l, ph))
}()
}
return nil
}
// setupCluster sets up an initial cluster definition for bootstrap or discovery.
func setupCluster(cfg *config) (*etcdserver.Cluster, error) {
var cls *etcdserver.Cluster
var err error
switch {
default:
// We're statically configured, and cluster has appropriately been set.
cls, err = etcdserver.NewClusterFromString(cfg.initialClusterToken, cfg.initialCluster)
}
return cls, err
}
// setupCluster sets up an initial cluster definition for bootstrap or discovery.
func setupCluster(cfg *config) (*etcdserver.Cluster, error) {
var cls *etcdserver.Cluster
var err error
switch {
case cfg.durl != "":
// If using discovery, generate a temporary cluster based on
// self's advertised peer URLs
clusterStr := genClusterString(cfg.name, cfg.apurls)
cls, err = etcdserver.NewClusterFromString(cfg.durl, clusterStr)
case cfg.dnsCluster != "":
clusterStr, clusterToken, err := discovery.SRVGetCluster(cfg.name, cfg.dnsCluster, cfg.initialClusterToken, cfg.apurls)
if err != nil {
return nil, err
}
cls, err = etcdserver.NewClusterFromString(clusterToken, clusterStr)
default:
// We're statically configured, and cluster has appropriately been set.
cls, err = etcdserver.NewClusterFromString(cfg.initialClusterToken, cfg.initialCluster)
}
return cls, err
}
// setupCluster sets up an initial cluster definition for bootstrap or discovery.
func setupCluster(apurls []url.URL) (*etcdserver.Cluster, error) {
set := make(map[string]bool)
fs.Visit(func(f *flag.Flag) {
set[f.Name] = true
})
if set["discovery"] && set["initial-cluster"] {
return nil, fmt.Errorf("both discovery and bootstrap-config are set")
}
var cls *etcdserver.Cluster
var err error
switch {
case set["discovery"]:
// If using discovery, generate a temporary cluster based on
// self's advertised peer URLs
clusterStr := genClusterString(*name, apurls)
cls, err = etcdserver.NewClusterFromString(*durl, clusterStr)
case set["initial-cluster"]:
fallthrough
default:
// We're statically configured, and cluster has appropriately been set.
cls, err = etcdserver.NewClusterFromString(*initialClusterToken, *initialCluster)
}
return cls, err
}
// TODO: support TLS
func (c *cluster) Launch(t *testing.T) {
if c.Size <= 0 {
t.Fatalf("cluster size <= 0")
}
lns := make([]net.Listener, c.Size)
addrs := make([]string, c.Size)
for i := 0; i < c.Size; i++ {
l := newLocalListener(t)
// each member claims only one peer listener
lns[i] = l
addrs[i] = fmt.Sprintf("%v=%v", c.name(i), "http://"+l.Addr().String())
}
clusterStr := strings.Join(addrs, ",")
var err error
for i := 0; i < c.Size; i++ {
m := member{}
m.PeerListeners = []net.Listener{lns[i]}
cln := newLocalListener(t)
m.ClientListeners = []net.Listener{cln}
m.Name = c.name(i)
m.ClientURLs, err = types.NewURLs([]string{"http://" + cln.Addr().String()})
if err != nil {
t.Fatal(err)
}
m.DataDir, err = ioutil.TempDir(os.TempDir(), "etcd")
if err != nil {
t.Fatal(err)
}
m.Cluster, err = etcdserver.NewClusterFromString(clusterName, clusterStr)
if err != nil {
t.Fatal(err)
}
m.ClusterState = etcdserver.ClusterStateValueNew
m.Transport, err = transport.NewTransport(transport.TLSInfo{})
if err != nil {
t.Fatal(err)
}
// TODO: need the support of graceful stop in Sender to remove this
m.Transport.DisableKeepAlives = true
m.Transport.Dial = (&net.Dialer{Timeout: 100 * time.Millisecond}).Dial
m.Launch(t)
c.Members = append(c.Members, m)
}
}
func fillClusterForMembers(ms []*member, cName string) error {
addrs := make([]string, 0)
for _, m := range ms {
for _, l := range m.PeerListeners {
addrs = append(addrs, fmt.Sprintf("%s=%s", m.Name, "http://"+l.Addr().String()))
}
}
clusterStr := strings.Join(addrs, ",")
var err error
for _, m := range ms {
m.Cluster, err = etcdserver.NewClusterFromString(cName, clusterStr)
if err != nil {
return err
}
}
return nil
}
// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes.
func startProxy() error {
apurls, err := flags.URLsFromFlags(fs, "initial-advertise-peer-urls", "addr", peerTLSInfo)
if err != nil {
return err
}
cls, err := setupCluster(apurls)
if err != nil {
return fmt.Errorf("error setting up initial cluster: %v", err)
}
if *durl != "" {
s, err := discovery.GetCluster(*durl, *dproxy)
if err != nil {
return err
}
if cls, err = etcdserver.NewClusterFromString(*durl, s); err != nil {
return err
}
}
pt, err := transport.NewTransport(clientTLSInfo)
if err != nil {
return err
}
// TODO(jonboulle): update peerURLs dynamically (i.e. when updating
// clientURLs) instead of just using the initial fixed list here
peerURLs := cls.PeerURLs()
uf := func() []string {
cls, err := etcdserver.GetClusterFromPeers(peerURLs)
if err != nil {
log.Printf("proxy: %v", err)
return []string{}
}
return cls.ClientURLs()
}
ph := proxy.NewHandler(pt, uf)
ph = &cors.CORSHandler{
Handler: ph,
Info: corsInfo,
}
if proxyFlag.String() == proxyFlagReadonly {
ph = proxy.NewReadonlyHandler(ph)
}
lcurls, err := flags.URLsFromFlags(fs, "listen-client-urls", "bind-addr", clientTLSInfo)
if err != nil {
return err
}
// Start a proxy server goroutine for each listen address
for _, u := range lcurls {
l, err := transport.NewListener(u.Host, u.Scheme, clientTLSInfo)
if err != nil {
return err
}
host := u.Host
go func() {
log.Print("proxy: listening for client requests on ", host)
log.Fatal(http.Serve(l, ph))
}()
}
return nil
}
// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes.
func startProxy(cfg *config) error {
cls, err := setupCluster(cfg)
if err != nil {
return fmt.Errorf("error setting up initial cluster: %v", err)
}
if cfg.durl != "" {
s, err := discovery.GetCluster(cfg.durl, cfg.dproxy)
if err != nil {
return err
}
if cls, err = etcdserver.NewClusterFromString(cfg.durl, s); err != nil {
return err
}
}
pt, err := transport.NewTransport(cfg.clientTLSInfo)
if err != nil {
return err
}
tr, err := transport.NewTransport(cfg.peerTLSInfo)
if err != nil {
return err
}
if cfg.dir == "" {
cfg.dir = fmt.Sprintf("%v.etcd", cfg.name)
log.Printf("no proxy data-dir provided, using default proxy data-dir ./%s", cfg.dir)
}
cfg.dir = path.Join(cfg.dir, "proxy")
err = os.MkdirAll(cfg.dir, 0700)
if err != nil {
return err
}
var peerURLs []string
clusterfile := path.Join(cfg.dir, "cluster")
b, err := ioutil.ReadFile(clusterfile)
switch {
case err == nil:
urls := struct{ PeerURLs []string }{}
err := json.Unmarshal(b, &urls)
if err != nil {
return err
}
peerURLs = urls.PeerURLs
log.Printf("proxy: using peer urls %v from cluster file ./%s", peerURLs, clusterfile)
case os.IsNotExist(err):
peerURLs = cls.PeerURLs()
log.Printf("proxy: using peer urls %v ", peerURLs)
default:
return err
}
uf := func() []string {
gcls, err := etcdserver.GetClusterFromPeers(peerURLs, tr)
// TODO: remove the 2nd check when we fix GetClusterFromPeers
// GetClusterFromPeers should not return nil error with an invaild empty cluster
if err != nil {
log.Printf("proxy: %v", err)
return []string{}
}
if len(gcls.Members()) == 0 {
return cls.ClientURLs()
}
cls = gcls
urls := struct{ PeerURLs []string }{cls.PeerURLs()}
b, err := json.Marshal(urls)
if err != nil {
log.Printf("proxy: error on marshal peer urls %s", err)
return cls.ClientURLs()
}
err = ioutil.WriteFile(clusterfile+".bak", b, 0600)
if err != nil {
log.Printf("proxy: error on writing urls %s", err)
return cls.ClientURLs()
}
err = os.Rename(clusterfile+".bak", clusterfile)
if err != nil {
log.Printf("proxy: error on updating clusterfile %s", err)
return cls.ClientURLs()
}
if !reflect.DeepEqual(cls.PeerURLs(), peerURLs) {
log.Printf("proxy: updated peer urls in cluster file from %v to %v", peerURLs, cls.PeerURLs())
}
peerURLs = cls.PeerURLs()
return cls.ClientURLs()
}
ph := proxy.NewHandler(pt, uf)
ph = &cors.CORSHandler{
Handler: ph,
Info: cfg.corsInfo,
}
if cfg.isReadonlyProxy() {
ph = proxy.NewReadonlyHandler(ph)
}
// Start a proxy server goroutine for each listen address
for _, u := range cfg.lcurls {
l, err := transport.NewListener(u.Host, u.Scheme, cfg.clientTLSInfo)
if err != nil {
return err
}
host := u.Host
go func() {
log.Print("proxy: listening for client requests on ", host)
log.Fatal(http.Serve(l, ph))
}()
}
return nil
}