// GetEtcdConfig will return reference to embed.Config object. This
// is to be passed to embed.StartEtcd() function.
func GetEtcdConfig(readConf bool) (*embed.Config, error) {
// NOTE: This sets most of the fields internally with default values.
// For example, most of *URL fields are filled with all available IPs
// of local node i.e binds on all addresses.
cfg := embed.NewConfig()
// etcd member names doesn't have to be unique as etcd internally uses
// UUIDs for member IDs. In practice, etcd instance names are usually
// set to hostname of node. But we also need to keep the mapping
// between peers and their etcd names simple. So etcd member names are
// set to (peer) UUID of glusterd instance.
cfg.Name = gdctx.MyUUID.String()
cfg.Dir = cfg.Name + ".etcd"
listenClientURL, err := url.Parse("http://" + config.GetString("etcdclientaddress"))
if err != nil {
return nil, err
}
cfg.ACUrls = []url.URL{*listenClientURL}
cfg.LCUrls = []url.URL{*listenClientURL}
listenPeerURL, err := url.Parse("http://" + config.GetString("etcdpeeraddress"))
if err != nil {
return nil, err
}
cfg.APUrls = []url.URL{*listenPeerURL}
cfg.LPUrls = []url.URL{*listenPeerURL}
cfg.InitialCluster = cfg.Name + "=" + listenPeerURL.String()
cfg.ClusterState = embed.ClusterStateFlagNew
if readConf {
oldCfg, err := readEtcdConfig()
if err == nil {
log.Info("Found saved etcd config file. Using that.")
cfg.InitialCluster = oldCfg.InitialCluster
cfg.ClusterState = oldCfg.ClusterState
cfg.Name = oldCfg.Name
cfg.Dir = oldCfg.Dir
}
}
return cfg, nil
}
func TestEmbedEtcd(t *testing.T) {
tests := []struct {
cfg embed.Config
werr string
wpeers int
wclients int
}{
{werr: "multiple discovery"},
{werr: "advertise-client-urls is required"},
{werr: "should be at least"},
{werr: "is too long"},
{wpeers: 1, wclients: 1},
{wpeers: 2, wclients: 1},
{wpeers: 1, wclients: 2},
}
urls := newEmbedURLs(10)
// setup defaults
for i := range tests {
tests[i].cfg = *embed.NewConfig()
}
tests[0].cfg.Durl = "abc"
setupEmbedCfg(&tests[1].cfg, []url.URL{urls[0]}, []url.URL{urls[1]})
tests[1].cfg.ACUrls = nil
tests[2].cfg.TickMs = tests[2].cfg.ElectionMs - 1
tests[3].cfg.ElectionMs = 999999
setupEmbedCfg(&tests[4].cfg, []url.URL{urls[2]}, []url.URL{urls[3]})
setupEmbedCfg(&tests[5].cfg, []url.URL{urls[4]}, []url.URL{urls[5], urls[6]})
setupEmbedCfg(&tests[6].cfg, []url.URL{urls[7], urls[8]}, []url.URL{urls[9]})
dir := path.Join(os.TempDir(), fmt.Sprintf("embed-etcd"))
os.RemoveAll(dir)
defer os.RemoveAll(dir)
for i, tt := range tests {
tests[i].cfg.Dir = dir
e, err := embed.StartEtcd(&tests[i].cfg)
if tt.werr != "" {
if err == nil || !strings.Contains(err.Error(), tt.werr) {
t.Errorf("%d: expected error with %q, got %v", i, tt.werr, err)
}
if e != nil {
e.Close()
}
continue
}
if err != nil {
t.Errorf("%d: expected success, got error %v", i, err)
continue
}
if len(e.Peers) != tt.wpeers {
t.Errorf("%d: expected %d peers, got %d", i, tt.wpeers, len(e.Peers))
}
if len(e.Clients) != tt.wclients {
t.Errorf("%d: expected %d peers, got %d", i, tt.wclients, len(e.Clients))
}
e.Close()
}
}
func newConfig() *config {
cfg := &config{
Config: *embed.NewConfig(),
configProxy: configProxy{
Proxy: proxyFlagOff,
ProxyFailureWaitMs: 5000,
ProxyRefreshIntervalMs: 30000,
ProxyDialTimeoutMs: 1000,
ProxyWriteTimeoutMs: 5000,
},
ignored: ignored,
}
cfg.configFlags = configFlags{
FlagSet: flag.NewFlagSet("etcd", flag.ContinueOnError),
clusterState: flags.NewStringsFlag(
embed.ClusterStateFlagNew,
embed.ClusterStateFlagExisting,
),
fallback: flags.NewStringsFlag(
fallbackFlagExit,
fallbackFlagProxy,
),
proxy: flags.NewStringsFlag(
proxyFlagOff,
proxyFlagReadonly,
proxyFlagOn,
),
}
fs := cfg.FlagSet
fs.Usage = func() {
fmt.Fprintln(os.Stderr, usageline)
}
fs.StringVar(&cfg.configFile, "config-file", "", "Path to the server configuration file")
// member
fs.Var(cfg.CorsInfo, "cors", "Comma-separated white list of origins for CORS (cross-origin resource sharing).")
fs.StringVar(&cfg.Dir, "data-dir", cfg.Dir, "Path to the data directory.")
fs.StringVar(&cfg.WalDir, "wal-dir", cfg.WalDir, "Path to the dedicated wal directory.")
fs.Var(flags.NewURLsValue(embed.DefaultListenPeerURLs), "listen-peer-urls", "List of URLs to listen on for peer traffic.")
fs.Var(flags.NewURLsValue(embed.DefaultListenClientURLs), "listen-client-urls", "List of URLs to listen on for client traffic.")
fs.UintVar(&cfg.MaxSnapFiles, "max-snapshots", cfg.MaxSnapFiles, "Maximum number of snapshot files to retain (0 is unlimited).")
fs.UintVar(&cfg.MaxWalFiles, "max-wals", cfg.MaxWalFiles, "Maximum number of wal files to retain (0 is unlimited).")
fs.StringVar(&cfg.Name, "name", cfg.Name, "Human-readable name for this member.")
fs.Uint64Var(&cfg.SnapCount, "snapshot-count", cfg.SnapCount, "Number of committed transactions to trigger a snapshot to disk.")
fs.UintVar(&cfg.TickMs, "heartbeat-interval", cfg.TickMs, "Time (in milliseconds) of a heartbeat interval.")
fs.UintVar(&cfg.ElectionMs, "election-timeout", cfg.ElectionMs, "Time (in milliseconds) for an election to timeout.")
fs.Int64Var(&cfg.QuotaBackendBytes, "quota-backend-bytes", cfg.QuotaBackendBytes, "Raise alarms when backend size exceeds the given quota. 0 means use the default quota.")
// clustering
fs.Var(flags.NewURLsValue(embed.DefaultInitialAdvertisePeerURLs), "initial-advertise-peer-urls", "List of this member's peer URLs to advertise to the rest of the cluster.")
fs.Var(flags.NewURLsValue(embed.DefaultAdvertiseClientURLs), "advertise-client-urls", "List of this member's client URLs to advertise to the public.")
fs.StringVar(&cfg.Durl, "discovery", cfg.Durl, "Discovery URL used to bootstrap the cluster.")
fs.Var(cfg.fallback, "discovery-fallback", fmt.Sprintf("Valid values include %s", strings.Join(cfg.fallback.Values, ", ")))
if err := cfg.fallback.Set(fallbackFlagProxy); err != nil {
// Should never happen.
plog.Panicf("unexpected error setting up discovery-fallback flag: %v", err)
}
fs.StringVar(&cfg.Dproxy, "discovery-proxy", cfg.Dproxy, "HTTP proxy to use for traffic to discovery service.")
fs.StringVar(&cfg.DNSCluster, "discovery-srv", cfg.DNSCluster, "DNS domain used to bootstrap initial cluster.")
fs.StringVar(&cfg.InitialCluster, "initial-cluster", cfg.InitialCluster, "Initial cluster configuration for bootstrapping.")
fs.StringVar(&cfg.InitialClusterToken, "initial-cluster-token", cfg.InitialClusterToken, "Initial cluster token for the etcd cluster during bootstrap.")
fs.Var(cfg.clusterState, "initial-cluster-state", "Initial cluster state ('new' or 'existing').")
if err := cfg.clusterState.Set(embed.ClusterStateFlagNew); err != nil {
// Should never happen.
plog.Panicf("unexpected error setting up clusterStateFlag: %v", err)
}
fs.BoolVar(&cfg.StrictReconfigCheck, "strict-reconfig-check", cfg.StrictReconfigCheck, "Reject reconfiguration requests that would cause quorum loss.")
// proxy
fs.Var(cfg.proxy, "proxy", fmt.Sprintf("Valid values include %s", strings.Join(cfg.proxy.Values, ", ")))
if err := cfg.proxy.Set(proxyFlagOff); err != nil {
// Should never happen.
plog.Panicf("unexpected error setting up proxyFlag: %v", err)
}
fs.UintVar(&cfg.ProxyFailureWaitMs, "proxy-failure-wait", cfg.ProxyFailureWaitMs, "Time (in milliseconds) an endpoint will be held in a failed state.")
fs.UintVar(&cfg.ProxyRefreshIntervalMs, "proxy-refresh-interval", cfg.ProxyRefreshIntervalMs, "Time (in milliseconds) of the endpoints refresh interval.")
fs.UintVar(&cfg.ProxyDialTimeoutMs, "proxy-dial-timeout", cfg.ProxyDialTimeoutMs, "Time (in milliseconds) for a dial to timeout.")
fs.UintVar(&cfg.ProxyWriteTimeoutMs, "proxy-write-timeout", cfg.ProxyWriteTimeoutMs, "Time (in milliseconds) for a write to timeout.")
fs.UintVar(&cfg.ProxyReadTimeoutMs, "proxy-read-timeout", cfg.ProxyReadTimeoutMs, "Time (in milliseconds) for a read to timeout.")
// security
fs.StringVar(&cfg.ClientTLSInfo.CAFile, "ca-file", "", "DEPRECATED: Path to the client server TLS CA file.")
fs.StringVar(&cfg.ClientTLSInfo.CertFile, "cert-file", "", "Path to the client server TLS cert file.")
fs.StringVar(&cfg.ClientTLSInfo.KeyFile, "key-file", "", "Path to the client server TLS key file.")
fs.BoolVar(&cfg.ClientTLSInfo.ClientCertAuth, "client-cert-auth", false, "Enable client cert authentication.")
fs.StringVar(&cfg.ClientTLSInfo.TrustedCAFile, "trusted-ca-file", "", "Path to the client server TLS trusted CA key file.")
fs.BoolVar(&cfg.ClientAutoTLS, "auto-tls", false, "Client TLS using generated certificates")
fs.StringVar(&cfg.PeerTLSInfo.CAFile, "peer-ca-file", "", "DEPRECATED: Path to the peer server TLS CA file.")
fs.StringVar(&cfg.PeerTLSInfo.CertFile, "peer-cert-file", "", "Path to the peer server TLS cert file.")
fs.StringVar(&cfg.PeerTLSInfo.KeyFile, "peer-key-file", "", "Path to the peer server TLS key file.")
fs.BoolVar(&cfg.PeerTLSInfo.ClientCertAuth, "peer-client-cert-auth", false, "Enable peer client cert authentication.")
fs.StringVar(&cfg.PeerTLSInfo.TrustedCAFile, "peer-trusted-ca-file", "", "Path to the peer server TLS trusted CA file.")
fs.BoolVar(&cfg.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates")
// logging
fs.BoolVar(&cfg.Debug, "debug", false, "Enable debug-level logging for etcd.")
fs.StringVar(&cfg.LogPkgLevels, "log-package-levels", "", "Specify a particular log level for each etcd package (eg: 'etcdmain=CRITICAL,etcdserver=DEBUG').")
fs.StringVar(&cfg.logOutput, "log-output", "default", "Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd.")
// unsafe
fs.BoolVar(&cfg.ForceNewCluster, "force-new-cluster", false, "Force to create a new one member cluster.")
// version
fs.BoolVar(&cfg.printVersion, "version", false, "Print the version and exit.")
fs.IntVar(&cfg.AutoCompactionRetention, "auto-compaction-retention", 0, "Auto compaction retention for mvcc key value store in hour. 0 means disable auto compaction.")
// pprof profiler via HTTP
fs.BoolVar(&cfg.EnablePprof, "enable-pprof", false, "Enable runtime profiling data via HTTP server. Address is at client URL + \"/debug/pprof/\"")
// ignored
for _, f := range cfg.ignored {
fs.Var(&flags.IgnoredFlag{Name: f}, f, "")
}
return cfg
}
// RunEtcd starts an etcd server and runs it forever
func RunEtcd(etcdServerConfig *configapi.EtcdConfig) {
cfg := embed.NewConfig()
cfg.Debug = true
cfg.Name = defaultName
cfg.Dir = etcdServerConfig.StorageDir
clientTLS := configapi.UseTLS(etcdServerConfig.ServingInfo)
if clientTLS {
cfg.ClientTLSInfo.CAFile = etcdServerConfig.ServingInfo.ClientCA
cfg.ClientTLSInfo.CertFile = etcdServerConfig.ServingInfo.ServerCert.CertFile
cfg.ClientTLSInfo.KeyFile = etcdServerConfig.ServingInfo.ServerCert.KeyFile
cfg.ClientTLSInfo.ClientCertAuth = len(cfg.ClientTLSInfo.CAFile) > 0
}
u, err := types.NewURLs(addressToURLs(etcdServerConfig.ServingInfo.BindAddress, clientTLS))
if err != nil {
glog.Fatalf("Unable to build etcd peer URLs: %v", err)
}
cfg.LCUrls = []url.URL(u)
peerTLS := configapi.UseTLS(etcdServerConfig.PeerServingInfo)
if peerTLS {
cfg.PeerTLSInfo.CAFile = etcdServerConfig.PeerServingInfo.ClientCA
cfg.PeerTLSInfo.CertFile = etcdServerConfig.PeerServingInfo.ServerCert.CertFile
cfg.PeerTLSInfo.KeyFile = etcdServerConfig.PeerServingInfo.ServerCert.KeyFile
cfg.PeerTLSInfo.ClientCertAuth = len(cfg.PeerTLSInfo.CAFile) > 0
}
u, err = types.NewURLs(addressToURLs(etcdServerConfig.PeerServingInfo.BindAddress, peerTLS))
if err != nil {
glog.Fatalf("Unable to build etcd peer URLs: %v", err)
}
cfg.LPUrls = []url.URL(u)
u, err = types.NewURLs(addressToURLs(etcdServerConfig.Address, clientTLS))
if err != nil {
glog.Fatalf("Unable to build etcd announce client URLs: %v", err)
}
cfg.ACUrls = []url.URL(u)
u, err = types.NewURLs(addressToURLs(etcdServerConfig.PeerAddress, peerTLS))
if err != nil {
glog.Fatalf("Unable to build etcd announce peer URLs: %v", err)
}
cfg.APUrls = []url.URL(u)
cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
osutil.HandleInterrupts()
e, err := embed.StartEtcd(cfg)
if err != nil {
glog.Fatalf("Unable to start etcd: %v", err)
}
go func() {
defer e.Close()
select {
case <-e.Server.ReadyNotify():
glog.Infof("Started etcd at %s", etcdServerConfig.Address)
case <-time.After(60 * time.Second):
glog.Warning("etcd took too long to start, stopped")
e.Server.Stop() // trigger a shutdown
}
glog.Fatalf("etcd has returned an error: %v", <-e.Err())
}()
}
