func withEtcd(t *testing.T, f func(*config.Config, *etcd_client.Client)) {
c := config.New()
c.Name = fmt.Sprintf("TEST%d", testNum)
c.Addr = fmt.Sprintf("127.0.0.1:%d", 4101+testNum)
c.Peer.Addr = fmt.Sprintf("127.0.0.1:%d", 7101+testNum)
testNum++
tmpdir, err := ioutil.TempDir("", "datad-etcd-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
c.DataDir = tmpdir
c.Force = true
c.Peer.HeartbeatInterval = 25
c.Peer.ElectionTimeout = 100
c.SnapshotCount = 10000
i := etcd.New(c)
go i.Run()
<-i.ReadyNotify()
// Run f.
f(c, etcd_client.NewClient([]string{i.Server.URL()}))
i.Stop()
}
// New returns a new Etcd instance.
func New(c *config.Config) *Etcd {
if c == nil {
c = config.New()
}
return &Etcd{
Config: c,
readyC: make(chan bool),
}
}
// New returns a new Etcd instance.
func New(c *config.Config) *Etcd {
if c == nil {
c = config.New()
}
return &Etcd{
Config: c,
closeChan: make(chan bool),
readyNotify: make(chan bool),
stopNotify: make(chan bool),
}
}
func TestRunStop(t *testing.T) {
path, _ := ioutil.TempDir("", "etcd-")
defer os.RemoveAll(path)
config := config.New()
config.Name = "ETCDTEST"
config.DataDir = path
config.Addr = "localhost:0"
config.Peer.Addr = "localhost:0"
etcd := New(config)
go etcd.Run()
<-etcd.ReadyNotify()
etcd.Stop()
}
func (c *config) runEtcd() {
etcdAddr := c.bindAddr + ":4001"
etcdConfig := etcdconfig.New()
etcdConfig.Addr = etcdAddr
etcdConfig.BindAddr = etcdAddr
etcdConfig.DataDir = "openshift.local.etcd"
etcdConfig.Name = "openshift.local"
// initialize etcd
etcdServer := etcd.New(etcdConfig)
go util.Forever(func() {
glog.Infof("Started etcd at http://%s", etcdAddr)
etcdServer.Run()
}, 0)
}
func main() {
var config = config.New()
if err := config.Load(os.Args[1:]); err != nil {
fmt.Println(server.Usage() + "\n")
fmt.Println(err.Error() + "\n")
os.Exit(1)
} else if config.ShowVersion {
fmt.Println("etcd version", server.ReleaseVersion)
os.Exit(0)
} else if config.ShowHelp {
fmt.Println(server.Usage() + "\n")
os.Exit(0)
}
var etcd = etcd.New(config)
etcd.Run()
}
// Starts a new server.
func RunServer(f func(*server.Server)) {
c := config.New()
c.Name = testName
c.Addr = testClientURL
c.Peer.Addr = testRaftURL
c.DataDir = testDataDir
c.Force = true
c.Peer.HeartbeatInterval = testHeartbeatInterval
c.Peer.ElectionTimeout = testElectionTimeout
c.SnapshotCount = testSnapshotCount
i := etcd.New(c)
go i.Run()
<-i.ReadyNotify()
// Execute the function passed in.
f(i.Server)
i.Stop()
}
func main() {
// Load configuration.
var config = config.New()
if err := config.Load(os.Args[1:]); err != nil {
fmt.Println(server.Usage() + "\n")
fmt.Println(err.Error() + "\n")
os.Exit(1)
} else if config.ShowVersion {
fmt.Println("etcd version", server.ReleaseVersion)
os.Exit(0)
} else if config.ShowHelp {
fmt.Println(server.Usage() + "\n")
os.Exit(0)
}
// Enable options.
if config.VeryVeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Trace)
} else if config.VeryVerbose {
log.Verbose = true
raft.SetLogLevel(raft.Debug)
} else if config.Verbose {
log.Verbose = true
}
if config.CPUProfileFile != "" {
profile(config.CPUProfileFile)
}
if config.DataDir == "" {
log.Fatal("The data dir was not set and could not be guessed from machine name")
}
// Create data directory if it doesn't already exist.
if err := os.MkdirAll(config.DataDir, 0744); err != nil {
log.Fatalf("Unable to create path: %s", err)
}
// Warn people if they have an info file
info := filepath.Join(config.DataDir, "info")
if _, err := os.Stat(info); err == nil {
log.Warnf("All cached configuration is now ignored. The file %s can be removed.", info)
}
var mbName string
if config.Trace() {
mbName = config.MetricsBucketName()
runtime.SetBlockProfileRate(1)
}
mb := metrics.NewBucket(mbName)
if config.GraphiteHost != "" {
err := mb.Publish(config.GraphiteHost)
if err != nil {
panic(err)
}
}
// Retrieve CORS configuration
corsInfo, err := ehttp.NewCORSInfo(config.CorsOrigins)
if err != nil {
log.Fatal("CORS:", err)
}
// Create etcd key-value store and registry.
store := store.New()
registry := server.NewRegistry(store)
// Create stats objects
followersStats := server.NewRaftFollowersStats(config.Name)
serverStats := server.NewRaftServerStats(config.Name)
// Calculate all of our timeouts
heartbeatInterval := time.Duration(config.Peer.HeartbeatInterval) * time.Millisecond
electionTimeout := time.Duration(config.Peer.ElectionTimeout) * time.Millisecond
dialTimeout := (3 * heartbeatInterval) + electionTimeout
responseHeaderTimeout := (3 * heartbeatInterval) + electionTimeout
// Create peer server
psConfig := server.PeerServerConfig{
Name: config.Name,
Scheme: config.PeerTLSInfo().Scheme(),
URL: config.Peer.Addr,
SnapshotCount: config.SnapshotCount,
MaxClusterSize: config.MaxClusterSize,
RetryTimes: config.MaxRetryAttempts,
RetryInterval: config.RetryInterval,
}
ps := server.NewPeerServer(psConfig, registry, store, &mb, followersStats, serverStats)
var psListener net.Listener
if psConfig.Scheme == "https" {
peerServerTLSConfig, err := config.PeerTLSInfo().ServerConfig()
if err != nil {
log.Fatal("peer server TLS error: ", err)
}
psListener, err = server.NewTLSListener(config.Peer.BindAddr, peerServerTLSConfig)
if err != nil {
log.Fatal("Failed to create peer listener: ", err)
}
} else {
psListener, err = server.NewListener(config.Peer.BindAddr)
if err != nil {
log.Fatal("Failed to create peer listener: ", err)
}
}
// Create raft transporter and server
raftTransporter := server.NewTransporter(followersStats, serverStats, registry, heartbeatInterval, dialTimeout, responseHeaderTimeout)
if psConfig.Scheme == "https" {
raftClientTLSConfig, err := config.PeerTLSInfo().ClientConfig()
if err != nil {
log.Fatal("raft client TLS error: ", err)
}
raftTransporter.SetTLSConfig(*raftClientTLSConfig)
}
raftServer, err := raft.NewServer(config.Name, config.DataDir, raftTransporter, store, ps, "")
if err != nil {
log.Fatal(err)
}
raftServer.SetElectionTimeout(electionTimeout)
raftServer.SetHeartbeatInterval(heartbeatInterval)
ps.SetRaftServer(raftServer)
// Create etcd server
s := server.New(config.Name, config.Addr, ps, registry, store, &mb)
if config.Trace() {
s.EnableTracing()
}
var sListener net.Listener
if config.EtcdTLSInfo().Scheme() == "https" {
etcdServerTLSConfig, err := config.EtcdTLSInfo().ServerConfig()
if err != nil {
log.Fatal("etcd TLS error: ", err)
}
sListener, err = server.NewTLSListener(config.BindAddr, etcdServerTLSConfig)
if err != nil {
log.Fatal("Failed to create TLS etcd listener: ", err)
}
} else {
sListener, err = server.NewListener(config.BindAddr)
if err != nil {
log.Fatal("Failed to create etcd listener: ", err)
}
}
ps.SetServer(s)
ps.Start(config.Snapshot, config.Discovery, config.Peers)
go func() {
log.Infof("peer server [name %s, listen on %s, advertised url %s]", ps.Config.Name, psListener.Addr(), ps.Config.URL)
sHTTP := &ehttp.CORSHandler{ps.HTTPHandler(), corsInfo}
log.Fatal(http.Serve(psListener, sHTTP))
}()
log.Infof("etcd server [name %s, listen on %s, advertised url %s]", s.Name, sListener.Addr(), s.URL())
sHTTP := &ehttp.CORSHandler{s.HTTPHandler(), corsInfo}
log.Fatal(http.Serve(sListener, sHTTP))
}
func (c *config) startAllInOne() {
minionHost := "127.0.0.1"
minionPort := 10250
rootDirectory := path.Clean("/var/lib/openshift")
osAddr := c.ListenAddr
osPrefix := "/osapi/v1beta1"
kubePrefix := "/api/v1beta1"
kubeClient, err := kubeclient.New("http://"+osAddr, nil)
if err != nil {
glog.Fatalf("Unable to configure client - bad URL: %v", err)
}
osClient, err := osclient.New("http://"+osAddr, nil)
if err != nil {
glog.Fatalf("Unable to configure client - bad URL: %v", err)
}
etcdAddr := "127.0.0.1:4001"
etcdServers := []string{} // default
etcdConfig := etcdconfig.New()
etcdConfig.BindAddr = etcdAddr
etcdConfig.DataDir = "openshift.local.etcd"
etcdConfig.Name = "openshift.local"
// check docker connection
dockerClient, dockerAddr := c.Docker.GetClientOrExit()
if err := dockerClient.Ping(); err != nil {
glog.Errorf("WARNING: Docker could not be reached at %s. Docker must be installed and running to start containers.\n%v", dockerAddr, err)
} else {
glog.Infof("Connecting to Docker at %s", dockerAddr)
}
cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194")
if err != nil {
glog.Errorf("Error on creating cadvisor client: %v", err)
}
// initialize etcd
etcdServer := etcd.New(etcdConfig)
go util.Forever(func() {
glog.Infof("Started etcd at http://%s", etcdAddr)
etcdServer.Run()
}, 0)
etcdClient := etcdclient.NewClient(etcdServers)
for i := 0; ; i += 1 {
_, err := etcdClient.Get("/", false, false)
if err == nil || tools.IsEtcdNotFound(err) {
break
}
if i > 100 {
glog.Fatal("Could not reach etcd: %v", err)
}
time.Sleep(50 * time.Millisecond)
}
// initialize Kubelet
os.MkdirAll(rootDirectory, 0750)
cfg := kconfig.NewPodConfig(kconfig.PodConfigNotificationSnapshotAndUpdates)
kconfig.NewSourceEtcd(kconfig.EtcdKeyForHost(minionHost), etcdClient, cfg.Channel("etcd"))
k := kubelet.NewMainKubelet(
minionHost,
dockerClient,
cadvisorClient,
etcdClient,
rootDirectory,
30*time.Second)
go util.Forever(func() { k.Run(cfg.Updates()) }, 0)
go util.Forever(func() {
kubelet.ListenAndServeKubeletServer(k, cfg.Channel("http"), minionHost, uint(minionPort))
}, 0)
imageRegistry := image.NewEtcdRegistry(etcdClient)
// initialize OpenShift API
storage := map[string]apiserver.RESTStorage{
"builds": buildregistry.NewStorage(build.NewEtcdRegistry(etcdClient)),
"buildConfigs": buildconfigregistry.NewStorage(build.NewEtcdRegistry(etcdClient)),
"images": image.NewImageStorage(imageRegistry),
"imageRepositories": image.NewImageRepositoryStorage(imageRegistry),
"imageRepositoryMappings": image.NewImageRepositoryMappingStorage(imageRegistry, imageRegistry),
"templateConfigs": template.NewStorage(),
}
osMux := http.NewServeMux()
// initialize Kubernetes API
podInfoGetter := &kubeclient.HTTPPodInfoGetter{
Client: http.DefaultClient,
Port: uint(minionPort),
}
masterConfig := &master.Config{
Client: kubeClient,
EtcdServers: etcdServers,
HealthCheckMinions: true,
Minions: []string{minionHost},
PodInfoGetter: podInfoGetter,
}
m := master.New(masterConfig)
apiserver.NewAPIGroup(m.API_v1beta1()).InstallREST(osMux, kubePrefix)
apiserver.NewAPIGroup(storage, runtime.Codec).InstallREST(osMux, osPrefix)
apiserver.InstallSupport(osMux)
osApi := &http.Server{
Addr: osAddr,
Handler: apiserver.RecoverPanics(osMux),
ReadTimeout: 5 * time.Minute,
WriteTimeout: 5 * time.Minute,
MaxHeaderBytes: 1 << 20,
}
go util.Forever(func() {
glog.Infof("Started Kubernetes API at http://%s%s", osAddr, kubePrefix)
glog.Infof("Started OpenShift API at http://%s%s", osAddr, osPrefix)
glog.Fatal(osApi.ListenAndServe())
}, 0)
// initialize kube proxy
serviceConfig := pconfig.NewServiceConfig()
endpointsConfig := pconfig.NewEndpointsConfig()
pconfig.NewConfigSourceEtcd(etcdClient,
serviceConfig.Channel("etcd"),
endpointsConfig.Channel("etcd"))
loadBalancer := proxy.NewLoadBalancerRR()
proxier := proxy.NewProxier(loadBalancer)
serviceConfig.RegisterHandler(proxier)
endpointsConfig.RegisterHandler(loadBalancer)
glog.Infof("Started Kubernetes Proxy")
// initialize replication manager
controllerManager := controller.NewReplicationManager(kubeClient)
controllerManager.Run(10 * time.Second)
glog.Infof("Started Kubernetes Replication Manager")
// initialize scheduler
configFactory := &factory.ConfigFactory{Client: kubeClient}
config := configFactory.Create()
s := scheduler.New(config)
s.Run()
glog.Infof("Started Kubernetes Scheduler")
// initialize build controller
dockerBuilderImage := env("OPENSHIFT_DOCKER_BUILDER_IMAGE", "openshift/docker-builder")
useHostDockerSocket := len(env("USE_HOST_DOCKER_SOCKET", "")) > 0
stiBuilderImage := env("OPENSHIFT_STI_BUILDER_IMAGE", "openshift/sti-builder")
dockerRegistry := env("DOCKER_REGISTRY", "")
buildStrategies := map[buildapi.BuildType]build.BuildJobStrategy{
buildapi.DockerBuildType: strategy.NewDockerBuildStrategy(dockerBuilderImage, useHostDockerSocket),
buildapi.STIBuildType: strategy.NewSTIBuildStrategy(stiBuilderImage, useHostDockerSocket),
}
buildController := build.NewBuildController(kubeClient, osClient, buildStrategies, dockerRegistry, 1200)
buildController.Run(10 * time.Second)
select {}
}
