func serve(listeners []net.Listener, servers []*http.Server) error {
hdServers := make([]httpdown.Server, len(listeners))
for i, l := range listeners {
if servers[i].TLSConfig != nil {
l = tls.NewListener(l, servers[i].TLSConfig)
}
hdServers[i] = httpdown.HTTP{}.Serve(servers[i], l)
}
_ = daemon.SdNotify(sdReady) // ignore error
ch := make(chan os.Signal)
signal.Notify(ch, syscall.SIGTERM)
<-ch
signal.Stop(ch)
errs := make([]error, len(listeners))
wg := sync.WaitGroup{}
for i := range hdServers {
wg.Add(1)
go func(i int) {
defer wg.Done()
errs[i] = hdServers[i].Stop()
}(i)
}
wg.Wait()
for _, err := range errs {
if err != nil {
return err
}
}
return nil
}
// StartAllInOne:
// 1. Creates the signer certificate if needed
// 2. Calls RunMaster
// 3. Calls RunNode
// 4. If only writing configs, it exits
// 5. Waits forever
func (o AllInOneOptions) StartAllInOne() error {
if o.PrintIP {
host, _, err := net.SplitHostPort(o.NodeArgs.DefaultKubernetesURL.Host)
if err != nil {
return err
}
fmt.Fprintf(o.Output, "%s\n", host)
return nil
}
masterOptions := *o.MasterOptions
if err := masterOptions.RunMaster(); err != nil {
return err
}
nodeOptions := NodeOptions{o.NodeArgs, o.NodeConfigFile, o.MasterOptions.Output}
if err := nodeOptions.RunNode(); err != nil {
return err
}
if o.IsWriteConfigOnly() {
return nil
}
daemon.SdNotify("READY=1")
select {}
}
// StartEtcdServer calls RunEtcdServer and then waits forever
func (o *EtcdOptions) StartEtcdServer() error {
if err := o.RunEtcdServer(); err != nil {
return err
}
go daemon.SdNotify("READY=1")
select {}
}
func (s *Server) AcceptConnections() {
go systemdDaemon.SdNotify("READY=1")
// close the lock so the listeners start accepting connections
select {
case <-s.start:
default:
close(s.start)
}
}
func main() {
flag.Parse()
exists, err := dirExists(*outputDir)
if err != nil {
log.Fatal(err)
}
if !exists {
if err := os.Mkdir(*outputDir, 0755); err != nil {
log.Fatal(err)
}
}
cfg := client.Config{
Endpoints: []string{*endpoint},
Transport: client.DefaultTransport,
// set timeout per request to fail fast when the target endpoint is unavailable
HeaderTimeoutPerRequest: time.Second,
}
c, err := client.New(cfg)
if err != nil {
log.Fatal(err)
}
if err := c.Sync(context.Background()); err != nil {
log.Fatal(err)
}
kapi := client.NewKeysAPI(c)
resp, err := generateConfig(kapi)
if err != nil {
log.Fatal(err)
}
if systemdutil.IsRunningSystemd() {
err := daemon.SdNotify("READY=1")
if err != nil {
log.Printf("failed to notify systemd for readiness: %v", err)
if err == daemon.SdNotifyNoSocket {
log.Printf("forgot to set Type=notify in systemd service file?")
}
}
}
if *watch {
for {
resp, err = generateConfigWatcher(kapi, resp)
if err != nil {
log.Fatal(err)
}
}
}
os.Exit(0)
}
func (r *SystemdConfig) Clean() {
daemon.SdNotify("STOPPING=1")
for service, filenames := range r.written {
log.Printf("systemd: %s: removing configs...", service)
for _, filename := range filenames {
os.Remove(filename)
}
reload(service)
}
}
// StartNode calls RunNode and then waits forever
func (o NodeOptions) StartNode() error {
if err := o.RunNode(); err != nil {
return err
}
if o.IsWriteConfigOnly() {
return nil
}
go daemon.SdNotify("READY=1")
select {}
}
// AcceptConnections allows clients to connect to the API server.
// Referenced Daemon is notified about this server, and waits for the
// daemon acknowledgement before the incoming connections are accepted.
func (s *Server) AcceptConnections(d *daemon.Daemon) {
// Tell the init daemon we are accepting requests
s.daemon = d
s.registerSubRouter()
go systemdDaemon.SdNotify("READY=1")
// close the lock so the listeners start accepting connections
select {
case <-s.start:
default:
close(s.start)
}
}
// StartMaster calls RunMaster and then waits forever
func (o MasterOptions) StartMaster() error {
if err := o.RunMaster(); err != nil {
return err
}
if o.IsWriteConfigOnly() {
return nil
}
// TODO: this should be encapsulated by RunMaster, but StartAllInOne has no
// way to communicate whether RunMaster should block.
go daemon.SdNotify("READY=1")
select {}
}
// StartMaster calls RunMaster and then waits forever
func (o MasterOptions) StartMaster() error {
if err := o.RunMaster(); err != nil {
return err
}
if o.IsWriteConfigOnly() {
return nil
}
go daemon.SdNotify("READY=1")
select {}
return nil
}
// This function provides a CLI handler for 'start' command
func NewCommandStartAllInOne(fullName string, out io.Writer) (*cobra.Command, *AllInOneOptions) {
options := &AllInOneOptions{}
cmds := &cobra.Command{
Use: "start",
Short: "Launch StraYard All-In-One",
Long: allinoneLongDesc,
Run: func(c *cobra.Command, args []string) {
fmt.Println("haha")
daemon.SdNotify("READY=1")
select {}
},
}
cmds.SetOutput(out)
return cmds, options
}
func RunServer(ctx context.Context, sm subnet.Manager, listenAddr, cafile, certfile, keyfile string) {
// {network} is always required a the API level but to
// keep backward compat, special "_" network is allowed
// that means "no network"
r := mux.NewRouter()
r.HandleFunc("/v1/{network}/config", bindHandler(handleGetNetworkConfig, ctx, sm)).Methods("GET")
r.HandleFunc("/v1/{network}/leases", bindHandler(handleAcquireLease, ctx, sm)).Methods("POST")
r.HandleFunc("/v1/{network}/leases/{subnet}", bindHandler(handleWatchLease, ctx, sm)).Methods("GET")
r.HandleFunc("/v1/{network}/leases/{subnet}", bindHandler(handleRenewLease, ctx, sm)).Methods("PUT")
r.HandleFunc("/v1/{network}/leases/{subnet}", bindHandler(handleRevokeLease, ctx, sm)).Methods("DELETE")
r.HandleFunc("/v1/{network}/leases", bindHandler(handleWatchLeases, ctx, sm)).Methods("GET")
r.HandleFunc("/v1/", bindHandler(handleNetworks, ctx, sm)).Methods("GET")
r.HandleFunc("/v1/{network}/reservations", bindHandler(handleListReservations, ctx, sm)).Methods("GET")
r.HandleFunc("/v1/{network}/reservations", bindHandler(handleAddReservation, ctx, sm)).Methods("POST")
r.HandleFunc("/v1/{network}/reservations/{subnet}", bindHandler(handleRemoveReservation, ctx, sm)).Methods("DELETE")
l, err := listener(listenAddr, cafile, certfile, keyfile)
if err != nil {
log.Errorf("Error listening on %v: %v", listenAddr, err)
return
}
c := make(chan error, 1)
go func() {
c <- http.Serve(l, httpLogger(r))
}()
daemon.SdNotify("READY=1")
select {
case <-ctx.Done():
l.Close()
<-c
case err := <-c:
log.Errorf("Error serving on %v: %v", listenAddr, err)
}
}
func (r *SystemdConfig) StoreAddress(address string) error {
data := templateArgs{address}
for _, s := range r.services {
pattern := filepath.Join(r.templatePath, s.dir, s.filepattern)
log.Printf("systemd: %s: loading config from %s", s.name, pattern)
templates, err := template.ParseGlob(pattern)
if err != nil {
log.Println("systemd:", err)
continue
}
var written []string
for _, t := range templates.Templates() {
dest := filepath.Join(r.destPath, s.dir, t.Name())
log.Println("systemd: generating", dest)
fp, err := os.Create(dest)
if err != nil {
log.Println("systemd:", err)
continue
}
written = append(written, dest)
t.Execute(fp, data)
fp.Close()
}
if written != nil {
r.written[s.name] = written
reload(s.name)
} else {
log.Println("systemd: %s: no configs written, skipping reload", s.name)
}
}
daemon.SdNotify("READY=1")
return nil
}
// Run spawns the http servers (secure and insecure). It only returns if stopCh is closed
// or one of the ports cannot be listened on initially.
func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) {
if s.SecureServingInfo != nil && s.Handler != nil {
if err := s.serveSecurely(stopCh); err != nil {
glog.Fatal(err)
}
}
if s.InsecureServingInfo != nil && s.InsecureHandler != nil {
if err := s.serveInsecurely(stopCh); err != nil {
glog.Fatal(err)
}
}
s.RunPostStartHooks()
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
<-stopCh
}
func (m *Manager) runNetwork(n *Network) {
n.Run(m.extIface, func(bn backend.Network) {
if m.isMultiNetwork() {
log.Infof("%v: lease acquired: %v", n.Name, bn.Lease().Subnet)
path := filepath.Join(opts.subnetDir, n.Name) + ".env"
if err := writeSubnetFile(path, n.Config.Network, m.ipMasq, bn); err != nil {
log.Warningf("%v failed to write subnet file: %s", n.Name, err)
return
}
} else {
log.Infof("Lease acquired: %v", bn.Lease().Subnet)
if err := writeSubnetFile(opts.subnetFile, n.Config.Network, m.ipMasq, bn); err != nil {
log.Warningf("%v failed to write subnet file: %s", n.Name, err)
return
}
daemon.SdNotify("READY=1")
}
})
m.delNetwork(n)
}
// StartAllInOne:
// 1. Creates the signer certificate if needed
// 2. Calls RunMaster
// 3. Calls RunNode
// 4. If only writing configs, it exits
// 5. Waits forever
func (o AllInOneOptions) StartAllInOne() error {
if !o.IsWriteConfigOnly() {
glog.Infof("Starting an OpenShift all-in-one")
}
masterOptions := MasterOptions{o.MasterArgs, o.CreateCerts, o.MasterConfigFile, o.Output}
if err := masterOptions.RunMaster(); err != nil {
return err
}
nodeOptions := NodeOptions{o.NodeArgs, o.NodeConfigFile, o.Output}
if err := nodeOptions.RunNode(); err != nil {
return err
}
if o.IsWriteConfigOnly() {
return nil
}
daemon.SdNotify("READY=1")
select {}
return nil
}
// Watchdog setup the watchdog and start then. This functoin will
// comunicate with systemd sending the pings to it, if this fails
// to send the ping systemd will reatart this daemon.
func Watchdog() (stop chan struct{}, err error) {
// Check if systemd exist.
if !util.IsRunningSystemd() {
return nil, e.New(ErrNotRunning)
}
// Get the periode and check if watchdog is on for this daemon
wPeriodeµsec := os.Getenv("WATCHDOG_USEC")
if wPeriodeµsec == "" {
return nil, e.New(ErrNoWatchdog)
}
wPerInt64, err := strconv.ParseInt(wPeriodeµsec, 10, 32)
if err != nil {
return nil, e.Push(err, ErrInvPeriode)
}
wPerHalf := time.Duration(int(wPerInt64)/2) * time.Microsecond
if wPerHalf <= 0 {
return nil, e.New(ErrInvInterval)
}
log.Tag("systemd", "watchdog").Printf("Starting the watchdog with interval of %v.", wPerHalf)
stop = make(chan struct{})
// Start the periodic pings
go func() {
for {
select {
case <-stop:
log.Tag("systemd", "watchdog").Println("By request watchdog is stoping.")
return
case <-time.After(wPerHalf):
// Send the ping.
log.ProtoLevel().Tag("systemd", "watchdog").Println("Ping.")
daemon.SdNotify(sdState)
}
}
}()
return
}
func Main() {
//载入命令中设置的配置项
cfg := NewConfig()
err := cfg.Parse(os.Args[1:])
if err != nil {
plog.Errorf("error verifying flags, %v. See 'etcd --help'.", err)
switch err {
case errUnsetAdvertiseClientURLsFlag:
plog.Errorf("When listening on specific address(es), this etcd process must advertise accessible url(s) to each connected client.")
}
os.Exit(1)
}
//配置信息记录到日志中
setupLogging(cfg)
var stopped <-chan struct{}
plog.Infof("etcd Version: %s\n", version.Version)
plog.Infof("Git SHA: %s\n", version.GitSHA)
plog.Infof("Go Version: %s\n", runtime.Version())
plog.Infof("Go OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
GoMaxProcs := runtime.GOMAXPROCS(0)
plog.Infof("setting maximum number of CPUs to %d, total number of available CPUs is %d", GoMaxProcs, runtime.NumCPU())
// TODO: check whether fields are set instead of whether fields have default value
if cfg.name != defaultName && cfg.initialCluster == initialClusterFromName(defaultName) {
cfg.initialCluster = initialClusterFromName(cfg.name)
}
if cfg.dir == "" {
cfg.dir = fmt.Sprintf("%v.etcd", cfg.name)
plog.Warningf("no data-dir provided, using default data-dir ./%s", cfg.dir)
}
//识别数据文件夹类型(会员或代理或为空)
which := identifyDataDirOrDie(cfg.dir)
if which != dirEmpty {
plog.Noticef("the server is already initialized as %v before, starting as etcd %v...", which, which)
switch which {
case dirMember:
stopped, err = startEtcd(cfg)
case dirProxy:
err = startProxy(cfg)
default:
plog.Panicf("unhandled dir type %v", which)
}
} else { //类型为空时(新建数据--???--)
shouldProxy := cfg.isProxy() //是否为代理模式
if !shouldProxy {
stopped, err = startEtcd(cfg)
if derr, ok := err.(*etcdserver.DiscoveryError); ok && derr.Err == discovery.ErrFullCluster {
if cfg.shouldFallbackToProxy() {
plog.Noticef("discovery cluster full, falling back to %s", fallbackFlagProxy)
shouldProxy = true
}
}
}
if shouldProxy { //代理模式
//不加入到etcd一致性集群中，纯粹进行代理转发。
err = startProxy(cfg)
}
}
if err != nil {
if derr, ok := err.(*etcdserver.DiscoveryError); ok {
switch derr.Err {
case discovery.ErrDuplicateID:
plog.Errorf("member %q has previously registered with discovery service token (%s).", cfg.name, cfg.durl)
plog.Errorf("But etcd could not find valid cluster configuration in the given data dir (%s).", cfg.dir)
plog.Infof("Please check the given data dir path if the previous bootstrap succeeded")
plog.Infof("or use a new discovery token if the previous bootstrap failed.")
case discovery.ErrDuplicateName:
plog.Errorf("member with duplicated name has registered with discovery service token(%s).", cfg.durl)
plog.Errorf("please check (cURL) the discovery token for more information.")
plog.Errorf("please do not reuse the discovery token and generate a new one to bootstrap the cluster.")
default:
plog.Errorf("%v", err)
plog.Infof("discovery token %s was used, but failed to bootstrap the cluster.", cfg.durl)
plog.Infof("please generate a new discovery token and try to bootstrap again.")
}
os.Exit(1)
}
if strings.Contains(err.Error(), "include") && strings.Contains(err.Error(), "--initial-cluster") {
plog.Infof("%v", err)
if cfg.initialCluster == initialClusterFromName(cfg.name) {
plog.Infof("forgot to set --initial-cluster flag?")
}
if types.URLs(cfg.apurls).String() == defaultInitialAdvertisePeerURLs {
plog.Infof("forgot to set --initial-advertise-peer-urls flag?")
}
if cfg.initialCluster == initialClusterFromName(cfg.name) && len(cfg.durl) == 0 {
plog.Infof("if you want to use discovery service, please set --discovery flag.")
}
os.Exit(1)
}
plog.Fatalf("%v", err)
}
osutil.HandleInterrupts()
if systemdutil.IsRunningSystemd() {
// At this point, the initialization of etcd is done.
// The listeners are listening on the TCP ports and ready
// for accepting connections.
// The http server is probably ready for serving incoming
// connections. If it is not, the connection might be pending
// for less than one second.
err := daemon.SdNotify("READY=1")
if err != nil {
plog.Errorf("failed to notify systemd for readiness: %v", err)
if err == daemon.SdNotifyNoSocket {
plog.Errorf("forgot to set Type=notify in systemd service file?")
}
}
}
<-stopped
osutil.Exit(0)
}
func (s *GenericAPIServer) Run(options *ServerRunOptions) {
if s.enableSwaggerSupport {
s.InstallSwaggerAPI()
}
// We serve on 2 ports. See docs/accessing_the_api.md
secureLocation := ""
if options.SecurePort != 0 {
secureLocation = net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort))
}
insecureLocation := net.JoinHostPort(options.InsecureBindAddress.String(), strconv.Itoa(options.InsecurePort))
var sem chan bool
if options.MaxRequestsInFlight > 0 {
sem = make(chan bool, options.MaxRequestsInFlight)
}
longRunningRE := regexp.MustCompile(options.LongRunningRequestRE)
longRunningRequestCheck := apiserver.BasicLongRunningRequestCheck(longRunningRE, map[string]string{"watch": "true"})
longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) {
// TODO unify this with apiserver.MaxInFlightLimit
if longRunningRequestCheck(req) {
return nil, ""
}
return time.After(globalTimeout), ""
}
if secureLocation != "" {
handler := apiserver.TimeoutHandler(s.Handler, longRunningTimeout)
secureServer := &http.Server{
Addr: secureLocation,
Handler: apiserver.MaxInFlightLimit(sem, longRunningRequestCheck, apiserver.RecoverPanics(handler)),
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
MinVersion: tls.VersionTLS10,
},
}
if len(options.ClientCAFile) > 0 {
clientCAs, err := crypto.CertPoolFromFile(options.ClientCAFile)
if err != nil {
glog.Fatalf("Unable to load client CA file: %v", err)
}
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
// Specify allowed CAs for client certificates
secureServer.TLSConfig.ClientCAs = clientCAs
}
glog.Infof("Serving securely on %s", secureLocation)
if options.TLSCertFile == "" && options.TLSPrivateKeyFile == "" {
options.TLSCertFile = path.Join(options.CertDirectory, "apiserver.crt")
options.TLSPrivateKeyFile = path.Join(options.CertDirectory, "apiserver.key")
// TODO (cjcullen): Is ClusterIP the right address to sign a cert with?
alternateIPs := []net.IP{s.ServiceReadWriteIP}
alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}
// It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
if shouldGenSelfSignedCerts(options.TLSCertFile, options.TLSPrivateKeyFile) {
if err := crypto.GenerateSelfSignedCert(s.ClusterIP.String(), options.TLSCertFile, options.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil {
glog.Errorf("Unable to generate self signed cert: %v", err)
} else {
glog.Infof("Using self-signed cert (%options, %options)", options.TLSCertFile, options.TLSPrivateKeyFile)
}
}
}
go func() {
defer utilruntime.HandleCrash()
for {
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
if err := secureServer.ListenAndServeTLS(options.TLSCertFile, options.TLSPrivateKeyFile); err != nil {
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
} else {
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
}
handler := apiserver.TimeoutHandler(s.InsecureHandler, longRunningTimeout)
http := &http.Server{
Addr: insecureLocation,
Handler: apiserver.RecoverPanics(handler),
MaxHeaderBytes: 1 << 20,
}
glog.Infof("Serving insecurely on %s", insecureLocation)
glog.Fatal(http.ListenAndServe())
}
// Run runs the specified APIServer. This should never exit.
func (s *APIServer) Run(_ []string) error {
s.verifyClusterIPFlags()
// If advertise-address is not specified, use bind-address. If bind-address
// is not usable (unset, 0.0.0.0, or loopback), setDefaults() in
// pkg/master/master.go will do the right thing and use the host's default
// interface.
if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
s.AdvertiseAddress = s.BindAddress
}
if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) {
glog.Fatalf("Specify either --etcd-servers or --etcd-config")
}
if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) {
glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort))
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
// Setup tunneler if needed
var tunneler master.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH master.InstallSSHKey
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
// Set up the tunneler
tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
kubeletClient, err := client.NewKubeletClient(&s.KubeletConfig)
if err != nil {
glog.Fatalf("Failure to start kubelet client: %v", err)
}
apiGroupVersionOverrides, err := s.parseRuntimeConfig()
if err != nil {
glog.Fatalf("error in parsing runtime-config: %s", err)
}
clientConfig := &client.Config{
Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
Version: s.DeprecatedStorageVersion,
}
client, err := client.New(clientConfig)
if err != nil {
glog.Fatalf("Invalid server address: %v", err)
}
legacyV1Group, err := latest.Group("")
if err != nil {
return err
}
storageDestinations := master.NewStorageDestinations()
storageVersions := generateStorageVersionMap(s.DeprecatedStorageVersion, s.StorageVersions)
if _, found := storageVersions[legacyV1Group.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.Group, storageVersions)
}
etcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, legacyV1Group.InterfacesFor, storageVersions[legacyV1Group.Group], s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup("", etcdStorage)
if !apiGroupVersionOverrides["extensions/v1beta1"].Disable {
expGroup, err := latest.Group("extensions")
if err != nil {
glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err)
}
if _, found := storageVersions[expGroup.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.Group, storageVersions)
}
expEtcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, expGroup.InterfacesFor, storageVersions[expGroup.Group], s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup("extensions", expEtcdStorage)
}
updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdPathPrefix, &storageDestinations, newEtcd)
n := s.ServiceClusterIPRange
// Default to the private server key for service account token signing
if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
if apiserver.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
} else {
glog.Warning("No RSA key provided, service account token authentication disabled")
}
}
authenticator, err := apiserver.NewAuthenticator(apiserver.AuthenticatorConfig{
BasicAuthFile: s.BasicAuthFile,
ClientCAFile: s.ClientCAFile,
TokenAuthFile: s.TokenAuthFile,
OIDCIssuerURL: s.OIDCIssuerURL,
OIDCClientID: s.OIDCClientID,
OIDCCAFile: s.OIDCCAFile,
OIDCUsernameClaim: s.OIDCUsernameClaim,
ServiceAccountKeyFile: s.ServiceAccountKeyFile,
ServiceAccountLookup: s.ServiceAccountLookup,
Storage: etcdStorage,
KeystoneURL: s.KeystoneURL,
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationPolicyFile)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
if len(s.ExternalHost) == 0 {
// TODO: extend for other providers
if s.CloudProvider == "gce" {
instances, supported := cloud.Instances()
if !supported {
glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.")
}
name, err := os.Hostname()
if err != nil {
glog.Fatalf("Failed to get hostname: %v", err)
}
addrs, err := instances.NodeAddresses(name)
if err != nil {
glog.Warningf("Unable to obtain external host address from cloud provider: %v", err)
} else {
for _, addr := range addrs {
if addr.Type == api.NodeExternalIP {
s.ExternalHost = addr.Address
}
}
}
}
}
config := &master.Config{
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
EventTTL: s.EventTTL,
KubeletClient: kubeletClient,
ServiceClusterIPRange: &n,
EnableCoreControllers: true,
EnableLogsSupport: s.EnableLogsSupport,
EnableUISupport: true,
EnableSwaggerSupport: true,
EnableProfiling: s.EnableProfiling,
EnableWatchCache: s.EnableWatchCache,
EnableIndex: true,
APIPrefix: s.APIPrefix,
APIGroupPrefix: s.APIGroupPrefix,
CorsAllowedOriginList: s.CorsAllowedOriginList,
ReadWritePort: s.SecurePort,
PublicAddress: s.AdvertiseAddress,
Authenticator: authenticator,
SupportsBasicAuth: len(s.BasicAuthFile) > 0,
Authorizer: authorizer,
AdmissionControl: admissionController,
APIGroupVersionOverrides: apiGroupVersionOverrides,
MasterServiceNamespace: s.MasterServiceNamespace,
ClusterName: s.ClusterName,
ExternalHost: s.ExternalHost,
MinRequestTimeout: s.MinRequestTimeout,
ProxyDialer: proxyDialerFn,
ProxyTLSClientConfig: proxyTLSClientConfig,
Tunneler: tunneler,
ServiceNodePortRange: s.ServiceNodePortRange,
KubernetesServiceNodePort: s.KubernetesServiceNodePort,
}
m := master.New(config)
// We serve on 2 ports. See docs/accessing_the_api.md
secureLocation := ""
if s.SecurePort != 0 {
secureLocation = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort))
}
insecureLocation := net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort))
// See the flag commentary to understand our assumptions when opening the read-only and read-write ports.
var sem chan bool
if s.MaxRequestsInFlight > 0 {
sem = make(chan bool, s.MaxRequestsInFlight)
}
longRunningRE := regexp.MustCompile(s.LongRunningRequestRE)
longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) {
// TODO unify this with apiserver.MaxInFlightLimit
if longRunningRE.MatchString(req.URL.Path) || req.URL.Query().Get("watch") == "true" {
return nil, ""
}
return time.After(time.Minute), ""
}
if secureLocation != "" {
handler := apiserver.TimeoutHandler(m.Handler, longRunningTimeout)
secureServer := &http.Server{
Addr: secureLocation,
Handler: apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(handler)),
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
MinVersion: tls.VersionTLS10,
},
}
if len(s.ClientCAFile) > 0 {
clientCAs, err := util.CertPoolFromFile(s.ClientCAFile)
if err != nil {
glog.Fatalf("Unable to load client CA file: %v", err)
}
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
// Specify allowed CAs for client certificates
secureServer.TLSConfig.ClientCAs = clientCAs
}
glog.Infof("Serving securely on %s", secureLocation)
if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
s.TLSCertFile = path.Join(s.CertDirectory, "apiserver.crt")
s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "apiserver.key")
// TODO (cjcullen): Is PublicAddress the right address to sign a cert with?
alternateIPs := []net.IP{config.ServiceReadWriteIP}
alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}
// It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil {
glog.Errorf("Unable to generate self signed cert: %v", err)
} else {
glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
}
}
go func() {
defer util.HandleCrash()
for {
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
if err := secureServer.ListenAndServeTLS(s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
}
handler := apiserver.TimeoutHandler(m.InsecureHandler, longRunningTimeout)
http := &http.Server{
Addr: insecureLocation,
Handler: apiserver.RecoverPanics(handler),
MaxHeaderBytes: 1 << 20,
}
if secureLocation == "" {
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
}
glog.Infof("Serving insecurely on %s", insecureLocation)
glog.Fatal(http.ListenAndServe())
return nil
}
func startEtcdOrProxyV2() {
grpc.EnableTracing = false
cfg := newConfig()
defaultInitialCluster := cfg.InitialCluster
err := cfg.parse(os.Args[1:])
if err != nil {
plog.Errorf("error verifying flags, %v. See 'etcd --help'.", err)
switch err {
case embed.ErrUnsetAdvertiseClientURLsFlag:
plog.Errorf("When listening on specific address(es), this etcd process must advertise accessible url(s) to each connected client.")
}
os.Exit(1)
}
setupLogging(cfg)
var stopped <-chan struct{}
var errc <-chan error
plog.Infof("etcd Version: %s\n", version.Version)
plog.Infof("Git SHA: %s\n", version.GitSHA)
plog.Infof("Go Version: %s\n", runtime.Version())
plog.Infof("Go OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
GoMaxProcs := runtime.GOMAXPROCS(0)
plog.Infof("setting maximum number of CPUs to %d, total number of available CPUs is %d", GoMaxProcs, runtime.NumCPU())
// TODO: check whether fields are set instead of whether fields have default value
defaultHost, defaultHostErr := cfg.IsDefaultHost()
defaultHostOverride := defaultHost == "" || defaultHostErr == nil
if (defaultHostOverride || cfg.Name != embed.DefaultName) && cfg.InitialCluster == defaultInitialCluster {
cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
}
if cfg.Dir == "" {
cfg.Dir = fmt.Sprintf("%v.etcd", cfg.Name)
plog.Warningf("no data-dir provided, using default data-dir ./%s", cfg.Dir)
}
which := identifyDataDirOrDie(cfg.Dir)
if which != dirEmpty {
plog.Noticef("the server is already initialized as %v before, starting as etcd %v...", which, which)
switch which {
case dirMember:
stopped, errc, err = startEtcd(&cfg.Config)
case dirProxy:
err = startProxy(cfg)
default:
plog.Panicf("unhandled dir type %v", which)
}
} else {
shouldProxy := cfg.isProxy()
if !shouldProxy {
stopped, errc, err = startEtcd(&cfg.Config)
if derr, ok := err.(*etcdserver.DiscoveryError); ok && derr.Err == discovery.ErrFullCluster {
if cfg.shouldFallbackToProxy() {
plog.Noticef("discovery cluster full, falling back to %s", fallbackFlagProxy)
shouldProxy = true
}
}
}
if shouldProxy {
err = startProxy(cfg)
}
}
if err != nil {
if derr, ok := err.(*etcdserver.DiscoveryError); ok {
switch derr.Err {
case discovery.ErrDuplicateID:
plog.Errorf("member %q has previously registered with discovery service token (%s).", cfg.Name, cfg.Durl)
plog.Errorf("But etcd could not find valid cluster configuration in the given data dir (%s).", cfg.Dir)
plog.Infof("Please check the given data dir path if the previous bootstrap succeeded")
plog.Infof("or use a new discovery token if the previous bootstrap failed.")
case discovery.ErrDuplicateName:
plog.Errorf("member with duplicated name has registered with discovery service token(%s).", cfg.Durl)
plog.Errorf("please check (cURL) the discovery token for more information.")
plog.Errorf("please do not reuse the discovery token and generate a new one to bootstrap the cluster.")
default:
plog.Errorf("%v", err)
plog.Infof("discovery token %s was used, but failed to bootstrap the cluster.", cfg.Durl)
plog.Infof("please generate a new discovery token and try to bootstrap again.")
}
os.Exit(1)
}
if strings.Contains(err.Error(), "include") && strings.Contains(err.Error(), "--initial-cluster") {
plog.Infof("%v", err)
if cfg.InitialCluster == cfg.InitialClusterFromName(cfg.Name) {
plog.Infof("forgot to set --initial-cluster flag?")
}
if types.URLs(cfg.APUrls).String() == embed.DefaultInitialAdvertisePeerURLs {
plog.Infof("forgot to set --initial-advertise-peer-urls flag?")
}
if cfg.InitialCluster == cfg.InitialClusterFromName(cfg.Name) && len(cfg.Durl) == 0 {
plog.Infof("if you want to use discovery service, please set --discovery flag.")
}
os.Exit(1)
}
plog.Fatalf("%v", err)
}
osutil.HandleInterrupts()
if systemdutil.IsRunningSystemd() {
// At this point, the initialization of etcd is done.
// The listeners are listening on the TCP ports and ready
// for accepting connections. The etcd instance should be
// joined with the cluster and ready to serve incoming
// connections.
sent, err := daemon.SdNotify(false, "READY=1")
if err != nil {
plog.Errorf("failed to notify systemd for readiness: %v", err)
}
if !sent {
plog.Errorf("forgot to set Type=notify in systemd service file?")
}
}
select {
case lerr := <-errc:
// fatal out on listener errors
plog.Fatal(lerr)
case <-stopped:
}
osutil.Exit(0)
}
func (s *GenericAPIServer) Run() {
// install APIs which depend on other APIs to be installed
if s.enableSwaggerSupport {
s.InstallSwaggerAPI()
}
if s.enableOpenAPISupport {
s.InstallOpenAPI()
}
if s.SecureServingInfo != nil {
secureServer := &http.Server{
Addr: s.SecureServingInfo.BindAddress,
Handler: s.Handler,
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
},
}
if len(s.SecureServingInfo.ClientCA) > 0 {
clientCAs, err := certutil.NewPool(s.SecureServingInfo.ClientCA)
if err != nil {
glog.Fatalf("Unable to load client CA file: %v", err)
}
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
// Specify allowed CAs for client certificates
secureServer.TLSConfig.ClientCAs = clientCAs
// "h2" NextProtos is necessary for enabling HTTP2 for go's 1.7 HTTP Server
secureServer.TLSConfig.NextProtos = []string{"h2"}
}
// It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
if s.SecureServingInfo.ServerCert.Generate && !certutil.CanReadCertOrKey(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile) {
// TODO (cjcullen): Is ClusterIP the right address to sign a cert with?
alternateIPs := []net.IP{s.ServiceReadWriteIP}
alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}
if err := certutil.GenerateSelfSignedCert(s.ClusterIP.String(), s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile, alternateIPs, alternateDNS); err != nil {
glog.Errorf("Unable to generate self signed cert: %v", err)
} else {
glog.Infof("Using self-signed cert (%s, %s)", s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile)
}
}
glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
go func() {
defer utilruntime.HandleCrash()
for {
if err := secureServer.ListenAndServeTLS(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile); err != nil {
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
}
if s.InsecureServingInfo != nil {
insecureServer := &http.Server{
Addr: s.InsecureServingInfo.BindAddress,
Handler: s.InsecureHandler,
MaxHeaderBytes: 1 << 20,
}
glog.Infof("Serving insecurely on %s", s.InsecureServingInfo.BindAddress)
go func() {
defer utilruntime.HandleCrash()
for {
if err := insecureServer.ListenAndServe(); err != nil {
glog.Errorf("Unable to listen for insecure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
}
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) per port
if s.SecureServingInfo != nil {
if err := waitForSuccessfulDial(true, "tcp", s.SecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil {
glog.Fatalf("Secure server never started: %v", err)
}
}
if s.InsecureServingInfo != nil {
if err := waitForSuccessfulDial(false, "tcp", s.InsecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil {
glog.Fatalf("Insecure server never started: %v", err)
}
}
s.RunPostStartHooks()
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
select {}
}
// Run runs the specified APIServer. This should never exit.
func (s *APIServer) Run(_ []string) error {
s.verifyClusterIPFlags()
// If advertise-address is not specified, use bind-address. If bind-address
// is not usable (unset, 0.0.0.0, or loopback), setDefaults() in
// pkg/master/master.go will do the right thing and use the host's default
// interface.
if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
s.AdvertiseAddress = s.BindAddress
}
if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) {
glog.Fatalf("specify either --etcd-servers or --etcd-config")
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
HostNetworkSources: []string{},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
kubeletClient, err := client.NewKubeletClient(&s.KubeletConfig)
if err != nil {
glog.Fatalf("Failure to start kubelet client: %v", err)
}
// "api/all=false" allows users to selectively enable specific api versions.
disableAllAPIs := false
allAPIFlagValue, ok := s.RuntimeConfig["api/all"]
if ok && allAPIFlagValue == "false" {
disableAllAPIs = true
}
// "api/legacy=false" allows users to disable legacy api versions.
disableLegacyAPIs := false
legacyAPIFlagValue, ok := s.RuntimeConfig["api/legacy"]
if ok && legacyAPIFlagValue == "false" {
disableLegacyAPIs = true
}
_ = disableLegacyAPIs // hush the compiler while we don't have legacy APIs to disable.
// "api/v1={true|false} allows users to enable/disable v1 API.
// This takes preference over api/all and api/legacy, if specified.
disableV1 := disableAllAPIs
disableV1 = !s.getRuntimeConfigValue("api/v1", !disableV1)
// "experimental/v1={true|false} allows users to enable/disable the experimental API.
// This takes preference over api/all, if specified.
enableExp := s.getRuntimeConfigValue("experimental/v1", false)
clientConfig := &client.Config{
Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
Version: s.StorageVersion,
}
client, err := client.New(clientConfig)
if err != nil {
glog.Fatalf("Invalid server address: %v", err)
}
etcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, latest.InterfacesFor, latest.Version, s.StorageVersion, s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
}
expEtcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, explatest.InterfacesFor, explatest.Version, s.ExpStorageVersion, s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid experimental storage version or misconfigured etcd: %v", err)
}
n := s.ServiceClusterIPRange
// Default to the private server key for service account token signing
if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
if apiserver.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
} else {
glog.Warning("no RSA key provided, service account token authentication disabled")
}
}
authenticator, err := apiserver.NewAuthenticator(s.BasicAuthFile, s.ClientCAFile, s.TokenAuthFile, s.ServiceAccountKeyFile, s.ServiceAccountLookup, etcdStorage, s.KeystoneURL)
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(s.AuthorizationMode, s.AuthorizationPolicyFile)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
if len(s.ExternalHost) == 0 {
// TODO: extend for other providers
if s.CloudProvider == "gce" {
instances, supported := cloud.Instances()
if !supported {
glog.Fatalf("gce cloud provider has no instances. this shouldn't happen. exiting.")
}
name, err := os.Hostname()
if err != nil {
glog.Fatalf("failed to get hostname: %v", err)
}
addrs, err := instances.NodeAddresses(name)
if err != nil {
glog.Warningf("unable to obtain external host address from cloud provider: %v", err)
} else {
for _, addr := range addrs {
if addr.Type == api.NodeExternalIP {
s.ExternalHost = addr.Address
}
}
}
}
}
var installSSH master.InstallSSHKey
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
config := &master.Config{
DatabaseStorage: etcdStorage,
ExpDatabaseStorage: expEtcdStorage,
EventTTL: s.EventTTL,
KubeletClient: kubeletClient,
ServiceClusterIPRange: &n,
EnableCoreControllers: true,
EnableLogsSupport: s.EnableLogsSupport,
EnableUISupport: true,
EnableSwaggerSupport: true,
EnableProfiling: s.EnableProfiling,
EnableIndex: true,
APIPrefix: s.APIPrefix,
ExpAPIPrefix: s.ExpAPIPrefix,
CorsAllowedOriginList: s.CorsAllowedOriginList,
ReadWritePort: s.SecurePort,
PublicAddress: s.AdvertiseAddress,
Authenticator: authenticator,
SupportsBasicAuth: len(s.BasicAuthFile) > 0,
Authorizer: authorizer,
AdmissionControl: admissionController,
DisableV1: disableV1,
EnableExp: enableExp,
MasterServiceNamespace: s.MasterServiceNamespace,
ClusterName: s.ClusterName,
ExternalHost: s.ExternalHost,
MinRequestTimeout: s.MinRequestTimeout,
SSHUser: s.SSHUser,
SSHKeyfile: s.SSHKeyfile,
InstallSSHKey: installSSH,
ServiceNodePortRange: s.ServiceNodePortRange,
}
m := master.New(config)
// We serve on 2 ports. See docs/accessing_the_api.md
secureLocation := ""
if s.SecurePort != 0 {
secureLocation = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort))
}
insecureLocation := net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort))
// See the flag commentary to understand our assumptions when opening the read-only and read-write ports.
var sem chan bool
if s.MaxRequestsInFlight > 0 {
sem = make(chan bool, s.MaxRequestsInFlight)
}
longRunningRE := regexp.MustCompile(s.LongRunningRequestRE)
longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) {
// TODO unify this with apiserver.MaxInFlightLimit
if longRunningRE.MatchString(req.URL.Path) || req.URL.Query().Get("watch") == "true" {
return nil, ""
}
return time.After(time.Minute), ""
}
if secureLocation != "" {
handler := apiserver.TimeoutHandler(m.Handler, longRunningTimeout)
secureServer := &http.Server{
Addr: secureLocation,
Handler: apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(handler)),
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
MinVersion: tls.VersionTLS10,
},
}
if len(s.ClientCAFile) > 0 {
clientCAs, err := util.CertPoolFromFile(s.ClientCAFile)
if err != nil {
glog.Fatalf("unable to load client CA file: %v", err)
}
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
// Specify allowed CAs for client certificates
secureServer.TLSConfig.ClientCAs = clientCAs
}
glog.Infof("Serving securely on %s", secureLocation)
go func() {
defer util.HandleCrash()
for {
if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
s.TLSCertFile = path.Join(s.CertDirectory, "apiserver.crt")
s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "apiserver.key")
// TODO (cjcullen): Is PublicAddress the right address to sign a cert with?
alternateIPs := []net.IP{config.ServiceReadWriteIP}
alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}
// It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil {
glog.Errorf("Unable to generate self signed cert: %v", err)
} else {
glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
}
}
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
if err := secureServer.ListenAndServeTLS(s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
}
handler := apiserver.TimeoutHandler(m.InsecureHandler, longRunningTimeout)
http := &http.Server{
Addr: insecureLocation,
Handler: apiserver.RecoverPanics(handler),
MaxHeaderBytes: 1 << 20,
}
if secureLocation == "" {
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
}
glog.Infof("Serving insecurely on %s", insecureLocation)
glog.Fatal(http.ListenAndServe())
return nil
}
func run(traefikConfiguration *TraefikConfiguration) {
fmtlog.SetFlags(fmtlog.Lshortfile | fmtlog.LstdFlags)
// load global configuration
globalConfiguration := traefikConfiguration.GlobalConfiguration
http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = globalConfiguration.MaxIdleConnsPerHost
if globalConfiguration.InsecureSkipVerify {
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
loggerMiddleware := middlewares.NewLogger(globalConfiguration.AccessLogsFile)
defer loggerMiddleware.Close()
if globalConfiguration.File != nil && len(globalConfiguration.File.Filename) == 0 {
// no filename, setting to global config file
if len(traefikConfiguration.ConfigFile) != 0 {
globalConfiguration.File.Filename = traefikConfiguration.ConfigFile
} else {
log.Errorln("Error using file configuration backend, no filename defined")
}
}
if len(globalConfiguration.EntryPoints) == 0 {
globalConfiguration.EntryPoints = map[string]*EntryPoint{"http": {Address: ":80"}}
globalConfiguration.DefaultEntryPoints = []string{"http"}
}
if globalConfiguration.Debug {
globalConfiguration.LogLevel = "DEBUG"
}
// logging
level, err := logrus.ParseLevel(strings.ToLower(globalConfiguration.LogLevel))
if err != nil {
log.Error("Error getting level", err)
}
log.SetLevel(level)
if len(globalConfiguration.TraefikLogsFile) > 0 {
fi, err := os.OpenFile(globalConfiguration.TraefikLogsFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
defer func() {
if err := fi.Close(); err != nil {
log.Error("Error closing file", err)
}
}()
if err != nil {
log.Error("Error opening file", err)
} else {
log.SetOutput(fi)
log.SetFormatter(&logrus.TextFormatter{DisableColors: true, FullTimestamp: true, DisableSorting: true})
}
} else {
log.SetFormatter(&logrus.TextFormatter{FullTimestamp: true, DisableSorting: true})
}
jsonConf, _ := json.Marshal(globalConfiguration)
log.Infof("Traefik version %s built on %s", version.Version, version.BuildDate)
if globalConfiguration.CheckNewVersion {
ticker := time.NewTicker(24 * time.Hour)
safe.Go(func() {
version.CheckNewVersion()
for {
select {
case <-ticker.C:
version.CheckNewVersion()
}
}
})
}
if len(traefikConfiguration.ConfigFile) != 0 {
log.Infof("Using TOML configuration file %s", traefikConfiguration.ConfigFile)
}
log.Debugf("Global configuration loaded %s", string(jsonConf))
server := NewServer(globalConfiguration)
server.Start()
defer server.Close()
sent, err := daemon.SdNotify("READY=1")
if !sent && err != nil {
log.Error("Fail to notify", err)
}
server.Wait()
log.Info("Shutting down")
}
func StartNode() error {
go daemon.SdNotify("READY=1")
return nil
}
func Main() {
cfg := NewConfig()
err := cfg.Parse(os.Args[1:])
if err != nil {
plog.Errorf("error verifying flags, %v. See 'etcd --help'.", err)
switch err {
case errUnsetAdvertiseClientURLsFlag:
plog.Errorf("When listening on specific address(es), this etcd process must advertise accessible url(s) to each connected client.")
}
os.Exit(1)
}
setupLogging(cfg)
var stopped <-chan struct{}
GoMaxProcs := 1
if envMaxProcs, err := strconv.Atoi(os.Getenv("GOMAXPROCS")); err == nil {
GoMaxProcs = envMaxProcs
}
plog.Infof("setting maximum number of CPUs to %d, total number of available CPUs is %d", GoMaxProcs, runtime.NumCPU())
runtime.GOMAXPROCS(GoMaxProcs)
// TODO: check whether fields are set instead of whether fields have default value
if cfg.name != defaultName && cfg.initialCluster == initialClusterFromName(defaultName) {
cfg.initialCluster = initialClusterFromName(cfg.name)
}
if cfg.dir == "" {
cfg.dir = fmt.Sprintf("%v.etcd", cfg.name)
plog.Warningf("no data-dir provided, using default data-dir ./%s", cfg.dir)
}
which := identifyDataDirOrDie(cfg.dir)
if which != dirEmpty {
plog.Noticef("the server is already initialized as %v before, starting as etcd %v...", which, which)
switch which {
case dirMember:
stopped, err = startEtcd(cfg)
case dirProxy:
err = startProxy(cfg)
default:
plog.Panicf("unhandled dir type %v", which)
}
} else {
shouldProxy := cfg.isProxy()
if !shouldProxy {
stopped, err = startEtcd(cfg)
if err == discovery.ErrFullCluster && cfg.shouldFallbackToProxy() {
plog.Noticef("discovery cluster full, falling back to %s", fallbackFlagProxy)
shouldProxy = true
}
}
if shouldProxy {
err = startProxy(cfg)
}
}
if err != nil {
switch err {
case discovery.ErrDuplicateID:
plog.Errorf("member %q has previously registered with discovery service token (%s).", cfg.name, cfg.durl)
plog.Errorf("But etcd could not find valid cluster configuration in the given data dir (%s).", cfg.dir)
plog.Infof("Please check the given data dir path if the previous bootstrap succeeded")
plog.Infof("or use a new discovery token if the previous bootstrap failed.")
os.Exit(1)
case discovery.ErrDuplicateName:
plog.Errorf("member with duplicated name has registered with discovery service token(%s).", cfg.durl)
plog.Errorf("please check (cURL) the discovery token for more information.")
plog.Errorf("please do not reuse the discovery token and generate a new one to bootstrap the cluster.")
default:
plog.Fatalf("%v", err)
}
}
osutil.HandleInterrupts()
if systemdutil.IsRunningSystemd() {
// At this point, the initialization of etcd is done.
// The listeners are listening on the TCP ports and ready
// for accepting connections.
// The http server is probably ready for serving incoming
// connections. If it is not, the connection might be pending
// for less than one second.
err := daemon.SdNotify("READY=1")
if err != nil {
plog.Errorf("failed to notify systemd for readiness")
}
}
<-stopped
osutil.Exit(0)
}
func (s *GenericAPIServer) Run() {
// install APIs which depend on other APIs to be installed
if s.enableSwaggerSupport {
s.InstallSwaggerAPI()
}
if s.enableOpenAPISupport {
s.InstallOpenAPI()
}
if s.SecureServingInfo != nil && s.Handler != nil {
secureServer := &http.Server{
Addr: s.SecureServingInfo.BindAddress,
Handler: s.Handler,
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
},
}
if len(s.SecureServingInfo.ClientCA) > 0 {
clientCAs, err := certutil.NewPool(s.SecureServingInfo.ClientCA)
if err != nil {
glog.Fatalf("Unable to load client CA file: %v", err)
}
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
// Specify allowed CAs for client certificates
secureServer.TLSConfig.ClientCAs = clientCAs
// "h2" NextProtos is necessary for enabling HTTP2 for go's 1.7 HTTP Server
secureServer.TLSConfig.NextProtos = []string{"h2"}
}
glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
go func() {
defer utilruntime.HandleCrash()
for {
if err := secureServer.ListenAndServeTLS(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile); err != nil {
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
}
if s.InsecureServingInfo != nil && s.InsecureHandler != nil {
insecureServer := &http.Server{
Addr: s.InsecureServingInfo.BindAddress,
Handler: s.InsecureHandler,
MaxHeaderBytes: 1 << 20,
}
glog.Infof("Serving insecurely on %s", s.InsecureServingInfo.BindAddress)
go func() {
defer utilruntime.HandleCrash()
for {
if err := insecureServer.ListenAndServe(); err != nil {
glog.Errorf("Unable to listen for insecure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
}
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) per port
if s.SecureServingInfo != nil {
if err := waitForSuccessfulDial(true, "tcp", s.SecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil {
glog.Fatalf("Secure server never started: %v", err)
}
}
if s.InsecureServingInfo != nil {
if err := waitForSuccessfulDial(false, "tcp", s.InsecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil {
glog.Fatalf("Insecure server never started: %v", err)
}
}
s.RunPostStartHooks()
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
select {}
}
func (s *GenericAPIServer) Run(options *options.ServerRunOptions) {
// install APIs which depend on other APIs to be installed
if s.enableSwaggerSupport {
s.InstallSwaggerAPI()
}
if s.enableOpenAPISupport {
s.InstallOpenAPI()
}
secureStartedCh := make(chan struct{})
if options.SecurePort != 0 {
secureLocation := net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort))
secureServer := &http.Server{
Addr: secureLocation,
Handler: s.Handler,
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
},
}
if len(options.ClientCAFile) > 0 {
clientCAs, err := certutil.NewPool(options.ClientCAFile)
if err != nil {
glog.Fatalf("Unable to load client CA file: %v", err)
}
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
// Specify allowed CAs for client certificates
secureServer.TLSConfig.ClientCAs = clientCAs
// "h2" NextProtos is necessary for enabling HTTP2 for go's 1.7 HTTP Server
secureServer.TLSConfig.NextProtos = []string{"h2"}
}
glog.Infof("Serving securely on %s", secureLocation)
if options.TLSCertFile == "" && options.TLSPrivateKeyFile == "" {
options.TLSCertFile = path.Join(options.CertDirectory, "apiserver.crt")
options.TLSPrivateKeyFile = path.Join(options.CertDirectory, "apiserver.key")
// TODO (cjcullen): Is ClusterIP the right address to sign a cert with?
alternateIPs := []net.IP{s.ServiceReadWriteIP}
alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}
// It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
if !certutil.CanReadCertOrKey(options.TLSCertFile, options.TLSPrivateKeyFile) {
if err := certutil.GenerateSelfSignedCert(s.ClusterIP.String(), options.TLSCertFile, options.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil {
glog.Errorf("Unable to generate self signed cert: %v", err)
} else {
glog.Infof("Using self-signed cert (%s, %s)", options.TLSCertFile, options.TLSPrivateKeyFile)
}
}
}
go func() {
defer utilruntime.HandleCrash()
notifyStarted := sync.Once{}
for {
if err := secureServer.ListenAndServeTLS(options.TLSCertFile, options.TLSPrivateKeyFile); err != nil {
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
} else {
notifyStarted.Do(func() {
close(secureStartedCh)
})
}
time.Sleep(15 * time.Second)
}
}()
} else {
close(secureStartedCh)
}
insecureLocation := net.JoinHostPort(options.InsecureBindAddress.String(), strconv.Itoa(options.InsecurePort))
insecureServer := &http.Server{
Addr: insecureLocation,
Handler: s.InsecureHandler,
MaxHeaderBytes: 1 << 20,
}
insecureStartedCh := make(chan struct{})
glog.Infof("Serving insecurely on %s", insecureLocation)
go func() {
defer utilruntime.HandleCrash()
notifyStarted := sync.Once{}
for {
if err := insecureServer.ListenAndServe(); err != nil {
glog.Errorf("Unable to listen for insecure (%v); will try again.", err)
} else {
notifyStarted.Do(func() {
close(insecureStartedCh)
})
}
time.Sleep(15 * time.Second)
}
}()
<-secureStartedCh
<-insecureStartedCh
s.RunPostStartHooks(PostStartHookContext{})
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
select {}
}
// notifySystem sends a message to the host when the server is ready to be used
func notifySystem() {
// Tell the init daemon we are accepting requests
go systemdDaemon.SdNotify("READY=1")
}
// Run runs the specified APIServer. This should never exit.
func (s *APIServer) Run(_ []string) error {
s.verifyClusterIPFlags()
// If advertise-address is not specified, use bind-address. If bind-address
// is also unset (or 0.0.0.0), setDefaults() in pkg/master/master.go will
// do the right thing and use the host's default interface.
if s.AdvertiseAddress == nil || net.IP(s.AdvertiseAddress).IsUnspecified() {
s.AdvertiseAddress = s.BindAddress
}
if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) {
glog.Fatalf("specify either --etcd-servers or --etcd-config")
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
HostNetworkSources: []string{},
})
cloud := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
qingletClient, err := client.NewQingletClient(&s.QingletConfig)
if err != nil {
glog.Fatalf("Failure to start qinglet client: %v", err)
}
// "api/all=false" allows users to selectively enable specific api versions.
disableAllAPIs := false
allAPIFlagValue, ok := s.RuntimeConfig["api/all"]
if ok && allAPIFlagValue == "false" {
disableAllAPIs = true
}
// "api/legacy=false" allows users to disable legacy api versions.
// Right now, v1beta1 and v1beta2 are considered legacy.
disableLegacyAPIs := false
legacyAPIFlagValue, ok := s.RuntimeConfig["api/legacy"]
if ok && legacyAPIFlagValue == "false" {
disableLegacyAPIs = true
}
_ = disableLegacyAPIs // hush the compiler while we don't have legacy APIs to disable.
// "api/v1beta3={true|false} allows users to enable/disable v1beta3 API.
// This takes preference over api/all and api/legacy, if specified.
disableV1beta3 := disableAllAPIs
disableV1beta3 = !s.getRuntimeConfigValue("api/v1beta3", !disableV1beta3)
// "api/v1={true|false} allows users to enable/disable v1 API.
// This takes preference over api/all and api/legacy, if specified.
disableV1 := disableAllAPIs
disableV1 = !s.getRuntimeConfigValue("api/v1", !disableV1)
// TODO: expose same flags as client.BindClientConfigFlags but for a server
clientConfig := &client.Config{
Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
Version: s.StorageVersion,
}
client, err := client.New(clientConfig)
if err != nil {
glog.Fatalf("Invalid server address: %v", err)
}
helper, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, s.StorageVersion, s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
}
// TODO Is this the right place for migration to happen? Must *both* old and
// new etcd prefix params be supplied for this to be valid?
if s.OldEtcdPathPrefix != "" {
if err = helper.MigrateKeys(s.OldEtcdPathPrefix); err != nil {
glog.Fatalf("Migration of old etcd keys failed: %v", err)
}
}
n := net.IPNet(s.ServiceClusterIPRange)
// Default to the private server key for service account token signing
if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
if apiserver.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
} else {
glog.Warning("no RSA key provided, service account token authentication disabled")
}
}
authenticator, err := apiserver.NewAuthenticator(s.BasicAuthFile, s.ClientCAFile, s.TokenAuthFile, s.ServiceAccountKeyFile, s.ServiceAccountLookup, helper)
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(s.AuthorizationMode, s.AuthorizationPolicyFile)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
if len(s.ExternalHost) == 0 {
// TODO: extend for other providers
if s.CloudProvider == "gce" {
instances, supported := cloud.Instances()
if !supported {
glog.Fatalf("gce cloud provider has no instances. this shouldn't happen. exiting.")
}
name, err := os.Hostname()
if err != nil {
glog.Fatalf("failed to get hostname: %v", err)
}
addrs, err := instances.NodeAddresses(name)
if err != nil {
glog.Warningf("unable to obtain external host address from cloud provider: %v", err)
} else {
for _, addr := range addrs {
if addr.Type == api.NodeExternalIP {
s.ExternalHost = addr.Address
}
}
}
}
}
var installSSH master.InstallSSHKey
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
config := &master.Config{
EtcdHelper: helper,
EventTTL: s.EventTTL,
QingletClient: qingletClient,
ServiceClusterIPRange: &n,
EnableCoreControllers: true,
EnableLogsSupport: s.EnableLogsSupport,
EnableUISupport: true,
EnableSwaggerSupport: true,
EnableProfiling: s.EnableProfiling,
EnableIndex: true,
APIPrefix: s.APIPrefix,
CorsAllowedOriginList: s.CorsAllowedOriginList,
ReadWritePort: s.SecurePort,
PublicAddress: net.IP(s.AdvertiseAddress),
Authenticator: authenticator,
SupportsBasicAuth: len(s.BasicAuthFile) > 0,
Authorizer: authorizer,
AdmissionControl: admissionController,
DisableV1Beta3: disableV1beta3,
DisableV1: disableV1,
MasterServiceNamespace: s.MasterServiceNamespace,
ClusterName: s.ClusterName,
ExternalHost: s.ExternalHost,
MinRequestTimeout: s.MinRequestTimeout,
SSHUser: s.SSHUser,
SSHKeyfile: s.SSHKeyfile,
InstallSSHKey: installSSH,
ServiceNodePortRange: s.ServiceNodePortRange,
}
m := master.New(config)
// We serve on 2 ports. See docs/accessing_the_api.md
secureLocation := ""
if s.SecurePort != 0 {
secureLocation = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort))
}
insecureLocation := net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort))
// See the flag commentary to understand our assumptions when opening the read-only and read-write ports.
var sem chan bool
if s.MaxRequestsInFlight > 0 {
sem = make(chan bool, s.MaxRequestsInFlight)
}
longRunningRE := regexp.MustCompile(s.LongRunningRequestRE)
if secureLocation != "" {
secureServer := &http.Server{
Addr: secureLocation,
Handler: apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(m.Handler)),
ReadTimeout: ReadWriteTimeout,
WriteTimeout: ReadWriteTimeout,
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
MinVersion: tls.VersionTLS10,
},
}
if len(s.ClientCAFile) > 0 {
clientCAs, err := util.CertPoolFromFile(s.ClientCAFile)
if err != nil {
glog.Fatalf("unable to load client CA file: %v", err)
}
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
// Specify allowed CAs for client certificates
secureServer.TLSConfig.ClientCAs = clientCAs
}
glog.Infof("Serving securely on %s", secureLocation)
go func() {
defer util.HandleCrash()
for {
if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
s.TLSCertFile = path.Join(s.CertDirectory, "apiserver.crt")
s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "apiserver.key")
// TODO (cjcullen): Is PublicAddress the right address to sign a cert with?
if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
glog.Errorf("Unable to generate self signed cert: %v", err)
} else {
glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
}
}
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon sucessful start message: %v\n", err)
}
if err := secureServer.ListenAndServeTLS(s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
}
http := &http.Server{
Addr: insecureLocation,
Handler: apiserver.RecoverPanics(m.InsecureHandler),
ReadTimeout: ReadWriteTimeout,
WriteTimeout: ReadWriteTimeout,
MaxHeaderBytes: 1 << 20,
}
if secureLocation == "" {
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon sucessful start message: %v\n", err)
}
}
glog.Infof("Serving insecurely on %s", insecureLocation)
glog.Fatal(http.ListenAndServe())
return nil
}
