// RunEtcd starts an etcd server and runs it forever
func RunEtcd(etcdServerConfig *configapi.EtcdConfig) {
cfg := &config{
name: defaultName,
dir: etcdServerConfig.StorageDir,
TickMs: 100,
ElectionMs: 1000,
maxSnapFiles: 5,
maxWalFiles: 5,
initialClusterToken: "etcd-cluster",
}
var err error
if configapi.UseTLS(etcdServerConfig.ServingInfo) {
cfg.clientTLSInfo.CAFile = etcdServerConfig.ServingInfo.ClientCA
cfg.clientTLSInfo.CertFile = etcdServerConfig.ServingInfo.ServerCert.CertFile
cfg.clientTLSInfo.KeyFile = etcdServerConfig.ServingInfo.ServerCert.KeyFile
}
if cfg.lcurls, err = urlsFromStrings(etcdServerConfig.ServingInfo.BindAddress, cfg.clientTLSInfo); err != nil {
glog.Fatalf("Unable to build etcd client URLs: %v", err)
}
if configapi.UseTLS(etcdServerConfig.PeerServingInfo) {
cfg.peerTLSInfo.CAFile = etcdServerConfig.PeerServingInfo.ClientCA
cfg.peerTLSInfo.CertFile = etcdServerConfig.PeerServingInfo.ServerCert.CertFile
cfg.peerTLSInfo.KeyFile = etcdServerConfig.PeerServingInfo.ServerCert.KeyFile
}
if cfg.lpurls, err = urlsFromStrings(etcdServerConfig.PeerServingInfo.BindAddress, cfg.peerTLSInfo); err != nil {
glog.Fatalf("Unable to build etcd peer URLs: %v", err)
}
if cfg.acurls, err = urlsFromStrings(etcdServerConfig.Address, cfg.clientTLSInfo); err != nil {
glog.Fatalf("Unable to build etcd announce client URLs: %v", err)
}
if cfg.apurls, err = urlsFromStrings(etcdServerConfig.PeerAddress, cfg.peerTLSInfo); err != nil {
glog.Fatalf("Unable to build etcd announce peer URLs: %v", err)
}
if err := cfg.resolveUrls(); err != nil {
glog.Fatalf("Unable to resolve etcd URLs: %v", err)
}
cfg.initialCluster = fmt.Sprintf("%s=%s", cfg.name, cfg.apurls[0].String())
stopped, err := startEtcd(cfg)
if err != nil {
glog.Fatalf("Unable to start etcd: %v", err)
}
go func() {
glog.Infof("Started etcd at %s", etcdServerConfig.Address)
<-stopped
}()
}
// Run starts an http server for the static assets listening on the configured
// bind address
func (c *AssetConfig) Run() {
if !c.OpenshiftEnabled {
return
}
assetHandler, err := c.buildHandler()
if err != nil {
glog.Fatal(err)
}
publicURL, err := url.Parse(c.Options.PublicURL)
if err != nil {
glog.Fatal(err)
}
mux := http.NewServeMux()
mux.Handle(publicURL.Path, http.StripPrefix(publicURL.Path, assetHandler))
if publicURL.Path != "/" {
mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
http.Redirect(w, req, publicURL.Path, http.StatusFound)
})
}
timeout := c.Options.ServingInfo.RequestTimeoutSeconds
if timeout == -1 {
timeout = 0
}
server := &http.Server{
Addr: c.Options.ServingInfo.BindAddress,
Handler: mux,
ReadTimeout: time.Duration(timeout) * time.Second,
WriteTimeout: time.Duration(timeout) * time.Second,
MaxHeaderBytes: 1 << 20,
}
isTLS := configapi.UseTLS(c.Options.ServingInfo.ServingInfo)
go util.Forever(func() {
if isTLS {
server.TLSConfig = &tls.Config{
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
MinVersion: tls.VersionTLS10,
}
glog.Infof("OpenShift UI listening at https://%s", c.Options.ServingInfo.BindAddress)
glog.Fatal(server.ListenAndServeTLS(c.Options.ServingInfo.ServerCert.CertFile, c.Options.ServingInfo.ServerCert.KeyFile))
} else {
glog.Infof("OpenShift UI listening at http://%s", c.Options.ServingInfo.BindAddress)
glog.Fatal(server.ListenAndServe())
}
}, 0)
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try)
cmdutil.WaitForSuccessfulDial(isTLS, "tcp", c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
glog.Infof("OpenShift UI available at %s", c.Options.PublicURL)
}
func newAuthenticator(config configapi.MasterConfig, etcdHelper tools.EtcdHelper, tokenGetter serviceaccount.ServiceAccountTokenGetter, apiClientCAs *x509.CertPool) authenticator.Request {
authenticators := []authenticator.Request{}
// ServiceAccount token
if len(config.ServiceAccountConfig.PublicKeyFiles) > 0 {
publicKeys := []*rsa.PublicKey{}
for _, keyFile := range config.ServiceAccountConfig.PublicKeyFiles {
publicKey, err := serviceaccount.ReadPublicKey(keyFile)
if err != nil {
glog.Fatalf("Error reading service account key file %s: %v", keyFile, err)
}
publicKeys = append(publicKeys, publicKey)
}
tokenAuthenticator := serviceaccount.JWTTokenAuthenticator(publicKeys, true, tokenGetter)
authenticators = append(authenticators, bearertoken.New(tokenAuthenticator, true))
}
// OAuth token
if config.OAuthConfig != nil {
tokenAuthenticator := getEtcdTokenAuthenticator(etcdHelper)
authenticators = append(authenticators, bearertoken.New(tokenAuthenticator, true))
// Allow token as access_token param for WebSockets
authenticators = append(authenticators, paramtoken.New("access_token", tokenAuthenticator, true))
}
if configapi.UseTLS(config.ServingInfo.ServingInfo) {
// build cert authenticator
// TODO: add "system:" prefix in authenticator, limit cert to username
// TODO: add "system:" prefix to groups in authenticator, limit cert to group name
opts := x509request.DefaultVerifyOptions()
opts.Roots = apiClientCAs
certauth := x509request.New(opts, x509request.SubjectToUserConversion)
authenticators = append(authenticators, certauth)
}
// TODO: make anonymous auth optional?
ret := &unionrequest.Authenticator{
FailOnError: true,
Handlers: []authenticator.Request{
group.NewGroupAdder(unionrequest.NewUnionAuthentication(authenticators...), []string{bootstrappolicy.AuthenticatedGroup}),
authenticator.RequestFunc(func(req *http.Request) (user.Info, bool, error) {
return &user.DefaultInfo{Name: unauthenticatedUsername, Groups: []string{bootstrappolicy.UnauthenticatedGroup}}, true, nil
}),
},
}
return ret
}
// ValidateEtcdConnectionInfo validates the connection info. If a server EtcdConfig is provided,
// it ensures the connection info includes a URL for it, and has a client cert/key if the server requires
// client certificate authentication
func ValidateEtcdConnectionInfo(config api.EtcdConnectionInfo, server *api.EtcdConfig) fielderrors.ValidationErrorList {
allErrs := fielderrors.ValidationErrorList{}
if len(config.URLs) == 0 {
allErrs = append(allErrs, fielderrors.NewFieldRequired("urls"))
}
for i, u := range config.URLs {
_, urlErrs := ValidateURL(u, fmt.Sprintf("urls[%d]", i))
if len(urlErrs) > 0 {
allErrs = append(allErrs, urlErrs...)
}
}
if len(config.CA) > 0 {
allErrs = append(allErrs, ValidateFile(config.CA, "ca")...)
}
allErrs = append(allErrs, ValidateCertInfo(config.ClientCert, false)...)
// If we have server config info, make sure the client connection info will work with it
if server != nil {
var builtInAddress string
if api.UseTLS(server.ServingInfo) {
builtInAddress = fmt.Sprintf("https://%s", server.Address)
} else {
builtInAddress = fmt.Sprintf("http://%s", server.Address)
}
// Require a client cert to connect to an etcd that requires client certs
if len(server.ServingInfo.ClientCA) > 0 {
if len(config.ClientCert.CertFile) == 0 {
allErrs = append(allErrs, fielderrors.NewFieldRequired("certFile"))
}
}
// Require the etcdClientInfo to include the address of the internal etcd
clientURLs := util.NewStringSet(config.URLs...)
if !clientURLs.Has(builtInAddress) {
allErrs = append(allErrs, fielderrors.NewFieldInvalid("urls", strings.Join(clientURLs.List(), ","), fmt.Sprintf("must include the etcd address %s", builtInAddress)))
}
}
return allErrs
}
func BuildKubernetesNodeConfig(options configapi.NodeConfig) (*NodeConfig, error) {
kubeClient, _, err := configapi.GetKubeClient(options.MasterKubeConfig)
if err != nil {
return nil, err
}
if options.NodeName == "localhost" {
glog.Warningf(`Using "localhost" as node name will not resolve from all locations`)
}
var dnsIP net.IP
if len(options.DNSIP) > 0 {
dnsIP = net.ParseIP(options.DNSIP)
if dnsIP == nil {
return nil, fmt.Errorf("Invalid DNS IP: %s", options.DNSIP)
}
}
clientCAs, err := util.CertPoolFromFile(options.ServingInfo.ClientCA)
if err != nil {
return nil, err
}
imageTemplate := variable.NewDefaultImageTemplate()
imageTemplate.Format = options.ImageConfig.Format
imageTemplate.Latest = options.ImageConfig.Latest
var path string
var fileCheckInterval int64
if options.PodManifestConfig != nil {
path = options.PodManifestConfig.Path
fileCheckInterval = options.PodManifestConfig.FileCheckIntervalSeconds
}
var dockerExecHandler dockertools.ExecHandler
switch options.DockerConfig.ExecHandlerName {
case configapi.DockerExecHandlerNative:
dockerExecHandler = &dockertools.NativeExecHandler{}
case configapi.DockerExecHandlerNsenter:
dockerExecHandler = &dockertools.NsenterExecHandler{}
}
kubeAddress, kubePortStr, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, fmt.Errorf("cannot parse node address: %v", err)
}
kubePort, err := strconv.Atoi(kubePortStr)
if err != nil {
return nil, fmt.Errorf("cannot parse node port: %v", err)
}
address := util.IP{}
if err := address.Set(kubeAddress); err != nil {
return nil, err
}
// declare the OpenShift defaults from config
server := kapp.NewKubeletServer()
server.Config = path
server.RootDirectory = options.VolumeDirectory
server.HostnameOverride = options.NodeName
server.AllowPrivileged = true
server.RegisterNode = true
server.Address = address
server.Port = uint(kubePort)
server.ReadOnlyPort = 0 // no read only access
server.ClusterDNS = util.IP(dnsIP)
server.ClusterDomain = options.DNSDomain
server.NetworkPluginName = options.NetworkPluginName
server.HostNetworkSources = strings.Join([]string{kubelet.ApiserverSource, kubelet.FileSource}, ",")
server.HTTPCheckFrequency = 0 // no remote HTTP pod creation access
server.FileCheckFrequency = time.Duration(fileCheckInterval) * time.Second
server.PodInfraContainerImage = imageTemplate.ExpandOrDie("pod")
// prevents kube from generating certs
server.TLSCertFile = options.ServingInfo.ServerCert.CertFile
server.TLSPrivateKeyFile = options.ServingInfo.ServerCert.KeyFile
if value := cmdutil.Env("OPENSHIFT_CONTAINERIZED", ""); len(value) > 0 {
server.Containerized = value == "true"
}
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubeletArguments, server.AddFlags); len(err) > 0 {
return nil, errors.NewAggregate(err)
}
cfg, err := server.KubeletConfig()
if err != nil {
return nil, err
}
// provide any config overrides
cfg.StreamingConnectionIdleTimeout = 5 * time.Minute // TODO: should be set
cfg.KubeClient = kubeClient
cfg.DockerExecHandler = dockerExecHandler
// TODO: could be cleaner
if configapi.UseTLS(options.ServingInfo) {
cfg.TLSOptions = &kubelet.TLSOptions{
Config: &tls.Config{
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
MinVersion: tls.VersionTLS10,
// RequireAndVerifyClientCert lets us limit requests to ones with a valid client certificate
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: clientCAs,
},
CertFile: options.ServingInfo.ServerCert.CertFile,
KeyFile: options.ServingInfo.ServerCert.KeyFile,
}
} else {
cfg.TLSOptions = nil
}
config := &NodeConfig{
BindAddress: options.ServingInfo.BindAddress,
AllowDisabledDocker: options.AllowDisabledDocker,
Client: kubeClient,
VolumeDir: options.VolumeDirectory,
KubeletServer: server,
KubeletConfig: cfg,
}
return config, nil
}
func BuildMasterConfig(options configapi.MasterConfig) (*MasterConfig, error) {
client, err := etcd.GetAndTestEtcdClient(options.EtcdClientInfo)
if err != nil {
return nil, err
}
etcdHelper, err := NewEtcdHelper(client, options.EtcdStorageConfig.OpenShiftStorageVersion, options.EtcdStorageConfig.OpenShiftStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up server storage: %v", err)
}
clientCAs, err := configapi.GetClientCertCAPool(options)
if err != nil {
return nil, err
}
apiClientCAs, err := configapi.GetAPIClientCertCAPool(options)
if err != nil {
return nil, err
}
privilegedLoopbackKubeClient, _, err := configapi.GetKubeClient(options.MasterClients.OpenShiftLoopbackKubeConfig)
if err != nil {
return nil, err
}
privilegedLoopbackOpenShiftClient, privilegedLoopbackClientConfig, err := configapi.GetOpenShiftClient(options.MasterClients.OpenShiftLoopbackKubeConfig)
if err != nil {
return nil, err
}
imageTemplate := variable.NewDefaultImageTemplate()
imageTemplate.Format = options.ImageConfig.Format
imageTemplate.Latest = options.ImageConfig.Latest
policyCache, policyClient := newReadOnlyCacheAndClient(etcdHelper)
requestContextMapper := kapi.NewRequestContextMapper()
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
// in-order list of plug-ins that should intercept admission decisions (origin only intercepts)
admissionControlPluginNames := []string{"OriginNamespaceLifecycle", "BuildByStrategy"}
admissionClient := admissionControlClient(privilegedLoopbackKubeClient, privilegedLoopbackOpenShiftClient)
admissionController := admission.NewFromPlugins(admissionClient, admissionControlPluginNames, "")
serviceAccountTokenGetter, err := newServiceAccountTokenGetter(options, client)
if err != nil {
return nil, err
}
config := &MasterConfig{
Options: options,
OpenshiftEnabled: options.OpenshiftEnabled,
Authenticator: newAuthenticator(options, etcdHelper, serviceAccountTokenGetter, apiClientCAs),
Authorizer: newAuthorizer(policyClient, options.ProjectConfig.ProjectRequestMessage),
AuthorizationAttributeBuilder: newAuthorizationAttributeBuilder(requestContextMapper),
PolicyCache: policyCache,
ProjectAuthorizationCache: newProjectAuthorizationCache(privilegedLoopbackOpenShiftClient, privilegedLoopbackKubeClient, policyClient),
RequestContextMapper: requestContextMapper,
AdmissionControl: admissionController,
TLS: configapi.UseTLS(options.ServingInfo.ServingInfo),
ControllerPlug: plug.NewPlug(!options.PauseControllers),
ImageFor: imageTemplate.ExpandOrDie,
EtcdHelper: etcdHelper,
KubeletClientConfig: kubeletClientConfig,
ClientCAs: clientCAs,
APIClientCAs: apiClientCAs,
PrivilegedLoopbackClientConfig: *privilegedLoopbackClientConfig,
PrivilegedLoopbackOpenShiftClient: privilegedLoopbackOpenShiftClient,
PrivilegedLoopbackKubernetesClient: privilegedLoopbackKubeClient,
BuildControllerServiceAccount: bootstrappolicy.InfraBuildControllerServiceAccountName,
DeploymentControllerServiceAccount: bootstrappolicy.InfraDeploymentControllerServiceAccountName,
ReplicationControllerServiceAccount: bootstrappolicy.InfraReplicationControllerServiceAccountName,
}
return config, nil
}