// startServiceAccountTestServer returns a started server
// It is the responsibility of the caller to ensure the returned stopFunc is called
func startServiceAccountTestServer(t *testing.T) (*clientset.Clientset, restclient.Config, func()) {
deleteAllEtcdKeys()
// Listener
var m *master.Master
apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
// Anonymous client config
clientConfig := restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}}
// Root client
// TODO: remove rootClient after we refactor pkg/admission to use the clientset.
rootClientset := clientset.NewForConfigOrDie(&restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}, BearerToken: rootToken})
// Set up two authenticators:
// 1. A token authenticator that maps the rootToken to the "root" user
// 2. A ServiceAccountToken authenticator that validates ServiceAccount tokens
rootTokenAuth := authenticator.TokenFunc(func(token string) (user.Info, bool, error) {
if token == rootToken {
return &user.DefaultInfo{rootUserName, "", []string{}}, true, nil
}
return nil, false, nil
})
serviceAccountKey, _ := rsa.GenerateKey(rand.Reader, 2048)
serviceAccountTokenGetter := serviceaccountcontroller.NewGetterFromClient(rootClientset)
serviceAccountTokenAuth := serviceaccount.JWTTokenAuthenticator([]*rsa.PublicKey{&serviceAccountKey.PublicKey}, true, serviceAccountTokenGetter)
authenticator := union.New(
bearertoken.New(rootTokenAuth),
bearertoken.New(serviceAccountTokenAuth),
)
// Set up a stub authorizer:
// 1. The "root" user is allowed to do anything
// 2. ServiceAccounts named "ro" are allowed read-only operations in their namespace
// 3. ServiceAccounts named "rw" are allowed any operation in their namespace
authorizer := authorizer.AuthorizerFunc(func(attrs authorizer.Attributes) error {
username := attrs.GetUserName()
ns := attrs.GetNamespace()
// If the user is "root"...
if username == rootUserName {
// allow them to do anything
return nil
}
// If the user is a service account...
if serviceAccountNamespace, serviceAccountName, err := serviceaccount.SplitUsername(username); err == nil {
// Limit them to their own namespace
if serviceAccountNamespace == ns {
switch serviceAccountName {
case readOnlyServiceAccountName:
if attrs.IsReadOnly() {
return nil
}
case readWriteServiceAccountName:
return nil
}
}
}
return fmt.Errorf("User %s is denied (ns=%s, readonly=%v, resource=%s)", username, ns, attrs.IsReadOnly(), attrs.GetResource())
})
// Set up admission plugin to auto-assign serviceaccounts to pods
serviceAccountAdmission := serviceaccountadmission.NewServiceAccount(rootClientset)
masterConfig := framework.NewMasterConfig()
masterConfig.EnableIndex = true
masterConfig.Authenticator = authenticator
masterConfig.Authorizer = authorizer
masterConfig.AdmissionControl = serviceAccountAdmission
// Create a master and install handlers into mux.
m, err := master.New(masterConfig)
if err != nil {
t.Fatalf("Error in bringing up the master: %v", err)
}
// Start the service account and service account token controllers
tokenController := serviceaccountcontroller.NewTokensController(rootClientset, serviceaccountcontroller.TokensControllerOptions{TokenGenerator: serviceaccount.JWTTokenGenerator(serviceAccountKey)})
tokenController.Run()
serviceAccountController := serviceaccountcontroller.NewServiceAccountsController(rootClientset, serviceaccountcontroller.DefaultServiceAccountsControllerOptions())
serviceAccountController.Run()
// Start the admission plugin reflectors
serviceAccountAdmission.Run()
stop := func() {
tokenController.Stop()
serviceAccountController.Stop()
serviceAccountAdmission.Stop()
// TODO: Uncomment when fix #19254
// apiServer.Close()
}
return rootClientset, clientConfig, stop
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
// Connect and setup etcd interfaces
etcdClient, err := etcd.EtcdClient(options.EtcdClientInfo)
if err != nil {
return nil, err
}
databaseStorage, err := master.NewEtcdStorage(etcdClient, kapilatest.InterfacesFor, options.EtcdStorageConfig.KubernetesStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err)
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kclient.NewKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, err
}
portRange, err := util.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, err
}
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
server := app.NewAPIServer()
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.AdmissionControl = strings.Join(AdmissionPlugins, ",")
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cmserver := cmapp.NewCMServer()
cmserver.PodEvictionTimeout = podEvictionTimeout
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
plugins := []admission.Interface{}
for _, pluginName := range strings.Split(server.AdmissionControl, ",") {
switch pluginName {
case saadmit.PluginName:
// we need to set some custom parameters on the service account admission controller, so create that one by hand
saAdmitter := saadmit.NewServiceAccount(kubeClient)
saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences
saAdmitter.Run()
plugins = append(plugins, saAdmitter)
default:
plugin := admission.InitPlugin(pluginName, kubeClient, server.AdmissionControlConfigFile)
if plugin != nil {
plugins = append(plugins, plugin)
}
}
}
admissionController := admission.NewChainHandler(plugins...)
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
m := &master.Config{
PublicAddress: net.ParseIP(options.KubernetesMasterConfig.MasterIP),
ReadWritePort: port,
DatabaseStorage: databaseStorage,
ExpDatabaseStorage: databaseStorage,
EventTTL: server.EventTTL,
//MinRequestTimeout: server.MinRequestTimeout,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
KubeletClient: kubeletClient,
APIPrefix: KubeAPIPrefix,
EnableCoreControllers: true,
MasterCount: options.KubernetesMasterConfig.MasterCount,
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionController,
EnableV1Beta3: configapi.HasKubernetesAPILevel(*options.KubernetesMasterConfig, "v1beta3"),
DisableV1: !configapi.HasKubernetesAPILevel(*options.KubernetesMasterConfig, "v1"),
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
}
return kmaster, nil
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
// Connect and setup etcd interfaces
etcdClient, err := etcd.EtcdClient(options.EtcdClientInfo)
if err != nil {
return nil, err
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kclient.NewKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, err
}
portRange, err := util.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, err
}
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
server := app.NewAPIServer()
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.AdmissionControl = strings.Join(AdmissionPlugins, ",")
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cmserver := cmapp.NewCMServer()
cmserver.PodEvictionTimeout = podEvictionTimeout
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile)
}
plugins := []admission.Interface{}
for _, pluginName := range strings.Split(server.AdmissionControl, ",") {
switch pluginName {
case saadmit.PluginName:
// we need to set some custom parameters on the service account admission controller, so create that one by hand
saAdmitter := saadmit.NewServiceAccount(kubeClient)
saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences
saAdmitter.Run()
plugins = append(plugins, saAdmitter)
default:
plugin := admission.InitPlugin(pluginName, kubeClient, server.AdmissionControlConfigFile)
if plugin != nil {
plugins = append(plugins, plugin)
}
}
}
admissionController := admission.NewChainHandler(plugins...)
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
// TODO you have to know every APIGroup you're enabling or upstream will panic. It's alternative to panicing is Fataling
// It needs a refactor to return errors
storageDestinations := master.NewStorageDestinations()
// storageVersions is a map from API group to allowed versions that must be a version exposed by the REST API or it breaks.
// We need to fix the upstream to stop using the storage version as a preferred api version.
storageVersions := map[string]string{}
enabledKubeVersions := configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupKube)
enabledKubeVersionSet := sets.NewString(enabledKubeVersions...)
if len(enabledKubeVersions) > 0 {
databaseStorage, err := master.NewEtcdStorage(etcdClient, kapilatest.InterfacesForLegacyGroup, options.EtcdStorageConfig.KubernetesStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupKube, databaseStorage)
storageVersions[configapi.APIGroupKube] = options.EtcdStorageConfig.KubernetesStorageVersion
}
enabledExtensionsVersions := configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupExtensions)
if len(enabledExtensionsVersions) > 0 {
groupMeta, err := kapilatest.Group(configapi.APIGroupExtensions)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes extensions server storage: %v", err)
}
// TODO expose storage version options for api groups
databaseStorage, err := master.NewEtcdStorage(etcdClient, groupMeta.InterfacesFor, groupMeta.GroupVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes extensions server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupExtensions, databaseStorage)
storageVersions[configapi.APIGroupExtensions] = enabledExtensionsVersions[0]
}
m := &master.Config{
PublicAddress: net.ParseIP(options.KubernetesMasterConfig.MasterIP),
ReadWritePort: port,
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
EventTTL: server.EventTTL,
//MinRequestTimeout: server.MinRequestTimeout,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
KubeletClient: kubeletClient,
APIPrefix: KubeAPIPrefix,
APIGroupPrefix: KubeAPIGroupPrefix,
EnableCoreControllers: true,
MasterCount: options.KubernetesMasterConfig.MasterCount,
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionController,
EnableExp: len(enabledExtensionsVersions) > 0,
DisableV1: !enabledKubeVersionSet.Has("v1"),
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
}
// set for consistency -- Origin only used m.EnableExp
cmserver.EnableExperimental = m.EnableExp
if options.DNSConfig != nil {
_, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress)
if err != nil {
return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err)
}
dnsPort, err := strconv.Atoi(dnsPortStr)
if err != nil {
return nil, fmt.Errorf("invalid DNS port: %v", err)
}
m.ExtraServicePorts = append(m.ExtraServicePorts,
kapi.ServicePort{Name: "dns", Port: dnsPort, Protocol: kapi.ProtocolUDP, TargetPort: util.NewIntOrStringFromInt(dnsPort)},
kapi.ServicePort{Name: "dns-tcp", Port: dnsPort, Protocol: kapi.ProtocolTCP, TargetPort: util.NewIntOrStringFromInt(dnsPort)},
)
m.ExtraEndpointPorts = append(m.ExtraEndpointPorts,
kapi.EndpointPort{Name: "dns", Port: dnsPort, Protocol: kapi.ProtocolUDP},
kapi.EndpointPort{Name: "dns-tcp", Port: dnsPort, Protocol: kapi.ProtocolTCP},
)
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
}
return kmaster, nil
}
// startServiceAccountTestServer returns a started server
// It is the responsibility of the caller to ensure the returned stopFunc is called
func startServiceAccountTestServer(t *testing.T) (*client.Client, client.Config, func()) {
deleteAllEtcdKeys()
// Etcd
etcdStorage, err := framework.NewEtcdStorage()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expEtcdStorage, err := framework.NewExtensionsEtcdStorage(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
storageDestinations := master.NewStorageDestinations()
storageDestinations.AddAPIGroup("", etcdStorage)
storageDestinations.AddAPIGroup("extensions", expEtcdStorage)
storageVersions := make(map[string]string)
storageVersions[""] = testapi.Default.Version()
storageVersions["extensions"] = testapi.Extensions.GroupAndVersion()
// Listener
var m *master.Master
apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
// Anonymous client config
clientConfig := client.Config{Host: apiServer.URL, Version: testapi.Default.Version()}
// Root client
rootClient := client.NewOrDie(&client.Config{Host: apiServer.URL, Version: testapi.Default.Version(), BearerToken: rootToken})
// Set up two authenticators:
// 1. A token authenticator that maps the rootToken to the "root" user
// 2. A ServiceAccountToken authenticator that validates ServiceAccount tokens
rootTokenAuth := authenticator.TokenFunc(func(token string) (user.Info, bool, error) {
if token == rootToken {
return &user.DefaultInfo{rootUserName, "", "", "", []string{}, ""}, true, nil
}
return nil, false, nil
})
serviceAccountKey, err := rsa.GenerateKey(rand.Reader, 2048)
serviceAccountTokenGetter := serviceaccount.NewGetterFromClient(rootClient)
serviceAccountTokenAuth := serviceaccount.JWTTokenAuthenticator([]*rsa.PublicKey{&serviceAccountKey.PublicKey}, true, serviceAccountTokenGetter)
authenticator := union.New(
bearertoken.New(rootTokenAuth),
bearertoken.New(serviceAccountTokenAuth),
)
// Set up a stub authorizer:
// 1. The "root" user is allowed to do anything
// 2. ServiceAccounts named "ro" are allowed read-only operations in their namespace
// 3. ServiceAccounts named "rw" are allowed any operation in their namespace
authorizer := authorizer.AuthorizerFunc(func(attrs authorizer.Attributes) (string, error) {
username := attrs.GetUserName()
ns := attrs.GetNamespace()
// If the user is "root"...
if username == rootUserName {
// allow them to do anything
return "", nil
}
// If the user is a service account...
if serviceAccountNamespace, serviceAccountName, err := serviceaccount.SplitUsername(username); err == nil {
// Limit them to their own namespace
if serviceAccountNamespace == ns {
switch serviceAccountName {
case readOnlyServiceAccountName:
if attrs.IsReadOnly() {
return "", nil
}
case readWriteServiceAccountName:
return "", nil
}
}
}
return "", fmt.Errorf("User %s is denied (ns=%s, readonly=%v, resource=%s)", username, ns, attrs.IsReadOnly(), attrs.GetResource())
})
// Set up admission plugin to auto-assign serviceaccounts to pods
serviceAccountAdmission := serviceaccountadmission.NewServiceAccount(rootClient)
// Create a master and install handlers into mux.
m = master.New(&master.Config{
StorageDestinations: storageDestinations,
KubeletClient: client.FakeKubeletClient{},
EnableLogsSupport: false,
EnableUISupport: false,
EnableIndex: true,
APIPrefix: "/api",
Authenticator: authenticator,
Authorizer: authorizer,
AdmissionControl: serviceAccountAdmission,
StorageVersions: storageVersions,
})
// Start the service account and service account token controllers
tokenController := serviceaccount.NewTokensController(rootClient, serviceaccount.TokensControllerOptions{TokenGenerator: serviceaccount.JWTTokenGenerator(serviceAccountKey)})
tokenController.Run()
serviceAccountController := serviceaccount.NewServiceAccountsController(rootClient, serviceaccount.DefaultServiceAccountsControllerOptions())
serviceAccountController.Run()
// Start the admission plugin reflectors
serviceAccountAdmission.Run()
stop := func() {
tokenController.Stop()
serviceAccountController.Stop()
serviceAccountAdmission.Stop()
apiServer.Close()
}
return rootClient, clientConfig, stop
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, pluginInitializer oadmission.PluginInitializer) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
// Connect and setup etcd interfaces
etcdClient, err := etcd.MakeNewEtcdClient(options.EtcdClientInfo)
if err != nil {
return nil, err
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, err
}
portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, err
}
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
// Defaults are tested in TestAPIServerDefaults
server := apiserveroptions.NewAPIServer()
// Adjust defaults
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.AdmissionControl = strings.Join(AdmissionPlugins, ",")
server.EnableLogsSupport = false // don't expose server logs
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
if len(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride) > 0 {
server.AdmissionControl = strings.Join(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride, ",")
}
// Defaults are tested in TestCMServerDefaults
cmserver := cmapp.NewCMServer()
// Adjust defaults
cmserver.Address = "" // no healthz endpoint
cmserver.Port = 0 // no healthz endpoint
cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout}
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile)
}
plugins := []admission.Interface{}
for _, pluginName := range strings.Split(server.AdmissionControl, ",") {
switch pluginName {
case serviceadmit.ExternalIPPluginName:
// this needs to be moved upstream to be part of core config
reject, admit, err := serviceadmit.ParseCIDRRules(options.NetworkConfig.ExternalIPNetworkCIDRs)
if err != nil {
// should have been caught with validation
return nil, err
}
plugins = append(plugins, serviceadmit.NewExternalIPRanger(reject, admit))
case saadmit.PluginName:
// we need to set some custom parameters on the service account admission controller, so create that one by hand
saAdmitter := saadmit.NewServiceAccount(internalclientset.FromUnversionedClient(kubeClient))
saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences
saAdmitter.Run()
plugins = append(plugins, saAdmitter)
default:
configFile, err := pluginconfig.GetPluginConfigFile(options.KubernetesMasterConfig.AdmissionConfig.PluginConfig, pluginName, server.AdmissionControlConfigFile)
if err != nil {
return nil, err
}
plugin := admission.InitPlugin(pluginName, internalclientset.FromUnversionedClient(kubeClient), configFile)
if plugin != nil {
plugins = append(plugins, plugin)
}
}
}
pluginInitializer.Initialize(plugins)
// ensure that plugins have been properly initialized
if err := oadmission.Validate(plugins); err != nil {
return nil, err
}
admissionController := admission.NewChainHandler(plugins...)
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
// TODO you have to know every APIGroup you're enabling or upstream will panic. It's alternative to panicing is Fataling
// It needs a refactor to return errors
storageDestinations := genericapiserver.NewStorageDestinations()
// storageVersions is a map from API group to allowed versions that must be a version exposed by the REST API or it breaks.
// We need to fix the upstream to stop using the storage version as a preferred api version.
storageVersions := map[string]string{}
enabledKubeVersions := configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupKube)
if len(enabledKubeVersions) > 0 {
kubeStorageVersion := unversioned.GroupVersion{Group: configapi.APIGroupKube, Version: options.EtcdStorageConfig.KubernetesStorageVersion}
databaseStorage, err := NewEtcdStorage(etcdClient, kubeStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupKube, databaseStorage)
storageVersions[configapi.APIGroupKube] = options.EtcdStorageConfig.KubernetesStorageVersion
}
// enable this if extensions API is enabled (or batch or autoscaling, since they persist to extensions/v1beta1 for now)
// TODO: replace this with a loop over configured storage versions
extensionsEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupExtensions)) > 0
batchEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupBatch)) > 0
autoscalingEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupAutoscaling)) > 0
if extensionsEnabled || autoscalingEnabled || batchEnabled {
// TODO: replace this with a configured storage version for extensions once configuration exposes this
extensionsStorageVersion := unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"}
databaseStorage, err := NewEtcdStorage(etcdClient, extensionsStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes extensions server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupExtensions, databaseStorage)
storageVersions[configapi.APIGroupExtensions] = extensionsStorageVersion.String()
}
// Preserve previous behavior of using the first non-loopback address
// TODO: Deprecate this behavior and just require a valid value to be passed in
publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP)
if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() {
hostIP, err := knet.ChooseHostInterface()
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err)
}
publicAddress = hostIP
glog.Infof("Will report %v as public IP address.", publicAddress)
}
m := &master.Config{
Config: &genericapiserver.Config{
PublicAddress: publicAddress,
ReadWritePort: port,
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionController,
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
APIGroupVersionOverrides: getAPIGroupVersionOverrides(options),
APIPrefix: KubeAPIPrefix,
APIGroupPrefix: KubeAPIGroupPrefix,
MasterCount: options.KubernetesMasterConfig.MasterCount,
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
Serializer: kapi.Codecs,
},
EventTTL: server.EventTTL,
//MinRequestTimeout: server.MinRequestTimeout,
KubeletClient: kubeletClient,
EnableCoreControllers: true,
}
if options.DNSConfig != nil {
_, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress)
if err != nil {
return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err)
}
dnsPort, err := strconv.Atoi(dnsPortStr)
if err != nil {
return nil, fmt.Errorf("invalid DNS port: %v", err)
}
m.ExtraServicePorts = append(m.ExtraServicePorts,
kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)},
kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)},
)
m.ExtraEndpointPorts = append(m.ExtraEndpointPorts,
kapi.EndpointPort{Name: "dns", Port: dnsPort, Protocol: kapi.ProtocolUDP},
kapi.EndpointPort{Name: "dns-tcp", Port: dnsPort, Protocol: kapi.ProtocolTCP},
)
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
}
return kmaster, nil
}
// startServiceAccountTestServer returns a started server
// It is the responsibility of the caller to ensure the returned stopFunc is called
func startServiceAccountTestServer(t *testing.T) (*clientset.Clientset, restclient.Config, func()) {
// Listener
h := &framework.MasterHolder{Initialized: make(chan struct{})}
apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
<-h.Initialized
h.M.GenericAPIServer.Handler.ServeHTTP(w, req)
}))
// Anonymous client config
clientConfig := restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: ®istered.GroupOrDie(v1.GroupName).GroupVersion}}
// Root client
// TODO: remove rootClient after we refactor pkg/admission to use the clientset.
rootClientset := clientset.NewForConfigOrDie(&restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: ®istered.GroupOrDie(v1.GroupName).GroupVersion}, BearerToken: rootToken})
internalRootClientset := internalclientset.NewForConfigOrDie(&restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: ®istered.GroupOrDie(v1.GroupName).GroupVersion}, BearerToken: rootToken})
// Set up two authenticators:
// 1. A token authenticator that maps the rootToken to the "root" user
// 2. A ServiceAccountToken authenticator that validates ServiceAccount tokens
rootTokenAuth := authenticator.TokenFunc(func(token string) (user.Info, bool, error) {
if token == rootToken {
return &user.DefaultInfo{Name: rootUserName}, true, nil
}
return nil, false, nil
})
serviceAccountKey, _ := rsa.GenerateKey(rand.Reader, 2048)
serviceAccountTokenGetter := serviceaccountcontroller.NewGetterFromClient(rootClientset)
serviceAccountTokenAuth := serviceaccount.JWTTokenAuthenticator([]interface{}{&serviceAccountKey.PublicKey}, true, serviceAccountTokenGetter)
authenticator := union.New(
bearertoken.New(rootTokenAuth),
bearertoken.New(serviceAccountTokenAuth),
)
// Set up a stub authorizer:
// 1. The "root" user is allowed to do anything
// 2. ServiceAccounts named "ro" are allowed read-only operations in their namespace
// 3. ServiceAccounts named "rw" are allowed any operation in their namespace
authorizer := authorizer.AuthorizerFunc(func(attrs authorizer.Attributes) (bool, string, error) {
username := ""
if user := attrs.GetUser(); user != nil {
username = user.GetName()
}
ns := attrs.GetNamespace()
// If the user is "root"...
if username == rootUserName {
// allow them to do anything
return true, "", nil
}
// If the user is a service account...
if serviceAccountNamespace, serviceAccountName, err := serviceaccount.SplitUsername(username); err == nil {
// Limit them to their own namespace
if serviceAccountNamespace == ns {
switch serviceAccountName {
case readOnlyServiceAccountName:
if attrs.IsReadOnly() {
return true, "", nil
}
case readWriteServiceAccountName:
return true, "", nil
}
}
}
return false, fmt.Sprintf("User %s is denied (ns=%s, readonly=%v, resource=%s)", username, ns, attrs.IsReadOnly(), attrs.GetResource()), nil
})
// Set up admission plugin to auto-assign serviceaccounts to pods
serviceAccountAdmission := serviceaccountadmission.NewServiceAccount(internalRootClientset)
masterConfig := framework.NewMasterConfig()
masterConfig.GenericConfig.EnableIndex = true
masterConfig.GenericConfig.Authenticator = authenticator
masterConfig.GenericConfig.Authorizer = authorizer
masterConfig.GenericConfig.AdmissionControl = serviceAccountAdmission
framework.RunAMasterUsingServer(masterConfig, apiServer, h)
// Start the service account and service account token controllers
stopCh := make(chan struct{})
tokenController := serviceaccountcontroller.NewTokensController(rootClientset, serviceaccountcontroller.TokensControllerOptions{TokenGenerator: serviceaccount.JWTTokenGenerator(serviceAccountKey)})
go tokenController.Run(1, stopCh)
informers := informers.NewSharedInformerFactory(rootClientset, nil, controller.NoResyncPeriodFunc())
serviceAccountController := serviceaccountcontroller.NewServiceAccountsController(informers.ServiceAccounts(), informers.Namespaces(), rootClientset, serviceaccountcontroller.DefaultServiceAccountsControllerOptions())
informers.Start(stopCh)
go serviceAccountController.Run(5, stopCh)
// Start the admission plugin reflectors
serviceAccountAdmission.Run()
stop := func() {
close(stopCh)
serviceAccountAdmission.Stop()
apiServer.Close()
}
return rootClientset, clientConfig, stop
}
func newAdmissionChain(pluginNames []string, admissionConfigFilename string, pluginConfig map[string]configapi.AdmissionPluginConfig, options configapi.MasterConfig, kubeClientSet *internalclientset.Clientset, pluginInitializer oadmission.PluginInitializer) (admission.Interface, error) {
plugins := []admission.Interface{}
for _, pluginName := range pluginNames {
switch pluginName {
case lifecycle.PluginName:
// We need to include our infrastructure and shared resource namespaces in the immortal namespaces list
immortalNamespaces := sets.NewString(kapi.NamespaceDefault)
if len(options.PolicyConfig.OpenShiftSharedResourcesNamespace) > 0 {
immortalNamespaces.Insert(options.PolicyConfig.OpenShiftSharedResourcesNamespace)
}
if len(options.PolicyConfig.OpenShiftInfrastructureNamespace) > 0 {
immortalNamespaces.Insert(options.PolicyConfig.OpenShiftInfrastructureNamespace)
}
plugins = append(plugins, lifecycle.NewLifecycle(kubeClientSet, immortalNamespaces))
case serviceadmit.ExternalIPPluginName:
// this needs to be moved upstream to be part of core config
reject, admit, err := serviceadmit.ParseRejectAdmitCIDRRules(options.NetworkConfig.ExternalIPNetworkCIDRs)
if err != nil {
// should have been caught with validation
return nil, err
}
plugins = append(plugins, serviceadmit.NewExternalIPRanger(reject, admit))
case serviceadmit.RestrictedEndpointsPluginName:
// we need to set some customer parameters, so create by hand
restrictedNetworks, err := serviceadmit.ParseSimpleCIDRRules([]string{options.NetworkConfig.ClusterNetworkCIDR, options.NetworkConfig.ServiceNetworkCIDR})
if err != nil {
// should have been caught with validation
return nil, err
}
plugins = append(plugins, serviceadmit.NewRestrictedEndpointsAdmission(restrictedNetworks))
case saadmit.PluginName:
// we need to set some custom parameters on the service account admission controller, so create that one by hand
saAdmitter := saadmit.NewServiceAccount(kubeClientSet)
saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences
saAdmitter.Run()
plugins = append(plugins, saAdmitter)
default:
configFile, err := pluginconfig.GetPluginConfigFile(pluginConfig, pluginName, admissionConfigFilename)
if err != nil {
return nil, err
}
plugin := admission.InitPlugin(pluginName, kubeClientSet, configFile)
if plugin != nil {
plugins = append(plugins, plugin)
}
}
}
pluginInitializer.Initialize(plugins)
// ensure that plugins have been properly initialized
if err := oadmission.Validate(plugins); err != nil {
return nil, err
}
return admission.NewChainHandler(plugins...), nil
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, informers shared.InformerFactory, pluginInitializer oadmission.PluginInitializer) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, err
}
portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, err
}
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
// Defaults are tested in TestAPIServerDefaults
server := apiserveroptions.NewAPIServer()
// Adjust defaults
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.AdmissionControl = strings.Join(AdmissionPlugins, ",")
server.EnableLogsSupport = false // don't expose server logs
server.EnableProfiling = false
server.APIPrefix = KubeAPIPrefix
server.APIGroupPrefix = KubeAPIGroupPrefix
server.SecurePort = port
server.MasterCount = options.KubernetesMasterConfig.MasterCount
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
if len(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride) > 0 {
server.AdmissionControl = strings.Join(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride, ",")
}
// Defaults are tested in TestCMServerDefaults
cmserver := cmapp.NewCMServer()
// Adjust defaults
cmserver.Address = "" // no healthz endpoint
cmserver.Port = 0 // no healthz endpoint
cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout}
cmserver.VolumeConfiguration.EnableDynamicProvisioning = options.VolumeConfig.DynamicProvisioningEnabled
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile)
}
plugins := []admission.Interface{}
for _, pluginName := range strings.Split(server.AdmissionControl, ",") {
switch pluginName {
case lifecycle.PluginName:
// We need to include our infrastructure and shared resource namespaces in the immortal namespaces list
immortalNamespaces := sets.NewString(kapi.NamespaceDefault)
if len(options.PolicyConfig.OpenShiftSharedResourcesNamespace) > 0 {
immortalNamespaces.Insert(options.PolicyConfig.OpenShiftSharedResourcesNamespace)
}
if len(options.PolicyConfig.OpenShiftInfrastructureNamespace) > 0 {
immortalNamespaces.Insert(options.PolicyConfig.OpenShiftInfrastructureNamespace)
}
plugins = append(plugins, lifecycle.NewLifecycle(clientadapter.FromUnversionedClient(kubeClient), immortalNamespaces))
case serviceadmit.ExternalIPPluginName:
// this needs to be moved upstream to be part of core config
reject, admit, err := serviceadmit.ParseCIDRRules(options.NetworkConfig.ExternalIPNetworkCIDRs)
if err != nil {
// should have been caught with validation
return nil, err
}
plugins = append(plugins, serviceadmit.NewExternalIPRanger(reject, admit))
case saadmit.PluginName:
// we need to set some custom parameters on the service account admission controller, so create that one by hand
saAdmitter := saadmit.NewServiceAccount(clientadapter.FromUnversionedClient(kubeClient))
saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences
saAdmitter.Run()
plugins = append(plugins, saAdmitter)
default:
configFile, err := pluginconfig.GetPluginConfigFile(options.KubernetesMasterConfig.AdmissionConfig.PluginConfig, pluginName, server.AdmissionControlConfigFile)
if err != nil {
return nil, err
}
plugin := admission.InitPlugin(pluginName, clientadapter.FromUnversionedClient(kubeClient), configFile)
if plugin != nil {
plugins = append(plugins, plugin)
}
}
}
pluginInitializer.Initialize(plugins)
// ensure that plugins have been properly initialized
if err := oadmission.Validate(plugins); err != nil {
return nil, err
}
admissionController := admission.NewChainHandler(plugins...)
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
resourceEncodingConfig := genericapiserver.NewDefaultResourceEncodingConfig()
resourceEncodingConfig.SetVersionEncoding(
kapi.GroupName,
unversioned.GroupVersion{Group: kapi.GroupName, Version: options.EtcdStorageConfig.KubernetesStorageVersion},
kapi.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
extensions.GroupName,
unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"},
extensions.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
batch.GroupName,
unversioned.GroupVersion{Group: batch.GroupName, Version: "v1"},
batch.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
autoscaling.GroupName,
unversioned.GroupVersion{Group: autoscaling.GroupName, Version: "v1"},
autoscaling.SchemeGroupVersion,
)
etcdConfig := storagebackend.Config{
Prefix: options.EtcdStorageConfig.KubernetesStoragePrefix,
ServerList: options.EtcdClientInfo.URLs,
KeyFile: options.EtcdClientInfo.ClientCert.KeyFile,
CertFile: options.EtcdClientInfo.ClientCert.CertFile,
CAFile: options.EtcdClientInfo.CA,
DeserializationCacheSize: genericapiserveroptions.DefaultDeserializationCacheSize,
}
storageFactory := genericapiserver.NewDefaultStorageFactory(etcdConfig, "", kapi.Codecs, resourceEncodingConfig, master.DefaultAPIResourceConfigSource())
// the order here is important, it defines which version will be used for storage
storageFactory.AddCohabitatingResources(extensions.Resource("jobs"), batch.Resource("jobs"))
storageFactory.AddCohabitatingResources(extensions.Resource("horizontalpodautoscalers"), autoscaling.Resource("horizontalpodautoscalers"))
// Preserve previous behavior of using the first non-loopback address
// TODO: Deprecate this behavior and just require a valid value to be passed in
publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP)
if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() {
hostIP, err := knet.ChooseHostInterface()
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err)
}
publicAddress = hostIP
glog.Infof("Will report %v as public IP address.", publicAddress)
}
m := &master.Config{
Config: &genericapiserver.Config{
PublicAddress: publicAddress,
ReadWritePort: port,
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionController,
StorageFactory: storageFactory,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
APIResourceConfigSource: getAPIResourceConfig(options),
APIPrefix: server.APIPrefix,
APIGroupPrefix: server.APIGroupPrefix,
MasterCount: server.MasterCount,
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
Serializer: kapi.Codecs,
EnableLogsSupport: server.EnableLogsSupport,
EnableProfiling: server.EnableProfiling,
EnableWatchCache: server.EnableWatchCache,
MasterServiceNamespace: server.MasterServiceNamespace,
ExternalHost: server.ExternalHost,
MinRequestTimeout: server.MinRequestTimeout,
KubernetesServiceNodePort: server.KubernetesServiceNodePort,
},
EventTTL: server.EventTTL,
KubeletClient: kubeletClient,
EnableCoreControllers: true,
DeleteCollectionWorkers: server.DeleteCollectionWorkers,
}
if server.EnableWatchCache {
cachesize.SetWatchCacheSizes(server.WatchCacheSizes)
}
if options.DNSConfig != nil {
_, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress)
if err != nil {
return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err)
}
dnsPort, err := strconv.Atoi(dnsPortStr)
if err != nil {
return nil, fmt.Errorf("invalid DNS port: %v", err)
}
m.ExtraServicePorts = append(m.ExtraServicePorts,
kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)},
kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)},
)
m.ExtraEndpointPorts = append(m.ExtraEndpointPorts,
kapi.EndpointPort{Name: "dns", Port: int32(dnsPort), Protocol: kapi.ProtocolUDP},
kapi.EndpointPort{Name: "dns-tcp", Port: int32(dnsPort), Protocol: kapi.ProtocolTCP},
)
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
Informers: informers,
}
return kmaster, nil
}