// getBatchResources returns the resources for batch api
func (m *Master) getBatchResources(c *Config, version unversioned.GroupVersion) map[string]rest.Storage {
storage := map[string]rest.Storage{}
if c.APIResourceConfigSource.ResourceEnabled(version.WithResource("jobs")) {
jobsStorage, jobsStatusStorage := jobetcd.NewREST(m.GetRESTOptionsOrDie(c, batch.Resource("jobs")))
storage["jobs"] = jobsStorage
storage["jobs/status"] = jobsStatusStorage
}
return storage
}
func (p RESTStorageProvider) v2alpha1Storage(apiResourceConfigSource genericapiserver.APIResourceConfigSource, restOptionsGetter genericapiserver.RESTOptionsGetter) map[string]rest.Storage {
version := batchapiv2alpha1.SchemeGroupVersion
storage := map[string]rest.Storage{}
if apiResourceConfigSource.ResourceEnabled(version.WithResource("jobs")) {
jobsStorage, jobsStatusStorage := jobetcd.NewREST(restOptionsGetter(batch.Resource("jobs")))
storage["jobs"] = jobsStorage
storage["jobs/status"] = jobsStatusStorage
}
if apiResourceConfigSource.ResourceEnabled(version.WithResource("cronjobs")) {
cronJobsStorage, cronJobsStatusStorage := cronjobetcd.NewREST(restOptionsGetter(batch.Resource("cronjobs")))
storage["cronjobs"] = cronJobsStorage
storage["cronjobs/status"] = cronJobsStatusStorage
storage["scheduledjobs"] = cronJobsStorage
storage["scheduledjobs/status"] = cronJobsStatusStorage
}
return storage
}
// NewREST returns a RESTStorage object that will work against Jobs.
func NewREST(opts generic.RESTOptions) (*REST, *StatusREST) {
prefix := "/" + opts.ResourcePrefix
newListFunc := func() runtime.Object { return &batch.JobList{} }
storageInterface, dFunc := opts.Decorator(
opts.StorageConfig,
cachesize.GetWatchCacheSizeByResource(cachesize.Jobs),
&batch.Job{},
prefix,
job.Strategy,
newListFunc,
job.GetAttrs,
storage.NoTriggerPublisher,
)
store := &genericregistry.Store{
NewFunc: func() runtime.Object { return &batch.Job{} },
// NewListFunc returns an object capable of storing results of an etcd list.
NewListFunc: newListFunc,
// Produces a path that etcd understands, to the root of the resource
// by combining the namespace in the context with the given prefix
KeyRootFunc: func(ctx api.Context) string {
return genericregistry.NamespaceKeyRootFunc(ctx, prefix)
},
// Produces a path that etcd understands, to the resource by combining
// the namespace in the context with the given prefix
KeyFunc: func(ctx api.Context, name string) (string, error) {
return genericregistry.NamespaceKeyFunc(ctx, prefix, name)
},
// Retrieve the name field of a job
ObjectNameFunc: func(obj runtime.Object) (string, error) {
return obj.(*batch.Job).Name, nil
},
// Used to match objects based on labels/fields for list and watch
PredicateFunc: job.MatchJob,
QualifiedResource: batch.Resource("jobs"),
EnableGarbageCollection: opts.EnableGarbageCollection,
DeleteCollectionWorkers: opts.DeleteCollectionWorkers,
// Used to validate job creation
CreateStrategy: job.Strategy,
// Used to validate job updates
UpdateStrategy: job.Strategy,
DeleteStrategy: job.Strategy,
Storage: storageInterface,
DestroyFunc: dFunc,
}
statusStore := *store
statusStore.UpdateStrategy = job.StatusStrategy
return &REST{store}, &StatusREST{store: &statusStore}
}
// Get retrieves the Job from the indexer for a given namespace and name.
func (s jobNamespaceLister) Get(name string) (*v1.Job, error) {
obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
if err != nil {
return nil, err
}
if !exists {
return nil, errors.NewNotFound(batch.Resource("job"), name)
}
return obj.(*v1.Job), nil
}
// getBatchResources returns the resources for batch api
func (m *Master) getBatchResources(c *Config) map[string]rest.Storage {
// TODO update when we support more than one version of this group
version := batchapiv1.SchemeGroupVersion
storage := map[string]rest.Storage{}
if c.APIResourceConfigSource.ResourceEnabled(version.WithResource("jobs")) {
jobsStorage, jobsStatusStorage := jobetcd.NewREST(m.GetRESTOptionsOrDie(c, batch.Resource("jobs")))
storage["jobs"] = jobsStorage
storage["jobs/status"] = jobsStatusStorage
}
return storage
}
// NewREST returns a RESTStorage object that will work against ScheduledJobs.
func NewREST(opts generic.RESTOptions) (*REST, *StatusREST) {
prefix := "/scheduledjobs"
newListFunc := func() runtime.Object { return &batch.ScheduledJobList{} }
storageInterface := opts.Decorator(
opts.Storage, cachesize.GetWatchCacheSizeByResource(cachesize.ScheduledJobs), &batch.ScheduledJob{}, prefix, scheduledjob.Strategy, newListFunc)
store := ®istry.Store{
NewFunc: func() runtime.Object { return &batch.ScheduledJob{} },
// NewListFunc returns an object capable of storing results of an etcd list.
NewListFunc: newListFunc,
// Produces a path that etcd understands, to the root of the resource
// by combining the namespace in the context with the given prefix
KeyRootFunc: func(ctx api.Context) string {
return registry.NamespaceKeyRootFunc(ctx, prefix)
},
// Produces a path that etcd understands, to the resource by combining
// the namespace in the context with the given prefix
KeyFunc: func(ctx api.Context, name string) (string, error) {
return registry.NamespaceKeyFunc(ctx, prefix, name)
},
// Retrieve the name field of a scheduled job
ObjectNameFunc: func(obj runtime.Object) (string, error) {
return obj.(*batch.ScheduledJob).Name, nil
},
// Used to match objects based on labels/fields for list and watch
PredicateFunc: func(label labels.Selector, field fields.Selector) generic.Matcher {
return scheduledjob.MatchScheduledJob(label, field)
},
QualifiedResource: batch.Resource("scheduledjobs"),
DeleteCollectionWorkers: opts.DeleteCollectionWorkers,
// Used to validate scheduled job creation
CreateStrategy: scheduledjob.Strategy,
// Used to validate scheduled job updates
UpdateStrategy: scheduledjob.Strategy,
DeleteStrategy: scheduledjob.Strategy,
Storage: storageInterface,
}
statusStore := *store
statusStore.UpdateStrategy = scheduledjob.StatusStrategy
return &REST{store}, &StatusREST{store: &statusStore}
}
// ForResource gives generic access to a shared informer of the matching type
// TODO extend this to unknown resources with a client pool
func (f *sharedInformerFactory) ForResource(resource unversioned.GroupResource) (GenericInformer, error) {
switch resource {
case api.Resource("pods"):
return &genericInformer{resource: resource, informer: f.Pods().Informer()}, nil
case api.Resource("limitranges"):
return &genericInformer{resource: resource, informer: f.LimitRanges().Informer()}, nil
case api.Resource("namespaces"):
return &genericInformer{resource: resource, informer: f.Namespaces().Informer()}, nil
case api.Resource("nodes"):
return &genericInformer{resource: resource, informer: f.Nodes().Informer()}, nil
case api.Resource("persistentvolumeclaims"):
return &genericInformer{resource: resource, informer: f.PersistentVolumeClaims().Informer()}, nil
case api.Resource("persistentvolumes"):
return &genericInformer{resource: resource, informer: f.PersistentVolumes().Informer()}, nil
case api.Resource("serviceaccounts"):
return &genericInformer{resource: resource, informer: f.ServiceAccounts().Informer()}, nil
case extensions.Resource("daemonsets"):
return &genericInformer{resource: resource, informer: f.DaemonSets().Informer()}, nil
case extensions.Resource("deployments"):
return &genericInformer{resource: resource, informer: f.Deployments().Informer()}, nil
case extensions.Resource("replicasets"):
return &genericInformer{resource: resource, informer: f.ReplicaSets().Informer()}, nil
case rbac.Resource("clusterrolebindings"):
return &genericInformer{resource: resource, informer: f.ClusterRoleBindings().Informer()}, nil
case rbac.Resource("clusterroles"):
return &genericInformer{resource: resource, informer: f.ClusterRoles().Informer()}, nil
case rbac.Resource("rolebindings"):
return &genericInformer{resource: resource, informer: f.RoleBindings().Informer()}, nil
case rbac.Resource("roles"):
return &genericInformer{resource: resource, informer: f.Roles().Informer()}, nil
case batch.Resource("jobs"):
return &genericInformer{resource: resource, informer: f.Jobs().Informer()}, nil
}
return nil, fmt.Errorf("no informer found for %v", resource)
}
func buildBatchResources(apiResourceConfigSource genericapiserver.APIResourceConfigSource, restOptionsGetter RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
apiGroupInfo := NewDefaultAPIGroupInfo(batch.GroupName)
storageForVersion := func(version unversioned.GroupVersion) map[string]rest.Storage {
storage := map[string]rest.Storage{}
if apiResourceConfigSource.ResourceEnabled(version.WithResource("jobs")) {
jobsStorage, jobsStatusStorage := jobetcd.NewREST(restOptionsGetter(batch.Resource("jobs")))
storage["jobs"] = jobsStorage
storage["jobs/status"] = jobsStatusStorage
}
return storage
}
if apiResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv2alpha1.SchemeGroupVersion) {
apiGroupInfo.VersionedResourcesStorageMap[batchapiv2alpha1.SchemeGroupVersion.Version] = storageForVersion(batchapiv2alpha1.SchemeGroupVersion)
apiGroupInfo.GroupMeta.GroupVersion = batchapiv2alpha1.SchemeGroupVersion
}
if apiResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv1.SchemeGroupVersion) {
apiGroupInfo.VersionedResourcesStorageMap[batchapiv1.SchemeGroupVersion.Version] = storageForVersion(batchapiv1.SchemeGroupVersion)
apiGroupInfo.GroupMeta.GroupVersion = batchapiv1.SchemeGroupVersion
}
return apiGroupInfo, true
}
// NewREST returns a RESTStorage object that will work against CronJobs.
func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) {
store := &genericregistry.Store{
NewFunc: func() runtime.Object { return &batch.CronJob{} },
NewListFunc: func() runtime.Object { return &batch.CronJobList{} },
ObjectNameFunc: func(obj runtime.Object) (string, error) {
return obj.(*batch.CronJob).Name, nil
},
PredicateFunc: cronjob.MatchCronJob,
QualifiedResource: batch.Resource("cronjobs"),
CreateStrategy: cronjob.Strategy,
UpdateStrategy: cronjob.Strategy,
DeleteStrategy: cronjob.Strategy,
}
options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: cronjob.GetAttrs}
if err := store.CompleteWithOptions(options); err != nil {
panic(err) // TODO: Propagate error up
}
statusStore := *store
statusStore.UpdateStrategy = cronjob.StatusStrategy
return &REST{store}, &StatusREST{store: &statusStore}
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.ServerRunOptions) error {
genericvalidation.VerifyEtcdServersList(s.GenericServerRunOptions)
genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
genericConfig := genericapiserver.NewConfig(). // create the new config
ApplyOptions(s.GenericServerRunOptions). // apply the options selected
Complete() // set default values based on the known values
serviceIPRange, apiServerServiceIP, err := genericapiserver.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange)
if err != nil {
glog.Fatalf("Error determining service IP ranges: %v", err)
}
if err := genericConfig.MaybeGenerateServingCerts(apiServerServiceIP); err != nil {
glog.Fatalf("Failed to generate service certificate: %v", err)
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
// Setup tunneler if needed
var tunneler genericapiserver.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH genericapiserver.InstallSSHKey
cloud, err := cloudprovider.InitCloudProvider(s.GenericServerRunOptions.CloudProvider, s.GenericServerRunOptions.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
if s.KubeletConfig.Port == 0 {
glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.")
}
// Set up the tunneler
// TODO(cjcullen): If we want this to handle per-kubelet ports or other
// kubelet listen-addresses, we need to plumb through options.
healthCheckPath := &url.URL{
Scheme: "https",
Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)),
Path: "healthz",
}
tunneler = genericapiserver.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
if s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize == 0 {
// When size of cache is not explicitly set, estimate its size based on
// target memory usage.
glog.V(2).Infof("Initalizing deserialization cache size based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB)
// This is the heuristics that from memory capacity is trying to infer
// the maximum number of nodes in the cluster and set cache sizes based
// on that value.
// From our documentation, we officially recomment 120GB machines for
// 2000 nodes, and we scale from that point. Thus we assume ~60MB of
// capacity per node.
// TODO: We may consider deciding that some percentage of memory will
// be used for the deserialization cache and divide it by the max object
// size to compute its size. We may even go further and measure
// collective sizes of the objects in the cache.
clusterSize := s.GenericServerRunOptions.TargetRAMMB / 60
s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize = 25 * clusterSize
if s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize < 1000 {
s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize = 1000
}
}
storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion()
if err != nil {
glog.Fatalf("error generating storage version map: %s", err)
}
storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
s.GenericServerRunOptions.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs,
genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
// FIXME: this GroupVersionResource override should be configurable
[]schema.GroupVersionResource{batch.Resource("cronjobs").WithVersion("v2alpha1")},
master.DefaultAPIResourceConfigSource(), s.GenericServerRunOptions.RuntimeConfig)
if err != nil {
glog.Fatalf("error in initializing storage factory: %s", err)
}
storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs"))
storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers"))
for _, override := range s.GenericServerRunOptions.EtcdServersOverrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
continue
}
group := apiresource[0]
resource := apiresource[1]
groupResource := schema.GroupResource{Group: group, Resource: resource}
servers := strings.Split(tokens[1], ";")
storageFactory.SetEtcdLocation(groupResource, servers)
}
// Default to the private server key for service account token signing
if len(s.ServiceAccountKeyFiles) == 0 && s.GenericServerRunOptions.TLSPrivateKeyFile != "" {
if authenticator.IsValidServiceAccountKeyFile(s.GenericServerRunOptions.TLSPrivateKeyFile) {
s.ServiceAccountKeyFiles = []string{s.GenericServerRunOptions.TLSPrivateKeyFile}
} else {
glog.Warning("No TLS key provided, service account token authentication disabled")
}
}
var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter
if s.ServiceAccountLookup {
// If we need to look up service accounts and tokens,
// go directly to etcd to avoid recursive auth insanity
storageConfig, err := storageFactory.NewConfig(api.Resource("serviceaccounts"))
if err != nil {
glog.Fatalf("Unable to get serviceaccounts storage: %v", err)
}
serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets")))
}
apiAuthenticator, securityDefinitions, err := authenticator.New(authenticator.AuthenticatorConfig{
Anonymous: s.GenericServerRunOptions.AnonymousAuth,
AnyToken: s.GenericServerRunOptions.EnableAnyToken,
BasicAuthFile: s.GenericServerRunOptions.BasicAuthFile,
ClientCAFile: s.GenericServerRunOptions.ClientCAFile,
TokenAuthFile: s.GenericServerRunOptions.TokenAuthFile,
OIDCIssuerURL: s.GenericServerRunOptions.OIDCIssuerURL,
OIDCClientID: s.GenericServerRunOptions.OIDCClientID,
OIDCCAFile: s.GenericServerRunOptions.OIDCCAFile,
OIDCUsernameClaim: s.GenericServerRunOptions.OIDCUsernameClaim,
OIDCGroupsClaim: s.GenericServerRunOptions.OIDCGroupsClaim,
ServiceAccountKeyFiles: s.ServiceAccountKeyFiles,
ServiceAccountLookup: s.ServiceAccountLookup,
ServiceAccountTokenGetter: serviceAccountGetter,
KeystoneURL: s.GenericServerRunOptions.KeystoneURL,
KeystoneCAFile: s.GenericServerRunOptions.KeystoneCAFile,
WebhookTokenAuthnConfigFile: s.WebhookTokenAuthnConfigFile,
WebhookTokenAuthnCacheTTL: s.WebhookTokenAuthnCacheTTL,
RequestHeaderConfig: s.GenericServerRunOptions.AuthenticationRequestHeaderConfig(),
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
privilegedLoopbackToken := uuid.NewRandom().String()
selfClientConfig, err := s.GenericServerRunOptions.NewSelfClientConfig(privilegedLoopbackToken)
if err != nil {
glog.Fatalf("Failed to create clientset: %v", err)
}
client, err := s.GenericServerRunOptions.NewSelfClient(privilegedLoopbackToken)
if err != nil {
glog.Errorf("Failed to create clientset: %v", err)
}
sharedInformers := informers.NewSharedInformerFactory(nil, client, 10*time.Minute)
authorizationConfig := authorizer.AuthorizationConfig{
PolicyFile: s.GenericServerRunOptions.AuthorizationPolicyFile,
WebhookConfigFile: s.GenericServerRunOptions.AuthorizationWebhookConfigFile,
WebhookCacheAuthorizedTTL: s.GenericServerRunOptions.AuthorizationWebhookCacheAuthorizedTTL,
WebhookCacheUnauthorizedTTL: s.GenericServerRunOptions.AuthorizationWebhookCacheUnauthorizedTTL,
RBACSuperUser: s.GenericServerRunOptions.AuthorizationRBACSuperUser,
InformerFactory: sharedInformers,
}
authorizationModeNames := strings.Split(s.GenericServerRunOptions.AuthorizationMode, ",")
apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.GenericServerRunOptions.AdmissionControl, ",")
// TODO(dims): We probably need to add an option "EnableLoopbackToken"
if apiAuthenticator != nil {
var uid = uuid.NewRandom().String()
tokens := make(map[string]*user.DefaultInfo)
tokens[privilegedLoopbackToken] = &user.DefaultInfo{
Name: user.APIServerUser,
UID: uid,
Groups: []string{user.SystemPrivilegedGroup},
}
tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens)
apiAuthenticator = authenticatorunion.New(tokenAuthenticator, apiAuthenticator)
tokenAuthorizer := authorizer.NewPrivilegedGroups(user.SystemPrivilegedGroup)
apiAuthorizer = authorizerunion.New(tokenAuthorizer, apiAuthorizer)
}
pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer)
admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.GenericServerRunOptions.AdmissionControlConfigFile, pluginInitializer)
if err != nil {
glog.Fatalf("Failed to initialize plugins: %v", err)
}
proxyTransport := utilnet.SetTransportDefaults(&http.Transport{
Dial: proxyDialerFn,
TLSClientConfig: proxyTLSClientConfig,
})
kubeVersion := version.Get()
genericConfig.Version = &kubeVersion
genericConfig.LoopbackClientConfig = selfClientConfig
genericConfig.Authenticator = apiAuthenticator
genericConfig.Authorizer = apiAuthorizer
genericConfig.AdmissionControl = admissionController
genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"
genericConfig.OpenAPIConfig.Definitions = generatedopenapi.OpenAPIDefinitions
genericConfig.EnableOpenAPISupport = true
genericConfig.EnableMetrics = true
genericConfig.OpenAPIConfig.SecurityDefinitions = securityDefinitions
config := &master.Config{
GenericConfig: genericConfig.Config,
StorageFactory: storageFactory,
EnableWatchCache: s.GenericServerRunOptions.EnableWatchCache,
EnableCoreControllers: true,
DeleteCollectionWorkers: s.GenericServerRunOptions.DeleteCollectionWorkers,
EventTTL: s.EventTTL,
KubeletClientConfig: s.KubeletConfig,
EnableUISupport: true,
EnableLogsSupport: true,
ProxyTransport: proxyTransport,
Tunneler: tunneler,
ServiceIPRange: serviceIPRange,
APIServerServiceIP: apiServerServiceIP,
APIServerServicePort: 443,
ServiceNodePortRange: s.GenericServerRunOptions.ServiceNodePortRange,
KubernetesServiceNodePort: s.GenericServerRunOptions.KubernetesServiceNodePort,
MasterCount: s.GenericServerRunOptions.MasterCount,
}
if s.GenericServerRunOptions.EnableWatchCache {
glog.V(2).Infof("Initalizing cache sizes based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB)
cachesize.InitializeWatchCacheSizes(s.GenericServerRunOptions.TargetRAMMB)
cachesize.SetWatchCacheSizes(s.GenericServerRunOptions.WatchCacheSizes)
}
m, err := config.Complete().New()
if err != nil {
return err
}
sharedInformers.Start(wait.NeverStop)
m.GenericAPIServer.PrepareRun().Run(wait.NeverStop)
return nil
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, informers shared.InformerFactory, admissionControl admission.Interface, originAuthenticator authenticator.Request) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, err
}
portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, err
}
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
// Defaults are tested in TestAPIServerDefaults
server := apiserveroptions.NewAPIServer()
// Adjust defaults
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.EnableLogsSupport = false // don't expose server logs
server.EnableProfiling = false
server.APIPrefix = KubeAPIPrefix
server.APIGroupPrefix = KubeAPIGroupPrefix
server.SecurePort = port
server.MasterCount = options.KubernetesMasterConfig.MasterCount
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
// Defaults are tested in TestCMServerDefaults
cmserver := cmapp.NewCMServer()
// Adjust defaults
cmserver.Address = "" // no healthz endpoint
cmserver.Port = 0 // no healthz endpoint
cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout}
cmserver.VolumeConfiguration.EnableDynamicProvisioning = options.VolumeConfig.DynamicProvisioningEnabled
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile)
}
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
resourceEncodingConfig := genericapiserver.NewDefaultResourceEncodingConfig()
resourceEncodingConfig.SetVersionEncoding(
kapi.GroupName,
unversioned.GroupVersion{Group: kapi.GroupName, Version: options.EtcdStorageConfig.KubernetesStorageVersion},
kapi.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
extensions.GroupName,
unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"},
extensions.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
batch.GroupName,
unversioned.GroupVersion{Group: batch.GroupName, Version: "v1"},
batch.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
autoscaling.GroupName,
unversioned.GroupVersion{Group: autoscaling.GroupName, Version: "v1"},
autoscaling.SchemeGroupVersion,
)
etcdConfig := storagebackend.Config{
Prefix: options.EtcdStorageConfig.KubernetesStoragePrefix,
ServerList: options.EtcdClientInfo.URLs,
KeyFile: options.EtcdClientInfo.ClientCert.KeyFile,
CertFile: options.EtcdClientInfo.ClientCert.CertFile,
CAFile: options.EtcdClientInfo.CA,
DeserializationCacheSize: genericapiserveroptions.DefaultDeserializationCacheSize,
}
storageFactory := genericapiserver.NewDefaultStorageFactory(etcdConfig, "", kapi.Codecs, resourceEncodingConfig, master.DefaultAPIResourceConfigSource())
// the order here is important, it defines which version will be used for storage
storageFactory.AddCohabitatingResources(extensions.Resource("jobs"), batch.Resource("jobs"))
storageFactory.AddCohabitatingResources(extensions.Resource("horizontalpodautoscalers"), autoscaling.Resource("horizontalpodautoscalers"))
// Preserve previous behavior of using the first non-loopback address
// TODO: Deprecate this behavior and just require a valid value to be passed in
publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP)
if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() {
hostIP, err := knet.ChooseHostInterface()
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err)
}
publicAddress = hostIP
glog.Infof("Will report %v as public IP address.", publicAddress)
}
m := &master.Config{
Config: &genericapiserver.Config{
PublicAddress: publicAddress,
ReadWritePort: port,
Authenticator: originAuthenticator, // this is used to fulfill the tokenreviews endpoint which is used by node authentication
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionControl,
StorageFactory: storageFactory,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
APIResourceConfigSource: getAPIResourceConfig(options),
APIPrefix: server.APIPrefix,
APIGroupPrefix: server.APIGroupPrefix,
MasterCount: server.MasterCount,
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
Serializer: kapi.Codecs,
EnableLogsSupport: server.EnableLogsSupport,
EnableProfiling: server.EnableProfiling,
EnableWatchCache: server.EnableWatchCache,
MasterServiceNamespace: server.MasterServiceNamespace,
ExternalHost: server.ExternalHost,
MinRequestTimeout: server.MinRequestTimeout,
KubernetesServiceNodePort: server.KubernetesServiceNodePort,
},
EventTTL: server.EventTTL,
KubeletClient: kubeletClient,
EnableCoreControllers: true,
DeleteCollectionWorkers: server.DeleteCollectionWorkers,
}
if server.EnableWatchCache {
cachesize.SetWatchCacheSizes(server.WatchCacheSizes)
}
if options.DNSConfig != nil {
_, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress)
if err != nil {
return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err)
}
dnsPort, err := strconv.Atoi(dnsPortStr)
if err != nil {
return nil, fmt.Errorf("invalid DNS port: %v", err)
}
m.ExtraServicePorts = append(m.ExtraServicePorts,
kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)},
kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)},
)
m.ExtraEndpointPorts = append(m.ExtraEndpointPorts,
kapi.EndpointPort{Name: "dns", Port: int32(dnsPort), Protocol: kapi.ProtocolUDP},
kapi.EndpointPort{Name: "dns-tcp", Port: int32(dnsPort), Protocol: kapi.ProtocolTCP},
)
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
Informers: informers,
}
return kmaster, nil
}
type podNodeConstraints struct {
*admission.Handler
selectorLabelBlacklist sets.String
config *api.PodNodeConstraintsConfig
authorizer authorizer.Authorizer
}
// resourcesToCheck is a map of resources and corresponding kinds of things that
// we want handled in this plugin
// TODO: Include a function that will extract the PodSpec from the resource for
// each type added here.
var resourcesToCheck = map[unversioned.GroupResource]unversioned.GroupKind{
kapi.Resource("pods"): kapi.Kind("Pod"),
kapi.Resource("podtemplates"): kapi.Kind("PodTemplate"),
kapi.Resource("replicationcontrollers"): kapi.Kind("ReplicationController"),
batch.Resource("jobs"): batch.Kind("Job"),
extensions.Resource("deployments"): extensions.Kind("Deployment"),
extensions.Resource("replicasets"): extensions.Kind("ReplicaSet"),
extensions.Resource("jobs"): extensions.Kind("Job"),
deployapi.Resource("deploymentconfigs"): deployapi.Kind("DeploymentConfig"),
}
// resourcesToIgnore is a list of resource kinds that contain a PodSpec that
// we choose not to handle in this plugin
var resourcesToIgnore = []unversioned.GroupKind{
extensions.Kind("DaemonSet"),
}
func shouldCheckResource(resource unversioned.GroupResource, kind unversioned.GroupKind) (bool, error) {
expectedKind, shouldCheck := resourcesToCheck[resource]
if !shouldCheck {
func TestJobScale(t *testing.T) {
fakeClientset := fake.NewSimpleClientset(job())
scaler := JobScaler{fakeClientset.Batch()}
preconditions := ScalePrecondition{-1, ""}
count := uint(3)
name := "foo"
scaler.Scale("default", name, count, &preconditions, nil, nil)
actions := fakeClientset.Actions()
if len(actions) != 2 {
t.Errorf("unexpected actions: %v, expected 2 actions (get, update)", actions)
}
if action, ok := actions[0].(testcore.GetAction); !ok || action.GetResource().GroupResource() != batch.Resource("jobs") || action.GetName() != name {
t.Errorf("unexpected action: %v, expected get-replicationController %s", actions[0], name)
}
if action, ok := actions[1].(testcore.UpdateAction); !ok || action.GetResource().GroupResource() != batch.Resource("jobs") || *action.GetObject().(*batch.Job).Spec.Parallelism != int32(count) {
t.Errorf("unexpected action %v, expected update-job with parallelism = %d", actions[1], count)
}
}
func TestPodNodeConstraintsResources(t *testing.T) {
ns := kapi.NamespaceDefault
testconfigs := []struct {
config *api.PodNodeConstraintsConfig
userinfo user.Info
reviewResponse *authorizationapi.SubjectAccessReviewResponse
}{
{
config: testConfig(),
userinfo: serviceaccount.UserInfo("", "", ""),
reviewResponse: reviewResponse(false, ""),
},
}
testresources := []struct {
resource func(bool) runtime.Object
kind unversioned.GroupKind
groupresource unversioned.GroupResource
prefix string
}{
{
resource: replicationController,
kind: kapi.Kind("ReplicationController"),
groupresource: kapi.Resource("replicationcontrollers"),
prefix: "ReplicationController",
},
{
resource: deployment,
kind: extensions.Kind("Deployment"),
groupresource: extensions.Resource("deployments"),
prefix: "Deployment",
},
{
resource: replicaSet,
kind: extensions.Kind("ReplicaSet"),
groupresource: extensions.Resource("replicasets"),
prefix: "ReplicaSet",
},
{
resource: job,
kind: extensions.Kind("Job"),
groupresource: extensions.Resource("jobs"),
prefix: "Job",
},
{
resource: job,
kind: batch.Kind("Job"),
groupresource: batch.Resource("jobs"),
prefix: "Job",
},
{
resource: deploymentConfig,
kind: deployapi.Kind("DeploymentConfig"),
groupresource: deployapi.Resource("deploymentconfigs"),
prefix: "DeploymentConfig",
},
{
resource: podTemplate,
kind: deployapi.Kind("PodTemplate"),
groupresource: deployapi.Resource("podtemplates"),
prefix: "PodTemplate",
},
{
resource: podSecurityPolicySubjectReview,
kind: securityapi.Kind("PodSecurityPolicySubjectReview"),
groupresource: securityapi.Resource("podsecuritypolicysubjectreviews"),
prefix: "PodSecurityPolicy",
},
{
resource: podSecurityPolicySelfSubjectReview,
kind: securityapi.Kind("PodSecurityPolicySelfSubjectReview"),
groupresource: securityapi.Resource("podsecuritypolicyselfsubjectreviews"),
prefix: "PodSecurityPolicy",
},
{
resource: podSecurityPolicyReview,
kind: securityapi.Kind("PodSecurityPolicyReview"),
groupresource: securityapi.Resource("podsecuritypolicyreviews"),
prefix: "PodSecurityPolicy",
},
}
testparams := []struct {
nodeselector bool
expectedErrorMsg string
prefix string
}{
{
nodeselector: true,
expectedErrorMsg: "node selection by label(s) [bogus] is prohibited by policy for your role",
prefix: "with nodeSelector",
},
{
nodeselector: false,
expectedErrorMsg: "",
prefix: "without nodeSelector",
},
}
testops := []struct {
operation admission.Operation
}{
{
operation: admission.Create,
},
{
operation: admission.Update,
},
}
for _, tc := range testconfigs {
for _, tr := range testresources {
for _, tp := range testparams {
for _, top := range testops {
var expectedError error
errPrefix := fmt.Sprintf("%s; %s; %s", tr.prefix, tp.prefix, top.operation)
prc := NewPodNodeConstraints(tc.config)
prc.(oadmission.WantsAuthorizer).SetAuthorizer(fakeAuthorizer(t))
err := prc.(oadmission.Validator).Validate()
if err != nil {
checkAdmitError(t, err, expectedError, errPrefix)
continue
}
attrs := admission.NewAttributesRecord(tr.resource(tp.nodeselector), nil, tr.kind.WithVersion("version"), ns, "test", tr.groupresource.WithVersion("version"), "", top.operation, tc.userinfo)
if tp.expectedErrorMsg != "" {
expectedError = admission.NewForbidden(attrs, fmt.Errorf(tp.expectedErrorMsg))
}
err = prc.Admit(attrs)
checkAdmitError(t, err, expectedError, errPrefix)
}
}
}
}
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.ServerRunOptions) error {
// set defaults
if err := s.GenericServerRunOptions.DefaultAdvertiseAddress(s.SecureServing, s.InsecureServing); err != nil {
return err
}
serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
if err != nil {
return fmt.Errorf("error determining service IP ranges: %v", err)
}
if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), apiServerServiceIP); err != nil {
return fmt.Errorf("error creating self-signed certificates: %v", err)
}
if err := s.CloudProvider.DefaultExternalHost(s.GenericServerRunOptions); err != nil {
return fmt.Errorf("error setting the external host value: %v", err)
}
s.Authentication.ApplyAuthorization(s.Authorization)
// validate options
if errs := s.Validate(); len(errs) != 0 {
return utilerrors.NewAggregate(errs)
}
// create config from options
genericConfig := genericapiserver.NewConfig().
ApplyOptions(s.GenericServerRunOptions).
ApplyInsecureServingOptions(s.InsecureServing)
if _, err := genericConfig.ApplySecureServingOptions(s.SecureServing); err != nil {
return fmt.Errorf("failed to configure https: %s", err)
}
if err = s.Authentication.Apply(genericConfig); err != nil {
return fmt.Errorf("failed to configure authentication: %s", err)
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
// Setup nodeTunneler if needed
var nodeTunneler tunneler.Tunneler
var proxyDialerFn utilnet.DialFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSHKey tunneler.InstallSSHKey
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider.CloudProvider, s.CloudProvider.CloudConfigFile)
if err != nil {
return fmt.Errorf("cloud provider could not be initialized: %v", err)
}
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSHKey = instances.AddSSHKeyToAllInstances
}
}
if s.KubeletConfig.Port == 0 {
return fmt.Errorf("must enable kubelet port if proxy ssh-tunneling is specified")
}
if s.KubeletConfig.ReadOnlyPort == 0 {
return fmt.Errorf("must enable kubelet readonly port if proxy ssh-tunneling is specified")
}
// Set up the nodeTunneler
// TODO(cjcullen): If we want this to handle per-kubelet ports or other
// kubelet listen-addresses, we need to plumb through options.
healthCheckPath := &url.URL{
Scheme: "http",
Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.ReadOnlyPort), 10)),
Path: "healthz",
}
nodeTunneler = tunneler.New(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSHKey)
// Use the nodeTunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = nodeTunneler.Dial
// Use the nodeTunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = nodeTunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
if s.Etcd.StorageConfig.DeserializationCacheSize == 0 {
// When size of cache is not explicitly set, estimate its size based on
// target memory usage.
glog.V(2).Infof("Initializing deserialization cache size based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB)
// This is the heuristics that from memory capacity is trying to infer
// the maximum number of nodes in the cluster and set cache sizes based
// on that value.
// From our documentation, we officially recomment 120GB machines for
// 2000 nodes, and we scale from that point. Thus we assume ~60MB of
// capacity per node.
// TODO: We may consider deciding that some percentage of memory will
// be used for the deserialization cache and divide it by the max object
// size to compute its size. We may even go further and measure
// collective sizes of the objects in the cache.
clusterSize := s.GenericServerRunOptions.TargetRAMMB / 60
s.Etcd.StorageConfig.DeserializationCacheSize = 25 * clusterSize
if s.Etcd.StorageConfig.DeserializationCacheSize < 1000 {
s.Etcd.StorageConfig.DeserializationCacheSize = 1000
}
}
storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion()
if err != nil {
return fmt.Errorf("error generating storage version map: %s", err)
}
storageFactory, err := kubeapiserver.BuildDefaultStorageFactory(
s.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs,
genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
// FIXME: this GroupVersionResource override should be configurable
[]schema.GroupVersionResource{batch.Resource("cronjobs").WithVersion("v2alpha1")},
master.DefaultAPIResourceConfigSource(), s.GenericServerRunOptions.RuntimeConfig)
if err != nil {
return fmt.Errorf("error in initializing storage factory: %s", err)
}
storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers"))
for _, override := range s.Etcd.EtcdServersOverrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
continue
}
group := apiresource[0]
resource := apiresource[1]
groupResource := schema.GroupResource{Group: group, Resource: resource}
servers := strings.Split(tokens[1], ";")
storageFactory.SetEtcdLocation(groupResource, servers)
}
// Default to the private server key for service account token signing
if len(s.Authentication.ServiceAccounts.KeyFiles) == 0 && s.SecureServing.ServerCert.CertKey.KeyFile != "" {
if kubeauthenticator.IsValidServiceAccountKeyFile(s.SecureServing.ServerCert.CertKey.KeyFile) {
s.Authentication.ServiceAccounts.KeyFiles = []string{s.SecureServing.ServerCert.CertKey.KeyFile}
} else {
glog.Warning("No TLS key provided, service account token authentication disabled")
}
}
authenticatorConfig := s.Authentication.ToAuthenticationConfig()
if s.Authentication.ServiceAccounts.Lookup {
// If we need to look up service accounts and tokens,
// go directly to etcd to avoid recursive auth insanity
storageConfig, err := storageFactory.NewConfig(api.Resource("serviceaccounts"))
if err != nil {
return fmt.Errorf("unable to get serviceaccounts storage: %v", err)
}
authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets")))
}
apiAuthenticator, securityDefinitions, err := authenticatorConfig.New()
if err != nil {
return fmt.Errorf("invalid Authentication Config: %v", err)
}
privilegedLoopbackToken := uuid.NewRandom().String()
selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken)
if err != nil {
return fmt.Errorf("failed to create clientset: %v", err)
}
client, err := internalclientset.NewForConfig(selfClientConfig)
if err != nil {
kubeAPIVersions := os.Getenv("KUBE_API_VERSIONS")
if len(kubeAPIVersions) == 0 {
return fmt.Errorf("failed to create clientset: %v", err)
}
// KUBE_API_VERSIONS is used in test-update-storage-objects.sh, disabling a number of API
// groups. This leads to a nil client above and undefined behaviour further down.
// TODO: get rid of KUBE_API_VERSIONS or define sane behaviour if set
glog.Errorf("Failed to create clientset with KUBE_API_VERSIONS=%q. KUBE_API_VERSIONS is only for testing. Things will break.", kubeAPIVersions)
}
sharedInformers := informers.NewSharedInformerFactory(nil, client, 10*time.Minute)
authorizationConfig := s.Authorization.ToAuthorizationConfig(sharedInformers)
apiAuthorizer, err := authorizationConfig.New()
if err != nil {
return fmt.Errorf("invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.GenericServerRunOptions.AdmissionControl, ",")
pluginInitializer := kubeadmission.NewPluginInitializer(client, sharedInformers, apiAuthorizer)
admissionConfigProvider, err := kubeadmission.ReadAdmissionConfiguration(admissionControlPluginNames, s.GenericServerRunOptions.AdmissionControlConfigFile)
if err != nil {
return fmt.Errorf("failed to read plugin config: %v", err)
}
admissionController, err := admission.NewFromPlugins(admissionControlPluginNames, admissionConfigProvider, pluginInitializer)
if err != nil {
return fmt.Errorf("failed to initialize plugins: %v", err)
}
proxyTransport := utilnet.SetTransportDefaults(&http.Transport{
Dial: proxyDialerFn,
TLSClientConfig: proxyTLSClientConfig,
})
kubeVersion := version.Get()
genericConfig.Version = &kubeVersion
genericConfig.LoopbackClientConfig = selfClientConfig
genericConfig.Authenticator = apiAuthenticator
genericConfig.Authorizer = apiAuthorizer
genericConfig.AdmissionControl = admissionController
genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.OpenAPIDefinitions)
genericConfig.OpenAPIConfig.SecurityDefinitions = securityDefinitions
genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"
genericConfig.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
genericConfig.EnableMetrics = true
genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
sets.NewString("watch", "proxy"),
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
)
config := &master.Config{
GenericConfig: genericConfig,
APIResourceConfigSource: storageFactory.APIResourceConfigSource,
StorageFactory: storageFactory,
EnableWatchCache: s.GenericServerRunOptions.EnableWatchCache,
EnableCoreControllers: true,
DeleteCollectionWorkers: s.GenericServerRunOptions.DeleteCollectionWorkers,
EventTTL: s.EventTTL,
KubeletClientConfig: s.KubeletConfig,
EnableUISupport: true,
EnableLogsSupport: true,
ProxyTransport: proxyTransport,
Tunneler: nodeTunneler,
ServiceIPRange: serviceIPRange,
APIServerServiceIP: apiServerServiceIP,
APIServerServicePort: 443,
ServiceNodePortRange: s.ServiceNodePortRange,
KubernetesServiceNodePort: s.KubernetesServiceNodePort,
MasterCount: s.MasterCount,
}
if s.GenericServerRunOptions.EnableWatchCache {
glog.V(2).Infof("Initializing cache sizes based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB)
cachesize.InitializeWatchCacheSizes(s.GenericServerRunOptions.TargetRAMMB)
cachesize.SetWatchCacheSizes(s.GenericServerRunOptions.WatchCacheSizes)
}
m, err := config.Complete().New()
if err != nil {
return err
}
sharedInformers.Start(wait.NeverStop)
m.GenericAPIServer.PrepareRun().Run(wait.NeverStop)
return nil
}
func TestJobScaleFailsPreconditions(t *testing.T) {
ten := int32(10)
fake := fake.NewSimpleClientset(&batch.Job{
ObjectMeta: metav1.ObjectMeta{
Namespace: api.NamespaceDefault,
Name: "foo",
},
Spec: batch.JobSpec{
Parallelism: &ten,
},
})
scaler := JobScaler{fake.Batch()}
preconditions := ScalePrecondition{2, ""}
count := uint(3)
name := "foo"
scaler.Scale("default", name, count, &preconditions, nil, nil)
actions := fake.Actions()
if len(actions) != 1 {
t.Errorf("unexpected actions: %v, expected 1 actions (get)", actions)
}
if action, ok := actions[0].(testcore.GetAction); !ok || action.GetResource().GroupResource() != batch.Resource("jobs") || action.GetName() != name {
t.Errorf("unexpected action: %v, expected get-job %s", actions[0], name)
}
}
func (f *Factory) PodForResource(resource string, timeout time.Duration) (string, error) {
sortBy := func(pods []*api.Pod) sort.Interface { return sort.Reverse(controller.ActivePods(pods)) }
namespace, _, err := f.DefaultNamespace()
if err != nil {
return "", err
}
mapper, _ := f.Object(false)
resourceType, name, err := util.ResolveResource(api.Resource("pods"), resource, mapper)
if err != nil {
return "", err
}
switch resourceType {
case api.Resource("pods"):
return name, nil
case api.Resource("replicationcontrollers"):
kc, err := f.Client()
if err != nil {
return "", err
}
rc, err := kc.ReplicationControllers(namespace).Get(name)
if err != nil {
return "", err
}
selector := labels.SelectorFromSet(rc.Spec.Selector)
pod, _, err := cmdutil.GetFirstPod(kc, namespace, selector, timeout, sortBy)
if err != nil {
return "", err
}
return pod.Name, nil
case deployapi.Resource("deploymentconfigs"):
oc, kc, err := f.Clients()
if err != nil {
return "", err
}
dc, err := oc.DeploymentConfigs(namespace).Get(name)
if err != nil {
return "", err
}
selector := labels.SelectorFromSet(dc.Spec.Selector)
pod, _, err := cmdutil.GetFirstPod(kc, namespace, selector, timeout, sortBy)
if err != nil {
return "", err
}
return pod.Name, nil
case extensions.Resource("daemonsets"):
kc, err := f.Client()
if err != nil {
return "", err
}
ds, err := kc.Extensions().DaemonSets(namespace).Get(name)
if err != nil {
return "", err
}
selector, err := unversioned.LabelSelectorAsSelector(ds.Spec.Selector)
if err != nil {
return "", err
}
pod, _, err := cmdutil.GetFirstPod(kc, namespace, selector, timeout, sortBy)
if err != nil {
return "", err
}
return pod.Name, nil
case extensions.Resource("jobs"):
kc, err := f.Client()
if err != nil {
return "", err
}
job, err := kc.Extensions().Jobs(namespace).Get(name)
if err != nil {
return "", err
}
return podNameForJob(job, kc, timeout, sortBy)
case batch.Resource("jobs"):
kc, err := f.Client()
if err != nil {
return "", err
}
job, err := kc.Batch().Jobs(namespace).Get(name)
if err != nil {
return "", err
}
return podNameForJob(job, kc, timeout, sortBy)
default:
return "", fmt.Errorf("remote shell for %s is not supported", resourceType)
}
}
// BuildDefaultAPIServer constructs the appropriate APIServer and StorageFactory for the kubernetes server.
// It returns an error if no KubernetesMasterConfig was defined.
func BuildDefaultAPIServer(options configapi.MasterConfig) (*apiserveroptions.APIServer, genericapiserver.StorageFactory, error) {
if options.KubernetesMasterConfig == nil {
return nil, nil, fmt.Errorf("no kubernetesMasterConfig defined, unable to load settings")
}
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, nil, err
}
portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, nil, err
}
// Defaults are tested in TestAPIServerDefaults
server := apiserveroptions.NewAPIServer()
// Adjust defaults
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.EnableLogsSupport = false // don't expose server logs
server.EnableProfiling = false
server.APIPrefix = KubeAPIPrefix
server.APIGroupPrefix = KubeAPIGroupPrefix
server.SecurePort = port
server.MasterCount = options.KubernetesMasterConfig.MasterCount
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, nil, kerrors.NewAggregate(err)
}
resourceEncodingConfig := genericapiserver.NewDefaultResourceEncodingConfig()
resourceEncodingConfig.SetVersionEncoding(
kapi.GroupName,
unversioned.GroupVersion{Group: kapi.GroupName, Version: options.EtcdStorageConfig.KubernetesStorageVersion},
kapi.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
extensions.GroupName,
unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"},
extensions.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
batch.GroupName,
unversioned.GroupVersion{Group: batch.GroupName, Version: "v1"},
batch.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
autoscaling.GroupName,
unversioned.GroupVersion{Group: autoscaling.GroupName, Version: "v1"},
autoscaling.SchemeGroupVersion,
)
storageGroupsToEncodingVersion, err := server.StorageGroupsToEncodingVersion()
if err != nil {
return nil, nil, err
}
// use the stock storage config based on args, but override bits from our config where appropriate
etcdConfig := server.StorageConfig
etcdConfig.Prefix = options.EtcdStorageConfig.KubernetesStoragePrefix
etcdConfig.ServerList = options.EtcdClientInfo.URLs
etcdConfig.KeyFile = options.EtcdClientInfo.ClientCert.KeyFile
etcdConfig.CertFile = options.EtcdClientInfo.ClientCert.CertFile
etcdConfig.CAFile = options.EtcdClientInfo.CA
storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
etcdConfig,
server.DefaultStorageMediaType,
kapi.Codecs,
genericapiserver.NewDefaultResourceEncodingConfig(),
storageGroupsToEncodingVersion,
// FIXME: this GroupVersionResource override should be configurable
[]unversioned.GroupVersionResource{batch.Resource("scheduledjobs").WithVersion("v2alpha1")},
master.DefaultAPIResourceConfigSource(), server.RuntimeConfig,
)
if err != nil {
return nil, nil, err
}
/*storageFactory := genericapiserver.NewDefaultStorageFactory(
etcdConfig,
server.DefaultStorageMediaType,
kapi.Codecs,
resourceEncodingConfig,
master.DefaultAPIResourceConfigSource(),
)*/
// the order here is important, it defines which version will be used for storage
storageFactory.AddCohabitatingResources(extensions.Resource("jobs"), batch.Resource("jobs"))
storageFactory.AddCohabitatingResources(extensions.Resource("horizontalpodautoscalers"), autoscaling.Resource("horizontalpodautoscalers"))
return server, storageFactory, nil
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.APIServer) error {
genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions)
if len(s.StorageConfig.ServerList) == 0 {
glog.Fatalf("--etcd-servers must be specified")
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
// Setup tunneler if needed
var tunneler genericapiserver.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH genericapiserver.InstallSSHKey
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
if s.KubeletConfig.Port == 0 {
glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.")
}
// Set up the tunneler
// TODO(cjcullen): If we want this to handle per-kubelet ports or other
// kubelet listen-addresses, we need to plumb through options.
healthCheckPath := &url.URL{
Scheme: "https",
Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)),
Path: "healthz",
}
tunneler = genericapiserver.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig)
if err != nil {
glog.Fatalf("Failure to start kubelet client: %v", err)
}
apiResourceConfigSource, err := parseRuntimeConfig(s)
if err != nil {
glog.Fatalf("error in parsing runtime-config: %s", err)
}
clientConfig := &restclient.Config{
Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
// Increase QPS limits. The client is currently passed to all admission plugins,
// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
// for more details. Once #22422 is fixed, we may want to remove it.
QPS: 50,
Burst: 100,
}
if len(s.DeprecatedStorageVersion) != 0 {
gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion)
if err != nil {
glog.Fatalf("error in parsing group version: %s", err)
}
clientConfig.GroupVersion = &gv
}
client, err := clientset.NewForConfig(clientConfig)
if err != nil {
glog.Errorf("Failed to create clientset: %v", err)
}
resourceEncoding := genericapiserver.NewDefaultResourceEncodingConfig()
groupToEncoding, err := s.StorageGroupsToEncodingVersion()
if err != nil {
glog.Fatalf("error getting group encoding: %s", err)
}
for group, storageEncodingVersion := range groupToEncoding {
resourceEncoding.SetVersionEncoding(group, storageEncodingVersion, unversioned.GroupVersion{Group: group, Version: runtime.APIVersionInternal})
}
storageFactory := genericapiserver.NewDefaultStorageFactory(s.StorageConfig, api.Codecs, resourceEncoding, apiResourceConfigSource)
storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs"))
storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers"))
for _, override := range s.EtcdServersOverrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
continue
}
group := apiresource[0]
resource := apiresource[1]
groupResource := unversioned.GroupResource{Group: group, Resource: resource}
servers := strings.Split(tokens[1], ";")
storageFactory.SetEtcdLocation(groupResource, servers)
}
// Default to the private server key for service account token signing
if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
} else {
glog.Warning("No RSA key provided, service account token authentication disabled")
}
}
var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter
if s.ServiceAccountLookup {
// If we need to look up service accounts and tokens,
// go directly to etcd to avoid recursive auth insanity
storage, err := storageFactory.New(api.Resource("serviceaccounts"))
if err != nil {
glog.Fatalf("Unable to get serviceaccounts storage: %v", err)
}
serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storage)
}
authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
BasicAuthFile: s.BasicAuthFile,
ClientCAFile: s.ClientCAFile,
TokenAuthFile: s.TokenAuthFile,
OIDCIssuerURL: s.OIDCIssuerURL,
OIDCClientID: s.OIDCClientID,
OIDCCAFile: s.OIDCCAFile,
OIDCUsernameClaim: s.OIDCUsernameClaim,
OIDCGroupsClaim: s.OIDCGroupsClaim,
ServiceAccountKeyFile: s.ServiceAccountKeyFile,
ServiceAccountLookup: s.ServiceAccountLookup,
ServiceAccountTokenGetter: serviceAccountGetter,
KeystoneURL: s.KeystoneURL,
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationConfig)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
if len(s.ExternalHost) == 0 {
// TODO: extend for other providers
if s.CloudProvider == "gce" {
instances, supported := cloud.Instances()
if !supported {
glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.")
}
name, err := os.Hostname()
if err != nil {
glog.Fatalf("Failed to get hostname: %v", err)
}
addrs, err := instances.NodeAddresses(name)
if err != nil {
glog.Warningf("Unable to obtain external host address from cloud provider: %v", err)
} else {
for _, addr := range addrs {
if addr.Type == api.NodeExternalIP {
s.ExternalHost = addr.Address
}
}
}
}
}
genericConfig := genericapiserver.NewConfig(s.ServerRunOptions)
// TODO: Move the following to generic api server as well.
genericConfig.StorageFactory = storageFactory
genericConfig.Authenticator = authenticator
genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0
genericConfig.Authorizer = authorizer
genericConfig.AdmissionControl = admissionController
genericConfig.APIResourceConfigSource = apiResourceConfigSource
genericConfig.MasterServiceNamespace = s.MasterServiceNamespace
genericConfig.ProxyDialer = proxyDialerFn
genericConfig.ProxyTLSClientConfig = proxyTLSClientConfig
genericConfig.Serializer = api.Codecs
config := &master.Config{
Config: genericConfig,
EnableCoreControllers: true,
DeleteCollectionWorkers: s.DeleteCollectionWorkers,
EventTTL: s.EventTTL,
KubeletClient: kubeletClient,
Tunneler: tunneler,
}
if s.EnableWatchCache {
cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
}
m, err := master.New(config)
if err != nil {
return err
}
m.Run(s.ServerRunOptions)
return nil
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.APIServer) error {
genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions)
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
// Setup tunneler if needed
var tunneler genericapiserver.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH genericapiserver.InstallSSHKey
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
if s.KubeletConfig.Port == 0 {
glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.")
}
// Set up the tunneler
// TODO(cjcullen): If we want this to handle per-kubelet ports or other
// kubelet listen-addresses, we need to plumb through options.
healthCheckPath := &url.URL{
Scheme: "https",
Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)),
Path: "healthz",
}
tunneler = genericapiserver.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig)
if err != nil {
glog.Fatalf("Failure to start kubelet client: %v", err)
}
storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion()
if err != nil {
glog.Fatalf("error generating storage version map: %s", err)
}
storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
s.StorageConfig, s.DefaultStorageMediaType, api.Codecs,
genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
master.DefaultAPIResourceConfigSource(), s.RuntimeConfig)
if err != nil {
glog.Fatalf("error in initializing storage factory: %s", err)
}
storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs"))
storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers"))
for _, override := range s.EtcdServersOverrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
continue
}
group := apiresource[0]
resource := apiresource[1]
groupResource := unversioned.GroupResource{Group: group, Resource: resource}
servers := strings.Split(tokens[1], ";")
storageFactory.SetEtcdLocation(groupResource, servers)
}
// Default to the private server key for service account token signing
if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
} else {
glog.Warning("No RSA key provided, service account token authentication disabled")
}
}
var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter
if s.ServiceAccountLookup {
// If we need to look up service accounts and tokens,
// go directly to etcd to avoid recursive auth insanity
storage, err := storageFactory.New(api.Resource("serviceaccounts"))
if err != nil {
glog.Fatalf("Unable to get serviceaccounts storage: %v", err)
}
serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storage)
}
authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
BasicAuthFile: s.BasicAuthFile,
ClientCAFile: s.ClientCAFile,
TokenAuthFile: s.TokenAuthFile,
OIDCIssuerURL: s.OIDCIssuerURL,
OIDCClientID: s.OIDCClientID,
OIDCCAFile: s.OIDCCAFile,
OIDCUsernameClaim: s.OIDCUsernameClaim,
OIDCGroupsClaim: s.OIDCGroupsClaim,
ServiceAccountKeyFile: s.ServiceAccountKeyFile,
ServiceAccountLookup: s.ServiceAccountLookup,
ServiceAccountTokenGetter: serviceAccountGetter,
KeystoneURL: s.KeystoneURL,
WebhookTokenAuthnConfigFile: s.WebhookTokenAuthnConfigFile,
WebhookTokenAuthnCacheTTL: s.WebhookTokenAuthnCacheTTL,
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationConfig)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
client, err := s.NewSelfClient()
if err != nil {
glog.Errorf("Failed to create clientset: %v", err)
}
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
genericConfig := genericapiserver.NewConfig(s.ServerRunOptions)
// TODO: Move the following to generic api server as well.
genericConfig.StorageFactory = storageFactory
genericConfig.Authenticator = authenticator
genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0
genericConfig.Authorizer = authorizer
genericConfig.AdmissionControl = admissionController
genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
genericConfig.MasterServiceNamespace = s.MasterServiceNamespace
genericConfig.ProxyDialer = proxyDialerFn
genericConfig.ProxyTLSClientConfig = proxyTLSClientConfig
genericConfig.Serializer = api.Codecs
config := &master.Config{
Config: genericConfig,
EnableCoreControllers: true,
DeleteCollectionWorkers: s.DeleteCollectionWorkers,
EventTTL: s.EventTTL,
KubeletClient: kubeletClient,
Tunneler: tunneler,
}
if s.EnableWatchCache {
cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
}
m, err := master.New(config)
if err != nil {
return err
}
m.Run(s.ServerRunOptions)
return nil
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.APIServer) error {
genericvalidation.VerifyEtcdServersList(s.ServerRunOptions)
genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions)
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
// Setup tunneler if needed
var tunneler genericapiserver.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH genericapiserver.InstallSSHKey
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
if s.KubeletConfig.Port == 0 {
glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.")
}
// Set up the tunneler
// TODO(cjcullen): If we want this to handle per-kubelet ports or other
// kubelet listen-addresses, we need to plumb through options.
healthCheckPath := &url.URL{
Scheme: "https",
Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)),
Path: "healthz",
}
tunneler = genericapiserver.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig)
if err != nil {
glog.Fatalf("Failed to start kubelet client: %v", err)
}
storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion()
if err != nil {
glog.Fatalf("error generating storage version map: %s", err)
}
storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
s.StorageConfig, s.DefaultStorageMediaType, api.Codecs,
genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
// FIXME: this GroupVersionResource override should be configurable
[]unversioned.GroupVersionResource{batch.Resource("scheduledjobs").WithVersion("v2alpha1")},
master.DefaultAPIResourceConfigSource(), s.RuntimeConfig)
if err != nil {
glog.Fatalf("error in initializing storage factory: %s", err)
}
storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs"))
storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers"))
for _, override := range s.EtcdServersOverrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
continue
}
group := apiresource[0]
resource := apiresource[1]
groupResource := unversioned.GroupResource{Group: group, Resource: resource}
servers := strings.Split(tokens[1], ";")
storageFactory.SetEtcdLocation(groupResource, servers)
}
// Default to the private server key for service account token signing
if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
} else {
glog.Warning("No RSA key provided, service account token authentication disabled")
}
}
var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter
if s.ServiceAccountLookup {
// If we need to look up service accounts and tokens,
// go directly to etcd to avoid recursive auth insanity
storageConfig, err := storageFactory.NewConfig(api.Resource("serviceaccounts"))
if err != nil {
glog.Fatalf("Unable to get serviceaccounts storage: %v", err)
}
serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets")))
}
apiAuthenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
Anonymous: s.AnonymousAuth,
AnyToken: s.EnableAnyToken,
BasicAuthFile: s.BasicAuthFile,
ClientCAFile: s.ClientCAFile,
TokenAuthFile: s.TokenAuthFile,
OIDCIssuerURL: s.OIDCIssuerURL,
OIDCClientID: s.OIDCClientID,
OIDCCAFile: s.OIDCCAFile,
OIDCUsernameClaim: s.OIDCUsernameClaim,
OIDCGroupsClaim: s.OIDCGroupsClaim,
ServiceAccountKeyFile: s.ServiceAccountKeyFile,
ServiceAccountLookup: s.ServiceAccountLookup,
ServiceAccountTokenGetter: serviceAccountGetter,
KeystoneURL: s.KeystoneURL,
WebhookTokenAuthnConfigFile: s.WebhookTokenAuthnConfigFile,
WebhookTokenAuthnCacheTTL: s.WebhookTokenAuthnCacheTTL,
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
modeEnabled := func(mode string) bool {
for _, m := range authorizationModeNames {
if m == mode {
return true
}
}
return false
}
authorizationConfig := authorizer.AuthorizationConfig{
PolicyFile: s.AuthorizationPolicyFile,
WebhookConfigFile: s.AuthorizationWebhookConfigFile,
WebhookCacheAuthorizedTTL: s.AuthorizationWebhookCacheAuthorizedTTL,
WebhookCacheUnauthorizedTTL: s.AuthorizationWebhookCacheUnauthorizedTTL,
RBACSuperUser: s.AuthorizationRBACSuperUser,
}
if modeEnabled(genericoptions.ModeRBAC) {
mustGetRESTOptions := func(resource string) generic.RESTOptions {
config, err := storageFactory.NewConfig(rbac.Resource(resource))
if err != nil {
glog.Fatalf("Unable to get %s storage: %v", resource, err)
}
return generic.RESTOptions{StorageConfig: config, Decorator: generic.UndecoratedStorage, ResourcePrefix: storageFactory.ResourcePrefix(rbac.Resource(resource))}
}
// For initial bootstrapping go directly to etcd to avoid privillege escalation check.
authorizationConfig.RBACRoleRegistry = role.NewRegistry(roleetcd.NewREST(mustGetRESTOptions("roles")))
authorizationConfig.RBACRoleBindingRegistry = rolebinding.NewRegistry(rolebindingetcd.NewREST(mustGetRESTOptions("rolebindings")))
authorizationConfig.RBACClusterRoleRegistry = clusterrole.NewRegistry(clusterroleetcd.NewREST(mustGetRESTOptions("clusterroles")))
authorizationConfig.RBACClusterRoleBindingRegistry = clusterrolebinding.NewRegistry(clusterrolebindingetcd.NewREST(mustGetRESTOptions("clusterrolebindings")))
}
apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
privilegedLoopbackToken := uuid.NewRandom().String()
client, err := s.NewSelfClient(privilegedLoopbackToken)
if err != nil {
glog.Errorf("Failed to create clientset: %v", err)
}
// TODO(dims): We probably need to add an option "EnableLoopbackToken"
if apiAuthenticator != nil {
var uid = uuid.NewRandom().String()
tokens := make(map[string]*user.DefaultInfo)
tokens[privilegedLoopbackToken] = &user.DefaultInfo{
Name: "system:apiserver",
UID: uid,
Groups: []string{"system:masters"},
}
tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens)
apiAuthenticator = authenticatorunion.New(tokenAuthenticator, apiAuthenticator)
tokenAuthorizer := authorizer.NewPrivilegedGroups("system:masters")
apiAuthorizer = authorizerunion.New(tokenAuthorizer, apiAuthorizer)
}
sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)
pluginInitializer := admission.NewPluginInitializer(sharedInformers)
admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile, pluginInitializer)
if err != nil {
glog.Fatalf("Failed to initialize plugins: %v", err)
}
genericConfig := genericapiserver.NewConfig(s.ServerRunOptions)
// TODO: Move the following to generic api server as well.
genericConfig.Authenticator = apiAuthenticator
genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0
genericConfig.Authorizer = apiAuthorizer
genericConfig.AuthorizerRBACSuperUser = s.AuthorizationRBACSuperUser
genericConfig.AdmissionControl = admissionController
genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
genericConfig.MasterServiceNamespace = s.MasterServiceNamespace
genericConfig.ProxyDialer = proxyDialerFn
genericConfig.ProxyTLSClientConfig = proxyTLSClientConfig
genericConfig.Serializer = api.Codecs
genericConfig.OpenAPIInfo.Title = "Kubernetes"
genericConfig.OpenAPIDefinitions = openapi.OpenAPIDefinitions
genericConfig.EnableOpenAPISupport = true
config := &master.Config{
GenericConfig: genericConfig,
StorageFactory: storageFactory,
EnableWatchCache: s.EnableWatchCache,
EnableCoreControllers: true,
DeleteCollectionWorkers: s.DeleteCollectionWorkers,
EventTTL: s.EventTTL,
KubeletClient: kubeletClient,
EnableUISupport: true,
EnableLogsSupport: true,
Tunneler: tunneler,
}
if s.EnableWatchCache {
glog.V(2).Infof("Initalizing cache sizes based on %dMB limit", s.TargetRAMMB)
cachesize.InitializeWatchCacheSizes(s.TargetRAMMB)
cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
}
m, err := config.Complete().New()
if err != nil {
return err
}
sharedInformers.Start(wait.NeverStop)
m.Run(s.ServerRunOptions)
return nil
}
type podNodeConstraints struct {
*admission.Handler
selectorLabelBlacklist sets.String
config *api.PodNodeConstraintsConfig
authorizer authorizer.Authorizer
}
// resourcesToCheck is a map of resources and corresponding kinds of things that
// we want handled in this plugin
// TODO: Include a function that will extract the PodSpec from the resource for
// each type added here.
var resourcesToCheck = map[unversioned.GroupResource]unversioned.GroupKind{
kapi.Resource("pods"): kapi.Kind("Pod"),
kapi.Resource("podtemplates"): kapi.Kind("PodTemplate"),
kapi.Resource("replicationcontrollers"): kapi.Kind("ReplicationController"),
batch.Resource("jobs"): batch.Kind("Job"),
batch.Resource("jobtemplates"): batch.Kind("JobTemplate"),
batch.Resource("scheduledjobs"): batch.Kind("ScheduledJob"),
extensions.Resource("deployments"): extensions.Kind("Deployment"),
extensions.Resource("replicasets"): extensions.Kind("ReplicaSet"),
extensions.Resource("jobs"): extensions.Kind("Job"),
extensions.Resource("jobtemplates"): extensions.Kind("JobTemplate"),
apps.Resource("petsets"): apps.Kind("PetSet"),
deployapi.Resource("deploymentconfigs"): deployapi.Kind("DeploymentConfig"),
securityapi.Resource("podsecuritypolicysubjectreviews"): securityapi.Kind("PodSecurityPolicySubjectReview"),
securityapi.Resource("podsecuritypolicyselfsubjectreviews"): securityapi.Kind("PodSecurityPolicySelfSubjectReview"),
securityapi.Resource("podsecuritypolicyreviews"): securityapi.Kind("PodSecurityPolicyReview"),
}
// resourcesToIgnore is a list of resource kinds that contain a PodSpec that
// we choose not to handle in this plugin