// TestValidOpenAPISpec verifies that the open api is added
// at the proper endpoint and the spec is valid.
func TestValidOpenAPISpec(t *testing.T) {
_, etcdserver, config, assert := setUp(t)
defer etcdserver.Terminate(t)
config.GenericConfig.EnableIndex = true
config.GenericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(openapigen.OpenAPIDefinitions)
config.GenericConfig.OpenAPIConfig.Info = &spec.Info{
InfoProps: spec.InfoProps{
Title: "Kubernetes",
Version: "unversioned",
},
}
config.GenericConfig.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
master, err := config.Complete().New()
if err != nil {
t.Fatalf("Error in bringing up the master: %v", err)
}
// make sure swagger.json is not registered before calling PrepareRun.
server := httptest.NewServer(master.GenericAPIServer.HandlerContainer.ServeMux)
defer server.Close()
resp, err := http.Get(server.URL + "/swagger.json")
if !assert.NoError(err) {
t.Errorf("unexpected error: %v", err)
}
assert.Equal(http.StatusNotFound, resp.StatusCode)
master.GenericAPIServer.PrepareRun()
resp, err = http.Get(server.URL + "/swagger.json")
if !assert.NoError(err) {
t.Errorf("unexpected error: %v", err)
}
assert.Equal(http.StatusOK, resp.StatusCode)
// as json schema
var sch spec.Schema
if assert.NoError(decodeResponse(resp, &sch)) {
validator := validate.NewSchemaValidator(spec.MustLoadSwagger20Schema(), nil, "", strfmt.Default)
res := validator.Validate(&sch)
assert.NoError(res.AsError())
}
// Validate OpenApi spec
doc, err := loads.Spec(server.URL + "/swagger.json")
if assert.NoError(err) {
validator := validate.NewSpecValidator(doc.Schema(), strfmt.Default)
res, warns := validator.Validate(doc)
assert.NoError(res.AsError())
if !warns.IsValid() {
t.Logf("Open API spec on root has some warnings : %v", warns)
}
}
}
func Run(serverOptions *genericoptions.ServerRunOptions, stopCh <-chan struct{}) error {
// Set ServiceClusterIPRange
_, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
serverOptions.ServiceClusterIPRange = *serviceClusterIPRange
serverOptions.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"}
genericvalidation.ValidateRunOptions(serverOptions)
genericvalidation.VerifyEtcdServersList(serverOptions)
config := genericapiserver.NewConfig().ApplyOptions(serverOptions).Complete()
if err := config.MaybeGenerateServingCerts(); err != nil {
// this wasn't treated as fatal for this process before
fmt.Printf("Error creating cert: %v", err)
}
config.Authorizer = authorizer.NewAlwaysAllowAuthorizer()
config.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
s, err := config.New()
if err != nil {
return fmt.Errorf("Error in bringing up the server: %v", err)
}
groupVersion := v1.SchemeGroupVersion
groupName := groupVersion.Group
groupMeta, err := registered.Group(groupName)
if err != nil {
return fmt.Errorf("%v", err)
}
storageFactory := newStorageFactory()
storageConfig, err := storageFactory.NewConfig(unversioned.GroupResource{Group: groupName, Resource: "testtype"})
if err != nil {
return fmt.Errorf("Unable to get storage config: %v", err)
}
restStorageMap := map[string]rest.Storage{
"testtypes": testgroupetcd.NewREST(storageConfig, generic.UndecoratedStorage),
}
apiGroupInfo := genericapiserver.APIGroupInfo{
GroupMeta: *groupMeta,
VersionedResourcesStorageMap: map[string]map[string]rest.Storage{
groupVersion.Version: restStorageMap,
},
Scheme: api.Scheme,
NegotiatedSerializer: api.Codecs,
}
if err := s.InstallAPIGroup(&apiGroupInfo); err != nil {
return fmt.Errorf("Error in installing API: %v", err)
}
s.PrepareRun().Run(stopCh)
return nil
}
func (c *MasterConfig) RunInProxyMode(proxy *kubernetes.ProxyConfig, assetConfig *AssetConfig) {
handlerChain, messages, err := c.buildHandlerChain(assetConfig)
if err != nil {
glog.Fatalf("Failed to launch master: %v", err)
}
// TODO(sttts): create a genericapiserver here
container := genericmux.NewAPIContainer(http.NewServeMux(), kapi.Codecs)
// install /api proxy forwarder
proxyMessages, err := proxy.InstallAPI(container.Container)
if err != nil {
glog.Fatalf("Failed to launch master: %v", err)
}
messages = append(messages, proxyMessages...)
// install GenericAPIServer handlers manually, usually done by GenericAPIServer.PrepareRun()
healthz.InstallHandler(&container.NonSwaggerRoutes, healthz.PingHealthz)
swaggerConfig := genericapiserver.DefaultSwaggerConfig()
swaggerConfig.WebServicesUrl = c.Options.MasterPublicURL
genericroutes.Swagger{Config: swaggerConfig}.Install(container)
messages = append(messages, fmt.Sprintf("Started Swagger Schema API at %%s%s", swaggerConfig.ApiPath))
genericroutes.OpenAPI{Config: kubernetes.DefaultOpenAPIConfig()}.Install(container)
messages = append(messages, fmt.Sprintf("Started OpenAPI Schema at %%s%s", openAPIServePath))
// install origin handlers
c.InstallProtectedAPI(container)
// TODO(sttts): split cmd/server/kubernetes config generation into generic and master-specific
// until then: create ad-hoc config
genericConfig := genericapiserver.NewConfig()
genericConfig.RequestContextMapper = c.RequestContextMapper
genericConfig.LegacyAPIGroupPrefixes = kubernetes.LegacyAPIGroupPrefixes
genericConfig.MaxRequestsInFlight = c.Options.ServingInfo.MaxRequestsInFlight
secureHandler, _ := handlerChain(container.ServeMux, genericConfig)
c.serve(secureHandler, messages)
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try)
cmdutil.WaitForSuccessfulDial(c.TLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.ServerRunOptions) error {
// set defaults
if err := s.GenericServerRunOptions.DefaultAdvertiseAddress(s.SecureServing, s.InsecureServing); err != nil {
return err
}
if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String()); err != nil {
return fmt.Errorf("error creating self-signed certificates: %v", err)
}
if err := s.GenericServerRunOptions.DefaultExternalHost(); err != nil {
return fmt.Errorf("error setting the external host value: %v", err)
}
// validate options
if errs := s.Validate(); len(errs) != 0 {
return utilerrors.NewAggregate(errs)
}
// create config from options
genericConfig := genericapiserver.NewConfig(). // create the new config
ApplyOptions(s.GenericServerRunOptions). // apply the options selected
ApplyInsecureServingOptions(s.InsecureServing)
if _, err := genericConfig.ApplySecureServingOptions(s.SecureServing); err != nil {
return fmt.Errorf("failed to configure https: %s", err)
}
if _, err := genericConfig.ApplyAuthenticationOptions(s.Authentication); err != nil {
return fmt.Errorf("failed to configure authentication: %s", err)
}
// TODO: register cluster federation resources here.
resourceConfig := genericapiserver.NewResourceConfig()
if s.Etcd.StorageConfig.DeserializationCacheSize == 0 {
// When size of cache is not explicitly set, set it to 50000
s.Etcd.StorageConfig.DeserializationCacheSize = 50000
}
storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion()
if err != nil {
return fmt.Errorf("error generating storage version map: %s", err)
}
storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
s.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs,
genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
[]schema.GroupVersionResource{}, resourceConfig, s.GenericServerRunOptions.RuntimeConfig)
if err != nil {
return fmt.Errorf("error in initializing storage factory: %s", err)
}
for _, override := range s.Etcd.EtcdServersOverrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
continue
}
group := apiresource[0]
resource := apiresource[1]
groupResource := schema.GroupResource{Group: group, Resource: resource}
servers := strings.Split(tokens[1], ";")
storageFactory.SetEtcdLocation(groupResource, servers)
}
apiAuthenticator, securityDefinitions, err := authenticator.New(s.Authentication.ToAuthenticationConfig())
if err != nil {
return fmt.Errorf("invalid Authentication Config: %v", err)
}
privilegedLoopbackToken := uuid.NewRandom().String()
selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken)
if err != nil {
return fmt.Errorf("failed to create clientset: %v", err)
}
client, err := internalclientset.NewForConfig(selfClientConfig)
if err != nil {
return fmt.Errorf("failed to create clientset: %v", err)
}
sharedInformers := informers.NewSharedInformerFactory(nil, client, 10*time.Minute)
authorizerconfig := s.Authorization.ToAuthorizationConfig(sharedInformers)
apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizerconfig)
if err != nil {
return fmt.Errorf("invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.GenericServerRunOptions.AdmissionControl, ",")
pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer)
admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.GenericServerRunOptions.AdmissionControlConfigFile, pluginInitializer)
if err != nil {
return fmt.Errorf("failed to initialize plugins: %v", err)
}
kubeVersion := version.Get()
genericConfig.Version = &kubeVersion
genericConfig.LoopbackClientConfig = selfClientConfig
genericConfig.Authenticator = apiAuthenticator
genericConfig.Authorizer = apiAuthorizer
genericConfig.AdmissionControl = admissionController
genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(openapi.OpenAPIDefinitions)
genericConfig.OpenAPIConfig.SecurityDefinitions = securityDefinitions
genericConfig.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
sets.NewString("watch", "proxy"),
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
)
// TODO: Move this to generic api server (Need to move the command line flag).
if s.GenericServerRunOptions.EnableWatchCache {
cachesize.InitializeWatchCacheSizes(s.GenericServerRunOptions.TargetRAMMB)
cachesize.SetWatchCacheSizes(s.GenericServerRunOptions.WatchCacheSizes)
}
m, err := genericConfig.Complete().New()
if err != nil {
return err
}
routes.UIRedirect{}.Install(m.HandlerContainer)
routes.Logs{}.Install(m.HandlerContainer)
// TODO: Refactor this code to share it with kube-apiserver rather than duplicating it here.
restOptionsFactory := &restOptionsFactory{
storageFactory: storageFactory,
enableGarbageCollection: s.GenericServerRunOptions.EnableGarbageCollection,
deleteCollectionWorkers: s.GenericServerRunOptions.DeleteCollectionWorkers,
}
if s.GenericServerRunOptions.EnableWatchCache {
restOptionsFactory.storageDecorator = genericregistry.StorageWithCacher
} else {
restOptionsFactory.storageDecorator = generic.UndecoratedStorage
}
installFederationAPIs(m, restOptionsFactory)
installCoreAPIs(s, m, restOptionsFactory)
installExtensionsAPIs(m, restOptionsFactory)
sharedInformers.Start(wait.NeverStop)
m.PrepareRun().Run(wait.NeverStop)
return nil
}
// startMasterOrDie starts a kubernetes master and an httpserver to handle api requests
func startMasterOrDie(masterConfig *master.Config, incomingServer *httptest.Server, masterReceiver MasterReceiver) (*master.Master, *httptest.Server) {
var m *master.Master
var s *httptest.Server
if incomingServer != nil {
s = incomingServer
} else {
s = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.GenericAPIServer.Handler.ServeHTTP(w, req)
}))
}
if masterConfig == nil {
masterConfig = NewMasterConfig()
masterConfig.GenericConfig.EnableProfiling = true
masterConfig.GenericConfig.EnableMetrics = true
masterConfig.GenericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(openapi.OpenAPIDefinitions)
masterConfig.GenericConfig.OpenAPIConfig.Info = &spec.Info{
InfoProps: spec.InfoProps{
Title: "Kubernetes",
Version: "unversioned",
},
}
masterConfig.GenericConfig.OpenAPIConfig.DefaultResponse = &spec.Response{
ResponseProps: spec.ResponseProps{
Description: "Default Response.",
},
}
masterConfig.GenericConfig.OpenAPIConfig.Definitions = openapi.OpenAPIDefinitions
masterConfig.GenericConfig.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
}
// set the loopback client config
if masterConfig.GenericConfig.LoopbackClientConfig == nil {
masterConfig.GenericConfig.LoopbackClientConfig = &restclient.Config{QPS: 50, Burst: 100, ContentConfig: restclient.ContentConfig{NegotiatedSerializer: api.Codecs}}
}
masterConfig.GenericConfig.LoopbackClientConfig.Host = s.URL
privilegedLoopbackToken := uuid.NewRandom().String()
// wrap any available authorizer
tokens := make(map[string]*user.DefaultInfo)
tokens[privilegedLoopbackToken] = &user.DefaultInfo{
Name: user.APIServerUser,
UID: uuid.NewRandom().String(),
Groups: []string{user.SystemPrivilegedGroup},
}
tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens)
if masterConfig.GenericConfig.Authenticator == nil {
masterConfig.GenericConfig.Authenticator = authenticatorunion.New(tokenAuthenticator, authauthenticator.RequestFunc(alwaysEmpty))
} else {
masterConfig.GenericConfig.Authenticator = authenticatorunion.New(tokenAuthenticator, masterConfig.GenericConfig.Authenticator)
}
if masterConfig.GenericConfig.Authorizer != nil {
tokenAuthorizer := authorizer.NewPrivilegedGroups(user.SystemPrivilegedGroup)
masterConfig.GenericConfig.Authorizer = authorizerunion.New(tokenAuthorizer, masterConfig.GenericConfig.Authorizer)
} else {
masterConfig.GenericConfig.Authorizer = alwaysAllow{}
}
masterConfig.GenericConfig.LoopbackClientConfig.BearerToken = privilegedLoopbackToken
m, err := masterConfig.Complete().New()
if err != nil {
glog.Fatalf("error in bringing up the master: %v", err)
}
if masterReceiver != nil {
masterReceiver.SetMaster(m)
}
// TODO have this start method actually use the normal start sequence for the API server
// this method never actually calls the `Run` method for the API server
// fire the post hooks ourselves
m.GenericAPIServer.PrepareRun()
m.GenericAPIServer.RunPostStartHooks()
cfg := *masterConfig.GenericConfig.LoopbackClientConfig
cfg.ContentConfig.GroupVersion = &schema.GroupVersion{}
privilegedClient, err := restclient.RESTClientFor(&cfg)
if err != nil {
glog.Fatal(err)
}
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
result := privilegedClient.Get().AbsPath("/healthz").Do()
status := 0
result.StatusCode(&status)
if status == 200 {
return true, nil
}
return false, nil
})
if err != nil {
glog.Fatal(err)
}
// wait for services to be ready
if masterConfig.EnableCoreControllers {
// TODO Once /healthz is updated for posthooks, we'll wait for good health
coreClient := coreclient.NewForConfigOrDie(&cfg)
svcWatch, err := coreClient.Services(v1.NamespaceDefault).Watch(v1.ListOptions{})
if err != nil {
glog.Fatal(err)
}
_, err = watch.Until(30*time.Second, svcWatch, func(event watch.Event) (bool, error) {
if event.Type != watch.Added {
return false, nil
}
if event.Object.(*v1.Service).Name == "kubernetes" {
return true, nil
}
return false, nil
})
if err != nil {
glog.Fatal(err)
}
}
return m, s
}
func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error {
serverOptions.Etcd.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"}
// set defaults
if err := serverOptions.GenericServerRunOptions.DefaultExternalHost(); err != nil {
return err
}
if err := serverOptions.SecureServing.MaybeDefaultWithSelfSignedCerts(serverOptions.GenericServerRunOptions.AdvertiseAddress.String()); err != nil {
glog.Fatalf("Error creating self-signed certificates: %v", err)
}
// validate options
if errs := serverOptions.Etcd.Validate(); len(errs) > 0 {
return utilerrors.NewAggregate(errs)
}
if errs := serverOptions.SecureServing.Validate(); len(errs) > 0 {
return utilerrors.NewAggregate(errs)
}
if errs := serverOptions.InsecureServing.Validate("insecure-port"); len(errs) > 0 {
return utilerrors.NewAggregate(errs)
}
// create config from options
config := genericapiserver.NewConfig().
ApplyOptions(serverOptions.GenericServerRunOptions).
ApplyInsecureServingOptions(serverOptions.InsecureServing)
if _, err := config.ApplySecureServingOptions(serverOptions.SecureServing); err != nil {
return fmt.Errorf("failed to configure https: %s", err)
}
if _, err := config.ApplyAuthenticationOptions(serverOptions.Authentication); err != nil {
return fmt.Errorf("failed to configure authentication: %s", err)
}
config.Authorizer = authorizer.NewAlwaysAllowAuthorizer()
config.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
s, err := config.Complete().New()
if err != nil {
return fmt.Errorf("Error in bringing up the server: %v", err)
}
groupVersion := v1.SchemeGroupVersion
groupName := groupVersion.Group
groupMeta, err := registered.Group(groupName)
if err != nil {
return fmt.Errorf("%v", err)
}
storageFactory := newStorageFactory()
storageConfig, err := storageFactory.NewConfig(schema.GroupResource{Group: groupName, Resource: "testtype"})
if err != nil {
return fmt.Errorf("Unable to get storage config: %v", err)
}
testTypeOpts := generic.RESTOptions{
StorageConfig: storageConfig,
Decorator: generic.UndecoratedStorage,
ResourcePrefix: "testtypes",
DeleteCollectionWorkers: 1,
}
restStorageMap := map[string]rest.Storage{
"testtypes": testgroupetcd.NewREST(testTypeOpts),
}
apiGroupInfo := genericapiserver.APIGroupInfo{
GroupMeta: *groupMeta,
VersionedResourcesStorageMap: map[string]map[string]rest.Storage{
groupVersion.Version: restStorageMap,
},
Scheme: api.Scheme,
NegotiatedSerializer: api.Codecs,
}
if err := s.InstallAPIGroup(&apiGroupInfo); err != nil {
return fmt.Errorf("Error in installing API: %v", err)
}
s.PrepareRun().Run(stopCh)
return nil
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.ServerRunOptions) error {
if errs := s.Etcd.Validate(); len(errs) > 0 {
return utilerrors.NewAggregate(errs)
}
if err := s.GenericServerRunOptions.DefaultExternalAddress(s.SecureServing, s.InsecureServing); err != nil {
return err
}
serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange)
if err != nil {
return fmt.Errorf("error determining service IP ranges: %v", err)
}
if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), apiServerServiceIP); err != nil {
return fmt.Errorf("error creating self-signed certificates: %v", err)
}
genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
genericConfig := genericapiserver.NewConfig(). // create the new config
ApplyOptions(s.GenericServerRunOptions). // apply the options selected
ApplyInsecureServingOptions(s.InsecureServing)
if _, err := genericConfig.ApplySecureServingOptions(s.SecureServing); err != nil {
return fmt.Errorf("failed to configure https: %s", err)
}
if _, err = genericConfig.ApplyAuthenticationOptions(s.Authentication); err != nil {
return fmt.Errorf("failed to configure authentication: %s", err)
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
// Setup tunneler if needed
var tunneler genericapiserver.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH genericapiserver.InstallSSHKey
cloud, err := cloudprovider.InitCloudProvider(s.GenericServerRunOptions.CloudProvider, s.GenericServerRunOptions.CloudConfigFile)
if err != nil {
return fmt.Errorf("cloud provider could not be initialized: %v", err)
}
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
if s.KubeletConfig.Port == 0 {
return fmt.Errorf("must enable kubelet port if proxy ssh-tunneling is specified")
}
// Set up the tunneler
// TODO(cjcullen): If we want this to handle per-kubelet ports or other
// kubelet listen-addresses, we need to plumb through options.
healthCheckPath := &url.URL{
Scheme: "https",
Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)),
Path: "healthz",
}
tunneler = genericapiserver.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
if s.Etcd.StorageConfig.DeserializationCacheSize == 0 {
// When size of cache is not explicitly set, estimate its size based on
// target memory usage.
glog.V(2).Infof("Initalizing deserialization cache size based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB)
// This is the heuristics that from memory capacity is trying to infer
// the maximum number of nodes in the cluster and set cache sizes based
// on that value.
// From our documentation, we officially recomment 120GB machines for
// 2000 nodes, and we scale from that point. Thus we assume ~60MB of
// capacity per node.
// TODO: We may consider deciding that some percentage of memory will
// be used for the deserialization cache and divide it by the max object
// size to compute its size. We may even go further and measure
// collective sizes of the objects in the cache.
clusterSize := s.GenericServerRunOptions.TargetRAMMB / 60
s.Etcd.StorageConfig.DeserializationCacheSize = 25 * clusterSize
if s.Etcd.StorageConfig.DeserializationCacheSize < 1000 {
s.Etcd.StorageConfig.DeserializationCacheSize = 1000
}
}
storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion()
if err != nil {
return fmt.Errorf("error generating storage version map: %s", err)
}
storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
s.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs,
genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
// FIXME: this GroupVersionResource override should be configurable
[]schema.GroupVersionResource{batch.Resource("cronjobs").WithVersion("v2alpha1")},
master.DefaultAPIResourceConfigSource(), s.GenericServerRunOptions.RuntimeConfig)
if err != nil {
return fmt.Errorf("error in initializing storage factory: %s", err)
}
storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs"))
storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers"))
for _, override := range s.Etcd.EtcdServersOverrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
continue
}
group := apiresource[0]
resource := apiresource[1]
groupResource := schema.GroupResource{Group: group, Resource: resource}
servers := strings.Split(tokens[1], ";")
storageFactory.SetEtcdLocation(groupResource, servers)
}
// Default to the private server key for service account token signing
if len(s.Authentication.ServiceAccounts.KeyFiles) == 0 && s.SecureServing.ServerCert.CertKey.KeyFile != "" {
if authenticator.IsValidServiceAccountKeyFile(s.SecureServing.ServerCert.CertKey.KeyFile) {
s.Authentication.ServiceAccounts.KeyFiles = []string{s.SecureServing.ServerCert.CertKey.KeyFile}
} else {
glog.Warning("No TLS key provided, service account token authentication disabled")
}
}
authenticatorConfig := s.Authentication.ToAuthenticationConfig()
if s.Authentication.ServiceAccounts.Lookup {
// If we need to look up service accounts and tokens,
// go directly to etcd to avoid recursive auth insanity
storageConfig, err := storageFactory.NewConfig(api.Resource("serviceaccounts"))
if err != nil {
return fmt.Errorf("unable to get serviceaccounts storage: %v", err)
}
authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets")))
}
apiAuthenticator, securityDefinitions, err := authenticator.New(authenticatorConfig)
if err != nil {
return fmt.Errorf("invalid Authentication Config: %v", err)
}
privilegedLoopbackToken := uuid.NewRandom().String()
selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken)
if err != nil {
return fmt.Errorf("failed to create clientset: %v", err)
}
client, err := internalclientset.NewForConfig(selfClientConfig)
if err != nil {
kubeAPIVersions := os.Getenv("KUBE_API_VERSIONS")
if len(kubeAPIVersions) == 0 {
return fmt.Errorf("failed to create clientset: %v", err)
}
// KUBE_API_VERSIONS is used in test-update-storage-objects.sh, disabling a number of API
// groups. This leads to a nil client above and undefined behaviour further down.
// TODO: get rid of KUBE_API_VERSIONS or define sane behaviour if set
glog.Errorf("Failed to create clientset with KUBE_API_VERSIONS=%q. KUBE_API_VERSIONS is only for testing. Things will break.", kubeAPIVersions)
}
sharedInformers := informers.NewSharedInformerFactory(nil, client, 10*time.Minute)
authorizationConfig := s.Authorization.ToAuthorizationConfig(sharedInformers)
apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationConfig)
if err != nil {
return fmt.Errorf("invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.GenericServerRunOptions.AdmissionControl, ",")
pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer)
admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.GenericServerRunOptions.AdmissionControlConfigFile, pluginInitializer)
if err != nil {
return fmt.Errorf("failed to initialize plugins: %v", err)
}
proxyTransport := utilnet.SetTransportDefaults(&http.Transport{
Dial: proxyDialerFn,
TLSClientConfig: proxyTLSClientConfig,
})
kubeVersion := version.Get()
genericConfig.Version = &kubeVersion
genericConfig.LoopbackClientConfig = selfClientConfig
genericConfig.Authenticator = apiAuthenticator
genericConfig.Authorizer = apiAuthorizer
genericConfig.AdmissionControl = admissionController
genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.OpenAPIDefinitions)
genericConfig.OpenAPIConfig.SecurityDefinitions = securityDefinitions
genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"
genericConfig.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
genericConfig.EnableMetrics = true
genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
sets.NewString("watch", "proxy"),
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
)
config := &master.Config{
GenericConfig: genericConfig,
APIResourceConfigSource: storageFactory.APIResourceConfigSource,
StorageFactory: storageFactory,
EnableWatchCache: s.GenericServerRunOptions.EnableWatchCache,
EnableCoreControllers: true,
DeleteCollectionWorkers: s.GenericServerRunOptions.DeleteCollectionWorkers,
EventTTL: s.EventTTL,
KubeletClientConfig: s.KubeletConfig,
EnableUISupport: true,
EnableLogsSupport: true,
ProxyTransport: proxyTransport,
Tunneler: tunneler,
ServiceIPRange: serviceIPRange,
APIServerServiceIP: apiServerServiceIP,
APIServerServicePort: 443,
ServiceNodePortRange: s.GenericServerRunOptions.ServiceNodePortRange,
KubernetesServiceNodePort: s.GenericServerRunOptions.KubernetesServiceNodePort,
MasterCount: s.GenericServerRunOptions.MasterCount,
}
if s.GenericServerRunOptions.EnableWatchCache {
glog.V(2).Infof("Initalizing cache sizes based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB)
cachesize.InitializeWatchCacheSizes(s.GenericServerRunOptions.TargetRAMMB)
cachesize.SetWatchCacheSizes(s.GenericServerRunOptions.WatchCacheSizes)
}
m, err := config.Complete().New()
if err != nil {
return err
}
sharedInformers.Start(wait.NeverStop)
m.GenericAPIServer.PrepareRun().Run(wait.NeverStop)
return nil
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.ServerRunOptions) error {
genericvalidation.VerifyEtcdServersList(s.GenericServerRunOptions)
genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
genericConfig := genericapiserver.NewConfig(). // create the new config
ApplyOptions(s.GenericServerRunOptions). // apply the options selected
Complete() // set default values based on the known values
serviceIPRange, apiServerServiceIP, err := genericapiserver.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange)
if err != nil {
glog.Fatalf("Error determining service IP ranges: %v", err)
}
if err := genericConfig.MaybeGenerateServingCerts(apiServerServiceIP); err != nil {
glog.Fatalf("Failed to generate service certificate: %v", err)
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
// Setup tunneler if needed
var tunneler genericapiserver.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH genericapiserver.InstallSSHKey
cloud, err := cloudprovider.InitCloudProvider(s.GenericServerRunOptions.CloudProvider, s.GenericServerRunOptions.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
if s.KubeletConfig.Port == 0 {
glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.")
}
// Set up the tunneler
// TODO(cjcullen): If we want this to handle per-kubelet ports or other
// kubelet listen-addresses, we need to plumb through options.
healthCheckPath := &url.URL{
Scheme: "https",
Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)),
Path: "healthz",
}
tunneler = genericapiserver.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
if s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize == 0 {
// When size of cache is not explicitly set, estimate its size based on
// target memory usage.
glog.V(2).Infof("Initalizing deserialization cache size based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB)
// This is the heuristics that from memory capacity is trying to infer
// the maximum number of nodes in the cluster and set cache sizes based
// on that value.
// From our documentation, we officially recomment 120GB machines for
// 2000 nodes, and we scale from that point. Thus we assume ~60MB of
// capacity per node.
// TODO: We may consider deciding that some percentage of memory will
// be used for the deserialization cache and divide it by the max object
// size to compute its size. We may even go further and measure
// collective sizes of the objects in the cache.
clusterSize := s.GenericServerRunOptions.TargetRAMMB / 60
s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize = 25 * clusterSize
if s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize < 1000 {
s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize = 1000
}
}
storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion()
if err != nil {
glog.Fatalf("error generating storage version map: %s", err)
}
storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
s.GenericServerRunOptions.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs,
genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
// FIXME: this GroupVersionResource override should be configurable
[]unversioned.GroupVersionResource{batch.Resource("cronjobs").WithVersion("v2alpha1")},
master.DefaultAPIResourceConfigSource(), s.GenericServerRunOptions.RuntimeConfig)
if err != nil {
glog.Fatalf("error in initializing storage factory: %s", err)
}
storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs"))
storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers"))
for _, override := range s.GenericServerRunOptions.EtcdServersOverrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
continue
}
group := apiresource[0]
resource := apiresource[1]
groupResource := unversioned.GroupResource{Group: group, Resource: resource}
servers := strings.Split(tokens[1], ";")
storageFactory.SetEtcdLocation(groupResource, servers)
}
// Default to the private server key for service account token signing
if len(s.ServiceAccountKeyFiles) == 0 && s.GenericServerRunOptions.TLSPrivateKeyFile != "" {
if authenticator.IsValidServiceAccountKeyFile(s.GenericServerRunOptions.TLSPrivateKeyFile) {
s.ServiceAccountKeyFiles = []string{s.GenericServerRunOptions.TLSPrivateKeyFile}
} else {
glog.Warning("No TLS key provided, service account token authentication disabled")
}
}
var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter
if s.ServiceAccountLookup {
// If we need to look up service accounts and tokens,
// go directly to etcd to avoid recursive auth insanity
storageConfig, err := storageFactory.NewConfig(api.Resource("serviceaccounts"))
if err != nil {
glog.Fatalf("Unable to get serviceaccounts storage: %v", err)
}
serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets")))
}
apiAuthenticator, securityDefinitions, err := authenticator.New(authenticator.AuthenticatorConfig{
Anonymous: s.GenericServerRunOptions.AnonymousAuth,
AnyToken: s.GenericServerRunOptions.EnableAnyToken,
BasicAuthFile: s.GenericServerRunOptions.BasicAuthFile,
ClientCAFile: s.GenericServerRunOptions.ClientCAFile,
TokenAuthFile: s.GenericServerRunOptions.TokenAuthFile,
OIDCIssuerURL: s.GenericServerRunOptions.OIDCIssuerURL,
OIDCClientID: s.GenericServerRunOptions.OIDCClientID,
OIDCCAFile: s.GenericServerRunOptions.OIDCCAFile,
OIDCUsernameClaim: s.GenericServerRunOptions.OIDCUsernameClaim,
OIDCGroupsClaim: s.GenericServerRunOptions.OIDCGroupsClaim,
ServiceAccountKeyFiles: s.ServiceAccountKeyFiles,
ServiceAccountLookup: s.ServiceAccountLookup,
ServiceAccountTokenGetter: serviceAccountGetter,
KeystoneURL: s.GenericServerRunOptions.KeystoneURL,
KeystoneCAFile: s.GenericServerRunOptions.KeystoneCAFile,
WebhookTokenAuthnConfigFile: s.WebhookTokenAuthnConfigFile,
WebhookTokenAuthnCacheTTL: s.WebhookTokenAuthnCacheTTL,
RequestHeaderConfig: s.GenericServerRunOptions.AuthenticationRequestHeaderConfig(),
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
privilegedLoopbackToken := uuid.NewRandom().String()
selfClientConfig, err := s.GenericServerRunOptions.NewSelfClientConfig(privilegedLoopbackToken)
if err != nil {
glog.Fatalf("Failed to create clientset: %v", err)
}
client, err := s.GenericServerRunOptions.NewSelfClient(privilegedLoopbackToken)
if err != nil {
glog.Errorf("Failed to create clientset: %v", err)
}
sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)
authorizationConfig := authorizer.AuthorizationConfig{
PolicyFile: s.GenericServerRunOptions.AuthorizationPolicyFile,
WebhookConfigFile: s.GenericServerRunOptions.AuthorizationWebhookConfigFile,
WebhookCacheAuthorizedTTL: s.GenericServerRunOptions.AuthorizationWebhookCacheAuthorizedTTL,
WebhookCacheUnauthorizedTTL: s.GenericServerRunOptions.AuthorizationWebhookCacheUnauthorizedTTL,
RBACSuperUser: s.GenericServerRunOptions.AuthorizationRBACSuperUser,
InformerFactory: sharedInformers,
}
authorizationModeNames := strings.Split(s.GenericServerRunOptions.AuthorizationMode, ",")
apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.GenericServerRunOptions.AdmissionControl, ",")
// TODO(dims): We probably need to add an option "EnableLoopbackToken"
if apiAuthenticator != nil {
var uid = uuid.NewRandom().String()
tokens := make(map[string]*user.DefaultInfo)
tokens[privilegedLoopbackToken] = &user.DefaultInfo{
Name: user.APIServerUser,
UID: uid,
Groups: []string{user.SystemPrivilegedGroup},
}
tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens)
apiAuthenticator = authenticatorunion.New(tokenAuthenticator, apiAuthenticator)
tokenAuthorizer := authorizer.NewPrivilegedGroups(user.SystemPrivilegedGroup)
apiAuthorizer = authorizerunion.New(tokenAuthorizer, apiAuthorizer)
}
pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer)
admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.GenericServerRunOptions.AdmissionControlConfigFile, pluginInitializer)
if err != nil {
glog.Fatalf("Failed to initialize plugins: %v", err)
}
proxyTransport := utilnet.SetTransportDefaults(&http.Transport{
Dial: proxyDialerFn,
TLSClientConfig: proxyTLSClientConfig,
})
kubeVersion := version.Get()
genericConfig.Version = &kubeVersion
genericConfig.LoopbackClientConfig = selfClientConfig
genericConfig.Authenticator = apiAuthenticator
genericConfig.Authorizer = apiAuthorizer
genericConfig.AdmissionControl = admissionController
genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.OpenAPIDefinitions)
genericConfig.OpenAPIConfig.SecurityDefinitions = securityDefinitions
genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"
genericConfig.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
genericConfig.EnableMetrics = true
config := &master.Config{
GenericConfig: genericConfig.Config,
StorageFactory: storageFactory,
EnableWatchCache: s.GenericServerRunOptions.EnableWatchCache,
EnableCoreControllers: true,
DeleteCollectionWorkers: s.GenericServerRunOptions.DeleteCollectionWorkers,
EventTTL: s.EventTTL,
KubeletClientConfig: s.KubeletConfig,
EnableUISupport: true,
EnableLogsSupport: true,
ProxyTransport: proxyTransport,
Tunneler: tunneler,
ServiceIPRange: serviceIPRange,
APIServerServiceIP: apiServerServiceIP,
APIServerServicePort: 443,
ServiceNodePortRange: s.GenericServerRunOptions.ServiceNodePortRange,
KubernetesServiceNodePort: s.GenericServerRunOptions.KubernetesServiceNodePort,
MasterCount: s.GenericServerRunOptions.MasterCount,
}
if s.GenericServerRunOptions.EnableWatchCache {
glog.V(2).Infof("Initalizing cache sizes based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB)
cachesize.InitializeWatchCacheSizes(s.GenericServerRunOptions.TargetRAMMB)
cachesize.SetWatchCacheSizes(s.GenericServerRunOptions.WatchCacheSizes)
}
m, err := config.Complete().New()
if err != nil {
return err
}
sharedInformers.Start(wait.NeverStop)
m.GenericAPIServer.PrepareRun().Run(wait.NeverStop)
return nil
}