// Run runs the specified APIServer. This should never exit. func Run(s *options.APIServer) error { verifyClusterIPFlags(s) // If advertise-address is not specified, use bind-address. If bind-address // is not usable (unset, 0.0.0.0, or loopback), we will use the host's default // interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster) if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() { hostIP, err := utilnet.ChooseBindAddress(s.BindAddress) if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v' . "+ "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) } s.AdvertiseAddress = hostIP } glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress) if len(s.EtcdServerList) == 0 { glog.Fatalf("--etcd-servers must be specified") } if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) { glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort)) } capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: s.AllowPrivileged, // TODO(vmarmol): Implement support for HostNetworkSources. PrivilegedSources: capabilities.PrivilegedSources{ HostNetworkSources: []string{}, HostPIDSources: []string{}, HostIPCSources: []string{}, }, PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, }) cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } // Setup tunneler if needed var tunneler master.Tunneler var proxyDialerFn apiserver.ProxyDialerFunc if len(s.SSHUser) > 0 { // Get ssh key distribution func, if supported var installSSH master.InstallSSHKey if cloud != nil { if instances, supported := cloud.Instances(); supported { installSSH = instances.AddSSHKeyToAllInstances } } // Set up the tunneler tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, installSSH) // Use the tunneler's dialer to connect to the kubelet s.KubeletConfig.Dial = tunneler.Dial // Use the tunneler's dialer when proxying to pods, services, and nodes proxyDialerFn = tunneler.Dial } // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig) if err != nil { glog.Fatalf("Failure to start kubelet client: %v", err) } apiGroupVersionOverrides, err := parseRuntimeConfig(s) if err != nil { glog.Fatalf("error in parsing runtime-config: %s", err) } clientConfig := &client.Config{ Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)), } if len(s.DeprecatedStorageVersion) != 0 { gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion) if err != nil { glog.Fatalf("error in parsing group version: %s", err) } clientConfig.GroupVersion = &gv } client, err := client.New(clientConfig) if err != nil { glog.Fatalf("Invalid server address: %v", err) } legacyV1Group, err := registered.Group(api.GroupName) if err != nil { return err } storageDestinations := genericapiserver.NewStorageDestinations() storageVersions := generateStorageVersionMap(s.DeprecatedStorageVersion, s.StorageVersions) if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.GroupVersion.Group, storageVersions) } etcdStorage, err := newEtcd(s.EtcdServerList, api.Codecs, storageVersions[legacyV1Group.GroupVersion.Group], s.EtcdPathPrefix) if err != nil { glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup("", etcdStorage) if !apiGroupVersionOverrides["extensions/v1beta1"].Disable { expGroup, err := registered.Group(extensions.GroupName) if err != nil { glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) } if _, found := storageVersions[expGroup.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.GroupVersion.Group, storageVersions) } expEtcdStorage, err := newEtcd(s.EtcdServerList, api.Codecs, storageVersions[expGroup.GroupVersion.Group], s.EtcdPathPrefix) if err != nil { glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup(extensions.GroupName, expEtcdStorage) } updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdPathPrefix, &storageDestinations, newEtcd) n := s.ServiceClusterIPRange // Default to the private server key for service account token signing if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" { if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) { s.ServiceAccountKeyFile = s.TLSPrivateKeyFile } else { glog.Warning("No RSA key provided, service account token authentication disabled") } } var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter if s.ServiceAccountLookup { // If we need to look up service accounts and tokens, // go directly to etcd to avoid recursive auth insanity serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(etcdStorage) } authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{ BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, ServiceAccountKeyFile: s.ServiceAccountKeyFile, ServiceAccountLookup: s.ServiceAccountLookup, ServiceAccountTokenGetter: serviceAccountGetter, KeystoneURL: s.KeystoneURL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationPolicyFile) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile) if len(s.ExternalHost) == 0 { // TODO: extend for other providers if s.CloudProvider == "gce" { instances, supported := cloud.Instances() if !supported { glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.") } name, err := os.Hostname() if err != nil { glog.Fatalf("Failed to get hostname: %v", err) } addrs, err := instances.NodeAddresses(name) if err != nil { glog.Warningf("Unable to obtain external host address from cloud provider: %v", err) } else { for _, addr := range addrs { if addr.Type == api.NodeExternalIP { s.ExternalHost = addr.Address } } } } } config := &master.Config{ Config: &genericapiserver.Config{ StorageDestinations: storageDestinations, StorageVersions: storageVersions, ServiceClusterIPRange: &n, EnableLogsSupport: s.EnableLogsSupport, EnableUISupport: true, EnableSwaggerSupport: true, EnableProfiling: s.EnableProfiling, EnableWatchCache: s.EnableWatchCache, EnableIndex: true, APIPrefix: s.APIPrefix, APIGroupPrefix: s.APIGroupPrefix, CorsAllowedOriginList: s.CorsAllowedOriginList, ReadWritePort: s.SecurePort, PublicAddress: s.AdvertiseAddress, Authenticator: authenticator, SupportsBasicAuth: len(s.BasicAuthFile) > 0, Authorizer: authorizer, AdmissionControl: admissionController, APIGroupVersionOverrides: apiGroupVersionOverrides, MasterServiceNamespace: s.MasterServiceNamespace, MasterCount: s.MasterCount, ExternalHost: s.ExternalHost, MinRequestTimeout: s.MinRequestTimeout, ProxyDialer: proxyDialerFn, ProxyTLSClientConfig: proxyTLSClientConfig, ServiceNodePortRange: s.ServiceNodePortRange, KubernetesServiceNodePort: s.KubernetesServiceNodePort, Serializer: api.Codecs, }, EnableCoreControllers: true, EventTTL: s.EventTTL, KubeletClient: kubeletClient, Tunneler: tunneler, } m := master.New(config) m.Run(s.ServerRunOptions) return nil }
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, pluginInitializer oadmission.PluginInitializer) (*MasterConfig, error) { if options.KubernetesMasterConfig == nil { return nil, errors.New("insufficient information to build KubernetesMasterConfig") } // Connect and setup etcd interfaces etcdClient, err := etcd.MakeNewEtcdClient(options.EtcdClientInfo) if err != nil { return nil, err } kubeletClientConfig := configapi.GetKubeletClientConfig(options) kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig) if err != nil { return nil, fmt.Errorf("unable to configure Kubelet client: %v", err) } // in-order list of plug-ins that should intercept admission decisions // TODO: Push node environment support to upstream in future _, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress) if err != nil { return nil, err } port, err := strconv.Atoi(portString) if err != nil { return nil, err } portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange) if err != nil { return nil, err } podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout) if err != nil { return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err) } // Defaults are tested in TestAPIServerDefaults server := apiserveroptions.NewAPIServer() // Adjust defaults server.EventTTL = 2 * time.Hour server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet)) server.ServiceNodePortRange = *portRange server.AdmissionControl = strings.Join(AdmissionPlugins, ",") server.EnableLogsSupport = false // don't expose server logs // resolve extended arguments // TODO: this should be done in config validation (along with the above) so we can provide // proper errors if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 { return nil, kerrors.NewAggregate(err) } if len(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride) > 0 { server.AdmissionControl = strings.Join(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride, ",") } // Defaults are tested in TestCMServerDefaults cmserver := cmapp.NewCMServer() // Adjust defaults cmserver.Address = "" // no healthz endpoint cmserver.Port = 0 // no healthz endpoint cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout} // resolve extended arguments // TODO: this should be done in config validation (along with the above) so we can provide // proper errors if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 { return nil, kerrors.NewAggregate(err) } cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile) if err != nil { return nil, err } if cloud != nil { glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile) } plugins := []admission.Interface{} for _, pluginName := range strings.Split(server.AdmissionControl, ",") { switch pluginName { case serviceadmit.ExternalIPPluginName: // this needs to be moved upstream to be part of core config reject, admit, err := serviceadmit.ParseCIDRRules(options.NetworkConfig.ExternalIPNetworkCIDRs) if err != nil { // should have been caught with validation return nil, err } plugins = append(plugins, serviceadmit.NewExternalIPRanger(reject, admit)) case saadmit.PluginName: // we need to set some custom parameters on the service account admission controller, so create that one by hand saAdmitter := saadmit.NewServiceAccount(internalclientset.FromUnversionedClient(kubeClient)) saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences saAdmitter.Run() plugins = append(plugins, saAdmitter) default: configFile, err := pluginconfig.GetPluginConfigFile(options.KubernetesMasterConfig.AdmissionConfig.PluginConfig, pluginName, server.AdmissionControlConfigFile) if err != nil { return nil, err } plugin := admission.InitPlugin(pluginName, internalclientset.FromUnversionedClient(kubeClient), configFile) if plugin != nil { plugins = append(plugins, plugin) } } } pluginInitializer.Initialize(plugins) // ensure that plugins have been properly initialized if err := oadmission.Validate(plugins); err != nil { return nil, err } admissionController := admission.NewChainHandler(plugins...) var proxyClientCerts []tls.Certificate if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 { clientCert, err := tls.LoadX509KeyPair( options.KubernetesMasterConfig.ProxyClientInfo.CertFile, options.KubernetesMasterConfig.ProxyClientInfo.KeyFile, ) if err != nil { return nil, err } proxyClientCerts = append(proxyClientCerts, clientCert) } // TODO you have to know every APIGroup you're enabling or upstream will panic. It's alternative to panicing is Fataling // It needs a refactor to return errors storageDestinations := genericapiserver.NewStorageDestinations() // storageVersions is a map from API group to allowed versions that must be a version exposed by the REST API or it breaks. // We need to fix the upstream to stop using the storage version as a preferred api version. storageVersions := map[string]string{} enabledKubeVersions := configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupKube) if len(enabledKubeVersions) > 0 { kubeStorageVersion := unversioned.GroupVersion{Group: configapi.APIGroupKube, Version: options.EtcdStorageConfig.KubernetesStorageVersion} databaseStorage, err := NewEtcdStorage(etcdClient, kubeStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix) if err != nil { return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err) } storageDestinations.AddAPIGroup(configapi.APIGroupKube, databaseStorage) storageVersions[configapi.APIGroupKube] = options.EtcdStorageConfig.KubernetesStorageVersion } // enable this if extensions API is enabled (or batch or autoscaling, since they persist to extensions/v1beta1 for now) // TODO: replace this with a loop over configured storage versions extensionsEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupExtensions)) > 0 batchEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupBatch)) > 0 autoscalingEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupAutoscaling)) > 0 if extensionsEnabled || autoscalingEnabled || batchEnabled { // TODO: replace this with a configured storage version for extensions once configuration exposes this extensionsStorageVersion := unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"} databaseStorage, err := NewEtcdStorage(etcdClient, extensionsStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix) if err != nil { return nil, fmt.Errorf("Error setting up Kubernetes extensions server storage: %v", err) } storageDestinations.AddAPIGroup(configapi.APIGroupExtensions, databaseStorage) storageVersions[configapi.APIGroupExtensions] = extensionsStorageVersion.String() } // Preserve previous behavior of using the first non-loopback address // TODO: Deprecate this behavior and just require a valid value to be passed in publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP) if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() { hostIP, err := knet.ChooseHostInterface() if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err) } publicAddress = hostIP glog.Infof("Will report %v as public IP address.", publicAddress) } m := &master.Config{ Config: &genericapiserver.Config{ PublicAddress: publicAddress, ReadWritePort: port, Authorizer: apiserver.NewAlwaysAllowAuthorizer(), AdmissionControl: admissionController, StorageDestinations: storageDestinations, StorageVersions: storageVersions, ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange), ServiceNodePortRange: server.ServiceNodePortRange, RequestContextMapper: requestContextMapper, APIGroupVersionOverrides: getAPIGroupVersionOverrides(options), APIPrefix: KubeAPIPrefix, APIGroupPrefix: KubeAPIGroupPrefix, MasterCount: options.KubernetesMasterConfig.MasterCount, // Set the TLS options for proxying to pods and services // Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname) ProxyTLSClientConfig: &tls.Config{ // Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs InsecureSkipVerify: true, Certificates: proxyClientCerts, }, Serializer: kapi.Codecs, }, EventTTL: server.EventTTL, //MinRequestTimeout: server.MinRequestTimeout, KubeletClient: kubeletClient, EnableCoreControllers: true, } if options.DNSConfig != nil { _, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress) if err != nil { return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err) } dnsPort, err := strconv.Atoi(dnsPortStr) if err != nil { return nil, fmt.Errorf("invalid DNS port: %v", err) } m.ExtraServicePorts = append(m.ExtraServicePorts, kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)}, kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)}, ) m.ExtraEndpointPorts = append(m.ExtraEndpointPorts, kapi.EndpointPort{Name: "dns", Port: dnsPort, Protocol: kapi.ProtocolUDP}, kapi.EndpointPort{Name: "dns-tcp", Port: dnsPort, Protocol: kapi.ProtocolTCP}, ) } kmaster := &MasterConfig{ Options: *options.KubernetesMasterConfig, KubeClient: kubeClient, Master: m, ControllerManager: cmserver, CloudProvider: cloud, } return kmaster, nil }
// Run runs the specified APIServer. This should never exit. func Run(s *options.APIServer) error { genericvalidation.VerifyEtcdServersList(s.ServerRunOptions) genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions) capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: s.AllowPrivileged, // TODO(vmarmol): Implement support for HostNetworkSources. PrivilegedSources: capabilities.PrivilegedSources{ HostNetworkSources: []string{}, HostPIDSources: []string{}, HostIPCSources: []string{}, }, PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, }) // Setup tunneler if needed var tunneler genericapiserver.Tunneler var proxyDialerFn apiserver.ProxyDialerFunc if len(s.SSHUser) > 0 { // Get ssh key distribution func, if supported var installSSH genericapiserver.InstallSSHKey cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } if cloud != nil { if instances, supported := cloud.Instances(); supported { installSSH = instances.AddSSHKeyToAllInstances } } if s.KubeletConfig.Port == 0 { glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.") } // Set up the tunneler // TODO(cjcullen): If we want this to handle per-kubelet ports or other // kubelet listen-addresses, we need to plumb through options. healthCheckPath := &url.URL{ Scheme: "https", Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)), Path: "healthz", } tunneler = genericapiserver.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH) // Use the tunneler's dialer to connect to the kubelet s.KubeletConfig.Dial = tunneler.Dial // Use the tunneler's dialer when proxying to pods, services, and nodes proxyDialerFn = tunneler.Dial } // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig) if err != nil { glog.Fatalf("Failed to start kubelet client: %v", err) } storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion() if err != nil { glog.Fatalf("error generating storage version map: %s", err) } storageFactory, err := genericapiserver.BuildDefaultStorageFactory( s.StorageConfig, s.DefaultStorageMediaType, api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion, // FIXME: this GroupVersionResource override should be configurable []unversioned.GroupVersionResource{batch.Resource("scheduledjobs").WithVersion("v2alpha1")}, master.DefaultAPIResourceConfigSource(), s.RuntimeConfig) if err != nil { glog.Fatalf("error in initializing storage factory: %s", err) } storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs")) storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers")) for _, override := range s.EtcdServersOverrides { tokens := strings.Split(override, "#") if len(tokens) != 2 { glog.Errorf("invalid value of etcd server overrides: %s", override) continue } apiresource := strings.Split(tokens[0], "/") if len(apiresource) != 2 { glog.Errorf("invalid resource definition: %s", tokens[0]) continue } group := apiresource[0] resource := apiresource[1] groupResource := unversioned.GroupResource{Group: group, Resource: resource} servers := strings.Split(tokens[1], ";") storageFactory.SetEtcdLocation(groupResource, servers) } // Default to the private server key for service account token signing if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" { if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) { s.ServiceAccountKeyFile = s.TLSPrivateKeyFile } else { glog.Warning("No RSA key provided, service account token authentication disabled") } } var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter if s.ServiceAccountLookup { // If we need to look up service accounts and tokens, // go directly to etcd to avoid recursive auth insanity storageConfig, err := storageFactory.NewConfig(api.Resource("serviceaccounts")) if err != nil { glog.Fatalf("Unable to get serviceaccounts storage: %v", err) } serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets"))) } apiAuthenticator, err := authenticator.New(authenticator.AuthenticatorConfig{ Anonymous: s.AnonymousAuth, AnyToken: s.EnableAnyToken, BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, OIDCGroupsClaim: s.OIDCGroupsClaim, ServiceAccountKeyFile: s.ServiceAccountKeyFile, ServiceAccountLookup: s.ServiceAccountLookup, ServiceAccountTokenGetter: serviceAccountGetter, KeystoneURL: s.KeystoneURL, WebhookTokenAuthnConfigFile: s.WebhookTokenAuthnConfigFile, WebhookTokenAuthnCacheTTL: s.WebhookTokenAuthnCacheTTL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") modeEnabled := func(mode string) bool { for _, m := range authorizationModeNames { if m == mode { return true } } return false } authorizationConfig := authorizer.AuthorizationConfig{ PolicyFile: s.AuthorizationPolicyFile, WebhookConfigFile: s.AuthorizationWebhookConfigFile, WebhookCacheAuthorizedTTL: s.AuthorizationWebhookCacheAuthorizedTTL, WebhookCacheUnauthorizedTTL: s.AuthorizationWebhookCacheUnauthorizedTTL, RBACSuperUser: s.AuthorizationRBACSuperUser, } if modeEnabled(genericoptions.ModeRBAC) { mustGetRESTOptions := func(resource string) generic.RESTOptions { config, err := storageFactory.NewConfig(rbac.Resource(resource)) if err != nil { glog.Fatalf("Unable to get %s storage: %v", resource, err) } return generic.RESTOptions{StorageConfig: config, Decorator: generic.UndecoratedStorage, ResourcePrefix: storageFactory.ResourcePrefix(rbac.Resource(resource))} } // For initial bootstrapping go directly to etcd to avoid privillege escalation check. authorizationConfig.RBACRoleRegistry = role.NewRegistry(roleetcd.NewREST(mustGetRESTOptions("roles"))) authorizationConfig.RBACRoleBindingRegistry = rolebinding.NewRegistry(rolebindingetcd.NewREST(mustGetRESTOptions("rolebindings"))) authorizationConfig.RBACClusterRoleRegistry = clusterrole.NewRegistry(clusterroleetcd.NewREST(mustGetRESTOptions("clusterroles"))) authorizationConfig.RBACClusterRoleBindingRegistry = clusterrolebinding.NewRegistry(clusterrolebindingetcd.NewREST(mustGetRESTOptions("clusterrolebindings"))) } apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") privilegedLoopbackToken := uuid.NewRandom().String() client, err := s.NewSelfClient(privilegedLoopbackToken) if err != nil { glog.Errorf("Failed to create clientset: %v", err) } // TODO(dims): We probably need to add an option "EnableLoopbackToken" if apiAuthenticator != nil { var uid = uuid.NewRandom().String() tokens := make(map[string]*user.DefaultInfo) tokens[privilegedLoopbackToken] = &user.DefaultInfo{ Name: "system:apiserver", UID: uid, Groups: []string{"system:masters"}, } tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens) apiAuthenticator = authenticatorunion.New(tokenAuthenticator, apiAuthenticator) tokenAuthorizer := authorizer.NewPrivilegedGroups("system:masters") apiAuthorizer = authorizerunion.New(tokenAuthorizer, apiAuthorizer) } sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute) pluginInitializer := admission.NewPluginInitializer(sharedInformers) admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile, pluginInitializer) if err != nil { glog.Fatalf("Failed to initialize plugins: %v", err) } genericConfig := genericapiserver.NewConfig(s.ServerRunOptions) // TODO: Move the following to generic api server as well. genericConfig.Authenticator = apiAuthenticator genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0 genericConfig.Authorizer = apiAuthorizer genericConfig.AuthorizerRBACSuperUser = s.AuthorizationRBACSuperUser genericConfig.AdmissionControl = admissionController genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource genericConfig.MasterServiceNamespace = s.MasterServiceNamespace genericConfig.ProxyDialer = proxyDialerFn genericConfig.ProxyTLSClientConfig = proxyTLSClientConfig genericConfig.Serializer = api.Codecs genericConfig.OpenAPIInfo.Title = "Kubernetes" genericConfig.OpenAPIDefinitions = openapi.OpenAPIDefinitions genericConfig.EnableOpenAPISupport = true config := &master.Config{ GenericConfig: genericConfig, StorageFactory: storageFactory, EnableWatchCache: s.EnableWatchCache, EnableCoreControllers: true, DeleteCollectionWorkers: s.DeleteCollectionWorkers, EventTTL: s.EventTTL, KubeletClient: kubeletClient, EnableUISupport: true, EnableLogsSupport: true, Tunneler: tunneler, } if s.EnableWatchCache { glog.V(2).Infof("Initalizing cache sizes based on %dMB limit", s.TargetRAMMB) cachesize.InitializeWatchCacheSizes(s.TargetRAMMB) cachesize.SetWatchCacheSizes(s.WatchCacheSizes) } m, err := config.Complete().New() if err != nil { return err } sharedInformers.Start(wait.NeverStop) m.Run(s.ServerRunOptions) return nil }
// Run runs the specified APIServer. This should never exit. func Run(s *options.APIServer) error { verifyClusterIPFlags(s) // If advertise-address is not specified, use bind-address. If bind-address // is not usable (unset, 0.0.0.0, or loopback), we will use the host's default // interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster) if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() { hostIP, err := utilnet.ChooseBindAddress(s.BindAddress) if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v' . "+ "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) } s.AdvertiseAddress = hostIP } glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress) if len(s.EtcdConfig.ServerList) == 0 { glog.Fatalf("--etcd-servers must be specified") } if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) { glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort)) } capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: s.AllowPrivileged, // TODO(vmarmol): Implement support for HostNetworkSources. PrivilegedSources: capabilities.PrivilegedSources{ HostNetworkSources: []string{}, HostPIDSources: []string{}, HostIPCSources: []string{}, }, PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, }) cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } // Setup tunneler if needed var tunneler master.Tunneler var proxyDialerFn apiserver.ProxyDialerFunc if len(s.SSHUser) > 0 { // Get ssh key distribution func, if supported var installSSH master.InstallSSHKey if cloud != nil { if instances, supported := cloud.Instances(); supported { installSSH = instances.AddSSHKeyToAllInstances } } if s.KubeletConfig.Port == 0 { glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.") } // Set up the tunneler // TODO(cjcullen): If we want this to handle per-kubelet ports or other // kubelet listen-addresses, we need to plumb through options. healthCheckPath := &url.URL{ Scheme: "https", Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)), Path: "healthz", } tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH) // Use the tunneler's dialer to connect to the kubelet s.KubeletConfig.Dial = tunneler.Dial // Use the tunneler's dialer when proxying to pods, services, and nodes proxyDialerFn = tunneler.Dial } // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig) if err != nil { glog.Fatalf("Failure to start kubelet client: %v", err) } apiResourceConfigSource, err := parseRuntimeConfig(s) if err != nil { glog.Fatalf("error in parsing runtime-config: %s", err) } clientConfig := &restclient.Config{ Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)), // Increase QPS limits. The client is currently passed to all admission plugins, // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 // for more details. Once #22422 is fixed, we may want to remove it. QPS: 50, Burst: 100, } if len(s.DeprecatedStorageVersion) != 0 { gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion) if err != nil { glog.Fatalf("error in parsing group version: %s", err) } clientConfig.GroupVersion = &gv } client, err := clientset.NewForConfig(clientConfig) if err != nil { glog.Errorf("Failed to create clientset: %v", err) } legacyV1Group, err := registered.Group(api.GroupName) if err != nil { return err } storageDestinations := genericapiserver.NewStorageDestinations() storageVersions := s.StorageGroupsToGroupVersions() if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.GroupVersion.Group, storageVersions) } etcdStorage, err := newEtcd(api.Codecs, storageVersions[legacyV1Group.GroupVersion.Group], "/__internal", s.EtcdConfig) if err != nil { glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup("", etcdStorage) if apiResourceConfigSource.AnyResourcesForVersionEnabled(extensionsapiv1beta1.SchemeGroupVersion) { glog.Infof("Configuring extensions/v1beta1 storage destination") expGroup, err := registered.Group(extensions.GroupName) if err != nil { glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) } if _, found := storageVersions[expGroup.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.GroupVersion.Group, storageVersions) } expEtcdStorage, err := newEtcd(api.Codecs, storageVersions[expGroup.GroupVersion.Group], "extensions/__internal", s.EtcdConfig) if err != nil { glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup(extensions.GroupName, expEtcdStorage) // Since HPA has been moved to the autoscaling group, we need to make // sure autoscaling has a storage destination. If the autoscaling group // itself is on, it will overwrite this decision below. storageDestinations.AddAPIGroup(autoscaling.GroupName, expEtcdStorage) // Since Job has been moved to the batch group, we need to make // sure batch has a storage destination. If the batch group // itself is on, it will overwrite this decision below. storageDestinations.AddAPIGroup(batch.GroupName, expEtcdStorage) } // autoscaling/v1/horizontalpodautoscalers is a move from extensions/v1beta1/horizontalpodautoscalers. // The storage version needs to be either extensions/v1beta1 or autoscaling/v1. // Users must roll forward while using 1.2, because we will require the latter for 1.3. if apiResourceConfigSource.AnyResourcesForVersionEnabled(autoscalingapiv1.SchemeGroupVersion) { glog.Infof("Configuring autoscaling/v1 storage destination") autoscalingGroup, err := registered.Group(autoscaling.GroupName) if err != nil { glog.Fatalf("Autoscaling API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) } // Figure out what storage group/version we should use. storageGroupVersion, found := storageVersions[autoscalingGroup.GroupVersion.Group] if !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", autoscalingGroup.GroupVersion.Group, storageVersions) } if storageGroupVersion != "autoscaling/v1" && storageGroupVersion != "extensions/v1beta1" { glog.Fatalf("The storage version for autoscaling must be either 'autoscaling/v1' or 'extensions/v1beta1'") } glog.Infof("Using %v for autoscaling group storage version", storageGroupVersion) autoscalingEtcdStorage, err := newEtcd(api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdConfig) if err != nil { glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup(autoscaling.GroupName, autoscalingEtcdStorage) } // batch/v1/job is a move from extensions/v1beta1/job. The storage // version needs to be either extensions/v1beta1 or batch/v1. Users // must roll forward while using 1.2, because we will require the // latter for 1.3. if apiResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv1.SchemeGroupVersion) { glog.Infof("Configuring batch/v1 storage destination") batchGroup, err := registered.Group(batch.GroupName) if err != nil { glog.Fatalf("Batch API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) } // Figure out what storage group/version we should use. storageGroupVersion, found := storageVersions[batchGroup.GroupVersion.Group] if !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", batchGroup.GroupVersion.Group, storageVersions) } if storageGroupVersion != "batch/v1" && storageGroupVersion != "extensions/v1beta1" { glog.Fatalf("The storage version for batch must be either 'batch/v1' or 'extensions/v1beta1'") } glog.Infof("Using %v for batch group storage version", storageGroupVersion) batchEtcdStorage, err := newEtcd(api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdConfig) if err != nil { glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup(batch.GroupName, batchEtcdStorage) } updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdConfig, &storageDestinations, newEtcd) n := s.ServiceClusterIPRange // Default to the private server key for service account token signing if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" { if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) { s.ServiceAccountKeyFile = s.TLSPrivateKeyFile } else { glog.Warning("No RSA key provided, service account token authentication disabled") } } var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter if s.ServiceAccountLookup { // If we need to look up service accounts and tokens, // go directly to etcd to avoid recursive auth insanity serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(etcdStorage) } authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{ BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, OIDCGroupsClaim: s.OIDCGroupsClaim, ServiceAccountKeyFile: s.ServiceAccountKeyFile, ServiceAccountLookup: s.ServiceAccountLookup, ServiceAccountTokenGetter: serviceAccountGetter, KeystoneURL: s.KeystoneURL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationConfig) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile) if len(s.ExternalHost) == 0 { // TODO: extend for other providers if s.CloudProvider == "gce" { instances, supported := cloud.Instances() if !supported { glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.") } name, err := os.Hostname() if err != nil { glog.Fatalf("Failed to get hostname: %v", err) } addrs, err := instances.NodeAddresses(name) if err != nil { glog.Warningf("Unable to obtain external host address from cloud provider: %v", err) } else { for _, addr := range addrs { if addr.Type == api.NodeExternalIP { s.ExternalHost = addr.Address } } } } } config := &master.Config{ Config: &genericapiserver.Config{ StorageDestinations: storageDestinations, StorageVersions: storageVersions, ServiceClusterIPRange: &n, EnableLogsSupport: s.EnableLogsSupport, EnableUISupport: true, EnableSwaggerSupport: true, EnableSwaggerUI: s.EnableSwaggerUI, EnableProfiling: s.EnableProfiling, EnableWatchCache: s.EnableWatchCache, EnableIndex: true, APIPrefix: s.APIPrefix, APIGroupPrefix: s.APIGroupPrefix, CorsAllowedOriginList: s.CorsAllowedOriginList, ReadWritePort: s.SecurePort, PublicAddress: s.AdvertiseAddress, Authenticator: authenticator, SupportsBasicAuth: len(s.BasicAuthFile) > 0, Authorizer: authorizer, AdmissionControl: admissionController, APIResourceConfigSource: apiResourceConfigSource, MasterServiceNamespace: s.MasterServiceNamespace, MasterCount: s.MasterCount, ExternalHost: s.ExternalHost, MinRequestTimeout: s.MinRequestTimeout, ProxyDialer: proxyDialerFn, ProxyTLSClientConfig: proxyTLSClientConfig, ServiceNodePortRange: s.ServiceNodePortRange, KubernetesServiceNodePort: s.KubernetesServiceNodePort, Serializer: api.Codecs, }, EnableCoreControllers: true, DeleteCollectionWorkers: s.DeleteCollectionWorkers, EventTTL: s.EventTTL, KubeletClient: kubeletClient, Tunneler: tunneler, } if s.EnableWatchCache { cachesize.SetWatchCacheSizes(s.WatchCacheSizes) } m, err := master.New(config) if err != nil { return err } m.Run(s.ServerRunOptions) return nil }
func (c *MasterConfig) GetRestStorage() map[string]rest.Storage { kubeletClient, err := kubeletclient.NewStaticKubeletClient(c.KubeletClientConfig) if err != nil { glog.Fatalf("Unable to configure Kubelet client: %v", err) } // TODO: allow the system CAs and the local CAs to be joined together. importTransport, err := restclient.TransportFor(&restclient.Config{}) if err != nil { glog.Fatalf("Unable to configure a default transport for importing: %v", err) } insecureImportTransport, err := restclient.TransportFor(&restclient.Config{Insecure: true}) if err != nil { glog.Fatalf("Unable to configure a default transport for importing: %v", err) } buildStorage, buildDetailsStorage, err := buildetcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) buildRegistry := buildregistry.NewRegistry(buildStorage) buildConfigStorage, err := buildconfigetcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) buildConfigRegistry := buildconfigregistry.NewRegistry(buildConfigStorage) deployConfigStorage, deployConfigStatusStorage, deployConfigScaleStorage, err := deployconfigetcd.NewREST(c.RESTOptionsGetter) dcInstantiateOriginClient, dcInstantiateKubeClient := c.DeploymentConfigInstantiateClients() dcInstantiateStorage := deployconfiginstantiate.NewREST( *deployConfigStorage.Store, dcInstantiateOriginClient, dcInstantiateKubeClient, c.ExternalVersionCodec, c.AdmissionControl, ) checkStorageErr(err) deployConfigRegistry := deployconfigregistry.NewRegistry(deployConfigStorage) routeAllocator := c.RouteAllocator() routeStorage, routeStatusStorage, err := routeetcd.NewREST(c.RESTOptionsGetter, routeAllocator) checkStorageErr(err) hostSubnetStorage, err := hostsubnetetcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) netNamespaceStorage, err := netnamespaceetcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) clusterNetworkStorage, err := clusternetworketcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) egressNetworkPolicyStorage, err := egressnetworkpolicyetcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) userStorage, err := useretcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) userRegistry := userregistry.NewRegistry(userStorage) identityStorage, err := identityetcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) identityRegistry := identityregistry.NewRegistry(identityStorage) userIdentityMappingStorage := useridentitymapping.NewREST(userRegistry, identityRegistry) groupStorage, err := groupetcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) policyStorage, err := policyetcd.NewStorage(c.RESTOptionsGetter) checkStorageErr(err) policyRegistry := policyregistry.NewRegistry(policyStorage) policyBindingStorage, err := policybindingetcd.NewStorage(c.RESTOptionsGetter) checkStorageErr(err) policyBindingRegistry := policybindingregistry.NewRegistry(policyBindingStorage) clusterPolicyStorage, err := clusterpolicystorage.NewStorage(c.RESTOptionsGetter) checkStorageErr(err) clusterPolicyRegistry := clusterpolicyregistry.NewRegistry(clusterPolicyStorage) clusterPolicyBindingStorage, err := clusterpolicybindingstorage.NewStorage(c.RESTOptionsGetter) checkStorageErr(err) clusterPolicyBindingRegistry := clusterpolicybindingregistry.NewRegistry(clusterPolicyBindingStorage) selfSubjectRulesReviewStorage := selfsubjectrulesreview.NewREST(c.RuleResolver, c.Informers.ClusterPolicies().Lister().ClusterPolicies()) subjectRulesReviewStorage := subjectrulesreview.NewREST(c.RuleResolver, c.Informers.ClusterPolicies().Lister().ClusterPolicies()) roleStorage := rolestorage.NewVirtualStorage(policyRegistry, c.RuleResolver) roleBindingStorage := rolebindingstorage.NewVirtualStorage(policyBindingRegistry, c.RuleResolver) clusterRoleStorage := clusterrolestorage.NewClusterRoleStorage(clusterPolicyRegistry, clusterPolicyBindingRegistry) clusterRoleBindingStorage := clusterrolebindingstorage.NewClusterRoleBindingStorage(clusterPolicyRegistry, clusterPolicyBindingRegistry) subjectAccessReviewStorage := subjectaccessreview.NewREST(c.Authorizer) subjectAccessReviewRegistry := subjectaccessreview.NewRegistry(subjectAccessReviewStorage) localSubjectAccessReviewStorage := localsubjectaccessreview.NewREST(subjectAccessReviewRegistry) resourceAccessReviewStorage := resourceaccessreview.NewREST(c.Authorizer) resourceAccessReviewRegistry := resourceaccessreview.NewRegistry(resourceAccessReviewStorage) localResourceAccessReviewStorage := localresourceaccessreview.NewREST(resourceAccessReviewRegistry) podSecurityPolicyReviewStorage := podsecuritypolicyreview.NewREST(oscc.NewDefaultSCCMatcher(c.Informers.SecurityContextConstraints().Lister()), clientadapter.FromUnversionedClient(c.PrivilegedLoopbackKubernetesClient)) podSecurityPolicySubjectStorage := podsecuritypolicysubjectreview.NewREST(oscc.NewDefaultSCCMatcher(c.Informers.SecurityContextConstraints().Lister()), clientadapter.FromUnversionedClient(c.PrivilegedLoopbackKubernetesClient)) podSecurityPolicySelfSubjectReviewStorage := podsecuritypolicyselfsubjectreview.NewREST(oscc.NewDefaultSCCMatcher(c.Informers.SecurityContextConstraints().Lister()), clientadapter.FromUnversionedClient(c.PrivilegedLoopbackKubernetesClient)) imageStorage, err := imageetcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) imageRegistry := image.NewRegistry(imageStorage) imageSignatureStorage := imagesignature.NewREST(c.PrivilegedLoopbackOpenShiftClient.Images()) imageStreamSecretsStorage := imagesecret.NewREST(c.ImageStreamSecretClient()) imageStreamStorage, imageStreamStatusStorage, internalImageStreamStorage, err := imagestreametcd.NewREST(c.RESTOptionsGetter, c.RegistryNameFn, subjectAccessReviewRegistry, c.LimitVerifier) checkStorageErr(err) imageStreamRegistry := imagestream.NewRegistry(imageStreamStorage, imageStreamStatusStorage, internalImageStreamStorage) imageStreamMappingStorage := imagestreammapping.NewREST(imageRegistry, imageStreamRegistry, c.RegistryNameFn) imageStreamTagStorage := imagestreamtag.NewREST(imageRegistry, imageStreamRegistry) imageStreamTagRegistry := imagestreamtag.NewRegistry(imageStreamTagStorage) importerFn := func(r importer.RepositoryRetriever) imageimporter.Interface { return imageimporter.NewImageStreamImporter(r, c.Options.ImagePolicyConfig.MaxImagesBulkImportedPerRepository, flowcontrol.NewTokenBucketRateLimiter(2.0, 3)) } importerDockerClientFn := func() dockerregistry.Client { return dockerregistry.NewClient(20*time.Second, false) } imageStreamImportStorage := imagestreamimport.NewREST(importerFn, imageStreamRegistry, internalImageStreamStorage, imageStorage, c.ImageStreamImportSecretClient(), importTransport, insecureImportTransport, importerDockerClientFn) imageStreamImageStorage := imagestreamimage.NewREST(imageRegistry, imageStreamRegistry) imageStreamImageRegistry := imagestreamimage.NewRegistry(imageStreamImageStorage) buildGenerator := &buildgenerator.BuildGenerator{ Client: buildgenerator.Client{ GetBuildConfigFunc: buildConfigRegistry.GetBuildConfig, UpdateBuildConfigFunc: buildConfigRegistry.UpdateBuildConfig, GetBuildFunc: buildRegistry.GetBuild, CreateBuildFunc: buildRegistry.CreateBuild, GetImageStreamFunc: imageStreamRegistry.GetImageStream, GetImageStreamImageFunc: imageStreamImageRegistry.GetImageStreamImage, GetImageStreamTagFunc: imageStreamTagRegistry.GetImageStreamTag, }, ServiceAccounts: c.KubeClient(), Secrets: c.KubeClient(), } // TODO: with sharding, this needs to be changed deployConfigGenerator := &deployconfiggenerator.DeploymentConfigGenerator{ Client: deployconfiggenerator.Client{ DCFn: deployConfigRegistry.GetDeploymentConfig, ISFn: imageStreamRegistry.GetImageStream, LISFn2: imageStreamRegistry.ListImageStreams, }, } configClient, kclient := c.DeploymentConfigClients() deployRollbackClient := deployrollback.Client{ DCFn: deployConfigRegistry.GetDeploymentConfig, RCFn: clientDeploymentInterface{kclient}.GetDeployment, GRFn: deployrollback.NewRollbackGenerator().GenerateRollback, } deployConfigRollbackStorage := deployrollback.NewREST(configClient, kclient, c.ExternalVersionCodec) projectStorage := projectproxy.NewREST(c.PrivilegedLoopbackKubernetesClient.Namespaces(), c.ProjectAuthorizationCache, c.ProjectAuthorizationCache, c.ProjectCache) namespace, templateName, err := configapi.ParseNamespaceAndName(c.Options.ProjectConfig.ProjectRequestTemplate) if err != nil { glog.Errorf("Error parsing project request template value: %v", err) // we can continue on, the storage that gets created will be valid, it simply won't work properly. There's no reason to kill the master } projectRequestStorage := projectrequeststorage.NewREST(c.Options.ProjectConfig.ProjectRequestMessage, namespace, templateName, c.PrivilegedLoopbackOpenShiftClient, c.PrivilegedLoopbackKubernetesClient, c.Informers.PolicyBindings().Lister()) bcClient := c.BuildConfigWebHookClient() buildConfigWebHooks := buildconfigregistry.NewWebHookREST( buildConfigRegistry, buildclient.NewOSClientBuildConfigInstantiatorClient(bcClient), map[string]webhook.Plugin{ "generic": generic.New(), "github": github.New(), }, ) clientStorage, err := clientetcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) clientRegistry := clientregistry.NewRegistry(clientStorage) // If OAuth is disabled, set the strategy to Deny saAccountGrantMethod := oauthapi.GrantHandlerDeny if c.Options.OAuthConfig != nil { // Otherwise, take the value provided in master-config.yaml saAccountGrantMethod = oauthapi.GrantHandlerType(c.Options.OAuthConfig.GrantConfig.ServiceAccountMethod) } combinedOAuthClientGetter := saoauth.NewServiceAccountOAuthClientGetter(c.KubeClient(), c.KubeClient(), clientRegistry, saAccountGrantMethod) authorizeTokenStorage, err := authorizetokenetcd.NewREST(c.RESTOptionsGetter, combinedOAuthClientGetter) checkStorageErr(err) accessTokenStorage, err := accesstokenetcd.NewREST(c.RESTOptionsGetter, combinedOAuthClientGetter) checkStorageErr(err) clientAuthorizationStorage, err := clientauthetcd.NewREST(c.RESTOptionsGetter, combinedOAuthClientGetter) checkStorageErr(err) templateStorage, err := templateetcd.NewREST(c.RESTOptionsGetter) checkStorageErr(err) storage := map[string]rest.Storage{ "images": imageStorage, "imagesignatures": imageSignatureStorage, "imageStreams/secrets": imageStreamSecretsStorage, "imageStreams": imageStreamStorage, "imageStreams/status": imageStreamStatusStorage, "imageStreamImports": imageStreamImportStorage, "imageStreamImages": imageStreamImageStorage, "imageStreamMappings": imageStreamMappingStorage, "imageStreamTags": imageStreamTagStorage, "deploymentConfigs": deployConfigStorage, "deploymentConfigs/scale": deployConfigScaleStorage, "deploymentConfigs/status": deployConfigStatusStorage, "deploymentConfigs/rollback": deployConfigRollbackStorage, "deploymentConfigs/log": deploylogregistry.NewREST(configClient, kclient, c.DeploymentLogClient(), kubeletClient), "deploymentConfigs/instantiate": dcInstantiateStorage, // TODO: Deprecate these "generateDeploymentConfigs": deployconfiggenerator.NewREST(deployConfigGenerator, c.ExternalVersionCodec), "deploymentConfigRollbacks": deployrollback.NewDeprecatedREST(deployRollbackClient, c.ExternalVersionCodec), "processedTemplates": templateregistry.NewREST(), "templates": templateStorage, "routes": routeStorage, "routes/status": routeStatusStorage, "projects": projectStorage, "projectRequests": projectRequestStorage, "hostSubnets": hostSubnetStorage, "netNamespaces": netNamespaceStorage, "clusterNetworks": clusterNetworkStorage, "egressNetworkPolicies": egressNetworkPolicyStorage, "users": userStorage, "groups": groupStorage, "identities": identityStorage, "userIdentityMappings": userIdentityMappingStorage, "oAuthAuthorizeTokens": authorizeTokenStorage, "oAuthAccessTokens": accessTokenStorage, "oAuthClients": clientStorage, "oAuthClientAuthorizations": clientAuthorizationStorage, "resourceAccessReviews": resourceAccessReviewStorage, "subjectAccessReviews": subjectAccessReviewStorage, "localSubjectAccessReviews": localSubjectAccessReviewStorage, "localResourceAccessReviews": localResourceAccessReviewStorage, "selfSubjectRulesReviews": selfSubjectRulesReviewStorage, "subjectRulesReviews": subjectRulesReviewStorage, "podSecurityPolicyReviews": podSecurityPolicyReviewStorage, "podSecurityPolicySubjectReviews": podSecurityPolicySubjectStorage, "podSecurityPolicySelfSubjectReviews": podSecurityPolicySelfSubjectReviewStorage, "policies": policyStorage, "policyBindings": policyBindingStorage, "roles": roleStorage, "roleBindings": roleBindingStorage, "clusterPolicies": clusterPolicyStorage, "clusterPolicyBindings": clusterPolicyBindingStorage, "clusterRoleBindings": clusterRoleBindingStorage, "clusterRoles": clusterRoleStorage, "clusterResourceQuotas": restInPeace(clusterresourcequotaregistry.NewStorage(c.RESTOptionsGetter)), "clusterResourceQuotas/status": updateInPeace(clusterresourcequotaregistry.NewStatusStorage(c.RESTOptionsGetter)), "appliedClusterResourceQuotas": appliedclusterresourcequotaregistry.NewREST( c.ClusterQuotaMappingController.GetClusterQuotaMapper(), c.Informers.ClusterResourceQuotas().Lister(), c.Informers.Namespaces().Lister()), } if configapi.IsBuildEnabled(&c.Options) { storage["builds"] = buildStorage storage["buildConfigs"] = buildConfigStorage storage["buildConfigs/webhooks"] = buildConfigWebHooks storage["builds/clone"] = buildclone.NewStorage(buildGenerator) storage["buildConfigs/instantiate"] = buildconfiginstantiate.NewStorage(buildGenerator) storage["buildConfigs/instantiatebinary"] = buildconfiginstantiate.NewBinaryStorage(buildGenerator, buildStorage, c.BuildLogClient(), kubeletClient) storage["builds/log"] = buildlogregistry.NewREST(buildStorage, buildStorage, c.BuildLogClient(), kubeletClient) storage["builds/details"] = buildDetailsStorage } return storage }
// Run runs the specified APIServer. This should never exit. func (s *APIServer) Run(_ []string) error { s.verifyClusterIPFlags() // If advertise-address is not specified, use bind-address. If bind-address // is not usable (unset, 0.0.0.0, or loopback), we will use the host's default // interface as valid public addr for master (see: util#ValidPublicAddrForMaster) if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() { hostIP, err := util.ValidPublicAddrForMaster(s.BindAddress) if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v' . "+ "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) } s.AdvertiseAddress = hostIP } glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress) if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) { glog.Fatalf("Specify either --etcd-servers or --etcd-config") } if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) { glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort)) } capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: s.AllowPrivileged, // TODO(vmarmol): Implement support for HostNetworkSources. PrivilegedSources: capabilities.PrivilegedSources{ HostNetworkSources: []string{}, HostPIDSources: []string{}, HostIPCSources: []string{}, }, PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, }) cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } // Setup tunneler if needed var tunneler master.Tunneler var proxyDialerFn apiserver.ProxyDialerFunc if len(s.SSHUser) > 0 { // Get ssh key distribution func, if supported var installSSH master.InstallSSHKey if cloud != nil { if instances, supported := cloud.Instances(); supported { installSSH = instances.AddSSHKeyToAllInstances } } // Set up the tunneler tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, installSSH) // Use the tunneler's dialer to connect to the kubelet s.KubeletConfig.Dial = tunneler.Dial // Use the tunneler's dialer when proxying to pods, services, and nodes proxyDialerFn = tunneler.Dial } // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig) if err != nil { glog.Fatalf("Failure to start kubelet client: %v", err) } apiGroupVersionOverrides, err := s.parseRuntimeConfig() if err != nil { glog.Fatalf("error in parsing runtime-config: %s", err) } clientConfig := &client.Config{ Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)), } if len(s.DeprecatedStorageVersion) != 0 { gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion) if err != nil { glog.Fatalf("error in parsing group version: %s", err) } clientConfig.GroupVersion = &gv } client, err := client.New(clientConfig) if err != nil { glog.Fatalf("Invalid server address: %v", err) } legacyV1Group, err := latest.Group("") if err != nil { return err } storageDestinations := master.NewStorageDestinations() storageVersions := generateStorageVersionMap(s.DeprecatedStorageVersion, s.StorageVersions) if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.GroupVersion.Group, storageVersions) } etcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, legacyV1Group.InterfacesFor, storageVersions[legacyV1Group.GroupVersion.Group], s.EtcdPathPrefix) if err != nil { glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup("", etcdStorage) if !apiGroupVersionOverrides["extensions/v1beta1"].Disable { expGroup, err := latest.Group("extensions") if err != nil { glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) } if _, found := storageVersions[expGroup.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.GroupVersion.Group, storageVersions) } expEtcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, expGroup.InterfacesFor, storageVersions[expGroup.GroupVersion.Group], s.EtcdPathPrefix) if err != nil { glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup("extensions", expEtcdStorage) } updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdPathPrefix, &storageDestinations, newEtcd) n := s.ServiceClusterIPRange // Default to the private server key for service account token signing if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" { if apiserver.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) { s.ServiceAccountKeyFile = s.TLSPrivateKeyFile } else { glog.Warning("No RSA key provided, service account token authentication disabled") } } authenticator, err := apiserver.NewAuthenticator(apiserver.AuthenticatorConfig{ BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, ServiceAccountKeyFile: s.ServiceAccountKeyFile, ServiceAccountLookup: s.ServiceAccountLookup, Storage: etcdStorage, KeystoneURL: s.KeystoneURL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationPolicyFile) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile) if len(s.ExternalHost) == 0 { // TODO: extend for other providers if s.CloudProvider == "gce" { instances, supported := cloud.Instances() if !supported { glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.") } name, err := os.Hostname() if err != nil { glog.Fatalf("Failed to get hostname: %v", err) } addrs, err := instances.NodeAddresses(name) if err != nil { glog.Warningf("Unable to obtain external host address from cloud provider: %v", err) } else { for _, addr := range addrs { if addr.Type == api.NodeExternalIP { s.ExternalHost = addr.Address } } } } } config := &master.Config{ StorageDestinations: storageDestinations, StorageVersions: storageVersions, EventTTL: s.EventTTL, KubeletClient: kubeletClient, ServiceClusterIPRange: &n, EnableCoreControllers: true, EnableLogsSupport: s.EnableLogsSupport, EnableUISupport: true, EnableSwaggerSupport: true, EnableProfiling: s.EnableProfiling, EnableWatchCache: s.EnableWatchCache, EnableIndex: true, APIPrefix: s.APIPrefix, APIGroupPrefix: s.APIGroupPrefix, CorsAllowedOriginList: s.CorsAllowedOriginList, ReadWritePort: s.SecurePort, PublicAddress: s.AdvertiseAddress, Authenticator: authenticator, SupportsBasicAuth: len(s.BasicAuthFile) > 0, Authorizer: authorizer, AdmissionControl: admissionController, APIGroupVersionOverrides: apiGroupVersionOverrides, MasterServiceNamespace: s.MasterServiceNamespace, ClusterName: s.ClusterName, ExternalHost: s.ExternalHost, MinRequestTimeout: s.MinRequestTimeout, ProxyDialer: proxyDialerFn, ProxyTLSClientConfig: proxyTLSClientConfig, Tunneler: tunneler, ServiceNodePortRange: s.ServiceNodePortRange, KubernetesServiceNodePort: s.KubernetesServiceNodePort, } m := master.New(config) // We serve on 2 ports. See docs/accessing_the_api.md secureLocation := "" if s.SecurePort != 0 { secureLocation = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort)) } insecureLocation := net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)) // See the flag commentary to understand our assumptions when opening the read-only and read-write ports. var sem chan bool if s.MaxRequestsInFlight > 0 { sem = make(chan bool, s.MaxRequestsInFlight) } longRunningRE := regexp.MustCompile(s.LongRunningRequestRE) longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) { // TODO unify this with apiserver.MaxInFlightLimit if longRunningRE.MatchString(req.URL.Path) || req.URL.Query().Get("watch") == "true" { return nil, "" } return time.After(time.Minute), "" } if secureLocation != "" { handler := apiserver.TimeoutHandler(m.Handler, longRunningTimeout) secureServer := &http.Server{ Addr: secureLocation, Handler: apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(handler)), MaxHeaderBytes: 1 << 20, TLSConfig: &tls.Config{ // Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability) MinVersion: tls.VersionTLS10, }, } if len(s.ClientCAFile) > 0 { clientCAs, err := util.CertPoolFromFile(s.ClientCAFile) if err != nil { glog.Fatalf("Unable to load client CA file: %v", err) } // Populate PeerCertificates in requests, but don't reject connections without certificates // This allows certificates to be validated by authenticators, while still allowing other auth types secureServer.TLSConfig.ClientAuth = tls.RequestClientCert // Specify allowed CAs for client certificates secureServer.TLSConfig.ClientCAs = clientCAs } glog.Infof("Serving securely on %s", secureLocation) if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" { s.TLSCertFile = path.Join(s.CertDirectory, "apiserver.crt") s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "apiserver.key") // TODO (cjcullen): Is PublicAddress the right address to sign a cert with? alternateIPs := []net.IP{config.ServiceReadWriteIP} alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"} // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil { glog.Errorf("Unable to generate self signed cert: %v", err) } else { glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile) } } go func() { defer util.HandleCrash() for { // err == systemd.SdNotifyNoSocket when not running on a systemd system if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) } if err := secureServer.ListenAndServeTLS(s.TLSCertFile, s.TLSPrivateKeyFile); err != nil { glog.Errorf("Unable to listen for secure (%v); will try again.", err) } time.Sleep(15 * time.Second) } }() } handler := apiserver.TimeoutHandler(m.InsecureHandler, longRunningTimeout) http := &http.Server{ Addr: insecureLocation, Handler: apiserver.RecoverPanics(handler), MaxHeaderBytes: 1 << 20, } if secureLocation == "" { // err == systemd.SdNotifyNoSocket when not running on a systemd system if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) } } glog.Infof("Serving insecurely on %s", insecureLocation) glog.Fatal(http.ListenAndServe()) return nil }
func (c *MasterConfig) GetRestStorage() map[string]rest.Storage { defaultRegistry := env("OPENSHIFT_DEFAULT_REGISTRY", "${DOCKER_REGISTRY_SERVICE_HOST}:${DOCKER_REGISTRY_SERVICE_PORT}") svcCache := service.NewServiceResolverCache(c.KubeClient().Services(kapi.NamespaceDefault).Get) defaultRegistryFunc, err := svcCache.Defer(defaultRegistry) if err != nil { glog.Fatalf("OPENSHIFT_DEFAULT_REGISTRY variable is invalid %q: %v", defaultRegistry, err) } kubeletClient, err := kubeletclient.NewStaticKubeletClient(c.KubeletClientConfig) if err != nil { glog.Fatalf("Unable to configure Kubelet client: %v", err) } // TODO: allow the system CAs and the local CAs to be joined together. importTransport, err := restclient.TransportFor(&restclient.Config{}) if err != nil { glog.Fatalf("Unable to configure a default transport for importing: %v", err) } insecureImportTransport, err := restclient.TransportFor(&restclient.Config{Insecure: true}) if err != nil { glog.Fatalf("Unable to configure a default transport for importing: %v", err) } applicationStorage := application.NewREST(c.EtcdHelper, c.PrivilegedLoopbackOpenShiftClient, c.PrivilegedLoopbackKubernetesClient) serviceBrokerStorage := servicebroker.NewREST(c.EtcdHelper, c.BackingServiceInstanceControllerClients()) backingServiceStorage := backingservice.NewREST(c.EtcdHelper, c.BackingServiceInstanceControllerClients()) buildStorage, buildDetailsStorage := buildetcd.NewREST(c.EtcdHelper) buildRegistry := buildregistry.NewRegistry(buildStorage) buildConfigStorage := buildconfigetcd.NewREST(c.EtcdHelper) buildConfigRegistry := buildconfigregistry.NewRegistry(buildConfigStorage) deployConfigStorage, deployConfigScaleStorage := deployconfigetcd.NewREST(c.EtcdHelper, c.DeploymentConfigScaleClient()) deployConfigRegistry := deployconfigregistry.NewRegistry(deployConfigStorage) routeAllocator := c.RouteAllocator() routeStorage, routeStatusStorage := routeetcd.NewREST(c.EtcdHelper, routeAllocator) hostSubnetStorage := hostsubnetetcd.NewREST(c.EtcdHelper) netNamespaceStorage := netnamespaceetcd.NewREST(c.EtcdHelper) clusterNetworkStorage := clusternetworketcd.NewREST(c.EtcdHelper) userStorage := useretcd.NewREST(c.EtcdHelper) userRegistry := userregistry.NewRegistry(userStorage) identityStorage := identityetcd.NewREST(c.EtcdHelper) identityRegistry := identityregistry.NewRegistry(identityStorage) userIdentityMappingStorage := useridentitymapping.NewREST(userRegistry, identityRegistry) policyStorage := policyetcd.NewStorage(c.EtcdHelper) policyRegistry := policyregistry.NewRegistry(policyStorage) policyBindingStorage := policybindingetcd.NewStorage(c.EtcdHelper) policyBindingRegistry := policybindingregistry.NewRegistry(policyBindingStorage) clusterPolicyStorage := clusterpolicystorage.NewStorage(c.EtcdHelper) clusterPolicyRegistry := clusterpolicyregistry.NewRegistry(clusterPolicyStorage) clusterPolicyBindingStorage := clusterpolicybindingstorage.NewStorage(c.EtcdHelper) clusterPolicyBindingRegistry := clusterpolicybindingregistry.NewRegistry(clusterPolicyBindingStorage) ruleResolver := rulevalidation.NewDefaultRuleResolver( policyRegistry, policyBindingRegistry, clusterPolicyRegistry, clusterPolicyBindingRegistry, ) roleStorage := rolestorage.NewVirtualStorage(policyRegistry, ruleResolver) roleBindingStorage := rolebindingstorage.NewVirtualStorage(policyBindingRegistry, ruleResolver) clusterRoleStorage := clusterrolestorage.NewClusterRoleStorage(clusterPolicyRegistry, clusterPolicyBindingRegistry) clusterRoleBindingStorage := clusterrolebindingstorage.NewClusterRoleBindingStorage(clusterPolicyRegistry, clusterPolicyBindingRegistry) subjectAccessReviewStorage := subjectaccessreview.NewREST(c.Authorizer) subjectAccessReviewRegistry := subjectaccessreview.NewRegistry(subjectAccessReviewStorage) localSubjectAccessReviewStorage := localsubjectaccessreview.NewREST(subjectAccessReviewRegistry) resourceAccessReviewStorage := resourceaccessreview.NewREST(c.Authorizer) resourceAccessReviewRegistry := resourceaccessreview.NewRegistry(resourceAccessReviewStorage) localResourceAccessReviewStorage := localresourceaccessreview.NewREST(resourceAccessReviewRegistry) imageStorage := imageetcd.NewREST(c.EtcdHelper) imageRegistry := image.NewRegistry(imageStorage) imageStreamSecretsStorage := imagesecret.NewREST(c.ImageStreamSecretClient()) imageStreamStorage, imageStreamStatusStorage, internalImageStreamStorage := imagestreametcd.NewREST(c.EtcdHelper, imagestream.DefaultRegistryFunc(defaultRegistryFunc), subjectAccessReviewRegistry) imageStreamRegistry := imagestream.NewRegistry(imageStreamStorage, imageStreamStatusStorage, internalImageStreamStorage) imageStreamMappingStorage := imagestreammapping.NewREST(imageRegistry, imageStreamRegistry) imageStreamTagStorage := imagestreamtag.NewREST(imageRegistry, imageStreamRegistry) imageStreamTagRegistry := imagestreamtag.NewRegistry(imageStreamTagStorage) importerFn := func(r importer.RepositoryRetriever) imageimporter.Interface { return imageimporter.NewImageStreamImporter(r, c.Options.ImagePolicyConfig.MaxImagesBulkImportedPerRepository, util.NewTokenBucketRateLimiter(2.0, 3)) } importerDockerClientFn := func() dockerregistry.Client { return dockerregistry.NewClient(20*time.Second, false) } imageStreamImportStorage := imagestreamimport.NewREST(importerFn, imageStreamRegistry, internalImageStreamStorage, imageStorage, c.ImageStreamImportSecretClient(), importTransport, insecureImportTransport, importerDockerClientFn) imageStreamImageStorage := imagestreamimage.NewREST(imageRegistry, imageStreamRegistry) imageStreamImageRegistry := imagestreamimage.NewRegistry(imageStreamImageStorage) backingServiceInstanceEtcd := backingserviceinstanceetcd.NewREST(c.EtcdHelper) backingServiceInstanceRegistry := backingserviceinstanceregistry.NewRegistry(backingServiceInstanceEtcd) backingServiceInstanceBindingEtcd := backingserviceinstanceetcd.NewBindingREST(backingServiceInstanceRegistry, deployConfigRegistry) buildGenerator := &buildgenerator.BuildGenerator{ Client: buildgenerator.Client{ GetBuildConfigFunc: buildConfigRegistry.GetBuildConfig, UpdateBuildConfigFunc: buildConfigRegistry.UpdateBuildConfig, GetBuildFunc: buildRegistry.GetBuild, CreateBuildFunc: buildRegistry.CreateBuild, GetImageStreamFunc: imageStreamRegistry.GetImageStream, GetImageStreamImageFunc: imageStreamImageRegistry.GetImageStreamImage, GetImageStreamTagFunc: imageStreamTagRegistry.GetImageStreamTag, }, ServiceAccounts: c.KubeClient(), Secrets: c.KubeClient(), } // TODO: with sharding, this needs to be changed deployConfigGenerator := &deployconfiggenerator.DeploymentConfigGenerator{ Client: deployconfiggenerator.Client{ DCFn: deployConfigRegistry.GetDeploymentConfig, ISFn: imageStreamRegistry.GetImageStream, LISFn2: imageStreamRegistry.ListImageStreams, }, } configClient, kclient := c.DeploymentConfigClients() deployRollback := &deployrollback.RollbackGenerator{} deployRollbackClient := deployrollback.Client{ DCFn: deployConfigRegistry.GetDeploymentConfig, RCFn: clientDeploymentInterface{kclient}.GetDeployment, GRFn: deployRollback.GenerateRollback, } projectStorage := projectproxy.NewREST(kclient.Namespaces(), c.ProjectAuthorizationCache) namespace, templateName, err := configapi.ParseNamespaceAndName(c.Options.ProjectConfig.ProjectRequestTemplate) if err != nil { glog.Errorf("Error parsing project request template value: %v", err) // we can continue on, the storage that gets created will be valid, it simply won't work properly. There's no reason to kill the master } projectRequestStorage := projectrequeststorage.NewREST(c.Options.ProjectConfig.ProjectRequestMessage, namespace, templateName, c.PrivilegedLoopbackOpenShiftClient, c.PrivilegedLoopbackKubernetesClient) bcClient := c.BuildConfigWebHookClient() buildConfigWebHooks := buildconfigregistry.NewWebHookREST( buildConfigRegistry, buildclient.NewOSClientBuildConfigInstantiatorClient(bcClient), map[string]webhook.Plugin{ "generic": generic.New(), "github": github.New(), }, ) storage := map[string]rest.Storage{ "images": imageStorage, "imageStreams/secrets": imageStreamSecretsStorage, "imageStreams": imageStreamStorage, "imageStreams/status": imageStreamStatusStorage, "imageStreamImports": imageStreamImportStorage, "imageStreamImages": imageStreamImageStorage, "imageStreamMappings": imageStreamMappingStorage, "imageStreamTags": imageStreamTagStorage, "applications": applicationStorage, "serviceBrokers": serviceBrokerStorage, "backingServices": backingServiceStorage, "backingServiceInstances": backingServiceInstanceEtcd, "backingServiceInstances/binding": backingServiceInstanceBindingEtcd, "deploymentConfigs": deployConfigStorage, "deploymentConfigs/scale": deployConfigScaleStorage, "generateDeploymentConfigs": deployconfiggenerator.NewREST(deployConfigGenerator, c.EtcdHelper.Codec()), "deploymentConfigRollbacks": deployrollback.NewREST(deployRollbackClient, c.EtcdHelper.Codec()), "deploymentConfigs/log": deploylogregistry.NewREST(configClient, kclient, c.DeploymentLogClient(), kubeletClient), "processedTemplates": templateregistry.NewREST(), "templates": templateetcd.NewREST(c.EtcdHelper), "routes": routeStorage, "routes/status": routeStatusStorage, "projects": projectStorage, "projectRequests": projectRequestStorage, "hostSubnets": hostSubnetStorage, "netNamespaces": netNamespaceStorage, "clusterNetworks": clusterNetworkStorage, "users": userStorage, "groups": groupetcd.NewREST(c.EtcdHelper), "identities": identityStorage, "userIdentityMappings": userIdentityMappingStorage, "oAuthAuthorizeTokens": authorizetokenetcd.NewREST(c.EtcdHelper), "oAuthAccessTokens": accesstokenetcd.NewREST(c.EtcdHelper), "oAuthClients": clientetcd.NewREST(c.EtcdHelper), "oAuthClientAuthorizations": clientauthetcd.NewREST(c.EtcdHelper), "resourceAccessReviews": resourceAccessReviewStorage, "subjectAccessReviews": subjectAccessReviewStorage, "localSubjectAccessReviews": localSubjectAccessReviewStorage, "localResourceAccessReviews": localResourceAccessReviewStorage, "policies": policyStorage, "policyBindings": policyBindingStorage, "roles": roleStorage, "roleBindings": roleBindingStorage, "clusterPolicies": clusterPolicyStorage, "clusterPolicyBindings": clusterPolicyBindingStorage, "clusterRoleBindings": clusterRoleBindingStorage, "clusterRoles": clusterRoleStorage, } if configapi.IsBuildEnabled(&c.Options) { storage["builds"] = buildStorage storage["buildConfigs"] = buildConfigStorage storage["buildConfigs/webhooks"] = buildConfigWebHooks storage["builds/clone"] = buildclone.NewStorage(buildGenerator) storage["buildConfigs/instantiate"] = buildconfiginstantiate.NewStorage(buildGenerator) storage["buildConfigs/instantiatebinary"] = buildconfiginstantiate.NewBinaryStorage(buildGenerator, buildStorage, c.BuildLogClient(), kubeletClient) storage["builds/log"] = buildlogregistry.NewREST(buildStorage, buildStorage, c.BuildLogClient(), kubeletClient) storage["builds/details"] = buildDetailsStorage } return storage }
// Run runs the specified APIServer. This should never exit. func Run(s *options.APIServer) error { genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions) if len(s.StorageConfig.ServerList) == 0 { glog.Fatalf("--etcd-servers must be specified") } capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: s.AllowPrivileged, // TODO(vmarmol): Implement support for HostNetworkSources. PrivilegedSources: capabilities.PrivilegedSources{ HostNetworkSources: []string{}, HostPIDSources: []string{}, HostIPCSources: []string{}, }, PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, }) cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } // Setup tunneler if needed var tunneler genericapiserver.Tunneler var proxyDialerFn apiserver.ProxyDialerFunc if len(s.SSHUser) > 0 { // Get ssh key distribution func, if supported var installSSH genericapiserver.InstallSSHKey if cloud != nil { if instances, supported := cloud.Instances(); supported { installSSH = instances.AddSSHKeyToAllInstances } } if s.KubeletConfig.Port == 0 { glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.") } // Set up the tunneler // TODO(cjcullen): If we want this to handle per-kubelet ports or other // kubelet listen-addresses, we need to plumb through options. healthCheckPath := &url.URL{ Scheme: "https", Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)), Path: "healthz", } tunneler = genericapiserver.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH) // Use the tunneler's dialer to connect to the kubelet s.KubeletConfig.Dial = tunneler.Dial // Use the tunneler's dialer when proxying to pods, services, and nodes proxyDialerFn = tunneler.Dial } // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig) if err != nil { glog.Fatalf("Failure to start kubelet client: %v", err) } apiResourceConfigSource, err := parseRuntimeConfig(s) if err != nil { glog.Fatalf("error in parsing runtime-config: %s", err) } clientConfig := &restclient.Config{ Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)), // Increase QPS limits. The client is currently passed to all admission plugins, // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 // for more details. Once #22422 is fixed, we may want to remove it. QPS: 50, Burst: 100, } if len(s.DeprecatedStorageVersion) != 0 { gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion) if err != nil { glog.Fatalf("error in parsing group version: %s", err) } clientConfig.GroupVersion = &gv } client, err := clientset.NewForConfig(clientConfig) if err != nil { glog.Errorf("Failed to create clientset: %v", err) } resourceEncoding := genericapiserver.NewDefaultResourceEncodingConfig() groupToEncoding, err := s.StorageGroupsToEncodingVersion() if err != nil { glog.Fatalf("error getting group encoding: %s", err) } for group, storageEncodingVersion := range groupToEncoding { resourceEncoding.SetVersionEncoding(group, storageEncodingVersion, unversioned.GroupVersion{Group: group, Version: runtime.APIVersionInternal}) } storageFactory := genericapiserver.NewDefaultStorageFactory(s.StorageConfig, api.Codecs, resourceEncoding, apiResourceConfigSource) storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs")) storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers")) for _, override := range s.EtcdServersOverrides { tokens := strings.Split(override, "#") if len(tokens) != 2 { glog.Errorf("invalid value of etcd server overrides: %s", override) continue } apiresource := strings.Split(tokens[0], "/") if len(apiresource) != 2 { glog.Errorf("invalid resource definition: %s", tokens[0]) continue } group := apiresource[0] resource := apiresource[1] groupResource := unversioned.GroupResource{Group: group, Resource: resource} servers := strings.Split(tokens[1], ";") storageFactory.SetEtcdLocation(groupResource, servers) } // Default to the private server key for service account token signing if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" { if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) { s.ServiceAccountKeyFile = s.TLSPrivateKeyFile } else { glog.Warning("No RSA key provided, service account token authentication disabled") } } var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter if s.ServiceAccountLookup { // If we need to look up service accounts and tokens, // go directly to etcd to avoid recursive auth insanity storage, err := storageFactory.New(api.Resource("serviceaccounts")) if err != nil { glog.Fatalf("Unable to get serviceaccounts storage: %v", err) } serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storage) } authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{ BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, OIDCGroupsClaim: s.OIDCGroupsClaim, ServiceAccountKeyFile: s.ServiceAccountKeyFile, ServiceAccountLookup: s.ServiceAccountLookup, ServiceAccountTokenGetter: serviceAccountGetter, KeystoneURL: s.KeystoneURL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationConfig) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile) if len(s.ExternalHost) == 0 { // TODO: extend for other providers if s.CloudProvider == "gce" { instances, supported := cloud.Instances() if !supported { glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.") } name, err := os.Hostname() if err != nil { glog.Fatalf("Failed to get hostname: %v", err) } addrs, err := instances.NodeAddresses(name) if err != nil { glog.Warningf("Unable to obtain external host address from cloud provider: %v", err) } else { for _, addr := range addrs { if addr.Type == api.NodeExternalIP { s.ExternalHost = addr.Address } } } } } genericConfig := genericapiserver.NewConfig(s.ServerRunOptions) // TODO: Move the following to generic api server as well. genericConfig.StorageFactory = storageFactory genericConfig.Authenticator = authenticator genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0 genericConfig.Authorizer = authorizer genericConfig.AdmissionControl = admissionController genericConfig.APIResourceConfigSource = apiResourceConfigSource genericConfig.MasterServiceNamespace = s.MasterServiceNamespace genericConfig.ProxyDialer = proxyDialerFn genericConfig.ProxyTLSClientConfig = proxyTLSClientConfig genericConfig.Serializer = api.Codecs config := &master.Config{ Config: genericConfig, EnableCoreControllers: true, DeleteCollectionWorkers: s.DeleteCollectionWorkers, EventTTL: s.EventTTL, KubeletClient: kubeletClient, Tunneler: tunneler, } if s.EnableWatchCache { cachesize.SetWatchCacheSizes(s.WatchCacheSizes) } m, err := master.New(config) if err != nil { return err } m.Run(s.ServerRunOptions) return nil }
// Run runs the specified APIServer. This should never exit. func Run(s *options.APIServer) error { genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions) capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: s.AllowPrivileged, // TODO(vmarmol): Implement support for HostNetworkSources. PrivilegedSources: capabilities.PrivilegedSources{ HostNetworkSources: []string{}, HostPIDSources: []string{}, HostIPCSources: []string{}, }, PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, }) // Setup tunneler if needed var tunneler genericapiserver.Tunneler var proxyDialerFn apiserver.ProxyDialerFunc if len(s.SSHUser) > 0 { // Get ssh key distribution func, if supported var installSSH genericapiserver.InstallSSHKey cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } if cloud != nil { if instances, supported := cloud.Instances(); supported { installSSH = instances.AddSSHKeyToAllInstances } } if s.KubeletConfig.Port == 0 { glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.") } // Set up the tunneler // TODO(cjcullen): If we want this to handle per-kubelet ports or other // kubelet listen-addresses, we need to plumb through options. healthCheckPath := &url.URL{ Scheme: "https", Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)), Path: "healthz", } tunneler = genericapiserver.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH) // Use the tunneler's dialer to connect to the kubelet s.KubeletConfig.Dial = tunneler.Dial // Use the tunneler's dialer when proxying to pods, services, and nodes proxyDialerFn = tunneler.Dial } // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig) if err != nil { glog.Fatalf("Failure to start kubelet client: %v", err) } storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion() if err != nil { glog.Fatalf("error generating storage version map: %s", err) } storageFactory, err := genericapiserver.BuildDefaultStorageFactory( s.StorageConfig, s.DefaultStorageMediaType, api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion, master.DefaultAPIResourceConfigSource(), s.RuntimeConfig) if err != nil { glog.Fatalf("error in initializing storage factory: %s", err) } storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs")) storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers")) for _, override := range s.EtcdServersOverrides { tokens := strings.Split(override, "#") if len(tokens) != 2 { glog.Errorf("invalid value of etcd server overrides: %s", override) continue } apiresource := strings.Split(tokens[0], "/") if len(apiresource) != 2 { glog.Errorf("invalid resource definition: %s", tokens[0]) continue } group := apiresource[0] resource := apiresource[1] groupResource := unversioned.GroupResource{Group: group, Resource: resource} servers := strings.Split(tokens[1], ";") storageFactory.SetEtcdLocation(groupResource, servers) } // Default to the private server key for service account token signing if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" { if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) { s.ServiceAccountKeyFile = s.TLSPrivateKeyFile } else { glog.Warning("No RSA key provided, service account token authentication disabled") } } var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter if s.ServiceAccountLookup { // If we need to look up service accounts and tokens, // go directly to etcd to avoid recursive auth insanity storage, err := storageFactory.New(api.Resource("serviceaccounts")) if err != nil { glog.Fatalf("Unable to get serviceaccounts storage: %v", err) } serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storage) } authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{ BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, OIDCGroupsClaim: s.OIDCGroupsClaim, ServiceAccountKeyFile: s.ServiceAccountKeyFile, ServiceAccountLookup: s.ServiceAccountLookup, ServiceAccountTokenGetter: serviceAccountGetter, KeystoneURL: s.KeystoneURL, WebhookTokenAuthnConfigFile: s.WebhookTokenAuthnConfigFile, WebhookTokenAuthnCacheTTL: s.WebhookTokenAuthnCacheTTL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationConfig) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") client, err := s.NewSelfClient() if err != nil { glog.Errorf("Failed to create clientset: %v", err) } admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile) genericConfig := genericapiserver.NewConfig(s.ServerRunOptions) // TODO: Move the following to generic api server as well. genericConfig.StorageFactory = storageFactory genericConfig.Authenticator = authenticator genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0 genericConfig.Authorizer = authorizer genericConfig.AdmissionControl = admissionController genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource genericConfig.MasterServiceNamespace = s.MasterServiceNamespace genericConfig.ProxyDialer = proxyDialerFn genericConfig.ProxyTLSClientConfig = proxyTLSClientConfig genericConfig.Serializer = api.Codecs config := &master.Config{ Config: genericConfig, EnableCoreControllers: true, DeleteCollectionWorkers: s.DeleteCollectionWorkers, EventTTL: s.EventTTL, KubeletClient: kubeletClient, Tunneler: tunneler, } if s.EnableWatchCache { cachesize.SetWatchCacheSizes(s.WatchCacheSizes) } m, err := master.New(config) if err != nil { return err } m.Run(s.ServerRunOptions) return nil }
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, informers shared.InformerFactory, admissionControl admission.Interface, originAuthenticator authenticator.Request) (*MasterConfig, error) { if options.KubernetesMasterConfig == nil { return nil, errors.New("insufficient information to build KubernetesMasterConfig") } kubeletClientConfig := configapi.GetKubeletClientConfig(options) kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig) if err != nil { return nil, fmt.Errorf("unable to configure Kubelet client: %v", err) } // in-order list of plug-ins that should intercept admission decisions // TODO: Push node environment support to upstream in future podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout) if err != nil { return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err) } // Defaults are tested in TestCMServerDefaults cmserver := cmapp.NewCMServer() // Adjust defaults cmserver.Address = "" // no healthz endpoint cmserver.Port = 0 // no healthz endpoint cmserver.EnableGarbageCollector = false // disabled until we add the controller cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout} cmserver.VolumeConfiguration.EnableDynamicProvisioning = options.VolumeConfig.DynamicProvisioningEnabled // resolve extended arguments // TODO: this should be done in config validation (along with the above) so we can provide // proper errors if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 { return nil, kerrors.NewAggregate(err) } // resolve extended arguments // TODO: this should be done in config validation (along with the above) so we can provide // proper errors schedulerserver := scheduleroptions.NewSchedulerServer() schedulerserver.PolicyConfigFile = options.KubernetesMasterConfig.SchedulerConfigFile if err := cmdflags.Resolve(options.KubernetesMasterConfig.SchedulerArguments, schedulerserver.AddFlags); len(err) > 0 { return nil, kerrors.NewAggregate(err) } cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile) if err != nil { return nil, err } if cloud != nil { glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", cmserver.CloudProvider, cmserver.CloudConfigFile) } var proxyClientCerts []tls.Certificate if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 { clientCert, err := tls.LoadX509KeyPair( options.KubernetesMasterConfig.ProxyClientInfo.CertFile, options.KubernetesMasterConfig.ProxyClientInfo.KeyFile, ) if err != nil { return nil, err } proxyClientCerts = append(proxyClientCerts, clientCert) } server, storageFactory, err := BuildDefaultAPIServer(options) if err != nil { return nil, err } // Preserve previous behavior of using the first non-loopback address // TODO: Deprecate this behavior and just require a valid value to be passed in publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP) if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() { hostIP, err := knet.ChooseHostInterface() if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err) } publicAddress = hostIP glog.Infof("Will report %v as public IP address.", publicAddress) } m := &master.Config{ Config: &genericapiserver.Config{ PublicAddress: publicAddress, ReadWritePort: server.SecurePort, Authenticator: originAuthenticator, // this is used to fulfill the tokenreviews endpoint which is used by node authentication Authorizer: authorizer.NewAlwaysAllowAuthorizer(), AdmissionControl: admissionControl, StorageFactory: storageFactory, ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange), ServiceNodePortRange: server.ServiceNodePortRange, RequestContextMapper: requestContextMapper, APIResourceConfigSource: getAPIResourceConfig(options), APIPrefix: server.APIPrefix, APIGroupPrefix: server.APIGroupPrefix, MasterCount: server.MasterCount, // Set the TLS options for proxying to pods and services // Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname) ProxyTLSClientConfig: &tls.Config{ // Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs InsecureSkipVerify: true, Certificates: proxyClientCerts, }, Serializer: kapi.Codecs, EnableLogsSupport: server.EnableLogsSupport, EnableProfiling: server.EnableProfiling, EnableWatchCache: server.EnableWatchCache, MasterServiceNamespace: server.MasterServiceNamespace, ExternalHost: server.ExternalHost, MinRequestTimeout: server.MinRequestTimeout, KubernetesServiceNodePort: server.KubernetesServiceNodePort, }, EventTTL: server.EventTTL, KubeletClient: kubeletClient, EnableCoreControllers: true, DeleteCollectionWorkers: server.DeleteCollectionWorkers, } if server.EnableWatchCache { cachesize.SetWatchCacheSizes(server.WatchCacheSizes) } if options.DNSConfig != nil { _, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress) if err != nil { return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err) } dnsPort, err := strconv.Atoi(dnsPortStr) if err != nil { return nil, fmt.Errorf("invalid DNS port: %v", err) } m.ExtraServicePorts = append(m.ExtraServicePorts, kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)}, kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)}, ) m.ExtraEndpointPorts = append(m.ExtraEndpointPorts, kapi.EndpointPort{Name: "dns", Port: int32(dnsPort), Protocol: kapi.ProtocolUDP}, kapi.EndpointPort{Name: "dns-tcp", Port: int32(dnsPort), Protocol: kapi.ProtocolTCP}, ) } kmaster := &MasterConfig{ Options: *options.KubernetesMasterConfig, KubeClient: kubeClient, Master: m, ControllerManager: cmserver, CloudProvider: cloud, SchedulerServer: schedulerserver, Informers: informers, } return kmaster, nil }
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, informers shared.InformerFactory, pluginInitializer oadmission.PluginInitializer) (*MasterConfig, error) { if options.KubernetesMasterConfig == nil { return nil, errors.New("insufficient information to build KubernetesMasterConfig") } kubeletClientConfig := configapi.GetKubeletClientConfig(options) kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig) if err != nil { return nil, fmt.Errorf("unable to configure Kubelet client: %v", err) } // in-order list of plug-ins that should intercept admission decisions // TODO: Push node environment support to upstream in future _, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress) if err != nil { return nil, err } port, err := strconv.Atoi(portString) if err != nil { return nil, err } portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange) if err != nil { return nil, err } podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout) if err != nil { return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err) } // Defaults are tested in TestAPIServerDefaults server := apiserveroptions.NewAPIServer() // Adjust defaults server.EventTTL = 2 * time.Hour server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet)) server.ServiceNodePortRange = *portRange server.AdmissionControl = strings.Join(AdmissionPlugins, ",") server.EnableLogsSupport = false // don't expose server logs server.EnableProfiling = false server.APIPrefix = KubeAPIPrefix server.APIGroupPrefix = KubeAPIGroupPrefix server.SecurePort = port server.MasterCount = options.KubernetesMasterConfig.MasterCount // resolve extended arguments // TODO: this should be done in config validation (along with the above) so we can provide // proper errors if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 { return nil, kerrors.NewAggregate(err) } if len(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride) > 0 { server.AdmissionControl = strings.Join(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride, ",") } // Defaults are tested in TestCMServerDefaults cmserver := cmapp.NewCMServer() // Adjust defaults cmserver.Address = "" // no healthz endpoint cmserver.Port = 0 // no healthz endpoint cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout} cmserver.VolumeConfiguration.EnableDynamicProvisioning = options.VolumeConfig.DynamicProvisioningEnabled // resolve extended arguments // TODO: this should be done in config validation (along with the above) so we can provide // proper errors if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 { return nil, kerrors.NewAggregate(err) } cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile) if err != nil { return nil, err } if cloud != nil { glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile) } plugins := []admission.Interface{} for _, pluginName := range strings.Split(server.AdmissionControl, ",") { switch pluginName { case lifecycle.PluginName: // We need to include our infrastructure and shared resource namespaces in the immortal namespaces list immortalNamespaces := sets.NewString(kapi.NamespaceDefault) if len(options.PolicyConfig.OpenShiftSharedResourcesNamespace) > 0 { immortalNamespaces.Insert(options.PolicyConfig.OpenShiftSharedResourcesNamespace) } if len(options.PolicyConfig.OpenShiftInfrastructureNamespace) > 0 { immortalNamespaces.Insert(options.PolicyConfig.OpenShiftInfrastructureNamespace) } plugins = append(plugins, lifecycle.NewLifecycle(clientadapter.FromUnversionedClient(kubeClient), immortalNamespaces)) case serviceadmit.ExternalIPPluginName: // this needs to be moved upstream to be part of core config reject, admit, err := serviceadmit.ParseCIDRRules(options.NetworkConfig.ExternalIPNetworkCIDRs) if err != nil { // should have been caught with validation return nil, err } plugins = append(plugins, serviceadmit.NewExternalIPRanger(reject, admit)) case saadmit.PluginName: // we need to set some custom parameters on the service account admission controller, so create that one by hand saAdmitter := saadmit.NewServiceAccount(clientadapter.FromUnversionedClient(kubeClient)) saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences saAdmitter.Run() plugins = append(plugins, saAdmitter) default: configFile, err := pluginconfig.GetPluginConfigFile(options.KubernetesMasterConfig.AdmissionConfig.PluginConfig, pluginName, server.AdmissionControlConfigFile) if err != nil { return nil, err } plugin := admission.InitPlugin(pluginName, clientadapter.FromUnversionedClient(kubeClient), configFile) if plugin != nil { plugins = append(plugins, plugin) } } } pluginInitializer.Initialize(plugins) // ensure that plugins have been properly initialized if err := oadmission.Validate(plugins); err != nil { return nil, err } admissionController := admission.NewChainHandler(plugins...) var proxyClientCerts []tls.Certificate if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 { clientCert, err := tls.LoadX509KeyPair( options.KubernetesMasterConfig.ProxyClientInfo.CertFile, options.KubernetesMasterConfig.ProxyClientInfo.KeyFile, ) if err != nil { return nil, err } proxyClientCerts = append(proxyClientCerts, clientCert) } resourceEncodingConfig := genericapiserver.NewDefaultResourceEncodingConfig() resourceEncodingConfig.SetVersionEncoding( kapi.GroupName, unversioned.GroupVersion{Group: kapi.GroupName, Version: options.EtcdStorageConfig.KubernetesStorageVersion}, kapi.SchemeGroupVersion, ) resourceEncodingConfig.SetVersionEncoding( extensions.GroupName, unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"}, extensions.SchemeGroupVersion, ) resourceEncodingConfig.SetVersionEncoding( batch.GroupName, unversioned.GroupVersion{Group: batch.GroupName, Version: "v1"}, batch.SchemeGroupVersion, ) resourceEncodingConfig.SetVersionEncoding( autoscaling.GroupName, unversioned.GroupVersion{Group: autoscaling.GroupName, Version: "v1"}, autoscaling.SchemeGroupVersion, ) etcdConfig := storagebackend.Config{ Prefix: options.EtcdStorageConfig.KubernetesStoragePrefix, ServerList: options.EtcdClientInfo.URLs, KeyFile: options.EtcdClientInfo.ClientCert.KeyFile, CertFile: options.EtcdClientInfo.ClientCert.CertFile, CAFile: options.EtcdClientInfo.CA, DeserializationCacheSize: genericapiserveroptions.DefaultDeserializationCacheSize, } storageFactory := genericapiserver.NewDefaultStorageFactory(etcdConfig, "", kapi.Codecs, resourceEncodingConfig, master.DefaultAPIResourceConfigSource()) // the order here is important, it defines which version will be used for storage storageFactory.AddCohabitatingResources(extensions.Resource("jobs"), batch.Resource("jobs")) storageFactory.AddCohabitatingResources(extensions.Resource("horizontalpodautoscalers"), autoscaling.Resource("horizontalpodautoscalers")) // Preserve previous behavior of using the first non-loopback address // TODO: Deprecate this behavior and just require a valid value to be passed in publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP) if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() { hostIP, err := knet.ChooseHostInterface() if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err) } publicAddress = hostIP glog.Infof("Will report %v as public IP address.", publicAddress) } m := &master.Config{ Config: &genericapiserver.Config{ PublicAddress: publicAddress, ReadWritePort: port, Authorizer: apiserver.NewAlwaysAllowAuthorizer(), AdmissionControl: admissionController, StorageFactory: storageFactory, ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange), ServiceNodePortRange: server.ServiceNodePortRange, RequestContextMapper: requestContextMapper, APIResourceConfigSource: getAPIResourceConfig(options), APIPrefix: server.APIPrefix, APIGroupPrefix: server.APIGroupPrefix, MasterCount: server.MasterCount, // Set the TLS options for proxying to pods and services // Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname) ProxyTLSClientConfig: &tls.Config{ // Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs InsecureSkipVerify: true, Certificates: proxyClientCerts, }, Serializer: kapi.Codecs, EnableLogsSupport: server.EnableLogsSupport, EnableProfiling: server.EnableProfiling, EnableWatchCache: server.EnableWatchCache, MasterServiceNamespace: server.MasterServiceNamespace, ExternalHost: server.ExternalHost, MinRequestTimeout: server.MinRequestTimeout, KubernetesServiceNodePort: server.KubernetesServiceNodePort, }, EventTTL: server.EventTTL, KubeletClient: kubeletClient, EnableCoreControllers: true, DeleteCollectionWorkers: server.DeleteCollectionWorkers, } if server.EnableWatchCache { cachesize.SetWatchCacheSizes(server.WatchCacheSizes) } if options.DNSConfig != nil { _, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress) if err != nil { return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err) } dnsPort, err := strconv.Atoi(dnsPortStr) if err != nil { return nil, fmt.Errorf("invalid DNS port: %v", err) } m.ExtraServicePorts = append(m.ExtraServicePorts, kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)}, kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)}, ) m.ExtraEndpointPorts = append(m.ExtraEndpointPorts, kapi.EndpointPort{Name: "dns", Port: int32(dnsPort), Protocol: kapi.ProtocolUDP}, kapi.EndpointPort{Name: "dns-tcp", Port: int32(dnsPort), Protocol: kapi.ProtocolTCP}, ) } kmaster := &MasterConfig{ Options: *options.KubernetesMasterConfig, KubeClient: kubeClient, Master: m, ControllerManager: cmserver, CloudProvider: cloud, Informers: informers, } return kmaster, nil }
// Run runs the specified APIServer. This should never exit. func Run(s *options.APIServer) error { genericvalidation.VerifyEtcdServersList(s.ServerRunOptions) genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions) genericConfig := genericapiserver.NewConfig(). // create the new config ApplyOptions(s.ServerRunOptions). // apply the options selected Complete() // set default values based on the known values if err := genericConfig.MaybeGenerateServingCerts(); err != nil { glog.Fatalf("Failed to generate service certificate: %v", err) } capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: s.AllowPrivileged, // TODO(vmarmol): Implement support for HostNetworkSources. PrivilegedSources: capabilities.PrivilegedSources{ HostNetworkSources: []string{}, HostPIDSources: []string{}, HostIPCSources: []string{}, }, PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, }) // Setup tunneler if needed var tunneler genericapiserver.Tunneler var proxyDialerFn apiserver.ProxyDialerFunc if len(s.SSHUser) > 0 { // Get ssh key distribution func, if supported var installSSH genericapiserver.InstallSSHKey cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } if cloud != nil { if instances, supported := cloud.Instances(); supported { installSSH = instances.AddSSHKeyToAllInstances } } if s.KubeletConfig.Port == 0 { glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.") } // Set up the tunneler // TODO(cjcullen): If we want this to handle per-kubelet ports or other // kubelet listen-addresses, we need to plumb through options. healthCheckPath := &url.URL{ Scheme: "https", Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)), Path: "healthz", } tunneler = genericapiserver.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH) // Use the tunneler's dialer to connect to the kubelet s.KubeletConfig.Dial = tunneler.Dial // Use the tunneler's dialer when proxying to pods, services, and nodes proxyDialerFn = tunneler.Dial } // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig) if err != nil { glog.Fatalf("Failed to start kubelet client: %v", err) } if s.StorageConfig.DeserializationCacheSize == 0 { // When size of cache is not explicitly set, estimate its size based on // target memory usage. glog.V(2).Infof("Initalizing deserialization cache size based on %dMB limit", s.TargetRAMMB) // This is the heuristics that from memory capacity is trying to infer // the maximum number of nodes in the cluster and set cache sizes based // on that value. // From our documentation, we officially recomment 120GB machines for // 2000 nodes, and we scale from that point. Thus we assume ~60MB of // capacity per node. // TODO: We may consider deciding that some percentage of memory will // be used for the deserialization cache and divide it by the max object // size to compute its size. We may even go further and measure // collective sizes of the objects in the cache. clusterSize := s.TargetRAMMB / 60 s.StorageConfig.DeserializationCacheSize = 25 * clusterSize if s.StorageConfig.DeserializationCacheSize < 1000 { s.StorageConfig.DeserializationCacheSize = 1000 } } storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion() if err != nil { glog.Fatalf("error generating storage version map: %s", err) } storageFactory, err := genericapiserver.BuildDefaultStorageFactory( s.StorageConfig, s.DefaultStorageMediaType, api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion, // FIXME: this GroupVersionResource override should be configurable []unversioned.GroupVersionResource{batch.Resource("scheduledjobs").WithVersion("v2alpha1")}, master.DefaultAPIResourceConfigSource(), s.RuntimeConfig) if err != nil { glog.Fatalf("error in initializing storage factory: %s", err) } storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs")) storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers")) for _, override := range s.EtcdServersOverrides { tokens := strings.Split(override, "#") if len(tokens) != 2 { glog.Errorf("invalid value of etcd server overrides: %s", override) continue } apiresource := strings.Split(tokens[0], "/") if len(apiresource) != 2 { glog.Errorf("invalid resource definition: %s", tokens[0]) continue } group := apiresource[0] resource := apiresource[1] groupResource := unversioned.GroupResource{Group: group, Resource: resource} servers := strings.Split(tokens[1], ";") storageFactory.SetEtcdLocation(groupResource, servers) } // Default to the private server key for service account token signing if len(s.ServiceAccountKeyFiles) == 0 && s.TLSPrivateKeyFile != "" { if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) { s.ServiceAccountKeyFiles = []string{s.TLSPrivateKeyFile} } else { glog.Warning("No TLS key provided, service account token authentication disabled") } } var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter if s.ServiceAccountLookup { // If we need to look up service accounts and tokens, // go directly to etcd to avoid recursive auth insanity storageConfig, err := storageFactory.NewConfig(api.Resource("serviceaccounts")) if err != nil { glog.Fatalf("Unable to get serviceaccounts storage: %v", err) } serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets"))) } apiAuthenticator, err := authenticator.New(authenticator.AuthenticatorConfig{ Anonymous: s.AnonymousAuth, AnyToken: s.EnableAnyToken, BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, OIDCGroupsClaim: s.OIDCGroupsClaim, ServiceAccountKeyFiles: s.ServiceAccountKeyFiles, ServiceAccountLookup: s.ServiceAccountLookup, ServiceAccountTokenGetter: serviceAccountGetter, KeystoneURL: s.KeystoneURL, WebhookTokenAuthnConfigFile: s.WebhookTokenAuthnConfigFile, WebhookTokenAuthnCacheTTL: s.WebhookTokenAuthnCacheTTL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } privilegedLoopbackToken := uuid.NewRandom().String() selfClientConfig, err := s.NewSelfClientConfig(privilegedLoopbackToken) if err != nil { glog.Fatalf("Failed to create clientset: %v", err) } client, err := s.NewSelfClient(privilegedLoopbackToken) if err != nil { glog.Errorf("Failed to create clientset: %v", err) } sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute) authorizationConfig := authorizer.AuthorizationConfig{ PolicyFile: s.AuthorizationPolicyFile, WebhookConfigFile: s.AuthorizationWebhookConfigFile, WebhookCacheAuthorizedTTL: s.AuthorizationWebhookCacheAuthorizedTTL, WebhookCacheUnauthorizedTTL: s.AuthorizationWebhookCacheUnauthorizedTTL, RBACSuperUser: s.AuthorizationRBACSuperUser, InformerFactory: sharedInformers, } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") // TODO(dims): We probably need to add an option "EnableLoopbackToken" if apiAuthenticator != nil { var uid = uuid.NewRandom().String() tokens := make(map[string]*user.DefaultInfo) tokens[privilegedLoopbackToken] = &user.DefaultInfo{ Name: user.APIServerUser, UID: uid, Groups: []string{user.SystemPrivilegedGroup}, } tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens) apiAuthenticator = authenticatorunion.New(tokenAuthenticator, apiAuthenticator) tokenAuthorizer := authorizer.NewPrivilegedGroups(user.SystemPrivilegedGroup) apiAuthorizer = authorizerunion.New(tokenAuthorizer, apiAuthorizer) } pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer) admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile, pluginInitializer) if err != nil { glog.Fatalf("Failed to initialize plugins: %v", err) } kubeVersion := version.Get() genericConfig.Version = &kubeVersion genericConfig.LoopbackClientConfig = selfClientConfig genericConfig.Authenticator = apiAuthenticator genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0 genericConfig.Authorizer = apiAuthorizer genericConfig.AuthorizerRBACSuperUser = s.AuthorizationRBACSuperUser genericConfig.AdmissionControl = admissionController genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource genericConfig.MasterServiceNamespace = s.MasterServiceNamespace genericConfig.ProxyDialer = proxyDialerFn genericConfig.ProxyTLSClientConfig = proxyTLSClientConfig genericConfig.OpenAPIConfig.Info.Title = "Kubernetes" genericConfig.OpenAPIConfig.Definitions = generatedopenapi.OpenAPIDefinitions genericConfig.OpenAPIConfig.GetOperationID = openapi.GetOperationID genericConfig.EnableOpenAPISupport = true config := &master.Config{ GenericConfig: genericConfig.Config, StorageFactory: storageFactory, EnableWatchCache: s.EnableWatchCache, EnableCoreControllers: true, DeleteCollectionWorkers: s.DeleteCollectionWorkers, EventTTL: s.EventTTL, KubeletClient: kubeletClient, EnableUISupport: true, EnableLogsSupport: true, Tunneler: tunneler, } if s.EnableWatchCache { glog.V(2).Infof("Initalizing cache sizes based on %dMB limit", s.TargetRAMMB) cachesize.InitializeWatchCacheSizes(s.TargetRAMMB) cachesize.SetWatchCacheSizes(s.WatchCacheSizes) } m, err := config.Complete().New() if err != nil { return err } sharedInformers.Start(wait.NeverStop) m.GenericAPIServer.Run() return nil }