// Returns a basic master config. func NewMasterConfig() *master.Config { etcdClient := NewEtcdClient() storageVersions := make(map[string]string) etcdStorage := etcdstorage.NewEtcdStorage(etcdClient, testapi.Default.Codec(), etcdtest.PathPrefix(), false) storageVersions[api.GroupName] = testapi.Default.GroupVersion().String() autoscalingEtcdStorage := NewAutoscalingEtcdStorage(etcdClient) storageVersions[autoscaling.GroupName] = testapi.Autoscaling.GroupVersion().String() batchEtcdStorage := NewBatchEtcdStorage(etcdClient) storageVersions[batch.GroupName] = testapi.Batch.GroupVersion().String() expEtcdStorage := NewExtensionsEtcdStorage(etcdClient) storageVersions[extensions.GroupName] = testapi.Extensions.GroupVersion().String() storageDestinations := genericapiserver.NewStorageDestinations() storageDestinations.AddAPIGroup(api.GroupName, etcdStorage) storageDestinations.AddAPIGroup(autoscaling.GroupName, autoscalingEtcdStorage) storageDestinations.AddAPIGroup(batch.GroupName, batchEtcdStorage) storageDestinations.AddAPIGroup(extensions.GroupName, expEtcdStorage) return &master.Config{ Config: &genericapiserver.Config{ StorageDestinations: storageDestinations, StorageVersions: storageVersions, APIPrefix: "/api", APIGroupPrefix: "/apis", Authorizer: apiserver.NewAlwaysAllowAuthorizer(), AdmissionControl: admit.NewAlwaysAdmit(), Serializer: api.Codecs, }, KubeletClient: kubeletclient.FakeKubeletClient{}, } }
// setUp is a convience function for setting up for (most) tests. func setUp(t *testing.T) (Master, *etcdtesting.EtcdTestServer, Config, *assert.Assertions) { server := etcdtesting.NewEtcdTestClientServer(t) master := Master{ GenericAPIServer: &genericapiserver.GenericAPIServer{}, } config := Config{ Config: &genericapiserver.Config{}, } storageVersions := make(map[string]string) storageDestinations := genericapiserver.NewStorageDestinations() storageDestinations.AddAPIGroup( api.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix(), false)) storageDestinations.AddAPIGroup( extensions.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Extensions.Codec(), etcdtest.PathPrefix(), false)) config.StorageDestinations = storageDestinations storageVersions[api.GroupName] = testapi.Default.GroupVersion().String() storageVersions[extensions.GroupName] = testapi.Extensions.GroupVersion().String() config.StorageVersions = storageVersions config.PublicAddress = net.ParseIP("192.168.10.4") master.nodeRegistry = registrytest.NewNodeRegistry([]string{"node1", "node2"}, api.NodeResources{}) return master, server, config, assert.New(t) }
func TestUpdateEtcdOverrides(t *testing.T) { storageVersions := map[string]string{ "": "v1", "extensions": "extensions/v1beta1", } testCases := []struct { apigroup string resource string servers []string }{ { apigroup: api.GroupName, resource: "resource", servers: []string{"http://127.0.0.1:10000"}, }, { apigroup: api.GroupName, resource: "resource", servers: []string{"http://127.0.0.1:10000", "http://127.0.0.1:20000"}, }, { apigroup: extensions.GroupName, resource: "resource", servers: []string{"http://127.0.0.1:10000"}, }, } for _, test := range testCases { newEtcd := func(_ runtime.NegotiatedSerializer, _, _ string, etcdConfig etcdstorage.EtcdConfig) (storage.Interface, error) { if !reflect.DeepEqual(test.servers, etcdConfig.ServerList) { t.Errorf("unexpected server list, expected: %#v, got: %#v", test.servers, etcdConfig.ServerList) } return nil, nil } storageDestinations := genericapiserver.NewStorageDestinations() override := test.apigroup + "/" + test.resource + "#" + strings.Join(test.servers, ";") defaultEtcdConfig := etcdstorage.EtcdConfig{ Prefix: genericapiserver.DefaultEtcdPathPrefix, ServerList: []string{"http://127.0.0.1"}, } updateEtcdOverrides([]string{override}, storageVersions, defaultEtcdConfig, &storageDestinations, newEtcd) apigroup, ok := storageDestinations.APIGroups[test.apigroup] if !ok { t.Errorf("apigroup: %s not created", test.apigroup) continue } if apigroup.Overrides == nil { t.Errorf("Overrides not created for: %s", test.apigroup) continue } if _, ok := apigroup.Overrides[test.resource]; !ok { t.Errorf("override not created for: %s", test.resource) continue } } }
func newStorageDestinations(groupName string, groupMeta *apimachinery.GroupMeta) (*genericapiserver.StorageDestinations, error) { storageDestinations := genericapiserver.NewStorageDestinations() var storageConfig etcdstorage.EtcdConfig storageConfig.ServerList = []string{"http://127.0.0.1:4001"} storageConfig.Prefix = genericapiserver.DefaultEtcdPathPrefix storageConfig.Codec = groupMeta.Codec storageInterface, err := storageConfig.NewStorage() if err != nil { return nil, err } storageDestinations.AddAPIGroup(groupName, storageInterface) return &storageDestinations, nil }
func TestUpdateEtcdOverrides(t *testing.T) { storageVersions := generateStorageVersionMap("", "v1,extensions/v1beta1") testCases := []struct { apigroup string resource string servers []string }{ { apigroup: api.GroupName, resource: "resource", servers: []string{"http://127.0.0.1:10000"}, }, { apigroup: api.GroupName, resource: "resource", servers: []string{"http://127.0.0.1:10000", "http://127.0.0.1:20000"}, }, { apigroup: extensions.GroupName, resource: "resource", servers: []string{"http://127.0.0.1:10000"}, }, } for _, test := range testCases { newEtcd := func(serverList []string, _ meta.VersionInterfacesFunc, _, _ string) (storage.Interface, error) { if !reflect.DeepEqual(test.servers, serverList) { t.Errorf("unexpected server list, expected: %#v, got: %#v", test.servers, serverList) } return nil, nil } storageDestinations := genericapiserver.NewStorageDestinations() override := test.apigroup + "/" + test.resource + "#" + strings.Join(test.servers, ";") updateEtcdOverrides([]string{override}, storageVersions, "", &storageDestinations, newEtcd) apigroup, ok := storageDestinations.APIGroups[test.apigroup] if !ok { t.Errorf("apigroup: %s not created", test.apigroup) continue } if apigroup.Overrides == nil { t.Errorf("Overrides not created for: %s", test.apigroup) continue } if _, ok := apigroup.Overrides[test.resource]; !ok { t.Errorf("override not created for: %s", test.resource) continue } } }
// setUp is a convience function for setting up for (most) tests. func setUp(t *testing.T) (*Master, *etcdtesting.EtcdTestServer, Config, *assert.Assertions) { server := etcdtesting.NewEtcdTestClientServer(t) master := &Master{ GenericAPIServer: &genericapiserver.GenericAPIServer{}, } config := Config{ Config: &genericapiserver.Config{}, } storageVersions := make(map[string]string) storageDestinations := genericapiserver.NewStorageDestinations() storageDestinations.AddAPIGroup( api.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)) storageDestinations.AddAPIGroup( autoscaling.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Autoscaling.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)) storageDestinations.AddAPIGroup( batch.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Batch.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)) storageDestinations.AddAPIGroup( extensions.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Extensions.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)) config.StorageDestinations = storageDestinations storageVersions[api.GroupName] = testapi.Default.GroupVersion().String() storageVersions[autoscaling.GroupName] = testapi.Autoscaling.GroupVersion().String() storageVersions[batch.GroupName] = testapi.Batch.GroupVersion().String() storageVersions[extensions.GroupName] = testapi.Extensions.GroupVersion().String() config.StorageVersions = storageVersions config.PublicAddress = net.ParseIP("192.168.10.4") config.Serializer = api.Codecs config.KubeletClient = client.FakeKubeletClient{} config.APIPrefix = "/api" config.APIGroupPrefix = "/apis" config.APIResourceConfigSource = DefaultAPIResourceConfigSource() config.ProxyDialer = func(network, addr string) (net.Conn, error) { return nil, nil } config.ProxyTLSClientConfig = &tls.Config{} // TODO: this is kind of hacky. The trouble is that the sync loop // runs in a go-routine and there is no way to validate in the test // that the sync routine has actually run. The right answer here // is probably to add some sort of callback that we can register // to validate that it's actually been run, but for now we don't // run the sync routine and register types manually. config.disableThirdPartyControllerForTesting = true master.nodeRegistry = registrytest.NewNodeRegistry([]string{"node1", "node2"}, api.NodeResources{}) return master, server, config, assert.New(t) }
// Run runs the specified APIServer. This should never exit. func Run(s *options.APIServer) error { verifyClusterIPFlags(s) // If advertise-address is not specified, use bind-address. If bind-address // is not usable (unset, 0.0.0.0, or loopback), we will use the host's default // interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster) if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() { hostIP, err := utilnet.ChooseBindAddress(s.BindAddress) if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v' . "+ "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) } s.AdvertiseAddress = hostIP } glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress) if len(s.EtcdServerList) == 0 { glog.Fatalf("--etcd-servers must be specified") } if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) { glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort)) } capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: s.AllowPrivileged, // TODO(vmarmol): Implement support for HostNetworkSources. PrivilegedSources: capabilities.PrivilegedSources{ HostNetworkSources: []string{}, HostPIDSources: []string{}, HostIPCSources: []string{}, }, PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, }) cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } // Setup tunneler if needed var tunneler master.Tunneler var proxyDialerFn apiserver.ProxyDialerFunc if len(s.SSHUser) > 0 { // Get ssh key distribution func, if supported var installSSH master.InstallSSHKey if cloud != nil { if instances, supported := cloud.Instances(); supported { installSSH = instances.AddSSHKeyToAllInstances } } // Set up the tunneler tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, installSSH) // Use the tunneler's dialer to connect to the kubelet s.KubeletConfig.Dial = tunneler.Dial // Use the tunneler's dialer when proxying to pods, services, and nodes proxyDialerFn = tunneler.Dial } // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig) if err != nil { glog.Fatalf("Failure to start kubelet client: %v", err) } apiGroupVersionOverrides, err := parseRuntimeConfig(s) if err != nil { glog.Fatalf("error in parsing runtime-config: %s", err) } clientConfig := &client.Config{ Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)), } if len(s.DeprecatedStorageVersion) != 0 { gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion) if err != nil { glog.Fatalf("error in parsing group version: %s", err) } clientConfig.GroupVersion = &gv } client, err := client.New(clientConfig) if err != nil { glog.Fatalf("Invalid server address: %v", err) } legacyV1Group, err := registered.Group(api.GroupName) if err != nil { return err } storageDestinations := genericapiserver.NewStorageDestinations() storageVersions := generateStorageVersionMap(s.DeprecatedStorageVersion, s.StorageVersions) if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.GroupVersion.Group, storageVersions) } etcdStorage, err := newEtcd(s.EtcdServerList, api.Codecs, storageVersions[legacyV1Group.GroupVersion.Group], s.EtcdPathPrefix) if err != nil { glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup("", etcdStorage) if !apiGroupVersionOverrides["extensions/v1beta1"].Disable { expGroup, err := registered.Group(extensions.GroupName) if err != nil { glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) } if _, found := storageVersions[expGroup.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.GroupVersion.Group, storageVersions) } expEtcdStorage, err := newEtcd(s.EtcdServerList, api.Codecs, storageVersions[expGroup.GroupVersion.Group], s.EtcdPathPrefix) if err != nil { glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup(extensions.GroupName, expEtcdStorage) } updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdPathPrefix, &storageDestinations, newEtcd) n := s.ServiceClusterIPRange // Default to the private server key for service account token signing if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" { if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) { s.ServiceAccountKeyFile = s.TLSPrivateKeyFile } else { glog.Warning("No RSA key provided, service account token authentication disabled") } } var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter if s.ServiceAccountLookup { // If we need to look up service accounts and tokens, // go directly to etcd to avoid recursive auth insanity serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(etcdStorage) } authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{ BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, ServiceAccountKeyFile: s.ServiceAccountKeyFile, ServiceAccountLookup: s.ServiceAccountLookup, ServiceAccountTokenGetter: serviceAccountGetter, KeystoneURL: s.KeystoneURL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationPolicyFile) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile) if len(s.ExternalHost) == 0 { // TODO: extend for other providers if s.CloudProvider == "gce" { instances, supported := cloud.Instances() if !supported { glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.") } name, err := os.Hostname() if err != nil { glog.Fatalf("Failed to get hostname: %v", err) } addrs, err := instances.NodeAddresses(name) if err != nil { glog.Warningf("Unable to obtain external host address from cloud provider: %v", err) } else { for _, addr := range addrs { if addr.Type == api.NodeExternalIP { s.ExternalHost = addr.Address } } } } } config := &master.Config{ Config: &genericapiserver.Config{ StorageDestinations: storageDestinations, StorageVersions: storageVersions, ServiceClusterIPRange: &n, EnableLogsSupport: s.EnableLogsSupport, EnableUISupport: true, EnableSwaggerSupport: true, EnableProfiling: s.EnableProfiling, EnableWatchCache: s.EnableWatchCache, EnableIndex: true, APIPrefix: s.APIPrefix, APIGroupPrefix: s.APIGroupPrefix, CorsAllowedOriginList: s.CorsAllowedOriginList, ReadWritePort: s.SecurePort, PublicAddress: s.AdvertiseAddress, Authenticator: authenticator, SupportsBasicAuth: len(s.BasicAuthFile) > 0, Authorizer: authorizer, AdmissionControl: admissionController, APIGroupVersionOverrides: apiGroupVersionOverrides, MasterServiceNamespace: s.MasterServiceNamespace, MasterCount: s.MasterCount, ExternalHost: s.ExternalHost, MinRequestTimeout: s.MinRequestTimeout, ProxyDialer: proxyDialerFn, ProxyTLSClientConfig: proxyTLSClientConfig, ServiceNodePortRange: s.ServiceNodePortRange, KubernetesServiceNodePort: s.KubernetesServiceNodePort, Serializer: api.Codecs, }, EnableCoreControllers: true, EventTTL: s.EventTTL, KubeletClient: kubeletClient, Tunneler: tunneler, } m := master.New(config) m.Run(s.ServerRunOptions) return nil }
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, pluginInitializer oadmission.PluginInitializer) (*MasterConfig, error) { if options.KubernetesMasterConfig == nil { return nil, errors.New("insufficient information to build KubernetesMasterConfig") } // Connect and setup etcd interfaces etcdClient, err := etcd.MakeNewEtcdClient(options.EtcdClientInfo) if err != nil { return nil, err } kubeletClientConfig := configapi.GetKubeletClientConfig(options) kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig) if err != nil { return nil, fmt.Errorf("unable to configure Kubelet client: %v", err) } // in-order list of plug-ins that should intercept admission decisions // TODO: Push node environment support to upstream in future _, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress) if err != nil { return nil, err } port, err := strconv.Atoi(portString) if err != nil { return nil, err } portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange) if err != nil { return nil, err } podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout) if err != nil { return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err) } // Defaults are tested in TestAPIServerDefaults server := apiserveroptions.NewAPIServer() // Adjust defaults server.EventTTL = 2 * time.Hour server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet)) server.ServiceNodePortRange = *portRange server.AdmissionControl = strings.Join(AdmissionPlugins, ",") server.EnableLogsSupport = false // don't expose server logs // resolve extended arguments // TODO: this should be done in config validation (along with the above) so we can provide // proper errors if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 { return nil, kerrors.NewAggregate(err) } if len(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride) > 0 { server.AdmissionControl = strings.Join(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride, ",") } // Defaults are tested in TestCMServerDefaults cmserver := cmapp.NewCMServer() // Adjust defaults cmserver.Address = "" // no healthz endpoint cmserver.Port = 0 // no healthz endpoint cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout} // resolve extended arguments // TODO: this should be done in config validation (along with the above) so we can provide // proper errors if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 { return nil, kerrors.NewAggregate(err) } cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile) if err != nil { return nil, err } if cloud != nil { glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile) } plugins := []admission.Interface{} for _, pluginName := range strings.Split(server.AdmissionControl, ",") { switch pluginName { case serviceadmit.ExternalIPPluginName: // this needs to be moved upstream to be part of core config reject, admit, err := serviceadmit.ParseCIDRRules(options.NetworkConfig.ExternalIPNetworkCIDRs) if err != nil { // should have been caught with validation return nil, err } plugins = append(plugins, serviceadmit.NewExternalIPRanger(reject, admit)) case saadmit.PluginName: // we need to set some custom parameters on the service account admission controller, so create that one by hand saAdmitter := saadmit.NewServiceAccount(internalclientset.FromUnversionedClient(kubeClient)) saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences saAdmitter.Run() plugins = append(plugins, saAdmitter) default: configFile, err := pluginconfig.GetPluginConfigFile(options.KubernetesMasterConfig.AdmissionConfig.PluginConfig, pluginName, server.AdmissionControlConfigFile) if err != nil { return nil, err } plugin := admission.InitPlugin(pluginName, internalclientset.FromUnversionedClient(kubeClient), configFile) if plugin != nil { plugins = append(plugins, plugin) } } } pluginInitializer.Initialize(plugins) // ensure that plugins have been properly initialized if err := oadmission.Validate(plugins); err != nil { return nil, err } admissionController := admission.NewChainHandler(plugins...) var proxyClientCerts []tls.Certificate if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 { clientCert, err := tls.LoadX509KeyPair( options.KubernetesMasterConfig.ProxyClientInfo.CertFile, options.KubernetesMasterConfig.ProxyClientInfo.KeyFile, ) if err != nil { return nil, err } proxyClientCerts = append(proxyClientCerts, clientCert) } // TODO you have to know every APIGroup you're enabling or upstream will panic. It's alternative to panicing is Fataling // It needs a refactor to return errors storageDestinations := genericapiserver.NewStorageDestinations() // storageVersions is a map from API group to allowed versions that must be a version exposed by the REST API or it breaks. // We need to fix the upstream to stop using the storage version as a preferred api version. storageVersions := map[string]string{} enabledKubeVersions := configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupKube) if len(enabledKubeVersions) > 0 { kubeStorageVersion := unversioned.GroupVersion{Group: configapi.APIGroupKube, Version: options.EtcdStorageConfig.KubernetesStorageVersion} databaseStorage, err := NewEtcdStorage(etcdClient, kubeStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix) if err != nil { return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err) } storageDestinations.AddAPIGroup(configapi.APIGroupKube, databaseStorage) storageVersions[configapi.APIGroupKube] = options.EtcdStorageConfig.KubernetesStorageVersion } // enable this if extensions API is enabled (or batch or autoscaling, since they persist to extensions/v1beta1 for now) // TODO: replace this with a loop over configured storage versions extensionsEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupExtensions)) > 0 batchEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupBatch)) > 0 autoscalingEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupAutoscaling)) > 0 if extensionsEnabled || autoscalingEnabled || batchEnabled { // TODO: replace this with a configured storage version for extensions once configuration exposes this extensionsStorageVersion := unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"} databaseStorage, err := NewEtcdStorage(etcdClient, extensionsStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix) if err != nil { return nil, fmt.Errorf("Error setting up Kubernetes extensions server storage: %v", err) } storageDestinations.AddAPIGroup(configapi.APIGroupExtensions, databaseStorage) storageVersions[configapi.APIGroupExtensions] = extensionsStorageVersion.String() } // Preserve previous behavior of using the first non-loopback address // TODO: Deprecate this behavior and just require a valid value to be passed in publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP) if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() { hostIP, err := knet.ChooseHostInterface() if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err) } publicAddress = hostIP glog.Infof("Will report %v as public IP address.", publicAddress) } m := &master.Config{ Config: &genericapiserver.Config{ PublicAddress: publicAddress, ReadWritePort: port, Authorizer: apiserver.NewAlwaysAllowAuthorizer(), AdmissionControl: admissionController, StorageDestinations: storageDestinations, StorageVersions: storageVersions, ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange), ServiceNodePortRange: server.ServiceNodePortRange, RequestContextMapper: requestContextMapper, APIGroupVersionOverrides: getAPIGroupVersionOverrides(options), APIPrefix: KubeAPIPrefix, APIGroupPrefix: KubeAPIGroupPrefix, MasterCount: options.KubernetesMasterConfig.MasterCount, // Set the TLS options for proxying to pods and services // Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname) ProxyTLSClientConfig: &tls.Config{ // Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs InsecureSkipVerify: true, Certificates: proxyClientCerts, }, Serializer: kapi.Codecs, }, EventTTL: server.EventTTL, //MinRequestTimeout: server.MinRequestTimeout, KubeletClient: kubeletClient, EnableCoreControllers: true, } if options.DNSConfig != nil { _, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress) if err != nil { return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err) } dnsPort, err := strconv.Atoi(dnsPortStr) if err != nil { return nil, fmt.Errorf("invalid DNS port: %v", err) } m.ExtraServicePorts = append(m.ExtraServicePorts, kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)}, kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)}, ) m.ExtraEndpointPorts = append(m.ExtraEndpointPorts, kapi.EndpointPort{Name: "dns", Port: dnsPort, Protocol: kapi.ProtocolUDP}, kapi.EndpointPort{Name: "dns-tcp", Port: dnsPort, Protocol: kapi.ProtocolTCP}, ) } kmaster := &MasterConfig{ Options: *options.KubernetesMasterConfig, KubeClient: kubeClient, Master: m, ControllerManager: cmserver, CloudProvider: cloud, } return kmaster, nil }
// Run runs the specified APIServer. This should never exit. func Run(s *options.APIServer) error { verifyClusterIPFlags(s) // If advertise-address is not specified, use bind-address. If bind-address // is not usable (unset, 0.0.0.0, or loopback), we will use the host's default // interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster) if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() { hostIP, err := utilnet.ChooseBindAddress(s.BindAddress) if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v' . "+ "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) } s.AdvertiseAddress = hostIP } glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress) if len(s.EtcdConfig.ServerList) == 0 { glog.Fatalf("--etcd-servers must be specified") } if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) { glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort)) } capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: s.AllowPrivileged, // TODO(vmarmol): Implement support for HostNetworkSources. PrivilegedSources: capabilities.PrivilegedSources{ HostNetworkSources: []string{}, HostPIDSources: []string{}, HostIPCSources: []string{}, }, PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, }) cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } // Setup tunneler if needed var tunneler master.Tunneler var proxyDialerFn apiserver.ProxyDialerFunc if len(s.SSHUser) > 0 { // Get ssh key distribution func, if supported var installSSH master.InstallSSHKey if cloud != nil { if instances, supported := cloud.Instances(); supported { installSSH = instances.AddSSHKeyToAllInstances } } if s.KubeletConfig.Port == 0 { glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.") } // Set up the tunneler // TODO(cjcullen): If we want this to handle per-kubelet ports or other // kubelet listen-addresses, we need to plumb through options. healthCheckPath := &url.URL{ Scheme: "https", Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)), Path: "healthz", } tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH) // Use the tunneler's dialer to connect to the kubelet s.KubeletConfig.Dial = tunneler.Dial // Use the tunneler's dialer when proxying to pods, services, and nodes proxyDialerFn = tunneler.Dial } // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig) if err != nil { glog.Fatalf("Failure to start kubelet client: %v", err) } apiResourceConfigSource, err := parseRuntimeConfig(s) if err != nil { glog.Fatalf("error in parsing runtime-config: %s", err) } clientConfig := &restclient.Config{ Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)), // Increase QPS limits. The client is currently passed to all admission plugins, // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 // for more details. Once #22422 is fixed, we may want to remove it. QPS: 50, Burst: 100, } if len(s.DeprecatedStorageVersion) != 0 { gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion) if err != nil { glog.Fatalf("error in parsing group version: %s", err) } clientConfig.GroupVersion = &gv } client, err := clientset.NewForConfig(clientConfig) if err != nil { glog.Errorf("Failed to create clientset: %v", err) } legacyV1Group, err := registered.Group(api.GroupName) if err != nil { return err } storageDestinations := genericapiserver.NewStorageDestinations() storageVersions := s.StorageGroupsToGroupVersions() if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.GroupVersion.Group, storageVersions) } etcdStorage, err := newEtcd(api.Codecs, storageVersions[legacyV1Group.GroupVersion.Group], "/__internal", s.EtcdConfig) if err != nil { glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup("", etcdStorage) if apiResourceConfigSource.AnyResourcesForVersionEnabled(extensionsapiv1beta1.SchemeGroupVersion) { glog.Infof("Configuring extensions/v1beta1 storage destination") expGroup, err := registered.Group(extensions.GroupName) if err != nil { glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) } if _, found := storageVersions[expGroup.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.GroupVersion.Group, storageVersions) } expEtcdStorage, err := newEtcd(api.Codecs, storageVersions[expGroup.GroupVersion.Group], "extensions/__internal", s.EtcdConfig) if err != nil { glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup(extensions.GroupName, expEtcdStorage) // Since HPA has been moved to the autoscaling group, we need to make // sure autoscaling has a storage destination. If the autoscaling group // itself is on, it will overwrite this decision below. storageDestinations.AddAPIGroup(autoscaling.GroupName, expEtcdStorage) // Since Job has been moved to the batch group, we need to make // sure batch has a storage destination. If the batch group // itself is on, it will overwrite this decision below. storageDestinations.AddAPIGroup(batch.GroupName, expEtcdStorage) } // autoscaling/v1/horizontalpodautoscalers is a move from extensions/v1beta1/horizontalpodautoscalers. // The storage version needs to be either extensions/v1beta1 or autoscaling/v1. // Users must roll forward while using 1.2, because we will require the latter for 1.3. if apiResourceConfigSource.AnyResourcesForVersionEnabled(autoscalingapiv1.SchemeGroupVersion) { glog.Infof("Configuring autoscaling/v1 storage destination") autoscalingGroup, err := registered.Group(autoscaling.GroupName) if err != nil { glog.Fatalf("Autoscaling API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) } // Figure out what storage group/version we should use. storageGroupVersion, found := storageVersions[autoscalingGroup.GroupVersion.Group] if !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", autoscalingGroup.GroupVersion.Group, storageVersions) } if storageGroupVersion != "autoscaling/v1" && storageGroupVersion != "extensions/v1beta1" { glog.Fatalf("The storage version for autoscaling must be either 'autoscaling/v1' or 'extensions/v1beta1'") } glog.Infof("Using %v for autoscaling group storage version", storageGroupVersion) autoscalingEtcdStorage, err := newEtcd(api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdConfig) if err != nil { glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup(autoscaling.GroupName, autoscalingEtcdStorage) } // batch/v1/job is a move from extensions/v1beta1/job. The storage // version needs to be either extensions/v1beta1 or batch/v1. Users // must roll forward while using 1.2, because we will require the // latter for 1.3. if apiResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv1.SchemeGroupVersion) { glog.Infof("Configuring batch/v1 storage destination") batchGroup, err := registered.Group(batch.GroupName) if err != nil { glog.Fatalf("Batch API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) } // Figure out what storage group/version we should use. storageGroupVersion, found := storageVersions[batchGroup.GroupVersion.Group] if !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", batchGroup.GroupVersion.Group, storageVersions) } if storageGroupVersion != "batch/v1" && storageGroupVersion != "extensions/v1beta1" { glog.Fatalf("The storage version for batch must be either 'batch/v1' or 'extensions/v1beta1'") } glog.Infof("Using %v for batch group storage version", storageGroupVersion) batchEtcdStorage, err := newEtcd(api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdConfig) if err != nil { glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup(batch.GroupName, batchEtcdStorage) } updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdConfig, &storageDestinations, newEtcd) n := s.ServiceClusterIPRange // Default to the private server key for service account token signing if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" { if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) { s.ServiceAccountKeyFile = s.TLSPrivateKeyFile } else { glog.Warning("No RSA key provided, service account token authentication disabled") } } var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter if s.ServiceAccountLookup { // If we need to look up service accounts and tokens, // go directly to etcd to avoid recursive auth insanity serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(etcdStorage) } authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{ BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, OIDCGroupsClaim: s.OIDCGroupsClaim, ServiceAccountKeyFile: s.ServiceAccountKeyFile, ServiceAccountLookup: s.ServiceAccountLookup, ServiceAccountTokenGetter: serviceAccountGetter, KeystoneURL: s.KeystoneURL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationConfig) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile) if len(s.ExternalHost) == 0 { // TODO: extend for other providers if s.CloudProvider == "gce" { instances, supported := cloud.Instances() if !supported { glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.") } name, err := os.Hostname() if err != nil { glog.Fatalf("Failed to get hostname: %v", err) } addrs, err := instances.NodeAddresses(name) if err != nil { glog.Warningf("Unable to obtain external host address from cloud provider: %v", err) } else { for _, addr := range addrs { if addr.Type == api.NodeExternalIP { s.ExternalHost = addr.Address } } } } } config := &master.Config{ Config: &genericapiserver.Config{ StorageDestinations: storageDestinations, StorageVersions: storageVersions, ServiceClusterIPRange: &n, EnableLogsSupport: s.EnableLogsSupport, EnableUISupport: true, EnableSwaggerSupport: true, EnableSwaggerUI: s.EnableSwaggerUI, EnableProfiling: s.EnableProfiling, EnableWatchCache: s.EnableWatchCache, EnableIndex: true, APIPrefix: s.APIPrefix, APIGroupPrefix: s.APIGroupPrefix, CorsAllowedOriginList: s.CorsAllowedOriginList, ReadWritePort: s.SecurePort, PublicAddress: s.AdvertiseAddress, Authenticator: authenticator, SupportsBasicAuth: len(s.BasicAuthFile) > 0, Authorizer: authorizer, AdmissionControl: admissionController, APIResourceConfigSource: apiResourceConfigSource, MasterServiceNamespace: s.MasterServiceNamespace, MasterCount: s.MasterCount, ExternalHost: s.ExternalHost, MinRequestTimeout: s.MinRequestTimeout, ProxyDialer: proxyDialerFn, ProxyTLSClientConfig: proxyTLSClientConfig, ServiceNodePortRange: s.ServiceNodePortRange, KubernetesServiceNodePort: s.KubernetesServiceNodePort, Serializer: api.Codecs, }, EnableCoreControllers: true, DeleteCollectionWorkers: s.DeleteCollectionWorkers, EventTTL: s.EventTTL, KubeletClient: kubeletClient, Tunneler: tunneler, } if s.EnableWatchCache { cachesize.SetWatchCacheSizes(s.WatchCacheSizes) } m, err := master.New(config) if err != nil { return err } m.Run(s.ServerRunOptions) return nil }
// Run runs the specified APIServer. This should never exit. func Run(s *options.APIServer) error { verifyClusterIPFlags(s) // If advertise-address is not specified, use bind-address. If bind-address // is not usable (unset, 0.0.0.0, or loopback), we will use the host's default // interface as valid public addr for master (see: util#ValidPublicAddrForMaster) if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() { hostIP, err := util.ValidPublicAddrForMaster(s.BindAddress) if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v' . "+ "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) } s.AdvertiseAddress = hostIP } glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress) if len(s.EtcdServerList) == 0 { glog.Fatalf("--etcd-servers must be specified") } if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) { glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort)) } capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: s.AllowPrivileged, // TODO(vmarmol): Implement support for HostNetworkSources. PrivilegedSources: capabilities.PrivilegedSources{ HostNetworkSources: []string{}, HostPIDSources: []string{}, HostIPCSources: []string{}, }, PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, }) cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } // Setup tunneler if needed var tunneler master.Tunneler var proxyDialerFn apiserver.ProxyDialerFunc if len(s.SSHUser) > 0 { // Get ssh key distribution func, if supported var installSSH master.InstallSSHKey if cloud != nil { if instances, supported := cloud.Instances(); supported { installSSH = instances.AddSSHKeyToAllInstances } } // Set up the tunneler tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, installSSH) // Use the tunneler's dialer to connect to the kubelet s.KubeletConfig.Dial = tunneler.Dial // Use the tunneler's dialer when proxying to pods, services, and nodes proxyDialerFn = tunneler.Dial } // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig) if err != nil { glog.Fatalf("Failure to start kubelet client: %v", err) } apiGroupVersionOverrides, err := parseRuntimeConfig(s) if err != nil { glog.Fatalf("error in parsing runtime-config: %s", err) } clientConfig := &client.Config{ Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)), } if len(s.DeprecatedStorageVersion) != 0 { gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion) if err != nil { glog.Fatalf("error in parsing group version: %s", err) } clientConfig.GroupVersion = &gv } client, err := client.New(clientConfig) if err != nil { glog.Fatalf("Invalid server address: %v", err) } legacyV1Group, err := latest.Group(api.GroupName) if err != nil { return err } storageDestinations := genericapiserver.NewStorageDestinations() storageVersions := generateStorageVersionMap(s.DeprecatedStorageVersion, s.StorageVersions) if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.GroupVersion.Group, storageVersions) } etcdStorage, err := newEtcd(s.EtcdServerList, legacyV1Group.InterfacesFor, storageVersions[legacyV1Group.GroupVersion.Group], s.EtcdPathPrefix) if err != nil { glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup("", etcdStorage) if !apiGroupVersionOverrides["extensions/v1beta1"].Disable { expGroup, err := latest.Group(extensions.GroupName) if err != nil { glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) } if _, found := storageVersions[expGroup.GroupVersion.Group]; !found { glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.GroupVersion.Group, storageVersions) } expEtcdStorage, err := newEtcd(s.EtcdServerList, expGroup.InterfacesFor, storageVersions[expGroup.GroupVersion.Group], s.EtcdPathPrefix) if err != nil { glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) } storageDestinations.AddAPIGroup(extensions.GroupName, expEtcdStorage) } updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdPathPrefix, &storageDestinations, newEtcd) n := s.ServiceClusterIPRange // Default to the private server key for service account token signing if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" { if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) { s.ServiceAccountKeyFile = s.TLSPrivateKeyFile } else { glog.Warning("No RSA key provided, service account token authentication disabled") } } var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter if s.ServiceAccountLookup { // If we need to look up service accounts and tokens, // go directly to etcd to avoid recursive auth insanity serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(etcdStorage) } authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{ BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, ServiceAccountKeyFile: s.ServiceAccountKeyFile, ServiceAccountLookup: s.ServiceAccountLookup, ServiceAccountTokenGetter: serviceAccountGetter, KeystoneURL: s.KeystoneURL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationPolicyFile) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile) if len(s.ExternalHost) == 0 { // TODO: extend for other providers if s.CloudProvider == "gce" { instances, supported := cloud.Instances() if !supported { glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.") } name, err := os.Hostname() if err != nil { glog.Fatalf("Failed to get hostname: %v", err) } addrs, err := instances.NodeAddresses(name) if err != nil { glog.Warningf("Unable to obtain external host address from cloud provider: %v", err) } else { for _, addr := range addrs { if addr.Type == api.NodeExternalIP { s.ExternalHost = addr.Address } } } } } config := &master.Config{ Config: &genericapiserver.Config{ StorageDestinations: storageDestinations, StorageVersions: storageVersions, ServiceClusterIPRange: &n, EnableLogsSupport: s.EnableLogsSupport, EnableUISupport: true, EnableSwaggerSupport: true, EnableProfiling: s.EnableProfiling, EnableWatchCache: s.EnableWatchCache, EnableIndex: true, APIPrefix: s.APIPrefix, APIGroupPrefix: s.APIGroupPrefix, CorsAllowedOriginList: s.CorsAllowedOriginList, ReadWritePort: s.SecurePort, PublicAddress: s.AdvertiseAddress, Authenticator: authenticator, SupportsBasicAuth: len(s.BasicAuthFile) > 0, Authorizer: authorizer, AdmissionControl: admissionController, APIGroupVersionOverrides: apiGroupVersionOverrides, MasterServiceNamespace: s.MasterServiceNamespace, MasterCount: s.MasterCount, ExternalHost: s.ExternalHost, MinRequestTimeout: s.MinRequestTimeout, ProxyDialer: proxyDialerFn, ProxyTLSClientConfig: proxyTLSClientConfig, ServiceNodePortRange: s.ServiceNodePortRange, KubernetesServiceNodePort: s.KubernetesServiceNodePort, }, EnableCoreControllers: true, EventTTL: s.EventTTL, KubeletClient: kubeletClient, Tunneler: tunneler, } m := master.New(config) // We serve on 2 ports. See docs/accessing_the_api.md secureLocation := "" if s.SecurePort != 0 { secureLocation = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort)) } insecureLocation := net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)) // See the flag commentary to understand our assumptions when opening the read-only and read-write ports. var sem chan bool if s.MaxRequestsInFlight > 0 { sem = make(chan bool, s.MaxRequestsInFlight) } longRunningRE := regexp.MustCompile(s.LongRunningRequestRE) longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) { // TODO unify this with apiserver.MaxInFlightLimit if longRunningRE.MatchString(req.URL.Path) || req.URL.Query().Get("watch") == "true" { return nil, "" } return time.After(time.Minute), "" } if secureLocation != "" { handler := apiserver.TimeoutHandler(m.Handler, longRunningTimeout) secureServer := &http.Server{ Addr: secureLocation, Handler: apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(handler)), MaxHeaderBytes: 1 << 20, TLSConfig: &tls.Config{ // Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability) MinVersion: tls.VersionTLS10, }, } if len(s.ClientCAFile) > 0 { clientCAs, err := util.CertPoolFromFile(s.ClientCAFile) if err != nil { glog.Fatalf("Unable to load client CA file: %v", err) } // Populate PeerCertificates in requests, but don't reject connections without certificates // This allows certificates to be validated by authenticators, while still allowing other auth types secureServer.TLSConfig.ClientAuth = tls.RequestClientCert // Specify allowed CAs for client certificates secureServer.TLSConfig.ClientCAs = clientCAs } glog.Infof("Serving securely on %s", secureLocation) if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" { s.TLSCertFile = path.Join(s.CertDirectory, "apiserver.crt") s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "apiserver.key") // TODO (cjcullen): Is PublicAddress the right address to sign a cert with? alternateIPs := []net.IP{config.ServiceReadWriteIP} alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"} // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil { glog.Errorf("Unable to generate self signed cert: %v", err) } else { glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile) } } go func() { defer util.HandleCrash() for { // err == systemd.SdNotifyNoSocket when not running on a systemd system if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) } if err := secureServer.ListenAndServeTLS(s.TLSCertFile, s.TLSPrivateKeyFile); err != nil { glog.Errorf("Unable to listen for secure (%v); will try again.", err) } time.Sleep(15 * time.Second) } }() } handler := apiserver.TimeoutHandler(m.InsecureHandler, longRunningTimeout) http := &http.Server{ Addr: insecureLocation, Handler: apiserver.RecoverPanics(handler), MaxHeaderBytes: 1 << 20, } if secureLocation == "" { // err == systemd.SdNotifyNoSocket when not running on a systemd system if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) } } glog.Infof("Serving insecurely on %s", insecureLocation) glog.Fatal(http.ListenAndServe()) return nil }