func DefaultAndValidateRunOptions(options *options.ServerRunOptions) {
genericvalidation.ValidateRunOptions(options)
// If advertise-address is not specified, use bind-address. If bind-address
// is not usable (unset, 0.0.0.0, or loopback), we will use the host's default
// interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster)
if options.AdvertiseAddress == nil || options.AdvertiseAddress.IsUnspecified() {
hostIP, err := utilnet.ChooseBindAddress(options.BindAddress)
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v' . "+
"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
}
options.AdvertiseAddress = hostIP
}
glog.Infof("Will report %v as public IP address.", options.AdvertiseAddress)
// Set default value for ExternalHost if not specified.
if len(options.ExternalHost) == 0 {
// TODO: extend for other providers
if options.CloudProvider == "gce" {
cloud, err := cloudprovider.InitCloudProvider(options.CloudProvider, options.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
instances, supported := cloud.Instances()
if !supported {
glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.")
}
hostname, err := os.Hostname()
if err != nil {
glog.Fatalf("Failed to get hostname: %v", err)
}
nodeName, err := instances.CurrentNodeName(hostname)
if err != nil {
glog.Fatalf("Failed to get NodeName: %v", err)
}
addrs, err := instances.NodeAddresses(nodeName)
if err != nil {
glog.Warningf("Unable to obtain external host address from cloud provider: %v", err)
} else {
for _, addr := range addrs {
if addr.Type == api.NodeExternalIP {
options.ExternalHost = addr.Address
}
}
}
}
}
}
func DefaultAndValidateRunOptions(options *ServerRunOptions) {
ValidateRunOptions(options)
// If advertise-address is not specified, use bind-address. If bind-address
// is not usable (unset, 0.0.0.0, or loopback), we will use the host's default
// interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster)
if options.AdvertiseAddress == nil || options.AdvertiseAddress.IsUnspecified() {
hostIP, err := utilnet.ChooseBindAddress(options.BindAddress)
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v' . "+
"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
}
options.AdvertiseAddress = hostIP
}
glog.Infof("Will report %v as public IP address.", options.AdvertiseAddress)
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.APIServer) error {
verifyClusterIPFlags(s)
// If advertise-address is not specified, use bind-address. If bind-address
// is not usable (unset, 0.0.0.0, or loopback), we will use the host's default
// interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster)
if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
hostIP, err := utilnet.ChooseBindAddress(s.BindAddress)
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v' . "+
"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
}
s.AdvertiseAddress = hostIP
}
glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress)
if len(s.EtcdServerList) == 0 {
glog.Fatalf("--etcd-servers must be specified")
}
if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) {
glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort))
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
// Setup tunneler if needed
var tunneler master.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH master.InstallSSHKey
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
// Set up the tunneler
tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig)
if err != nil {
glog.Fatalf("Failure to start kubelet client: %v", err)
}
apiGroupVersionOverrides, err := parseRuntimeConfig(s)
if err != nil {
glog.Fatalf("error in parsing runtime-config: %s", err)
}
clientConfig := &client.Config{
Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
}
if len(s.DeprecatedStorageVersion) != 0 {
gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion)
if err != nil {
glog.Fatalf("error in parsing group version: %s", err)
}
clientConfig.GroupVersion = &gv
}
client, err := client.New(clientConfig)
if err != nil {
glog.Fatalf("Invalid server address: %v", err)
}
legacyV1Group, err := registered.Group(api.GroupName)
if err != nil {
return err
}
storageDestinations := genericapiserver.NewStorageDestinations()
storageVersions := generateStorageVersionMap(s.DeprecatedStorageVersion, s.StorageVersions)
if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.GroupVersion.Group, storageVersions)
}
etcdStorage, err := newEtcd(s.EtcdServerList, api.Codecs, storageVersions[legacyV1Group.GroupVersion.Group], s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup("", etcdStorage)
if !apiGroupVersionOverrides["extensions/v1beta1"].Disable {
expGroup, err := registered.Group(extensions.GroupName)
if err != nil {
glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err)
}
if _, found := storageVersions[expGroup.GroupVersion.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.GroupVersion.Group, storageVersions)
}
expEtcdStorage, err := newEtcd(s.EtcdServerList, api.Codecs, storageVersions[expGroup.GroupVersion.Group], s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup(extensions.GroupName, expEtcdStorage)
}
updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdPathPrefix, &storageDestinations, newEtcd)
n := s.ServiceClusterIPRange
// Default to the private server key for service account token signing
if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
} else {
glog.Warning("No RSA key provided, service account token authentication disabled")
}
}
var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter
if s.ServiceAccountLookup {
// If we need to look up service accounts and tokens,
// go directly to etcd to avoid recursive auth insanity
serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(etcdStorage)
}
authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
BasicAuthFile: s.BasicAuthFile,
ClientCAFile: s.ClientCAFile,
TokenAuthFile: s.TokenAuthFile,
OIDCIssuerURL: s.OIDCIssuerURL,
OIDCClientID: s.OIDCClientID,
OIDCCAFile: s.OIDCCAFile,
OIDCUsernameClaim: s.OIDCUsernameClaim,
ServiceAccountKeyFile: s.ServiceAccountKeyFile,
ServiceAccountLookup: s.ServiceAccountLookup,
ServiceAccountTokenGetter: serviceAccountGetter,
KeystoneURL: s.KeystoneURL,
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationPolicyFile)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
if len(s.ExternalHost) == 0 {
// TODO: extend for other providers
if s.CloudProvider == "gce" {
instances, supported := cloud.Instances()
if !supported {
glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.")
}
name, err := os.Hostname()
if err != nil {
glog.Fatalf("Failed to get hostname: %v", err)
}
addrs, err := instances.NodeAddresses(name)
if err != nil {
glog.Warningf("Unable to obtain external host address from cloud provider: %v", err)
} else {
for _, addr := range addrs {
if addr.Type == api.NodeExternalIP {
s.ExternalHost = addr.Address
}
}
}
}
}
config := &master.Config{
Config: &genericapiserver.Config{
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
ServiceClusterIPRange: &n,
EnableLogsSupport: s.EnableLogsSupport,
EnableUISupport: true,
EnableSwaggerSupport: true,
EnableProfiling: s.EnableProfiling,
EnableWatchCache: s.EnableWatchCache,
EnableIndex: true,
APIPrefix: s.APIPrefix,
APIGroupPrefix: s.APIGroupPrefix,
CorsAllowedOriginList: s.CorsAllowedOriginList,
ReadWritePort: s.SecurePort,
PublicAddress: s.AdvertiseAddress,
Authenticator: authenticator,
SupportsBasicAuth: len(s.BasicAuthFile) > 0,
Authorizer: authorizer,
AdmissionControl: admissionController,
APIGroupVersionOverrides: apiGroupVersionOverrides,
MasterServiceNamespace: s.MasterServiceNamespace,
MasterCount: s.MasterCount,
ExternalHost: s.ExternalHost,
MinRequestTimeout: s.MinRequestTimeout,
ProxyDialer: proxyDialerFn,
ProxyTLSClientConfig: proxyTLSClientConfig,
ServiceNodePortRange: s.ServiceNodePortRange,
KubernetesServiceNodePort: s.KubernetesServiceNodePort,
Serializer: api.Codecs,
},
EnableCoreControllers: true,
EventTTL: s.EventTTL,
KubeletClient: kubeletClient,
Tunneler: tunneler,
}
m := master.New(config)
m.Run(s.ServerRunOptions)
return nil
}
func startComponents(firstManifestURL, secondManifestURL string) (string, string) {
// Setup
handler := delegateHandler{}
apiServer := httptest.NewServer(&handler)
cfg := etcd.Config{
Endpoints: []string{"http://127.0.0.1:4001"},
}
etcdClient, err := etcd.New(cfg)
if err != nil {
glog.Fatalf("Error creating etcd client: %v", err)
}
glog.Infof("Creating etcd client pointing to %v", cfg.Endpoints)
keysAPI := etcd.NewKeysAPI(etcdClient)
sleep := 4 * time.Second
ok := false
for i := 0; i < 3; i++ {
keys, err := keysAPI.Get(context.TODO(), "/", nil)
if err != nil {
glog.Warningf("Unable to list root etcd keys: %v", err)
if i < 2 {
time.Sleep(sleep)
sleep = sleep * sleep
}
continue
}
for _, node := range keys.Node.Nodes {
if _, err := keysAPI.Delete(context.TODO(), node.Key, &etcd.DeleteOptions{Recursive: true}); err != nil {
glog.Fatalf("Unable delete key: %v", err)
}
}
ok = true
break
}
if !ok {
glog.Fatalf("Failed to connect to etcd")
}
cl := client.NewOrDie(&client.Config{Host: apiServer.URL, ContentConfig: client.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
clientset := clientset.NewForConfigOrDie(&client.Config{Host: apiServer.URL, ContentConfig: client.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
// TODO: caesarxuchao: hacky way to specify version of Experimental client.
// We will fix this by supporting multiple group versions in Config
cl.ExtensionsClient = client.NewExtensionsOrDie(&client.Config{Host: apiServer.URL, ContentConfig: client.ContentConfig{GroupVersion: testapi.Extensions.GroupVersion()}})
// Master
host, port, err := net.SplitHostPort(strings.TrimLeft(apiServer.URL, "http://"))
if err != nil {
glog.Fatalf("Unable to parse URL '%v': %v", apiServer.URL, err)
}
portNumber, err := strconv.Atoi(port)
if err != nil {
glog.Fatalf("Nonnumeric port? %v", err)
}
publicAddress := net.ParseIP(host)
if publicAddress == nil {
glog.Fatalf("No public address for %s", host)
}
// The caller of master.New should guarantee pulicAddress is properly set
hostIP, err := utilnet.ChooseBindAddress(publicAddress)
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v' . "+
"Fail to get a valid public address for master.", err)
}
masterConfig := framework.NewMasterConfig()
masterConfig.EnableCoreControllers = true
masterConfig.EnableProfiling = true
masterConfig.ReadWritePort = portNumber
masterConfig.PublicAddress = hostIP
masterConfig.CacheTimeout = 2 * time.Second
// Create a master and install handlers into mux.
m := master.New(masterConfig)
handler.delegate = m.Handler
// Scheduler
schedulerConfigFactory := factory.NewConfigFactory(cl, api.DefaultSchedulerName)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
glog.Fatalf("Couldn't create scheduler config: %v", err)
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName})
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(cl.Events(""))
scheduler.New(schedulerConfig).Run()
// ensure the service endpoints are sync'd several times within the window that the integration tests wait
go endpointcontroller.NewEndpointController(clientset, controller.NoResyncPeriodFunc).
Run(3, util.NeverStop)
// TODO: Write an integration test for the replication controllers watch.
go replicationcontroller.NewReplicationManager(clientset, controller.NoResyncPeriodFunc, replicationcontroller.BurstReplicas).
Run(3, util.NeverStop)
nodeController := nodecontroller.NewNodeController(nil, clientset, 5*time.Minute, util.NewFakeRateLimiter(), util.NewFakeRateLimiter(),
40*time.Second, 60*time.Second, 5*time.Second, nil, false)
nodeController.Run(5 * time.Second)
cadvisorInterface := new(cadvisor.Fake)
// Kubelet (localhost)
testRootDir := integration.MakeTempDirOrDie("kubelet_integ_1.", "")
configFilePath := integration.MakeTempDirOrDie("config", testRootDir)
glog.Infof("Using %s as root dir for kubelet #1", testRootDir)
cm := cm.NewStubContainerManager()
kcfg := kubeletapp.SimpleKubelet(
clientset,
fakeDocker1,
"localhost",
testRootDir,
firstManifestURL,
"127.0.0.1",
10250, /* KubeletPort */
0, /* ReadOnlyPort */
api.NamespaceDefault,
empty_dir.ProbeVolumePlugins(),
nil,
cadvisorInterface,
configFilePath,
nil,
kubecontainer.FakeOS{},
1*time.Second, /* FileCheckFrequency */
1*time.Second, /* HTTPCheckFrequency */
10*time.Second, /* MinimumGCAge */
3*time.Second, /* NodeStatusUpdateFrequency */
10*time.Second, /* SyncFrequency */
10*time.Second, /* OutOfDiskTransitionFrequency */
40, /* MaxPods */
cm, net.ParseIP("127.0.0.1"))
kubeletapp.RunKubelet(kcfg)
// Kubelet (machine)
// Create a second kubelet so that the guestbook example's two redis slaves both
// have a place they can schedule.
testRootDir = integration.MakeTempDirOrDie("kubelet_integ_2.", "")
glog.Infof("Using %s as root dir for kubelet #2", testRootDir)
kcfg = kubeletapp.SimpleKubelet(
clientset,
fakeDocker2,
"127.0.0.1",
testRootDir,
secondManifestURL,
"127.0.0.1",
10251, /* KubeletPort */
0, /* ReadOnlyPort */
api.NamespaceDefault,
empty_dir.ProbeVolumePlugins(),
nil,
cadvisorInterface,
"",
nil,
kubecontainer.FakeOS{},
1*time.Second, /* FileCheckFrequency */
1*time.Second, /* HTTPCheckFrequency */
10*time.Second, /* MinimumGCAge */
3*time.Second, /* NodeStatusUpdateFrequency */
10*time.Second, /* SyncFrequency */
10*time.Second, /* OutOfDiskTransitionFrequency */
40, /* MaxPods */
cm,
net.ParseIP("127.0.0.1"))
kubeletapp.RunKubelet(kcfg)
return apiServer.URL, configFilePath
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.APIServer) error {
verifyClusterIPFlags(s)
// If advertise-address is not specified, use bind-address. If bind-address
// is not usable (unset, 0.0.0.0, or loopback), we will use the host's default
// interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster)
if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
hostIP, err := utilnet.ChooseBindAddress(s.BindAddress)
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v' . "+
"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
}
s.AdvertiseAddress = hostIP
}
glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress)
if len(s.EtcdConfig.ServerList) == 0 {
glog.Fatalf("--etcd-servers must be specified")
}
if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) {
glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort))
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
// Setup tunneler if needed
var tunneler master.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH master.InstallSSHKey
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
if s.KubeletConfig.Port == 0 {
glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.")
}
// Set up the tunneler
// TODO(cjcullen): If we want this to handle per-kubelet ports or other
// kubelet listen-addresses, we need to plumb through options.
healthCheckPath := &url.URL{
Scheme: "https",
Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)),
Path: "healthz",
}
tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig)
if err != nil {
glog.Fatalf("Failure to start kubelet client: %v", err)
}
apiResourceConfigSource, err := parseRuntimeConfig(s)
if err != nil {
glog.Fatalf("error in parsing runtime-config: %s", err)
}
clientConfig := &restclient.Config{
Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
// Increase QPS limits. The client is currently passed to all admission plugins,
// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
// for more details. Once #22422 is fixed, we may want to remove it.
QPS: 50,
Burst: 100,
}
if len(s.DeprecatedStorageVersion) != 0 {
gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion)
if err != nil {
glog.Fatalf("error in parsing group version: %s", err)
}
clientConfig.GroupVersion = &gv
}
client, err := clientset.NewForConfig(clientConfig)
if err != nil {
glog.Errorf("Failed to create clientset: %v", err)
}
legacyV1Group, err := registered.Group(api.GroupName)
if err != nil {
return err
}
storageDestinations := genericapiserver.NewStorageDestinations()
storageVersions := s.StorageGroupsToGroupVersions()
if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.GroupVersion.Group, storageVersions)
}
etcdStorage, err := newEtcd(api.Codecs, storageVersions[legacyV1Group.GroupVersion.Group], "/__internal", s.EtcdConfig)
if err != nil {
glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup("", etcdStorage)
if apiResourceConfigSource.AnyResourcesForVersionEnabled(extensionsapiv1beta1.SchemeGroupVersion) {
glog.Infof("Configuring extensions/v1beta1 storage destination")
expGroup, err := registered.Group(extensions.GroupName)
if err != nil {
glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err)
}
if _, found := storageVersions[expGroup.GroupVersion.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.GroupVersion.Group, storageVersions)
}
expEtcdStorage, err := newEtcd(api.Codecs, storageVersions[expGroup.GroupVersion.Group], "extensions/__internal", s.EtcdConfig)
if err != nil {
glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup(extensions.GroupName, expEtcdStorage)
// Since HPA has been moved to the autoscaling group, we need to make
// sure autoscaling has a storage destination. If the autoscaling group
// itself is on, it will overwrite this decision below.
storageDestinations.AddAPIGroup(autoscaling.GroupName, expEtcdStorage)
// Since Job has been moved to the batch group, we need to make
// sure batch has a storage destination. If the batch group
// itself is on, it will overwrite this decision below.
storageDestinations.AddAPIGroup(batch.GroupName, expEtcdStorage)
}
// autoscaling/v1/horizontalpodautoscalers is a move from extensions/v1beta1/horizontalpodautoscalers.
// The storage version needs to be either extensions/v1beta1 or autoscaling/v1.
// Users must roll forward while using 1.2, because we will require the latter for 1.3.
if apiResourceConfigSource.AnyResourcesForVersionEnabled(autoscalingapiv1.SchemeGroupVersion) {
glog.Infof("Configuring autoscaling/v1 storage destination")
autoscalingGroup, err := registered.Group(autoscaling.GroupName)
if err != nil {
glog.Fatalf("Autoscaling API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err)
}
// Figure out what storage group/version we should use.
storageGroupVersion, found := storageVersions[autoscalingGroup.GroupVersion.Group]
if !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", autoscalingGroup.GroupVersion.Group, storageVersions)
}
if storageGroupVersion != "autoscaling/v1" && storageGroupVersion != "extensions/v1beta1" {
glog.Fatalf("The storage version for autoscaling must be either 'autoscaling/v1' or 'extensions/v1beta1'")
}
glog.Infof("Using %v for autoscaling group storage version", storageGroupVersion)
autoscalingEtcdStorage, err := newEtcd(api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdConfig)
if err != nil {
glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup(autoscaling.GroupName, autoscalingEtcdStorage)
}
// batch/v1/job is a move from extensions/v1beta1/job. The storage
// version needs to be either extensions/v1beta1 or batch/v1. Users
// must roll forward while using 1.2, because we will require the
// latter for 1.3.
if apiResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv1.SchemeGroupVersion) {
glog.Infof("Configuring batch/v1 storage destination")
batchGroup, err := registered.Group(batch.GroupName)
if err != nil {
glog.Fatalf("Batch API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err)
}
// Figure out what storage group/version we should use.
storageGroupVersion, found := storageVersions[batchGroup.GroupVersion.Group]
if !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", batchGroup.GroupVersion.Group, storageVersions)
}
if storageGroupVersion != "batch/v1" && storageGroupVersion != "extensions/v1beta1" {
glog.Fatalf("The storage version for batch must be either 'batch/v1' or 'extensions/v1beta1'")
}
glog.Infof("Using %v for batch group storage version", storageGroupVersion)
batchEtcdStorage, err := newEtcd(api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdConfig)
if err != nil {
glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup(batch.GroupName, batchEtcdStorage)
}
updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdConfig, &storageDestinations, newEtcd)
n := s.ServiceClusterIPRange
// Default to the private server key for service account token signing
if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
} else {
glog.Warning("No RSA key provided, service account token authentication disabled")
}
}
var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter
if s.ServiceAccountLookup {
// If we need to look up service accounts and tokens,
// go directly to etcd to avoid recursive auth insanity
serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(etcdStorage)
}
authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
BasicAuthFile: s.BasicAuthFile,
ClientCAFile: s.ClientCAFile,
TokenAuthFile: s.TokenAuthFile,
OIDCIssuerURL: s.OIDCIssuerURL,
OIDCClientID: s.OIDCClientID,
OIDCCAFile: s.OIDCCAFile,
OIDCUsernameClaim: s.OIDCUsernameClaim,
OIDCGroupsClaim: s.OIDCGroupsClaim,
ServiceAccountKeyFile: s.ServiceAccountKeyFile,
ServiceAccountLookup: s.ServiceAccountLookup,
ServiceAccountTokenGetter: serviceAccountGetter,
KeystoneURL: s.KeystoneURL,
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationConfig)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
if len(s.ExternalHost) == 0 {
// TODO: extend for other providers
if s.CloudProvider == "gce" {
instances, supported := cloud.Instances()
if !supported {
glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.")
}
name, err := os.Hostname()
if err != nil {
glog.Fatalf("Failed to get hostname: %v", err)
}
addrs, err := instances.NodeAddresses(name)
if err != nil {
glog.Warningf("Unable to obtain external host address from cloud provider: %v", err)
} else {
for _, addr := range addrs {
if addr.Type == api.NodeExternalIP {
s.ExternalHost = addr.Address
}
}
}
}
}
config := &master.Config{
Config: &genericapiserver.Config{
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
ServiceClusterIPRange: &n,
EnableLogsSupport: s.EnableLogsSupport,
EnableUISupport: true,
EnableSwaggerSupport: true,
EnableSwaggerUI: s.EnableSwaggerUI,
EnableProfiling: s.EnableProfiling,
EnableWatchCache: s.EnableWatchCache,
EnableIndex: true,
APIPrefix: s.APIPrefix,
APIGroupPrefix: s.APIGroupPrefix,
CorsAllowedOriginList: s.CorsAllowedOriginList,
ReadWritePort: s.SecurePort,
PublicAddress: s.AdvertiseAddress,
Authenticator: authenticator,
SupportsBasicAuth: len(s.BasicAuthFile) > 0,
Authorizer: authorizer,
AdmissionControl: admissionController,
APIResourceConfigSource: apiResourceConfigSource,
MasterServiceNamespace: s.MasterServiceNamespace,
MasterCount: s.MasterCount,
ExternalHost: s.ExternalHost,
MinRequestTimeout: s.MinRequestTimeout,
ProxyDialer: proxyDialerFn,
ProxyTLSClientConfig: proxyTLSClientConfig,
ServiceNodePortRange: s.ServiceNodePortRange,
KubernetesServiceNodePort: s.KubernetesServiceNodePort,
Serializer: api.Codecs,
},
EnableCoreControllers: true,
DeleteCollectionWorkers: s.DeleteCollectionWorkers,
EventTTL: s.EventTTL,
KubeletClient: kubeletClient,
Tunneler: tunneler,
}
if s.EnableWatchCache {
cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
}
m, err := master.New(config)
if err != nil {
return err
}
m.Run(s.ServerRunOptions)
return nil
}
// Get the host's public IP address
func (lk LocalkubeServer) GetHostIP() (net.IP, error) {
return utilnet.ChooseBindAddress(net.ParseIP("0.0.0.0"))
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.APIServer) error {
verifyClusterIPFlags(s)
// If advertise-address is not specified, use bind-address. If bind-address
// is not usable (unset, 0.0.0.0, or loopback), we will use the host's default
// interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster)
if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
hostIP, err := utilnet.ChooseBindAddress(s.BindAddress)
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v' . "+
"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
}
s.AdvertiseAddress = hostIP
}
glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress)
if len(s.EtcdConfig.ServerList) == 0 {
glog.Fatalf("--etcd-servers must be specified")
}
if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) {
glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort))
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH genericapiserver.InstallSSHKey
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
if s.KubeletConfig.Port == 0 {
glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.")
}
// Set up the tunneler
// TODO(cjcullen): If we want this to handle per-kubelet ports or other
// kubelet listen-addresses, we need to plumb through options.
healthCheckPath := &url.URL{
Scheme: "https",
Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)),
Path: "healthz",
}
tunneler := genericapiserver.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
apiResourceConfigSource, err := parseRuntimeConfig(s)
if err != nil {
glog.Fatalf("error in parsing runtime-config: %s", err)
}
clientConfig := &restclient.Config{
Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
// Increase QPS limits. The client is currently passed to all admission plugins,
// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
// for more details. Once #22422 is fixed, we may want to remove it.
QPS: 50,
Burst: 100,
}
if len(s.DeprecatedStorageVersion) != 0 {
gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion)
if err != nil {
glog.Fatalf("error in parsing group version: %s", err)
}
clientConfig.GroupVersion = &gv
}
client, err := clientset.NewForConfig(clientConfig)
if err != nil {
glog.Errorf("Failed to create clientset: %v", err)
}
// TODO: register cluster federation resources here.
n := s.ServiceClusterIPRange
resourceEncoding := genericapiserver.NewDefaultResourceEncodingConfig()
groupToEncoding, err := s.StorageGroupsToEncodingVersion()
if err != nil {
glog.Fatalf("error getting group encoding: %s", err)
}
for group, storageEncodingVersion := range groupToEncoding {
resourceEncoding.SetVersionEncoding(group, storageEncodingVersion, unversioned.GroupVersion{Group: group, Version: runtime.APIVersionInternal})
}
storageFactory := genericapiserver.NewDefaultStorageFactory(s.EtcdConfig, api.Codecs, resourceEncoding, apiResourceConfigSource)
for _, override := range s.EtcdServersOverrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
continue
}
group := apiresource[0]
resource := apiresource[1]
groupResource := unversioned.GroupResource{Group: group, Resource: resource}
servers := strings.Split(tokens[1], ";")
storageFactory.SetEtcdLocation(groupResource, servers)
}
authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
BasicAuthFile: s.BasicAuthFile,
ClientCAFile: s.ClientCAFile,
TokenAuthFile: s.TokenAuthFile,
OIDCIssuerURL: s.OIDCIssuerURL,
OIDCClientID: s.OIDCClientID,
OIDCCAFile: s.OIDCCAFile,
OIDCUsernameClaim: s.OIDCUsernameClaim,
OIDCGroupsClaim: s.OIDCGroupsClaim,
KeystoneURL: s.KeystoneURL,
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationConfig)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
if len(s.ExternalHost) == 0 {
// TODO: extend for other providers
if s.CloudProvider == "gce" {
instances, supported := cloud.Instances()
if !supported {
glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.")
}
name, err := os.Hostname()
if err != nil {
glog.Fatalf("Failed to get hostname: %v", err)
}
addrs, err := instances.NodeAddresses(name)
if err != nil {
glog.Warningf("Unable to obtain external host address from cloud provider: %v", err)
} else {
for _, addr := range addrs {
if addr.Type == api.NodeExternalIP {
s.ExternalHost = addr.Address
}
}
}
}
}
config := &genericapiserver.Config{
StorageFactory: storageFactory,
ServiceClusterIPRange: &n,
EnableLogsSupport: s.EnableLogsSupport,
EnableUISupport: true,
EnableSwaggerSupport: true,
EnableSwaggerUI: s.EnableSwaggerUI,
EnableProfiling: s.EnableProfiling,
EnableWatchCache: s.EnableWatchCache,
EnableIndex: true,
APIPrefix: s.APIPrefix,
APIGroupPrefix: s.APIGroupPrefix,
CorsAllowedOriginList: s.CorsAllowedOriginList,
ReadWritePort: s.SecurePort,
PublicAddress: s.AdvertiseAddress,
Authenticator: authenticator,
SupportsBasicAuth: len(s.BasicAuthFile) > 0,
Authorizer: authorizer,
AdmissionControl: admissionController,
APIResourceConfigSource: apiResourceConfigSource,
MasterServiceNamespace: s.MasterServiceNamespace,
MasterCount: s.MasterCount,
ExternalHost: s.ExternalHost,
MinRequestTimeout: s.MinRequestTimeout,
ProxyDialer: proxyDialerFn,
ProxyTLSClientConfig: proxyTLSClientConfig,
ServiceNodePortRange: s.ServiceNodePortRange,
KubernetesServiceNodePort: s.KubernetesServiceNodePort,
Serializer: api.Codecs,
}
// TODO: Move this to generic api server (Need to move the command line flag).
if s.EnableWatchCache {
cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
}
m, err := genericapiserver.New(config)
if err != nil {
return err
}
installFederationAPIs(s, m, storageFactory)
m.Run(s.ServerRunOptions)
return nil
}
func (s *ServingOptions) DefaultExternalAddress() (net.IP, error) {
return utilnet.ChooseBindAddress(s.BindAddress)
}
