func NewClusterClientSet(c *federation_v1alpha1.Cluster) (*ClusterClient, error) {
var serverAddress string
hostIP, err := utilnet.ChooseHostInterface()
if err != nil {
return nil, err
}
for _, item := range c.Spec.ServerAddressByClientCIDRs {
_, cidrnet, err := net.ParseCIDR(item.ClientCIDR)
if err != nil {
return nil, err
}
myaddr := net.ParseIP(hostIP.String())
if cidrnet.Contains(myaddr) == true {
serverAddress = item.ServerAddress
break
}
}
var clusterClientSet = ClusterClient{}
if serverAddress != "" {
clusterConfig, err := clientcmd.BuildConfigFromFlags(serverAddress, "")
if err != nil {
return nil, err
}
clusterConfig.QPS = KubeAPIQPS
clusterConfig.Burst = KubeAPIBurst
clusterClientSet.discoveryClient = discovery.NewDiscoveryClientForConfigOrDie((restclient.AddUserAgent(clusterConfig, UserAgentName)))
if clusterClientSet.discoveryClient == nil {
return nil, nil
}
}
return &clusterClientSet, err
}
func BuildClusterConfig(c *federation_v1beta1.Cluster) (*restclient.Config, error) {
var serverAddress string
var clusterConfig *restclient.Config
hostIP, err := utilnet.ChooseHostInterface()
if err != nil {
return nil, err
}
for _, item := range c.Spec.ServerAddressByClientCIDRs {
_, cidrnet, err := net.ParseCIDR(item.ClientCIDR)
if err != nil {
return nil, err
}
myaddr := net.ParseIP(hostIP.String())
if cidrnet.Contains(myaddr) == true {
serverAddress = item.ServerAddress
break
}
}
if serverAddress != "" {
if c.Spec.SecretRef == nil {
glog.Infof("didn't find secretRef for cluster %s. Trying insecure access", c.Name)
clusterConfig, err = clientcmd.BuildConfigFromFlags(serverAddress, "")
} else {
kubeconfigGetter := KubeconfigGetterForCluster(c)
clusterConfig, err = clientcmd.BuildConfigFromKubeconfigGetter(serverAddress, kubeconfigGetter)
}
if err != nil {
return nil, err
}
clusterConfig.QPS = KubeAPIQPS
clusterConfig.Burst = KubeAPIBurst
}
return clusterConfig, nil
}
// Called by higher layers to create the plugin SDN node instance
func NewNodePlugin(pluginName string, osClient *osclient.Client, kClient *kclient.Client, hostname string, selfIP string, iptablesSyncPeriod time.Duration, mtu uint32) (*OsdnNode, error) {
if !osapi.IsOpenShiftNetworkPlugin(pluginName) {
return nil, nil
}
log.Infof("Initializing SDN node of type %q with configured hostname %q (IP %q), iptables sync period %q", pluginName, hostname, selfIP, iptablesSyncPeriod.String())
if hostname == "" {
output, err := kexec.New().Command("uname", "-n").CombinedOutput()
if err != nil {
return nil, err
}
hostname = strings.TrimSpace(string(output))
log.Infof("Resolved hostname to %q", hostname)
}
if selfIP == "" {
var err error
selfIP, err = netutils.GetNodeIP(hostname)
if err != nil {
log.V(5).Infof("Failed to determine node address from hostname %s; using default interface (%v)", hostname, err)
var defaultIP net.IP
defaultIP, err = kubeutilnet.ChooseHostInterface()
if err != nil {
return nil, err
}
selfIP = defaultIP.String()
log.Infof("Resolved IP address to %q", selfIP)
}
}
ovsif, err := ovs.New(kexec.New(), BR)
if err != nil {
return nil, err
}
plugin := &OsdnNode{
multitenant: osapi.IsOpenShiftMultitenantNetworkPlugin(pluginName),
kClient: kClient,
osClient: osClient,
ovs: ovsif,
localIP: selfIP,
hostName: hostname,
vnids: newNodeVNIDMap(),
podNetworkReady: make(chan struct{}),
kubeletInitReady: make(chan struct{}),
iptablesSyncPeriod: iptablesSyncPeriod,
mtu: mtu,
egressPolicies: make(map[uint32][]*osapi.EgressNetworkPolicy),
}
if err := plugin.dockerPreCNICleanup(); err != nil {
return nil, err
}
return plugin, nil
}
// Set IP addresses for the node.
func (kl *Kubelet) setNodeAddress(node *api.Node) error {
if kl.cloud != nil {
instances, ok := kl.cloud.Instances()
if !ok {
return fmt.Errorf("failed to get instances from cloud provider")
}
// TODO(roberthbailey): Can we do this without having credentials to talk
// to the cloud provider?
// TODO(justinsb): We can if CurrentNodeName() was actually CurrentNode() and returned an interface
// TODO: If IP addresses couldn't be fetched from the cloud provider, should kubelet fallback on the other methods for getting the IP below?
nodeAddresses, err := instances.NodeAddresses(kl.nodeName)
if err != nil {
return fmt.Errorf("failed to get node address from cloud provider: %v", err)
}
node.Status.Addresses = nodeAddresses
} else {
var ipAddr net.IP
var err error
// 1) Use nodeIP if set
// 2) If the user has specified an IP to HostnameOverride, use it
// 3) Lookup the IP from node name by DNS and use the first non-loopback ipv4 address
// 4) Try to get the IP from the network interface used as default gateway
if kl.nodeIP != nil {
ipAddr = kl.nodeIP
} else if addr := net.ParseIP(kl.hostname); addr != nil {
ipAddr = addr
} else {
var addrs []net.IP
addrs, err = net.LookupIP(node.Name)
for _, addr := range addrs {
if !addr.IsLoopback() && addr.To4() != nil {
ipAddr = addr
break
}
}
if ipAddr == nil {
ipAddr, err = utilnet.ChooseHostInterface()
}
}
if ipAddr == nil {
// We tried everything we could, but the IP address wasn't fetchable; error out
return fmt.Errorf("can't get ip address of node %s. error: %v", node.Name, err)
} else {
node.Status.Addresses = []api.NodeAddress{
{Type: api.NodeLegacyHostIP, Address: ipAddr.String()},
{Type: api.NodeInternalIP, Address: ipAddr.String()},
}
}
}
return nil
}
func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight bool) (*Init, error) {
if cfgPath != "" {
b, err := ioutil.ReadFile(cfgPath)
if err != nil {
return nil, fmt.Errorf("unable to read config from %q [%v]", cfgPath, err)
}
if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), b, cfg); err != nil {
return nil, fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err)
}
}
// Auto-detect the IP
if len(cfg.API.AdvertiseAddresses) == 0 {
// TODO(phase1+) perhaps we could actually grab eth0 and eth1
ip, err := netutil.ChooseHostInterface()
if err != nil {
return nil, err
}
cfg.API.AdvertiseAddresses = []string{ip.String()}
}
if !skipPreFlight {
fmt.Println("Running pre-flight checks")
err := preflight.RunInitMasterChecks(cfg)
if err != nil {
return nil, &preflight.PreFlightError{Msg: err.Error()}
}
} else {
fmt.Println("Skipping pre-flight checks")
}
// validate version argument
ver, err := kubeadmutil.KubernetesReleaseVersion(cfg.KubernetesVersion)
if err != nil {
if cfg.KubernetesVersion != kubeadmapiext.DefaultKubernetesVersion {
return nil, err
} else {
ver = kubeadmapiext.DefaultKubernetesFallbackVersion
}
}
cfg.KubernetesVersion = ver
fmt.Println("Using Kubernetes version:", ver)
// TODO(phase1+) create a custom flag
if cfg.CloudProvider != "" {
if cloudprovider.IsCloudProvider(cfg.CloudProvider) {
fmt.Printf("cloud provider %q initialized for the control plane. Remember to set the same cloud provider flag on the kubelet.\n", cfg.CloudProvider)
} else {
return nil, fmt.Errorf("cloud provider %q is not supported, you can use any of %v, or leave it unset.\n", cfg.CloudProvider, cloudprovider.CloudProviders())
}
}
return &Init{cfg: cfg}, nil
}
func PrepareTokenDiscovery(d *kubeadmapi.TokenDiscovery) error {
if len(d.Addresses) == 0 {
ip, err := netutil.ChooseHostInterface()
if err != nil {
return err
}
d.Addresses = []string{ip.String() + ":" + strconv.Itoa(kubeadmapiext.DefaultDiscoveryBindPort)}
}
if err := kubeadmutil.GenerateTokenIfNeeded(d); err != nil {
return fmt.Errorf("failed to generate token(s) [%v]", err)
}
return nil
}
func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight bool) (*Init, error) {
if cfgPath != "" {
b, err := ioutil.ReadFile(cfgPath)
if err != nil {
return nil, fmt.Errorf("unable to read config from %q [%v]", cfgPath, err)
}
if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), b, cfg); err != nil {
return nil, fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err)
}
}
// Auto-detect the IP
if len(cfg.API.AdvertiseAddresses) == 0 {
// TODO(phase1+) perhaps we could actually grab eth0 and eth1
ip, err := netutil.ChooseHostInterface()
if err != nil {
return nil, err
}
cfg.API.AdvertiseAddresses = []string{ip.String()}
}
if !skipPreFlight {
fmt.Println("Running pre-flight checks")
err := preflight.RunInitMasterChecks(cfg)
if err != nil {
return nil, &preflight.PreFlightError{Msg: err.Error()}
}
} else {
fmt.Println("Skipping pre-flight checks")
}
// validate version argument
ver, err := kubeadmutil.KubernetesReleaseVersion(cfg.KubernetesVersion)
if err != nil {
if cfg.KubernetesVersion != kubeadmapiext.DefaultKubernetesVersion {
return nil, err
} else {
ver = kubeadmapiext.DefaultKubernetesFallbackVersion
}
}
cfg.KubernetesVersion = ver
fmt.Println("Using Kubernetes version:", ver)
// Warn about the limitations with the current cloudprovider solution.
if cfg.CloudProvider != "" {
fmt.Println("WARNING: For cloudprovider integrations to work --cloud-provider must be set for all kubelets in the cluster.")
fmt.Println("\t(/etc/systemd/system/kubelet.service.d/10-kubeadm.conf should be edited for this purpose)")
}
return &Init{cfg: cfg}, nil
}
// NewProxier returns a new Proxier given a LoadBalancer and an address on
// which to listen. Because of the iptables logic, It is assumed that there
// is only a single Proxier active on a machine. An error will be returned if
// the proxier cannot be started due to an invalid ListenIP (loopback) or
// if iptables fails to update or acquire the initial lock. Once a proxier is
// created, it will keep iptables up to date in the background and will not
// terminate if a particular iptables call fails.
func NewProxier(loadBalancer LoadBalancer, listenIP net.IP, netsh netsh.Interface, pr utilnet.PortRange, syncPeriod, udpIdleTimeout time.Duration) (*Proxier, error) {
if listenIP.Equal(localhostIPv4) || listenIP.Equal(localhostIPv6) {
return nil, ErrProxyOnLocalhost
}
hostIP, err := utilnet.ChooseHostInterface()
if err != nil {
return nil, fmt.Errorf("failed to select a host interface: %v", err)
}
proxyPorts := newPortAllocator(pr)
glog.V(2).Infof("Setting proxy IP to %v and initializing iptables", hostIP)
return createProxier(loadBalancer, listenIP, netsh, hostIP, proxyPorts, syncPeriod, udpIdleTimeout)
}
// NewProxier returns a new Proxier given a LoadBalancer and an address on
// which to listen. Because of the iptables logic, It is assumed that there
// is only a single Proxier active on a machine. An error will be returned if
// the proxier cannot be started due to an invalid ListenIP (loopback) or
// if iptables fails to update or acquire the initial lock. Once a proxier is
// created, it will keep iptables up to date in the background and will not
// terminate if a particular iptables call fails.
func NewProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables.Interface, pr utilnet.PortRange, syncPeriod, udpIdleTimeout time.Duration, kubeClient *kubeclient.Client, withHaproxier bool) (*Proxier, error) {
if listenIP.Equal(localhostIPv4) || listenIP.Equal(localhostIPv6) {
return nil, ErrProxyOnLocalhost
}
hostIP, err := utilnet.ChooseHostInterface()
if err != nil {
return nil, fmt.Errorf("failed to select a host interface: %v", err)
}
err = setRLimit(64 * 1000)
if err != nil {
return nil, fmt.Errorf("failed to set open file handler limit: %v", err)
}
proxyPorts := newPortAllocator(pr)
glog.V(2).Infof("Setting proxy IP to %v and initializing iptables", hostIP)
return createProxier(loadBalancer, listenIP, iptables, hostIP, proxyPorts, syncPeriod, udpIdleTimeout, kubeClient, withHaproxier)
}
// Called by higher layers to create the plugin SDN node instance
func NewNodePlugin(pluginName string, osClient *osclient.Client, kClient *kclient.Client, hostname string, selfIP string, iptablesSyncPeriod time.Duration, mtu uint) (api.OsdnNodePlugin, error) {
if !IsOpenShiftNetworkPlugin(pluginName) {
return nil, nil
}
log.Infof("Initializing SDN node of type %q with configured hostname %q (IP %q), iptables sync period %q", pluginName, hostname, selfIP, iptablesSyncPeriod.String())
if hostname == "" {
output, err := kexec.New().Command("uname", "-n").CombinedOutput()
if err != nil {
return nil, err
}
hostname = strings.TrimSpace(string(output))
log.Infof("Resolved hostname to %q", hostname)
}
if selfIP == "" {
var err error
selfIP, err = netutils.GetNodeIP(hostname)
if err != nil {
log.V(5).Infof("Failed to determine node address from hostname %s; using default interface (%v)", hostname, err)
var defaultIP net.IP
defaultIP, err = kubeutilnet.ChooseHostInterface()
if err != nil {
return nil, err
}
selfIP = defaultIP.String()
log.Infof("Resolved IP address to %q", selfIP)
}
}
plugin := &OsdnNode{
multitenant: IsOpenShiftMultitenantNetworkPlugin(pluginName),
registry: newRegistry(osClient, kClient),
localIP: selfIP,
hostName: hostname,
vnids: newVnidMap(),
podNetworkReady: make(chan struct{}),
iptablesSyncPeriod: iptablesSyncPeriod,
mtu: mtu,
}
return plugin, nil
}
// Called by plug factory functions to initialize the generic plugin instance
func (oc *OsdnController) BaseInit(registry *Registry, pluginHooks PluginHooks, multitenant bool, hostname string, selfIP string, iptablesSyncPeriod time.Duration) error {
log.Infof("Starting with configured hostname %q (IP %q), iptables sync period %q", hostname, selfIP, iptablesSyncPeriod.String())
if hostname == "" {
output, err := kexec.New().Command("uname", "-n").CombinedOutput()
if err != nil {
return err
}
hostname = strings.TrimSpace(string(output))
}
if selfIP == "" {
var err error
selfIP, err = netutils.GetNodeIP(hostname)
if err != nil {
log.V(5).Infof("Failed to determine node address from hostname %s; using default interface (%v)", hostname, err)
defaultIP, err := kubeutilnet.ChooseHostInterface()
if err != nil {
return err
}
selfIP = defaultIP.String()
}
}
if multitenant {
log.Infof("Initializing multi-tenant plugin for %s (%s)", hostname, selfIP)
} else {
log.Infof("Initializing single-tenant plugin for %s (%s)", hostname, selfIP)
}
oc.pluginHooks = pluginHooks
oc.Registry = registry
oc.localIP = selfIP
oc.HostName = hostname
oc.vnidMap = make(map[string]uint)
oc.podNetworkReady = make(chan struct{})
oc.adminNamespaces = make([]string, 0)
oc.iptablesSyncPeriod = iptablesSyncPeriod
return nil
}
// Called by plug factory functions to initialize the generic plugin instance
func (oc *OvsController) BaseInit(registry *Registry, flowController FlowController, pluginHooks PluginHooks, hostname string, selfIP string) error {
if hostname == "" {
output, err := kexec.New().Command("uname", "-n").CombinedOutput()
if err != nil {
return err
}
hostname = strings.TrimSpace(string(output))
}
if selfIP == "" {
var err error
selfIP, err = netutils.GetNodeIP(hostname)
if err != nil {
log.V(5).Infof("Failed to determine node address from hostname %s; using default interface (%v)", hostname, err)
defaultIP, err := kubeutilnet.ChooseHostInterface()
if err != nil {
return err
}
selfIP = defaultIP.String()
}
}
log.Infof("Self IP: %s.", selfIP)
oc.pluginHooks = pluginHooks
oc.Registry = registry
oc.flowController = flowController
oc.localIP = selfIP
oc.hostName = hostname
oc.VNIDMap = make(map[string]uint)
oc.sig = make(chan struct{})
oc.podNetworkReady = make(chan struct{})
oc.adminNamespaces = make([]string, 0)
oc.services = make(map[string]api.Service)
return nil
}
func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight bool) (*Init, error) {
fmt.Println("[kubeadm] WARNING: kubeadm is in alpha, please do not use it for production clusters.")
if cfgPath != "" {
b, err := ioutil.ReadFile(cfgPath)
if err != nil {
return nil, fmt.Errorf("unable to read config from %q [%v]", cfgPath, err)
}
if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), b, cfg); err != nil {
return nil, fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err)
}
}
// Auto-detect the IP
if len(cfg.API.AdvertiseAddresses) == 0 {
ip, err := netutil.ChooseHostInterface()
if err != nil {
return nil, err
}
cfg.API.AdvertiseAddresses = []string{ip.String()}
}
if !skipPreFlight {
fmt.Println("[preflight] Running pre-flight checks")
// First, check if we're root separately from the other preflight checks and fail fast
if err := preflight.RunRootCheckOnly(); err != nil {
return nil, err
}
// Then continue with the others...
if err := preflight.RunInitMasterChecks(cfg); err != nil {
return nil, err
}
} else {
fmt.Println("[preflight] Skipping pre-flight checks")
}
// Try to start the kubelet service in case it's inactive
preflight.TryStartKubelet()
// validate version argument
ver, err := kubeadmutil.KubernetesReleaseVersion(cfg.KubernetesVersion)
if err != nil {
if cfg.KubernetesVersion != kubeadmapiext.DefaultKubernetesVersion {
return nil, err
} else {
ver = kubeadmapiext.DefaultKubernetesFallbackVersion
}
}
cfg.KubernetesVersion = ver
fmt.Println("[init] Using Kubernetes version:", ver)
// Warn about the limitations with the current cloudprovider solution.
if cfg.CloudProvider != "" {
fmt.Println("WARNING: For cloudprovider integrations to work --cloud-provider must be set for all kubelets in the cluster.")
fmt.Println("\t(/etc/systemd/system/kubelet.service.d/10-kubeadm.conf should be edited for this purpose)")
}
return &Init{cfg: cfg}, nil
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, pluginInitializer oadmission.PluginInitializer) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
// Connect and setup etcd interfaces
etcdClient, err := etcd.MakeNewEtcdClient(options.EtcdClientInfo)
if err != nil {
return nil, err
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, err
}
portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, err
}
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
// Defaults are tested in TestAPIServerDefaults
server := apiserveroptions.NewAPIServer()
// Adjust defaults
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.AdmissionControl = strings.Join(AdmissionPlugins, ",")
server.EnableLogsSupport = false // don't expose server logs
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
if len(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride) > 0 {
server.AdmissionControl = strings.Join(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride, ",")
}
// Defaults are tested in TestCMServerDefaults
cmserver := cmapp.NewCMServer()
// Adjust defaults
cmserver.Address = "" // no healthz endpoint
cmserver.Port = 0 // no healthz endpoint
cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout}
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile)
}
plugins := []admission.Interface{}
for _, pluginName := range strings.Split(server.AdmissionControl, ",") {
switch pluginName {
case serviceadmit.ExternalIPPluginName:
// this needs to be moved upstream to be part of core config
reject, admit, err := serviceadmit.ParseCIDRRules(options.NetworkConfig.ExternalIPNetworkCIDRs)
if err != nil {
// should have been caught with validation
return nil, err
}
plugins = append(plugins, serviceadmit.NewExternalIPRanger(reject, admit))
case saadmit.PluginName:
// we need to set some custom parameters on the service account admission controller, so create that one by hand
saAdmitter := saadmit.NewServiceAccount(internalclientset.FromUnversionedClient(kubeClient))
saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences
saAdmitter.Run()
plugins = append(plugins, saAdmitter)
default:
configFile, err := pluginconfig.GetPluginConfigFile(options.KubernetesMasterConfig.AdmissionConfig.PluginConfig, pluginName, server.AdmissionControlConfigFile)
if err != nil {
return nil, err
}
plugin := admission.InitPlugin(pluginName, internalclientset.FromUnversionedClient(kubeClient), configFile)
if plugin != nil {
plugins = append(plugins, plugin)
}
}
}
pluginInitializer.Initialize(plugins)
// ensure that plugins have been properly initialized
if err := oadmission.Validate(plugins); err != nil {
return nil, err
}
admissionController := admission.NewChainHandler(plugins...)
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
// TODO you have to know every APIGroup you're enabling or upstream will panic. It's alternative to panicing is Fataling
// It needs a refactor to return errors
storageDestinations := genericapiserver.NewStorageDestinations()
// storageVersions is a map from API group to allowed versions that must be a version exposed by the REST API or it breaks.
// We need to fix the upstream to stop using the storage version as a preferred api version.
storageVersions := map[string]string{}
enabledKubeVersions := configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupKube)
if len(enabledKubeVersions) > 0 {
kubeStorageVersion := unversioned.GroupVersion{Group: configapi.APIGroupKube, Version: options.EtcdStorageConfig.KubernetesStorageVersion}
databaseStorage, err := NewEtcdStorage(etcdClient, kubeStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupKube, databaseStorage)
storageVersions[configapi.APIGroupKube] = options.EtcdStorageConfig.KubernetesStorageVersion
}
// enable this if extensions API is enabled (or batch or autoscaling, since they persist to extensions/v1beta1 for now)
// TODO: replace this with a loop over configured storage versions
extensionsEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupExtensions)) > 0
batchEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupBatch)) > 0
autoscalingEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupAutoscaling)) > 0
if extensionsEnabled || autoscalingEnabled || batchEnabled {
// TODO: replace this with a configured storage version for extensions once configuration exposes this
extensionsStorageVersion := unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"}
databaseStorage, err := NewEtcdStorage(etcdClient, extensionsStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes extensions server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupExtensions, databaseStorage)
storageVersions[configapi.APIGroupExtensions] = extensionsStorageVersion.String()
}
// Preserve previous behavior of using the first non-loopback address
// TODO: Deprecate this behavior and just require a valid value to be passed in
publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP)
if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() {
hostIP, err := knet.ChooseHostInterface()
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err)
}
publicAddress = hostIP
glog.Infof("Will report %v as public IP address.", publicAddress)
}
m := &master.Config{
Config: &genericapiserver.Config{
PublicAddress: publicAddress,
ReadWritePort: port,
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionController,
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
APIGroupVersionOverrides: getAPIGroupVersionOverrides(options),
APIPrefix: KubeAPIPrefix,
APIGroupPrefix: KubeAPIGroupPrefix,
MasterCount: options.KubernetesMasterConfig.MasterCount,
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
Serializer: kapi.Codecs,
},
EventTTL: server.EventTTL,
//MinRequestTimeout: server.MinRequestTimeout,
KubeletClient: kubeletClient,
EnableCoreControllers: true,
}
if options.DNSConfig != nil {
_, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress)
if err != nil {
return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err)
}
dnsPort, err := strconv.Atoi(dnsPortStr)
if err != nil {
return nil, fmt.Errorf("invalid DNS port: %v", err)
}
m.ExtraServicePorts = append(m.ExtraServicePorts,
kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)},
kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)},
)
m.ExtraEndpointPorts = append(m.ExtraEndpointPorts,
kapi.EndpointPort{Name: "dns", Port: dnsPort, Protocol: kapi.ProtocolUDP},
kapi.EndpointPort{Name: "dns-tcp", Port: dnsPort, Protocol: kapi.ProtocolTCP},
)
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
}
return kmaster, nil
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, informers shared.InformerFactory, admissionControl admission.Interface, originAuthenticator authenticator.Request) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
// Defaults are tested in TestCMServerDefaults
cmserver := cmapp.NewCMServer()
// Adjust defaults
cmserver.Address = "" // no healthz endpoint
cmserver.Port = 0 // no healthz endpoint
cmserver.EnableGarbageCollector = false // disabled until we add the controller
cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout}
cmserver.VolumeConfiguration.EnableDynamicProvisioning = options.VolumeConfig.DynamicProvisioningEnabled
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
schedulerserver := scheduleroptions.NewSchedulerServer()
schedulerserver.PolicyConfigFile = options.KubernetesMasterConfig.SchedulerConfigFile
if err := cmdflags.Resolve(options.KubernetesMasterConfig.SchedulerArguments, schedulerserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", cmserver.CloudProvider, cmserver.CloudConfigFile)
}
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
server, storageFactory, err := BuildDefaultAPIServer(options)
if err != nil {
return nil, err
}
// Preserve previous behavior of using the first non-loopback address
// TODO: Deprecate this behavior and just require a valid value to be passed in
publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP)
if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() {
hostIP, err := knet.ChooseHostInterface()
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err)
}
publicAddress = hostIP
glog.Infof("Will report %v as public IP address.", publicAddress)
}
m := &master.Config{
Config: &genericapiserver.Config{
PublicAddress: publicAddress,
ReadWritePort: server.SecurePort,
Authenticator: originAuthenticator, // this is used to fulfill the tokenreviews endpoint which is used by node authentication
Authorizer: authorizer.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionControl,
StorageFactory: storageFactory,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
APIResourceConfigSource: getAPIResourceConfig(options),
APIPrefix: server.APIPrefix,
APIGroupPrefix: server.APIGroupPrefix,
MasterCount: server.MasterCount,
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
Serializer: kapi.Codecs,
EnableLogsSupport: server.EnableLogsSupport,
EnableProfiling: server.EnableProfiling,
EnableWatchCache: server.EnableWatchCache,
MasterServiceNamespace: server.MasterServiceNamespace,
ExternalHost: server.ExternalHost,
MinRequestTimeout: server.MinRequestTimeout,
KubernetesServiceNodePort: server.KubernetesServiceNodePort,
},
EventTTL: server.EventTTL,
KubeletClient: kubeletClient,
EnableCoreControllers: true,
DeleteCollectionWorkers: server.DeleteCollectionWorkers,
}
if server.EnableWatchCache {
cachesize.SetWatchCacheSizes(server.WatchCacheSizes)
}
if options.DNSConfig != nil {
_, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress)
if err != nil {
return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err)
}
dnsPort, err := strconv.Atoi(dnsPortStr)
if err != nil {
return nil, fmt.Errorf("invalid DNS port: %v", err)
}
m.ExtraServicePorts = append(m.ExtraServicePorts,
kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)},
kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)},
)
m.ExtraEndpointPorts = append(m.ExtraEndpointPorts,
kapi.EndpointPort{Name: "dns", Port: int32(dnsPort), Protocol: kapi.ProtocolUDP},
kapi.EndpointPort{Name: "dns-tcp", Port: int32(dnsPort), Protocol: kapi.ProtocolTCP},
)
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
SchedulerServer: schedulerserver,
Informers: informers,
}
return kmaster, nil
}
// Set IP and hostname addresses for the node.
func (kl *Kubelet) setNodeAddress(node *v1.Node) error {
if kl.nodeIP != nil {
if err := kl.validateNodeIP(); err != nil {
return fmt.Errorf("failed to validate nodeIP: %v", err)
}
glog.V(2).Infof("Using node IP: %q", kl.nodeIP.String())
}
if kl.cloud != nil {
instances, ok := kl.cloud.Instances()
if !ok {
return fmt.Errorf("failed to get instances from cloud provider")
}
// TODO(roberthbailey): Can we do this without having credentials to talk
// to the cloud provider?
// TODO(justinsb): We can if CurrentNodeName() was actually CurrentNode() and returned an interface
// TODO: If IP addresses couldn't be fetched from the cloud provider, should kubelet fallback on the other methods for getting the IP below?
nodeAddresses, err := instances.NodeAddresses(kl.nodeName)
if err != nil {
return fmt.Errorf("failed to get node address from cloud provider: %v", err)
}
if kl.nodeIP != nil {
for _, nodeAddress := range nodeAddresses {
if nodeAddress.Address == kl.nodeIP.String() {
node.Status.Addresses = []v1.NodeAddress{
{Type: nodeAddress.Type, Address: nodeAddress.Address},
{Type: v1.NodeHostName, Address: kl.GetHostname()},
}
return nil
}
}
return fmt.Errorf("failed to get node address from cloud provider that matches ip: %v", kl.nodeIP)
}
// Only add a NodeHostName address if the cloudprovider did not specify one
// (we assume the cloudprovider knows best)
var addressNodeHostName *v1.NodeAddress
for i := range nodeAddresses {
if nodeAddresses[i].Type == v1.NodeHostName {
addressNodeHostName = &nodeAddresses[i]
break
}
}
if addressNodeHostName == nil {
hostnameAddress := v1.NodeAddress{Type: v1.NodeHostName, Address: kl.GetHostname()}
nodeAddresses = append(nodeAddresses, hostnameAddress)
} else {
glog.V(2).Infof("Using Node Hostname from cloudprovider: %q", addressNodeHostName.Address)
}
node.Status.Addresses = nodeAddresses
} else {
var ipAddr net.IP
var err error
// 1) Use nodeIP if set
// 2) If the user has specified an IP to HostnameOverride, use it
// 3) Lookup the IP from node name by DNS and use the first non-loopback ipv4 address
// 4) Try to get the IP from the network interface used as default gateway
if kl.nodeIP != nil {
ipAddr = kl.nodeIP
} else if addr := net.ParseIP(kl.hostname); addr != nil {
ipAddr = addr
} else {
var addrs []net.IP
addrs, err = net.LookupIP(node.Name)
for _, addr := range addrs {
if !addr.IsLoopback() && addr.To4() != nil {
ipAddr = addr
break
}
}
if ipAddr == nil {
ipAddr, err = utilnet.ChooseHostInterface()
}
}
if ipAddr == nil {
// We tried everything we could, but the IP address wasn't fetchable; error out
return fmt.Errorf("can't get ip address of node %s. error: %v", node.Name, err)
} else {
node.Status.Addresses = []v1.NodeAddress{
{Type: v1.NodeLegacyHostIP, Address: ipAddr.String()},
{Type: v1.NodeInternalIP, Address: ipAddr.String()},
{Type: v1.NodeHostName, Address: kl.GetHostname()},
}
}
}
return nil
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, informers shared.InformerFactory, pluginInitializer oadmission.PluginInitializer) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, err
}
portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, err
}
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
// Defaults are tested in TestAPIServerDefaults
server := apiserveroptions.NewAPIServer()
// Adjust defaults
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.AdmissionControl = strings.Join(AdmissionPlugins, ",")
server.EnableLogsSupport = false // don't expose server logs
server.EnableProfiling = false
server.APIPrefix = KubeAPIPrefix
server.APIGroupPrefix = KubeAPIGroupPrefix
server.SecurePort = port
server.MasterCount = options.KubernetesMasterConfig.MasterCount
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
if len(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride) > 0 {
server.AdmissionControl = strings.Join(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride, ",")
}
// Defaults are tested in TestCMServerDefaults
cmserver := cmapp.NewCMServer()
// Adjust defaults
cmserver.Address = "" // no healthz endpoint
cmserver.Port = 0 // no healthz endpoint
cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout}
cmserver.VolumeConfiguration.EnableDynamicProvisioning = options.VolumeConfig.DynamicProvisioningEnabled
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile)
}
plugins := []admission.Interface{}
for _, pluginName := range strings.Split(server.AdmissionControl, ",") {
switch pluginName {
case lifecycle.PluginName:
// We need to include our infrastructure and shared resource namespaces in the immortal namespaces list
immortalNamespaces := sets.NewString(kapi.NamespaceDefault)
if len(options.PolicyConfig.OpenShiftSharedResourcesNamespace) > 0 {
immortalNamespaces.Insert(options.PolicyConfig.OpenShiftSharedResourcesNamespace)
}
if len(options.PolicyConfig.OpenShiftInfrastructureNamespace) > 0 {
immortalNamespaces.Insert(options.PolicyConfig.OpenShiftInfrastructureNamespace)
}
plugins = append(plugins, lifecycle.NewLifecycle(clientadapter.FromUnversionedClient(kubeClient), immortalNamespaces))
case serviceadmit.ExternalIPPluginName:
// this needs to be moved upstream to be part of core config
reject, admit, err := serviceadmit.ParseCIDRRules(options.NetworkConfig.ExternalIPNetworkCIDRs)
if err != nil {
// should have been caught with validation
return nil, err
}
plugins = append(plugins, serviceadmit.NewExternalIPRanger(reject, admit))
case saadmit.PluginName:
// we need to set some custom parameters on the service account admission controller, so create that one by hand
saAdmitter := saadmit.NewServiceAccount(clientadapter.FromUnversionedClient(kubeClient))
saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences
saAdmitter.Run()
plugins = append(plugins, saAdmitter)
default:
configFile, err := pluginconfig.GetPluginConfigFile(options.KubernetesMasterConfig.AdmissionConfig.PluginConfig, pluginName, server.AdmissionControlConfigFile)
if err != nil {
return nil, err
}
plugin := admission.InitPlugin(pluginName, clientadapter.FromUnversionedClient(kubeClient), configFile)
if plugin != nil {
plugins = append(plugins, plugin)
}
}
}
pluginInitializer.Initialize(plugins)
// ensure that plugins have been properly initialized
if err := oadmission.Validate(plugins); err != nil {
return nil, err
}
admissionController := admission.NewChainHandler(plugins...)
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
resourceEncodingConfig := genericapiserver.NewDefaultResourceEncodingConfig()
resourceEncodingConfig.SetVersionEncoding(
kapi.GroupName,
unversioned.GroupVersion{Group: kapi.GroupName, Version: options.EtcdStorageConfig.KubernetesStorageVersion},
kapi.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
extensions.GroupName,
unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"},
extensions.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
batch.GroupName,
unversioned.GroupVersion{Group: batch.GroupName, Version: "v1"},
batch.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
autoscaling.GroupName,
unversioned.GroupVersion{Group: autoscaling.GroupName, Version: "v1"},
autoscaling.SchemeGroupVersion,
)
etcdConfig := storagebackend.Config{
Prefix: options.EtcdStorageConfig.KubernetesStoragePrefix,
ServerList: options.EtcdClientInfo.URLs,
KeyFile: options.EtcdClientInfo.ClientCert.KeyFile,
CertFile: options.EtcdClientInfo.ClientCert.CertFile,
CAFile: options.EtcdClientInfo.CA,
DeserializationCacheSize: genericapiserveroptions.DefaultDeserializationCacheSize,
}
storageFactory := genericapiserver.NewDefaultStorageFactory(etcdConfig, "", kapi.Codecs, resourceEncodingConfig, master.DefaultAPIResourceConfigSource())
// the order here is important, it defines which version will be used for storage
storageFactory.AddCohabitatingResources(extensions.Resource("jobs"), batch.Resource("jobs"))
storageFactory.AddCohabitatingResources(extensions.Resource("horizontalpodautoscalers"), autoscaling.Resource("horizontalpodautoscalers"))
// Preserve previous behavior of using the first non-loopback address
// TODO: Deprecate this behavior and just require a valid value to be passed in
publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP)
if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() {
hostIP, err := knet.ChooseHostInterface()
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err)
}
publicAddress = hostIP
glog.Infof("Will report %v as public IP address.", publicAddress)
}
m := &master.Config{
Config: &genericapiserver.Config{
PublicAddress: publicAddress,
ReadWritePort: port,
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionController,
StorageFactory: storageFactory,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
APIResourceConfigSource: getAPIResourceConfig(options),
APIPrefix: server.APIPrefix,
APIGroupPrefix: server.APIGroupPrefix,
MasterCount: server.MasterCount,
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
Serializer: kapi.Codecs,
EnableLogsSupport: server.EnableLogsSupport,
EnableProfiling: server.EnableProfiling,
EnableWatchCache: server.EnableWatchCache,
MasterServiceNamespace: server.MasterServiceNamespace,
ExternalHost: server.ExternalHost,
MinRequestTimeout: server.MinRequestTimeout,
KubernetesServiceNodePort: server.KubernetesServiceNodePort,
},
EventTTL: server.EventTTL,
KubeletClient: kubeletClient,
EnableCoreControllers: true,
DeleteCollectionWorkers: server.DeleteCollectionWorkers,
}
if server.EnableWatchCache {
cachesize.SetWatchCacheSizes(server.WatchCacheSizes)
}
if options.DNSConfig != nil {
_, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress)
if err != nil {
return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err)
}
dnsPort, err := strconv.Atoi(dnsPortStr)
if err != nil {
return nil, fmt.Errorf("invalid DNS port: %v", err)
}
m.ExtraServicePorts = append(m.ExtraServicePorts,
kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)},
kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)},
)
m.ExtraEndpointPorts = append(m.ExtraEndpointPorts,
kapi.EndpointPort{Name: "dns", Port: int32(dnsPort), Protocol: kapi.ProtocolUDP},
kapi.EndpointPort{Name: "dns-tcp", Port: int32(dnsPort), Protocol: kapi.ProtocolTCP},
)
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
Informers: informers,
}
return kmaster, nil
}
// Set addresses for the node.
func (kl *Kubelet) setNodeAddress(node *api.Node) error {
// Set addresses for the node.
if kl.cloud != nil {
instances, ok := kl.cloud.Instances()
if !ok {
return fmt.Errorf("failed to get instances from cloud provider")
}
// TODO(roberthbailey): Can we do this without having credentials to talk
// to the cloud provider?
// TODO(justinsb): We can if CurrentNodeName() was actually CurrentNode() and returned an interface
nodeAddresses, err := instances.NodeAddresses(kl.nodeName)
if err != nil {
return fmt.Errorf("failed to get node address from cloud provider: %v", err)
}
node.Status.Addresses = nodeAddresses
} else {
if kl.nodeIP != nil {
node.Status.Addresses = []api.NodeAddress{
{Type: api.NodeLegacyHostIP, Address: kl.nodeIP.String()},
{Type: api.NodeInternalIP, Address: kl.nodeIP.String()},
}
} else if addr := net.ParseIP(kl.hostname); addr != nil {
node.Status.Addresses = []api.NodeAddress{
{Type: api.NodeLegacyHostIP, Address: addr.String()},
{Type: api.NodeInternalIP, Address: addr.String()},
}
} else {
addrs, err := net.LookupIP(node.Name)
if err != nil {
return fmt.Errorf("can't get ip address of node %s: %v", node.Name, err)
} else if len(addrs) == 0 {
return fmt.Errorf("no ip address for node %v", node.Name)
} else {
// check all ip addresses for this node.Name and try to find the first non-loopback IPv4 address.
// If no match is found, it uses the IP of the interface with gateway on it.
for _, ip := range addrs {
if ip.IsLoopback() {
continue
}
if ip.To4() != nil {
node.Status.Addresses = []api.NodeAddress{
{Type: api.NodeLegacyHostIP, Address: ip.String()},
{Type: api.NodeInternalIP, Address: ip.String()},
}
break
}
}
if len(node.Status.Addresses) == 0 {
ip, err := utilnet.ChooseHostInterface()
if err != nil {
return err
}
node.Status.Addresses = []api.NodeAddress{
{Type: api.NodeLegacyHostIP, Address: ip.String()},
{Type: api.NodeInternalIP, Address: ip.String()},
}
}
}
}
}
return nil
}
// RunInit executes master node provisioning, including certificates, needed static pod manifests, etc.
func RunInit(out io.Writer, cmd *cobra.Command, args []string, cfg *kubeadmapi.MasterConfiguration) error {
// Auto-detect the IP
if len(cfg.API.AdvertiseAddresses) == 0 {
// TODO(phase1+) perhaps we could actually grab eth0 and eth1
ip, err := netutil.ChooseHostInterface()
if err != nil {
return err
}
cfg.API.AdvertiseAddresses = []string{ip.String()}
}
// TODO(phase1+) create a custom flag
if cfg.CloudProvider != "" {
if cloudprovider.IsCloudProvider(cfg.CloudProvider) {
fmt.Printf("<cmd/init> cloud provider %q initialized for the control plane. Remember to set the same cloud provider flag on the kubelet.\n", cfg.CloudProvider)
} else {
return fmt.Errorf("<cmd/init> cloud provider %q is not supported, you can use any of %v, or leave it unset.\n", cfg.CloudProvider, cloudprovider.CloudProviders())
}
}
if err := kubemaster.CreateTokenAuthFile(&cfg.Secrets); err != nil {
return err
}
if err := kubemaster.WriteStaticPodManifests(cfg); err != nil {
return err
}
caKey, caCert, err := kubemaster.CreatePKIAssets(cfg)
if err != nil {
return err
}
kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(cfg.API.AdvertiseAddresses, []string{"kubelet", "admin"}, caKey, caCert)
if err != nil {
return err
}
// kubeadm is responsible for writing the following kubeconfig file, which
// kubelet should be waiting for. Help user avoid foot-shooting by refusing to
// write a file that has already been written (the kubelet will be up and
// running in that case - they'd need to stop the kubelet, remove the file, and
// start it again in that case).
// TODO(phase1+) this is no longer the right place to guard agains foo-shooting,
// we need to decide how to handle existing files (it may be handy to support
// importing existing files, may be we could even make our command idempotant,
// or at least allow for external PKI and stuff)
for name, kubeconfig := range kubeconfigs {
if err := kubeadmutil.WriteKubeconfigIfNotExists(name, kubeconfig); err != nil {
return err
}
}
client, err := kubemaster.CreateClientAndWaitForAPI(kubeconfigs["admin"])
if err != nil {
return err
}
schedulePodsOnMaster := false
if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client, schedulePodsOnMaster); err != nil {
return err
}
if err := kubemaster.CreateDiscoveryDeploymentAndSecret(cfg, client, caCert); err != nil {
return err
}
if err := kubemaster.CreateEssentialAddons(cfg, client); err != nil {
return err
}
// TODO(phase1+) use templates to reference struct fields directly as order of args is fragile
fmt.Fprintf(out, initDoneMsgf,
cfg.Secrets.GivenToken,
cfg.API.AdvertiseAddresses[0],
)
return nil
}
