func StartNode(nodeConfig configapi.NodeConfig) error {
config, err := kubernetes.BuildKubernetesNodeConfig(nodeConfig)
if err != nil {
return err
}
glog.Infof("Starting node %s (%s)", config.KubeletServer.HostnameOverride, version.Get().String())
// preconditions
config.EnsureVolumeDir()
config.EnsureDocker(docker.NewHelper())
// async starts
config.RunKubelet()
config.RunSDN()
config.RunProxy()
// HACK: RunProxy resets bridge-nf-call-iptables from what openshift-sdn requires
if config.SDNPlugin != nil {
sdnPluginName := nodeConfig.NetworkConfig.NetworkPluginName
if sdnPluginName == osdn.SingleTenantPluginName() || sdnPluginName == osdn.MultiTenantPluginName() {
if err := sysctl.SetSysctl("net/bridge/bridge-nf-call-iptables", 0); err != nil {
glog.Warningf("Could not set net.bridge.bridge-nf-call-iptables sysctl: %s", err)
}
}
}
return nil
}
func NewCmdJoinProjectsNetwork(commandName, fullName string, f *clientcmd.Factory, out io.Writer) *cobra.Command {
opts := &ProjectOptions{}
joinOp := &JoinOptions{Options: opts}
cmd := &cobra.Command{
Use: commandName,
Short: "Join project network",
Long: fmt.Sprintf(joinProjectsNetworkLong, ovs.MultiTenantPluginName()),
Example: fmt.Sprintf(joinProjectsNetworkExample, fullName),
Run: func(c *cobra.Command, args []string) {
if err := opts.Complete(f, c, args, out); err != nil {
kcmdutil.CheckErr(err)
}
opts.CheckSelector = c.Flag("selector").Changed
if err := joinOp.Validate(); err != nil {
kcmdutil.CheckErr(kcmdutil.UsageError(c, err.Error()))
}
err := joinOp.Run()
kcmdutil.CheckErr(err)
},
}
flags := cmd.Flags()
// Supported operations
flags.StringVar(&joinOp.joinProjectName, "to", "", "Join network of the given project name")
// Common optional params
flags.StringVar(&opts.Selector, "selector", "", "Label selector to filter projects. Either pass one/more projects as arguments or use this project selector")
return cmd
}
// Call by higher layers to create the plugin instance
func NewPlugin(pluginType string, osClient *osclient.Client, kClient *kclient.Client, hostname string, selfIP string, ready chan struct{}) (api.OsdnPlugin, oskserver.FilteringEndpointsConfigHandler, error) {
switch strings.ToLower(pluginType) {
case ovs.SingleTenantPluginName():
return ovs.CreatePlugin(osdn.NewRegistry(osClient, kClient), false, hostname, selfIP, ready)
case ovs.MultiTenantPluginName():
return ovs.CreatePlugin(osdn.NewRegistry(osClient, kClient), true, hostname, selfIP, ready)
}
return nil, nil, nil
}
func ValidateRuntime(config *configapi.NodeConfig, components *utilflags.ComponentFlag) error {
actual, err := components.Validate()
if err != nil {
return err
}
if actual.Len() == 0 {
return fmt.Errorf("at least one node component must be enabled (%s)", strings.Join(components.Allowed().List(), ", "))
}
switch strings.ToLower(config.NetworkConfig.NetworkPluginName) {
case ovs.MultiTenantPluginName():
if actual.Has(ComponentKubelet) && !actual.Has(ComponentPlugins) {
return fmt.Errorf("the multi-tenant SDN plugin must be run in the same process as the kubelet")
}
if actual.Has(ComponentProxy) && !actual.Has(ComponentPlugins) {
return fmt.Errorf("the multi-tenant SDN plugin requires the proxy and plugins components be enabled in the same process")
}
case ovs.SingleTenantPluginName():
if actual.Has(ComponentKubelet) && !actual.Has(ComponentPlugins) {
return fmt.Errorf("the SDN plugin must be run in the same process as the kubelet")
}
}
return nil
}
func BuildKubernetesNodeConfig(options configapi.NodeConfig) (*NodeConfig, error) {
originClient, _, err := configapi.GetOpenShiftClient(options.MasterKubeConfig)
if err != nil {
return nil, err
}
kubeClient, _, err := configapi.GetKubeClient(options.MasterKubeConfig)
if err != nil {
return nil, err
}
// Make a separate client for event reporting, to avoid event QPS blocking node calls
eventClient, _, err := configapi.GetKubeClient(options.MasterKubeConfig)
if err != nil {
return nil, err
}
if options.NodeName == "localhost" {
glog.Warningf(`Using "localhost" as node name will not resolve from all locations`)
}
clientCAs, err := util.CertPoolFromFile(options.ServingInfo.ClientCA)
if err != nil {
return nil, err
}
imageTemplate := variable.NewDefaultImageTemplate()
imageTemplate.Format = options.ImageConfig.Format
imageTemplate.Latest = options.ImageConfig.Latest
var path string
var fileCheckInterval int64
if options.PodManifestConfig != nil {
path = options.PodManifestConfig.Path
fileCheckInterval = options.PodManifestConfig.FileCheckIntervalSeconds
}
var dockerExecHandler dockertools.ExecHandler
switch options.DockerConfig.ExecHandlerName {
case configapi.DockerExecHandlerNative:
dockerExecHandler = &dockertools.NativeExecHandler{}
case configapi.DockerExecHandlerNsenter:
dockerExecHandler = &dockertools.NsenterExecHandler{}
}
kubeAddressStr, kubePortStr, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, fmt.Errorf("cannot parse node address: %v", err)
}
kubePort, err := strconv.Atoi(kubePortStr)
if err != nil {
return nil, fmt.Errorf("cannot parse node port: %v", err)
}
// Defaults are tested in TestKubeletDefaults
server := kubeletoptions.NewKubeletServer()
// Adjust defaults
server.Config = path
server.RootDirectory = options.VolumeDirectory
server.NodeIP = options.NodeIP
server.HostnameOverride = options.NodeName
server.AllowPrivileged = true
server.RegisterNode = true
server.Address = kubeAddressStr
server.Port = uint(kubePort)
server.ReadOnlyPort = 0 // no read only access
server.CAdvisorPort = 0 // no unsecured cadvisor access
server.HealthzPort = 0 // no unsecured healthz access
server.HealthzBindAddress = "" // no unsecured healthz access
server.ClusterDNS = options.DNSIP
server.ClusterDomain = options.DNSDomain
server.NetworkPluginName = options.NetworkConfig.NetworkPluginName
server.HostNetworkSources = strings.Join([]string{kubelettypes.ApiserverSource, kubelettypes.FileSource}, ",")
server.HostPIDSources = strings.Join([]string{kubelettypes.ApiserverSource, kubelettypes.FileSource}, ",")
server.HostIPCSources = strings.Join([]string{kubelettypes.ApiserverSource, kubelettypes.FileSource}, ",")
server.HTTPCheckFrequency = unversioned.Duration{Duration: time.Duration(0)} // no remote HTTP pod creation access
server.FileCheckFrequency = unversioned.Duration{Duration: time.Duration(fileCheckInterval) * time.Second}
server.PodInfraContainerImage = imageTemplate.ExpandOrDie("pod")
server.CPUCFSQuota = true // enable cpu cfs quota enforcement by default
server.MaxPods = 110
switch server.NetworkPluginName {
case ovs.SingleTenantPluginName(), ovs.MultiTenantPluginName():
// set defaults for openshift-sdn
server.HairpinMode = componentconfig.HairpinNone
server.ConfigureCBR0 = false
}
// prevents kube from generating certs
server.TLSCertFile = options.ServingInfo.ServerCert.CertFile
server.TLSPrivateKeyFile = options.ServingInfo.ServerCert.KeyFile
containerized := cmdutil.Env("OPENSHIFT_CONTAINERIZED", "") == "true"
server.Containerized = containerized
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubeletArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
proxyconfig, err := buildKubeProxyConfig(options)
if err != nil {
return nil, err
}
cfg, err := kubeletapp.UnsecuredKubeletConfig(server)
if err != nil {
return nil, err
}
// provide any config overrides
cfg.NodeName = options.NodeName
cfg.KubeClient = internalclientset.FromUnversionedClient(kubeClient)
cfg.EventClient = internalclientset.FromUnversionedClient(eventClient)
cfg.DockerExecHandler = dockerExecHandler
// docker-in-docker (dind) deployments are used for testing
// networking plugins. Running openshift under dind won't work
// with the real oom adjuster due to the state of the cgroups path
// in a dind container that uses systemd for init. Similarly,
// cgroup manipulation of the nested docker daemon doesn't work
// properly under centos/rhel and should be disabled by setting
// the name of the container to an empty string.
//
// This workaround should become unnecessary once user namespaces
if value := cmdutil.Env("OPENSHIFT_DIND", ""); value == "true" {
glog.Warningf("Using FakeOOMAdjuster for docker-in-docker compatibility")
cfg.OOMAdjuster = oom.NewFakeOOMAdjuster()
}
// Setup auth
osClient, osClientConfig, err := configapi.GetOpenShiftClient(options.MasterKubeConfig)
if err != nil {
return nil, err
}
authnTTL, err := time.ParseDuration(options.AuthConfig.AuthenticationCacheTTL)
if err != nil {
return nil, err
}
authn, err := newAuthenticator(clientCAs, clientcmd.AnonymousClientConfig(osClientConfig), authnTTL, options.AuthConfig.AuthenticationCacheSize)
if err != nil {
return nil, err
}
authzAttr, err := newAuthorizerAttributesGetter(options.NodeName)
if err != nil {
return nil, err
}
authzTTL, err := time.ParseDuration(options.AuthConfig.AuthorizationCacheTTL)
if err != nil {
return nil, err
}
authz, err := newAuthorizer(osClient, authzTTL, options.AuthConfig.AuthorizationCacheSize)
if err != nil {
return nil, err
}
cfg.Auth = kubeletserver.NewKubeletAuth(authn, authzAttr, authz)
// Make sure the node doesn't think it is in standalone mode
// This is required for the node to enforce nodeSelectors on pods, to set hostIP on pod status updates, etc
cfg.StandaloneMode = false
// TODO: could be cleaner
if configapi.UseTLS(options.ServingInfo) {
extraCerts, err := configapi.GetNamedCertificateMap(options.ServingInfo.NamedCertificates)
if err != nil {
return nil, err
}
cfg.TLSOptions = &kubeletserver.TLSOptions{
Config: crypto.SecureTLSConfig(&tls.Config{
// RequestClientCert lets us request certs, but allow requests without client certs
// Verification is done by the authn layer
ClientAuth: tls.RequestClientCert,
ClientCAs: clientCAs,
// Set SNI certificate func
// Do not use NameToCertificate, since that requires certificates be included in the server's tlsConfig.Certificates list,
// which we do not control when running with http.Server#ListenAndServeTLS
GetCertificate: cmdutil.GetCertificateFunc(extraCerts),
}),
CertFile: options.ServingInfo.ServerCert.CertFile,
KeyFile: options.ServingInfo.ServerCert.KeyFile,
}
} else {
cfg.TLSOptions = nil
}
// Prepare cloud provider
cloud, err := cloudprovider.InitCloudProvider(server.CloudProvider, server.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile)
}
cfg.Cloud = cloud
sdnPlugin, endpointFilter, err := factory.NewPlugin(options.NetworkConfig.NetworkPluginName, originClient, kubeClient, options.NodeName, options.NodeIP)
if err != nil {
return nil, fmt.Errorf("SDN initialization failed: %v", err)
}
if sdnPlugin != nil {
cfg.NetworkPlugins = append(cfg.NetworkPlugins, sdnPlugin)
}
config := &NodeConfig{
BindAddress: options.ServingInfo.BindAddress,
AllowDisabledDocker: options.AllowDisabledDocker,
Containerized: containerized,
Client: kubeClient,
VolumeDir: options.VolumeDirectory,
KubeletServer: server,
KubeletConfig: cfg,
ProxyConfig: proxyconfig,
MTU: options.NetworkConfig.MTU,
SDNPlugin: sdnPlugin,
FilteringEndpointsHandler: endpointFilter,
}
return config, nil
}