Example #1
0
func (s *KubeletExecutorServer) runKubelet(execUpdates <-chan kubetypes.PodUpdate, nodeInfos <-chan executor.NodeInfo, kubeletDone chan<- struct{},
	staticPodsConfigPath string, apiclient *client.Client) error {
	kcfg, err := s.UnsecuredKubeletConfig()
	if err == nil {
		// apply Messo specific settings
		executorDone := make(chan struct{})
		kcfg.Builder = func(kc *app.KubeletConfig) (app.KubeletBootstrap, *kconfig.PodConfig, error) {
			k, pc, err := app.CreateAndInitKubelet(kc)
			if err != nil {
				return k, pc, err
			}

			klet := k.(*kubelet.Kubelet)

			s.kletLock.Lock()
			s.klet = klet
			s.kletLock.Unlock()

			// decorate kubelet such that it shuts down when the executor is
			decorated := &executorKubelet{
				Kubelet:      klet,
				kubeletDone:  kubeletDone,
				executorDone: executorDone,
			}

			return decorated, pc, nil
		}
		kcfg.DockerDaemonContainer = "" // don't move the docker daemon into a cgroup
		kcfg.Hostname = kcfg.HostnameOverride
		kcfg.KubeClient = apiclient
		kcfg.NodeName = kcfg.HostnameOverride
		kcfg.PodConfig = kconfig.NewPodConfig(kconfig.PodConfigNotificationIncremental, kcfg.Recorder) // override the default pod source
		kcfg.StandaloneMode = false
		kcfg.SystemContainer = "" // don't take control over other system processes.
		if kcfg.Cloud != nil {
			// fail early and hard because having the cloud provider loaded would go unnoticed,
			// but break bigger cluster because accessing the state.json from every slave kills the master.
			panic("cloud provider must not be set")
		}

		// create custom cAdvisor interface which return the resource values that Mesos reports
		ni := <-nodeInfos
		cAdvisorInterface, err := NewMesosCadvisor(ni.Cores, ni.Mem, s.CAdvisorPort)
		if err != nil {
			return err
		}
		kcfg.CAdvisorInterface = cAdvisorInterface
		go func() {
			for ni := range nodeInfos {
				// TODO(sttts): implement with MachineAllocable mechanism when https://github.com/kubernetes/kubernetes/issues/13984 is finished
				log.V(3).Infof("ignoring updated node resources: %v", ni)
			}
		}()

		// create main pod source
		updates := kcfg.PodConfig.Channel(MESOS_CFG_SOURCE)
		go func() {
			// execUpdates will be closed by the executor on shutdown
			defer close(executorDone)

			for u := range execUpdates {
				u.Source = MESOS_CFG_SOURCE
				updates <- u
			}
		}()

		// create static-pods directory file source
		log.V(2).Infof("initializing static pods source factory, configured at path %q", staticPodsConfigPath)
		fileSourceUpdates := kcfg.PodConfig.Channel(kubetypes.FileSource)
		kconfig.NewSourceFile(staticPodsConfigPath, kcfg.HostnameOverride, kcfg.FileCheckFrequency, fileSourceUpdates)

		// run the kubelet, until execUpdates is closed
		// NOTE: because kcfg != nil holds, the upstream Run function will not
		//       initialize the cloud provider. We explicitly wouldn't want
		//       that because then every kubelet instance would query the master
		//       state.json which does not scale.
		err = s.KubeletServer.Run(kcfg)
	}

	if err != nil {
		// close the channel here. When Run returns without error, the executorKubelet is
		// responsible to do this. If it returns with an error, we are responsible here.
		close(kubeletDone)
	}
	return err
}
Example #2
0
func (s *KubeletExecutorServer) runKubelet(
	nodeInfos <-chan executor.NodeInfo,
	kubeletDone chan<- struct{},
	staticPodsConfigPath string,
	apiclient *clientset.Clientset,
	podLW *cache.ListWatch,
	registry executor.Registry,
	executorDone <-chan struct{},
) (err error) {
	defer func() {
		if err != nil {
			// close the channel here. When Run returns without error, the executorKubelet is
			// responsible to do this. If it returns with an error, we are responsible here.
			close(kubeletDone)
		}
	}()

	kcfg, err := kubeletapp.UnsecuredKubeletConfig(s.KubeletServer)
	if err != nil {
		return err
	}

	// apply Mesos specific settings
	kcfg.Builder = func(kc *kubeletapp.KubeletConfig) (kubeletapp.KubeletBootstrap, *kconfig.PodConfig, error) {
		k, pc, err := kubeletapp.CreateAndInitKubelet(kc)
		if err != nil {
			return k, pc, err
		}

		// decorate kubelet such that it shuts down when the executor is
		decorated := &executorKubelet{
			Kubelet:      k.(*kubelet.Kubelet),
			kubeletDone:  kubeletDone,
			executorDone: executorDone,
		}

		return decorated, pc, nil
	}
	kcfg.DockerDaemonContainer = "" // don't move the docker daemon into a cgroup
	kcfg.Hostname = kcfg.HostnameOverride
	kcfg.KubeClient = apiclient

	// taken from KubeletServer#Run(*KubeletConfig)
	eventClientConfig, err := kubeletapp.CreateAPIServerClientConfig(s.KubeletServer)
	if err != nil {
		return err
	}

	// make a separate client for events
	eventClientConfig.QPS = s.EventRecordQPS
	eventClientConfig.Burst = s.EventBurst
	kcfg.EventClient, err = clientset.NewForConfig(eventClientConfig)
	if err != nil {
		return err
	}

	kcfg.NodeName = kcfg.HostnameOverride
	kcfg.PodConfig = kconfig.NewPodConfig(kconfig.PodConfigNotificationIncremental, kcfg.Recorder) // override the default pod source
	kcfg.StandaloneMode = false
	kcfg.SystemContainer = "" // don't take control over other system processes.
	if kcfg.Cloud != nil {
		// fail early and hard because having the cloud provider loaded would go unnoticed,
		// but break bigger cluster because accessing the state.json from every slave kills the master.
		panic("cloud provider must not be set")
	}

	// create custom cAdvisor interface which return the resource values that Mesos reports
	ni := <-nodeInfos
	cAdvisorInterface, err := NewMesosCadvisor(ni.Cores, ni.Mem, s.CAdvisorPort)
	if err != nil {
		return err
	}

	kcfg.CAdvisorInterface = cAdvisorInterface
	kcfg.ContainerManager, err = cm.NewContainerManager(kcfg.Mounter, cAdvisorInterface)
	if err != nil {
		return err
	}

	go func() {
		for ni := range nodeInfos {
			// TODO(sttts): implement with MachineAllocable mechanism when https://github.com/kubernetes/kubernetes/issues/13984 is finished
			log.V(3).Infof("ignoring updated node resources: %v", ni)
		}
	}()

	// create main pod source, it will stop generating events once executorDone is closed
	newSourceMesos(executorDone, kcfg.PodConfig.Channel(mesosSource), podLW, registry)

	// create static-pods directory file source
	log.V(2).Infof("initializing static pods source factory, configured at path %q", staticPodsConfigPath)
	fileSourceUpdates := kcfg.PodConfig.Channel(kubetypes.FileSource)
	kconfig.NewSourceFile(staticPodsConfigPath, kcfg.HostnameOverride, kcfg.FileCheckFrequency, fileSourceUpdates)

	// run the kubelet
	// NOTE: because kcfg != nil holds, the upstream Run function will not
	//       initialize the cloud provider. We explicitly wouldn't want
	//       that because then every kubelet instance would query the master
	//       state.json which does not scale.
	err = kubeletapp.Run(s.KubeletServer, kcfg)
	return
}
Example #3
0
func (s *KubeletExecutorServer) runKubelet(
	nodeInfos <-chan executor.NodeInfo,
	kubeletDone chan<- struct{},
	staticPodsConfigPath string,
	apiclient *clientset.Clientset,
	podLW *cache.ListWatch,
	registry executor.Registry,
	executorDone <-chan struct{},
) (err error) {
	defer func() {
		if err != nil {
			// close the channel here. When Run returns without error, the executorKubelet is
			// responsible to do this. If it returns with an error, we are responsible here.
			close(kubeletDone)
		}
	}()

	kubeDeps, err := kubeletapp.UnsecuredKubeletDeps(s.KubeletServer)
	if err != nil {
		return err
	}

	// apply Mesos specific settings
	kubeDeps.Builder = func(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.KubeletDeps, standaloneMode bool) (kubelet.KubeletBootstrap, error) {
		k, err := kubeletapp.CreateAndInitKubelet(kubeCfg, kubeDeps, standaloneMode)
		if err != nil {
			return k, err
		}

		// decorate kubelet such that it shuts down when the executor is
		decorated := &executorKubelet{
			Kubelet:      k.(*kubelet.Kubelet),
			kubeletDone:  kubeletDone,
			executorDone: executorDone,
		}

		return decorated, nil
	}
	s.RuntimeCgroups = "" // don't move the docker daemon into a cgroup
	kubeDeps.KubeClient = apiclient

	// taken from KubeletServer#Run(*KubeletConfig)
	eventClientConfig, err := kubeletapp.CreateAPIServerClientConfig(s.KubeletServer)
	if err != nil {
		return err
	}

	// make a separate client for events
	eventClientConfig.QPS = float32(s.EventRecordQPS)
	eventClientConfig.Burst = int(s.EventBurst)
	kubeDeps.EventClient, err = clientset.NewForConfig(eventClientConfig)
	if err != nil {
		return err
	}

	kubeDeps.PodConfig = kconfig.NewPodConfig(kconfig.PodConfigNotificationIncremental, kubeDeps.Recorder) // override the default pod source

	s.SystemCgroups = "" // don't take control over other system processes.

	if kubeDeps.Cloud != nil {
		// fail early and hard because having the cloud provider loaded would go unnoticed,
		// but break bigger cluster because accessing the state.json from every slave kills the master.
		panic("cloud provider must not be set")
	}

	// create custom cAdvisor interface which return the resource values that Mesos reports
	ni := <-nodeInfos
	cAdvisorInterface, err := NewMesosCadvisor(ni.Cores, ni.Mem, uint(s.CAdvisorPort), s.ContainerRuntime)
	if err != nil {
		return err
	}

	kubeDeps.CAdvisorInterface = cAdvisorInterface
	kubeDeps.ContainerManager, err = cm.NewContainerManager(kubeDeps.Mounter, cAdvisorInterface, cm.NodeConfig{
		RuntimeCgroupsName: s.RuntimeCgroups,
		SystemCgroupsName:  s.SystemCgroups,
		KubeletCgroupsName: s.KubeletCgroups,
		ContainerRuntime:   s.ContainerRuntime,
	})
	if err != nil {
		return err
	}

	go func() {
		for ni := range nodeInfos {
			// TODO(sttts): implement with MachineAllocable mechanism when https://github.com/kubernetes/kubernetes/issues/13984 is finished
			log.V(3).Infof("ignoring updated node resources: %v", ni)
		}
	}()

	// create main pod source, it will stop generating events once executorDone is closed
	var containerOptions []podsource.Option
	if s.containerID != "" {
		// tag all pod containers with the containerID so that they can be properly GC'd by Mesos
		containerOptions = append(containerOptions, podsource.ContainerEnvOverlay([]api.EnvVar{
			{Name: envContainerID, Value: s.containerID},
		}))
		kubeDeps.ContainerRuntimeOptions = append(kubeDeps.ContainerRuntimeOptions,
			dockertools.PodInfraContainerEnv(map[string]string{
				envContainerID: s.containerID,
			}))
	}

	podsource.Mesos(executorDone, kubeDeps.PodConfig.Channel(podsource.MesosSource), podLW, registry, containerOptions...)

	// create static-pods directory file source
	log.V(2).Infof("initializing static pods source factory, configured at path %q", staticPodsConfigPath)
	fileSourceUpdates := kubeDeps.PodConfig.Channel(kubetypes.FileSource)
	kconfig.NewSourceFile(staticPodsConfigPath, s.HostnameOverride, s.FileCheckFrequency.Duration, fileSourceUpdates)

	// run the kubelet
	// NOTE: because kubeDeps != nil holds, the upstream Run function will not
	//       initialize the cloud provider. We explicitly wouldn't want
	//       that because then every kubelet instance would query the master
	//       state.json which does not scale.
	s.KubeletServer.LockFilePath = "" // disable lock file
	err = kubeletapp.Run(s.KubeletServer, kubeDeps)
	return
}