// TestExecutorShutdown ensures that the executor properly shuts down // when Shutdown is called. func TestExecutorShutdown(t *testing.T) { var ( mockDriver = &MockExecutorDriver{} kubeletFinished = make(chan struct{}) exitCalled = int32(0) executor = New(Config{ Docker: dockertools.CreateDockerClientOrDie("fake://", 0), NodeInfos: make(chan NodeInfo, 1), ShutdownAlert: func() { close(kubeletFinished) }, KubeletFinished: kubeletFinished, ExitFunc: func(_ int) { atomic.AddInt32(&exitCalled, 1) }, Registry: newFakeRegistry(), }) ) executor.Init(mockDriver) executor.Registered(mockDriver, nil, nil, nil) mockDriver.On("Stop").Return(mesosproto.Status_DRIVER_STOPPED, nil).Once() executor.Shutdown(mockDriver) assert.Equal(t, false, executor.isConnected(), "executor should not be connected after Shutdown") assert.Equal(t, true, executor.isDone(), "executor should be in Done state after Shutdown") assert.Equal(t, true, atomic.LoadInt32(&exitCalled) > 0, "the executor should call its ExitFunc when it is ready to close down") mockDriver.AssertExpectations(t) }
func (s *KubeletExecutorServer) runExecutor( nodeInfos chan<- executor.NodeInfo, kubeletFinished <-chan struct{}, staticPodsConfigPath string, apiclient *clientset.Clientset, registry executor.Registry, ) (<-chan struct{}, error) { staticPodFilters := podutil.Filters{ // annotate the pod with BindingHostKey so that the scheduler will ignore the pod // once it appears in the pod registry. the stock kubelet sets the pod host in order // to accomplish the same; we do this because the k8sm scheduler works differently. podutil.Annotator(map[string]string{ meta.BindingHostKey: s.HostnameOverride, }), } if s.containerID != "" { // tag all pod containers with the containerID so that they can be properly GC'd by Mesos staticPodFilters = append(staticPodFilters, podutil.Environment([]api.EnvVar{ {Name: envContainerID, Value: s.containerID}, })) } exec := executor.New(executor.Config{ Registry: registry, APIClient: apiclient, Docker: dockertools.CreateDockerClientOrDie(s.DockerEndpoint, 0), SuicideTimeout: s.SuicideTimeout, KubeletFinished: kubeletFinished, ExitFunc: os.Exit, NodeInfos: nodeInfos, Options: []executor.Option{ executor.StaticPods(staticPodsConfigPath, staticPodFilters), }, }) // initialize driver and initialize the executor with it dconfig := bindings.DriverConfig{ Executor: exec, HostnameOverride: s.HostnameOverride, BindingAddress: net.ParseIP(s.Address), } driver, err := bindings.NewMesosExecutorDriver(dconfig) if err != nil { return nil, fmt.Errorf("failed to create executor driver: %v", err) } log.V(2).Infof("Initialize executor driver...") exec.Init(driver) // start the driver go func() { if _, err := driver.Run(); err != nil { log.Fatalf("executor driver failed: %v", err) } log.Info("executor Run completed") }() return exec.Done(), nil }
//TODO: do not expose kubelet implementation details after we refactor the runtime API. func dockerRuntime() kubecontainer.Runtime { dockerClient := dockertools.CreateDockerClientOrDie("", 0) pm := kubepod.NewBasicPodManager(nil) dm := dockertools.NewDockerManager( dockerClient, nil, nil, nil, pm, nil, "", 0, 0, "", nil, nil, nil, nil, nil, nil, nil, false, nil, true, false, false, "", ) return dm }
// UnsecuredKubeletDeps returns a KubeletDeps suitable for being run, or an error if the server setup // is not valid. It will not start any background processes, and does not include authentication/authorization func UnsecuredKubeletDeps(s *options.KubeletServer) (*kubelet.KubeletDeps, error) { // Initialize the TLS Options tlsOptions, err := InitializeTLS(&s.KubeletConfiguration) if err != nil { return nil, err } mounter := mount.New() var writer kubeio.Writer = &kubeio.StdWriter{} if s.Containerized { glog.V(2).Info("Running kubelet in containerized mode (experimental)") mounter = mount.NewNsenterMounter() writer = &kubeio.NsenterWriter{} } var dockerClient dockertools.DockerInterface if s.ContainerRuntime == "docker" { dockerClient = dockertools.CreateDockerClientOrDie(s.DockerEndpoint, s.RuntimeRequestTimeout.Duration) } else { dockerClient = nil } return &kubelet.KubeletDeps{ Auth: nil, // default does not enforce auth[nz] CAdvisorInterface: nil, // cadvisor.New launches background processes (bg http.ListenAndServe, and some bg cleaners), not set here Cloud: nil, // cloud provider might start background processes ContainerManager: nil, DockerClient: dockerClient, KubeClient: nil, Mounter: mounter, NetworkPlugins: ProbeNetworkPlugins(s.NetworkPluginDir, s.CNIConfDir, s.CNIBinDir), OOMAdjuster: oom.NewOOMAdjuster(), OSInterface: kubecontainer.RealOS{}, Writer: writer, VolumePlugins: ProbeVolumePlugins(s.VolumePluginDir), TLSOptions: tlsOptions, }, nil }
func NewTestKubernetesExecutor() *Executor { return New(Config{ Docker: dockertools.CreateDockerClientOrDie("fake://", 0), Registry: newFakeRegistry(), }) }
// TestExecutorFrameworkMessage ensures that the executor is able to // handle messages from the framework, specifically about lost tasks // and Kamikaze. When a task is lost, the executor needs to clean up // its state. When a Kamikaze message is received, the executor should // attempt suicide. func TestExecutorFrameworkMessage(t *testing.T) { // TODO(jdef): Fix the unexpected call in the mocking system. t.Skip("This test started failing when panic catching was disabled.") var ( mockDriver = &MockExecutorDriver{} kubeletFinished = make(chan struct{}) registry = newFakeRegistry() executor = New(Config{ Docker: dockertools.CreateDockerClientOrDie("fake://", 0), NodeInfos: make(chan NodeInfo, 1), ShutdownAlert: func() { close(kubeletFinished) }, KubeletFinished: kubeletFinished, Registry: registry, }) pod = NewTestPod(1) mockKubeAPI = &mockKubeAPI{} ) executor.kubeAPI = mockKubeAPI executor.Init(mockDriver) executor.Registered(mockDriver, nil, nil, nil) executor.FrameworkMessage(mockDriver, "test framework message") // set up a pod to then lose executorinfo := &mesosproto.ExecutorInfo{} podTask, _ := podtask.New( api.NewDefaultContext(), podtask.Config{ ID: "foo", Prototype: executorinfo, HostPortStrategy: hostport.StrategyWildcard, }, pod, ) pod.Annotations = map[string]string{ "k8s.mesosphere.io/taskId": podTask.ID, } podTask.Spec = &podtask.Spec{ Executor: executorinfo, } taskInfo, err := podTask.BuildTaskInfo() assert.Equal(t, nil, err, "must be able to build task info") data, _ := runtime.Encode(testapi.Default.Codec(), pod) taskInfo.Data = data mockDriver.On( "SendStatusUpdate", mesosproto.TaskState_TASK_STARTING, ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Once() called := make(chan struct{}) mockDriver.On( "SendStatusUpdate", mesosproto.TaskState_TASK_RUNNING, ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Run(func(_ mock.Arguments) { close(called) }).Once() executor.LaunchTask(mockDriver, taskInfo) // must wait for this otherwise phase changes may not apply assertext.EventuallyTrue(t, wait.ForeverTestTimeout, func() bool { executor.lock.Lock() defer executor.lock.Unlock() return !registry.empty() }, "executor must be able to create a task and a pod") err = registry.phaseChange(pod, api.PodPending) assert.NoError(t, err) err = registry.phaseChange(pod, api.PodRunning) assert.NoError(t, err) // waiting until the pod is really running b/c otherwise a TASK_FAILED could be // triggered by the asynchronously running executor methods when removing the task // from k.tasks through the "task-lost:foo" message below. select { case <-called: case <-time.After(wait.ForeverTestTimeout): t.Fatalf("timed out waiting for SendStatusUpdate for the running task") } // send task-lost message for it called = make(chan struct{}) mockDriver.On( "SendStatusUpdate", mesosproto.TaskState_TASK_LOST, ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Run(func(_ mock.Arguments) { close(called) }).Once() // simulate what happens when the apiserver is told to delete a pod mockKubeAPI.On("killPod", pod.Namespace, pod.Name).Return(nil).Run(func(_ mock.Arguments) { registry.Remove(podTask.ID) }) executor.FrameworkMessage(mockDriver, "task-lost:foo") assertext.EventuallyTrue(t, wait.ForeverTestTimeout, func() bool { executor.lock.Lock() defer executor.lock.Unlock() return registry.empty() }, "executor must be able to kill a created task and pod") select { case <-called: case <-time.After(wait.ForeverTestTimeout): t.Fatalf("timed out waiting for SendStatusUpdate") } mockDriver.On("Stop").Return(mesosproto.Status_DRIVER_STOPPED, nil).Once() executor.FrameworkMessage(mockDriver, messages.Kamikaze) assert.Equal(t, true, executor.isDone(), "executor should have shut down after receiving a Kamikaze message") mockDriver.AssertExpectations(t) mockKubeAPI.AssertExpectations(t) }
// TestExecutorLaunchAndKillTask ensures that the executor is able to launch tasks and generates // appropriate status messages for mesos. It then kills the task and validates that appropriate // actions are taken by the executor. func TestExecutorLaunchAndKillTask(t *testing.T) { var ( mockDriver = &MockExecutorDriver{} registry = newFakeRegistry() executor = New(Config{ Docker: dockertools.CreateDockerClientOrDie("fake://", 0), NodeInfos: make(chan NodeInfo, 1), Registry: registry, }) mockKubeAPI = &mockKubeAPI{} pod = NewTestPod(1) executorinfo = &mesosproto.ExecutorInfo{} ) executor.kubeAPI = mockKubeAPI executor.Init(mockDriver) executor.Registered(mockDriver, nil, nil, nil) podTask, err := podtask.New( api.NewDefaultContext(), podtask.Config{ Prototype: executorinfo, HostPortStrategy: hostport.StrategyWildcard, }, pod, ) assert.Equal(t, nil, err, "must be able to create a task from a pod") pod.Annotations = map[string]string{ "k8s.mesosphere.io/taskId": podTask.ID, } podTask.Spec = &podtask.Spec{Executor: executorinfo} taskInfo, err := podTask.BuildTaskInfo() assert.Equal(t, nil, err, "must be able to build task info") data, err := runtime.Encode(testapi.Default.Codec(), pod) assert.Equal(t, nil, err, "must be able to encode a pod's spec data") taskInfo.Data = data var statusUpdateCalls sync.WaitGroup statusUpdateCalls.Add(1) statusUpdateDone := func(_ mock.Arguments) { statusUpdateCalls.Done() } mockDriver.On( "SendStatusUpdate", mesosproto.TaskState_TASK_STARTING, ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Run(statusUpdateDone).Once() statusUpdateCalls.Add(1) mockDriver.On( "SendStatusUpdate", mesosproto.TaskState_TASK_RUNNING, ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Run(statusUpdateDone).Once() executor.LaunchTask(mockDriver, taskInfo) assertext.EventuallyTrue(t, wait.ForeverTestTimeout, func() bool { executor.lock.Lock() defer executor.lock.Unlock() return !registry.empty() }, "executor must be able to create a task and a pod") // simulate a pod source update; normally this update is generated when binding a pod err = registry.phaseChange(pod, api.PodPending) assert.NoError(t, err) // simulate a pod source update; normally this update is generated by the kubelet once the pod is healthy err = registry.phaseChange(pod, api.PodRunning) assert.NoError(t, err) // Allow some time for asynchronous requests to the driver. finished := kmruntime.After(statusUpdateCalls.Wait) select { case <-finished: case <-time.After(wait.ForeverTestTimeout): t.Fatalf("timed out waiting for status update calls to finish") } statusUpdateCalls.Add(1) mockDriver.On( "SendStatusUpdate", mesosproto.TaskState_TASK_KILLED, ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Run(statusUpdateDone).Once() // simulate what happens when the apiserver is told to delete a pod mockKubeAPI.On("killPod", pod.Namespace, pod.Name).Return(nil).Run(func(_ mock.Arguments) { registry.Remove(podTask.ID) }) executor.KillTask(mockDriver, taskInfo.TaskId) assertext.EventuallyTrue(t, wait.ForeverTestTimeout, func() bool { executor.lock.Lock() defer executor.lock.Unlock() return registry.empty() }, "executor must be able to kill a created task and pod") // Allow some time for asynchronous requests to the driver. finished = kmruntime.After(statusUpdateCalls.Wait) select { case <-finished: case <-time.After(wait.ForeverTestTimeout): t.Fatalf("timed out waiting for status update calls to finish") } mockDriver.AssertExpectations(t) mockKubeAPI.AssertExpectations(t) }