func (s *SchedulerServer) bootstrap(hks hyperkube.Interface, sc *schedcfg.Config) (*ha.SchedulerProcess, ha.DriverFactory, tools.EtcdGetSet, *uid.UID) { s.FrameworkName = strings.TrimSpace(s.FrameworkName) if s.FrameworkName == "" { log.Fatalf("framework-name must be a non-empty string") } s.FrameworkWebURI = strings.TrimSpace(s.FrameworkWebURI) metrics.Register() runtime.Register() s.mux.Handle("/metrics", prometheus.Handler()) if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) { log.Fatalf("specify either --etcd-servers or --etcd-config") } if len(s.APIServerList) < 1 { log.Fatal("No api servers specified.") } client, err := s.createAPIServerClient() if err != nil { log.Fatalf("Unable to make apiserver client: %v", err) } s.client = client if s.ReconcileCooldown < defaultReconcileCooldown { s.ReconcileCooldown = defaultReconcileCooldown log.Warningf("user-specified reconcile cooldown too small, defaulting to %v", s.ReconcileCooldown) } executor, eid, err := s.prepareExecutorInfo(hks) if err != nil { log.Fatalf("misconfigured executor: %v", err) } // TODO(jdef): remove the dependency on etcd as soon as // (1) the generic config store is available for the FrameworkId storage // (2) the generic master election is provided by the apiserver // Compare docs/proposals/high-availability.md etcdClient, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList) if err != nil { log.Fatalf("misconfigured etcd: %v", err) } mesosPodScheduler := scheduler.New(scheduler.Config{ Schedcfg: *sc, Executor: executor, ScheduleFunc: scheduler.FCFSScheduleFunc, Client: client, EtcdClient: etcdClient, FailoverTimeout: s.FailoverTimeout, ReconcileInterval: s.ReconcileInterval, ReconcileCooldown: s.ReconcileCooldown, }) masterUri := s.MesosMaster info, cred, err := s.buildFrameworkInfo() if err != nil { log.Fatalf("Misconfigured mesos framework: %v", err) } schedulerProcess := ha.New(mesosPodScheduler) dconfig := &bindings.DriverConfig{ Scheduler: schedulerProcess, Framework: info, Master: masterUri, Credential: cred, BindingAddress: net.IP(s.Address), BindingPort: uint16(s.DriverPort), HostnameOverride: s.HostnameOverride, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, s.MesosAuthProvider) ctx = sasl.WithBindingAddress(ctx, net.IP(s.Address)) return ctx }, } kpl := scheduler.NewPlugin(mesosPodScheduler.NewDefaultPluginConfig(schedulerProcess.Terminal(), s.mux)) runtime.On(mesosPodScheduler.Registration(), func() { kpl.Run(schedulerProcess.Terminal()) }) runtime.On(mesosPodScheduler.Registration(), s.newServiceWriter(schedulerProcess.Terminal())) driverFactory := ha.DriverFactory(func() (drv bindings.SchedulerDriver, err error) { log.V(1).Infoln("performing deferred initialization") if err = mesosPodScheduler.Init(schedulerProcess.Master(), kpl, s.mux); err != nil { return nil, fmt.Errorf("failed to initialize pod scheduler: %v", err) } log.V(1).Infoln("deferred init complete") // defer obtaining framework ID to prevent multiple schedulers // from overwriting each other's framework IDs dconfig.Framework.Id, err = s.fetchFrameworkID(etcdClient) if err != nil { return nil, fmt.Errorf("failed to fetch framework ID from etcd: %v", err) } log.V(1).Infoln("constructing mesos scheduler driver") drv, err = bindings.NewMesosSchedulerDriver(*dconfig) if err != nil { return nil, fmt.Errorf("failed to construct scheduler driver: %v", err) } log.V(1).Infoln("constructed mesos scheduler driver:", drv) s.setDriver(drv) return drv, nil }) return schedulerProcess, driverFactory, etcdClient, eid }
// Test to create the scheduler plugin with the config returned by the scheduler, // and play through the whole life cycle of the plugin while creating pods, deleting // and failing them. func TestPlugin_LifeCycle(t *testing.T) { t.Skip("disabled due to flakiness; see #10795") assert := &EventAssertions{*assert.New(t)} // create a fake pod watch. We use that below to submit new pods to the scheduler podListWatch := NewMockPodsListWatch(api.PodList{}) // create fake apiserver testApiServer := NewTestServer(t, api.NamespaceDefault, podListWatch) defer testApiServer.server.Close() // create executor with some data for static pods if set executor := util.NewExecutorInfo( util.NewExecutorID("executor-id"), util.NewCommandInfo("executor-cmd"), ) executor.Data = []byte{0, 1, 2} // create scheduler testScheduler := New(Config{ Executor: executor, Client: client.NewOrDie(&client.Config{Host: testApiServer.server.URL, Version: testapi.Version()}), ScheduleFunc: FCFSScheduleFunc, Schedcfg: *schedcfg.CreateDefaultConfig(), }) assert.NotNil(testScheduler.client, "client is nil") assert.NotNil(testScheduler.executor, "executor is nil") assert.NotNil(testScheduler.offers, "offer registry is nil") // create scheduler process schedulerProcess := ha.New(testScheduler) // get plugin config from it c := testScheduler.NewPluginConfig(schedulerProcess.Terminal(), http.DefaultServeMux, &podListWatch.ListWatch) assert.NotNil(c) // make events observable eventObserver := NewEventObserver() c.Recorder = eventObserver // create plugin p := NewPlugin(c) assert.NotNil(p) // run plugin p.Run(schedulerProcess.Terminal()) defer schedulerProcess.End() // init scheduler err := testScheduler.Init(schedulerProcess.Master(), p, http.DefaultServeMux) assert.NoError(err) // create mock mesos scheduler driver mockDriver := &joinableDriver{} mockDriver.On("Start").Return(mesos.Status_DRIVER_RUNNING, nil).Once() started := mockDriver.Upon() mAny := mock.AnythingOfType mockDriver.On("ReconcileTasks", mAny("[]*mesosproto.TaskStatus")).Return(mesos.Status_DRIVER_RUNNING, nil) mockDriver.On("SendFrameworkMessage", mAny("*mesosproto.ExecutorID"), mAny("*mesosproto.SlaveID"), mAny("string")). Return(mesos.Status_DRIVER_RUNNING, nil) launchedTasks := make(chan *mesos.TaskInfo, 1) launchTasksCalledFunc := func(args mock.Arguments) { taskInfos := args.Get(1).([]*mesos.TaskInfo) assert.Equal(1, len(taskInfos)) launchedTasks <- taskInfos[0] } mockDriver.On("LaunchTasks", mAny("[]*mesosproto.OfferID"), mAny("[]*mesosproto.TaskInfo"), mAny("*mesosproto.Filters")). Return(mesos.Status_DRIVER_RUNNING, nil).Run(launchTasksCalledFunc) // elect master with mock driver driverFactory := ha.DriverFactory(func() (bindings.SchedulerDriver, error) { return mockDriver, nil }) schedulerProcess.Elect(driverFactory) elected := schedulerProcess.Elected() // driver will be started <-started // tell scheduler to be registered testScheduler.Registered( mockDriver, util.NewFrameworkID("kubernetes-id"), util.NewMasterInfo("master-id", (192<<24)+(168<<16)+(0<<8)+1, 5050), ) // wait for being elected <-elected //TODO(jdef) refactor things above here into a test suite setup of some sort // fake new, unscheduled pod pod1 := NewTestPod(1) podListWatch.Add(pod1, true) // notify watchers // wait for failedScheduling event because there is no offer assert.EventWithReason(eventObserver, "failedScheduling", "failedScheduling event not received") // add some matching offer offers1 := []*mesos.Offer{NewTestOffer(1)} testScheduler.ResourceOffers(nil, offers1) // and wait for scheduled pod assert.EventWithReason(eventObserver, "scheduled") select { case launchedTask := <-launchedTasks: // report back that the task has been staged, and then started by mesos testScheduler.StatusUpdate(mockDriver, newTaskStatusForTask(launchedTask, mesos.TaskState_TASK_STAGING)) testScheduler.StatusUpdate(mockDriver, newTaskStatusForTask(launchedTask, mesos.TaskState_TASK_RUNNING)) // check that ExecutorInfo.data has the static pod data assert.Len(launchedTask.Executor.Data, 3) // report back that the task has been lost mockDriver.AssertNumberOfCalls(t, "SendFrameworkMessage", 0) testScheduler.StatusUpdate(mockDriver, newTaskStatusForTask(launchedTask, mesos.TaskState_TASK_LOST)) // and wait that framework message is sent to executor mockDriver.AssertNumberOfCalls(t, "SendFrameworkMessage", 1) case <-time.After(5 * time.Second): t.Fatalf("timed out waiting for launchTasks call") } // start another pod podNum := 1 startPod := func(offers []*mesos.Offer) (*api.Pod, *mesos.TaskInfo) { podNum = podNum + 1 // create pod and matching offer pod := NewTestPod(podNum) podListWatch.Add(pod, true) // notify watchers testScheduler.ResourceOffers(mockDriver, offers) assert.EventWithReason(eventObserver, "scheduled") // wait for driver.launchTasks call select { case launchedTask := <-launchedTasks: testScheduler.StatusUpdate(mockDriver, newTaskStatusForTask(launchedTask, mesos.TaskState_TASK_STAGING)) testScheduler.StatusUpdate(mockDriver, newTaskStatusForTask(launchedTask, mesos.TaskState_TASK_RUNNING)) return pod, launchedTask case <-time.After(5 * time.Second): t.Fatal("timed out waiting for launchTasks") return nil, nil } } pod, launchedTask := startPod(offers1) // mock drvier.KillTask, should be invoked when a pod is deleted mockDriver.On("KillTask", mAny("*mesosproto.TaskID")).Return(mesos.Status_DRIVER_RUNNING, nil).Run(func(args mock.Arguments) { killedTaskId := *(args.Get(0).(*mesos.TaskID)) assert.Equal(*launchedTask.TaskId, killedTaskId, "expected same TaskID as during launch") }) killTaskCalled := mockDriver.Upon() // stop it again via the apiserver mock podListWatch.Delete(pod, true) // notify watchers // and wait for the driver killTask call with the correct TaskId select { case <-killTaskCalled: // report back that the task is finished testScheduler.StatusUpdate(mockDriver, newTaskStatusForTask(launchedTask, mesos.TaskState_TASK_FINISHED)) case <-time.After(5 * time.Second): t.Fatal("timed out waiting for KillTask") } // start pods: // - which are failing while binding, // - leading to reconciliation // - with different states on the apiserver failPodFromExecutor := func(task *mesos.TaskInfo) { beforePodLookups := testApiServer.Stats(pod.Name) status := newTaskStatusForTask(task, mesos.TaskState_TASK_FAILED) message := messages.CreateBindingFailure status.Message = &message testScheduler.StatusUpdate(mockDriver, status) // wait until pod is looked up at the apiserver assertext.EventuallyTrue(t, time.Second, func() bool { return testApiServer.Stats(pod.Name) == beforePodLookups+1 }, "expect that reconcilePod will access apiserver for pod %v", pod.Name) } // 1. with pod deleted from the apiserver pod, launchedTask = startPod(offers1) podListWatch.Delete(pod, false) // not notifying the watchers failPodFromExecutor(launchedTask) // 2. with pod still on the apiserver, not bound pod, launchedTask = startPod(offers1) failPodFromExecutor(launchedTask) // 3. with pod still on the apiserver, bound i.e. host!="" pod, launchedTask = startPod(offers1) pod.Spec.NodeName = *offers1[0].Hostname podListWatch.Modify(pod, false) // not notifying the watchers failPodFromExecutor(launchedTask) // 4. with pod still on the apiserver, bound i.e. host!="", notified via ListWatch pod, launchedTask = startPod(offers1) pod.Spec.NodeName = *offers1[0].Hostname podListWatch.Modify(pod, true) // notifying the watchers time.Sleep(time.Second / 2) failPodFromExecutor(launchedTask) }