func (s *SchedulerServer) bootstrap(hks hyperkube.Interface, sc *schedcfg.Config) (*ha.SchedulerProcess, ha.DriverFactory, tools.EtcdClient, *uid.UID) { s.FrameworkName = strings.TrimSpace(s.FrameworkName) if s.FrameworkName == "" { log.Fatalf("framework-name must be a non-empty string") } s.FrameworkWebURI = strings.TrimSpace(s.FrameworkWebURI) metrics.Register() runtime.Register() s.mux.Handle("/metrics", prometheus.Handler()) healthz.InstallHandler(s.mux) if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) { log.Fatalf("specify either --etcd-servers or --etcd-config") } if len(s.APIServerList) < 1 { log.Fatal("No api servers specified.") } client, err := s.createAPIServerClient() if err != nil { log.Fatalf("Unable to make apiserver client: %v", err) } s.client = client if s.ReconcileCooldown < defaultReconcileCooldown { s.ReconcileCooldown = defaultReconcileCooldown log.Warningf("user-specified reconcile cooldown too small, defaulting to %v", s.ReconcileCooldown) } executor, eid, err := s.prepareExecutorInfo(hks) if err != nil { log.Fatalf("misconfigured executor: %v", err) } // TODO(jdef): remove the dependency on etcd as soon as // (1) the generic config store is available for the FrameworkId storage // (2) the generic master election is provided by the apiserver // Compare docs/proposals/high-availability.md etcdClient, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList) if err != nil { log.Fatalf("misconfigured etcd: %v", err) } as := scheduler.NewAllocationStrategy( podtask.NewDefaultPredicate( s.DefaultContainerCPULimit, s.DefaultContainerMemLimit, ), podtask.NewDefaultProcurement( s.DefaultContainerCPULimit, s.DefaultContainerMemLimit, ), ) // downgrade allocation strategy if user disables "account-for-pod-resources" if !s.AccountForPodResources { as = scheduler.NewAllocationStrategy( podtask.DefaultMinimalPredicate, podtask.DefaultMinimalProcurement) } // mirror all nodes into the nodeStore nodesClient, err := s.createAPIServerClient() if err != nil { log.Fatalf("Cannot create client to watch nodes: %v", err) } nodeStore := cache.NewStore(cache.MetaNamespaceKeyFunc) nodeLW := cache.NewListWatchFromClient(nodesClient, "nodes", api.NamespaceAll, fields.Everything()) cache.NewReflector(nodeLW, &api.Node{}, nodeStore, s.nodeRelistPeriod).Run() lookupNode := func(hostName string) *api.Node { n, _, _ := nodeStore.GetByKey(hostName) // ignore error and return nil then if n == nil { return nil } return n.(*api.Node) } fcfs := scheduler.NewFCFSPodScheduler(as, lookupNode) mesosPodScheduler := scheduler.New(scheduler.Config{ Schedcfg: *sc, Executor: executor, Scheduler: fcfs, Client: client, EtcdClient: etcdClient, FailoverTimeout: s.FailoverTimeout, ReconcileInterval: s.ReconcileInterval, ReconcileCooldown: s.ReconcileCooldown, LookupNode: lookupNode, }) masterUri := s.MesosMaster info, cred, err := s.buildFrameworkInfo() if err != nil { log.Fatalf("Misconfigured mesos framework: %v", err) } schedulerProcess := ha.New(mesosPodScheduler) dconfig := &bindings.DriverConfig{ Scheduler: schedulerProcess, Framework: info, Master: masterUri, Credential: cred, BindingAddress: s.Address, BindingPort: uint16(s.DriverPort), HostnameOverride: s.HostnameOverride, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, s.MesosAuthProvider) ctx = sasl.WithBindingAddress(ctx, s.Address) return ctx }, } kpl := scheduler.NewPlugin(mesosPodScheduler.NewDefaultPluginConfig(schedulerProcess.Terminal(), s.mux)) runtime.On(mesosPodScheduler.Registration(), func() { kpl.Run(schedulerProcess.Terminal()) }) runtime.On(mesosPodScheduler.Registration(), s.newServiceWriter(schedulerProcess.Terminal())) driverFactory := ha.DriverFactory(func() (drv bindings.SchedulerDriver, err error) { log.V(1).Infoln("performing deferred initialization") if err = mesosPodScheduler.Init(schedulerProcess.Master(), kpl, s.mux); err != nil { return nil, fmt.Errorf("failed to initialize pod scheduler: %v", err) } log.V(1).Infoln("deferred init complete") // defer obtaining framework ID to prevent multiple schedulers // from overwriting each other's framework IDs dconfig.Framework.Id, err = s.fetchFrameworkID(etcdClient) if err != nil { return nil, fmt.Errorf("failed to fetch framework ID from etcd: %v", err) } log.V(1).Infoln("constructing mesos scheduler driver") drv, err = bindings.NewMesosSchedulerDriver(*dconfig) if err != nil { return nil, fmt.Errorf("failed to construct scheduler driver: %v", err) } log.V(1).Infoln("constructed mesos scheduler driver:", drv) s.setDriver(drv) return drv, nil }) return schedulerProcess, driverFactory, etcdClient, eid }
func newLifecycleTest(t *testing.T) lifecycleTest { assert := &EventAssertions{*assert.New(t)} // create a fake pod watch. We use that below to submit new pods to the scheduler podsListWatch := NewMockPodsListWatch(api.PodList{}) // create fake apiserver apiServer := NewTestServer(t, api.NamespaceDefault, podsListWatch) // create ExecutorInfo with some data for static pods if set ei := mesosutil.NewExecutorInfo( mesosutil.NewExecutorID("executor-id"), mesosutil.NewCommandInfo("executor-cmd"), ) ei.Data = []byte{0, 1, 2} // create framework client := client.NewOrDie(&client.Config{ Host: apiServer.server.URL, GroupVersion: testapi.Default.GroupVersion(), }) c := *schedcfg.CreateDefaultConfig() fw := framework.New(framework.Config{ Executor: ei, Client: client, SchedulerConfig: c, LookupNode: apiServer.LookupNode, }) // TODO(sttts): re-enable the following tests // assert.NotNil(framework.client, "client is nil") // assert.NotNil(framework.executor, "executor is nil") // assert.NotNil(framework.offers, "offer registry is nil") // create pod scheduler strategy := podschedulers.NewAllocationStrategy( podtask.NewDefaultPredicate( mresource.DefaultDefaultContainerCPULimit, mresource.DefaultDefaultContainerMemLimit, ), podtask.NewDefaultProcurement( mresource.DefaultDefaultContainerCPULimit, mresource.DefaultDefaultContainerMemLimit, ), ) fcfs := podschedulers.NewFCFSPodScheduler(strategy, apiServer.LookupNode) // create scheduler process schedulerProc := ha.New(fw) // create scheduler eventObs := NewEventObserver() scheduler := components.New(&c, fw, fcfs, client, eventObs, schedulerProc.Terminal(), http.DefaultServeMux, &podsListWatch.ListWatch) assert.NotNil(scheduler) // create mock mesos scheduler driver driver := &framework.JoinableDriver{} return lifecycleTest{ apiServer: apiServer, driver: driver, eventObs: eventObs, podsListWatch: podsListWatch, framework: fw, schedulerProc: schedulerProc, sched: scheduler, t: t, } }
func newLifecycleTest(t *testing.T) lifecycleTest { assert := &EventAssertions{*assert.New(t)} // create a fake pod watch. We use that below to submit new pods to the scheduler podsListWatch := NewMockPodsListWatch(api.PodList{}) // create fake apiserver apiServer := NewTestServer(t, api.NamespaceDefault, podsListWatch) // create executor with some data for static pods if set executor := mesosutil.NewExecutorInfo( mesosutil.NewExecutorID("executor-id"), mesosutil.NewCommandInfo("executor-cmd"), ) executor.Data = []byte{0, 1, 2} // create scheduler strategy := NewAllocationStrategy( podtask.NewDefaultPredicate( mresource.DefaultDefaultContainerCPULimit, mresource.DefaultDefaultContainerMemLimit, ), podtask.NewDefaultProcurement( mresource.DefaultDefaultContainerCPULimit, mresource.DefaultDefaultContainerMemLimit, ), ) scheduler := New(Config{ Executor: executor, Client: client.NewOrDie(&client.Config{ Host: apiServer.server.URL, Version: testapi.Default.Version(), }), Scheduler: NewFCFSPodScheduler(strategy, apiServer.LookupNode), Schedcfg: *schedcfg.CreateDefaultConfig(), LookupNode: apiServer.LookupNode, }) assert.NotNil(scheduler.client, "client is nil") assert.NotNil(scheduler.executor, "executor is nil") assert.NotNil(scheduler.offers, "offer registry is nil") // create scheduler process schedulerProc := ha.New(scheduler) // get plugin config from it config := scheduler.NewPluginConfig( schedulerProc.Terminal(), http.DefaultServeMux, &podsListWatch.ListWatch, ) assert.NotNil(config) // make events observable eventObs := NewEventObserver() config.Recorder = eventObs // create plugin plugin := NewPlugin(config).(*schedulingPlugin) assert.NotNil(plugin) // create mock mesos scheduler driver driver := &joinableDriver{} return lifecycleTest{ apiServer: apiServer, driver: driver, eventObs: eventObs, plugin: plugin, podsListWatch: podsListWatch, scheduler: scheduler, schedulerProc: schedulerProc, t: t, } }