// create the driver data structure func prepareDriver(scheduler sched.Scheduler, ld LeaderDetector, fwinfo *mesos.FrameworkInfo, cred *mesos.Credential) (sched.DriverConfig, error) { if len(*master) == 0 { return sched.DriverConfig{}, fmt.Errorf("--master is a mandatory flag.") } if strings.HasPrefix(*master, "zk://") { m, err := ld.Detect(master) if err != nil { return sched.DriverConfig{}, err } master = m } bindingAddress := parseIP(*address) return sched.DriverConfig{ Scheduler: scheduler, Framework: fwinfo, Master: *master, Credential: cred, HostnameOverride: *hostname, BindingAddress: bindingAddress, BindingPort: uint16(*port), WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, *authProvider) ctx = sasl.WithBindingAddress(ctx, bindingAddress) return ctx }, }, nil }
func driverConfig(fwinfo *mesos.FrameworkInfo) sched.DriverConfig { // build command executor exec := prepareExecutorInfo() publishedAddress := parseIP(config.Runonce.Address) dConfig := sched.DriverConfig{ Scheduler: newMesosRunonceScheduler(exec), Framework: fwinfo, Master: config.Runonce.Master, Credential: cred(fwinfo), PublishedAddress: publishedAddress, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, config.Runonce.AuthProvider) ctx = sasl.WithBindingAddress(ctx, publishedAddress) return ctx }, } // Allow listening port to be configurable so we can run this inside of // mesos if desired. // NOTE only affects main PID, meaning the authentication step uses // another PID which picks a random (32K range) port :(. Opened // https://github.com/mesos/mesos-go/issues/229 to discuss. if config.Runonce.BindingPort != 0 { dConfig.BindingPort = uint16(config.Runonce.BindingPort) } return dConfig }
func main() { // build command executor exec := prepareExecutorInfo() // the framework fwinfo := &mesos.FrameworkInfo{ User: proto.String(""), // Mesos-go will fill in user. Name: proto.String("Test Framework (Go)"), } cred := (*mesos.Credential)(nil) if *mesosAuthPrincipal != "" { fwinfo.Principal = proto.String(*mesosAuthPrincipal) cred = &mesos.Credential{ Principal: proto.String(*mesosAuthPrincipal), } if *mesosAuthSecretFile != "" { _, err := os.Stat(*mesosAuthSecretFile) if err != nil { log.Fatal("missing secret file: ", err.Error()) } secret, err := ioutil.ReadFile(*mesosAuthSecretFile) if err != nil { log.Fatal("failed to read secret file: ", err.Error()) } cred.Secret = proto.String(string(secret)) } } bindingAddress := parseIP(*address) config := sched.DriverConfig{ Scheduler: newExampleScheduler(exec), Framework: fwinfo, Master: *master, Credential: cred, BindingAddress: bindingAddress, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, *authProvider) ctx = sasl.WithBindingAddress(ctx, bindingAddress) return ctx }, } driver, err := sched.NewMesosSchedulerDriver(config) if err != nil { log.Errorln("Unable to create a SchedulerDriver ", err.Error()) } if stat, err := driver.Run(); err != nil { log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error()) time.Sleep(2 * time.Second) os.Exit(1) } log.Infof("framework terminating") }
func main() { startDrone() // build command executor exec := prepareExecutorInfo() // the framework fwinfo := &mesos.FrameworkInfo{ User: proto.String(""), // Mesos-go will fill in user. Name: proto.String("Drone Framework"), } cred := (*mesos.Credential)(nil) if *mesosAuthPrincipal != "" { fwinfo.Principal = proto.String(*mesosAuthPrincipal) secret, err := ioutil.ReadFile(*mesosAuthSecretFile) if err != nil { log.Fatal(err) } cred = &mesos.Credential{ Principal: proto.String(*mesosAuthPrincipal), Secret: secret, } } fmt.Println("Address: ", address) fmt.Println("Master: ", master) bindingAddress := parseIP(address) fmt.Println("bindingAddress: ", bindingAddress) config := sched.DriverConfig{ Scheduler: newExampleScheduler(exec), Framework: fwinfo, Master: master, Credential: cred, BindingAddress: bindingAddress, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, *authProvider) ctx = sasl.WithBindingAddress(ctx, bindingAddress) return ctx }, } driver, err := sched.NewMesosSchedulerDriver(config) if err != nil { log.Errorln("Unable to create a SchedulerDriver ", err.Error()) } if stat, err := driver.Run(); err != nil { log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error()) } }
func main() { flag.Parse() var user, pass string if flag.NArg() == 1 { user = "" pass = flag.Arg(0) } else if flag.NArg() == 2 { user = flag.Arg(0) pass = flag.Arg(1) } else { printUsage() println(flag.NArg()) return } fwinfo := &mesos.FrameworkInfo{ User: proto.String(""), Name: proto.String("BTC Mining Framework (Go)"), } cred := (*mesos.Credential)(nil) if *mesosAuthPrincipal != "" { fwinfo.Principal = proto.String(*mesosAuthPrincipal) secret, err := ioutil.ReadFile(*mesosAuthSecretFile) if err != nil { log.Fatal(err) } cred = &mesos.Credential{ Principal: proto.String(*mesosAuthPrincipal), Secret: secret, } } config := sched.DriverConfig{ Scheduler: newMinerScheduler(user, pass), Framework: fwinfo, Master: *master, Credential: cred, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, *authProvider) return ctx }, } driver, err := sched.NewMesosSchedulerDriver(config) if err != nil { log.Errorln("Unable to create a SchedulerDriver ", err.Error()) } if stat, err := driver.Run(); err != nil { log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error()) } }
func InitializeScheduler() (*DiegoScheduler, *sched.MesosSchedulerDriver) { exec := prepareExecutorInfo() fwinfo := &mesos.FrameworkInfo{ User: proto.String(""), // Mesos-go will fill in user. Name: proto.String("Diego Scheduler"), } cred := (*mesos.Credential)(nil) if *mesosAuthPrincipal != "" { fwinfo.Principal = proto.String(*mesosAuthPrincipal) secret_bytes, err := ioutil.ReadFile(*mesosAuthSecretFile) secret := string(secret_bytes[:]) if err != nil { log.Fatal(err) } cred = &mesos.Credential{ Principal: proto.String(*mesosAuthPrincipal), Secret: &secret, } } bindingAddressParsed := parseIP(*bindingAddress) publishedAddressParsed := bindingAddressParsed if *publishedAddress != "" { publishedAddressParsed = parseIP(*publishedAddress) } digoScheduler := NewDiegoScheduler(exec) config := sched.DriverConfig{ Scheduler: digoScheduler, Framework: fwinfo, Master: *master, Credential: cred, BindingAddress: bindingAddressParsed, PublishedAddress: publishedAddressParsed, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, *authProvider) ctx = sasl.WithBindingAddress(ctx, bindingAddressParsed) return ctx }, } driver, err := sched.NewMesosSchedulerDriver(config) if err != nil { log.Fatal("Unable to create a SchedulerDriver ", err.Error()) } return digoScheduler, driver }
func main() { execUri, execCmd = serveExecutorArtifact(*executorPath) // the framework fwinfo := &mesos.FrameworkInfo{ User: proto.String(""), // Mesos-go will fill in user. Name: proto.String("Test Framework (Go)"), Role: proto.String(*role), } cred := (*mesos.Credential)(nil) if *mesosAuthPrincipal != "" { fwinfo.Principal = proto.String(*mesosAuthPrincipal) if *mesosAuthSecretFile != "" { secret, err := ioutil.ReadFile(*mesosAuthSecretFile) if err != nil { log.Fatal(err) } cred = &mesos.Credential{ Principal: proto.String(*mesosAuthPrincipal), Secret: secret, } } } bindingAddress := parseIP(*address) config := sched.DriverConfig{ Scheduler: newExampleScheduler(), Framework: fwinfo, Master: *master, Credential: cred, BindingAddress: bindingAddress, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, *authProvider) ctx = sasl.WithBindingAddress(ctx, bindingAddress) return ctx }, } driver, err := sched.NewMesosSchedulerDriver(config) if err != nil { log.Errorln("Unable to create a SchedulerDriver ", err.Error()) } if stat, err := driver.Run(); err != nil { log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error()) } log.Infof("framework terminating") }
func main() { // start json server go func() { fs := http.FileServer(http.Dir("static")) http.Handle("/", fs) http.HandleFunc("/json", jsonHandleFunc) http.ListenAndServe(":8888", nil) }() // build command executor // args := []string{"/bin/cat /var/tmp/1.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/2.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/3.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/intermediate.txt | /bin/grep abe > /var/tmp/grep-result.txt", "date", "/bin/rm /var/tmp/intermediate.txt"} args := []string{"/bin/cat /var/tmp/1.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/2.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/3.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/intermediate.txt | /bin/grep abe > /var/tmp/grep-result.txt", "/bin/rm /var/tmp/intermediate.txt"} args2 := []string{"/bin/cat /var/tmp/1.txt", "/bin/date"} exec := prepareExecutorInfo(args) cmdQueue.PushBack(exec) exec = prepareExecutorInfo(args2) cmdQueue.PushBack(exec) // the framework fwinfo := &mesos.FrameworkInfo{ User: proto.String(""), // Mesos-go will fill in user. Name: proto.String("Test Framework (SDC)"), } cred := (*mesos.Credential)(nil) if *mesosAuthPrincipal != "" { fwinfo.Principal = proto.String(*mesosAuthPrincipal) secret, err := ioutil.ReadFile(*mesosAuthSecretFile) if err != nil { log.Fatal(err) } cred = &mesos.Credential{ Principal: proto.String(*mesosAuthPrincipal), Secret: secret, } } bindingAddress := parseIP(*address) execinfo := cmdQueue.Remove(cmdQueue.Front()).(*mesos.ExecutorInfo) config := sched.DriverConfig{ Scheduler: newSdcScheduler(execinfo, len(execinfo.Command.Arguments)), Framework: fwinfo, Master: *master, Credential: cred, BindingAddress: bindingAddress, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, *authProvider) ctx = sasl.WithBindingAddress(ctx, bindingAddress) return ctx }, } driver, err := sched.NewMesosSchedulerDriver(config) if err != nil { log.Errorln("Unable to create a SchedulerDriver ", err.Error()) } if stat, err := driver.Run(); err != nil { log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error()) } log.Infof("framework terminating") }
func (s *SchedulerServer) bootstrap(hks hyperkube.Interface, sc *schedcfg.Config) (*ha.SchedulerProcess, ha.DriverFactory, etcd.Client, *mesos.ExecutorID) { s.frameworkName = strings.TrimSpace(s.frameworkName) if s.frameworkName == "" { log.Fatalf("framework-name must be a non-empty string") } s.frameworkWebURI = strings.TrimSpace(s.frameworkWebURI) metrics.Register() runtime.Register() s.mux.Handle("/metrics", prometheus.Handler()) healthz.InstallHandler(s.mux) if len(s.etcdServerList) == 0 { log.Fatalf("specify --etcd-servers must be specified") } if len(s.apiServerList) < 1 { log.Fatal("No api servers specified.") } clientConfig, err := s.createAPIServerClientConfig() if err != nil { log.Fatalf("Unable to make apiserver client config: %v", err) } s.client, err = clientset.NewForConfig(clientConfig) if err != nil { log.Fatalf("Unable to make apiserver clientset: %v", err) } if s.reconcileCooldown < defaultReconcileCooldown { s.reconcileCooldown = defaultReconcileCooldown log.Warningf("user-specified reconcile cooldown too small, defaulting to %v", s.reconcileCooldown) } eiPrototype, err := s.prepareExecutorInfo(hks) if err != nil { log.Fatalf("misconfigured executor: %v", err) } // TODO(jdef): remove the dependency on etcd as soon as // (1) the generic config store is available for the FrameworkId storage // (2) the generic master election is provided by the apiserver // Compare docs/proposals/high-availability.md etcdClient, err := newEtcd(s.etcdServerList) if err != nil { log.Fatalf("misconfigured etcd: %v", err) } keysAPI := etcd.NewKeysAPI(etcdClient) // mirror all nodes into the nodeStore var eiRegistry executorinfo.Registry nodesClientConfig := *clientConfig nodesClient, err := clientset.NewForConfig(&nodesClientConfig) if err != nil { log.Fatalf("Cannot create client to watch nodes: %v", err) } nodeLW := cache.NewListWatchFromClient(nodesClient.CoreClient, "nodes", api.NamespaceAll, fields.Everything()) nodeStore, nodeCtl := controllerfw.NewInformer(nodeLW, &api.Node{}, s.nodeRelistPeriod, &controllerfw.ResourceEventHandlerFuncs{ DeleteFunc: func(obj interface{}) { if eiRegistry != nil { // TODO(jdef) use controllerfw.DeletionHandlingMetaNamespaceKeyFunc at some point? nodeName := "" if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { nodeName = tombstone.Key } else if node, ok := obj.(*api.Node); ok { nodeName = node.Name } if nodeName != "" { log.V(2).Infof("deleting node %q from registry", nodeName) eiRegistry.Invalidate(nodeName) } } }, }) lookupNode := func(hostName string) *api.Node { n, _, _ := nodeStore.GetByKey(hostName) // ignore error and return nil then if n == nil { return nil } return n.(*api.Node) } execInfoCache, err := executorinfo.NewCache(defaultExecutorInfoCacheSize) if err != nil { log.Fatalf("cannot create executorinfo cache: %v", err) } eiRegistry, err = executorinfo.NewRegistry(lookupNode, eiPrototype, execInfoCache) if err != nil { log.Fatalf("cannot create executorinfo registry: %v", err) } pr := podtask.NewDefaultProcurement(eiPrototype, eiRegistry) fcfs := podschedulers.NewFCFSPodScheduler(pr, lookupNode) frameworkIDStorage, err := s.frameworkIDStorage(keysAPI) if err != nil { log.Fatalf("cannot init framework ID storage: %v", err) } framework := framework.New(framework.Config{ SchedulerConfig: *sc, Client: s.client, FailoverTimeout: s.failoverTimeout, ReconcileInterval: s.reconcileInterval, ReconcileCooldown: s.reconcileCooldown, LookupNode: lookupNode, StoreFrameworkId: frameworkIDStorage.Set, ExecutorId: eiPrototype.GetExecutorId(), }) masterUri := s.mesosMaster info, cred, err := s.buildFrameworkInfo() if err != nil { log.Fatalf("Misconfigured mesos framework: %v", err) } schedulerProcess := ha.New(framework) // try publishing on the same IP as the slave var publishedAddress net.IP if libprocessIP := os.Getenv("LIBPROCESS_IP"); libprocessIP != "" { publishedAddress = net.ParseIP(libprocessIP) } if publishedAddress != nil { log.V(1).Infof("driver will publish address %v", publishedAddress) } dconfig := &bindings.DriverConfig{ Scheduler: schedulerProcess, Framework: info, Master: masterUri, Credential: cred, BindingAddress: s.address, BindingPort: uint16(s.driverPort), PublishedAddress: publishedAddress, HostnameOverride: s.hostnameOverride, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, s.mesosAuthProvider) ctx = sasl.WithBindingAddress(ctx, s.address) return ctx }, } // create event recorder sending events to the "" namespace of the apiserver eventsClientConfig := *clientConfig eventsClient, err := clientset.NewForConfig(&eventsClientConfig) if err != nil { log.Fatalf("Invalid API configuration: %v", err) } broadcaster := record.NewBroadcaster() recorder := broadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName}) broadcaster.StartLogging(log.Infof) broadcaster.StartRecordingToSink(&unversionedcore.EventSinkImpl{Interface: eventsClient.Events("")}) lw := cache.NewListWatchFromClient(s.client.CoreClient, "pods", api.NamespaceAll, fields.Everything()) hostPortStrategy := hostport.StrategyFixed if s.useHostPortEndpoints { hostPortStrategy = hostport.StrategyWildcard } // create scheduler core with all components arranged around it sched := components.New( sc, framework, fcfs, s.client, recorder, schedulerProcess.Terminal(), s.mux, lw, podtask.Config{ DefaultPodRoles: s.defaultPodRoles, FrameworkRoles: s.frameworkRoles, GenerateTaskDiscoveryEnabled: s.generateTaskDiscovery, HostPortStrategy: hostPortStrategy, Prototype: eiPrototype, }, s.defaultContainerCPULimit, s.defaultContainerMemLimit, ) runtime.On(framework.Registration(), func() { sched.Run(schedulerProcess.Terminal()) }) runtime.On(framework.Registration(), s.newServiceWriter(publishedAddress, schedulerProcess.Terminal())) runtime.On(framework.Registration(), func() { nodeCtl.Run(schedulerProcess.Terminal()) }) driverFactory := ha.DriverFactory(func() (drv bindings.SchedulerDriver, err error) { log.V(1).Infoln("performing deferred initialization") if err = framework.Init(sched, schedulerProcess.Master(), s.mux); err != nil { return nil, fmt.Errorf("failed to initialize pod scheduler: %v", err) } log.V(1).Infoln("deferred init complete") if s.failoverTimeout > 0 { // defer obtaining framework ID to prevent multiple schedulers // from overwriting each other's framework IDs var frameworkID string frameworkID, err = frameworkIDStorage.Get(context.TODO()) if err != nil { return nil, fmt.Errorf("failed to fetch framework ID from storage: %v", err) } if frameworkID != "" { log.Infof("configuring FrameworkInfo with ID found in storage: %q", frameworkID) dconfig.Framework.Id = &mesos.FrameworkID{Value: &frameworkID} } else { log.V(1).Infof("did not find framework ID in storage") } } else { // TODO(jdef) this is a hack, really for development, to simplify clean up of old framework IDs frameworkIDStorage.Remove(context.TODO()) } log.V(1).Infoln("constructing mesos scheduler driver") drv, err = bindings.NewMesosSchedulerDriver(*dconfig) if err != nil { return nil, fmt.Errorf("failed to construct scheduler driver: %v", err) } log.V(1).Infoln("constructed mesos scheduler driver:", drv) s.setDriver(drv) return drv, nil }) return schedulerProcess, driverFactory, etcdClient, eiPrototype.GetExecutorId() }
func (s *SchedulerServer) bootstrap(hks hyperkube.Interface, sc *schedcfg.Config) (*ha.SchedulerProcess, ha.DriverFactory, tools.EtcdGetSet, *uid.UID) { s.FrameworkName = strings.TrimSpace(s.FrameworkName) if s.FrameworkName == "" { log.Fatalf("framework-name must be a non-empty string") } s.FrameworkWebURI = strings.TrimSpace(s.FrameworkWebURI) metrics.Register() runtime.Register() s.mux.Handle("/metrics", prometheus.Handler()) if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) { log.Fatalf("specify either --etcd-servers or --etcd-config") } if len(s.APIServerList) < 1 { log.Fatal("No api servers specified.") } client, err := s.createAPIServerClient() if err != nil { log.Fatalf("Unable to make apiserver client: %v", err) } s.client = client if s.ReconcileCooldown < defaultReconcileCooldown { s.ReconcileCooldown = defaultReconcileCooldown log.Warningf("user-specified reconcile cooldown too small, defaulting to %v", s.ReconcileCooldown) } executor, eid, err := s.prepareExecutorInfo(hks) if err != nil { log.Fatalf("misconfigured executor: %v", err) } // TODO(jdef): remove the dependency on etcd as soon as // (1) the generic config store is available for the FrameworkId storage // (2) the generic master election is provided by the apiserver // Compare docs/proposals/high-availability.md etcdClient, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList) if err != nil { log.Fatalf("misconfigured etcd: %v", err) } mesosPodScheduler := scheduler.New(scheduler.Config{ Schedcfg: *sc, Executor: executor, ScheduleFunc: scheduler.FCFSScheduleFunc, Client: client, EtcdClient: etcdClient, FailoverTimeout: s.FailoverTimeout, ReconcileInterval: s.ReconcileInterval, ReconcileCooldown: s.ReconcileCooldown, }) masterUri := s.MesosMaster info, cred, err := s.buildFrameworkInfo() if err != nil { log.Fatalf("Misconfigured mesos framework: %v", err) } schedulerProcess := ha.New(mesosPodScheduler) dconfig := &bindings.DriverConfig{ Scheduler: schedulerProcess, Framework: info, Master: masterUri, Credential: cred, BindingAddress: net.IP(s.Address), BindingPort: uint16(s.DriverPort), HostnameOverride: s.HostnameOverride, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, s.MesosAuthProvider) ctx = sasl.WithBindingAddress(ctx, net.IP(s.Address)) return ctx }, } kpl := scheduler.NewPlugin(mesosPodScheduler.NewDefaultPluginConfig(schedulerProcess.Terminal(), s.mux)) runtime.On(mesosPodScheduler.Registration(), func() { kpl.Run(schedulerProcess.Terminal()) }) runtime.On(mesosPodScheduler.Registration(), s.newServiceWriter(schedulerProcess.Terminal())) driverFactory := ha.DriverFactory(func() (drv bindings.SchedulerDriver, err error) { log.V(1).Infoln("performing deferred initialization") if err = mesosPodScheduler.Init(schedulerProcess.Master(), kpl, s.mux); err != nil { return nil, fmt.Errorf("failed to initialize pod scheduler: %v", err) } log.V(1).Infoln("deferred init complete") // defer obtaining framework ID to prevent multiple schedulers // from overwriting each other's framework IDs dconfig.Framework.Id, err = s.fetchFrameworkID(etcdClient) if err != nil { return nil, fmt.Errorf("failed to fetch framework ID from etcd: %v", err) } log.V(1).Infoln("constructing mesos scheduler driver") drv, err = bindings.NewMesosSchedulerDriver(*dconfig) if err != nil { return nil, fmt.Errorf("failed to construct scheduler driver: %v", err) } log.V(1).Infoln("constructed mesos scheduler driver:", drv) s.setDriver(drv) return drv, nil }) return schedulerProcess, driverFactory, etcdClient, eid }
func (s *SchedulerServer) bootstrap(hks hyperkube.Interface, sc *schedcfg.Config) (*ha.SchedulerProcess, ha.DriverFactory, tools.EtcdClient, *uid.UID) { s.FrameworkName = strings.TrimSpace(s.FrameworkName) if s.FrameworkName == "" { log.Fatalf("framework-name must be a non-empty string") } s.FrameworkWebURI = strings.TrimSpace(s.FrameworkWebURI) metrics.Register() runtime.Register() s.mux.Handle("/metrics", prometheus.Handler()) healthz.InstallHandler(s.mux) if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) { log.Fatalf("specify either --etcd-servers or --etcd-config") } if len(s.APIServerList) < 1 { log.Fatal("No api servers specified.") } client, err := s.createAPIServerClient() if err != nil { log.Fatalf("Unable to make apiserver client: %v", err) } s.client = client if s.ReconcileCooldown < defaultReconcileCooldown { s.ReconcileCooldown = defaultReconcileCooldown log.Warningf("user-specified reconcile cooldown too small, defaulting to %v", s.ReconcileCooldown) } executor, eid, err := s.prepareExecutorInfo(hks) if err != nil { log.Fatalf("misconfigured executor: %v", err) } // TODO(jdef): remove the dependency on etcd as soon as // (1) the generic config store is available for the FrameworkId storage // (2) the generic master election is provided by the apiserver // Compare docs/proposals/high-availability.md etcdClient, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList) if err != nil { log.Fatalf("misconfigured etcd: %v", err) } as := scheduler.NewAllocationStrategy( podtask.NewDefaultPredicate( s.DefaultContainerCPULimit, s.DefaultContainerMemLimit, ), podtask.NewDefaultProcurement( s.DefaultContainerCPULimit, s.DefaultContainerMemLimit, ), ) // downgrade allocation strategy if user disables "account-for-pod-resources" if !s.AccountForPodResources { as = scheduler.NewAllocationStrategy( podtask.DefaultMinimalPredicate, podtask.DefaultMinimalProcurement) } // mirror all nodes into the nodeStore nodesClient, err := s.createAPIServerClient() if err != nil { log.Fatalf("Cannot create client to watch nodes: %v", err) } nodeStore := cache.NewStore(cache.MetaNamespaceKeyFunc) nodeLW := cache.NewListWatchFromClient(nodesClient, "nodes", api.NamespaceAll, fields.Everything()) cache.NewReflector(nodeLW, &api.Node{}, nodeStore, s.nodeRelistPeriod).Run() lookupNode := func(hostName string) *api.Node { n, _, _ := nodeStore.GetByKey(hostName) // ignore error and return nil then if n == nil { return nil } return n.(*api.Node) } fcfs := scheduler.NewFCFSPodScheduler(as, lookupNode) mesosPodScheduler := scheduler.New(scheduler.Config{ Schedcfg: *sc, Executor: executor, Scheduler: fcfs, Client: client, EtcdClient: etcdClient, FailoverTimeout: s.FailoverTimeout, ReconcileInterval: s.ReconcileInterval, ReconcileCooldown: s.ReconcileCooldown, LookupNode: lookupNode, }) masterUri := s.MesosMaster info, cred, err := s.buildFrameworkInfo() if err != nil { log.Fatalf("Misconfigured mesos framework: %v", err) } schedulerProcess := ha.New(mesosPodScheduler) dconfig := &bindings.DriverConfig{ Scheduler: schedulerProcess, Framework: info, Master: masterUri, Credential: cred, BindingAddress: s.Address, BindingPort: uint16(s.DriverPort), HostnameOverride: s.HostnameOverride, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, s.MesosAuthProvider) ctx = sasl.WithBindingAddress(ctx, s.Address) return ctx }, } kpl := scheduler.NewPlugin(mesosPodScheduler.NewDefaultPluginConfig(schedulerProcess.Terminal(), s.mux)) runtime.On(mesosPodScheduler.Registration(), func() { kpl.Run(schedulerProcess.Terminal()) }) runtime.On(mesosPodScheduler.Registration(), s.newServiceWriter(schedulerProcess.Terminal())) driverFactory := ha.DriverFactory(func() (drv bindings.SchedulerDriver, err error) { log.V(1).Infoln("performing deferred initialization") if err = mesosPodScheduler.Init(schedulerProcess.Master(), kpl, s.mux); err != nil { return nil, fmt.Errorf("failed to initialize pod scheduler: %v", err) } log.V(1).Infoln("deferred init complete") // defer obtaining framework ID to prevent multiple schedulers // from overwriting each other's framework IDs dconfig.Framework.Id, err = s.fetchFrameworkID(etcdClient) if err != nil { return nil, fmt.Errorf("failed to fetch framework ID from etcd: %v", err) } log.V(1).Infoln("constructing mesos scheduler driver") drv, err = bindings.NewMesosSchedulerDriver(*dconfig) if err != nil { return nil, fmt.Errorf("failed to construct scheduler driver: %v", err) } log.V(1).Infoln("constructed mesos scheduler driver:", drv) s.setDriver(drv) return drv, nil }) return schedulerProcess, driverFactory, etcdClient, eid }
func (sc *SchedulerCore) Run(mesosMaster string) { sc.schedulerState.MesosMaster = mesosMaster var frameworkId *mesos.FrameworkID if sc.schedulerState.FrameworkID == nil { frameworkId = nil } else { frameworkId = &mesos.FrameworkID{ Value: sc.schedulerState.FrameworkID, } } fwinfo := &mesos.FrameworkInfo{ Name: proto.String(sc.frameworkName), Id: frameworkId, FailoverTimeout: proto.Float64(86400), WebuiUrl: proto.String(sc.schedulerHTTPServer.GetURI()), Checkpoint: proto.Bool(true), Role: proto.String(sc.frameworkRole), } if sc.user != "" { fwinfo.User = proto.String(sc.user) } else { guestUser := "******" fwinfo.User = &guestUser } cred := (*mesos.Credential)(nil) if sc.mesosAuthPrincipal != "" { fwinfo.Principal = proto.String(sc.mesosAuthPrincipal) } if sc.mesosAuthSecretFile != "" && sc.mesosAuthPrincipal != "" { secret, err := ioutil.ReadFile(sc.mesosAuthSecretFile) if err != nil { log.Fatal(err) } cred = &mesos.Credential{ Principal: proto.String(sc.mesosAuthPrincipal), Secret: secret, } } hostname, err := os.Hostname() if err != nil { log.Fatal(err) } config := sched.DriverConfig{ Scheduler: sc, Framework: fwinfo, Master: mesosMaster, Credential: cred, HostnameOverride: hostname, } if sc.schedulerIPAddr != "" { config.BindingAddress = parseIP(sc.schedulerIPAddr) } if sc.authProvider != "" && sc.mesosAuthPrincipal != "" { config.WithAuthContext = func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, sc.authProvider) if sc.schedulerIPAddr != "" { ctx = sasl.WithBindingAddress(ctx, parseIP(sc.schedulerIPAddr)) } return ctx } } log.Infof("Running scheduler with FrameworkInfo: %v and DriverConfig: %v", fwinfo, config) driver, err := sched.NewMesosSchedulerDriver(config) if err != nil { log.Error("Unable to create a SchedulerDriver ", err.Error()) } sc.rServer = newReconciliationServer(driver, sc) sc.mgr.SetupFramework(sc.schedulerHTTPServer.URI) if stat, err := driver.Run(); err != nil { log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error()) } }
func main() { if *ghsNode { ghsNodeMain() return } // Allocate a port for GRPC lis, err := net.Listen("tcp", ":0") if err != nil { log.Fatalf("grpc failed to listen: %v", err) } grpcServer := grpc.NewServer() // build command executor nexec := prepareExecutorInfo(lis.Addr()) s := newvisghsScheduler(nexec) t := tracer.NewHTTPDisplay(http.DefaultServeMux, s.OnRun) server := tracer.NewGRPCServer(t) tracer.RegisterTraceServiceServer(grpcServer, server) go grpcServer.Serve(lis) time.Sleep(2 * time.Second) go http.ListenAndServe(":12345", nil) // the framework fwinfo := &mesos.FrameworkInfo{ User: proto.String(""), // Mesos-go will fill in user. Name: proto.String("visghs"), } cred := (*mesos.Credential)(nil) if *mesosAuthPrincipal != "" { fwinfo.Principal = proto.String(*mesosAuthPrincipal) cred = &mesos.Credential{ Principal: proto.String(*mesosAuthPrincipal), } if *mesosAuthSecretFile != "" { _, err := os.Stat(*mesosAuthSecretFile) if err != nil { log.Fatal("missing secret file: ", err.Error()) } secret, err := ioutil.ReadFile(*mesosAuthSecretFile) if err != nil { log.Fatal("failed to read secret file: ", err.Error()) } cred.Secret = proto.String(string(secret)) } } bindingAddress := parseIP(*address) config := sched.DriverConfig{ Scheduler: s, Framework: fwinfo, Master: *master, Credential: cred, BindingAddress: bindingAddress, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, *authProvider) ctx = sasl.WithBindingAddress(ctx, bindingAddress) return ctx }, } driver, err := sched.NewMesosSchedulerDriver(config) if err != nil { log.Errorln("Unable to create a SchedulerDriver ", err.Error()) } if stat, err := driver.Run(); err != nil { log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error()) time.Sleep(2 * time.Second) os.Exit(1) } log.Infof("framework terminating") }
func (s *SchedulerServer) bootstrap(hks hyperkube.Interface, sc *schedcfg.Config) (*ha.SchedulerProcess, ha.DriverFactory, *etcd.Client, *mesos.ExecutorID) { s.frameworkName = strings.TrimSpace(s.frameworkName) if s.frameworkName == "" { log.Fatalf("framework-name must be a non-empty string") } s.frameworkWebURI = strings.TrimSpace(s.frameworkWebURI) metrics.Register() runtime.Register() s.mux.Handle("/metrics", prometheus.Handler()) healthz.InstallHandler(s.mux) if (s.etcdConfigFile != "" && len(s.etcdServerList) != 0) || (s.etcdConfigFile == "" && len(s.etcdServerList) == 0) { log.Fatalf("specify either --etcd-servers or --etcd-config") } if len(s.apiServerList) < 1 { log.Fatal("No api servers specified.") } client, err := s.createAPIServerClient() if err != nil { log.Fatalf("Unable to make apiserver client: %v", err) } s.client = client if s.reconcileCooldown < defaultReconcileCooldown { s.reconcileCooldown = defaultReconcileCooldown log.Warningf("user-specified reconcile cooldown too small, defaulting to %v", s.reconcileCooldown) } eiPrototype, err := s.prepareExecutorInfo(hks) if err != nil { log.Fatalf("misconfigured executor: %v", err) } // TODO(jdef): remove the dependency on etcd as soon as // (1) the generic config store is available for the FrameworkId storage // (2) the generic master election is provided by the apiserver // Compare docs/proposals/high-availability.md etcdClient, err := newEtcd(s.etcdConfigFile, s.etcdServerList) if err != nil { log.Fatalf("misconfigured etcd: %v", err) } // mirror all nodes into the nodeStore var eiRegistry executorinfo.Registry nodesClient, err := s.createAPIServerClient() if err != nil { log.Fatalf("Cannot create client to watch nodes: %v", err) } nodeLW := cache.NewListWatchFromClient(nodesClient, "nodes", api.NamespaceAll, fields.Everything()) nodeStore, nodeCtl := controllerfw.NewInformer(nodeLW, &api.Node{}, s.nodeRelistPeriod, &controllerfw.ResourceEventHandlerFuncs{ DeleteFunc: func(obj interface{}) { node := obj.(*api.Node) if eiRegistry != nil { log.V(2).Infof("deleting node %q from registry", node.Name) eiRegistry.Invalidate(node.Name) } }, }) lookupNode := func(hostName string) *api.Node { n, _, _ := nodeStore.GetByKey(hostName) // ignore error and return nil then if n == nil { return nil } return n.(*api.Node) } execInfoCache, err := executorinfo.NewCache(defaultExecutorInfoCacheSize) if err != nil { log.Fatalf("cannot create executorinfo cache: %v", err) } eiRegistry, err = executorinfo.NewRegistry(lookupNode, eiPrototype, execInfoCache) if err != nil { log.Fatalf("cannot create executorinfo registry: %v", err) } pr := podtask.NewDefaultProcurement(eiPrototype, eiRegistry) fcfs := podschedulers.NewFCFSPodScheduler(pr, lookupNode) framework := framework.New(framework.Config{ SchedulerConfig: *sc, Client: client, FailoverTimeout: s.failoverTimeout, ReconcileInterval: s.reconcileInterval, ReconcileCooldown: s.reconcileCooldown, LookupNode: lookupNode, StoreFrameworkId: func(id string) { // TODO(jdef): port FrameworkId store to generic Kubernetes config store as soon as available _, err := etcdClient.Set(meta.FrameworkIDKey, id, uint64(s.failoverTimeout)) if err != nil { log.Errorf("failed to renew frameworkId TTL: %v", err) } }, ExecutorId: eiPrototype.GetExecutorId(), }) masterUri := s.mesosMaster info, cred, err := s.buildFrameworkInfo() if err != nil { log.Fatalf("Misconfigured mesos framework: %v", err) } schedulerProcess := ha.New(framework) dconfig := &bindings.DriverConfig{ Scheduler: schedulerProcess, Framework: info, Master: masterUri, Credential: cred, BindingAddress: s.address, BindingPort: uint16(s.driverPort), HostnameOverride: s.hostnameOverride, WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, s.mesosAuthProvider) ctx = sasl.WithBindingAddress(ctx, s.address) return ctx }, } // create event recorder sending events to the "" namespace of the apiserver broadcaster := record.NewBroadcaster() recorder := broadcaster.NewRecorder(api.EventSource{Component: "scheduler"}) broadcaster.StartRecordingToSink(client.Events("")) // create scheduler core with all components arranged around it lw := cache.NewListWatchFromClient(client, "pods", api.NamespaceAll, fields.Everything()) sched := components.New( sc, framework, fcfs, client, recorder, schedulerProcess.Terminal(), s.mux, lw, eiPrototype, s.mesosRoles, s.defaultContainerCPULimit, s.defaultContainerMemLimit, ) runtime.On(framework.Registration(), func() { sched.Run(schedulerProcess.Terminal()) }) runtime.On(framework.Registration(), s.newServiceWriter(schedulerProcess.Terminal())) runtime.On(framework.Registration(), func() { nodeCtl.Run(schedulerProcess.Terminal()) }) driverFactory := ha.DriverFactory(func() (drv bindings.SchedulerDriver, err error) { log.V(1).Infoln("performing deferred initialization") if err = framework.Init(sched, schedulerProcess.Master(), s.mux); err != nil { return nil, fmt.Errorf("failed to initialize pod scheduler: %v", err) } log.V(1).Infoln("deferred init complete") // defer obtaining framework ID to prevent multiple schedulers // from overwriting each other's framework IDs dconfig.Framework.Id, err = s.fetchFrameworkID(etcdClient) if err != nil { return nil, fmt.Errorf("failed to fetch framework ID from etcd: %v", err) } log.V(1).Infoln("constructing mesos scheduler driver") drv, err = bindings.NewMesosSchedulerDriver(*dconfig) if err != nil { return nil, fmt.Errorf("failed to construct scheduler driver: %v", err) } log.V(1).Infoln("constructed mesos scheduler driver:", drv) s.setDriver(drv) return drv, nil }) return schedulerProcess, driverFactory, etcdClient, eiPrototype.GetExecutorId() }
func main() { frameworkName := flag.String("framework-name", "etcd", "Unique name of this etcd cluster") master := flag.String("master", "127.0.0.1:5050", "Master address <ip:port>") zkFrameworkPersist := flag.String("zk-framework-persist", "", "Zookeeper URI of the form zk://host1:port1,host2:port2/chroot/path") taskCount := flag.Int("cluster-size", 5, "Total task count to run") adminPort := flag.Int("admin-port", 23400, "Binding port for admin interface") reseedTimeout := flag.Int("reseed-timeout", 240, "Seconds of etcd livelock to wait for before attempting a cluster re-seed") autoReseed := flag.Bool("auto-reseed", true, "Perform automatic cluster reseed when the "+ "cluster has been livelocked for -reseed-timeout seconds") artifactPort := flag.Int("artifact-port", 12300, "Binding port for artifact server") sandboxDisk := flag.Float64("sandbox-disk-limit", 4096, "Max disk usage for the etcd mesos sandbox in MB") sandboxCpu := flag.Float64("sandbox-cpu-limit", 4, "Max cpu usage for the etcd mesos sandbox in MB") sandboxMem := flag.Float64("sandbox-mem-limit", 2048, "Max memory usage for the etcd mesos sandbox in MB") executorPath := flag.String("executor-bin", "./bin/etcd-mesos-executor", "Path to executor binary") etcdPath := flag.String("etcd-bin", "./bin/etcd", "Path to etcd binary") etcdctlPath := flag.String("etcdctl-bin", "./bin/etcdctl", "Path to etcdctl binary") address := flag.String("address", "", "Binding address for scheduler and artifact server") driverPort := flag.Int("driver-port", 0, "Binding port for scheduler driver") mesosAuthPrincipal := flag.String("mesos-authentication-principal", "", "Mesos authentication principal") mesosAuthSecretFile := flag.String("mesos-authentication-secret-file", "", "Mesos authentication secret file") authProvider := flag.String("mesos-authentication-provider", sasl.ProviderName, fmt.Sprintf("Authentication provider to use, default is SASL that supports mechanisms: %+v", mech.ListSupported())) singleInstancePerSlave := flag.Bool("single-instance-per-slave", true, "Only allow one etcd instance to be started per slave") failoverTimeoutSeconds := flag.Float64("failover-timeout-seconds", 60*60*24*7, "Mesos framework failover timeout in seconds") weburi := flag.String("framework-weburi", "", "A URI that points to a web-based interface for interacting with the framework.") flag.Parse() if *zkFrameworkPersist == "" { log.Fatal("No value provided for -zk-framework-persist !") } if !*singleInstancePerSlave { log.Warning("-single-instance-per-slave=false is dangerous because it may lead to " + "multiple etcd instances in the same cluster on a single node, amplifying " + "the cost of a single node being lost, livelock, and data loss.") } if *address == "" { hostname, err := os.Hostname() if err == nil { *address = hostname } else { log.Errorf("Could not set default binding to hostname. Defaulting to 127.0.0.1") *address = "127.0.0.1" } } if *weburi == "" { *weburi = fmt.Sprintf("http://%s:%d/", *address, *adminPort) } executorUris := []*mesos.CommandInfo_URI{} execUri, err := etcdscheduler.ServeExecutorArtifact(*executorPath, *address, *artifactPort) if err != nil { log.Errorf("Could not stat executor binary: %v", err) return } executorUris = append(executorUris, &mesos.CommandInfo_URI{ Value: execUri, Executable: proto.Bool(true), }) etcdUri, err := etcdscheduler.ServeExecutorArtifact(*etcdPath, *address, *artifactPort) if err != nil { log.Errorf("Could not stat etcd binary: %v", err) return } executorUris = append(executorUris, &mesos.CommandInfo_URI{ Value: etcdUri, Executable: proto.Bool(true), }) etcdctlUri, err := etcdscheduler.ServeExecutorArtifact(*etcdctlPath, *address, *artifactPort) if err != nil { log.Errorf("Could not stat etcd binary: %v", err) return } executorUris = append(executorUris, &mesos.CommandInfo_URI{ Value: etcdctlUri, Executable: proto.Bool(true), }) go http.ListenAndServe(fmt.Sprintf("%s:%d", *address, *artifactPort), nil) log.V(2).Info("Serving executor artifacts...") bindingAddress := parseIP(*address) // chillFactor is the number of seconds that are slept for to allow for // convergence across the cluster during mutations. chillFactor := 10 etcdScheduler := etcdscheduler.NewEtcdScheduler( *taskCount, chillFactor, *reseedTimeout, *autoReseed, executorUris, *singleInstancePerSlave, *sandboxDisk, *sandboxCpu, *sandboxMem, ) etcdScheduler.ExecutorPath = *executorPath etcdScheduler.Master = *master etcdScheduler.FrameworkName = *frameworkName etcdScheduler.ZkConnect = *zkFrameworkPersist fwinfo := &mesos.FrameworkInfo{ User: proto.String(""), // Mesos-go will fill in user. Name: proto.String(*frameworkName), Checkpoint: proto.Bool(true), FailoverTimeout: proto.Float64(*failoverTimeoutSeconds), WebuiUrl: proto.String(*weburi), } cred := (*mesos.Credential)(nil) if *mesosAuthPrincipal != "" { fwinfo.Principal = proto.String(*mesosAuthPrincipal) secret, err := ioutil.ReadFile(*mesosAuthSecretFile) if err != nil { log.Fatal(err) } cred = &mesos.Credential{ Principal: proto.String(*mesosAuthPrincipal), Secret: secret, } } zkServers, zkChroot, err := rpc.ParseZKURI(*zkFrameworkPersist) etcdScheduler.ZkServers = zkServers etcdScheduler.ZkChroot = zkChroot if err != nil && *zkFrameworkPersist != "" { log.Fatalf("Error parsing zookeeper URI of %s: %s", *zkFrameworkPersist, err) } else if *zkFrameworkPersist != "" { previous, err := rpc.GetPreviousFrameworkID( zkServers, zkChroot, etcdScheduler.FrameworkName, ) if err != nil && err != zk.ErrNoNode { log.Fatalf("Could not retrieve previous framework ID: %s", err) } else if err == zk.ErrNoNode { log.Info("No previous persisted framework ID exists in zookeeper.") } else { log.Infof("Found stored framework ID in Zookeeper, "+ "attempting to re-use: %s", previous) fwinfo.Id = &mesos.FrameworkID{ Value: proto.String(previous), } } } config := scheduler.DriverConfig{ Scheduler: etcdScheduler, Framework: fwinfo, Master: etcdScheduler.Master, Credential: cred, BindingAddress: bindingAddress, BindingPort: uint16(*driverPort), WithAuthContext: func(ctx context.Context) context.Context { ctx = auth.WithLoginProvider(ctx, *authProvider) ctx = sasl.WithBindingAddress(ctx, bindingAddress) return ctx }, } driver, err := scheduler.NewMesosSchedulerDriver(config) if err != nil { log.Errorln("Unable to create a SchedulerDriver ", err.Error()) } go etcdScheduler.SerialLauncher(driver) go etcdScheduler.PeriodicReconciler(driver) go etcdScheduler.PeriodicHealthChecker() go etcdScheduler.PeriodicLaunchRequestor() go etcdScheduler.AdminHTTP(*adminPort, driver) if stat, err := driver.Run(); err != nil { log.Infof("Framework stopped with status %s and error: %s", stat.String(), err.Error()) } }
func getAuthContext(ctx context.Context) context.Context { return auth.WithLoginProvider(ctx, "SASL") }