// create the driver data structure
func prepareDriver(scheduler sched.Scheduler, ld LeaderDetector, fwinfo *mesos.FrameworkInfo, cred *mesos.Credential) (sched.DriverConfig, error) {
if len(*master) == 0 {
return sched.DriverConfig{}, fmt.Errorf("--master is a mandatory flag.")
}
if strings.HasPrefix(*master, "zk://") {
m, err := ld.Detect(master)
if err != nil {
return sched.DriverConfig{}, err
}
master = m
}
bindingAddress := parseIP(*address)
return sched.DriverConfig{
Scheduler: scheduler,
Framework: fwinfo,
Master: *master,
Credential: cred,
HostnameOverride: *hostname,
BindingAddress: bindingAddress,
BindingPort: uint16(*port),
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, *authProvider)
ctx = sasl.WithBindingAddress(ctx, bindingAddress)
return ctx
},
}, nil
}
func driverConfig(fwinfo *mesos.FrameworkInfo) sched.DriverConfig {
// build command executor
exec := prepareExecutorInfo()
publishedAddress := parseIP(config.Runonce.Address)
dConfig := sched.DriverConfig{
Scheduler: newMesosRunonceScheduler(exec),
Framework: fwinfo,
Master: config.Runonce.Master,
Credential: cred(fwinfo),
PublishedAddress: publishedAddress,
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, config.Runonce.AuthProvider)
ctx = sasl.WithBindingAddress(ctx, publishedAddress)
return ctx
},
}
// Allow listening port to be configurable so we can run this inside of
// mesos if desired.
// NOTE only affects main PID, meaning the authentication step uses
// another PID which picks a random (32K range) port :(. Opened
// https://github.com/mesos/mesos-go/issues/229 to discuss.
if config.Runonce.BindingPort != 0 {
dConfig.BindingPort = uint16(config.Runonce.BindingPort)
}
return dConfig
}
func main() {
// build command executor
exec := prepareExecutorInfo()
// the framework
fwinfo := &mesos.FrameworkInfo{
User: proto.String(""), // Mesos-go will fill in user.
Name: proto.String("Test Framework (Go)"),
}
cred := (*mesos.Credential)(nil)
if *mesosAuthPrincipal != "" {
fwinfo.Principal = proto.String(*mesosAuthPrincipal)
cred = &mesos.Credential{
Principal: proto.String(*mesosAuthPrincipal),
}
if *mesosAuthSecretFile != "" {
_, err := os.Stat(*mesosAuthSecretFile)
if err != nil {
log.Fatal("missing secret file: ", err.Error())
}
secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
if err != nil {
log.Fatal("failed to read secret file: ", err.Error())
}
cred.Secret = proto.String(string(secret))
}
}
bindingAddress := parseIP(*address)
config := sched.DriverConfig{
Scheduler: newExampleScheduler(exec),
Framework: fwinfo,
Master: *master,
Credential: cred,
BindingAddress: bindingAddress,
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, *authProvider)
ctx = sasl.WithBindingAddress(ctx, bindingAddress)
return ctx
},
}
driver, err := sched.NewMesosSchedulerDriver(config)
if err != nil {
log.Errorln("Unable to create a SchedulerDriver ", err.Error())
}
if stat, err := driver.Run(); err != nil {
log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
time.Sleep(2 * time.Second)
os.Exit(1)
}
log.Infof("framework terminating")
}
func main() {
startDrone()
// build command executor
exec := prepareExecutorInfo()
// the framework
fwinfo := &mesos.FrameworkInfo{
User: proto.String(""), // Mesos-go will fill in user.
Name: proto.String("Drone Framework"),
}
cred := (*mesos.Credential)(nil)
if *mesosAuthPrincipal != "" {
fwinfo.Principal = proto.String(*mesosAuthPrincipal)
secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
if err != nil {
log.Fatal(err)
}
cred = &mesos.Credential{
Principal: proto.String(*mesosAuthPrincipal),
Secret: secret,
}
}
fmt.Println("Address: ", address)
fmt.Println("Master: ", master)
bindingAddress := parseIP(address)
fmt.Println("bindingAddress: ", bindingAddress)
config := sched.DriverConfig{
Scheduler: newExampleScheduler(exec),
Framework: fwinfo,
Master: master,
Credential: cred,
BindingAddress: bindingAddress,
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, *authProvider)
ctx = sasl.WithBindingAddress(ctx, bindingAddress)
return ctx
},
}
driver, err := sched.NewMesosSchedulerDriver(config)
if err != nil {
log.Errorln("Unable to create a SchedulerDriver ", err.Error())
}
if stat, err := driver.Run(); err != nil {
log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
}
}
func InitializeScheduler() (*DiegoScheduler, *sched.MesosSchedulerDriver) {
exec := prepareExecutorInfo()
fwinfo := &mesos.FrameworkInfo{
User: proto.String(""), // Mesos-go will fill in user.
Name: proto.String("Diego Scheduler"),
}
cred := (*mesos.Credential)(nil)
if *mesosAuthPrincipal != "" {
fwinfo.Principal = proto.String(*mesosAuthPrincipal)
secret_bytes, err := ioutil.ReadFile(*mesosAuthSecretFile)
secret := string(secret_bytes[:])
if err != nil {
log.Fatal(err)
}
cred = &mesos.Credential{
Principal: proto.String(*mesosAuthPrincipal),
Secret: &secret,
}
}
bindingAddressParsed := parseIP(*bindingAddress)
publishedAddressParsed := bindingAddressParsed
if *publishedAddress != "" {
publishedAddressParsed = parseIP(*publishedAddress)
}
digoScheduler := NewDiegoScheduler(exec)
config := sched.DriverConfig{
Scheduler: digoScheduler,
Framework: fwinfo,
Master: *master,
Credential: cred,
BindingAddress: bindingAddressParsed,
PublishedAddress: publishedAddressParsed,
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, *authProvider)
ctx = sasl.WithBindingAddress(ctx, bindingAddressParsed)
return ctx
},
}
driver, err := sched.NewMesosSchedulerDriver(config)
if err != nil {
log.Fatal("Unable to create a SchedulerDriver ", err.Error())
}
return digoScheduler, driver
}
func main() {
execUri, execCmd = serveExecutorArtifact(*executorPath)
// the framework
fwinfo := &mesos.FrameworkInfo{
User: proto.String(""), // Mesos-go will fill in user.
Name: proto.String("Test Framework (Go)"),
Role: proto.String(*role),
}
cred := (*mesos.Credential)(nil)
if *mesosAuthPrincipal != "" {
fwinfo.Principal = proto.String(*mesosAuthPrincipal)
if *mesosAuthSecretFile != "" {
secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
if err != nil {
log.Fatal(err)
}
cred = &mesos.Credential{
Principal: proto.String(*mesosAuthPrincipal),
Secret: secret,
}
}
}
bindingAddress := parseIP(*address)
config := sched.DriverConfig{
Scheduler: newExampleScheduler(),
Framework: fwinfo,
Master: *master,
Credential: cred,
BindingAddress: bindingAddress,
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, *authProvider)
ctx = sasl.WithBindingAddress(ctx, bindingAddress)
return ctx
},
}
driver, err := sched.NewMesosSchedulerDriver(config)
if err != nil {
log.Errorln("Unable to create a SchedulerDriver ", err.Error())
}
if stat, err := driver.Run(); err != nil {
log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
}
log.Infof("framework terminating")
}
func main() {
// start json server
go func() {
fs := http.FileServer(http.Dir("static"))
http.Handle("/", fs)
http.HandleFunc("/json", jsonHandleFunc)
http.ListenAndServe(":8888", nil)
}()
// build command executor
// args := []string{"/bin/cat /var/tmp/1.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/2.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/3.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/intermediate.txt | /bin/grep abe > /var/tmp/grep-result.txt", "date", "/bin/rm /var/tmp/intermediate.txt"}
args := []string{"/bin/cat /var/tmp/1.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/2.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/3.txt >> /var/tmp/intermediate.txt", "/bin/cat /var/tmp/intermediate.txt | /bin/grep abe > /var/tmp/grep-result.txt", "/bin/rm /var/tmp/intermediate.txt"}
args2 := []string{"/bin/cat /var/tmp/1.txt", "/bin/date"}
exec := prepareExecutorInfo(args)
cmdQueue.PushBack(exec)
exec = prepareExecutorInfo(args2)
cmdQueue.PushBack(exec)
// the framework
fwinfo := &mesos.FrameworkInfo{
User: proto.String(""), // Mesos-go will fill in user.
Name: proto.String("Test Framework (SDC)"),
}
cred := (*mesos.Credential)(nil)
if *mesosAuthPrincipal != "" {
fwinfo.Principal = proto.String(*mesosAuthPrincipal)
secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
if err != nil {
log.Fatal(err)
}
cred = &mesos.Credential{
Principal: proto.String(*mesosAuthPrincipal),
Secret: secret,
}
}
bindingAddress := parseIP(*address)
execinfo := cmdQueue.Remove(cmdQueue.Front()).(*mesos.ExecutorInfo)
config := sched.DriverConfig{
Scheduler: newSdcScheduler(execinfo, len(execinfo.Command.Arguments)),
Framework: fwinfo,
Master: *master,
Credential: cred,
BindingAddress: bindingAddress,
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, *authProvider)
ctx = sasl.WithBindingAddress(ctx, bindingAddress)
return ctx
},
}
driver, err := sched.NewMesosSchedulerDriver(config)
if err != nil {
log.Errorln("Unable to create a SchedulerDriver ", err.Error())
}
if stat, err := driver.Run(); err != nil {
log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
}
log.Infof("framework terminating")
}
func (s *SchedulerServer) bootstrap(hks hyperkube.Interface, sc *schedcfg.Config) (*ha.SchedulerProcess, ha.DriverFactory, etcd.Client, *mesos.ExecutorID) {
s.frameworkName = strings.TrimSpace(s.frameworkName)
if s.frameworkName == "" {
log.Fatalf("framework-name must be a non-empty string")
}
s.frameworkWebURI = strings.TrimSpace(s.frameworkWebURI)
metrics.Register()
runtime.Register()
s.mux.Handle("/metrics", prometheus.Handler())
healthz.InstallHandler(s.mux)
if len(s.etcdServerList) == 0 {
log.Fatalf("specify --etcd-servers must be specified")
}
if len(s.apiServerList) < 1 {
log.Fatal("No api servers specified.")
}
clientConfig, err := s.createAPIServerClientConfig()
if err != nil {
log.Fatalf("Unable to make apiserver client config: %v", err)
}
s.client, err = clientset.NewForConfig(clientConfig)
if err != nil {
log.Fatalf("Unable to make apiserver clientset: %v", err)
}
if s.reconcileCooldown < defaultReconcileCooldown {
s.reconcileCooldown = defaultReconcileCooldown
log.Warningf("user-specified reconcile cooldown too small, defaulting to %v", s.reconcileCooldown)
}
eiPrototype, err := s.prepareExecutorInfo(hks)
if err != nil {
log.Fatalf("misconfigured executor: %v", err)
}
// TODO(jdef): remove the dependency on etcd as soon as
// (1) the generic config store is available for the FrameworkId storage
// (2) the generic master election is provided by the apiserver
// Compare docs/proposals/high-availability.md
etcdClient, err := newEtcd(s.etcdServerList)
if err != nil {
log.Fatalf("misconfigured etcd: %v", err)
}
keysAPI := etcd.NewKeysAPI(etcdClient)
// mirror all nodes into the nodeStore
var eiRegistry executorinfo.Registry
nodesClientConfig := *clientConfig
nodesClient, err := clientset.NewForConfig(&nodesClientConfig)
if err != nil {
log.Fatalf("Cannot create client to watch nodes: %v", err)
}
nodeLW := cache.NewListWatchFromClient(nodesClient.CoreClient, "nodes", api.NamespaceAll, fields.Everything())
nodeStore, nodeCtl := controllerfw.NewInformer(nodeLW, &api.Node{}, s.nodeRelistPeriod, &controllerfw.ResourceEventHandlerFuncs{
DeleteFunc: func(obj interface{}) {
if eiRegistry != nil {
// TODO(jdef) use controllerfw.DeletionHandlingMetaNamespaceKeyFunc at some point?
nodeName := ""
if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
nodeName = tombstone.Key
} else if node, ok := obj.(*api.Node); ok {
nodeName = node.Name
}
if nodeName != "" {
log.V(2).Infof("deleting node %q from registry", nodeName)
eiRegistry.Invalidate(nodeName)
}
}
},
})
lookupNode := func(hostName string) *api.Node {
n, _, _ := nodeStore.GetByKey(hostName) // ignore error and return nil then
if n == nil {
return nil
}
return n.(*api.Node)
}
execInfoCache, err := executorinfo.NewCache(defaultExecutorInfoCacheSize)
if err != nil {
log.Fatalf("cannot create executorinfo cache: %v", err)
}
eiRegistry, err = executorinfo.NewRegistry(lookupNode, eiPrototype, execInfoCache)
if err != nil {
log.Fatalf("cannot create executorinfo registry: %v", err)
}
pr := podtask.NewDefaultProcurement(eiPrototype, eiRegistry)
fcfs := podschedulers.NewFCFSPodScheduler(pr, lookupNode)
frameworkIDStorage, err := s.frameworkIDStorage(keysAPI)
if err != nil {
log.Fatalf("cannot init framework ID storage: %v", err)
}
framework := framework.New(framework.Config{
SchedulerConfig: *sc,
Client: s.client,
FailoverTimeout: s.failoverTimeout,
ReconcileInterval: s.reconcileInterval,
ReconcileCooldown: s.reconcileCooldown,
LookupNode: lookupNode,
StoreFrameworkId: frameworkIDStorage.Set,
ExecutorId: eiPrototype.GetExecutorId(),
})
masterUri := s.mesosMaster
info, cred, err := s.buildFrameworkInfo()
if err != nil {
log.Fatalf("Misconfigured mesos framework: %v", err)
}
schedulerProcess := ha.New(framework)
// try publishing on the same IP as the slave
var publishedAddress net.IP
if libprocessIP := os.Getenv("LIBPROCESS_IP"); libprocessIP != "" {
publishedAddress = net.ParseIP(libprocessIP)
}
if publishedAddress != nil {
log.V(1).Infof("driver will publish address %v", publishedAddress)
}
dconfig := &bindings.DriverConfig{
Scheduler: schedulerProcess,
Framework: info,
Master: masterUri,
Credential: cred,
BindingAddress: s.address,
BindingPort: uint16(s.driverPort),
PublishedAddress: publishedAddress,
HostnameOverride: s.hostnameOverride,
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, s.mesosAuthProvider)
ctx = sasl.WithBindingAddress(ctx, s.address)
return ctx
},
}
// create event recorder sending events to the "" namespace of the apiserver
eventsClientConfig := *clientConfig
eventsClient, err := clientset.NewForConfig(&eventsClientConfig)
if err != nil {
log.Fatalf("Invalid API configuration: %v", err)
}
broadcaster := record.NewBroadcaster()
recorder := broadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName})
broadcaster.StartLogging(log.Infof)
broadcaster.StartRecordingToSink(&unversionedcore.EventSinkImpl{Interface: eventsClient.Events("")})
lw := cache.NewListWatchFromClient(s.client.CoreClient, "pods", api.NamespaceAll, fields.Everything())
hostPortStrategy := hostport.StrategyFixed
if s.useHostPortEndpoints {
hostPortStrategy = hostport.StrategyWildcard
}
// create scheduler core with all components arranged around it
sched := components.New(
sc,
framework,
fcfs,
s.client,
recorder,
schedulerProcess.Terminal(),
s.mux,
lw,
podtask.Config{
DefaultPodRoles: s.defaultPodRoles,
FrameworkRoles: s.frameworkRoles,
GenerateTaskDiscoveryEnabled: s.generateTaskDiscovery,
HostPortStrategy: hostPortStrategy,
Prototype: eiPrototype,
},
s.defaultContainerCPULimit,
s.defaultContainerMemLimit,
)
runtime.On(framework.Registration(), func() { sched.Run(schedulerProcess.Terminal()) })
runtime.On(framework.Registration(), s.newServiceWriter(publishedAddress, schedulerProcess.Terminal()))
runtime.On(framework.Registration(), func() { nodeCtl.Run(schedulerProcess.Terminal()) })
driverFactory := ha.DriverFactory(func() (drv bindings.SchedulerDriver, err error) {
log.V(1).Infoln("performing deferred initialization")
if err = framework.Init(sched, schedulerProcess.Master(), s.mux); err != nil {
return nil, fmt.Errorf("failed to initialize pod scheduler: %v", err)
}
log.V(1).Infoln("deferred init complete")
if s.failoverTimeout > 0 {
// defer obtaining framework ID to prevent multiple schedulers
// from overwriting each other's framework IDs
var frameworkID string
frameworkID, err = frameworkIDStorage.Get(context.TODO())
if err != nil {
return nil, fmt.Errorf("failed to fetch framework ID from storage: %v", err)
}
if frameworkID != "" {
log.Infof("configuring FrameworkInfo with ID found in storage: %q", frameworkID)
dconfig.Framework.Id = &mesos.FrameworkID{Value: &frameworkID}
} else {
log.V(1).Infof("did not find framework ID in storage")
}
} else {
// TODO(jdef) this is a hack, really for development, to simplify clean up of old framework IDs
frameworkIDStorage.Remove(context.TODO())
}
log.V(1).Infoln("constructing mesos scheduler driver")
drv, err = bindings.NewMesosSchedulerDriver(*dconfig)
if err != nil {
return nil, fmt.Errorf("failed to construct scheduler driver: %v", err)
}
log.V(1).Infoln("constructed mesos scheduler driver:", drv)
s.setDriver(drv)
return drv, nil
})
return schedulerProcess, driverFactory, etcdClient, eiPrototype.GetExecutorId()
}
func (s *SchedulerServer) bootstrap(hks hyperkube.Interface, sc *schedcfg.Config) (*ha.SchedulerProcess, ha.DriverFactory, tools.EtcdGetSet, *uid.UID) {
s.FrameworkName = strings.TrimSpace(s.FrameworkName)
if s.FrameworkName == "" {
log.Fatalf("framework-name must be a non-empty string")
}
s.FrameworkWebURI = strings.TrimSpace(s.FrameworkWebURI)
metrics.Register()
runtime.Register()
s.mux.Handle("/metrics", prometheus.Handler())
if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) {
log.Fatalf("specify either --etcd-servers or --etcd-config")
}
if len(s.APIServerList) < 1 {
log.Fatal("No api servers specified.")
}
client, err := s.createAPIServerClient()
if err != nil {
log.Fatalf("Unable to make apiserver client: %v", err)
}
s.client = client
if s.ReconcileCooldown < defaultReconcileCooldown {
s.ReconcileCooldown = defaultReconcileCooldown
log.Warningf("user-specified reconcile cooldown too small, defaulting to %v", s.ReconcileCooldown)
}
executor, eid, err := s.prepareExecutorInfo(hks)
if err != nil {
log.Fatalf("misconfigured executor: %v", err)
}
// TODO(jdef): remove the dependency on etcd as soon as
// (1) the generic config store is available for the FrameworkId storage
// (2) the generic master election is provided by the apiserver
// Compare docs/proposals/high-availability.md
etcdClient, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList)
if err != nil {
log.Fatalf("misconfigured etcd: %v", err)
}
mesosPodScheduler := scheduler.New(scheduler.Config{
Schedcfg: *sc,
Executor: executor,
ScheduleFunc: scheduler.FCFSScheduleFunc,
Client: client,
EtcdClient: etcdClient,
FailoverTimeout: s.FailoverTimeout,
ReconcileInterval: s.ReconcileInterval,
ReconcileCooldown: s.ReconcileCooldown,
})
masterUri := s.MesosMaster
info, cred, err := s.buildFrameworkInfo()
if err != nil {
log.Fatalf("Misconfigured mesos framework: %v", err)
}
schedulerProcess := ha.New(mesosPodScheduler)
dconfig := &bindings.DriverConfig{
Scheduler: schedulerProcess,
Framework: info,
Master: masterUri,
Credential: cred,
BindingAddress: net.IP(s.Address),
BindingPort: uint16(s.DriverPort),
HostnameOverride: s.HostnameOverride,
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, s.MesosAuthProvider)
ctx = sasl.WithBindingAddress(ctx, net.IP(s.Address))
return ctx
},
}
kpl := scheduler.NewPlugin(mesosPodScheduler.NewDefaultPluginConfig(schedulerProcess.Terminal(), s.mux))
runtime.On(mesosPodScheduler.Registration(), func() { kpl.Run(schedulerProcess.Terminal()) })
runtime.On(mesosPodScheduler.Registration(), s.newServiceWriter(schedulerProcess.Terminal()))
driverFactory := ha.DriverFactory(func() (drv bindings.SchedulerDriver, err error) {
log.V(1).Infoln("performing deferred initialization")
if err = mesosPodScheduler.Init(schedulerProcess.Master(), kpl, s.mux); err != nil {
return nil, fmt.Errorf("failed to initialize pod scheduler: %v", err)
}
log.V(1).Infoln("deferred init complete")
// defer obtaining framework ID to prevent multiple schedulers
// from overwriting each other's framework IDs
dconfig.Framework.Id, err = s.fetchFrameworkID(etcdClient)
if err != nil {
return nil, fmt.Errorf("failed to fetch framework ID from etcd: %v", err)
}
log.V(1).Infoln("constructing mesos scheduler driver")
drv, err = bindings.NewMesosSchedulerDriver(*dconfig)
if err != nil {
return nil, fmt.Errorf("failed to construct scheduler driver: %v", err)
}
log.V(1).Infoln("constructed mesos scheduler driver:", drv)
s.setDriver(drv)
return drv, nil
})
return schedulerProcess, driverFactory, etcdClient, eid
}
func (s *SchedulerServer) bootstrap(hks hyperkube.Interface, sc *schedcfg.Config) (*ha.SchedulerProcess, ha.DriverFactory, tools.EtcdClient, *uid.UID) {
s.FrameworkName = strings.TrimSpace(s.FrameworkName)
if s.FrameworkName == "" {
log.Fatalf("framework-name must be a non-empty string")
}
s.FrameworkWebURI = strings.TrimSpace(s.FrameworkWebURI)
metrics.Register()
runtime.Register()
s.mux.Handle("/metrics", prometheus.Handler())
healthz.InstallHandler(s.mux)
if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) {
log.Fatalf("specify either --etcd-servers or --etcd-config")
}
if len(s.APIServerList) < 1 {
log.Fatal("No api servers specified.")
}
client, err := s.createAPIServerClient()
if err != nil {
log.Fatalf("Unable to make apiserver client: %v", err)
}
s.client = client
if s.ReconcileCooldown < defaultReconcileCooldown {
s.ReconcileCooldown = defaultReconcileCooldown
log.Warningf("user-specified reconcile cooldown too small, defaulting to %v", s.ReconcileCooldown)
}
executor, eid, err := s.prepareExecutorInfo(hks)
if err != nil {
log.Fatalf("misconfigured executor: %v", err)
}
// TODO(jdef): remove the dependency on etcd as soon as
// (1) the generic config store is available for the FrameworkId storage
// (2) the generic master election is provided by the apiserver
// Compare docs/proposals/high-availability.md
etcdClient, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList)
if err != nil {
log.Fatalf("misconfigured etcd: %v", err)
}
as := scheduler.NewAllocationStrategy(
podtask.NewDefaultPredicate(
s.DefaultContainerCPULimit,
s.DefaultContainerMemLimit,
),
podtask.NewDefaultProcurement(
s.DefaultContainerCPULimit,
s.DefaultContainerMemLimit,
),
)
// downgrade allocation strategy if user disables "account-for-pod-resources"
if !s.AccountForPodResources {
as = scheduler.NewAllocationStrategy(
podtask.DefaultMinimalPredicate,
podtask.DefaultMinimalProcurement)
}
// mirror all nodes into the nodeStore
nodesClient, err := s.createAPIServerClient()
if err != nil {
log.Fatalf("Cannot create client to watch nodes: %v", err)
}
nodeStore := cache.NewStore(cache.MetaNamespaceKeyFunc)
nodeLW := cache.NewListWatchFromClient(nodesClient, "nodes", api.NamespaceAll, fields.Everything())
cache.NewReflector(nodeLW, &api.Node{}, nodeStore, s.nodeRelistPeriod).Run()
lookupNode := func(hostName string) *api.Node {
n, _, _ := nodeStore.GetByKey(hostName) // ignore error and return nil then
if n == nil {
return nil
}
return n.(*api.Node)
}
fcfs := scheduler.NewFCFSPodScheduler(as, lookupNode)
mesosPodScheduler := scheduler.New(scheduler.Config{
Schedcfg: *sc,
Executor: executor,
Scheduler: fcfs,
Client: client,
EtcdClient: etcdClient,
FailoverTimeout: s.FailoverTimeout,
ReconcileInterval: s.ReconcileInterval,
ReconcileCooldown: s.ReconcileCooldown,
LookupNode: lookupNode,
})
masterUri := s.MesosMaster
info, cred, err := s.buildFrameworkInfo()
if err != nil {
log.Fatalf("Misconfigured mesos framework: %v", err)
}
schedulerProcess := ha.New(mesosPodScheduler)
dconfig := &bindings.DriverConfig{
Scheduler: schedulerProcess,
Framework: info,
Master: masterUri,
Credential: cred,
BindingAddress: s.Address,
BindingPort: uint16(s.DriverPort),
HostnameOverride: s.HostnameOverride,
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, s.MesosAuthProvider)
ctx = sasl.WithBindingAddress(ctx, s.Address)
return ctx
},
}
kpl := scheduler.NewPlugin(mesosPodScheduler.NewDefaultPluginConfig(schedulerProcess.Terminal(), s.mux))
runtime.On(mesosPodScheduler.Registration(), func() { kpl.Run(schedulerProcess.Terminal()) })
runtime.On(mesosPodScheduler.Registration(), s.newServiceWriter(schedulerProcess.Terminal()))
driverFactory := ha.DriverFactory(func() (drv bindings.SchedulerDriver, err error) {
log.V(1).Infoln("performing deferred initialization")
if err = mesosPodScheduler.Init(schedulerProcess.Master(), kpl, s.mux); err != nil {
return nil, fmt.Errorf("failed to initialize pod scheduler: %v", err)
}
log.V(1).Infoln("deferred init complete")
// defer obtaining framework ID to prevent multiple schedulers
// from overwriting each other's framework IDs
dconfig.Framework.Id, err = s.fetchFrameworkID(etcdClient)
if err != nil {
return nil, fmt.Errorf("failed to fetch framework ID from etcd: %v", err)
}
log.V(1).Infoln("constructing mesos scheduler driver")
drv, err = bindings.NewMesosSchedulerDriver(*dconfig)
if err != nil {
return nil, fmt.Errorf("failed to construct scheduler driver: %v", err)
}
log.V(1).Infoln("constructed mesos scheduler driver:", drv)
s.setDriver(drv)
return drv, nil
})
return schedulerProcess, driverFactory, etcdClient, eid
}
func (sc *SchedulerCore) Run(mesosMaster string) {
sc.schedulerState.MesosMaster = mesosMaster
var frameworkId *mesos.FrameworkID
if sc.schedulerState.FrameworkID == nil {
frameworkId = nil
} else {
frameworkId = &mesos.FrameworkID{
Value: sc.schedulerState.FrameworkID,
}
}
fwinfo := &mesos.FrameworkInfo{
Name: proto.String(sc.frameworkName),
Id: frameworkId,
FailoverTimeout: proto.Float64(86400),
WebuiUrl: proto.String(sc.schedulerHTTPServer.GetURI()),
Checkpoint: proto.Bool(true),
Role: proto.String(sc.frameworkRole),
}
if sc.user != "" {
fwinfo.User = proto.String(sc.user)
} else {
guestUser := "******"
fwinfo.User = &guestUser
}
cred := (*mesos.Credential)(nil)
if sc.mesosAuthPrincipal != "" {
fwinfo.Principal = proto.String(sc.mesosAuthPrincipal)
}
if sc.mesosAuthSecretFile != "" && sc.mesosAuthPrincipal != "" {
secret, err := ioutil.ReadFile(sc.mesosAuthSecretFile)
if err != nil {
log.Fatal(err)
}
cred = &mesos.Credential{
Principal: proto.String(sc.mesosAuthPrincipal),
Secret: secret,
}
}
hostname, err := os.Hostname()
if err != nil {
log.Fatal(err)
}
config := sched.DriverConfig{
Scheduler: sc,
Framework: fwinfo,
Master: mesosMaster,
Credential: cred,
HostnameOverride: hostname,
}
if sc.schedulerIPAddr != "" {
config.BindingAddress = parseIP(sc.schedulerIPAddr)
}
if sc.authProvider != "" && sc.mesosAuthPrincipal != "" {
config.WithAuthContext = func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, sc.authProvider)
if sc.schedulerIPAddr != "" {
ctx = sasl.WithBindingAddress(ctx, parseIP(sc.schedulerIPAddr))
}
return ctx
}
}
log.Infof("Running scheduler with FrameworkInfo: %v and DriverConfig: %v", fwinfo, config)
driver, err := sched.NewMesosSchedulerDriver(config)
if err != nil {
log.Error("Unable to create a SchedulerDriver ", err.Error())
}
sc.rServer = newReconciliationServer(driver, sc)
sc.mgr.SetupFramework(sc.schedulerHTTPServer.URI)
if stat, err := driver.Run(); err != nil {
log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
}
}
func main() {
if *ghsNode {
ghsNodeMain()
return
}
// Allocate a port for GRPC
lis, err := net.Listen("tcp", ":0")
if err != nil {
log.Fatalf("grpc failed to listen: %v", err)
}
grpcServer := grpc.NewServer()
// build command executor
nexec := prepareExecutorInfo(lis.Addr())
s := newvisghsScheduler(nexec)
t := tracer.NewHTTPDisplay(http.DefaultServeMux, s.OnRun)
server := tracer.NewGRPCServer(t)
tracer.RegisterTraceServiceServer(grpcServer, server)
go grpcServer.Serve(lis)
time.Sleep(2 * time.Second)
go http.ListenAndServe(":12345", nil)
// the framework
fwinfo := &mesos.FrameworkInfo{
User: proto.String(""), // Mesos-go will fill in user.
Name: proto.String("visghs"),
}
cred := (*mesos.Credential)(nil)
if *mesosAuthPrincipal != "" {
fwinfo.Principal = proto.String(*mesosAuthPrincipal)
cred = &mesos.Credential{
Principal: proto.String(*mesosAuthPrincipal),
}
if *mesosAuthSecretFile != "" {
_, err := os.Stat(*mesosAuthSecretFile)
if err != nil {
log.Fatal("missing secret file: ", err.Error())
}
secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
if err != nil {
log.Fatal("failed to read secret file: ", err.Error())
}
cred.Secret = proto.String(string(secret))
}
}
bindingAddress := parseIP(*address)
config := sched.DriverConfig{
Scheduler: s,
Framework: fwinfo,
Master: *master,
Credential: cred,
BindingAddress: bindingAddress,
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, *authProvider)
ctx = sasl.WithBindingAddress(ctx, bindingAddress)
return ctx
},
}
driver, err := sched.NewMesosSchedulerDriver(config)
if err != nil {
log.Errorln("Unable to create a SchedulerDriver ", err.Error())
}
if stat, err := driver.Run(); err != nil {
log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
time.Sleep(2 * time.Second)
os.Exit(1)
}
log.Infof("framework terminating")
}
func (s *SchedulerServer) bootstrap(hks hyperkube.Interface, sc *schedcfg.Config) (*ha.SchedulerProcess, ha.DriverFactory, *etcd.Client, *mesos.ExecutorID) {
s.frameworkName = strings.TrimSpace(s.frameworkName)
if s.frameworkName == "" {
log.Fatalf("framework-name must be a non-empty string")
}
s.frameworkWebURI = strings.TrimSpace(s.frameworkWebURI)
metrics.Register()
runtime.Register()
s.mux.Handle("/metrics", prometheus.Handler())
healthz.InstallHandler(s.mux)
if (s.etcdConfigFile != "" && len(s.etcdServerList) != 0) || (s.etcdConfigFile == "" && len(s.etcdServerList) == 0) {
log.Fatalf("specify either --etcd-servers or --etcd-config")
}
if len(s.apiServerList) < 1 {
log.Fatal("No api servers specified.")
}
client, err := s.createAPIServerClient()
if err != nil {
log.Fatalf("Unable to make apiserver client: %v", err)
}
s.client = client
if s.reconcileCooldown < defaultReconcileCooldown {
s.reconcileCooldown = defaultReconcileCooldown
log.Warningf("user-specified reconcile cooldown too small, defaulting to %v", s.reconcileCooldown)
}
eiPrototype, err := s.prepareExecutorInfo(hks)
if err != nil {
log.Fatalf("misconfigured executor: %v", err)
}
// TODO(jdef): remove the dependency on etcd as soon as
// (1) the generic config store is available for the FrameworkId storage
// (2) the generic master election is provided by the apiserver
// Compare docs/proposals/high-availability.md
etcdClient, err := newEtcd(s.etcdConfigFile, s.etcdServerList)
if err != nil {
log.Fatalf("misconfigured etcd: %v", err)
}
// mirror all nodes into the nodeStore
var eiRegistry executorinfo.Registry
nodesClient, err := s.createAPIServerClient()
if err != nil {
log.Fatalf("Cannot create client to watch nodes: %v", err)
}
nodeLW := cache.NewListWatchFromClient(nodesClient, "nodes", api.NamespaceAll, fields.Everything())
nodeStore, nodeCtl := controllerfw.NewInformer(nodeLW, &api.Node{}, s.nodeRelistPeriod, &controllerfw.ResourceEventHandlerFuncs{
DeleteFunc: func(obj interface{}) {
node := obj.(*api.Node)
if eiRegistry != nil {
log.V(2).Infof("deleting node %q from registry", node.Name)
eiRegistry.Invalidate(node.Name)
}
},
})
lookupNode := func(hostName string) *api.Node {
n, _, _ := nodeStore.GetByKey(hostName) // ignore error and return nil then
if n == nil {
return nil
}
return n.(*api.Node)
}
execInfoCache, err := executorinfo.NewCache(defaultExecutorInfoCacheSize)
if err != nil {
log.Fatalf("cannot create executorinfo cache: %v", err)
}
eiRegistry, err = executorinfo.NewRegistry(lookupNode, eiPrototype, execInfoCache)
if err != nil {
log.Fatalf("cannot create executorinfo registry: %v", err)
}
pr := podtask.NewDefaultProcurement(eiPrototype, eiRegistry)
fcfs := podschedulers.NewFCFSPodScheduler(pr, lookupNode)
framework := framework.New(framework.Config{
SchedulerConfig: *sc,
Client: client,
FailoverTimeout: s.failoverTimeout,
ReconcileInterval: s.reconcileInterval,
ReconcileCooldown: s.reconcileCooldown,
LookupNode: lookupNode,
StoreFrameworkId: func(id string) {
// TODO(jdef): port FrameworkId store to generic Kubernetes config store as soon as available
_, err := etcdClient.Set(meta.FrameworkIDKey, id, uint64(s.failoverTimeout))
if err != nil {
log.Errorf("failed to renew frameworkId TTL: %v", err)
}
},
ExecutorId: eiPrototype.GetExecutorId(),
})
masterUri := s.mesosMaster
info, cred, err := s.buildFrameworkInfo()
if err != nil {
log.Fatalf("Misconfigured mesos framework: %v", err)
}
schedulerProcess := ha.New(framework)
dconfig := &bindings.DriverConfig{
Scheduler: schedulerProcess,
Framework: info,
Master: masterUri,
Credential: cred,
BindingAddress: s.address,
BindingPort: uint16(s.driverPort),
HostnameOverride: s.hostnameOverride,
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, s.mesosAuthProvider)
ctx = sasl.WithBindingAddress(ctx, s.address)
return ctx
},
}
// create event recorder sending events to the "" namespace of the apiserver
broadcaster := record.NewBroadcaster()
recorder := broadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
broadcaster.StartRecordingToSink(client.Events(""))
// create scheduler core with all components arranged around it
lw := cache.NewListWatchFromClient(client, "pods", api.NamespaceAll, fields.Everything())
sched := components.New(
sc,
framework,
fcfs,
client,
recorder,
schedulerProcess.Terminal(),
s.mux,
lw,
eiPrototype,
s.mesosRoles,
s.defaultContainerCPULimit,
s.defaultContainerMemLimit,
)
runtime.On(framework.Registration(), func() { sched.Run(schedulerProcess.Terminal()) })
runtime.On(framework.Registration(), s.newServiceWriter(schedulerProcess.Terminal()))
runtime.On(framework.Registration(), func() { nodeCtl.Run(schedulerProcess.Terminal()) })
driverFactory := ha.DriverFactory(func() (drv bindings.SchedulerDriver, err error) {
log.V(1).Infoln("performing deferred initialization")
if err = framework.Init(sched, schedulerProcess.Master(), s.mux); err != nil {
return nil, fmt.Errorf("failed to initialize pod scheduler: %v", err)
}
log.V(1).Infoln("deferred init complete")
// defer obtaining framework ID to prevent multiple schedulers
// from overwriting each other's framework IDs
dconfig.Framework.Id, err = s.fetchFrameworkID(etcdClient)
if err != nil {
return nil, fmt.Errorf("failed to fetch framework ID from etcd: %v", err)
}
log.V(1).Infoln("constructing mesos scheduler driver")
drv, err = bindings.NewMesosSchedulerDriver(*dconfig)
if err != nil {
return nil, fmt.Errorf("failed to construct scheduler driver: %v", err)
}
log.V(1).Infoln("constructed mesos scheduler driver:", drv)
s.setDriver(drv)
return drv, nil
})
return schedulerProcess, driverFactory, etcdClient, eiPrototype.GetExecutorId()
}
func main() {
frameworkName :=
flag.String("framework-name", "etcd", "Unique name of this etcd cluster")
master :=
flag.String("master", "127.0.0.1:5050", "Master address <ip:port>")
zkFrameworkPersist :=
flag.String("zk-framework-persist", "", "Zookeeper URI of the form zk://host1:port1,host2:port2/chroot/path")
taskCount :=
flag.Int("cluster-size", 5, "Total task count to run")
adminPort :=
flag.Int("admin-port", 23400, "Binding port for admin interface")
reseedTimeout :=
flag.Int("reseed-timeout", 240, "Seconds of etcd livelock to wait for before attempting a cluster re-seed")
autoReseed :=
flag.Bool("auto-reseed", true, "Perform automatic cluster reseed when the "+
"cluster has been livelocked for -reseed-timeout seconds")
artifactPort :=
flag.Int("artifact-port", 12300, "Binding port for artifact server")
sandboxDisk :=
flag.Float64("sandbox-disk-limit", 4096, "Max disk usage for the etcd mesos sandbox in MB")
sandboxCpu :=
flag.Float64("sandbox-cpu-limit", 4, "Max cpu usage for the etcd mesos sandbox in MB")
sandboxMem :=
flag.Float64("sandbox-mem-limit", 2048, "Max memory usage for the etcd mesos sandbox in MB")
executorPath :=
flag.String("executor-bin", "./bin/etcd-mesos-executor", "Path to executor binary")
etcdPath :=
flag.String("etcd-bin", "./bin/etcd", "Path to etcd binary")
etcdctlPath :=
flag.String("etcdctl-bin", "./bin/etcdctl", "Path to etcdctl binary")
address :=
flag.String("address", "", "Binding address for scheduler and artifact server")
driverPort :=
flag.Int("driver-port", 0, "Binding port for scheduler driver")
mesosAuthPrincipal :=
flag.String("mesos-authentication-principal", "", "Mesos authentication principal")
mesosAuthSecretFile :=
flag.String("mesos-authentication-secret-file", "", "Mesos authentication secret file")
authProvider :=
flag.String("mesos-authentication-provider", sasl.ProviderName,
fmt.Sprintf("Authentication provider to use, default is SASL that supports mechanisms: %+v", mech.ListSupported()))
singleInstancePerSlave :=
flag.Bool("single-instance-per-slave", true, "Only allow one etcd instance to be started per slave")
failoverTimeoutSeconds :=
flag.Float64("failover-timeout-seconds", 60*60*24*7, "Mesos framework failover timeout in seconds")
weburi := flag.String("framework-weburi", "", "A URI that points to a web-based interface for interacting with the framework.")
flag.Parse()
if *zkFrameworkPersist == "" {
log.Fatal("No value provided for -zk-framework-persist !")
}
if !*singleInstancePerSlave {
log.Warning("-single-instance-per-slave=false is dangerous because it may lead to " +
"multiple etcd instances in the same cluster on a single node, amplifying " +
"the cost of a single node being lost, livelock, and data loss.")
}
if *address == "" {
hostname, err := os.Hostname()
if err == nil {
*address = hostname
} else {
log.Errorf("Could not set default binding to hostname. Defaulting to 127.0.0.1")
*address = "127.0.0.1"
}
}
if *weburi == "" {
*weburi = fmt.Sprintf("http://%s:%d/", *address, *adminPort)
}
executorUris := []*mesos.CommandInfo_URI{}
execUri, err := etcdscheduler.ServeExecutorArtifact(*executorPath, *address, *artifactPort)
if err != nil {
log.Errorf("Could not stat executor binary: %v", err)
return
}
executorUris = append(executorUris, &mesos.CommandInfo_URI{
Value: execUri,
Executable: proto.Bool(true),
})
etcdUri, err := etcdscheduler.ServeExecutorArtifact(*etcdPath, *address, *artifactPort)
if err != nil {
log.Errorf("Could not stat etcd binary: %v", err)
return
}
executorUris = append(executorUris, &mesos.CommandInfo_URI{
Value: etcdUri,
Executable: proto.Bool(true),
})
etcdctlUri, err := etcdscheduler.ServeExecutorArtifact(*etcdctlPath, *address, *artifactPort)
if err != nil {
log.Errorf("Could not stat etcd binary: %v", err)
return
}
executorUris = append(executorUris, &mesos.CommandInfo_URI{
Value: etcdctlUri,
Executable: proto.Bool(true),
})
go http.ListenAndServe(fmt.Sprintf("%s:%d", *address, *artifactPort), nil)
log.V(2).Info("Serving executor artifacts...")
bindingAddress := parseIP(*address)
// chillFactor is the number of seconds that are slept for to allow for
// convergence across the cluster during mutations.
chillFactor := 10
etcdScheduler := etcdscheduler.NewEtcdScheduler(
*taskCount,
chillFactor,
*reseedTimeout,
*autoReseed,
executorUris,
*singleInstancePerSlave,
*sandboxDisk,
*sandboxCpu,
*sandboxMem,
)
etcdScheduler.ExecutorPath = *executorPath
etcdScheduler.Master = *master
etcdScheduler.FrameworkName = *frameworkName
etcdScheduler.ZkConnect = *zkFrameworkPersist
fwinfo := &mesos.FrameworkInfo{
User: proto.String(""), // Mesos-go will fill in user.
Name: proto.String(*frameworkName),
Checkpoint: proto.Bool(true),
FailoverTimeout: proto.Float64(*failoverTimeoutSeconds),
WebuiUrl: proto.String(*weburi),
}
cred := (*mesos.Credential)(nil)
if *mesosAuthPrincipal != "" {
fwinfo.Principal = proto.String(*mesosAuthPrincipal)
secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
if err != nil {
log.Fatal(err)
}
cred = &mesos.Credential{
Principal: proto.String(*mesosAuthPrincipal),
Secret: secret,
}
}
zkServers, zkChroot, err := rpc.ParseZKURI(*zkFrameworkPersist)
etcdScheduler.ZkServers = zkServers
etcdScheduler.ZkChroot = zkChroot
if err != nil && *zkFrameworkPersist != "" {
log.Fatalf("Error parsing zookeeper URI of %s: %s", *zkFrameworkPersist, err)
} else if *zkFrameworkPersist != "" {
previous, err := rpc.GetPreviousFrameworkID(
zkServers,
zkChroot,
etcdScheduler.FrameworkName,
)
if err != nil && err != zk.ErrNoNode {
log.Fatalf("Could not retrieve previous framework ID: %s", err)
} else if err == zk.ErrNoNode {
log.Info("No previous persisted framework ID exists in zookeeper.")
} else {
log.Infof("Found stored framework ID in Zookeeper, "+
"attempting to re-use: %s", previous)
fwinfo.Id = &mesos.FrameworkID{
Value: proto.String(previous),
}
}
}
config := scheduler.DriverConfig{
Scheduler: etcdScheduler,
Framework: fwinfo,
Master: etcdScheduler.Master,
Credential: cred,
BindingAddress: bindingAddress,
BindingPort: uint16(*driverPort),
WithAuthContext: func(ctx context.Context) context.Context {
ctx = auth.WithLoginProvider(ctx, *authProvider)
ctx = sasl.WithBindingAddress(ctx, bindingAddress)
return ctx
},
}
driver, err := scheduler.NewMesosSchedulerDriver(config)
if err != nil {
log.Errorln("Unable to create a SchedulerDriver ", err.Error())
}
go etcdScheduler.SerialLauncher(driver)
go etcdScheduler.PeriodicReconciler(driver)
go etcdScheduler.PeriodicHealthChecker()
go etcdScheduler.PeriodicLaunchRequestor()
go etcdScheduler.AdminHTTP(*adminPort, driver)
if stat, err := driver.Run(); err != nil {
log.Infof("Framework stopped with status %s and error: %s",
stat.String(),
err.Error())
}
}