Beispiel #1
0
func createDriver(scheduler *Scheduler, settings *Settings) (*mesossched.MesosSchedulerDriver, error) {
	publishedAddr := net.ParseIP(settings.MessengerAddress)
	bindingPort := settings.MessengerPort
	credential, err := getCredential(settings)

	if err != nil {
		return nil, err
	}

	return mesossched.NewMesosSchedulerDriver(mesossched.DriverConfig{
		Master: settings.Master,
		Framework: &mesosproto.FrameworkInfo{
			Id:              getFrameworkID(settings),
			Name:            proto.String(settings.Name),
			User:            proto.String(settings.User),
			Checkpoint:      proto.Bool(settings.Checkpoint),
			FailoverTimeout: proto.Float64(settings.FailoverTimeout),
			Principal:       getPrincipalID(credential),
		},
		Scheduler:        scheduler,
		BindingAddress:   net.ParseIP("0.0.0.0"),
		PublishedAddress: publishedAddr,
		BindingPort:      bindingPort,
		Credential:       credential,
		WithAuthContext:  getAuthContext,
	})
}
Beispiel #2
0
func main() {
	minTasks := flag.Int("min", 1, "The start size of the cluster")
	port := flag.Int("port", 10001, "The port the framework should listen on")
	master := flag.String("master", "10.0.0.26:5050", "Location of leading Mesos master")

	flag.Parse()

	scheduler := newMemcacheScheduler(*minTasks)

	// PICK UP HERE, make this return registered memcache endpoints. Make memcache start on actually allocated ports
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		if err := json.NewEncoder(w).Encode(scheduler.TaskStatuses); err != nil {
			fmt.Println(err)
		}
	})

	go http.ListenAndServe(":"+strconv.Itoa(*port), nil)

	driver, err := sched.NewMesosSchedulerDriver(sched.DriverConfig{
		Master: *master,
		Framework: &mesos.FrameworkInfo{
			Name: proto.String("MEMCACHE"),
			User: proto.String(""),
		},
		Scheduler: scheduler,
	})
	if err != nil {
		log.Printf("Unable to create scheduler driver: %s", err)
		return
	}

	// Catch interrupt
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt, os.Kill)
		s := <-c
		if s != os.Interrupt {
			return
		}

		log.Println("Memcache is shutting down")
		close(scheduler.shutdown)

		select {
		case <-scheduler.done:
		case <-time.After(SHUTDOWN_TIMEOUT):
		}

		// we have shut down
		driver.Stop(false)
	}()

	log.Printf("Starting driver")
	if status, err := driver.Run(); err != nil {
		log.Printf("Framework stopped with status %s and error: %s\n", status.String(), err.Error())
	}
	log.Println("Exiting...")
}
Beispiel #3
0
func main() {

	// build command executor
	exec := prepareExecutorInfo()

	// the framework
	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""), // Mesos-go will fill in user.
		Name: proto.String("Test Framework (Go)"),
	}

	cred := (*mesos.Credential)(nil)
	if *mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(*mesosAuthPrincipal)
		cred = &mesos.Credential{
			Principal: proto.String(*mesosAuthPrincipal),
		}
		if *mesosAuthSecretFile != "" {
			_, err := os.Stat(*mesosAuthSecretFile)
			if err != nil {
				log.Fatal("missing secret file: ", err.Error())
			}
			secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
			if err != nil {
				log.Fatal("failed to read secret file: ", err.Error())
			}
			cred.Secret = proto.String(string(secret))
		}
	}
	bindingAddress := parseIP(*address)
	config := sched.DriverConfig{
		Scheduler:      newExampleScheduler(exec),
		Framework:      fwinfo,
		Master:         *master,
		Credential:     cred,
		BindingAddress: bindingAddress,
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, *authProvider)
			ctx = sasl.WithBindingAddress(ctx, bindingAddress)
			return ctx
		},
	}
	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver ", err.Error())
	}

	if stat, err := driver.Run(); err != nil {
		log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
		time.Sleep(2 * time.Second)
		os.Exit(1)
	}
	log.Infof("framework terminating")
}
func main() {
	flag.Parse()

	var user, pass string
	if flag.NArg() == 1 {
		user = ""
		pass = flag.Arg(0)
	} else if flag.NArg() == 2 {
		user = flag.Arg(0)
		pass = flag.Arg(1)
	} else {
		printUsage()
		println(flag.NArg())
		return
	}

	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""),
		Name: proto.String("BTC Mining Framework (Go)"),
	}

	cred := (*mesos.Credential)(nil)
	if *mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(*mesosAuthPrincipal)
		secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
		if err != nil {
			log.Fatal(err)
		}
		cred = &mesos.Credential{
			Principal: proto.String(*mesosAuthPrincipal),
			Secret:    secret,
		}
	}
	config := sched.DriverConfig{
		Scheduler:  newMinerScheduler(user, pass),
		Framework:  fwinfo,
		Master:     *master,
		Credential: cred,
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, *authProvider)
			return ctx
		},
	}

	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver ", err.Error())
	}

	if stat, err := driver.Run(); err != nil {
		log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
	}
}
Beispiel #5
0
func startScheduler() (reflex *scheduler.ReflexScheduler, start func()) {
	exec := &mesos.ExecutorInfo{
		ExecutorId: util.NewExecutorID("default"),
		Name:       proto.String("BH executor (Go)"),
		Source:     proto.String("bh_test"),
		Command: &mesos.CommandInfo{
			Value: proto.String(""),
			Uris:  []*mesos.CommandInfo_URI{},
		},
	}

	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""),
		Name: proto.String("reflex"),
	}

	// skipping creds for now...
	// cred := (*mesos.Credential)(nil)
	// if *mesosAuthPrincipal != "" {
	// 	fwinfo.Principal = proto.String(*mesosAuthPrincipal)
	// 	secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
	// 	if err != nil {
	// 		logrus.WithField("error", err).Fatal("failed reading secret file")
	// 	}
	// 	cred = &mesos.Credential{
	// 		Principal: proto.String(*mesosAuthPrincipal),
	// 		Secret:    secret,
	// 	}
	// }

	reflex = scheduler.NewScheduler(exec)

	config := sched.DriverConfig{
		Scheduler: reflex,
		Framework: fwinfo,
		Master:    "127.0.0.1:5050", // TODO: grab this from somewhere
		// Credential: cred,
	}

	start = func() {
		driver, err := sched.NewMesosSchedulerDriver(config)
		if err != nil {
			logrus.WithField("error", err).Fatal("unable to create a SchedulerDriver")
		}
		if stat, err := driver.Run(); err != nil {
			logrus.WithFields(logrus.Fields{
				"status": stat.String(),
				"error":  err,
			}).Info("framework stopped")
		}
	}

	return reflex, start
}
func main() {

	startDrone()

	// build command executor
	exec := prepareExecutorInfo()

	// the framework
	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""), // Mesos-go will fill in user.
		Name: proto.String("Drone Framework"),
	}

	cred := (*mesos.Credential)(nil)
	if *mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(*mesosAuthPrincipal)
		secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
		if err != nil {
			log.Fatal(err)
		}
		cred = &mesos.Credential{
			Principal: proto.String(*mesosAuthPrincipal),
			Secret:    secret,
		}
	}

	fmt.Println("Address: ", address)
	fmt.Println("Master: ", master)
	bindingAddress := parseIP(address)
	fmt.Println("bindingAddress: ", bindingAddress)
	config := sched.DriverConfig{
		Scheduler:      newExampleScheduler(exec),
		Framework:      fwinfo,
		Master:         master,
		Credential:     cred,
		BindingAddress: bindingAddress,
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, *authProvider)
			ctx = sasl.WithBindingAddress(ctx, bindingAddress)
			return ctx
		},
	}

	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver ", err.Error())
	}

	if stat, err := driver.Run(); err != nil {
		log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
	}

}
Beispiel #7
0
func (s *Scheduler) Start() error {
	Logger.Infof("Starting scheduler with configuration: \n%s", Config)
	sched = s // set this scheduler reachable for http server

	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	if err := s.resolveDeps(); err != nil {
		return err
	}

	listenAddr := s.listenAddr()
	s.httpServer = NewHttpServer(listenAddr)
	go s.httpServer.Start()

	s.cluster = NewCluster()

	s.labels = os.Getenv("STACK_LABELS")

	frameworkInfo := &mesos.FrameworkInfo{
		User:       proto.String(Config.User),
		Name:       proto.String(Config.FrameworkName),
		Role:       proto.String(Config.FrameworkRole),
		Checkpoint: proto.Bool(true),
		Labels:     utils.StringToLabels(s.labels),
	}

	driverConfig := scheduler.DriverConfig{
		Scheduler: s,
		Framework: frameworkInfo,
		Master:    Config.Master,
	}

	driver, err := scheduler.NewMesosSchedulerDriver(driverConfig)
	go func() {
		<-ctrlc
		s.Shutdown(driver)
	}()

	if err != nil {
		return fmt.Errorf("Unable to create SchedulerDriver: %s", err)
	}

	if stat, err := driver.Run(); err != nil {
		Logger.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err)
		return err
	}

	//TODO stop http server

	return nil
}
Beispiel #8
0
func main() {
	flag.Parse()

	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	frameworkInfo := &mesosproto.FrameworkInfo{
		User: proto.String(*user),
		Name: proto.String("Syphon Framework"),
	}

	schedulerConfig := framework.NewElodinaTransportSchedulerConfig()
	schedulerConfig.CpuPerTask = *cpuPerTask
	schedulerConfig.MemPerTask = *memPerTask
	schedulerConfig.Topics = strings.Split(*topics, ",")
	schedulerConfig.ExecutorBinaryName = "executor"
	schedulerConfig.ServiceHost = *artifactServerHost
	schedulerConfig.ServicePort = *artifactServerPort
	schedulerConfig.ThreadsPerTask = *threadsPerTask
	schedulerConfig.TargetURL = *targetUrl
	schedulerConfig.SSLCertFilePath = *certFile
	schedulerConfig.SSLKeyFilePath = *keyFile
	schedulerConfig.SSLCACertFilePath = *caFile
	schedulerConfig.ApiKey = *apiKey
	schedulerConfig.ApiUser = *apiUser
	schedulerConfig.Insecure = *insecure
	schedulerConfig.ConsumerConfig = mustReadConsumerConfig(*consumerConfigPath)

	transportScheduler := framework.NewElodinaTransportScheduler(schedulerConfig)
	driverConfig := scheduler.DriverConfig{
		Scheduler: transportScheduler,
		Framework: frameworkInfo,
		Master:    *master,
	}

	driver, err := scheduler.NewMesosSchedulerDriver(driverConfig)
	go func() {
		<-ctrlc
		transportScheduler.Shutdown(driver)
		driver.Stop(false)
	}()

	if err != nil {
		fmt.Println("Unable to create a SchedulerDriver ", err.Error())
	}

	go startArtifactServer()

	if stat, err := driver.Run(); err != nil {
		fmt.Println("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
	}
}
Beispiel #9
0
func NewDriver(master string, s sched.Scheduler) (sched.SchedulerDriver, error) {
	if master == "" {
		return nil, errors.New("driver: empty master")
	}
	return sched.NewMesosSchedulerDriver(sched.DriverConfig{
		Master: master,
		Framework: &pb.FrameworkInfo{
			Name: proto.String("Fuzzlr"),
			// User: proto.String(""),
		},
		Scheduler: s,
	})
}
Beispiel #10
0
func main() {

	// Start HTTP server hosting executor binary
	uri := ServeExecutorArtifact(*address, *artifactPort, *executorPath)

	// Executor
	exec := prepareExecutorInfo(uri, getExecutorCmd(*executorPath))

	// Scheduler
	numTasks, err := strconv.Atoi(*taskCount)
	if err != nil {
		log.Fatalf("Failed to convert '%v' to an integer with error: %v\n", taskCount, err)
		os.Exit(-1)
	}

	scheduler := NewExampleScheduler(exec, numTasks, CPUS_PER_TASK, MEM_PER_TASK, *externalServer)
	if err != nil {
		log.Fatalf("Failed to create scheduler with error: %v\n", err)
		os.Exit(-2)
	}

	//Start trigger server
	go trigger.RunTriggerServer(scheduler)

	// Framework
	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""), // Mesos-go will fill in user.
		Name: proto.String("Test Framework (Go)"),
	}

	// Scheduler Driver
	config := sched.DriverConfig{
		Scheduler:      scheduler,
		Framework:      fwinfo,
		Master:         *master,
		Credential:     (*mesos.Credential)(nil),
		BindingAddress: parseIP(*address),
	}

	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Fatalf("Unable to create a SchedulerDriver: %v\n", err.Error())
		os.Exit(-3)
	}

	if stat, err := driver.Run(); err != nil {
		log.Fatalf("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
		os.Exit(-4)
	}
}
Beispiel #11
0
// NewScheduler for Scheduler mesos driver creation
func NewScheduler(config mesosscheduler.DriverConfig, cluster *Cluster, sched *scheduler.Scheduler) (*Scheduler, error) {
	scheduler := Scheduler{
		Scheduler: *sched,
		cluster:   cluster,
	}

	config.Scheduler = &scheduler
	driver, err := mesosscheduler.NewMesosSchedulerDriver(config)
	if err != nil {
		return nil, err
	}
	scheduler.driver = driver
	return &scheduler, nil
}
Beispiel #12
0
func main() {
	seedURL := flag.String("seed", "http://mesosphere.com", "The first URL to crawl")
	master := flag.String("master", "127.0.1.1:5050", "Location of leading Mesos master")
	localMode := flag.Bool("local", true, "If true, saves rendered web pages on local disk")

	flag.Parse()

	scheduler := newRendlerScheduler(*seedURL, *localMode)
	driver, err := sched.NewMesosSchedulerDriver(sched.DriverConfig{
		Master: *master,
		Framework: &mesos.FrameworkInfo{
			Name: proto.String("RENDLER"),
			User: proto.String(""),
		},
		Scheduler: scheduler,
	})
	if err != nil {
		log.Printf("Unable to create scheduler driver: %s", err)
		return
	}

	// Catch interrupt
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt, os.Kill)
		s := <-c
		if s != os.Interrupt {
			return
		}

		log.Println("RENDLER is shutting down")
		close(scheduler.shutdown)

		select {
		case <-scheduler.done:
		case <-time.After(shutdownTimeout):
		}

		if err := rendler.WriteDOTFile(scheduler.crawlResults, scheduler.renderResults); err != nil {
			log.Printf("Could not write DOT file: %s", err)
		}
		// we have shut down
		driver.Stop(false)
	}()

	if status, err := driver.Run(); err != nil {
		log.Printf("Framework stopped with status %s and error: %s\n", status.String(), err.Error())
	}
	log.Println("Exiting...")
}
Beispiel #13
0
func InitializeScheduler() (*DiegoScheduler, *sched.MesosSchedulerDriver) {
	exec := prepareExecutorInfo()
	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""), // Mesos-go will fill in user.
		Name: proto.String("Diego Scheduler"),
	}

	cred := (*mesos.Credential)(nil)
	if *mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(*mesosAuthPrincipal)
		secret_bytes, err := ioutil.ReadFile(*mesosAuthSecretFile)
		secret := string(secret_bytes[:])
		if err != nil {
			log.Fatal(err)
		}
		cred = &mesos.Credential{
			Principal: proto.String(*mesosAuthPrincipal),
			Secret:    &secret,
		}
	}
	bindingAddressParsed := parseIP(*bindingAddress)
	publishedAddressParsed := bindingAddressParsed
	if *publishedAddress != "" {
		publishedAddressParsed = parseIP(*publishedAddress)
	}

	digoScheduler := NewDiegoScheduler(exec)
	config := sched.DriverConfig{
		Scheduler:        digoScheduler,
		Framework:        fwinfo,
		Master:           *master,
		Credential:       cred,
		BindingAddress:   bindingAddressParsed,
		PublishedAddress: publishedAddressParsed,
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, *authProvider)
			ctx = sasl.WithBindingAddress(ctx, bindingAddressParsed)
			return ctx
		},
	}
	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Fatal("Unable to create a SchedulerDriver ", err.Error())
	}

	return digoScheduler, driver

}
Beispiel #14
0
func NewDriver(master string, s sched.Scheduler) (sched.SchedulerDriver, error) {
	if master == "" {
		return nil, errors.New("driver: empty master")
	}
	return sched.NewMesosSchedulerDriver(sched.DriverConfig{
		Master: master,
		Framework: &pb.FrameworkInfo{
			Name:            proto.String("Fuzzlr"),
			Checkpoint:      proto.Bool(true),
			FailoverTimeout: proto.Float64(60 * 60 * 24 * 7),
			// User: proto.String(""),
		},
		Scheduler: s,
	})
}
Beispiel #15
0
func main() {
	if *version {
		fmt.Printf("NONE v%s\n", VERSION)
		os.Exit(0)
	}

	workdirPath := tarWorkdir()
	if workdirPath != nil {
		defer os.Remove(*workdirPath)
	}
	uris = exportArtifacts(workdirPath)
	containerInfo = prepareContainer()

	cmdq := NewCommandQueue()
	cs, err := ParseConstraints(constraints)
	if err != nil {
		log.Errorln("Error parsing constraints", err)
		os.Exit(10)
	}
	handler := NewCommandHandler()
	scheduler := NewNoneScheduler(cmdq, handler, prepareResourceFilter(cs))

	fwinfo := prepareFrameworkInfo()
	cred := prepateCredentials(fwinfo)
	config, err := prepareDriver(scheduler, NewZkLeaderDetector(), fwinfo, cred)
	if err != nil {
		log.Errorln("Unable to create a mesos driver:", err.Error())
		os.Exit(10)
	}

	driver, err := sched.NewMesosSchedulerDriver(config)
	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver:", err.Error())
		os.Exit(10)
	}

	queueCommands(cmdq)

	// run the driver and wait for it to finish
	if stat, err := driver.Run(); err != nil {
		log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
		os.Exit(2)
	}

	if handler.HasFailures() {
		os.Exit(1)
	}
}
Beispiel #16
0
func main() {
	execUri, execCmd = serveExecutorArtifact(*executorPath)

	// the framework
	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""), // Mesos-go will fill in user.
		Name: proto.String("Test Framework (Go)"),
		Role: proto.String(*role),
	}

	cred := (*mesos.Credential)(nil)
	if *mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(*mesosAuthPrincipal)
		if *mesosAuthSecretFile != "" {
			secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
			if err != nil {
				log.Fatal(err)
			}
			cred = &mesos.Credential{
				Principal: proto.String(*mesosAuthPrincipal),
				Secret:    secret,
			}
		}
	}
	bindingAddress := parseIP(*address)
	config := sched.DriverConfig{
		Scheduler:      newExampleScheduler(),
		Framework:      fwinfo,
		Master:         *master,
		Credential:     cred,
		BindingAddress: bindingAddress,
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, *authProvider)
			ctx = sasl.WithBindingAddress(ctx, bindingAddress)
			return ctx
		},
	}
	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver ", err.Error())
	}

	if stat, err := driver.Run(); err != nil {
		log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
	}
	log.Infof("framework terminating")
}
func (s *Scheduler) Start() error {
	Logger.Infof("Starting scheduler with configuration: \n%s", Config)
	sched = s // set this scheduler reachable for http server

	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	s.cluster = NewCluster()
	s.cluster.Load()

	s.httpServer = NewHttpServer(Config.Api)
	go s.httpServer.Start()

	frameworkInfo := &mesos.FrameworkInfo{
		User:            proto.String(Config.User),
		Name:            proto.String(Config.FrameworkName),
		Role:            proto.String(Config.FrameworkRole),
		FailoverTimeout: proto.Float64(float64(Config.FrameworkTimeout / 1e9)),
		Checkpoint:      proto.Bool(true),
	}

	if s.cluster.frameworkID != "" {
		frameworkInfo.Id = util.NewFrameworkID(s.cluster.frameworkID)
	}

	driverConfig := scheduler.DriverConfig{
		Scheduler: s,
		Framework: frameworkInfo,
		Master:    Config.Master,
	}

	driver, err := scheduler.NewMesosSchedulerDriver(driverConfig)
	s.schedulerDriver = driver

	if err != nil {
		return fmt.Errorf("Unable to create SchedulerDriver: %s", err)
	}

	go func() {
		if stat, err := driver.Run(); err != nil {
			Logger.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err)
			panic(err)
		}
	}()

	<-ctrlc
	return nil
}
Beispiel #18
0
func createDriver(scheduler *eremeticScheduler) (*sched.MesosSchedulerDriver, error) {
	publishedAddr := net.ParseIP(viper.GetString("messenger_address"))
	bindingPort := uint16(viper.GetInt("messenger_port"))

	return sched.NewMesosSchedulerDriver(sched.DriverConfig{
		Master: viper.GetString("master"),
		Framework: &mesos.FrameworkInfo{
			Name: proto.String("Eremetic"),
			User: proto.String(""),
		},
		Scheduler:        scheduler,
		BindingAddress:   net.ParseIP("0.0.0.0"),
		PublishedAddress: publishedAddr,
		BindingPort:      bindingPort,
	})
}
Beispiel #19
0
func main() {
	parseAndValidateSchedulerArgs()

	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	frameworkInfo := &mesosproto.FrameworkInfo{
		User: proto.String(""),
		Name: proto.String("Go LogLine Transform Framework"),
	}

	schedulerConfig := transform.NewTransformSchedulerConfig()
	schedulerConfig.ArtifactServerHost = *artifactServerHost
	schedulerConfig.ArtifactServerPort = *artifactServerPort
	schedulerConfig.ExecutorArchiveName = *executorArchiveName
	schedulerConfig.ExecutorBinaryName = *executorBinaryName
	schedulerConfig.Instances = *instances
	schedulerConfig.CpuPerTask = *cpuPerTask
	schedulerConfig.MemPerTask = *memPerTask
	schedulerConfig.ProducerConfig = *producerConfig
	schedulerConfig.Topic = *topic
	schedulerConfig.Sync = *sync

	go startArtifactServer(schedulerConfig)

	transformScheduler := transform.NewTransformScheduler(schedulerConfig)
	driverConfig := scheduler.DriverConfig{
		Scheduler: transformScheduler,
		Framework: frameworkInfo,
		Master:    *master,
	}

	driver, err := scheduler.NewMesosSchedulerDriver(driverConfig)
	go func() {
		<-ctrlc
		transformScheduler.Shutdown(driver)
		driver.Stop(false)
	}()

	if err != nil {
		fmt.Println("Unable to create a SchedulerDriver ", err.Error())
	}

	if stat, err := driver.Run(); err != nil {
		fmt.Println("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
	}
}
Beispiel #20
0
func (s *StackDeployScheduler) Start() error {
	Logger.Info("Starting scheduler")

	frameworkInfo := &mesos.FrameworkInfo{
		User:            proto.String(s.User),
		Name:            proto.String(s.FrameworkName),
		Role:            proto.String(s.FrameworkRole),
		FailoverTimeout: proto.Float64(float64(s.FailoverTimeout / 1e9)),
		Checkpoint:      proto.Bool(true),
	}

	if s.Storage.FrameworkID != "" {
		frameworkInfo.Id = mesosutil.NewFrameworkID(s.Storage.FrameworkID)
	}

	driverConfig := scheduler.DriverConfig{
		Scheduler: s,
		Framework: frameworkInfo,
		Master:    s.Master,
	}

	driver, err := scheduler.NewMesosSchedulerDriver(driverConfig)

	if err != nil {
		return fmt.Errorf("Unable to create SchedulerDriver: %s", err)
	}

	go func() {
		if stat, err := driver.Run(); err != nil {
			Logger.Info("Framework stopped with status %s and error: %s", stat.String(), err)
			panic(err)
		}
	}()

	s.Cron.Start()
	go func() {
		for {
			Logger.Info("Cron entries: %v\n", s.Cron.Entries())
			for _, entry := range s.Cron.Entries() {
				Logger.Info("Entry: %v, %v, %v, %v", entry.Prev, entry.Next, entry.Schedule, entry.Job)
			}
			time.Sleep(10 * time.Second)
		}
	}()

	return nil
}
Beispiel #21
0
func main() {
	var master = flag.String("master", "127.0.0.1:5050", "Master address <ip:port>")

	log.Infoln("Lancement PGAgentScheduler") //, time.Now())

	flag.Parse()

	config := scheduler.DriverConfig{
		Master: *master,
		Scheduler: forbin.NewDatabaseScheduler(
			&mesos.ExecutorInfo{
				ExecutorId: util.NewExecutorID("default"),
				Name:       proto.String("FBN"),
				Source:     proto.String("fb_test"),
				Command: &mesos.CommandInfo{
					Value: proto.String(CMD),
					Uris: []*mesos.CommandInfo_URI{
						&mesos.CommandInfo_URI{
							Value:   proto.String("http://localhost:8080/postgresql-bin.tar.gz"),
							Extract: proto.Bool(true),
						},
						&mesos.CommandInfo_URI{
							Value:   proto.String("http://localhost:8080/tools.tar.gz"),
							Extract: proto.Bool(true),
						},
					}},
			},
		),
		Framework: &mesos.FrameworkInfo{
			Name: proto.String(NAME),
			User: proto.String(""),
		},
	}

	driver, err := scheduler.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Fatalln("Unable to create a SchedulerDriver ", err.Error())
	}

	if stat, err := driver.Run(); err != nil {
		log.Infoln("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
	}

}
Beispiel #22
0
func createDriver(scheduler *eremeticScheduler) (*sched.MesosSchedulerDriver, error) {
	publishedAddr := net.ParseIP(viper.GetString("messenger_address"))
	bindingPort := uint16(viper.GetInt("messenger_port"))

	return sched.NewMesosSchedulerDriver(sched.DriverConfig{
		Master: viper.GetString("master"),
		Framework: &mesos.FrameworkInfo{
			Id:              getFrameworkID(),
			Name:            proto.String(viper.GetString("name")),
			User:            proto.String(viper.GetString("user")),
			Checkpoint:      proto.Bool(viper.GetBool("checkpoint")),
			FailoverTimeout: proto.Float64(viper.GetFloat64("failover_timeout")),
		},
		Scheduler:        scheduler,
		BindingAddress:   net.ParseIP("0.0.0.0"),
		PublishedAddress: publishedAddr,
		BindingPort:      bindingPort,
	})
}
Beispiel #23
0
func runScheduler() error {
	config := sched.DriverConfig{
		Scheduler: newScheduler(),
		Framework: &mesos.FrameworkInfo{
			User: proto.String(""),
			Name: proto.String("Elevators"),
		},
		Master: "127.0.0.1:5050",
	}
	driver, err := sched.NewMesosSchedulerDriver(config)
	if err != nil {
		return err
	}
	status, err := driver.Run()
	if err != nil {
		return fmt.Errorf("framework stopped with %s: %s", status, err)
	}
	return nil
}
func main() {

	log.Info("Creating scheduler")
	scheduler, err := NewScheduler()
	if err != nil {
		log.Fatalf("Unable to create scheduler: %s\n", err.Error())
		os.Exit(1)
	}
	log.Infof("Created scheduler %+v\n", scheduler)

	log.Info("Creating framework info")
	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""),
		Name: proto.String("basicdocker-2"),
	}
	log.Infof("Created fwinfo %+v\n", fwinfo)

	log.Info("Creating scheduler driver config")
	config := sched.DriverConfig{
		Scheduler:  scheduler,
		Framework:  fwinfo,
		Master:     *master,
		Credential: (*mesos.Credential)(nil),
	}
	log.Infof("Created driver config %+v\n", config)

	log.Info("Creating new scheduler driver from config")
	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Fatalf("Unable to create a SchedulerDriver: %v\n", err.Error())
		os.Exit(3)
	}
	log.Infof("Created scheduler driver %+v\n", driver)

	log.Info("Starting scheduler driver")
	if stat, err := driver.Run(); err != nil {
		log.Fatalf("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
		os.Exit(4)
	}
}
Beispiel #25
0
func main() {
	// Start HTTP server hosting executor binary
	uri := ServeSchedulerAPI(*address, *artifactPort)

	// Executor
	executorInfo := prepareExecutorInfo(uri, getExecutorCmd(*executorPath))

	// Scheduler
	scheduler, err := NewScraperScheduler(executorInfo, CPUS_PER_TASK, MEM_PER_TASK)
	if err != nil {
		log.Fatalf("Failed to create scheduler with error: %v\n", err)
		os.Exit(-2)
	}

	// Framework
	frameworkInfo := &mesos.FrameworkInfo{
		User: proto.String(""), // Mesos-go will fill in user.
		Name: proto.String("Web Scraper"),
	}

	// Scheduler Driver
	config := sched.DriverConfig{
		Scheduler:      scheduler,
		Framework:      frameworkInfo,
		Master:         *master,
		Credential:     (*mesos.Credential)(nil),
		BindingAddress: parseIP(*address),
	}

	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Fatalf("Unable to create a SchedulerDriver: %v\n", err.Error())
		os.Exit(-3)
	}

	if stat, err := driver.Run(); err != nil {
		log.Fatalf("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
		os.Exit(-4)
	}
}
Beispiel #26
0
func main() {
	log.V(1).Infoln("Initializing the Example Scheduler...")
	config = loadConfig()

	// the framework
	fwinfo := fwinfo()

	driver, err := sched.NewMesosSchedulerDriver(driverConfig(fwinfo))
	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver ", err.Error())
	}

	// Don't block on this call because we need printLogs to be the process blocker to ensure we retrieve all the logs.
	go func() {
		if stat, err := driver.Run(); err != nil {
			log.V(1).Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
		}
	}()

	printLogs()
}
func NewScheduler(mesosMaster, bridgeCIDR string) error {
	log.Info("Rancher Mesos Scheduler started")
	scheduler := &rancherScheduler{
		rancherExecutor: &mesos.ExecutorInfo{
			Command: &mesos.CommandInfo{
				Value: proto.String(cmd + " --bridge_cidr=" + bridgeCIDR),
				Uris:  []*mesos.CommandInfo_URI{},
			},
			ExecutorId: &mesos.ExecutorID{Value: proto.String("rancher-mesos-executor")},
			Name:       proto.String("RancherExecutor"),
		},
	}
	driver, err := sched.NewMesosSchedulerDriver(sched.DriverConfig{
		Master: mesosMaster,
		Framework: &mesos.FrameworkInfo{
			Name: proto.String("RancherMesos"),
			User: proto.String("root"), // make this configurable
		},
		Scheduler: scheduler,
	})

	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
		}).Error("Error starting scheduler driver")
		return err
	}

	_, err = driver.Run()
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
		}).Error("Error Running driver")
		return err
	}
	_, err = driver.Stop(false)
	return err
}
Beispiel #28
0
func (self *MesosManager) Start(engine *agro_engine.WorkEngine) {
	self.engine = engine
	self.executor = self.BuildExecutorInfo()

	//frameworkIdStr := FRAMEWORK_ID
	failoverTimeout := 0.0
	//frameworkId := &mesos.FrameworkID{Value: &frameworkIdStr}
	config := sched.DriverConfig{
		Master: self.master,
		Framework: &mesos.FrameworkInfo{
			Name:            proto.String("AgroFramework"),
			User:            proto.String(""),
			FailoverTimeout: &failoverTimeout,
			//Id:              frameworkId,
		},
		Scheduler: self,
	}

	driver, err := sched.NewMesosSchedulerDriver(config)
	if err != nil {
		log.Printf("Driver Error: %s", err)
		panic(err)
	}
	//driver.Init()
	//defer driver.Destroy()
	//go self.EventLoop()

	status, err := driver.Start()
	log.Printf("Driver Status:%s", status)
	if err != nil {
		log.Printf("Mesos Start Error: %s", err)
		panic(err)
	}
	//<-self.exit
	//log.Printf("Mesos Exit")
	//driver.Stop(false)
}
Beispiel #29
0
// NewFramework initializes Taurus framework with the provided configuration options
//
// NewFramework initializes Mesos Scheduler driver and creates HTTP API server
// It returns error if either Mesos driver or HTTP API server could not be created
func NewFramework(config *Config) (*Taurus, error) {
	fwInfo := &mesos.FrameworkInfo{
		User: proto.String(config.User),
		Name: proto.String(FrameworkName),
	}

	sched, err := NewScheduler(config.Worker, config.Master)
	if err != nil {
		return nil, fmt.Errorf("Unable to create %s Scheduler: %s", FrameworkName, err)
	}
	driverConfig := scheduler.DriverConfig{
		Scheduler: sched,
		Framework: fwInfo,
		Master:    config.Master,
	}

	driver, err := scheduler.NewMesosSchedulerDriver(driverConfig)
	if err != nil {
		return nil, fmt.Errorf("Unable to create a SchedulerDriver: %s", err)
	}

	api, err := NewApi(&ApiConfig{
		Address:   config.ListenAddr,
		TlsConfig: config.TlsConfig,
		Store:     config.Store,
	})
	if err != nil {
		return nil, fmt.Errorf("Could not create %s API server: %s", FrameworkName, err)
	}

	return &Taurus{
		fwInfo:    fwInfo,
		driver:    driver,
		scheduler: sched,
		api:       api,
	}, nil
}
Beispiel #30
0
// NewCluster for mesos Cluster creation
func NewCluster(scheduler *scheduler.Scheduler, store *state.Store, TLSConfig *tls.Config, master string, options cluster.DriverOpts) (cluster.Cluster, error) {
	log.WithFields(log.Fields{"name": "mesos"}).Debug("Initializing cluster")

	cluster := &Cluster{
		dockerEnginePort:    defaultDockerEnginePort,
		master:              master,
		slaves:              make(map[string]*slave),
		scheduler:           scheduler,
		store:               store,
		TLSConfig:           TLSConfig,
		options:             &options,
		offerTimeout:        defaultOfferTimeout,
		taskCreationTimeout: defaultTaskCreationTimeout,
	}

	cluster.pendingTasks = queue.NewQueue()

	// Empty string is accepted by the scheduler.
	user, _ := options.String("mesos.user", "SWARM_MESOS_USER")

	// Override the hostname here because mesos-go will try
	// to shell out to the hostname binary and it won't work with our official image.
	// Do not check error here, so mesos-go can still try.
	hostname, _ := os.Hostname()

	driverConfig := mesosscheduler.DriverConfig{
		Scheduler:        cluster,
		Framework:        &mesosproto.FrameworkInfo{Name: proto.String(frameworkName), User: &user},
		Master:           cluster.master,
		HostnameOverride: hostname,
	}

	if taskCreationTimeout, ok := options.String("mesos.tasktimeout", "SWARM_MESOS_TASK_TIMEOUT"); ok {
		d, err := time.ParseDuration(taskCreationTimeout)
		if err != nil {
			return nil, err
		}
		cluster.taskCreationTimeout = d
	}
	// Changing port for https
	if cluster.TLSConfig != nil {
		cluster.dockerEnginePort = defaultDockerEngineTLSPort
	}

	if bindingPort, ok := options.Uint("mesos.port", "SWARM_MESOS_PORT"); ok {
		driverConfig.BindingPort = uint16(bindingPort)
	}

	if bindingAddress, ok := options.IP("mesos.address", "SWARM_MESOS_ADDRESS"); ok {
		if bindingAddress == nil {
			return nil, fmt.Errorf("invalid address %s", bindingAddress)
		}
		driverConfig.BindingAddress = bindingAddress
	}

	if offerTimeout, ok := options.String("mesos.offertimeout", "SWARM_MESOS_OFFER_TIMEOUT"); ok {
		d, err := time.ParseDuration(offerTimeout)
		if err != nil {
			return nil, err
		}
		cluster.offerTimeout = d
	}

	driver, err := mesosscheduler.NewMesosSchedulerDriver(driverConfig)
	if err != nil {
		return nil, err
	}

	cluster.driver = driver

	status, err := driver.Start()
	if err != nil {
		log.Debugf("Mesos driver started, status/err %v: %v", status, err)
		return nil, err
	}
	log.Debugf("Mesos driver started, status %v", status)

	return cluster, nil
}