func (sc *SchedulerCore) acceptOffer(driver sched.SchedulerDriver, offer *mesos.Offer, operations []*mesos.Offer_Operation) { log.Infof("Accepting OfferID: %+v, Operations: %+v", *offer.Id.Value, operations) var status mesos.Status var err error if sc.compatibilityMode { tasks := []*mesos.TaskInfo{} for _, operation := range operations { if *operation.Type == mesos.Offer_Operation_LAUNCH { tasks = operation.Launch.TaskInfos } } status, err = driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(OFFER_INTERVAL)}) } else { status, err = driver.AcceptOffers([]*mesos.OfferID{offer.Id}, operations, &mesos.Filters{RefuseSeconds: proto.Float64(OFFER_INTERVAL)}) } if status != mesos.Status_DRIVER_RUNNING { log.Fatal("Driver not running, while trying to accept offers") } if err != nil { log.Panic("Failed to launch tasks: ", err) } }
func (s *Scheduler) launchTask(driver scheduler.SchedulerDriver, offer *mesos.Offer) { taskName := fmt.Sprintf("syslog-%s", offer.GetSlaveId().GetValue()) taskId := &mesos.TaskID{ Value: proto.String(fmt.Sprintf("%s-%s", taskName, uuid())), } data, err := json.Marshal(Config) if err != nil { panic(err) //shouldn't happen } Logger.Debugf("Task data: %s", string(data)) tcpPort := uint64(s.getPort(Config.TcpPort, offer, -1)) udpPort := uint64(s.getPort(Config.UdpPort, offer, int(tcpPort))) task := &mesos.TaskInfo{ Name: proto.String(taskName), TaskId: taskId, SlaveId: offer.GetSlaveId(), Executor: s.createExecutor(offer, tcpPort, udpPort), Resources: []*mesos.Resource{ util.NewScalarResource("cpus", Config.Cpus), util.NewScalarResource("mem", Config.Mem), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(tcpPort, tcpPort)}), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(udpPort, udpPort)}), }, Data: data, Labels: utils.StringToLabels(s.labels), } s.cluster.Add(offer.GetSlaveId().GetValue(), task) driver.LaunchTasks([]*mesos.OfferID{offer.GetId()}, []*mesos.TaskInfo{task}, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) }
func (sched *Scheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { logOffers(offers) jobs, err := getLaunchableJobs() if err != nil { log.Errorf("Unable to get pending jobs! %s\n", err.Error()) return } offersAndTasks, err := packJobsInOffers(jobs, offers) if err != nil { log.Errorf("Unable to pack jobs into offers! %s\n", err.Error()) return } for _, ot := range offersAndTasks { if len(ot.Tasks) == 0 { log.Infof("Declining unused offer %s", ot.Offer.Id.GetValue()) driver.DeclineOffer(ot.Offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) continue } else { log.Infof("Launching %d tasks for offer %s\n", len(ot.Tasks), ot.Offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{ot.Offer.Id}, ot.Tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) sched.tasksLaunched = sched.tasksLaunched + len(ot.Tasks) } } }
func (s *MemcacheScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { log.Printf("Received %d resource offers", len(offers)) for _, offer := range offers { select { case <-s.shutdown: log.Println("Shutting down: declining offer on [", offer.Hostname, "]") driver.DeclineOffer(offer.Id, defaultFilter) if s.tasksRunning == 0 { close(s.done) } continue default: } tasks := []*mesos.TaskInfo{} if canLaunchNewTask(offer) && s.shouldLaunchNewTask() { fmt.Println("Accepting Offer: ", offer) task := s.newMemcacheTask(offer) tasks = append(tasks, task) } if len(tasks) == 0 { driver.DeclineOffer(offer.Id, defaultFilter) } else { driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, defaultFilter) s.lastTaskCreatedAt = time.Now() return // limit one at a time } } }
func (s *Scheduler) launchTask(driver scheduler.SchedulerDriver, offer *mesos.Offer) { taskName := fmt.Sprintf("syscol-%s", offer.GetSlaveId().GetValue()) taskId := &mesos.TaskID{ Value: proto.String(fmt.Sprintf("%s-%s", taskName, uuid())), } data, err := json.Marshal(Config) if err != nil { panic(err) //shouldn't happen } Logger.Debugf("Task data: %s", string(data)) task := &mesos.TaskInfo{ Name: proto.String(taskName), TaskId: taskId, SlaveId: offer.GetSlaveId(), Executor: s.createExecutor(offer.GetSlaveId().GetValue()), Resources: []*mesos.Resource{ util.NewScalarResource("cpus", Config.Cpus), util.NewScalarResource("mem", Config.Mem), }, Data: data, } s.cluster.Add(offer.GetSlaveId().GetValue(), task) driver.LaunchTasks([]*mesos.OfferID{offer.GetId()}, []*mesos.TaskInfo{task}, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) }
// ResourceOffers handles the Resource Offers func (s *Scheduler) ResourceOffers(driver mesossched.SchedulerDriver, offers []*mesosproto.Offer) { logrus.WithField("offers", len(offers)).Debug("Received offers") var offer *mesosproto.Offer loop: for len(offers) > 0 { select { case <-s.shutdown: logrus.Info("Shutting down: declining offers") break loop case tid := <-s.tasks: logrus.WithField("task_id", tid).Debug("Trying to find offer to launch task with") t, _ := s.database.ReadUnmaskedTask(tid) if t.IsTerminating() { logrus.Debug("Dropping terminating task.") t.UpdateStatus(eremetic.Status{ Status: eremetic.TaskKilled, Time: time.Now().Unix(), }) s.database.PutTask(&t) continue } offer, offers = matchOffer(t, offers) if offer == nil { logrus.WithField("task_id", tid).Warn("Unable to find a matching offer") tasksDelayed.Inc() go func() { s.tasks <- tid }() break loop } logrus.WithFields(logrus.Fields{ "task_id": tid, "offer_id": offer.Id.GetValue(), }).Debug("Preparing to launch task") t, task := createTaskInfo(t, offer) t.UpdateStatus(eremetic.Status{ Status: eremetic.TaskStaging, Time: time.Now().Unix(), }) s.database.PutTask(&t) driver.LaunchTasks([]*mesosproto.OfferID{offer.Id}, []*mesosproto.TaskInfo{task}, defaultFilter) tasksLaunched.Inc() queueSize.Dec() continue default: break loop } } logrus.Debug("No tasks to launch. Declining offers.") for _, offer := range offers { driver.DeclineOffer(offer.Id, defaultFilter) } }
func (s *rancherScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { task := tasks.GetNextTask() if task == nil { for _, of := range offers { driver.DeclineOffer(of.Id, defaultFilter) } return } if task.RegistrationUrl == "" { tasks.AddTask(task) for _, of := range offers { driver.DeclineOffer(of.Id, defaultFilter) } return } taskBytes, err := task.Marshal() if err != nil { log.WithFields(log.Fields{ "err": err, }).Error("Error Marshalling task") for _, of := range offers { driver.DeclineOffer(of.Id, defaultFilter) } return } for _, offer := range offers { inadequate := false for _, res := range offer.GetResources() { if res.GetName() == "cpus" && *res.GetScalar().Value < taskCPUs { driver.DeclineOffer(offer.Id, defaultFilter) inadequate = true continue } if res.GetName() == "mem" && *res.GetScalar().Value < taskMem { driver.DeclineOffer(offer.Id, defaultFilter) inadequate = true continue } } if inadequate { continue } mesosTask := &mesos.TaskInfo{ TaskId: &mesos.TaskID{ Value: proto.String(task.HostUuid), }, SlaveId: offer.SlaveId, Resources: []*mesos.Resource{ mesosutil.NewScalarResource("cpus", taskCPUs), mesosutil.NewScalarResource("mem", taskMem), }, Data: taskBytes, Name: &task.Name, Executor: s.rancherExecutor, } driver.LaunchTasks([]*mesos.OfferID{offer.Id}, []*mesos.TaskInfo{mesosTask}, defaultFilter) } }
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { logOffers(offers) for _, offer := range offers { remainingCpus := getOfferCpu(offer) remainingMems := getOfferMem(offer) var tasks []*mesos.TaskInfo for sched.cpuPerTask <= remainingCpus && sched.memPerTask <= remainingMems && sched.tasksLaunched < sched.totalTasks { fmt.Printf("Tasks launched: %v Total tasks: %v\n", sched.tasksLaunched, sched.totalTasks) sched.tasksLaunched++ taskId := &mesos.TaskID{ Value: proto.String(strconv.Itoa(sched.tasksLaunched)), } dockerInfo := &mesos.ContainerInfo_DockerInfo{ Image: &sched.DockerImage, PortMappings: sched.DockerPorts, } containerType := mesos.ContainerInfo_DOCKER containerInfo := &mesos.ContainerInfo{ Type: &containerType, Docker: dockerInfo, } commandInfo := &mesos.CommandInfo{ Value: &sched.DockerCommand, } task := &mesos.TaskInfo{ Name: proto.String("go-task-" + taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", sched.cpuPerTask), util.NewScalarResource("mem", sched.memPerTask), }, Container: containerInfo, Command: commandInfo, } fmt.Printf("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) tasks = append(tasks, task) remainingCpus -= sched.cpuPerTask remainingMems -= sched.memPerTask } // fmt.Println("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) } }
func (sched *Scheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { for _, offer := range offers { taskId := &mesos.TaskID{ Value: proto.String(fmt.Sprintf("basicdocker-task-%d", time.Now().Unix())), } ports := util.FilterResources( offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "ports" }, ) if len(ports) > 0 && len(ports[0].GetRanges().GetRange()) > 0 { } else { return } task := &mesos.TaskInfo{ Name: proto.String(taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Container: &mesos.ContainerInfo{ Type: mesos.ContainerInfo_DOCKER.Enum(), Volumes: nil, Hostname: nil, Docker: &mesos.ContainerInfo_DockerInfo{ Image: &DOCKER_IMAGE_DEFAULT, Network: mesos.ContainerInfo_DockerInfo_BRIDGE.Enum(), }, }, Command: &mesos.CommandInfo{ Shell: proto.Bool(true), Value: proto.String("set -x ; /bin/date ; /bin/hostname ; sleep 200 ; echo done"), }, Executor: nil, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", getOfferCpu(offer)), util.NewScalarResource("mem", getOfferMem(offer)), util.NewRangesResource("ports", []*mesos.Value_Range{ util.NewValueRange( *ports[0].GetRanges().GetRange()[0].Begin, *ports[0].GetRanges().GetRange()[0].Begin+1, ), }), }, } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) var tasks []*mesos.TaskInfo = []*mesos.TaskInfo{task} log.Infoln("Launching ", len(tasks), " tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) sched.tasksLaunched++ time.Sleep(time.Second) } }
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { for _, offer := range offers { cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "cpus" }) cpus := 0.0 for _, res := range cpuResources { cpus += res.GetScalar().GetValue() } memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "mem" }) mems := 0.0 for _, res := range memResources { mems += res.GetScalar().GetValue() } log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems) remainingCpus := cpus remainingMems := mems var tasks []*mesos.TaskInfo for sched.tasksLaunched < sched.totalTasks && CPUS_PER_TASK <= remainingCpus && MEM_PER_TASK <= remainingMems { sched.tasksLaunched++ taskId := &mesos.TaskID{ Value: proto.String(strconv.Itoa(sched.tasksLaunched)), } task := &mesos.TaskInfo{ Name: proto.String("go-task-" + taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Executor: sched.executor, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", CPUS_PER_TASK), util.NewScalarResource("mem", MEM_PER_TASK), }, } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) tasks = append(tasks, task) remainingCpus -= CPUS_PER_TASK remainingMems -= MEM_PER_TASK } log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) } }
func (ctx *RunOnceApplicationContext) LaunchTask(driver scheduler.SchedulerDriver, offer *mesos.Offer) error { ctx.lock.Lock() defer ctx.lock.Unlock() ctx.InstancesLeftToRun-- taskInfo := ctx.newTaskInfo(offer) ctx.tasks = append(ctx.tasks, newRunOnceTask(offer, taskInfo.GetTaskId().GetValue())) _, err := driver.LaunchTasks([]*mesos.OfferID{offer.GetId()}, []*mesos.TaskInfo{taskInfo}, &mesos.Filters{RefuseSeconds: proto.Float64(10)}) return err }
// mesos.Scheduler interface method. // Invoked when resources have been offered to this framework. func (this *ElodinaTransportScheduler) ResourceOffers(driver scheduler.SchedulerDriver, offers []*mesos.Offer) { log.Logger.Info("Received offers") offersAndTasks := make(map[*mesos.Offer][]*mesos.TaskInfo) remainingPartitions, err := this.GetTopicPartitions() if err != nil { return } remainingPartitions.RemoveAll(this.TakenTopicPartitions.GetArray()) log.Logger.Debug("%v", remainingPartitions) tps := remainingPartitions.GetArray() offersAndResources := this.wrapInOfferAndResources(offers) for !remainingPartitions.IsEmpty() { log.Logger.Debug("Iteration %v", remainingPartitions) if this.hasEnoughInstances() { for _, transfer := range this.taskIdToTaskState { if len(transfer.assignment) < this.config.ThreadsPerTask { transfer.assignment = append(transfer.assignment, tps[0]) remainingPartitions.Remove(tps[0]) this.TakenTopicPartitions.Add(tps[0]) if len(tps) > 1 { tps = tps[1:] } else { tps = []consumer.TopicAndPartition{} } } } } else { log.Logger.Debug("Trying to launch new task") offer, task := this.launchNewTask(offersAndResources) if offer != nil && task != nil { offersAndTasks[offer] = append(offersAndTasks[offer], task) } else { for _, offer := range offers { if _, exists := offersAndTasks[offer]; !exists { offersAndTasks[offer] = make([]*mesos.TaskInfo, 0) } } break } } } this.assignPendingPartitions() for _, offer := range offers { if tasks, ok := offersAndTasks[offer]; ok { driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) } else { driver.DeclineOffer(offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(10)}) } } }
func (sc *SchedulerCore) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { sc.lock.Lock() defer sc.lock.Unlock() log.Info("Received resource offers: ", offers) launchTasks := make(map[string][]*mesos.TaskInfo) toBeScheduled := []*FrameworkRiakNode{} for _, cluster := range sc.schedulerState.Clusters { for _, riakNode := range cluster.Nodes { if riakNode.NeedsToBeScheduled() { log.Infof("Adding Riak node for scheduling: %+v", riakNode) // We need to schedule this task I guess? toBeScheduled = append(toBeScheduled, riakNode) } } } // Populate a mutable slice of offer resources allResources := [][]*mesos.Resource{} for _, offer := range offers { allResources = append(allResources, offer.Resources) } launchTasks, err := sc.spreadNodesAcrossOffers(offers, allResources, toBeScheduled, 0, 0, launchTasks) if err != nil { log.Error(err) } for _, offer := range offers { tasks := launchTasks[*offer.Id.Value] if tasks == nil { tasks = []*mesos.TaskInfo{} } // This is somewhat of a hack, to avoid synchronously calling the mesos-go SDK // to avoid a deadlock situation. // TODO: Fix and make actually queues around driver interactions // This is a massive hack // -Sargun Dhillon 2015-10-01 go func(offer *mesos.Offer) { log.Infof("Launching Tasks: %v for offer %v", tasks, *offer.Id.Value) status, err := driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(OFFER_INTERVAL)}) if status != mesos.Status_DRIVER_RUNNING { log.Fatal("Driver not running, while trying to launch tasks") } if err != nil { log.Panic("Failed to launch tasks: ", err) } }(offer) } }
func (s *DiegoScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { logOffers(offers) s.offersLock.Lock() defer s.offersLock.Unlock() if s.holdOffer { s.offers = append(s.offers, offers...) } else { offerIds := extractOfferIds(offers) driver.LaunchTasks(offerIds, nil, &mesos.Filters{RefuseSeconds: proto.Float64(30)}) } }
func (sched *ScraperScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { logOffers(offers) for _, offer := range offers { if sched.tasksLaunched >= sched.totalTasks || len(sched.urls) == 0 { log.Infof("Declining offer %s", offer.Id.GetValue()) driver.DeclineOffer(offer.Id, &mesos.Filters{}) continue } remainingCpus := getOfferCpu(offer) remainingMems := getOfferMem(offer) var tasks []*mesos.TaskInfo for sched.cpuPerTask <= remainingCpus && sched.memPerTask <= remainingMems && sched.tasksLaunched < sched.totalTasks { log.Infof("Processing url %v of %v\n", sched.tasksLaunched, sched.totalTasks) log.Infof("Total Tasks: %d", sched.totalTasks) log.Infof("Tasks Launched: %d", sched.tasksLaunched) uri := sched.urls[sched.tasksLaunched] log.Infof("URI: %s", uri) sched.tasksLaunched++ taskId := &mesos.TaskID{ Value: proto.String(strconv.Itoa(sched.tasksLaunched)), } task := &mesos.TaskInfo{ Name: proto.String("go-task-" + taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Executor: sched.executor, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", sched.cpuPerTask), util.NewScalarResource("mem", sched.memPerTask), }, Data: []byte(uri), } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) tasks = append(tasks, task) remainingCpus -= sched.cpuPerTask remainingMems -= sched.memPerTask } log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) } }
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { logOffers(offers) if sched.tasksLaunched >= sched.totalTasks { return } for _, offer := range offers { remainingCpus := getOfferCpu(offer) remainingMems := getOfferMem(offer) var tasks []*mesos.TaskInfo for sched.cpuPerTask <= remainingCpus && sched.memPerTask <= remainingMems && sched.tasksLaunched < sched.totalTasks { log.Infof("Processing image %v of %v\n", sched.tasksLaunched, sched.totalTasks) fileName := sched.images[sched.tasksLaunched] sched.tasksLaunched++ taskId := &mesos.TaskID{ Value: proto.String(strconv.Itoa(sched.tasksLaunched)), } task := &mesos.TaskInfo{ Name: proto.String("go-task-" + taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Executor: sched.executor, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", sched.cpuPerTask), util.NewScalarResource("mem", sched.memPerTask), }, Data: []byte(fileName), } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) tasks = append(tasks, task) remainingCpus -= sched.cpuPerTask remainingMems -= sched.memPerTask } log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) } }
func (s *rendlerScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { log.Printf("Received %d resource offers", len(offers)) for _, offer := range offers { select { case <-s.shutdown: log.Println("Shutting down: declining offer on [", offer.Hostname, "]") driver.DeclineOffer(offer.Id, defaultFilter) if s.tasksRunning == 0 { close(s.done) } continue default: } tasks := []*mesos.TaskInfo{} tasksToLaunch := maxTasksForOffer(offer) for tasksToLaunch > 0 { if s.crawlQueue.Front() != nil { url := s.crawlQueue.Front().Value.(string) s.crawlQueue.Remove(s.crawlQueue.Front()) task := s.newCrawlTask(url, offer) tasks = append(tasks, task) tasksToLaunch-- } if s.renderQueue.Front() != nil { url := s.renderQueue.Front().Value.(string) s.renderQueue.Remove(s.renderQueue.Front()) task := s.newRenderTask(url, offer) tasks = append(tasks, task) tasksToLaunch-- } if s.crawlQueue.Front() == nil && s.renderQueue.Front() == nil { break } } if len(tasks) == 0 { driver.DeclineOffer(offer.Id, defaultFilter) } else { driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, defaultFilter) } } }
func (s *DiegoScheduler) scheduleMatched(driver sched.SchedulerDriver, matches map[string]*OfferMatch) auctiontypes.AuctionResults { results := auctiontypes.AuctionResults{} for slaveId, match := range matches { if slaveId != "" { offers := match.Offers taskInfos := []*mesos.TaskInfo{} for _, lrpAuction := range match.LrpAuctions { taskInfo := s.createLrpTaskInfo(util.NewSlaveID(slaveId), &lrpAuction) taskInfos = append(taskInfos, taskInfo) results.SuccessfulLRPs = append(results.SuccessfulLRPs, lrpAuction) log.Infof("+scheduled lrp, lrp: %v/%v mem: %v, offers: mem: %v", lrpAuction.ProcessGuid, lrpAuction.Index, lrpAuction.MemoryMB, getOffersMem(offers)) } for _, taskAuction := range match.TaskAuctions { taskInfo := s.createTaskTaskInfo(util.NewSlaveID(slaveId), &taskAuction) taskInfos = append(taskInfos, taskInfo) results.SuccessfulTasks = append(results.SuccessfulTasks, taskAuction) log.Infof("+scheduled task, task: %v mem: %v, offers: mem: %v", taskAuction.TaskGuid, taskAuction.MemoryMB, getOffersMem(offers)) } driver.LaunchTasks(extractOfferIds(offers), taskInfos, // offer getting declied if no tasks &mesos.Filters{RefuseSeconds: proto.Float64(30)}) } else { for _, lrpAuction := range match.LrpAuctions { results.FailedLRPs = append(results.FailedLRPs, lrpAuction) log.Warningf("+schedule lrp failed, lrp: %v/%v mem: %v, offers: mem: %v", lrpAuction.GetProcessGuid(), lrpAuction.Index, lrpAuction.MemoryMB, getOffersMem(match.Offers)) } for _, taskAuction := range match.TaskAuctions { results.FailedTasks = append(results.FailedTasks, taskAuction) log.Warningf("+schedule task failed, task: %v mem: %v, offers: mem: %v", taskAuction.TaskGuid, taskAuction.MemoryMB, getOffersMem(match.Offers)) } } } return results }
// ResourceOffers handles the Resource Offers func (s *eremeticScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { log.Tracef("Received %d resource offers", len(offers)) var offer *mesos.Offer loop: for len(offers) > 0 { select { case <-s.shutdown: log.Info("Shutting down: declining offers") break loop case tid := <-s.tasks: log.Debugf("Trying to find offer to launch %s with", tid) t, _ := database.ReadTask(tid) offer, offers = matchOffer(t, offers) if offer == nil { log.Warnf("Could not find a matching offer for %s", tid) TasksDelayed.Inc() go func() { s.tasks <- tid }() break loop } log.Debugf("Preparing to launch task %s with offer %s", tid, offer.Id.GetValue()) t, task := s.newTask(t, offer) database.PutTask(&t) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, []*mesos.TaskInfo{task}, defaultFilter) TasksLaunched.Inc() QueueSize.Dec() continue default: break loop } } log.Trace("No tasks to launch. Declining offers.") for _, offer := range offers { driver.DeclineOffer(offer.Id, defaultFilter) } }
// ResourceOffers handles the Resource Offers func (s *eremeticScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { log.Tracef("Received %d resource offers", len(offers)) for _, offer := range offers { select { case <-s.shutdown: log.Infof("Shutting down: declining offer on [%s]", offer.Hostname) driver.DeclineOffer(offer.Id, defaultFilter) continue case tid := <-s.tasks: log.Debugf("Preparing to launch task %s with offer %s", tid, offer.Id.GetValue()) t, _ := database.ReadTask(tid) task := s.newTask(offer, &t) database.PutTask(&t) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, []*mesos.TaskInfo{task}, defaultFilter) continue default: } log.Trace("No tasks to launch. Declining offer.") driver.DeclineOffer(offer.Id, defaultFilter) } }
// process incoming offers and try to schedule new tasks as they come in on the channel func (sched *NoneScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { if sched.queue.Next() == nil { // no command to launch, we don't need to parse anything return } for _, offer := range offers { // match constraints if !sched.filter.FilterOffer(offer) { // skip offer if it does not match constraints continue } remainingCpus := SumScalarResources(sched.filter.FilterResources(offer, "cpus")) remainingMems := SumScalarResources(sched.filter.FilterResources(offer, "mem")) log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", remainingCpus, " mem=", remainingMems) // try to schedule as may tasks as possible for this single offer var tasks []*mesos.TaskInfo for sched.queue.GetCommand() != nil && sched.queue.GetCommand().MatchesResources(remainingCpus, remainingMems) { c := sched.queue.GetCommand() c.SlaveId = offer.SlaveId.GetValue() c.FrameworkId = sched.frameworkId sched.handler.CommandLaunched(c) task := sched.prepareTaskInfo(offer, c) tasks = append(tasks, task) remainingCpus -= c.CpuReq remainingMems -= c.MemReq sched.queue.Next() } log.Infoln("Launching", len(tasks), "tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) } }
func (self *MesosManager) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { log.Printf("ResourceOffers\n") computeCount := 0 for _, offer := range offers { offer_cpus := 0 for _, res := range offer.Resources { if *res.Name == "cpus" { computeCount += int(*res.Scalar.Value) offer_cpus += int(*res.Scalar.Value) } } tasks := make([]*mesos.TaskInfo, 0, computeCount) for offer_cpus_taken := 0; offer_cpus_taken < offer_cpus; { job := self.engine.GetJobToRun() if job != nil { log.Printf("Launch job: %s", job) mesos_taskinfo := self.BuildTaskInfo(job, offer) log.Printf("MesosTask: %s", mesos_taskinfo) tasks = append(tasks, mesos_taskinfo) offer_cpus_taken += 1 } else { offer_cpus_taken = offer_cpus } } if len(tasks) > 0 { _, err := driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{}) if err != nil { fmt.Print("Error: %s", err) } } else { driver.DeclineOffer(offer.Id, &mesos.Filters{}) } } self.computeCount = computeCount log.Printf("CPUsOffered: %d", self.computeCount) }
// mesos.Scheduler interface method. // Invoked when resources have been offered to this framework. func (this *TransformScheduler) ResourceOffers(driver scheduler.SchedulerDriver, offers []*mesos.Offer) { fmt.Println("Received offers") if int(this.runningInstances) > this.config.Instances { toKill := int(this.runningInstances) - this.config.Instances for i := 0; i < toKill; i++ { driver.KillTask(this.tasks[i]) } this.tasks = this.tasks[toKill:] } offersAndTasks := make(map[*mesos.Offer][]*mesos.TaskInfo) for _, offer := range offers { cpus := getScalarResources(offer, "cpus") mems := getScalarResources(offer, "mem") ports := getRangeResources(offer, "ports") remainingCpus := cpus remainingMems := mems var tasks []*mesos.TaskInfo for int(this.getRunningInstances()) < this.config.Instances && this.config.CpuPerTask <= remainingCpus && this.config.MemPerTask <= remainingMems && len(ports) > 0 { port := this.takePort(&ports) taskPort := &mesos.Value_Range{Begin: port, End: port} taskId := &mesos.TaskID{ Value: proto.String(fmt.Sprintf("golang-%s-%d", *offer.Hostname, *port)), } task := &mesos.TaskInfo{ Name: proto.String(taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Executor: this.createExecutor(this.getRunningInstances(), *port), Resources: []*mesos.Resource{ util.NewScalarResource("cpus", float64(this.config.CpuPerTask)), util.NewScalarResource("mem", float64(this.config.MemPerTask)), util.NewRangesResource("ports", []*mesos.Value_Range{taskPort}), }, } fmt.Printf("Prepared task: %s with offer %s for launch. Ports: %s\n", task.GetName(), offer.Id.GetValue(), taskPort) tasks = append(tasks, task) remainingCpus -= this.config.CpuPerTask remainingMems -= this.config.MemPerTask ports = ports[1:] this.tasks = append(this.tasks, taskId) this.incRunningInstances() } fmt.Printf("Launching %d tasks for offer %s\n", len(tasks), offer.Id.GetValue()) offersAndTasks[offer] = tasks } unlaunchedTasks := this.config.Instances - int(this.getRunningInstances()) if unlaunchedTasks > 0 { fmt.Printf("There are still %d tasks to be launched and no more resources are available.", unlaunchedTasks) } for _, offer := range offers { tasks := offersAndTasks[offer] driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) } }
func (sched *SdcScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { if sched.tasksLaunched >= sched.totalTasks { log.Info("decline all of the offers since all of our tasks are already launched") log.Infoln("sched.totalTasks ", sched.totalTasks) log.Infoln("sched.tasksFinished ", sched.tasksFinished) log.Infoln("sched.tasksLaunched ", sched.tasksLaunched) // cmdQueueからCommand.Argumentsをpopする // 将来的に外部のキューから取得できるように置き換える if sched.totalTasks == 0 && sched.tasksFinished == 0 && sched.tasksLaunched == 0 && cmdQueue.Len() != 0 { execinfo := cmdQueue.Remove(cmdQueue.Front()).(*mesos.ExecutorInfo) log.Infoln("execinfo ", execinfo.Command.Arguments) sched.totalTasks = len(execinfo.Command.Arguments) sched.executor.Command.Arguments = execinfo.Command.Arguments } if sched.totalTasks == 0 && sched.tasksFinished == 0 && sched.tasksLaunched == 0 { ids := make([]*mesos.OfferID, len(offers)) for i, offer := range offers { ids[i] = offer.Id } driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(5)}) return } } log.Info("prepare pass args: ", sched.executor.Command.Arguments) cmds := sched.executor.Command.Arguments for _, v := range cmds { fmt.Println("v = ", v) } // [/bin/cat /var/tmp/1.txt /var/tmp/2.txt /var/tmp/3.txt | /bin/grep abe > /var/tmp/grep-result.txt] // // rebuild args // 1. /bin/cat /var/tmp/1.txt >> /var/tmp/intermediate.txt // 2. /bin/cat /var/tmp/2.txt >> /var/tmp/intermediate.txt // 3. /bin/cat /var/tmp/3.txt >> /var/tmp/intermediate.txt // 4. /bin/cat /var/tmp/intermediate.txt | /bin/grep abe > /var/tmp/grep-result.txt // 5. /bin/rm /var/tmp/intermediate.txt for _, offer := range offers { cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "cpus" }) cpus := 0.0 for _, res := range cpuResources { cpus += res.GetScalar().GetValue() } memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "mem" }) mems := 0.0 for _, res := range memResources { mems += res.GetScalar().GetValue() } log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems) remainingCpus := cpus remainingMems := mems var tasks []*mesos.TaskInfo // $ cat 1.txt 2.txt. 3.txt | wc -lの場合 // 先に、後ろのタスクを上げておく必要がある? // // コンセプト実装はシンプルに中間ファイル方式で行く // 遅いけど for sched.tasksLaunched < sched.totalTasks && CPUS_PER_TASK <= remainingCpus && MEM_PER_TASK <= remainingMems { sched.tasksLaunched++ taskId := &mesos.TaskID{ Value: proto.String(strconv.Itoa(sched.tasksLaunched)), } // executionidの書き換え sched.executor.ExecutorId = util.NewExecutorID(taskId.GetValue()) log.Infof("sched.tasksLaunched = %d\n", sched.tasksLaunched) log.Infof("sched.totalTasks = %d\n", sched.totalTasks) log.Infof("sched.executor.Command.Value = %s\n", sched.executor.Command.GetValue()) log.Infof("sched.executor.GetExecutorId() = %s\n", sched.executor.GetExecutorId()) // sched.executor.Command.Arguments で書き換えても保持されている // 値はポインタなので、LaunchTasksするときに、複数のタスクでまとめられているので、 // 値は最後に上書きされた物になる // そこで、Argumentsがタスクごとにまとめられないように個別にオブジェクトを生成してタスクを // 起動する exec := &mesos.ExecutorInfo{ ExecutorId: sched.executor.GetExecutorId(), Name: proto.String(sched.executor.GetName()), Source: proto.String(sched.executor.GetSource()), Command: &mesos.CommandInfo{ Value: proto.String(sched.executor.Command.GetValue()), Uris: sched.executor.Command.GetUris(), }, } cmd := cmds[sched.tasksLaunched-1] log.Infof("cmd = %s\n", cmd) // Argumentsコマンドラインを使うと別Executorとみなされるので、色々面倒 // 以下はやってはいけない例 // exec.Command.Arguments = strings.Split(cmd, " ") task := &mesos.TaskInfo{ Name: proto.String("go-task-" + taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Executor: exec, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", CPUS_PER_TASK), util.NewScalarResource("mem", MEM_PER_TASK), }, // 実行したいコマンドラインはDataパラメータを使って渡せす Data: []byte(cmd), } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) tasks = append(tasks, task) remainingCpus -= CPUS_PER_TASK remainingMems -= MEM_PER_TASK } log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)}) } }
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { if *slowLaunch { time.Sleep(3 * time.Second) } if (sched.tasksLaunched - sched.tasksErrored) >= sched.totalTasks { log.Info("decline all of the offers since all of our tasks are already launched") ids := make([]*mesos.OfferID, len(offers)) for i, offer := range offers { ids[i] = offer.Id } driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(120)}) return } for _, offer := range offers { cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "cpus" }) cpus := 0.0 for _, res := range cpuResources { cpus += res.GetScalar().GetValue() } memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "mem" }) mems := 0.0 for _, res := range memResources { mems += res.GetScalar().GetValue() } log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems) remainingCpus := cpus remainingMems := mems // account for executor resources if there's not an executor already running on the slave if len(offer.ExecutorIds) == 0 { remainingCpus -= CPUS_PER_EXECUTOR remainingMems -= MEM_PER_EXECUTOR } var tasks []*mesos.TaskInfo for (sched.tasksLaunched-sched.tasksErrored) < sched.totalTasks && CPUS_PER_TASK <= remainingCpus && MEM_PER_TASK <= remainingMems { sched.tasksLaunched++ taskId := &mesos.TaskID{ Value: proto.String(strconv.Itoa(sched.tasksLaunched)), } task := &mesos.TaskInfo{ Name: proto.String("go-task-" + taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Executor: sched.executor, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", CPUS_PER_TASK), util.NewScalarResource("mem", MEM_PER_TASK), }, } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) tasks = append(tasks, task) remainingCpus -= CPUS_PER_TASK remainingMems -= MEM_PER_TASK } log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)}) } }
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { for _, offer := range offers { cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "cpus" }) cpus := 0.0 for _, res := range cpuResources { cpus += res.GetScalar().GetValue() } memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "mem" }) mems := 0.0 for _, res := range memResources { mems += res.GetScalar().GetValue() } log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems) remainingCpus := cpus remainingMems := mems var tasks []*mesos.TaskInfo queueItems := getQueue(sched) if len(queueItems) == 0 { fmt.Println("Queue is empty!") } // start a task of each item in the queue. before starting, check // if it has not already been launched. for i, item := range queueItems { if val, ok := launchedTasks[item.Commit.Sha]; ok { fmt.Println("item exists: ", val) continue } fmt.Println("Adding item: ", item.Commit.Sha) launchedTasks[item.Commit.Sha] = item.Commit.Sha if CPUS_PER_TASK <= remainingCpus && MEM_PER_TASK <= remainingMems { sched.tasksLaunched += i taskId := &mesos.TaskID{ Value: proto.String(strconv.Itoa(sched.tasksLaunched)), } stringArray := []string{"-addr=" + *droneServerIP, "-token=1"} dataString := strings.Join(stringArray, " ") task := &mesos.TaskInfo{ Name: proto.String("go-task-" + taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Executor: sched.executor, Data: []byte(dataString), Resources: []*mesos.Resource{ util.NewScalarResource("cpus", CPUS_PER_TASK), util.NewScalarResource("mem", MEM_PER_TASK), }, } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) tasks = append(tasks, task) remainingCpus -= CPUS_PER_TASK remainingMems -= MEM_PER_TASK } } // for sched.tasksLaunched < sched.totalTasks && // CPUS_PER_TASK <= remainingCpus && // MEM_PER_TASK <= remainingMems { // // sched.tasksLaunched++ // // taskId := &mesos.TaskID{ // Value: proto.String(strconv.Itoa(sched.tasksLaunched)), // } // // stringArray := []string{*droneServerIP, "-token=1"} // dataString := strings.Join(stringArray, " ") // // task := &mesos.TaskInfo{ // Name: proto.String("go-task-" + taskId.GetValue()), // TaskId: taskId, // SlaveId: offer.SlaveId, // Executor: sched.executor, // Data: []byte(dataString), // Resources: []*mesos.Resource{ // util.NewScalarResource("cpus", CPUS_PER_TASK), // util.NewScalarResource("mem", MEM_PER_TASK), // }, // } // // log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) // // tasks = append(tasks, task) // remainingCpus -= CPUS_PER_TASK // remainingMems -= MEM_PER_TASK // } log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) } }
// Schedule handles Mesos resource offers and based on the received offers launches PENDING Tasks // // It monitors PENDING Tasks Queue and tries to launch as many Tasks as possible for a single offer. // If no PENDING Tasks are available within QueueTimeout interval it declines the offer and retries Queue read again func (bw *BasicWorker) ScheduleTasks(driver scheduler.SchedulerDriver, offers []*mesos.Offer) { ReadOffers: for _, offer := range offers { remainingCpus := taurus.ScalarResourceVal("cpus", offer.Resources) remainingMems := taurus.ScalarResourceVal("mem", offer.Resources) log.Println("Taurus Received Offer <", offer.Id.GetValue(), "> with cpus=", remainingCpus, " mem=", remainingMems) // Map to avoid launching duplicate tasks in the same batch slice launchTaskMap := make(map[string]bool) launchTasks := make([]*mesos.TaskInfo, 0, 0) var taskCpu, taskMem float64 var retryCount int ReadTasks: for { task, err := bw.pending.ReadTask(QueueTimeout) if err != nil { retryCount += 1 switch { case bw.pending.TimedOut(err): log.Printf("No %s tasks available", taurus.PENDING) case bw.pending.Closed(err): break ReadTasks default: log.Printf("Failed to read from %s queue: %s", taurus.PENDING, err) } if retryCount == QueueRetry { break ReadTasks } continue ReadTasks } if task != nil { taskId := task.Info.TaskId.GetValue() // Don't add the same task bwice into launchTasks slice if launchTaskMap[taskId] { log.Printf("Skipping already queued Task %s", taskId) continue ReadTasks } taskCpu = taurus.ScalarResourceVal("cpus", task.Info.Resources) taskMem = taurus.ScalarResourceVal("mem", task.Info.Resources) if remainingCpus >= taskCpu && remainingMems >= taskMem { task.Info.SlaveId = offer.SlaveId launchTasks = append(launchTasks, task.Info) launchTaskMap[taskId] = true remainingCpus -= taskCpu remainingMems -= taskMem } else { break ReadTasks } } } if len(launchTasks) > 0 { log.Printf("Launching %d tasks for offer %s", len(launchTasks), offer.Id.GetValue()) launchStatus, err := driver.LaunchTasks( []*mesos.OfferID{offer.Id}, launchTasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) if err != nil { log.Printf("Mesos status: %#v Failed to launch Tasks %s: %s", launchStatus, launchTasks, err) continue ReadOffers } } else { log.Println("Declining offer ", offer.Id.GetValue()) declineStatus, err := driver.DeclineOffer( offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) if err != nil { log.Printf("Error declining offer for mesos status %#v: %s", declineStatus, err) } } } }
// TODO(tyler) split this long function up! func (s *EtcdScheduler) launchOne(driver scheduler.SchedulerDriver) { // Always ensure we've pruned any dead / unmanaged nodes before // launching new ones, or we may overconfigure the ensemble such // that it can not make progress if the next launch fails. err := s.Prune() if err != nil { log.Errorf("Failed to remove stale cluster members: %s", err) return } if !s.shouldLaunch(driver) { log.Infoln("Skipping launch attempt for now.") return } // validOffer filters out offers that are no longer // desirable, even though they may have been when // they were enqueued. validOffer := func(offer *mesos.Offer) bool { runningCopy := s.RunningCopy() for _, etcdConfig := range runningCopy { if etcdConfig.SlaveID == offer.SlaveId.GetValue() { if s.singleInstancePerSlave { log.Info("Skipping offer: already running on this slave.") return false } } } return true } // Issue BlockingPop until we get back an offer we can use. var offer *mesos.Offer for { offer = s.offerCache.BlockingPop() if validOffer(offer) { break } else { s.decline(driver, offer) } } // Do this again because BlockingPop may have taken a long time. if !s.shouldLaunch(driver) { log.Infoln("Skipping launch attempt for now.") s.decline(driver, offer) return } // TODO(tyler) this is a broken hack resources := parseOffer(offer) lowest := *resources.ports[0].Begin rpcPort := lowest clientPort := lowest + 1 httpPort := lowest + 2 s.mut.Lock() var clusterType string if len(s.running) == 0 { clusterType = "new" } else { clusterType = "existing" } s.highestInstanceID++ name := "etcd-" + strconv.FormatInt(s.highestInstanceID, 10) node := &config.Node{ Name: name, Host: *offer.Hostname, RPCPort: rpcPort, ClientPort: clientPort, ReseedPort: httpPort, Type: clusterType, SlaveID: offer.GetSlaveId().GetValue(), } running := []*config.Node{node} for _, r := range s.running { running = append(running, r) } serializedNodes, err := json.Marshal(running) log.Infof("Serialized running: %+v", string(serializedNodes)) if err != nil { log.Errorf("Could not serialize running list: %v", err) // This Unlock is not deferred because the test implementation of LaunchTasks // calls this scheduler's StatusUpdate method, causing the test to deadlock. s.decline(driver, offer) s.mut.Unlock() return } configSummary := node.String() taskID := &mesos.TaskID{Value: &configSummary} executor := s.newExecutorInfo(node, s.executorUris) task := &mesos.TaskInfo{ Data: serializedNodes, Name: proto.String("etcd-server"), TaskId: taskID, SlaveId: offer.SlaveId, Executor: executor, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", s.cpusPerTask), util.NewScalarResource("mem", s.memPerTask), util.NewScalarResource("disk", s.diskPerTask), util.NewRangesResource("ports", []*mesos.Value_Range{ util.NewValueRange(uint64(rpcPort), uint64(httpPort)), }), }, Discovery: &mesos.DiscoveryInfo{ Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(), Name: proto.String("etcd-server"), Ports: &mesos.Ports{ Ports: []*mesos.Port{ &mesos.Port{ Number: proto.Uint32(uint32(rpcPort)), Protocol: proto.String("tcp"), }, // HACK: "client" is not a real SRV protocol. This is so // that we can have etcd proxies use srv discovery on the // above tcp name. Mesos-dns does not yet care about // names for DiscoveryInfo. When it does, we should // create a name for clients to use. We want to keep // the rpcPort accessible at _etcd-server._tcp.<fwname>.mesos &mesos.Port{ Number: proto.Uint32(uint32(clientPort)), Protocol: proto.String("client"), }, }, }, }, } log.Infof( "Prepared task: %s with offer %s for launch", task.GetName(), offer.Id.GetValue(), ) log.Info("Launching etcd node.") tasks := []*mesos.TaskInfo{task} s.pending[node.Name] = struct{}{} // This Unlock is not deferred because the test implementation of LaunchTasks // calls this scheduler's StatusUpdate method, causing the test to deadlock. s.mut.Unlock() atomic.AddUint32(&s.Stats.LaunchedServers, 1) driver.LaunchTasks( []*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{ RefuseSeconds: proto.Float64(1), }, ) }
func (s *MinerScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { for i, offer := range offers { memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "mem" }) mems := 0.0 for _, res := range memResources { mems += res.GetScalar().GetValue() } cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "cpus" }) cpus := 0.0 for _, res := range cpuResources { cpus += res.GetScalar().GetValue() } portsResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "ports" }) var ports uint64 for _, res := range portsResources { port_ranges := res.GetRanges().GetRange() for _, port_range := range port_ranges { ports += port_range.GetEnd() - port_range.GetBegin() } } // If a miner server is running, we start a new miner daemon. Otherwise, we start a new miner server. tasks := make([]*mesos.TaskInfo, 0) if !s.minerServerRunning && mems >= MEM_PER_SERVER_TASK && cpus >= CPU_PER_SERVER_TASK && ports >= 2 { var taskId *mesos.TaskID var task *mesos.TaskInfo // we need two ports var p2pool_port uint64 var worker_port uint64 // A rather stupid algorithm for picking two ports // The difficulty here is that a range might only include one port, // in which case we will need to pick another port from another range. for _, res := range portsResources { r := res.GetRanges().GetRange()[0] begin := r.GetBegin() end := r.GetEnd() if p2pool_port == 0 { p2pool_port = begin if worker_port == 0 && (begin+1) <= end { worker_port = begin + 1 break } continue } if worker_port == 0 { worker_port = begin break } } taskId = &mesos.TaskID{ Value: proto.String("miner-server-" + strconv.Itoa(i)), } containerType := mesos.ContainerInfo_DOCKER task = &mesos.TaskInfo{ Name: proto.String("task-" + taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Container: &mesos.ContainerInfo{ Type: &containerType, Docker: &mesos.ContainerInfo_DockerInfo{ Image: proto.String(MINER_SERVER_DOCKER_IMAGE), }, }, Command: &mesos.CommandInfo{ Shell: proto.Bool(false), Arguments: []string{ // these arguments will be passed to run_p2pool.py "--bitcoind-address", *bitcoindAddr, "--p2pool-port", strconv.Itoa(int(p2pool_port)), "-w", strconv.Itoa(int(worker_port)), s.rpc_user, s.rpc_pass, }, }, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", CPU_PER_SERVER_TASK), util.NewScalarResource("mem", MEM_PER_SERVER_TASK), }, } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) cpus -= CPU_PER_SERVER_TASK mems -= MEM_PER_SERVER_TASK // update state s.minerServerHostname = offer.GetHostname() s.minerServerRunning = true s.minerServerPort = int(worker_port) tasks = append(tasks, task) } if s.minerServerRunning && mems >= MEM_PER_DAEMON_TASK { var taskId *mesos.TaskID var task *mesos.TaskInfo taskId = &mesos.TaskID{ Value: proto.String("miner-daemon-" + strconv.Itoa(i)), } containerType := mesos.ContainerInfo_DOCKER task = &mesos.TaskInfo{ Name: proto.String("task-" + taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Container: &mesos.ContainerInfo{ Type: &containerType, Docker: &mesos.ContainerInfo_DockerInfo{ Image: proto.String(MINER_DAEMON_DOCKER_IMAGE), }, }, Command: &mesos.CommandInfo{ Shell: proto.Bool(false), Arguments: []string{"-o", s.minerServerHostname + ":" + strconv.Itoa(s.minerServerPort)}, }, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", cpus), util.NewScalarResource("mem", MEM_PER_DAEMON_TASK), }, } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) tasks = append(tasks, task) } driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) } }
func (s *visghsScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { if (s.nodesLaunched - s.nodesErrored) >= s.nodeTasks { log.Info("decline all of the offers since all of our tasks are already launched") ids := make([]*mesos.OfferID, len(offers)) for i, offer := range offers { ids[i] = offer.Id } driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(120)}) return } for _, offer := range offers { cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "cpus" }) cpus := 0.0 for _, res := range cpuResources { cpus += res.GetScalar().GetValue() } memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "mem" }) mems := 0.0 for _, res := range memResources { mems += res.GetScalar().GetValue() } portResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "ports" }) ports := []*mesos.Value_Range{} portsCount := uint64(0) for _, res := range portResources { for _, rs := range res.GetRanges().GetRange() { ports = append(ports, rs) portsCount += 1 + rs.GetEnd() - rs.GetBegin() } } log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems, " ports=", ports) remainingCpus := cpus remainingMems := mems remainingPorts := ports remainingPortsCount := portsCount // account for executor resources if there's not an executor already running on the slave if len(offer.ExecutorIds) == 0 { remainingCpus -= CPUS_PER_EXECUTOR remainingMems -= MEM_PER_EXECUTOR } var tasks []*mesos.TaskInfo for (s.nodesLaunched-s.nodesErrored) < s.nodeTasks && CPUS_PER_TASK <= remainingCpus && MEM_PER_TASK <= remainingMems && PORTS_PER_TASK <= remainingPortsCount { log.Infoln("Ports <", remainingPortsCount, remainingPorts) s.nodesLaunched++ taskId := &mesos.TaskID{ Value: proto.String(strconv.Itoa(s.nodesLaunched)), } taskPorts := []*mesos.Value_Range{} leftOverPorts := []*mesos.Value_Range{} for t := 0; t < PORTS_PER_TASK; t++ { if len(remainingPorts) < 1 { // failed to allocate port, oh no! } ps := remainingPorts[0] //take the first port from the first pb := ps.GetBegin() pe := ps.GetEnd() //Create one range per port we need, it's easier this way tp := mesos.Value_Range{} p := pb tp.Begin = &p tp.End = &p taskPorts = append(taskPorts, &tp) pb++ if pb <= pe { rpb := pb rpe := pe rtp := mesos.Value_Range{Begin: &rpb, End: &rpe} leftOverPorts = append(leftOverPorts, &rtp) } for _, ps := range remainingPorts[1:] { leftOverPorts = append(leftOverPorts, ps) } } radiaPort := (uint32)(taskPorts[0].GetBegin()) task := &mesos.TaskInfo{ Name: proto.String("visghs-node-" + taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Executor: s.nexec, Discovery: &mesos.DiscoveryInfo{ Name: proto.String("visghs"), //Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(), Visibility: mesos.DiscoveryInfo_FRAMEWORK.Enum(), Ports: &mesos.Ports{ Ports: []*mesos.Port{ {Protocol: proto.String("UDP"), Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(), Name: proto.String("udpprobe"), Number: proto.Uint32(radiaPort)}, {Protocol: proto.String("TCP"), Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(), Name: proto.String("radiarpc"), Number: proto.Uint32(radiaPort)}, }, }, }, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", CPUS_PER_TASK), util.NewScalarResource("mem", MEM_PER_TASK), util.NewRangesResource("ports", taskPorts), }, } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) tasks = append(tasks, task) remainingCpus -= CPUS_PER_TASK remainingMems -= MEM_PER_TASK remainingPorts = leftOverPorts remainingPortsCount -= PORTS_PER_TASK } log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)}) } }