func (sc *SchedulerCore) acceptOffer(driver sched.SchedulerDriver, offer *mesos.Offer, operations []*mesos.Offer_Operation) {
log.Infof("Accepting OfferID: %+v, Operations: %+v", *offer.Id.Value, operations)
var status mesos.Status
var err error
if sc.compatibilityMode {
tasks := []*mesos.TaskInfo{}
for _, operation := range operations {
if *operation.Type == mesos.Offer_Operation_LAUNCH {
tasks = operation.Launch.TaskInfos
}
}
status, err = driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(OFFER_INTERVAL)})
} else {
status, err = driver.AcceptOffers([]*mesos.OfferID{offer.Id}, operations, &mesos.Filters{RefuseSeconds: proto.Float64(OFFER_INTERVAL)})
}
if status != mesos.Status_DRIVER_RUNNING {
log.Fatal("Driver not running, while trying to accept offers")
}
if err != nil {
log.Panic("Failed to launch tasks: ", err)
}
}
func (s *Scheduler) launchTask(driver scheduler.SchedulerDriver, offer *mesos.Offer) {
taskName := fmt.Sprintf("syslog-%s", offer.GetSlaveId().GetValue())
taskId := &mesos.TaskID{
Value: proto.String(fmt.Sprintf("%s-%s", taskName, uuid())),
}
data, err := json.Marshal(Config)
if err != nil {
panic(err) //shouldn't happen
}
Logger.Debugf("Task data: %s", string(data))
tcpPort := uint64(s.getPort(Config.TcpPort, offer, -1))
udpPort := uint64(s.getPort(Config.UdpPort, offer, int(tcpPort)))
task := &mesos.TaskInfo{
Name: proto.String(taskName),
TaskId: taskId,
SlaveId: offer.GetSlaveId(),
Executor: s.createExecutor(offer, tcpPort, udpPort),
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", Config.Cpus),
util.NewScalarResource("mem", Config.Mem),
util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(tcpPort, tcpPort)}),
util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(udpPort, udpPort)}),
},
Data: data,
Labels: utils.StringToLabels(s.labels),
}
s.cluster.Add(offer.GetSlaveId().GetValue(), task)
driver.LaunchTasks([]*mesos.OfferID{offer.GetId()}, []*mesos.TaskInfo{task}, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
func (sched *Scheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
logOffers(offers)
jobs, err := getLaunchableJobs()
if err != nil {
log.Errorf("Unable to get pending jobs! %s\n", err.Error())
return
}
offersAndTasks, err := packJobsInOffers(jobs, offers)
if err != nil {
log.Errorf("Unable to pack jobs into offers! %s\n", err.Error())
return
}
for _, ot := range offersAndTasks {
if len(ot.Tasks) == 0 {
log.Infof("Declining unused offer %s", ot.Offer.Id.GetValue())
driver.DeclineOffer(ot.Offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
continue
} else {
log.Infof("Launching %d tasks for offer %s\n", len(ot.Tasks), ot.Offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{ot.Offer.Id}, ot.Tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
sched.tasksLaunched = sched.tasksLaunched + len(ot.Tasks)
}
}
}
func (s *MemcacheScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
log.Printf("Received %d resource offers", len(offers))
for _, offer := range offers {
select {
case <-s.shutdown:
log.Println("Shutting down: declining offer on [", offer.Hostname, "]")
driver.DeclineOffer(offer.Id, defaultFilter)
if s.tasksRunning == 0 {
close(s.done)
}
continue
default:
}
tasks := []*mesos.TaskInfo{}
if canLaunchNewTask(offer) && s.shouldLaunchNewTask() {
fmt.Println("Accepting Offer: ", offer)
task := s.newMemcacheTask(offer)
tasks = append(tasks, task)
}
if len(tasks) == 0 {
driver.DeclineOffer(offer.Id, defaultFilter)
} else {
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, defaultFilter)
s.lastTaskCreatedAt = time.Now()
return // limit one at a time
}
}
}
func (s *Scheduler) launchTask(driver scheduler.SchedulerDriver, offer *mesos.Offer) {
taskName := fmt.Sprintf("syscol-%s", offer.GetSlaveId().GetValue())
taskId := &mesos.TaskID{
Value: proto.String(fmt.Sprintf("%s-%s", taskName, uuid())),
}
data, err := json.Marshal(Config)
if err != nil {
panic(err) //shouldn't happen
}
Logger.Debugf("Task data: %s", string(data))
task := &mesos.TaskInfo{
Name: proto.String(taskName),
TaskId: taskId,
SlaveId: offer.GetSlaveId(),
Executor: s.createExecutor(offer.GetSlaveId().GetValue()),
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", Config.Cpus),
util.NewScalarResource("mem", Config.Mem),
},
Data: data,
}
s.cluster.Add(offer.GetSlaveId().GetValue(), task)
driver.LaunchTasks([]*mesos.OfferID{offer.GetId()}, []*mesos.TaskInfo{task}, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
// ResourceOffers handles the Resource Offers
func (s *Scheduler) ResourceOffers(driver mesossched.SchedulerDriver, offers []*mesosproto.Offer) {
logrus.WithField("offers", len(offers)).Debug("Received offers")
var offer *mesosproto.Offer
loop:
for len(offers) > 0 {
select {
case <-s.shutdown:
logrus.Info("Shutting down: declining offers")
break loop
case tid := <-s.tasks:
logrus.WithField("task_id", tid).Debug("Trying to find offer to launch task with")
t, _ := s.database.ReadUnmaskedTask(tid)
if t.IsTerminating() {
logrus.Debug("Dropping terminating task.")
t.UpdateStatus(eremetic.Status{
Status: eremetic.TaskKilled,
Time: time.Now().Unix(),
})
s.database.PutTask(&t)
continue
}
offer, offers = matchOffer(t, offers)
if offer == nil {
logrus.WithField("task_id", tid).Warn("Unable to find a matching offer")
tasksDelayed.Inc()
go func() { s.tasks <- tid }()
break loop
}
logrus.WithFields(logrus.Fields{
"task_id": tid,
"offer_id": offer.Id.GetValue(),
}).Debug("Preparing to launch task")
t, task := createTaskInfo(t, offer)
t.UpdateStatus(eremetic.Status{
Status: eremetic.TaskStaging,
Time: time.Now().Unix(),
})
s.database.PutTask(&t)
driver.LaunchTasks([]*mesosproto.OfferID{offer.Id}, []*mesosproto.TaskInfo{task}, defaultFilter)
tasksLaunched.Inc()
queueSize.Dec()
continue
default:
break loop
}
}
logrus.Debug("No tasks to launch. Declining offers.")
for _, offer := range offers {
driver.DeclineOffer(offer.Id, defaultFilter)
}
}
func (s *rancherScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
task := tasks.GetNextTask()
if task == nil {
for _, of := range offers {
driver.DeclineOffer(of.Id, defaultFilter)
}
return
}
if task.RegistrationUrl == "" {
tasks.AddTask(task)
for _, of := range offers {
driver.DeclineOffer(of.Id, defaultFilter)
}
return
}
taskBytes, err := task.Marshal()
if err != nil {
log.WithFields(log.Fields{
"err": err,
}).Error("Error Marshalling task")
for _, of := range offers {
driver.DeclineOffer(of.Id, defaultFilter)
}
return
}
for _, offer := range offers {
inadequate := false
for _, res := range offer.GetResources() {
if res.GetName() == "cpus" && *res.GetScalar().Value < taskCPUs {
driver.DeclineOffer(offer.Id, defaultFilter)
inadequate = true
continue
}
if res.GetName() == "mem" && *res.GetScalar().Value < taskMem {
driver.DeclineOffer(offer.Id, defaultFilter)
inadequate = true
continue
}
}
if inadequate {
continue
}
mesosTask := &mesos.TaskInfo{
TaskId: &mesos.TaskID{
Value: proto.String(task.HostUuid),
},
SlaveId: offer.SlaveId,
Resources: []*mesos.Resource{
mesosutil.NewScalarResource("cpus", taskCPUs),
mesosutil.NewScalarResource("mem", taskMem),
},
Data: taskBytes,
Name: &task.Name,
Executor: s.rancherExecutor,
}
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, []*mesos.TaskInfo{mesosTask}, defaultFilter)
}
}
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
logOffers(offers)
for _, offer := range offers {
remainingCpus := getOfferCpu(offer)
remainingMems := getOfferMem(offer)
var tasks []*mesos.TaskInfo
for sched.cpuPerTask <= remainingCpus &&
sched.memPerTask <= remainingMems &&
sched.tasksLaunched < sched.totalTasks {
fmt.Printf("Tasks launched: %v Total tasks: %v\n", sched.tasksLaunched, sched.totalTasks)
sched.tasksLaunched++
taskId := &mesos.TaskID{
Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
}
dockerInfo := &mesos.ContainerInfo_DockerInfo{
Image: &sched.DockerImage,
PortMappings: sched.DockerPorts,
}
containerType := mesos.ContainerInfo_DOCKER
containerInfo := &mesos.ContainerInfo{
Type: &containerType,
Docker: dockerInfo,
}
commandInfo := &mesos.CommandInfo{
Value: &sched.DockerCommand,
}
task := &mesos.TaskInfo{
Name: proto.String("go-task-" + taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", sched.cpuPerTask),
util.NewScalarResource("mem", sched.memPerTask),
},
Container: containerInfo,
Command: commandInfo,
}
fmt.Printf("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
tasks = append(tasks, task)
remainingCpus -= sched.cpuPerTask
remainingMems -= sched.memPerTask
}
// fmt.Println("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
}
func (sched *Scheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
for _, offer := range offers {
taskId := &mesos.TaskID{
Value: proto.String(fmt.Sprintf("basicdocker-task-%d", time.Now().Unix())),
}
ports := util.FilterResources(
offer.Resources,
func(res *mesos.Resource) bool {
return res.GetName() == "ports"
},
)
if len(ports) > 0 && len(ports[0].GetRanges().GetRange()) > 0 {
} else {
return
}
task := &mesos.TaskInfo{
Name: proto.String(taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Container: &mesos.ContainerInfo{
Type: mesos.ContainerInfo_DOCKER.Enum(),
Volumes: nil,
Hostname: nil,
Docker: &mesos.ContainerInfo_DockerInfo{
Image: &DOCKER_IMAGE_DEFAULT,
Network: mesos.ContainerInfo_DockerInfo_BRIDGE.Enum(),
},
},
Command: &mesos.CommandInfo{
Shell: proto.Bool(true),
Value: proto.String("set -x ; /bin/date ; /bin/hostname ; sleep 200 ; echo done"),
},
Executor: nil,
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", getOfferCpu(offer)),
util.NewScalarResource("mem", getOfferMem(offer)),
util.NewRangesResource("ports", []*mesos.Value_Range{
util.NewValueRange(
*ports[0].GetRanges().GetRange()[0].Begin,
*ports[0].GetRanges().GetRange()[0].Begin+1,
),
}),
},
}
log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
var tasks []*mesos.TaskInfo = []*mesos.TaskInfo{task}
log.Infoln("Launching ", len(tasks), " tasks for offer", offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
sched.tasksLaunched++
time.Sleep(time.Second)
}
}
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
for _, offer := range offers {
cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "cpus"
})
cpus := 0.0
for _, res := range cpuResources {
cpus += res.GetScalar().GetValue()
}
memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "mem"
})
mems := 0.0
for _, res := range memResources {
mems += res.GetScalar().GetValue()
}
log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)
remainingCpus := cpus
remainingMems := mems
var tasks []*mesos.TaskInfo
for sched.tasksLaunched < sched.totalTasks &&
CPUS_PER_TASK <= remainingCpus &&
MEM_PER_TASK <= remainingMems {
sched.tasksLaunched++
taskId := &mesos.TaskID{
Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
}
task := &mesos.TaskInfo{
Name: proto.String("go-task-" + taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Executor: sched.executor,
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", CPUS_PER_TASK),
util.NewScalarResource("mem", MEM_PER_TASK),
},
}
log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
tasks = append(tasks, task)
remainingCpus -= CPUS_PER_TASK
remainingMems -= MEM_PER_TASK
}
log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
}
func (ctx *RunOnceApplicationContext) LaunchTask(driver scheduler.SchedulerDriver, offer *mesos.Offer) error {
ctx.lock.Lock()
defer ctx.lock.Unlock()
ctx.InstancesLeftToRun--
taskInfo := ctx.newTaskInfo(offer)
ctx.tasks = append(ctx.tasks, newRunOnceTask(offer, taskInfo.GetTaskId().GetValue()))
_, err := driver.LaunchTasks([]*mesos.OfferID{offer.GetId()}, []*mesos.TaskInfo{taskInfo}, &mesos.Filters{RefuseSeconds: proto.Float64(10)})
return err
}
// mesos.Scheduler interface method.
// Invoked when resources have been offered to this framework.
func (this *ElodinaTransportScheduler) ResourceOffers(driver scheduler.SchedulerDriver, offers []*mesos.Offer) {
log.Logger.Info("Received offers")
offersAndTasks := make(map[*mesos.Offer][]*mesos.TaskInfo)
remainingPartitions, err := this.GetTopicPartitions()
if err != nil {
return
}
remainingPartitions.RemoveAll(this.TakenTopicPartitions.GetArray())
log.Logger.Debug("%v", remainingPartitions)
tps := remainingPartitions.GetArray()
offersAndResources := this.wrapInOfferAndResources(offers)
for !remainingPartitions.IsEmpty() {
log.Logger.Debug("Iteration %v", remainingPartitions)
if this.hasEnoughInstances() {
for _, transfer := range this.taskIdToTaskState {
if len(transfer.assignment) < this.config.ThreadsPerTask {
transfer.assignment = append(transfer.assignment, tps[0])
remainingPartitions.Remove(tps[0])
this.TakenTopicPartitions.Add(tps[0])
if len(tps) > 1 {
tps = tps[1:]
} else {
tps = []consumer.TopicAndPartition{}
}
}
}
} else {
log.Logger.Debug("Trying to launch new task")
offer, task := this.launchNewTask(offersAndResources)
if offer != nil && task != nil {
offersAndTasks[offer] = append(offersAndTasks[offer], task)
} else {
for _, offer := range offers {
if _, exists := offersAndTasks[offer]; !exists {
offersAndTasks[offer] = make([]*mesos.TaskInfo, 0)
}
}
break
}
}
}
this.assignPendingPartitions()
for _, offer := range offers {
if tasks, ok := offersAndTasks[offer]; ok {
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
} else {
driver.DeclineOffer(offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(10)})
}
}
}
func (sc *SchedulerCore) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
sc.lock.Lock()
defer sc.lock.Unlock()
log.Info("Received resource offers: ", offers)
launchTasks := make(map[string][]*mesos.TaskInfo)
toBeScheduled := []*FrameworkRiakNode{}
for _, cluster := range sc.schedulerState.Clusters {
for _, riakNode := range cluster.Nodes {
if riakNode.NeedsToBeScheduled() {
log.Infof("Adding Riak node for scheduling: %+v", riakNode)
// We need to schedule this task I guess?
toBeScheduled = append(toBeScheduled, riakNode)
}
}
}
// Populate a mutable slice of offer resources
allResources := [][]*mesos.Resource{}
for _, offer := range offers {
allResources = append(allResources, offer.Resources)
}
launchTasks, err := sc.spreadNodesAcrossOffers(offers, allResources, toBeScheduled, 0, 0, launchTasks)
if err != nil {
log.Error(err)
}
for _, offer := range offers {
tasks := launchTasks[*offer.Id.Value]
if tasks == nil {
tasks = []*mesos.TaskInfo{}
}
// This is somewhat of a hack, to avoid synchronously calling the mesos-go SDK
// to avoid a deadlock situation.
// TODO: Fix and make actually queues around driver interactions
// This is a massive hack
// -Sargun Dhillon 2015-10-01
go func(offer *mesos.Offer) {
log.Infof("Launching Tasks: %v for offer %v", tasks, *offer.Id.Value)
status, err := driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(OFFER_INTERVAL)})
if status != mesos.Status_DRIVER_RUNNING {
log.Fatal("Driver not running, while trying to launch tasks")
}
if err != nil {
log.Panic("Failed to launch tasks: ", err)
}
}(offer)
}
}
func (s *DiegoScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
logOffers(offers)
s.offersLock.Lock()
defer s.offersLock.Unlock()
if s.holdOffer {
s.offers = append(s.offers, offers...)
} else {
offerIds := extractOfferIds(offers)
driver.LaunchTasks(offerIds, nil, &mesos.Filters{RefuseSeconds: proto.Float64(30)})
}
}
func (sched *ScraperScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
logOffers(offers)
for _, offer := range offers {
if sched.tasksLaunched >= sched.totalTasks || len(sched.urls) == 0 {
log.Infof("Declining offer %s", offer.Id.GetValue())
driver.DeclineOffer(offer.Id, &mesos.Filters{})
continue
}
remainingCpus := getOfferCpu(offer)
remainingMems := getOfferMem(offer)
var tasks []*mesos.TaskInfo
for sched.cpuPerTask <= remainingCpus &&
sched.memPerTask <= remainingMems &&
sched.tasksLaunched < sched.totalTasks {
log.Infof("Processing url %v of %v\n", sched.tasksLaunched, sched.totalTasks)
log.Infof("Total Tasks: %d", sched.totalTasks)
log.Infof("Tasks Launched: %d", sched.tasksLaunched)
uri := sched.urls[sched.tasksLaunched]
log.Infof("URI: %s", uri)
sched.tasksLaunched++
taskId := &mesos.TaskID{
Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
}
task := &mesos.TaskInfo{
Name: proto.String("go-task-" + taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Executor: sched.executor,
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", sched.cpuPerTask),
util.NewScalarResource("mem", sched.memPerTask),
},
Data: []byte(uri),
}
log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
tasks = append(tasks, task)
remainingCpus -= sched.cpuPerTask
remainingMems -= sched.memPerTask
}
log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
}
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
logOffers(offers)
if sched.tasksLaunched >= sched.totalTasks {
return
}
for _, offer := range offers {
remainingCpus := getOfferCpu(offer)
remainingMems := getOfferMem(offer)
var tasks []*mesos.TaskInfo
for sched.cpuPerTask <= remainingCpus &&
sched.memPerTask <= remainingMems &&
sched.tasksLaunched < sched.totalTasks {
log.Infof("Processing image %v of %v\n", sched.tasksLaunched, sched.totalTasks)
fileName := sched.images[sched.tasksLaunched]
sched.tasksLaunched++
taskId := &mesos.TaskID{
Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
}
task := &mesos.TaskInfo{
Name: proto.String("go-task-" + taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Executor: sched.executor,
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", sched.cpuPerTask),
util.NewScalarResource("mem", sched.memPerTask),
},
Data: []byte(fileName),
}
log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
tasks = append(tasks, task)
remainingCpus -= sched.cpuPerTask
remainingMems -= sched.memPerTask
}
log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
}
func (s *rendlerScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
log.Printf("Received %d resource offers", len(offers))
for _, offer := range offers {
select {
case <-s.shutdown:
log.Println("Shutting down: declining offer on [", offer.Hostname, "]")
driver.DeclineOffer(offer.Id, defaultFilter)
if s.tasksRunning == 0 {
close(s.done)
}
continue
default:
}
tasks := []*mesos.TaskInfo{}
tasksToLaunch := maxTasksForOffer(offer)
for tasksToLaunch > 0 {
if s.crawlQueue.Front() != nil {
url := s.crawlQueue.Front().Value.(string)
s.crawlQueue.Remove(s.crawlQueue.Front())
task := s.newCrawlTask(url, offer)
tasks = append(tasks, task)
tasksToLaunch--
}
if s.renderQueue.Front() != nil {
url := s.renderQueue.Front().Value.(string)
s.renderQueue.Remove(s.renderQueue.Front())
task := s.newRenderTask(url, offer)
tasks = append(tasks, task)
tasksToLaunch--
}
if s.crawlQueue.Front() == nil && s.renderQueue.Front() == nil {
break
}
}
if len(tasks) == 0 {
driver.DeclineOffer(offer.Id, defaultFilter)
} else {
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, defaultFilter)
}
}
}
func (s *DiegoScheduler) scheduleMatched(driver sched.SchedulerDriver, matches map[string]*OfferMatch) auctiontypes.AuctionResults {
results := auctiontypes.AuctionResults{}
for slaveId, match := range matches {
if slaveId != "" {
offers := match.Offers
taskInfos := []*mesos.TaskInfo{}
for _, lrpAuction := range match.LrpAuctions {
taskInfo := s.createLrpTaskInfo(util.NewSlaveID(slaveId), &lrpAuction)
taskInfos = append(taskInfos, taskInfo)
results.SuccessfulLRPs = append(results.SuccessfulLRPs, lrpAuction)
log.Infof("+scheduled lrp, lrp: %v/%v mem: %v, offers: mem: %v",
lrpAuction.ProcessGuid, lrpAuction.Index, lrpAuction.MemoryMB, getOffersMem(offers))
}
for _, taskAuction := range match.TaskAuctions {
taskInfo := s.createTaskTaskInfo(util.NewSlaveID(slaveId), &taskAuction)
taskInfos = append(taskInfos, taskInfo)
results.SuccessfulTasks = append(results.SuccessfulTasks, taskAuction)
log.Infof("+scheduled task, task: %v mem: %v, offers: mem: %v",
taskAuction.TaskGuid, taskAuction.MemoryMB, getOffersMem(offers))
}
driver.LaunchTasks(extractOfferIds(offers), taskInfos, // offer getting declied if no tasks
&mesos.Filters{RefuseSeconds: proto.Float64(30)})
} else {
for _, lrpAuction := range match.LrpAuctions {
results.FailedLRPs = append(results.FailedLRPs, lrpAuction)
log.Warningf("+schedule lrp failed, lrp: %v/%v mem: %v, offers: mem: %v",
lrpAuction.GetProcessGuid(), lrpAuction.Index, lrpAuction.MemoryMB, getOffersMem(match.Offers))
}
for _, taskAuction := range match.TaskAuctions {
results.FailedTasks = append(results.FailedTasks, taskAuction)
log.Warningf("+schedule task failed, task: %v mem: %v, offers: mem: %v",
taskAuction.TaskGuid, taskAuction.MemoryMB, getOffersMem(match.Offers))
}
}
}
return results
}
// ResourceOffers handles the Resource Offers
func (s *eremeticScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
log.Tracef("Received %d resource offers", len(offers))
var offer *mesos.Offer
loop:
for len(offers) > 0 {
select {
case <-s.shutdown:
log.Info("Shutting down: declining offers")
break loop
case tid := <-s.tasks:
log.Debugf("Trying to find offer to launch %s with", tid)
t, _ := database.ReadTask(tid)
offer, offers = matchOffer(t, offers)
if offer == nil {
log.Warnf("Could not find a matching offer for %s", tid)
TasksDelayed.Inc()
go func() { s.tasks <- tid }()
break loop
}
log.Debugf("Preparing to launch task %s with offer %s", tid, offer.Id.GetValue())
t, task := s.newTask(t, offer)
database.PutTask(&t)
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, []*mesos.TaskInfo{task}, defaultFilter)
TasksLaunched.Inc()
QueueSize.Dec()
continue
default:
break loop
}
}
log.Trace("No tasks to launch. Declining offers.")
for _, offer := range offers {
driver.DeclineOffer(offer.Id, defaultFilter)
}
}
// ResourceOffers handles the Resource Offers
func (s *eremeticScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
log.Tracef("Received %d resource offers", len(offers))
for _, offer := range offers {
select {
case <-s.shutdown:
log.Infof("Shutting down: declining offer on [%s]", offer.Hostname)
driver.DeclineOffer(offer.Id, defaultFilter)
continue
case tid := <-s.tasks:
log.Debugf("Preparing to launch task %s with offer %s", tid, offer.Id.GetValue())
t, _ := database.ReadTask(tid)
task := s.newTask(offer, &t)
database.PutTask(&t)
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, []*mesos.TaskInfo{task}, defaultFilter)
continue
default:
}
log.Trace("No tasks to launch. Declining offer.")
driver.DeclineOffer(offer.Id, defaultFilter)
}
}
// process incoming offers and try to schedule new tasks as they come in on the channel
func (sched *NoneScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
if sched.queue.Next() == nil {
// no command to launch, we don't need to parse anything
return
}
for _, offer := range offers {
// match constraints
if !sched.filter.FilterOffer(offer) {
// skip offer if it does not match constraints
continue
}
remainingCpus := SumScalarResources(sched.filter.FilterResources(offer, "cpus"))
remainingMems := SumScalarResources(sched.filter.FilterResources(offer, "mem"))
log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", remainingCpus, " mem=", remainingMems)
// try to schedule as may tasks as possible for this single offer
var tasks []*mesos.TaskInfo
for sched.queue.GetCommand() != nil &&
sched.queue.GetCommand().MatchesResources(remainingCpus, remainingMems) {
c := sched.queue.GetCommand()
c.SlaveId = offer.SlaveId.GetValue()
c.FrameworkId = sched.frameworkId
sched.handler.CommandLaunched(c)
task := sched.prepareTaskInfo(offer, c)
tasks = append(tasks, task)
remainingCpus -= c.CpuReq
remainingMems -= c.MemReq
sched.queue.Next()
}
log.Infoln("Launching", len(tasks), "tasks for offer", offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
}
func (self *MesosManager) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
log.Printf("ResourceOffers\n")
computeCount := 0
for _, offer := range offers {
offer_cpus := 0
for _, res := range offer.Resources {
if *res.Name == "cpus" {
computeCount += int(*res.Scalar.Value)
offer_cpus += int(*res.Scalar.Value)
}
}
tasks := make([]*mesos.TaskInfo, 0, computeCount)
for offer_cpus_taken := 0; offer_cpus_taken < offer_cpus; {
job := self.engine.GetJobToRun()
if job != nil {
log.Printf("Launch job: %s", job)
mesos_taskinfo := self.BuildTaskInfo(job, offer)
log.Printf("MesosTask: %s", mesos_taskinfo)
tasks = append(tasks, mesos_taskinfo)
offer_cpus_taken += 1
} else {
offer_cpus_taken = offer_cpus
}
}
if len(tasks) > 0 {
_, err := driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{})
if err != nil {
fmt.Print("Error: %s", err)
}
} else {
driver.DeclineOffer(offer.Id, &mesos.Filters{})
}
}
self.computeCount = computeCount
log.Printf("CPUsOffered: %d", self.computeCount)
}
// mesos.Scheduler interface method.
// Invoked when resources have been offered to this framework.
func (this *TransformScheduler) ResourceOffers(driver scheduler.SchedulerDriver, offers []*mesos.Offer) {
fmt.Println("Received offers")
if int(this.runningInstances) > this.config.Instances {
toKill := int(this.runningInstances) - this.config.Instances
for i := 0; i < toKill; i++ {
driver.KillTask(this.tasks[i])
}
this.tasks = this.tasks[toKill:]
}
offersAndTasks := make(map[*mesos.Offer][]*mesos.TaskInfo)
for _, offer := range offers {
cpus := getScalarResources(offer, "cpus")
mems := getScalarResources(offer, "mem")
ports := getRangeResources(offer, "ports")
remainingCpus := cpus
remainingMems := mems
var tasks []*mesos.TaskInfo
for int(this.getRunningInstances()) < this.config.Instances && this.config.CpuPerTask <= remainingCpus && this.config.MemPerTask <= remainingMems && len(ports) > 0 {
port := this.takePort(&ports)
taskPort := &mesos.Value_Range{Begin: port, End: port}
taskId := &mesos.TaskID{
Value: proto.String(fmt.Sprintf("golang-%s-%d", *offer.Hostname, *port)),
}
task := &mesos.TaskInfo{
Name: proto.String(taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Executor: this.createExecutor(this.getRunningInstances(), *port),
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", float64(this.config.CpuPerTask)),
util.NewScalarResource("mem", float64(this.config.MemPerTask)),
util.NewRangesResource("ports", []*mesos.Value_Range{taskPort}),
},
}
fmt.Printf("Prepared task: %s with offer %s for launch. Ports: %s\n", task.GetName(), offer.Id.GetValue(), taskPort)
tasks = append(tasks, task)
remainingCpus -= this.config.CpuPerTask
remainingMems -= this.config.MemPerTask
ports = ports[1:]
this.tasks = append(this.tasks, taskId)
this.incRunningInstances()
}
fmt.Printf("Launching %d tasks for offer %s\n", len(tasks), offer.Id.GetValue())
offersAndTasks[offer] = tasks
}
unlaunchedTasks := this.config.Instances - int(this.getRunningInstances())
if unlaunchedTasks > 0 {
fmt.Printf("There are still %d tasks to be launched and no more resources are available.", unlaunchedTasks)
}
for _, offer := range offers {
tasks := offersAndTasks[offer]
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
}
func (sched *SdcScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
if sched.tasksLaunched >= sched.totalTasks {
log.Info("decline all of the offers since all of our tasks are already launched")
log.Infoln("sched.totalTasks ", sched.totalTasks)
log.Infoln("sched.tasksFinished ", sched.tasksFinished)
log.Infoln("sched.tasksLaunched ", sched.tasksLaunched)
// cmdQueueからCommand.Argumentsをpopする
// 将来的に外部のキューから取得できるように置き換える
if sched.totalTasks == 0 && sched.tasksFinished == 0 && sched.tasksLaunched == 0 && cmdQueue.Len() != 0 {
execinfo := cmdQueue.Remove(cmdQueue.Front()).(*mesos.ExecutorInfo)
log.Infoln("execinfo ", execinfo.Command.Arguments)
sched.totalTasks = len(execinfo.Command.Arguments)
sched.executor.Command.Arguments = execinfo.Command.Arguments
}
if sched.totalTasks == 0 && sched.tasksFinished == 0 && sched.tasksLaunched == 0 {
ids := make([]*mesos.OfferID, len(offers))
for i, offer := range offers {
ids[i] = offer.Id
}
driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
return
}
}
log.Info("prepare pass args: ", sched.executor.Command.Arguments)
cmds := sched.executor.Command.Arguments
for _, v := range cmds {
fmt.Println("v = ", v)
}
// [/bin/cat /var/tmp/1.txt /var/tmp/2.txt /var/tmp/3.txt | /bin/grep abe > /var/tmp/grep-result.txt]
//
// rebuild args
// 1. /bin/cat /var/tmp/1.txt >> /var/tmp/intermediate.txt
// 2. /bin/cat /var/tmp/2.txt >> /var/tmp/intermediate.txt
// 3. /bin/cat /var/tmp/3.txt >> /var/tmp/intermediate.txt
// 4. /bin/cat /var/tmp/intermediate.txt | /bin/grep abe > /var/tmp/grep-result.txt
// 5. /bin/rm /var/tmp/intermediate.txt
for _, offer := range offers {
cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "cpus"
})
cpus := 0.0
for _, res := range cpuResources {
cpus += res.GetScalar().GetValue()
}
memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "mem"
})
mems := 0.0
for _, res := range memResources {
mems += res.GetScalar().GetValue()
}
log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)
remainingCpus := cpus
remainingMems := mems
var tasks []*mesos.TaskInfo
// $ cat 1.txt 2.txt. 3.txt | wc -lの場合
// 先に、後ろのタスクを上げておく必要がある？
//
// コンセプト実装はシンプルに中間ファイル方式で行く
// 遅いけど
for sched.tasksLaunched < sched.totalTasks &&
CPUS_PER_TASK <= remainingCpus &&
MEM_PER_TASK <= remainingMems {
sched.tasksLaunched++
taskId := &mesos.TaskID{
Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
}
// executionidの書き換え
sched.executor.ExecutorId = util.NewExecutorID(taskId.GetValue())
log.Infof("sched.tasksLaunched = %d\n", sched.tasksLaunched)
log.Infof("sched.totalTasks = %d\n", sched.totalTasks)
log.Infof("sched.executor.Command.Value = %s\n", sched.executor.Command.GetValue())
log.Infof("sched.executor.GetExecutorId() = %s\n", sched.executor.GetExecutorId())
// sched.executor.Command.Arguments で書き換えても保持されている
// 値はポインタなので、LaunchTasksするときに、複数のタスクでまとめられているので、
// 値は最後に上書きされた物になる
// そこで、Argumentsがタスクごとにまとめられないように個別にオブジェクトを生成してタスクを
// 起動する
exec := &mesos.ExecutorInfo{
ExecutorId: sched.executor.GetExecutorId(),
Name: proto.String(sched.executor.GetName()),
Source: proto.String(sched.executor.GetSource()),
Command: &mesos.CommandInfo{
Value: proto.String(sched.executor.Command.GetValue()),
Uris: sched.executor.Command.GetUris(),
},
}
cmd := cmds[sched.tasksLaunched-1]
log.Infof("cmd = %s\n", cmd)
// Argumentsコマンドラインを使うと別Executorとみなされるので、色々面倒
// 以下はやってはいけない例
// exec.Command.Arguments = strings.Split(cmd, " ")
task := &mesos.TaskInfo{
Name: proto.String("go-task-" + taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Executor: exec,
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", CPUS_PER_TASK),
util.NewScalarResource("mem", MEM_PER_TASK),
},
// 実行したいコマンドラインはDataパラメータを使って渡せす
Data: []byte(cmd),
}
log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
tasks = append(tasks, task)
remainingCpus -= CPUS_PER_TASK
remainingMems -= MEM_PER_TASK
}
log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
}
}
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
if *slowLaunch {
time.Sleep(3 * time.Second)
}
if (sched.tasksLaunched - sched.tasksErrored) >= sched.totalTasks {
log.Info("decline all of the offers since all of our tasks are already launched")
ids := make([]*mesos.OfferID, len(offers))
for i, offer := range offers {
ids[i] = offer.Id
}
driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(120)})
return
}
for _, offer := range offers {
cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "cpus"
})
cpus := 0.0
for _, res := range cpuResources {
cpus += res.GetScalar().GetValue()
}
memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "mem"
})
mems := 0.0
for _, res := range memResources {
mems += res.GetScalar().GetValue()
}
log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)
remainingCpus := cpus
remainingMems := mems
// account for executor resources if there's not an executor already running on the slave
if len(offer.ExecutorIds) == 0 {
remainingCpus -= CPUS_PER_EXECUTOR
remainingMems -= MEM_PER_EXECUTOR
}
var tasks []*mesos.TaskInfo
for (sched.tasksLaunched-sched.tasksErrored) < sched.totalTasks &&
CPUS_PER_TASK <= remainingCpus &&
MEM_PER_TASK <= remainingMems {
sched.tasksLaunched++
taskId := &mesos.TaskID{
Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
}
task := &mesos.TaskInfo{
Name: proto.String("go-task-" + taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Executor: sched.executor,
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", CPUS_PER_TASK),
util.NewScalarResource("mem", MEM_PER_TASK),
},
}
log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
tasks = append(tasks, task)
remainingCpus -= CPUS_PER_TASK
remainingMems -= MEM_PER_TASK
}
log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
}
}
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
for _, offer := range offers {
cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "cpus"
})
cpus := 0.0
for _, res := range cpuResources {
cpus += res.GetScalar().GetValue()
}
memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "mem"
})
mems := 0.0
for _, res := range memResources {
mems += res.GetScalar().GetValue()
}
log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)
remainingCpus := cpus
remainingMems := mems
var tasks []*mesos.TaskInfo
queueItems := getQueue(sched)
if len(queueItems) == 0 {
fmt.Println("Queue is empty!")
}
// start a task of each item in the queue. before starting, check
// if it has not already been launched.
for i, item := range queueItems {
if val, ok := launchedTasks[item.Commit.Sha]; ok {
fmt.Println("item exists: ", val)
continue
}
fmt.Println("Adding item: ", item.Commit.Sha)
launchedTasks[item.Commit.Sha] = item.Commit.Sha
if CPUS_PER_TASK <= remainingCpus && MEM_PER_TASK <= remainingMems {
sched.tasksLaunched += i
taskId := &mesos.TaskID{
Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
}
stringArray := []string{"-addr=" + *droneServerIP, "-token=1"}
dataString := strings.Join(stringArray, " ")
task := &mesos.TaskInfo{
Name: proto.String("go-task-" + taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Executor: sched.executor,
Data: []byte(dataString),
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", CPUS_PER_TASK),
util.NewScalarResource("mem", MEM_PER_TASK),
},
}
log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
tasks = append(tasks, task)
remainingCpus -= CPUS_PER_TASK
remainingMems -= MEM_PER_TASK
}
}
// for sched.tasksLaunched < sched.totalTasks &&
// CPUS_PER_TASK <= remainingCpus &&
// MEM_PER_TASK <= remainingMems {
//
// sched.tasksLaunched++
//
// taskId := &mesos.TaskID{
// Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
// }
//
// stringArray := []string{*droneServerIP, "-token=1"}
// dataString := strings.Join(stringArray, " ")
//
// task := &mesos.TaskInfo{
// Name: proto.String("go-task-" + taskId.GetValue()),
// TaskId: taskId,
// SlaveId: offer.SlaveId,
// Executor: sched.executor,
// Data: []byte(dataString),
// Resources: []*mesos.Resource{
// util.NewScalarResource("cpus", CPUS_PER_TASK),
// util.NewScalarResource("mem", MEM_PER_TASK),
// },
// }
//
// log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
//
// tasks = append(tasks, task)
// remainingCpus -= CPUS_PER_TASK
// remainingMems -= MEM_PER_TASK
// }
log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
}
// Schedule handles Mesos resource offers and based on the received offers launches PENDING Tasks
//
// It monitors PENDING Tasks Queue and tries to launch as many Tasks as possible for a single offer.
// If no PENDING Tasks are available within QueueTimeout interval it declines the offer and retries Queue read again
func (bw *BasicWorker) ScheduleTasks(driver scheduler.SchedulerDriver, offers []*mesos.Offer) {
ReadOffers:
for _, offer := range offers {
remainingCpus := taurus.ScalarResourceVal("cpus", offer.Resources)
remainingMems := taurus.ScalarResourceVal("mem", offer.Resources)
log.Println("Taurus Received Offer <", offer.Id.GetValue(), "> with cpus=", remainingCpus, " mem=", remainingMems)
// Map to avoid launching duplicate tasks in the same batch slice
launchTaskMap := make(map[string]bool)
launchTasks := make([]*mesos.TaskInfo, 0, 0)
var taskCpu, taskMem float64
var retryCount int
ReadTasks:
for {
task, err := bw.pending.ReadTask(QueueTimeout)
if err != nil {
retryCount += 1
switch {
case bw.pending.TimedOut(err):
log.Printf("No %s tasks available", taurus.PENDING)
case bw.pending.Closed(err):
break ReadTasks
default:
log.Printf("Failed to read from %s queue: %s", taurus.PENDING, err)
}
if retryCount == QueueRetry {
break ReadTasks
}
continue ReadTasks
}
if task != nil {
taskId := task.Info.TaskId.GetValue()
// Don't add the same task bwice into launchTasks slice
if launchTaskMap[taskId] {
log.Printf("Skipping already queued Task %s", taskId)
continue ReadTasks
}
taskCpu = taurus.ScalarResourceVal("cpus", task.Info.Resources)
taskMem = taurus.ScalarResourceVal("mem", task.Info.Resources)
if remainingCpus >= taskCpu && remainingMems >= taskMem {
task.Info.SlaveId = offer.SlaveId
launchTasks = append(launchTasks, task.Info)
launchTaskMap[taskId] = true
remainingCpus -= taskCpu
remainingMems -= taskMem
} else {
break ReadTasks
}
}
}
if len(launchTasks) > 0 {
log.Printf("Launching %d tasks for offer %s", len(launchTasks), offer.Id.GetValue())
launchStatus, err := driver.LaunchTasks(
[]*mesos.OfferID{offer.Id},
launchTasks,
&mesos.Filters{RefuseSeconds: proto.Float64(1)})
if err != nil {
log.Printf("Mesos status: %#v Failed to launch Tasks %s: %s", launchStatus, launchTasks, err)
continue ReadOffers
}
} else {
log.Println("Declining offer ", offer.Id.GetValue())
declineStatus, err := driver.DeclineOffer(
offer.Id,
&mesos.Filters{RefuseSeconds: proto.Float64(1)})
if err != nil {
log.Printf("Error declining offer for mesos status %#v: %s", declineStatus, err)
}
}
}
}
// TODO(tyler) split this long function up!
func (s *EtcdScheduler) launchOne(driver scheduler.SchedulerDriver) {
// Always ensure we've pruned any dead / unmanaged nodes before
// launching new ones, or we may overconfigure the ensemble such
// that it can not make progress if the next launch fails.
err := s.Prune()
if err != nil {
log.Errorf("Failed to remove stale cluster members: %s", err)
return
}
if !s.shouldLaunch(driver) {
log.Infoln("Skipping launch attempt for now.")
return
}
// validOffer filters out offers that are no longer
// desirable, even though they may have been when
// they were enqueued.
validOffer := func(offer *mesos.Offer) bool {
runningCopy := s.RunningCopy()
for _, etcdConfig := range runningCopy {
if etcdConfig.SlaveID == offer.SlaveId.GetValue() {
if s.singleInstancePerSlave {
log.Info("Skipping offer: already running on this slave.")
return false
}
}
}
return true
}
// Issue BlockingPop until we get back an offer we can use.
var offer *mesos.Offer
for {
offer = s.offerCache.BlockingPop()
if validOffer(offer) {
break
} else {
s.decline(driver, offer)
}
}
// Do this again because BlockingPop may have taken a long time.
if !s.shouldLaunch(driver) {
log.Infoln("Skipping launch attempt for now.")
s.decline(driver, offer)
return
}
// TODO(tyler) this is a broken hack
resources := parseOffer(offer)
lowest := *resources.ports[0].Begin
rpcPort := lowest
clientPort := lowest + 1
httpPort := lowest + 2
s.mut.Lock()
var clusterType string
if len(s.running) == 0 {
clusterType = "new"
} else {
clusterType = "existing"
}
s.highestInstanceID++
name := "etcd-" + strconv.FormatInt(s.highestInstanceID, 10)
node := &config.Node{
Name: name,
Host: *offer.Hostname,
RPCPort: rpcPort,
ClientPort: clientPort,
ReseedPort: httpPort,
Type: clusterType,
SlaveID: offer.GetSlaveId().GetValue(),
}
running := []*config.Node{node}
for _, r := range s.running {
running = append(running, r)
}
serializedNodes, err := json.Marshal(running)
log.Infof("Serialized running: %+v", string(serializedNodes))
if err != nil {
log.Errorf("Could not serialize running list: %v", err)
// This Unlock is not deferred because the test implementation of LaunchTasks
// calls this scheduler's StatusUpdate method, causing the test to deadlock.
s.decline(driver, offer)
s.mut.Unlock()
return
}
configSummary := node.String()
taskID := &mesos.TaskID{Value: &configSummary}
executor := s.newExecutorInfo(node, s.executorUris)
task := &mesos.TaskInfo{
Data: serializedNodes,
Name: proto.String("etcd-server"),
TaskId: taskID,
SlaveId: offer.SlaveId,
Executor: executor,
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", s.cpusPerTask),
util.NewScalarResource("mem", s.memPerTask),
util.NewScalarResource("disk", s.diskPerTask),
util.NewRangesResource("ports", []*mesos.Value_Range{
util.NewValueRange(uint64(rpcPort), uint64(httpPort)),
}),
},
Discovery: &mesos.DiscoveryInfo{
Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(),
Name: proto.String("etcd-server"),
Ports: &mesos.Ports{
Ports: []*mesos.Port{
&mesos.Port{
Number: proto.Uint32(uint32(rpcPort)),
Protocol: proto.String("tcp"),
},
// HACK: "client" is not a real SRV protocol. This is so
// that we can have etcd proxies use srv discovery on the
// above tcp name. Mesos-dns does not yet care about
// names for DiscoveryInfo. When it does, we should
// create a name for clients to use. We want to keep
// the rpcPort accessible at _etcd-server._tcp.<fwname>.mesos
&mesos.Port{
Number: proto.Uint32(uint32(clientPort)),
Protocol: proto.String("client"),
},
},
},
},
}
log.Infof(
"Prepared task: %s with offer %s for launch",
task.GetName(),
offer.Id.GetValue(),
)
log.Info("Launching etcd node.")
tasks := []*mesos.TaskInfo{task}
s.pending[node.Name] = struct{}{}
// This Unlock is not deferred because the test implementation of LaunchTasks
// calls this scheduler's StatusUpdate method, causing the test to deadlock.
s.mut.Unlock()
atomic.AddUint32(&s.Stats.LaunchedServers, 1)
driver.LaunchTasks(
[]*mesos.OfferID{offer.Id},
tasks,
&mesos.Filters{
RefuseSeconds: proto.Float64(1),
},
)
}
func (s *MinerScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
for i, offer := range offers {
memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "mem"
})
mems := 0.0
for _, res := range memResources {
mems += res.GetScalar().GetValue()
}
cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "cpus"
})
cpus := 0.0
for _, res := range cpuResources {
cpus += res.GetScalar().GetValue()
}
portsResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "ports"
})
var ports uint64
for _, res := range portsResources {
port_ranges := res.GetRanges().GetRange()
for _, port_range := range port_ranges {
ports += port_range.GetEnd() - port_range.GetBegin()
}
}
// If a miner server is running, we start a new miner daemon. Otherwise, we start a new miner server.
tasks := make([]*mesos.TaskInfo, 0)
if !s.minerServerRunning && mems >= MEM_PER_SERVER_TASK && cpus >= CPU_PER_SERVER_TASK && ports >= 2 {
var taskId *mesos.TaskID
var task *mesos.TaskInfo
// we need two ports
var p2pool_port uint64
var worker_port uint64
// A rather stupid algorithm for picking two ports
// The difficulty here is that a range might only include one port,
// in which case we will need to pick another port from another range.
for _, res := range portsResources {
r := res.GetRanges().GetRange()[0]
begin := r.GetBegin()
end := r.GetEnd()
if p2pool_port == 0 {
p2pool_port = begin
if worker_port == 0 && (begin+1) <= end {
worker_port = begin + 1
break
}
continue
}
if worker_port == 0 {
worker_port = begin
break
}
}
taskId = &mesos.TaskID{
Value: proto.String("miner-server-" + strconv.Itoa(i)),
}
containerType := mesos.ContainerInfo_DOCKER
task = &mesos.TaskInfo{
Name: proto.String("task-" + taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Container: &mesos.ContainerInfo{
Type: &containerType,
Docker: &mesos.ContainerInfo_DockerInfo{
Image: proto.String(MINER_SERVER_DOCKER_IMAGE),
},
},
Command: &mesos.CommandInfo{
Shell: proto.Bool(false),
Arguments: []string{
// these arguments will be passed to run_p2pool.py
"--bitcoind-address", *bitcoindAddr,
"--p2pool-port", strconv.Itoa(int(p2pool_port)),
"-w", strconv.Itoa(int(worker_port)),
s.rpc_user, s.rpc_pass,
},
},
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", CPU_PER_SERVER_TASK),
util.NewScalarResource("mem", MEM_PER_SERVER_TASK),
},
}
log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
cpus -= CPU_PER_SERVER_TASK
mems -= MEM_PER_SERVER_TASK
// update state
s.minerServerHostname = offer.GetHostname()
s.minerServerRunning = true
s.minerServerPort = int(worker_port)
tasks = append(tasks, task)
}
if s.minerServerRunning && mems >= MEM_PER_DAEMON_TASK {
var taskId *mesos.TaskID
var task *mesos.TaskInfo
taskId = &mesos.TaskID{
Value: proto.String("miner-daemon-" + strconv.Itoa(i)),
}
containerType := mesos.ContainerInfo_DOCKER
task = &mesos.TaskInfo{
Name: proto.String("task-" + taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Container: &mesos.ContainerInfo{
Type: &containerType,
Docker: &mesos.ContainerInfo_DockerInfo{
Image: proto.String(MINER_DAEMON_DOCKER_IMAGE),
},
},
Command: &mesos.CommandInfo{
Shell: proto.Bool(false),
Arguments: []string{"-o", s.minerServerHostname + ":" + strconv.Itoa(s.minerServerPort)},
},
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", cpus),
util.NewScalarResource("mem", MEM_PER_DAEMON_TASK),
},
}
log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
tasks = append(tasks, task)
}
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
}
func (s *visghsScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
if (s.nodesLaunched - s.nodesErrored) >= s.nodeTasks {
log.Info("decline all of the offers since all of our tasks are already launched")
ids := make([]*mesos.OfferID, len(offers))
for i, offer := range offers {
ids[i] = offer.Id
}
driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(120)})
return
}
for _, offer := range offers {
cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "cpus"
})
cpus := 0.0
for _, res := range cpuResources {
cpus += res.GetScalar().GetValue()
}
memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "mem"
})
mems := 0.0
for _, res := range memResources {
mems += res.GetScalar().GetValue()
}
portResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
return res.GetName() == "ports"
})
ports := []*mesos.Value_Range{}
portsCount := uint64(0)
for _, res := range portResources {
for _, rs := range res.GetRanges().GetRange() {
ports = append(ports, rs)
portsCount += 1 + rs.GetEnd() - rs.GetBegin()
}
}
log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems, " ports=", ports)
remainingCpus := cpus
remainingMems := mems
remainingPorts := ports
remainingPortsCount := portsCount
// account for executor resources if there's not an executor already running on the slave
if len(offer.ExecutorIds) == 0 {
remainingCpus -= CPUS_PER_EXECUTOR
remainingMems -= MEM_PER_EXECUTOR
}
var tasks []*mesos.TaskInfo
for (s.nodesLaunched-s.nodesErrored) < s.nodeTasks &&
CPUS_PER_TASK <= remainingCpus &&
MEM_PER_TASK <= remainingMems &&
PORTS_PER_TASK <= remainingPortsCount {
log.Infoln("Ports <", remainingPortsCount, remainingPorts)
s.nodesLaunched++
taskId := &mesos.TaskID{
Value: proto.String(strconv.Itoa(s.nodesLaunched)),
}
taskPorts := []*mesos.Value_Range{}
leftOverPorts := []*mesos.Value_Range{}
for t := 0; t < PORTS_PER_TASK; t++ {
if len(remainingPorts) < 1 {
// failed to allocate port, oh no!
}
ps := remainingPorts[0]
//take the first port from the first
pb := ps.GetBegin()
pe := ps.GetEnd()
//Create one range per port we need, it's easier this way
tp := mesos.Value_Range{}
p := pb
tp.Begin = &p
tp.End = &p
taskPorts = append(taskPorts, &tp)
pb++
if pb <= pe {
rpb := pb
rpe := pe
rtp := mesos.Value_Range{Begin: &rpb, End: &rpe}
leftOverPorts = append(leftOverPorts, &rtp)
}
for _, ps := range remainingPorts[1:] {
leftOverPorts = append(leftOverPorts, ps)
}
}
radiaPort := (uint32)(taskPorts[0].GetBegin())
task := &mesos.TaskInfo{
Name: proto.String("visghs-node-" + taskId.GetValue()),
TaskId: taskId,
SlaveId: offer.SlaveId,
Executor: s.nexec,
Discovery: &mesos.DiscoveryInfo{
Name: proto.String("visghs"),
//Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(),
Visibility: mesos.DiscoveryInfo_FRAMEWORK.Enum(),
Ports: &mesos.Ports{
Ports: []*mesos.Port{
{Protocol: proto.String("UDP"),
Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(),
Name: proto.String("udpprobe"),
Number: proto.Uint32(radiaPort)},
{Protocol: proto.String("TCP"),
Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(),
Name: proto.String("radiarpc"),
Number: proto.Uint32(radiaPort)},
},
},
},
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", CPUS_PER_TASK),
util.NewScalarResource("mem", MEM_PER_TASK),
util.NewRangesResource("ports", taskPorts),
},
}
log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
tasks = append(tasks, task)
remainingCpus -= CPUS_PER_TASK
remainingMems -= MEM_PER_TASK
remainingPorts = leftOverPorts
remainingPortsCount -= PORTS_PER_TASK
}
log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
}
}