Example #1
0
func (sc *SchedulerCore) acceptOffer(driver sched.SchedulerDriver, offer *mesos.Offer, operations []*mesos.Offer_Operation) {
	log.Infof("Accepting OfferID: %+v, Operations: %+v", *offer.Id.Value, operations)

	var status mesos.Status
	var err error

	if sc.compatibilityMode {
		tasks := []*mesos.TaskInfo{}
		for _, operation := range operations {
			if *operation.Type == mesos.Offer_Operation_LAUNCH {
				tasks = operation.Launch.TaskInfos
			}
		}
		status, err = driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(OFFER_INTERVAL)})
	} else {
		status, err = driver.AcceptOffers([]*mesos.OfferID{offer.Id}, operations, &mesos.Filters{RefuseSeconds: proto.Float64(OFFER_INTERVAL)})
	}

	if status != mesos.Status_DRIVER_RUNNING {
		log.Fatal("Driver not running, while trying to accept offers")
	}
	if err != nil {
		log.Panic("Failed to launch tasks: ", err)
	}
}
Example #2
0
func (s *Scheduler) launchTask(driver scheduler.SchedulerDriver, offer *mesos.Offer) {
	taskName := fmt.Sprintf("syslog-%s", offer.GetSlaveId().GetValue())
	taskId := &mesos.TaskID{
		Value: proto.String(fmt.Sprintf("%s-%s", taskName, uuid())),
	}

	data, err := json.Marshal(Config)
	if err != nil {
		panic(err) //shouldn't happen
	}
	Logger.Debugf("Task data: %s", string(data))

	tcpPort := uint64(s.getPort(Config.TcpPort, offer, -1))
	udpPort := uint64(s.getPort(Config.UdpPort, offer, int(tcpPort)))

	task := &mesos.TaskInfo{
		Name:     proto.String(taskName),
		TaskId:   taskId,
		SlaveId:  offer.GetSlaveId(),
		Executor: s.createExecutor(offer, tcpPort, udpPort),
		Resources: []*mesos.Resource{
			util.NewScalarResource("cpus", Config.Cpus),
			util.NewScalarResource("mem", Config.Mem),
			util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(tcpPort, tcpPort)}),
			util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(udpPort, udpPort)}),
		},
		Data:   data,
		Labels: utils.StringToLabels(s.labels),
	}

	s.cluster.Add(offer.GetSlaveId().GetValue(), task)

	driver.LaunchTasks([]*mesos.OfferID{offer.GetId()}, []*mesos.TaskInfo{task}, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
Example #3
0
func (sched *Scheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	logOffers(offers)
	jobs, err := getLaunchableJobs()
	if err != nil {
		log.Errorf("Unable to get pending jobs! %s\n", err.Error())
		return
	}

	offersAndTasks, err := packJobsInOffers(jobs, offers)
	if err != nil {
		log.Errorf("Unable to pack jobs into offers! %s\n", err.Error())
		return
	}

	for _, ot := range offersAndTasks {
		if len(ot.Tasks) == 0 {
			log.Infof("Declining unused offer %s", ot.Offer.Id.GetValue())
			driver.DeclineOffer(ot.Offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
			continue
		} else {
			log.Infof("Launching %d tasks for offer %s\n", len(ot.Tasks), ot.Offer.Id.GetValue())
			driver.LaunchTasks([]*mesos.OfferID{ot.Offer.Id}, ot.Tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
			sched.tasksLaunched = sched.tasksLaunched + len(ot.Tasks)
		}
	}

}
Example #4
0
func (s *MemcacheScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	log.Printf("Received %d resource offers", len(offers))
	for _, offer := range offers {
		select {
		case <-s.shutdown:
			log.Println("Shutting down: declining offer on [", offer.Hostname, "]")
			driver.DeclineOffer(offer.Id, defaultFilter)
			if s.tasksRunning == 0 {
				close(s.done)
			}
			continue
		default:
		}

		tasks := []*mesos.TaskInfo{}
		if canLaunchNewTask(offer) && s.shouldLaunchNewTask() {
			fmt.Println("Accepting Offer: ", offer)
			task := s.newMemcacheTask(offer)
			tasks = append(tasks, task)
		}

		if len(tasks) == 0 {
			driver.DeclineOffer(offer.Id, defaultFilter)
		} else {
			driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, defaultFilter)
			s.lastTaskCreatedAt = time.Now()
			return // limit one at a time
		}
	}
}
Example #5
0
func (s *Scheduler) launchTask(driver scheduler.SchedulerDriver, offer *mesos.Offer) {
	taskName := fmt.Sprintf("syscol-%s", offer.GetSlaveId().GetValue())
	taskId := &mesos.TaskID{
		Value: proto.String(fmt.Sprintf("%s-%s", taskName, uuid())),
	}

	data, err := json.Marshal(Config)
	if err != nil {
		panic(err) //shouldn't happen
	}
	Logger.Debugf("Task data: %s", string(data))

	task := &mesos.TaskInfo{
		Name:     proto.String(taskName),
		TaskId:   taskId,
		SlaveId:  offer.GetSlaveId(),
		Executor: s.createExecutor(offer.GetSlaveId().GetValue()),
		Resources: []*mesos.Resource{
			util.NewScalarResource("cpus", Config.Cpus),
			util.NewScalarResource("mem", Config.Mem),
		},
		Data: data,
	}

	s.cluster.Add(offer.GetSlaveId().GetValue(), task)

	driver.LaunchTasks([]*mesos.OfferID{offer.GetId()}, []*mesos.TaskInfo{task}, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
}
Example #6
0
// ResourceOffers handles the Resource Offers
func (s *Scheduler) ResourceOffers(driver mesossched.SchedulerDriver, offers []*mesosproto.Offer) {
	logrus.WithField("offers", len(offers)).Debug("Received offers")
	var offer *mesosproto.Offer

loop:
	for len(offers) > 0 {
		select {
		case <-s.shutdown:
			logrus.Info("Shutting down: declining offers")
			break loop
		case tid := <-s.tasks:
			logrus.WithField("task_id", tid).Debug("Trying to find offer to launch task with")
			t, _ := s.database.ReadUnmaskedTask(tid)

			if t.IsTerminating() {
				logrus.Debug("Dropping terminating task.")
				t.UpdateStatus(eremetic.Status{
					Status: eremetic.TaskKilled,
					Time:   time.Now().Unix(),
				})
				s.database.PutTask(&t)

				continue
			}
			offer, offers = matchOffer(t, offers)

			if offer == nil {
				logrus.WithField("task_id", tid).Warn("Unable to find a matching offer")
				tasksDelayed.Inc()
				go func() { s.tasks <- tid }()
				break loop
			}

			logrus.WithFields(logrus.Fields{
				"task_id":  tid,
				"offer_id": offer.Id.GetValue(),
			}).Debug("Preparing to launch task")

			t, task := createTaskInfo(t, offer)
			t.UpdateStatus(eremetic.Status{
				Status: eremetic.TaskStaging,
				Time:   time.Now().Unix(),
			})
			s.database.PutTask(&t)
			driver.LaunchTasks([]*mesosproto.OfferID{offer.Id}, []*mesosproto.TaskInfo{task}, defaultFilter)
			tasksLaunched.Inc()
			queueSize.Dec()

			continue
		default:
			break loop
		}
	}

	logrus.Debug("No tasks to launch. Declining offers.")
	for _, offer := range offers {
		driver.DeclineOffer(offer.Id, defaultFilter)
	}
}
func (s *rancherScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	task := tasks.GetNextTask()
	if task == nil {
		for _, of := range offers {
			driver.DeclineOffer(of.Id, defaultFilter)
		}
		return
	}
	if task.RegistrationUrl == "" {
		tasks.AddTask(task)
		for _, of := range offers {
			driver.DeclineOffer(of.Id, defaultFilter)
		}
		return
	}
	taskBytes, err := task.Marshal()
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
		}).Error("Error Marshalling task")
		for _, of := range offers {
			driver.DeclineOffer(of.Id, defaultFilter)
		}
		return
	}
	for _, offer := range offers {
		inadequate := false
		for _, res := range offer.GetResources() {
			if res.GetName() == "cpus" && *res.GetScalar().Value < taskCPUs {
				driver.DeclineOffer(offer.Id, defaultFilter)
				inadequate = true
				continue
			}
			if res.GetName() == "mem" && *res.GetScalar().Value < taskMem {
				driver.DeclineOffer(offer.Id, defaultFilter)
				inadequate = true
				continue
			}
		}
		if inadequate {
			continue
		}
		mesosTask := &mesos.TaskInfo{
			TaskId: &mesos.TaskID{
				Value: proto.String(task.HostUuid),
			},
			SlaveId: offer.SlaveId,
			Resources: []*mesos.Resource{
				mesosutil.NewScalarResource("cpus", taskCPUs),
				mesosutil.NewScalarResource("mem", taskMem),
			},
			Data:     taskBytes,
			Name:     &task.Name,
			Executor: s.rancherExecutor,
		}
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, []*mesos.TaskInfo{mesosTask}, defaultFilter)
	}
}
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	logOffers(offers)
	for _, offer := range offers {
		remainingCpus := getOfferCpu(offer)
		remainingMems := getOfferMem(offer)

		var tasks []*mesos.TaskInfo
		for sched.cpuPerTask <= remainingCpus &&
			sched.memPerTask <= remainingMems &&
			sched.tasksLaunched < sched.totalTasks {

			fmt.Printf("Tasks launched: %v Total tasks: %v\n", sched.tasksLaunched, sched.totalTasks)

			sched.tasksLaunched++

			taskId := &mesos.TaskID{
				Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
			}

			dockerInfo := &mesos.ContainerInfo_DockerInfo{
				Image:        &sched.DockerImage,
				PortMappings: sched.DockerPorts,
			}

			containerType := mesos.ContainerInfo_DOCKER

			containerInfo := &mesos.ContainerInfo{
				Type:   &containerType,
				Docker: dockerInfo,
			}

			commandInfo := &mesos.CommandInfo{
				Value: &sched.DockerCommand,
			}

			task := &mesos.TaskInfo{
				Name:    proto.String("go-task-" + taskId.GetValue()),
				TaskId:  taskId,
				SlaveId: offer.SlaveId,
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", sched.cpuPerTask),
					util.NewScalarResource("mem", sched.memPerTask),
				},
				Container: containerInfo,
				Command:   commandInfo,
			}
			fmt.Printf("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			tasks = append(tasks, task)
			remainingCpus -= sched.cpuPerTask
			remainingMems -= sched.memPerTask
		}
		//		fmt.Println("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
	}

}
func (sched *Scheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	for _, offer := range offers {
		taskId := &mesos.TaskID{
			Value: proto.String(fmt.Sprintf("basicdocker-task-%d", time.Now().Unix())),
		}

		ports := util.FilterResources(
			offer.Resources,
			func(res *mesos.Resource) bool {
				return res.GetName() == "ports"
			},
		)
		if len(ports) > 0 && len(ports[0].GetRanges().GetRange()) > 0 {

		} else {
			return
		}
		task := &mesos.TaskInfo{
			Name:    proto.String(taskId.GetValue()),
			TaskId:  taskId,
			SlaveId: offer.SlaveId,
			Container: &mesos.ContainerInfo{
				Type:     mesos.ContainerInfo_DOCKER.Enum(),
				Volumes:  nil,
				Hostname: nil,
				Docker: &mesos.ContainerInfo_DockerInfo{
					Image:   &DOCKER_IMAGE_DEFAULT,
					Network: mesos.ContainerInfo_DockerInfo_BRIDGE.Enum(),
				},
			},
			Command: &mesos.CommandInfo{
				Shell: proto.Bool(true),
				Value: proto.String("set -x ; /bin/date ; /bin/hostname ; sleep 200 ; echo done"),
			},
			Executor: nil,
			Resources: []*mesos.Resource{
				util.NewScalarResource("cpus", getOfferCpu(offer)),
				util.NewScalarResource("mem", getOfferMem(offer)),
				util.NewRangesResource("ports", []*mesos.Value_Range{
					util.NewValueRange(
						*ports[0].GetRanges().GetRange()[0].Begin,
						*ports[0].GetRanges().GetRange()[0].Begin+1,
					),
				}),
			},
		}

		log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

		var tasks []*mesos.TaskInfo = []*mesos.TaskInfo{task}
		log.Infoln("Launching ", len(tasks), " tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
		sched.tasksLaunched++
		time.Sleep(time.Second)
	}
}
Example #10
0
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {

	for _, offer := range offers {
		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)

		remainingCpus := cpus
		remainingMems := mems

		var tasks []*mesos.TaskInfo
		for sched.tasksLaunched < sched.totalTasks &&
			CPUS_PER_TASK <= remainingCpus &&
			MEM_PER_TASK <= remainingMems {

			sched.tasksLaunched++

			taskId := &mesos.TaskID{
				Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
			}

			task := &mesos.TaskInfo{
				Name:     proto.String("go-task-" + taskId.GetValue()),
				TaskId:   taskId,
				SlaveId:  offer.SlaveId,
				Executor: sched.executor,
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", CPUS_PER_TASK),
					util.NewScalarResource("mem", MEM_PER_TASK),
				},
			}
			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			tasks = append(tasks, task)
			remainingCpus -= CPUS_PER_TASK
			remainingMems -= MEM_PER_TASK
		}
		log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
	}
}
func (ctx *RunOnceApplicationContext) LaunchTask(driver scheduler.SchedulerDriver, offer *mesos.Offer) error {
	ctx.lock.Lock()
	defer ctx.lock.Unlock()

	ctx.InstancesLeftToRun--
	taskInfo := ctx.newTaskInfo(offer)
	ctx.tasks = append(ctx.tasks, newRunOnceTask(offer, taskInfo.GetTaskId().GetValue()))

	_, err := driver.LaunchTasks([]*mesos.OfferID{offer.GetId()}, []*mesos.TaskInfo{taskInfo}, &mesos.Filters{RefuseSeconds: proto.Float64(10)})
	return err
}
Example #12
0
// mesos.Scheduler interface method.
// Invoked when resources have been offered to this framework.
func (this *ElodinaTransportScheduler) ResourceOffers(driver scheduler.SchedulerDriver, offers []*mesos.Offer) {
	log.Logger.Info("Received offers")
	offersAndTasks := make(map[*mesos.Offer][]*mesos.TaskInfo)
	remainingPartitions, err := this.GetTopicPartitions()
	if err != nil {
		return
	}
	remainingPartitions.RemoveAll(this.TakenTopicPartitions.GetArray())
	log.Logger.Debug("%v", remainingPartitions)
	tps := remainingPartitions.GetArray()

	offersAndResources := this.wrapInOfferAndResources(offers)
	for !remainingPartitions.IsEmpty() {
		log.Logger.Debug("Iteration %v", remainingPartitions)
		if this.hasEnoughInstances() {
			for _, transfer := range this.taskIdToTaskState {
				if len(transfer.assignment) < this.config.ThreadsPerTask {
					transfer.assignment = append(transfer.assignment, tps[0])
					remainingPartitions.Remove(tps[0])
					this.TakenTopicPartitions.Add(tps[0])
					if len(tps) > 1 {
						tps = tps[1:]
					} else {
						tps = []consumer.TopicAndPartition{}
					}
				}
			}
		} else {
			log.Logger.Debug("Trying to launch new task")
			offer, task := this.launchNewTask(offersAndResources)
			if offer != nil && task != nil {
				offersAndTasks[offer] = append(offersAndTasks[offer], task)
			} else {
				for _, offer := range offers {
					if _, exists := offersAndTasks[offer]; !exists {
						offersAndTasks[offer] = make([]*mesos.TaskInfo, 0)
					}
				}
				break
			}
		}
	}

	this.assignPendingPartitions()

	for _, offer := range offers {
		if tasks, ok := offersAndTasks[offer]; ok {
			driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
		} else {
			driver.DeclineOffer(offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(10)})
		}
	}
}
Example #13
0
func (sc *SchedulerCore) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Received resource offers: ", offers)
	launchTasks := make(map[string][]*mesos.TaskInfo)
	toBeScheduled := []*FrameworkRiakNode{}
	for _, cluster := range sc.schedulerState.Clusters {
		for _, riakNode := range cluster.Nodes {
			if riakNode.NeedsToBeScheduled() {
				log.Infof("Adding Riak node for scheduling: %+v", riakNode)
				// We need to schedule this task I guess?
				toBeScheduled = append(toBeScheduled, riakNode)
			}
		}
	}

	// Populate a mutable slice of offer resources
	allResources := [][]*mesos.Resource{}
	for _, offer := range offers {
		allResources = append(allResources, offer.Resources)
	}

	launchTasks, err := sc.spreadNodesAcrossOffers(offers, allResources, toBeScheduled, 0, 0, launchTasks)
	if err != nil {
		log.Error(err)
	}

	for _, offer := range offers {
		tasks := launchTasks[*offer.Id.Value]

		if tasks == nil {
			tasks = []*mesos.TaskInfo{}
		}

		// This is somewhat of a hack, to avoid synchronously calling the mesos-go SDK
		// to avoid a deadlock situation.
		// TODO: Fix and make actually queues around driver interactions
		// This is a massive hack
		// -Sargun Dhillon 2015-10-01
		go func(offer *mesos.Offer) {
			log.Infof("Launching Tasks: %v for offer %v", tasks, *offer.Id.Value)
			status, err := driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(OFFER_INTERVAL)})

			if status != mesos.Status_DRIVER_RUNNING {
				log.Fatal("Driver not running, while trying to launch tasks")
			}
			if err != nil {
				log.Panic("Failed to launch tasks: ", err)
			}
		}(offer)
	}
}
Example #14
0
func (s *DiegoScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	logOffers(offers)

	s.offersLock.Lock()
	defer s.offersLock.Unlock()

	if s.holdOffer {
		s.offers = append(s.offers, offers...)
	} else {
		offerIds := extractOfferIds(offers)
		driver.LaunchTasks(offerIds, nil, &mesos.Filters{RefuseSeconds: proto.Float64(30)})
	}
}
Example #15
0
func (sched *ScraperScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	logOffers(offers)

	for _, offer := range offers {
		if sched.tasksLaunched >= sched.totalTasks || len(sched.urls) == 0 {
			log.Infof("Declining offer %s", offer.Id.GetValue())
			driver.DeclineOffer(offer.Id, &mesos.Filters{})
			continue
		}
		remainingCpus := getOfferCpu(offer)
		remainingMems := getOfferMem(offer)

		var tasks []*mesos.TaskInfo
		for sched.cpuPerTask <= remainingCpus &&
			sched.memPerTask <= remainingMems &&
			sched.tasksLaunched < sched.totalTasks {

			log.Infof("Processing url %v of %v\n", sched.tasksLaunched, sched.totalTasks)
			log.Infof("Total Tasks: %d", sched.totalTasks)
			log.Infof("Tasks Launched: %d", sched.tasksLaunched)
			uri := sched.urls[sched.tasksLaunched]
			log.Infof("URI: %s", uri)

			sched.tasksLaunched++

			taskId := &mesos.TaskID{
				Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
			}

			task := &mesos.TaskInfo{
				Name:     proto.String("go-task-" + taskId.GetValue()),
				TaskId:   taskId,
				SlaveId:  offer.SlaveId,
				Executor: sched.executor,
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", sched.cpuPerTask),
					util.NewScalarResource("mem", sched.memPerTask),
				},
				Data: []byte(uri),
			}
			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			tasks = append(tasks, task)
			remainingCpus -= sched.cpuPerTask
			remainingMems -= sched.memPerTask
		}
		log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
	}
}
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	logOffers(offers)

	if sched.tasksLaunched >= sched.totalTasks {
		return
	}

	for _, offer := range offers {
		remainingCpus := getOfferCpu(offer)
		remainingMems := getOfferMem(offer)

		var tasks []*mesos.TaskInfo
		for sched.cpuPerTask <= remainingCpus &&
			sched.memPerTask <= remainingMems &&
			sched.tasksLaunched < sched.totalTasks {

			log.Infof("Processing image %v of %v\n", sched.tasksLaunched, sched.totalTasks)
			fileName := sched.images[sched.tasksLaunched]
			sched.tasksLaunched++

			taskId := &mesos.TaskID{
				Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
			}

			task := &mesos.TaskInfo{
				Name:     proto.String("go-task-" + taskId.GetValue()),
				TaskId:   taskId,
				SlaveId:  offer.SlaveId,
				Executor: sched.executor,
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", sched.cpuPerTask),
					util.NewScalarResource("mem", sched.memPerTask),
				},
				Data: []byte(fileName),
			}
			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			tasks = append(tasks, task)
			remainingCpus -= sched.cpuPerTask
			remainingMems -= sched.memPerTask
		}
		log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
	}
}
Example #17
0
func (s *rendlerScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	log.Printf("Received %d resource offers", len(offers))
	for _, offer := range offers {
		select {
		case <-s.shutdown:
			log.Println("Shutting down: declining offer on [", offer.Hostname, "]")
			driver.DeclineOffer(offer.Id, defaultFilter)
			if s.tasksRunning == 0 {
				close(s.done)
			}
			continue
		default:
		}

		tasks := []*mesos.TaskInfo{}
		tasksToLaunch := maxTasksForOffer(offer)
		for tasksToLaunch > 0 {
			if s.crawlQueue.Front() != nil {
				url := s.crawlQueue.Front().Value.(string)
				s.crawlQueue.Remove(s.crawlQueue.Front())
				task := s.newCrawlTask(url, offer)
				tasks = append(tasks, task)
				tasksToLaunch--
			}
			if s.renderQueue.Front() != nil {
				url := s.renderQueue.Front().Value.(string)
				s.renderQueue.Remove(s.renderQueue.Front())
				task := s.newRenderTask(url, offer)
				tasks = append(tasks, task)
				tasksToLaunch--
			}
			if s.crawlQueue.Front() == nil && s.renderQueue.Front() == nil {
				break
			}
		}

		if len(tasks) == 0 {
			driver.DeclineOffer(offer.Id, defaultFilter)
		} else {
			driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, defaultFilter)
		}
	}
}
Example #18
0
func (s *DiegoScheduler) scheduleMatched(driver sched.SchedulerDriver, matches map[string]*OfferMatch) auctiontypes.AuctionResults {
	results := auctiontypes.AuctionResults{}

	for slaveId, match := range matches {
		if slaveId != "" {
			offers := match.Offers

			taskInfos := []*mesos.TaskInfo{}
			for _, lrpAuction := range match.LrpAuctions {
				taskInfo := s.createLrpTaskInfo(util.NewSlaveID(slaveId), &lrpAuction)
				taskInfos = append(taskInfos, taskInfo)
				results.SuccessfulLRPs = append(results.SuccessfulLRPs, lrpAuction)
				log.Infof("+scheduled lrp, lrp: %v/%v mem: %v, offers: mem: %v",
					lrpAuction.ProcessGuid, lrpAuction.Index, lrpAuction.MemoryMB, getOffersMem(offers))
			}
			for _, taskAuction := range match.TaskAuctions {
				taskInfo := s.createTaskTaskInfo(util.NewSlaveID(slaveId), &taskAuction)
				taskInfos = append(taskInfos, taskInfo)
				results.SuccessfulTasks = append(results.SuccessfulTasks, taskAuction)
				log.Infof("+scheduled task, task: %v mem: %v, offers: mem: %v",
					taskAuction.TaskGuid, taskAuction.MemoryMB, getOffersMem(offers))
			}

			driver.LaunchTasks(extractOfferIds(offers), taskInfos, // offer getting declied if no tasks
				&mesos.Filters{RefuseSeconds: proto.Float64(30)})

		} else {
			for _, lrpAuction := range match.LrpAuctions {
				results.FailedLRPs = append(results.FailedLRPs, lrpAuction)
				log.Warningf("+schedule lrp failed, lrp: %v/%v mem: %v, offers: mem: %v",
					lrpAuction.GetProcessGuid(), lrpAuction.Index, lrpAuction.MemoryMB, getOffersMem(match.Offers))
			}
			for _, taskAuction := range match.TaskAuctions {
				results.FailedTasks = append(results.FailedTasks, taskAuction)
				log.Warningf("+schedule task failed, task: %v mem: %v, offers: mem: %v",
					taskAuction.TaskGuid, taskAuction.MemoryMB, getOffersMem(match.Offers))
			}
		}
	}

	return results
}
Example #19
0
// ResourceOffers handles the Resource Offers
func (s *eremeticScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	log.Tracef("Received %d resource offers", len(offers))
	var offer *mesos.Offer

loop:
	for len(offers) > 0 {
		select {
		case <-s.shutdown:
			log.Info("Shutting down: declining offers")
			break loop
		case tid := <-s.tasks:
			log.Debugf("Trying to find offer to launch %s with", tid)
			t, _ := database.ReadTask(tid)
			offer, offers = matchOffer(t, offers)

			if offer == nil {
				log.Warnf("Could not find a matching offer for %s", tid)
				TasksDelayed.Inc()
				go func() { s.tasks <- tid }()
				break loop
			}

			log.Debugf("Preparing to launch task %s with offer %s", tid, offer.Id.GetValue())
			t, task := s.newTask(t, offer)
			database.PutTask(&t)
			driver.LaunchTasks([]*mesos.OfferID{offer.Id}, []*mesos.TaskInfo{task}, defaultFilter)
			TasksLaunched.Inc()
			QueueSize.Dec()

			continue
		default:
			break loop
		}
	}

	log.Trace("No tasks to launch. Declining offers.")
	for _, offer := range offers {
		driver.DeclineOffer(offer.Id, defaultFilter)
	}
}
Example #20
0
// ResourceOffers handles the Resource Offers
func (s *eremeticScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	log.Tracef("Received %d resource offers", len(offers))
	for _, offer := range offers {
		select {
		case <-s.shutdown:
			log.Infof("Shutting down: declining offer on [%s]", offer.Hostname)
			driver.DeclineOffer(offer.Id, defaultFilter)
			continue
		case tid := <-s.tasks:
			log.Debugf("Preparing to launch task %s with offer %s", tid, offer.Id.GetValue())
			t, _ := database.ReadTask(tid)
			task := s.newTask(offer, &t)
			database.PutTask(&t)
			driver.LaunchTasks([]*mesos.OfferID{offer.Id}, []*mesos.TaskInfo{task}, defaultFilter)
			continue
		default:
		}

		log.Trace("No tasks to launch. Declining offer.")
		driver.DeclineOffer(offer.Id, defaultFilter)
	}
}
Example #21
0
// process incoming offers and try to schedule new tasks as they come in on the channel
func (sched *NoneScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	if sched.queue.Next() == nil {
		// no command to launch, we don't need to parse anything
		return
	}

	for _, offer := range offers {
		// match constraints
		if !sched.filter.FilterOffer(offer) {
			// skip offer if it does not match constraints
			continue
		}

		remainingCpus := SumScalarResources(sched.filter.FilterResources(offer, "cpus"))
		remainingMems := SumScalarResources(sched.filter.FilterResources(offer, "mem"))

		log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", remainingCpus, " mem=", remainingMems)

		// try to schedule as may tasks as possible for this single offer
		var tasks []*mesos.TaskInfo
		for sched.queue.GetCommand() != nil &&
			sched.queue.GetCommand().MatchesResources(remainingCpus, remainingMems) {

			c := sched.queue.GetCommand()
			c.SlaveId = offer.SlaveId.GetValue()
			c.FrameworkId = sched.frameworkId
			sched.handler.CommandLaunched(c)
			task := sched.prepareTaskInfo(offer, c)
			tasks = append(tasks, task)

			remainingCpus -= c.CpuReq
			remainingMems -= c.MemReq
			sched.queue.Next()
		}
		log.Infoln("Launching", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
	}
}
Example #22
0
func (self *MesosManager) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	log.Printf("ResourceOffers\n")
	computeCount := 0
	for _, offer := range offers {
		offer_cpus := 0
		for _, res := range offer.Resources {
			if *res.Name == "cpus" {
				computeCount += int(*res.Scalar.Value)
				offer_cpus += int(*res.Scalar.Value)
			}
		}

		tasks := make([]*mesos.TaskInfo, 0, computeCount)

		for offer_cpus_taken := 0; offer_cpus_taken < offer_cpus; {
			job := self.engine.GetJobToRun()
			if job != nil {
				log.Printf("Launch job: %s", job)
				mesos_taskinfo := self.BuildTaskInfo(job, offer)
				log.Printf("MesosTask: %s", mesos_taskinfo)
				tasks = append(tasks, mesos_taskinfo)
				offer_cpus_taken += 1
			} else {
				offer_cpus_taken = offer_cpus
			}
		}
		if len(tasks) > 0 {
			_, err := driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{})
			if err != nil {
				fmt.Print("Error: %s", err)
			}
		} else {
			driver.DeclineOffer(offer.Id, &mesos.Filters{})
		}
	}
	self.computeCount = computeCount
	log.Printf("CPUsOffered: %d", self.computeCount)
}
Example #23
0
// mesos.Scheduler interface method.
// Invoked when resources have been offered to this framework.
func (this *TransformScheduler) ResourceOffers(driver scheduler.SchedulerDriver, offers []*mesos.Offer) {
	fmt.Println("Received offers")

	if int(this.runningInstances) > this.config.Instances {
		toKill := int(this.runningInstances) - this.config.Instances
		for i := 0; i < toKill; i++ {
			driver.KillTask(this.tasks[i])
		}

		this.tasks = this.tasks[toKill:]
	}

	offersAndTasks := make(map[*mesos.Offer][]*mesos.TaskInfo)
	for _, offer := range offers {
		cpus := getScalarResources(offer, "cpus")
		mems := getScalarResources(offer, "mem")
		ports := getRangeResources(offer, "ports")

		remainingCpus := cpus
		remainingMems := mems

		var tasks []*mesos.TaskInfo
		for int(this.getRunningInstances()) < this.config.Instances && this.config.CpuPerTask <= remainingCpus && this.config.MemPerTask <= remainingMems && len(ports) > 0 {
			port := this.takePort(&ports)
			taskPort := &mesos.Value_Range{Begin: port, End: port}
			taskId := &mesos.TaskID{
				Value: proto.String(fmt.Sprintf("golang-%s-%d", *offer.Hostname, *port)),
			}

			task := &mesos.TaskInfo{
				Name:     proto.String(taskId.GetValue()),
				TaskId:   taskId,
				SlaveId:  offer.SlaveId,
				Executor: this.createExecutor(this.getRunningInstances(), *port),
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", float64(this.config.CpuPerTask)),
					util.NewScalarResource("mem", float64(this.config.MemPerTask)),
					util.NewRangesResource("ports", []*mesos.Value_Range{taskPort}),
				},
			}
			fmt.Printf("Prepared task: %s with offer %s for launch. Ports: %s\n", task.GetName(), offer.Id.GetValue(), taskPort)

			tasks = append(tasks, task)
			remainingCpus -= this.config.CpuPerTask
			remainingMems -= this.config.MemPerTask
			ports = ports[1:]

			this.tasks = append(this.tasks, taskId)
			this.incRunningInstances()
		}
		fmt.Printf("Launching %d tasks for offer %s\n", len(tasks), offer.Id.GetValue())
		offersAndTasks[offer] = tasks
	}

	unlaunchedTasks := this.config.Instances - int(this.getRunningInstances())
	if unlaunchedTasks > 0 {
		fmt.Printf("There are still %d tasks to be launched and no more resources are available.", unlaunchedTasks)
	}

	for _, offer := range offers {
		tasks := offersAndTasks[offer]
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
	}
}
Example #24
0
func (sched *SdcScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	if sched.tasksLaunched >= sched.totalTasks {
		log.Info("decline all of the offers since all of our tasks are already launched")
		log.Infoln("sched.totalTasks ", sched.totalTasks)
		log.Infoln("sched.tasksFinished ", sched.tasksFinished)
		log.Infoln("sched.tasksLaunched ", sched.tasksLaunched)

		// cmdQueueからCommand.Argumentsをpopする
		// 将来的に外部のキューから取得できるように置き換える
		if sched.totalTasks == 0 && sched.tasksFinished == 0 && sched.tasksLaunched == 0 && cmdQueue.Len() != 0 {
			execinfo := cmdQueue.Remove(cmdQueue.Front()).(*mesos.ExecutorInfo)
			log.Infoln("execinfo ", execinfo.Command.Arguments)

			sched.totalTasks = len(execinfo.Command.Arguments)
			sched.executor.Command.Arguments = execinfo.Command.Arguments
		}

		if sched.totalTasks == 0 && sched.tasksFinished == 0 && sched.tasksLaunched == 0 {
			ids := make([]*mesos.OfferID, len(offers))
			for i, offer := range offers {
				ids[i] = offer.Id
			}
			driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
			return
		}
	}

	log.Info("prepare pass args: ", sched.executor.Command.Arguments)
	cmds := sched.executor.Command.Arguments
	for _, v := range cmds {
		fmt.Println("v = ", v)
	}

	// [/bin/cat /var/tmp/1.txt /var/tmp/2.txt /var/tmp/3.txt | /bin/grep abe > /var/tmp/grep-result.txt]
	//
	// rebuild args
	// 1. /bin/cat /var/tmp/1.txt >> /var/tmp/intermediate.txt
	// 2. /bin/cat /var/tmp/2.txt >> /var/tmp/intermediate.txt
	// 3. /bin/cat /var/tmp/3.txt >> /var/tmp/intermediate.txt
	// 4. /bin/cat /var/tmp/intermediate.txt | /bin/grep abe > /var/tmp/grep-result.txt
	// 5. /bin/rm /var/tmp/intermediate.txt

	for _, offer := range offers {
		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)

		remainingCpus := cpus
		remainingMems := mems

		var tasks []*mesos.TaskInfo

		// $ cat 1.txt 2.txt. 3.txt | wc -lの場合
		// 先に、後ろのタスクを上げておく必要がある?
		//
		// コンセプト実装はシンプルに中間ファイル方式で行く
		// 遅いけど

		for sched.tasksLaunched < sched.totalTasks &&
			CPUS_PER_TASK <= remainingCpus &&
			MEM_PER_TASK <= remainingMems {

			sched.tasksLaunched++

			taskId := &mesos.TaskID{
				Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
			}

			// executionidの書き換え
			sched.executor.ExecutorId = util.NewExecutorID(taskId.GetValue())

			log.Infof("sched.tasksLaunched = %d\n", sched.tasksLaunched)
			log.Infof("sched.totalTasks = %d\n", sched.totalTasks)
			log.Infof("sched.executor.Command.Value = %s\n", sched.executor.Command.GetValue())
			log.Infof("sched.executor.GetExecutorId() = %s\n", sched.executor.GetExecutorId())

			// sched.executor.Command.Arguments で書き換えても保持されている
			// 値はポインタなので、LaunchTasksするときに、複数のタスクでまとめられているので、
			// 値は最後に上書きされた物になる
			// そこで、Argumentsがタスクごとにまとめられないように個別にオブジェクトを生成してタスクを
			// 起動する
			exec := &mesos.ExecutorInfo{
				ExecutorId: sched.executor.GetExecutorId(),
				Name:       proto.String(sched.executor.GetName()),
				Source:     proto.String(sched.executor.GetSource()),
				Command: &mesos.CommandInfo{
					Value: proto.String(sched.executor.Command.GetValue()),
					Uris:  sched.executor.Command.GetUris(),
				},
			}

			cmd := cmds[sched.tasksLaunched-1]
			log.Infof("cmd = %s\n", cmd)
			// Argumentsコマンドラインを使うと別Executorとみなされるので、色々面倒
			// 以下はやってはいけない例
			// exec.Command.Arguments = strings.Split(cmd, " ")

			task := &mesos.TaskInfo{
				Name:     proto.String("go-task-" + taskId.GetValue()),
				TaskId:   taskId,
				SlaveId:  offer.SlaveId,
				Executor: exec,
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", CPUS_PER_TASK),
					util.NewScalarResource("mem", MEM_PER_TASK),
				},
				// 実行したいコマンドラインはDataパラメータを使って渡せす
				Data: []byte(cmd),
			}
			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			tasks = append(tasks, task)
			remainingCpus -= CPUS_PER_TASK
			remainingMems -= MEM_PER_TASK
		}

		log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
	}
}
Example #25
0
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {

	if *slowLaunch {
		time.Sleep(3 * time.Second)
	}

	if (sched.tasksLaunched - sched.tasksErrored) >= sched.totalTasks {
		log.Info("decline all of the offers since all of our tasks are already launched")
		ids := make([]*mesos.OfferID, len(offers))
		for i, offer := range offers {
			ids[i] = offer.Id
		}
		driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(120)})
		return
	}
	for _, offer := range offers {
		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)

		remainingCpus := cpus
		remainingMems := mems

		// account for executor resources if there's not an executor already running on the slave
		if len(offer.ExecutorIds) == 0 {
			remainingCpus -= CPUS_PER_EXECUTOR
			remainingMems -= MEM_PER_EXECUTOR
		}

		var tasks []*mesos.TaskInfo
		for (sched.tasksLaunched-sched.tasksErrored) < sched.totalTasks &&
			CPUS_PER_TASK <= remainingCpus &&
			MEM_PER_TASK <= remainingMems {

			sched.tasksLaunched++

			taskId := &mesos.TaskID{
				Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
			}

			task := &mesos.TaskInfo{
				Name:     proto.String("go-task-" + taskId.GetValue()),
				TaskId:   taskId,
				SlaveId:  offer.SlaveId,
				Executor: sched.executor,
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", CPUS_PER_TASK),
					util.NewScalarResource("mem", MEM_PER_TASK),
				},
			}
			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			tasks = append(tasks, task)
			remainingCpus -= CPUS_PER_TASK
			remainingMems -= MEM_PER_TASK
		}
		log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
	}
}
func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {

	for _, offer := range offers {
		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)

		remainingCpus := cpus
		remainingMems := mems
		var tasks []*mesos.TaskInfo
		queueItems := getQueue(sched)
		if len(queueItems) == 0 {
			fmt.Println("Queue is empty!")
		}

		// start a task of each item in the queue. before starting, check
		// if it has not already been launched.
		for i, item := range queueItems {
			if val, ok := launchedTasks[item.Commit.Sha]; ok {
				fmt.Println("item exists: ", val)
				continue
			}

			fmt.Println("Adding item: ", item.Commit.Sha)
			launchedTasks[item.Commit.Sha] = item.Commit.Sha

			if CPUS_PER_TASK <= remainingCpus && MEM_PER_TASK <= remainingMems {

				sched.tasksLaunched += i

				taskId := &mesos.TaskID{
					Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
				}

				stringArray := []string{"-addr=" + *droneServerIP, "-token=1"}
				dataString := strings.Join(stringArray, " ")

				task := &mesos.TaskInfo{
					Name:     proto.String("go-task-" + taskId.GetValue()),
					TaskId:   taskId,
					SlaveId:  offer.SlaveId,
					Executor: sched.executor,
					Data:     []byte(dataString),
					Resources: []*mesos.Resource{
						util.NewScalarResource("cpus", CPUS_PER_TASK),
						util.NewScalarResource("mem", MEM_PER_TASK),
					},
				}

				log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

				tasks = append(tasks, task)
				remainingCpus -= CPUS_PER_TASK
				remainingMems -= MEM_PER_TASK
			}

		}

		//		for sched.tasksLaunched < sched.totalTasks &&
		//		CPUS_PER_TASK <= remainingCpus &&
		//		MEM_PER_TASK <= remainingMems {
		//
		//			sched.tasksLaunched++
		//
		//			taskId := &mesos.TaskID{
		//				Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
		//			}
		//
		//			stringArray := []string{*droneServerIP, "-token=1"}
		//			dataString := strings.Join(stringArray, " ")
		//
		//			task := &mesos.TaskInfo{
		//				Name:     proto.String("go-task-" + taskId.GetValue()),
		//				TaskId:   taskId,
		//				SlaveId:  offer.SlaveId,
		//				Executor: sched.executor,
		//				Data: []byte(dataString),
		//				Resources: []*mesos.Resource{
		//					util.NewScalarResource("cpus", CPUS_PER_TASK),
		//					util.NewScalarResource("mem", MEM_PER_TASK),
		//				},
		//			}
		//
		//			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())
		//
		//			tasks = append(tasks, task)
		//			remainingCpus -= CPUS_PER_TASK
		//			remainingMems -= MEM_PER_TASK
		//		}
		log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
	}
}
Example #27
0
// Schedule handles Mesos resource offers and based on the received offers launches PENDING Tasks
//
// It monitors PENDING Tasks Queue and tries to launch as many Tasks as possible for a single offer.
// If no PENDING Tasks are available within QueueTimeout interval it declines the offer and retries Queue read again
func (bw *BasicWorker) ScheduleTasks(driver scheduler.SchedulerDriver, offers []*mesos.Offer) {
ReadOffers:
	for _, offer := range offers {
		remainingCpus := taurus.ScalarResourceVal("cpus", offer.Resources)
		remainingMems := taurus.ScalarResourceVal("mem", offer.Resources)

		log.Println("Taurus Received Offer <", offer.Id.GetValue(), "> with cpus=", remainingCpus, " mem=", remainingMems)

		// Map to avoid launching duplicate tasks in the same batch slice
		launchTaskMap := make(map[string]bool)
		launchTasks := make([]*mesos.TaskInfo, 0, 0)
		var taskCpu, taskMem float64
		var retryCount int
	ReadTasks:
		for {
			task, err := bw.pending.ReadTask(QueueTimeout)
			if err != nil {
				retryCount += 1
				switch {
				case bw.pending.TimedOut(err):
					log.Printf("No %s tasks available", taurus.PENDING)
				case bw.pending.Closed(err):
					break ReadTasks
				default:
					log.Printf("Failed to read from %s queue: %s", taurus.PENDING, err)
				}
				if retryCount == QueueRetry {
					break ReadTasks
				}
				continue ReadTasks
			}
			if task != nil {
				taskId := task.Info.TaskId.GetValue()
				// Don't add the same task bwice into launchTasks slice
				if launchTaskMap[taskId] {
					log.Printf("Skipping already queued Task %s", taskId)
					continue ReadTasks
				}
				taskCpu = taurus.ScalarResourceVal("cpus", task.Info.Resources)
				taskMem = taurus.ScalarResourceVal("mem", task.Info.Resources)
				if remainingCpus >= taskCpu && remainingMems >= taskMem {
					task.Info.SlaveId = offer.SlaveId
					launchTasks = append(launchTasks, task.Info)
					launchTaskMap[taskId] = true
					remainingCpus -= taskCpu
					remainingMems -= taskMem
				} else {
					break ReadTasks
				}
			}
		}

		if len(launchTasks) > 0 {
			log.Printf("Launching %d tasks for offer %s", len(launchTasks), offer.Id.GetValue())
			launchStatus, err := driver.LaunchTasks(
				[]*mesos.OfferID{offer.Id},
				launchTasks,
				&mesos.Filters{RefuseSeconds: proto.Float64(1)})
			if err != nil {
				log.Printf("Mesos status: %#v Failed to launch Tasks %s: %s", launchStatus, launchTasks, err)
				continue ReadOffers
			}
		} else {
			log.Println("Declining offer ", offer.Id.GetValue())
			declineStatus, err := driver.DeclineOffer(
				offer.Id,
				&mesos.Filters{RefuseSeconds: proto.Float64(1)})
			if err != nil {
				log.Printf("Error declining offer for mesos status %#v: %s", declineStatus, err)
			}
		}
	}
}
Example #28
0
// TODO(tyler) split this long function up!
func (s *EtcdScheduler) launchOne(driver scheduler.SchedulerDriver) {
	// Always ensure we've pruned any dead / unmanaged nodes before
	// launching new ones, or we may overconfigure the ensemble such
	// that it can not make progress if the next launch fails.
	err := s.Prune()
	if err != nil {
		log.Errorf("Failed to remove stale cluster members: %s", err)
		return
	}

	if !s.shouldLaunch(driver) {
		log.Infoln("Skipping launch attempt for now.")
		return
	}

	// validOffer filters out offers that are no longer
	// desirable, even though they may have been when
	// they were enqueued.
	validOffer := func(offer *mesos.Offer) bool {
		runningCopy := s.RunningCopy()
		for _, etcdConfig := range runningCopy {
			if etcdConfig.SlaveID == offer.SlaveId.GetValue() {
				if s.singleInstancePerSlave {
					log.Info("Skipping offer: already running on this slave.")
					return false
				}
			}
		}
		return true
	}

	// Issue BlockingPop until we get back an offer we can use.
	var offer *mesos.Offer
	for {
		offer = s.offerCache.BlockingPop()
		if validOffer(offer) {
			break
		} else {
			s.decline(driver, offer)
		}
	}

	// Do this again because BlockingPop may have taken a long time.
	if !s.shouldLaunch(driver) {
		log.Infoln("Skipping launch attempt for now.")
		s.decline(driver, offer)
		return
	}

	// TODO(tyler) this is a broken hack
	resources := parseOffer(offer)
	lowest := *resources.ports[0].Begin
	rpcPort := lowest
	clientPort := lowest + 1
	httpPort := lowest + 2

	s.mut.Lock()
	var clusterType string
	if len(s.running) == 0 {
		clusterType = "new"
	} else {
		clusterType = "existing"
	}

	s.highestInstanceID++
	name := "etcd-" + strconv.FormatInt(s.highestInstanceID, 10)

	node := &config.Node{
		Name:       name,
		Host:       *offer.Hostname,
		RPCPort:    rpcPort,
		ClientPort: clientPort,
		ReseedPort: httpPort,
		Type:       clusterType,
		SlaveID:    offer.GetSlaveId().GetValue(),
	}
	running := []*config.Node{node}
	for _, r := range s.running {
		running = append(running, r)
	}
	serializedNodes, err := json.Marshal(running)
	log.Infof("Serialized running: %+v", string(serializedNodes))
	if err != nil {
		log.Errorf("Could not serialize running list: %v", err)
		// This Unlock is not deferred because the test implementation of LaunchTasks
		// calls this scheduler's StatusUpdate method, causing the test to deadlock.
		s.decline(driver, offer)
		s.mut.Unlock()
		return
	}

	configSummary := node.String()
	taskID := &mesos.TaskID{Value: &configSummary}
	executor := s.newExecutorInfo(node, s.executorUris)
	task := &mesos.TaskInfo{
		Data:     serializedNodes,
		Name:     proto.String("etcd-server"),
		TaskId:   taskID,
		SlaveId:  offer.SlaveId,
		Executor: executor,
		Resources: []*mesos.Resource{
			util.NewScalarResource("cpus", s.cpusPerTask),
			util.NewScalarResource("mem", s.memPerTask),
			util.NewScalarResource("disk", s.diskPerTask),
			util.NewRangesResource("ports", []*mesos.Value_Range{
				util.NewValueRange(uint64(rpcPort), uint64(httpPort)),
			}),
		},
		Discovery: &mesos.DiscoveryInfo{
			Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(),
			Name:       proto.String("etcd-server"),
			Ports: &mesos.Ports{
				Ports: []*mesos.Port{
					&mesos.Port{
						Number:   proto.Uint32(uint32(rpcPort)),
						Protocol: proto.String("tcp"),
					},
					// HACK: "client" is not a real SRV protocol.  This is so
					// that we can have etcd proxies use srv discovery on the
					// above tcp name.  Mesos-dns does not yet care about
					// names for DiscoveryInfo.  When it does, we should
					// create a name for clients to use.  We want to keep
					// the rpcPort accessible at _etcd-server._tcp.<fwname>.mesos
					&mesos.Port{
						Number:   proto.Uint32(uint32(clientPort)),
						Protocol: proto.String("client"),
					},
				},
			},
		},
	}

	log.Infof(
		"Prepared task: %s with offer %s for launch",
		task.GetName(),
		offer.Id.GetValue(),
	)
	log.Info("Launching etcd node.")

	tasks := []*mesos.TaskInfo{task}

	s.pending[node.Name] = struct{}{}

	// This Unlock is not deferred because the test implementation of LaunchTasks
	// calls this scheduler's StatusUpdate method, causing the test to deadlock.
	s.mut.Unlock()

	atomic.AddUint32(&s.Stats.LaunchedServers, 1)
	driver.LaunchTasks(
		[]*mesos.OfferID{offer.Id},
		tasks,
		&mesos.Filters{
			RefuseSeconds: proto.Float64(1),
		},
	)
}
Example #29
0
func (s *MinerScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	for i, offer := range offers {
		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		portsResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "ports"
		})
		var ports uint64
		for _, res := range portsResources {
			port_ranges := res.GetRanges().GetRange()
			for _, port_range := range port_ranges {
				ports += port_range.GetEnd() - port_range.GetBegin()
			}
		}

		// If a miner server is running, we start a new miner daemon.  Otherwise, we start a new miner server.
		tasks := make([]*mesos.TaskInfo, 0)
		if !s.minerServerRunning && mems >= MEM_PER_SERVER_TASK && cpus >= CPU_PER_SERVER_TASK && ports >= 2 {
			var taskId *mesos.TaskID
			var task *mesos.TaskInfo

			// we need two ports
			var p2pool_port uint64
			var worker_port uint64
			// A rather stupid algorithm for picking two ports
			// The difficulty here is that a range might only include one port,
			// in which case we will need to pick another port from another range.
			for _, res := range portsResources {
				r := res.GetRanges().GetRange()[0]
				begin := r.GetBegin()
				end := r.GetEnd()
				if p2pool_port == 0 {
					p2pool_port = begin
					if worker_port == 0 && (begin+1) <= end {
						worker_port = begin + 1
						break
					}
					continue
				}
				if worker_port == 0 {
					worker_port = begin
					break
				}
			}

			taskId = &mesos.TaskID{
				Value: proto.String("miner-server-" + strconv.Itoa(i)),
			}

			containerType := mesos.ContainerInfo_DOCKER
			task = &mesos.TaskInfo{
				Name:    proto.String("task-" + taskId.GetValue()),
				TaskId:  taskId,
				SlaveId: offer.SlaveId,
				Container: &mesos.ContainerInfo{
					Type: &containerType,
					Docker: &mesos.ContainerInfo_DockerInfo{
						Image: proto.String(MINER_SERVER_DOCKER_IMAGE),
					},
				},
				Command: &mesos.CommandInfo{
					Shell: proto.Bool(false),
					Arguments: []string{
						// these arguments will be passed to run_p2pool.py
						"--bitcoind-address", *bitcoindAddr,
						"--p2pool-port", strconv.Itoa(int(p2pool_port)),
						"-w", strconv.Itoa(int(worker_port)),
						s.rpc_user, s.rpc_pass,
					},
				},
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", CPU_PER_SERVER_TASK),
					util.NewScalarResource("mem", MEM_PER_SERVER_TASK),
				},
			}
			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			cpus -= CPU_PER_SERVER_TASK
			mems -= MEM_PER_SERVER_TASK

			// update state
			s.minerServerHostname = offer.GetHostname()
			s.minerServerRunning = true
			s.minerServerPort = int(worker_port)

			tasks = append(tasks, task)
		}

		if s.minerServerRunning && mems >= MEM_PER_DAEMON_TASK {
			var taskId *mesos.TaskID
			var task *mesos.TaskInfo

			taskId = &mesos.TaskID{
				Value: proto.String("miner-daemon-" + strconv.Itoa(i)),
			}

			containerType := mesos.ContainerInfo_DOCKER
			task = &mesos.TaskInfo{
				Name:    proto.String("task-" + taskId.GetValue()),
				TaskId:  taskId,
				SlaveId: offer.SlaveId,
				Container: &mesos.ContainerInfo{
					Type: &containerType,
					Docker: &mesos.ContainerInfo_DockerInfo{
						Image: proto.String(MINER_DAEMON_DOCKER_IMAGE),
					},
				},
				Command: &mesos.CommandInfo{
					Shell:     proto.Bool(false),
					Arguments: []string{"-o", s.minerServerHostname + ":" + strconv.Itoa(s.minerServerPort)},
				},
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", cpus),
					util.NewScalarResource("mem", MEM_PER_DAEMON_TASK),
				},
			}
			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			tasks = append(tasks, task)
		}

		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
	}
}
Example #30
0
func (s *visghsScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	if (s.nodesLaunched - s.nodesErrored) >= s.nodeTasks {
		log.Info("decline all of the offers since all of our tasks are already launched")
		ids := make([]*mesos.OfferID, len(offers))
		for i, offer := range offers {
			ids[i] = offer.Id
		}
		driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(120)})
		return
	}
	for _, offer := range offers {
		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		portResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "ports"
		})
		ports := []*mesos.Value_Range{}
		portsCount := uint64(0)
		for _, res := range portResources {
			for _, rs := range res.GetRanges().GetRange() {
				ports = append(ports, rs)
				portsCount += 1 + rs.GetEnd() - rs.GetBegin()
			}
		}

		log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems, " ports=", ports)

		remainingCpus := cpus
		remainingMems := mems
		remainingPorts := ports
		remainingPortsCount := portsCount

		// account for executor resources if there's not an executor already running on the slave
		if len(offer.ExecutorIds) == 0 {
			remainingCpus -= CPUS_PER_EXECUTOR
			remainingMems -= MEM_PER_EXECUTOR
		}

		var tasks []*mesos.TaskInfo
		for (s.nodesLaunched-s.nodesErrored) < s.nodeTasks &&
			CPUS_PER_TASK <= remainingCpus &&
			MEM_PER_TASK <= remainingMems &&
			PORTS_PER_TASK <= remainingPortsCount {
			log.Infoln("Ports <", remainingPortsCount, remainingPorts)

			s.nodesLaunched++

			taskId := &mesos.TaskID{
				Value: proto.String(strconv.Itoa(s.nodesLaunched)),
			}

			taskPorts := []*mesos.Value_Range{}
			leftOverPorts := []*mesos.Value_Range{}
			for t := 0; t < PORTS_PER_TASK; t++ {
				if len(remainingPorts) < 1 {
					// failed to allocate port, oh no!
				}
				ps := remainingPorts[0]
				//take the first port from the first
				pb := ps.GetBegin()
				pe := ps.GetEnd()
				//Create one range per port we need, it's easier this way
				tp := mesos.Value_Range{}
				p := pb
				tp.Begin = &p
				tp.End = &p
				taskPorts = append(taskPorts, &tp)

				pb++
				if pb <= pe {
					rpb := pb
					rpe := pe
					rtp := mesos.Value_Range{Begin: &rpb, End: &rpe}
					leftOverPorts = append(leftOverPorts, &rtp)
				}
				for _, ps := range remainingPorts[1:] {
					leftOverPorts = append(leftOverPorts, ps)
				}
			}

			radiaPort := (uint32)(taskPorts[0].GetBegin())
			task := &mesos.TaskInfo{
				Name:     proto.String("visghs-node-" + taskId.GetValue()),
				TaskId:   taskId,
				SlaveId:  offer.SlaveId,
				Executor: s.nexec,
				Discovery: &mesos.DiscoveryInfo{
					Name: proto.String("visghs"),
					//Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(),
					Visibility: mesos.DiscoveryInfo_FRAMEWORK.Enum(),
					Ports: &mesos.Ports{
						Ports: []*mesos.Port{
							{Protocol: proto.String("UDP"),
								Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(),
								Name:       proto.String("udpprobe"),
								Number:     proto.Uint32(radiaPort)},
							{Protocol: proto.String("TCP"),
								Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(),
								Name:       proto.String("radiarpc"),
								Number:     proto.Uint32(radiaPort)},
						},
					},
				},
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", CPUS_PER_TASK),
					util.NewScalarResource("mem", MEM_PER_TASK),
					util.NewRangesResource("ports", taskPorts),
				},
			}
			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			tasks = append(tasks, task)
			remainingCpus -= CPUS_PER_TASK
			remainingMems -= MEM_PER_TASK
			remainingPorts = leftOverPorts
			remainingPortsCount -= PORTS_PER_TASK
		}
		log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
	}
}