// StatusUpdate takes care of updating the status
func (s *eremeticScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) {
id := status.TaskId.GetValue()
log.Debugf("Received task status [%s] for task [%s]", status.State.String(), id)
task, err := database.ReadTask(id)
if err != nil {
log.Debugf("Error reading task from database: %s", err)
}
if task.ID == "" {
task = types.EremeticTask{
ID: id,
SlaveId: status.SlaveId.GetValue(),
}
}
if !task.IsRunning() && *status.State == mesos.TaskState_TASK_RUNNING {
TasksRunning.Inc()
}
if types.IsTerminal(status.State) {
TasksTerminated.With(prometheus.Labels{"status": status.State.String()}).Inc()
if task.WasRunning() {
TasksRunning.Dec()
}
}
task.UpdateStatus(types.Status{
Status: status.State.String(),
Time: time.Now().Unix(),
})
if *status.State == mesos.TaskState_TASK_FAILED && !task.WasRunning() {
if task.Retry >= maxRetries {
log.Warnf("giving up on %s after %d retry attempts", id, task.Retry)
} else {
log.Infof("task %s was never running. re-scheduling", id)
task.UpdateStatus(types.Status{
Status: mesos.TaskState_TASK_STAGING.String(),
Time: time.Now().Unix(),
})
task.Retry++
go func() {
QueueSize.Inc()
s.tasks <- id
}()
}
}
if types.IsTerminal(status.State) {
handler.NotifyCallback(&task)
}
database.PutTask(&task)
}
func (s *eremeticScheduler) ScheduleTask(request types.Request) (string, error) {
log.Debugf(
"Adding task running on %s to queue",
request.DockerImage)
request.Name = fmt.Sprintf("Eremetic task %d", nextID(s))
task, err := createEremeticTask(request)
if err != nil {
log.Error(err.Error())
return "", err
}
TasksCreated.Inc()
QueueSize.Inc()
database.PutTask(&task)
s.tasks <- task.ID
return task.ID, nil
}
// ResourceOffers handles the Resource Offers
func (s *eremeticScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
log.Tracef("Received %d resource offers", len(offers))
var offer *mesos.Offer
loop:
for len(offers) > 0 {
select {
case <-s.shutdown:
log.Info("Shutting down: declining offers")
break loop
case tid := <-s.tasks:
log.Debugf("Trying to find offer to launch %s with", tid)
t, _ := database.ReadTask(tid)
offer, offers = matchOffer(t, offers)
if offer == nil {
log.Warnf("Could not find a matching offer for %s", tid)
TasksDelayed.Inc()
go func() { s.tasks <- tid }()
break loop
}
log.Debugf("Preparing to launch task %s with offer %s", tid, offer.Id.GetValue())
t, task := s.newTask(t, offer)
database.PutTask(&t)
driver.LaunchTasks([]*mesos.OfferID{offer.Id}, []*mesos.TaskInfo{task}, defaultFilter)
TasksLaunched.Inc()
QueueSize.Dec()
continue
default:
break loop
}
}
log.Trace("No tasks to launch. Declining offers.")
for _, offer := range offers {
driver.DeclineOffer(offer.Id, defaultFilter)
}
}
func TestReconcile(t *testing.T) {
dir, _ := os.Getwd()
database.NewDB(fmt.Sprintf("%s/../db/test.db", dir))
database.Clean()
defer database.Close()
maxReconciliationDelay = 1
Convey("ReconcileTasks", t, func() {
Convey("Finishes when there are no tasks", func() {
driver := NewMockScheduler()
r := ReconcileTasks(driver)
select {
case <-r.done:
}
So(driver.AssertNotCalled(t, "ReconcileTasks"), ShouldBeTrue)
})
Convey("Sends reconcile request", func() {
driver := NewMockScheduler()
driver.On("ReconcileTasks").Run(func(mock.Arguments) {
t, err := database.ReadTask("1234")
if err != nil {
panic("mock error")
}
t.UpdateStatus(types.Status{
Status: mesos.TaskState_TASK_RUNNING.String(),
Time: time.Now().Unix() + 1,
})
database.PutTask(&t)
}).Once()
database.PutTask(&types.EremeticTask{
ID: "1234",
Status: []types.Status{
types.Status{
Status: mesos.TaskState_TASK_STAGING.String(),
Time: time.Now().Unix(),
},
},
})
r := ReconcileTasks(driver)
select {
case <-r.done:
}
So(driver.AssertCalled(t, "ReconcileTasks"), ShouldBeTrue)
})
Convey("Cancel reconciliation", func() {
driver := NewMockScheduler()
database.PutTask(&types.EremeticTask{
ID: "1234",
Status: []types.Status{
types.Status{
Status: mesos.TaskState_TASK_STAGING.String(),
Time: time.Now().Unix(),
},
},
})
r := ReconcileTasks(driver)
r.Cancel()
select {
case <-r.done:
}
So(driver.AssertNotCalled(t, "ReconcileTasks"), ShouldBeTrue)
})
})
}
func TestScheduler(t *testing.T) {
dir, _ := os.Getwd()
database.NewDB(fmt.Sprintf("%s/../db/test.db", dir))
database.Clean()
defer database.Close()
Convey("eremeticScheduler", t, func() {
s := eremeticScheduler{}
id := "eremetic-task.9999"
database.PutTask(&types.EremeticTask{ID: id})
Convey("newTask", func() {
task := types.EremeticTask{
ID: "eremetic-task.1234",
}
offer := mesos.Offer{
FrameworkId: &mesos.FrameworkID{
Value: proto.String("framework-id"),
},
SlaveId: &mesos.SlaveID{
Value: proto.String("slave-id"),
},
Hostname: proto.String("hostname"),
}
taskData, mesosTask := s.newTask(task, &offer)
So(mesosTask.GetTaskId().GetValue(), ShouldEqual, task.ID)
So(taskData.SlaveId, ShouldEqual, "slave-id")
})
Convey("createEremeticScheduler", func() {
s := createEremeticScheduler()
So(s.tasksCreated, ShouldEqual, 0)
})
Convey("API", func() {
Convey("Registered", func() {
driver := NewMockScheduler()
driver.On("ReconcileTasks").Return("ok").Once()
fID := mesos.FrameworkID{Value: proto.String("1234")}
mInfo := mesos.MasterInfo{}
s.Registered(driver, &fID, &mInfo)
So(driver.AssertCalled(t, "ReconcileTasks"), ShouldBeTrue)
})
Convey("Reregistered", func() {
driver := NewMockScheduler()
driver.On("ReconcileTasks").Return("ok").Once()
database.Clean()
s.Reregistered(driver, &mesos.MasterInfo{})
So(driver.AssertCalled(t, "ReconcileTasks"), ShouldBeTrue)
})
Convey("Disconnected", func() {
s.Disconnected(nil)
})
Convey("ResourceOffers", func() {
driver := NewMockScheduler()
var offers []*mesos.Offer
Convey("No offers", func() {
s.ResourceOffers(driver, offers)
So(driver.AssertNotCalled(t, "DeclineOffer"), ShouldBeTrue)
So(driver.AssertNotCalled(t, "LaunchTasks"), ShouldBeTrue)
})
Convey("No tasks", func() {
offers = append(offers, &mesos.Offer{Id: &mesos.OfferID{Value: proto.String("1234")}})
driver.On("DeclineOffer").Return("declined").Once()
s.ResourceOffers(driver, offers)
So(driver.AssertCalled(t, "DeclineOffer"), ShouldBeTrue)
So(driver.AssertNotCalled(t, "LaunchTasks"), ShouldBeTrue)
})
})
Convey("StatusUpdate", func() {
Convey("Running then failing", func() {
s.StatusUpdate(nil, &mesos.TaskStatus{
TaskId: &mesos.TaskID{
Value: proto.String(id),
},
State: mesos.TaskState_TASK_RUNNING.Enum(),
})
task, _ := database.ReadTask(id)
So(len(task.Status), ShouldEqual, 1)
So(task.Status[0].Status, ShouldEqual, mesos.TaskState_TASK_RUNNING.String())
s.StatusUpdate(nil, &mesos.TaskStatus{
TaskId: &mesos.TaskID{
Value: proto.String(id),
},
State: mesos.TaskState_TASK_FAILED.Enum(),
})
task, _ = database.ReadTask(id)
So(len(task.Status), ShouldEqual, 2)
So(task.Status[0].Status, ShouldEqual, mesos.TaskState_TASK_RUNNING.String())
So(task.Status[1].Status, ShouldEqual, mesos.TaskState_TASK_FAILED.String())
})
Convey("Failing immediatly", func() {
s.tasks = make(chan string, 100)
s.StatusUpdate(nil, &mesos.TaskStatus{
TaskId: &mesos.TaskID{
Value: proto.String(id),
},
State: mesos.TaskState_TASK_FAILED.Enum(),
})
task, _ := database.ReadTask(id)
So(len(task.Status), ShouldEqual, 2)
So(task.Status[0].Status, ShouldEqual, mesos.TaskState_TASK_FAILED.String())
So(task.Status[1].Status, ShouldEqual, mesos.TaskState_TASK_STAGING.String())
select {
case c := <-s.tasks:
So(c, ShouldEqual, id)
}
})
})
Convey("FrameworkMessage", func() {
driver := NewMockScheduler()
message := `{"message": "this is a message"}`
Convey("From Eremetic", func() {
source := "eremetic-executor"
executor := mesos.ExecutorID{
Value: proto.String(source),
}
s.FrameworkMessage(driver, &executor, &mesos.SlaveID{}, message)
})
Convey("From an unknown source", func() {
source := "other-source"
executor := mesos.ExecutorID{
Value: proto.String(source),
}
s.FrameworkMessage(driver, &executor, &mesos.SlaveID{}, message)
})
Convey("A bad json", func() {
source := "eremetic-executor"
executor := mesos.ExecutorID{
Value: proto.String(source),
}
s.FrameworkMessage(driver, &executor, &mesos.SlaveID{}, "not a json")
})
})
Convey("OfferRescinded", func() {
s.OfferRescinded(nil, &mesos.OfferID{})
})
Convey("SlaveLost", func() {
s.SlaveLost(nil, &mesos.SlaveID{})
})
Convey("ExecutorLost", func() {
s.ExecutorLost(nil, &mesos.ExecutorID{}, &mesos.SlaveID{}, 2)
})
Convey("Error", func() {
s.Error(nil, "Error")
})
})
})
Convey("ScheduleTask", t, func() {
Convey("Given a valid Request", func() {
scheduler := &eremeticScheduler{
tasks: make(chan string, 100),
}
request := types.Request{
TaskCPUs: 0.5,
TaskMem: 22.0,
DockerImage: "busybox",
Command: "echo hello",
}
Convey("It should put a task id on the channel", func() {
taskID, err := scheduler.ScheduleTask(request)
So(err, ShouldBeNil)
select {
case c := <-scheduler.tasks:
So(c, ShouldEqual, taskID)
task, _ := database.ReadTask(taskID)
So(task.TaskCPUs, ShouldEqual, request.TaskCPUs)
So(task.TaskMem, ShouldEqual, request.TaskMem)
So(task.Command, ShouldEqual, request.Command)
So(task.User, ShouldEqual, "root")
So(task.Environment, ShouldBeEmpty)
So(task.Image, ShouldEqual, request.DockerImage)
So(task.ID, ShouldStartWith, "eremetic-task.")
}
})
})
})
}
func TestHandling(t *testing.T) {
scheduler := &mockScheduler{}
status := []types.Status{
types.Status{
Status: mesos.TaskState_TASK_RUNNING.String(),
Time: time.Now().Unix(),
},
}
dir, _ := os.Getwd()
database.NewDB(fmt.Sprintf("%s/../db/test.db", dir))
database.Clean()
defer database.Close()
Convey("writeJSON", t, func() {
Convey("Should respond with a JSON and the appropriate status code", func() {
var wr = httptest.NewRecorder()
writeJSON(200, "foo", wr)
contentType := wr.HeaderMap["Content-Type"][0]
So(contentType, ShouldEqual, "application/json; charset=UTF-8")
So(wr.Code, ShouldEqual, http.StatusOK)
})
})
Convey("HandleError", t, func() {
wr := httptest.NewRecorder()
Convey("It should return an error status code", func() {
var err = mockError{
message: "Error",
}
handleError(err, wr, "A test error")
So(wr.Code, ShouldEqual, 422)
So(strings.TrimSpace(wr.Body.String()), ShouldEqual, "{\"error\":\"Error\",\"message\":\"A test error\"}")
})
})
Convey("GetTaskInfo", t, func() {
wr := httptest.NewRecorder()
r, _ := http.NewRequest("GET", "/task/eremetic-task.1234", nil)
m := mux.NewRouter()
m.HandleFunc("/task/{taskId}", GetTaskInfo(scheduler))
Convey("Not Found", func() {
id := "eremetic-task.5678"
task := types.EremeticTask{
TaskCPUs: 0.2,
TaskMem: 0.5,
Command: "test",
Image: "test",
Status: status,
ID: id,
}
database.PutTask(&task)
m.ServeHTTP(wr, r)
So(wr.Code, ShouldEqual, http.StatusNotFound)
})
Convey("Found", func() {
id := "eremetic-task.1234"
task := types.EremeticTask{
TaskCPUs: 0.2,
TaskMem: 0.5,
Command: "test",
Image: "test",
Status: status,
ID: id,
}
database.PutTask(&task)
m.ServeHTTP(wr, r)
So(wr.Code, ShouldEqual, http.StatusOK)
})
})
Convey("AddTask", t, func() {
wr := httptest.NewRecorder()
Convey("It should respond with a location header", func() {
data := []byte(`{"task_mem":22.0, "docker_image": "busybox", "command": "echo hello", "task_cpus":0.5, "tasks_to_launch": 1}`)
r, _ := http.NewRequest("POST", "/task", bytes.NewBuffer(data))
r.Host = "localhost"
handler := AddTask(scheduler)
handler(wr, r)
location := wr.HeaderMap["Location"][0]
So(location, ShouldStartWith, "http://localhost/task/eremetic-task.")
So(wr.Code, ShouldEqual, http.StatusAccepted)
})
})
}