Esempio n. 1
0
// StatusUpdate takes care of updating the status
func (s *eremeticScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) {
	id := status.TaskId.GetValue()

	log.Debugf("Received task status [%s] for task [%s]", status.State.String(), id)

	task, err := database.ReadTask(id)
	if err != nil {
		log.Debugf("Error reading task from database: %s", err)
	}

	if task.ID == "" {
		task = types.EremeticTask{
			ID:      id,
			SlaveId: status.SlaveId.GetValue(),
		}
	}

	if !task.IsRunning() && *status.State == mesos.TaskState_TASK_RUNNING {
		TasksRunning.Inc()
	}

	if types.IsTerminal(status.State) {
		TasksTerminated.With(prometheus.Labels{"status": status.State.String()}).Inc()
		if task.WasRunning() {
			TasksRunning.Dec()
		}
	}

	task.UpdateStatus(types.Status{
		Status: status.State.String(),
		Time:   time.Now().Unix(),
	})

	if *status.State == mesos.TaskState_TASK_FAILED && !task.WasRunning() {
		if task.Retry >= maxRetries {
			log.Warnf("giving up on %s after %d retry attempts", id, task.Retry)
		} else {
			log.Infof("task %s was never running. re-scheduling", id)
			task.UpdateStatus(types.Status{
				Status: mesos.TaskState_TASK_STAGING.String(),
				Time:   time.Now().Unix(),
			})
			task.Retry++
			go func() {
				QueueSize.Inc()
				s.tasks <- id
			}()
		}
	}

	if types.IsTerminal(status.State) {
		handler.NotifyCallback(&task)
	}

	database.PutTask(&task)
}
Esempio n. 2
0
func (s *eremeticScheduler) ScheduleTask(request types.Request) (string, error) {
	log.Debugf(
		"Adding task running on %s to queue",
		request.DockerImage)

	request.Name = fmt.Sprintf("Eremetic task %d", nextID(s))

	task, err := createEremeticTask(request)
	if err != nil {
		log.Error(err.Error())
		return "", err
	}

	TasksCreated.Inc()
	QueueSize.Inc()
	database.PutTask(&task)
	s.tasks <- task.ID
	return task.ID, nil
}
Esempio n. 3
0
// ResourceOffers handles the Resource Offers
func (s *eremeticScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	log.Tracef("Received %d resource offers", len(offers))
	var offer *mesos.Offer

loop:
	for len(offers) > 0 {
		select {
		case <-s.shutdown:
			log.Info("Shutting down: declining offers")
			break loop
		case tid := <-s.tasks:
			log.Debugf("Trying to find offer to launch %s with", tid)
			t, _ := database.ReadTask(tid)
			offer, offers = matchOffer(t, offers)

			if offer == nil {
				log.Warnf("Could not find a matching offer for %s", tid)
				TasksDelayed.Inc()
				go func() { s.tasks <- tid }()
				break loop
			}

			log.Debugf("Preparing to launch task %s with offer %s", tid, offer.Id.GetValue())
			t, task := s.newTask(t, offer)
			database.PutTask(&t)
			driver.LaunchTasks([]*mesos.OfferID{offer.Id}, []*mesos.TaskInfo{task}, defaultFilter)
			TasksLaunched.Inc()
			QueueSize.Dec()

			continue
		default:
			break loop
		}
	}

	log.Trace("No tasks to launch. Declining offers.")
	for _, offer := range offers {
		driver.DeclineOffer(offer.Id, defaultFilter)
	}
}
Esempio n. 4
0
func TestReconcile(t *testing.T) {
	dir, _ := os.Getwd()
	database.NewDB(fmt.Sprintf("%s/../db/test.db", dir))
	database.Clean()
	defer database.Close()

	maxReconciliationDelay = 1

	Convey("ReconcileTasks", t, func() {
		Convey("Finishes when there are no tasks", func() {
			driver := NewMockScheduler()
			r := ReconcileTasks(driver)

			select {
			case <-r.done:
			}

			So(driver.AssertNotCalled(t, "ReconcileTasks"), ShouldBeTrue)
		})

		Convey("Sends reconcile request", func() {
			driver := NewMockScheduler()
			driver.On("ReconcileTasks").Run(func(mock.Arguments) {
				t, err := database.ReadTask("1234")
				if err != nil {
					panic("mock error")
				}
				t.UpdateStatus(types.Status{
					Status: mesos.TaskState_TASK_RUNNING.String(),
					Time:   time.Now().Unix() + 1,
				})
				database.PutTask(&t)
			}).Once()

			database.PutTask(&types.EremeticTask{
				ID: "1234",
				Status: []types.Status{
					types.Status{
						Status: mesos.TaskState_TASK_STAGING.String(),
						Time:   time.Now().Unix(),
					},
				},
			})

			r := ReconcileTasks(driver)

			select {
			case <-r.done:
			}

			So(driver.AssertCalled(t, "ReconcileTasks"), ShouldBeTrue)
		})

		Convey("Cancel reconciliation", func() {
			driver := NewMockScheduler()

			database.PutTask(&types.EremeticTask{
				ID: "1234",
				Status: []types.Status{
					types.Status{
						Status: mesos.TaskState_TASK_STAGING.String(),
						Time:   time.Now().Unix(),
					},
				},
			})

			r := ReconcileTasks(driver)
			r.Cancel()

			select {
			case <-r.done:
			}

			So(driver.AssertNotCalled(t, "ReconcileTasks"), ShouldBeTrue)
		})
	})
}
Esempio n. 5
0
func TestScheduler(t *testing.T) {
	dir, _ := os.Getwd()
	database.NewDB(fmt.Sprintf("%s/../db/test.db", dir))
	database.Clean()
	defer database.Close()

	Convey("eremeticScheduler", t, func() {
		s := eremeticScheduler{}
		id := "eremetic-task.9999"
		database.PutTask(&types.EremeticTask{ID: id})

		Convey("newTask", func() {
			task := types.EremeticTask{
				ID: "eremetic-task.1234",
			}
			offer := mesos.Offer{
				FrameworkId: &mesos.FrameworkID{
					Value: proto.String("framework-id"),
				},
				SlaveId: &mesos.SlaveID{
					Value: proto.String("slave-id"),
				},
				Hostname: proto.String("hostname"),
			}
			taskData, mesosTask := s.newTask(task, &offer)

			So(mesosTask.GetTaskId().GetValue(), ShouldEqual, task.ID)
			So(taskData.SlaveId, ShouldEqual, "slave-id")
		})

		Convey("createEremeticScheduler", func() {
			s := createEremeticScheduler()
			So(s.tasksCreated, ShouldEqual, 0)
		})

		Convey("API", func() {
			Convey("Registered", func() {
				driver := NewMockScheduler()
				driver.On("ReconcileTasks").Return("ok").Once()
				fID := mesos.FrameworkID{Value: proto.String("1234")}
				mInfo := mesos.MasterInfo{}
				s.Registered(driver, &fID, &mInfo)
				So(driver.AssertCalled(t, "ReconcileTasks"), ShouldBeTrue)
			})

			Convey("Reregistered", func() {
				driver := NewMockScheduler()
				driver.On("ReconcileTasks").Return("ok").Once()
				database.Clean()
				s.Reregistered(driver, &mesos.MasterInfo{})
				So(driver.AssertCalled(t, "ReconcileTasks"), ShouldBeTrue)
			})

			Convey("Disconnected", func() {
				s.Disconnected(nil)
			})

			Convey("ResourceOffers", func() {
				driver := NewMockScheduler()
				var offers []*mesos.Offer

				Convey("No offers", func() {
					s.ResourceOffers(driver, offers)
					So(driver.AssertNotCalled(t, "DeclineOffer"), ShouldBeTrue)
					So(driver.AssertNotCalled(t, "LaunchTasks"), ShouldBeTrue)
				})

				Convey("No tasks", func() {
					offers = append(offers, &mesos.Offer{Id: &mesos.OfferID{Value: proto.String("1234")}})
					driver.On("DeclineOffer").Return("declined").Once()
					s.ResourceOffers(driver, offers)
					So(driver.AssertCalled(t, "DeclineOffer"), ShouldBeTrue)
					So(driver.AssertNotCalled(t, "LaunchTasks"), ShouldBeTrue)
				})
			})

			Convey("StatusUpdate", func() {
				Convey("Running then failing", func() {
					s.StatusUpdate(nil, &mesos.TaskStatus{
						TaskId: &mesos.TaskID{
							Value: proto.String(id),
						},
						State: mesos.TaskState_TASK_RUNNING.Enum(),
					})
					task, _ := database.ReadTask(id)
					So(len(task.Status), ShouldEqual, 1)
					So(task.Status[0].Status, ShouldEqual, mesos.TaskState_TASK_RUNNING.String())

					s.StatusUpdate(nil, &mesos.TaskStatus{
						TaskId: &mesos.TaskID{
							Value: proto.String(id),
						},
						State: mesos.TaskState_TASK_FAILED.Enum(),
					})
					task, _ = database.ReadTask(id)

					So(len(task.Status), ShouldEqual, 2)
					So(task.Status[0].Status, ShouldEqual, mesos.TaskState_TASK_RUNNING.String())
					So(task.Status[1].Status, ShouldEqual, mesos.TaskState_TASK_FAILED.String())
				})

				Convey("Failing immediatly", func() {
					s.tasks = make(chan string, 100)
					s.StatusUpdate(nil, &mesos.TaskStatus{
						TaskId: &mesos.TaskID{
							Value: proto.String(id),
						},
						State: mesos.TaskState_TASK_FAILED.Enum(),
					})
					task, _ := database.ReadTask(id)
					So(len(task.Status), ShouldEqual, 2)
					So(task.Status[0].Status, ShouldEqual, mesos.TaskState_TASK_FAILED.String())
					So(task.Status[1].Status, ShouldEqual, mesos.TaskState_TASK_STAGING.String())

					select {
					case c := <-s.tasks:
						So(c, ShouldEqual, id)
					}
				})
			})

			Convey("FrameworkMessage", func() {
				driver := NewMockScheduler()
				message := `{"message": "this is a message"}`
				Convey("From Eremetic", func() {
					source := "eremetic-executor"
					executor := mesos.ExecutorID{
						Value: proto.String(source),
					}
					s.FrameworkMessage(driver, &executor, &mesos.SlaveID{}, message)
				})

				Convey("From an unknown source", func() {
					source := "other-source"
					executor := mesos.ExecutorID{
						Value: proto.String(source),
					}
					s.FrameworkMessage(driver, &executor, &mesos.SlaveID{}, message)
				})

				Convey("A bad json", func() {
					source := "eremetic-executor"
					executor := mesos.ExecutorID{
						Value: proto.String(source),
					}
					s.FrameworkMessage(driver, &executor, &mesos.SlaveID{}, "not a json")
				})
			})

			Convey("OfferRescinded", func() {
				s.OfferRescinded(nil, &mesos.OfferID{})
			})

			Convey("SlaveLost", func() {
				s.SlaveLost(nil, &mesos.SlaveID{})
			})

			Convey("ExecutorLost", func() {
				s.ExecutorLost(nil, &mesos.ExecutorID{}, &mesos.SlaveID{}, 2)
			})

			Convey("Error", func() {
				s.Error(nil, "Error")
			})
		})
	})

	Convey("ScheduleTask", t, func() {
		Convey("Given a valid Request", func() {
			scheduler := &eremeticScheduler{
				tasks: make(chan string, 100),
			}

			request := types.Request{
				TaskCPUs:    0.5,
				TaskMem:     22.0,
				DockerImage: "busybox",
				Command:     "echo hello",
			}

			Convey("It should put a task id on the channel", func() {
				taskID, err := scheduler.ScheduleTask(request)

				So(err, ShouldBeNil)

				select {
				case c := <-scheduler.tasks:
					So(c, ShouldEqual, taskID)
					task, _ := database.ReadTask(taskID)
					So(task.TaskCPUs, ShouldEqual, request.TaskCPUs)
					So(task.TaskMem, ShouldEqual, request.TaskMem)
					So(task.Command, ShouldEqual, request.Command)
					So(task.User, ShouldEqual, "root")
					So(task.Environment, ShouldBeEmpty)
					So(task.Image, ShouldEqual, request.DockerImage)
					So(task.ID, ShouldStartWith, "eremetic-task.")
				}
			})
		})
	})
}
Esempio n. 6
0
func TestHandling(t *testing.T) {
	scheduler := &mockScheduler{}
	status := []types.Status{
		types.Status{
			Status: mesos.TaskState_TASK_RUNNING.String(),
			Time:   time.Now().Unix(),
		},
	}

	dir, _ := os.Getwd()
	database.NewDB(fmt.Sprintf("%s/../db/test.db", dir))
	database.Clean()
	defer database.Close()

	Convey("writeJSON", t, func() {
		Convey("Should respond with a JSON and the appropriate status code", func() {
			var wr = httptest.NewRecorder()

			writeJSON(200, "foo", wr)
			contentType := wr.HeaderMap["Content-Type"][0]
			So(contentType, ShouldEqual, "application/json; charset=UTF-8")
			So(wr.Code, ShouldEqual, http.StatusOK)
		})
	})

	Convey("HandleError", t, func() {
		wr := httptest.NewRecorder()

		Convey("It should return an error status code", func() {
			var err = mockError{
				message: "Error",
			}

			handleError(err, wr, "A test error")

			So(wr.Code, ShouldEqual, 422)
			So(strings.TrimSpace(wr.Body.String()), ShouldEqual, "{\"error\":\"Error\",\"message\":\"A test error\"}")
		})
	})

	Convey("GetTaskInfo", t, func() {
		wr := httptest.NewRecorder()
		r, _ := http.NewRequest("GET", "/task/eremetic-task.1234", nil)
		m := mux.NewRouter()
		m.HandleFunc("/task/{taskId}", GetTaskInfo(scheduler))

		Convey("Not Found", func() {
			id := "eremetic-task.5678"
			task := types.EremeticTask{
				TaskCPUs: 0.2,
				TaskMem:  0.5,
				Command:  "test",
				Image:    "test",
				Status:   status,
				ID:       id,
			}
			database.PutTask(&task)
			m.ServeHTTP(wr, r)

			So(wr.Code, ShouldEqual, http.StatusNotFound)
		})

		Convey("Found", func() {
			id := "eremetic-task.1234"
			task := types.EremeticTask{
				TaskCPUs: 0.2,
				TaskMem:  0.5,
				Command:  "test",
				Image:    "test",
				Status:   status,
				ID:       id,
			}
			database.PutTask(&task)
			m.ServeHTTP(wr, r)

			So(wr.Code, ShouldEqual, http.StatusOK)
		})
	})

	Convey("AddTask", t, func() {
		wr := httptest.NewRecorder()

		Convey("It should respond with a location header", func() {
			data := []byte(`{"task_mem":22.0, "docker_image": "busybox", "command": "echo hello", "task_cpus":0.5, "tasks_to_launch": 1}`)
			r, _ := http.NewRequest("POST", "/task", bytes.NewBuffer(data))
			r.Host = "localhost"

			handler := AddTask(scheduler)
			handler(wr, r)

			location := wr.HeaderMap["Location"][0]
			So(location, ShouldStartWith, "http://localhost/task/eremetic-task.")
			So(wr.Code, ShouldEqual, http.StatusAccepted)
		})
	})
}