예제 #1
0
func TestRoutes(t *testing.T) {
	routes := routes(Handler{}, &config.Config{})

	db := eremetic.NewDefaultTaskDB()

	Convey("Create", t, func() {
		Convey("Should build the expected routes", func() {
			m := NewRouter(nil, &config.Config{}, db)
			for _, route := range routes {
				So(m.GetRoute(route.Name), ShouldNotBeNil)
			}
		})
	})

	Convey("Expected number of routes", t, func() {
		ExpectedNumberOfRoutes := 9 // Magic numbers FTW

		So(len(routes), ShouldEqual, ExpectedNumberOfRoutes)
	})
}
예제 #2
0
func TestReconcile(t *testing.T) {
	db := eremetic.NewDefaultTaskDB()

	maxReconciliationDelay = 1

	Convey("ReconcileTasks", t, func() {
		Convey("Finishes when there are no tasks", func() {
			driver := mock.NewMesosScheduler()
			r := reconcileTasks(driver, db)

			select {
			case <-r.done:
			}

			So(driver.ReconcileTasksFnInvoked, ShouldBeFalse)
		})

		Convey("Sends reconcile request", func() {
			driver := mock.NewMesosScheduler()
			driver.ReconcileTasksFn = func(ts []*mesosproto.TaskStatus) (mesosproto.Status, error) {
				t, err := db.ReadTask("1234")
				if err != nil {
					return mesosproto.Status_DRIVER_RUNNING, err
				}
				t.UpdateStatus(eremetic.Status{
					Status: eremetic.TaskRunning,
					Time:   time.Now().Unix() + 1,
				})
				db.PutTask(&t)

				return mesosproto.Status_DRIVER_RUNNING, nil
			}

			db.PutTask(&eremetic.Task{
				ID: "1234",
				Status: []eremetic.Status{
					eremetic.Status{
						Status: eremetic.TaskStaging,
						Time:   time.Now().Unix(),
					},
				},
			})

			r := reconcileTasks(driver, db)

			select {
			case <-r.done:
			}

			So(driver.ReconcileTasksFnInvoked, ShouldBeTrue)
		})

		Convey("Cancel reconciliation", func() {
			driver := mock.NewMesosScheduler()

			db.PutTask(&eremetic.Task{
				ID: "1234",
				Status: []eremetic.Status{
					eremetic.Status{
						Status: eremetic.TaskStaging,
						Time:   time.Now().Unix(),
					},
				},
			})

			r := reconcileTasks(driver, db)
			r.Cancel()

			select {
			case <-r.done:
			}

			So(driver.ReconcileTasksFnInvoked, ShouldBeFalse)
		})
	})
}
예제 #3
0
func TestScheduler(t *testing.T) {
	logrus.SetOutput(ioutil.Discard)

	db := eremetic.NewDefaultTaskDB()

	Convey("Scheduling", t, func() {
		Convey("Given a scheduler with one task", func() {
			s := &Scheduler{
				tasks:    make(chan string, 1),
				database: db,
			}

			id := "eremetic-task.9999"

			db.PutTask(&eremetic.Task{ID: id})

			Convey("When creating a new scheduler", func() {
				queueSize := 200
				s = NewScheduler(queueSize, db)

				Convey("The settings should have default values", func() {
					So(s.tasksCreated, ShouldEqual, 0)
					So(cap(s.tasks), ShouldEqual, queueSize)
				})
			})

			Convey("When the scheduler is registered", func() {
				driver := mock.NewMesosScheduler()
				driver.ReconcileTasksFn = func(ts []*mesosproto.TaskStatus) (mesosproto.Status, error) {
					return mesosproto.Status_DRIVER_RUNNING, nil
				}

				fid := mesosproto.FrameworkID{Value: proto.String("1234")}
				info := mesosproto.MasterInfo{}

				s.Registered(driver, &fid, &info)

				Convey("The tasks should be reconciled", func() {
					So(driver.ReconcileTasksFnInvoked, ShouldBeTrue)
				})
			})

			Convey("When the scheduler is reregistered", func() {
				driver := mock.NewMesosScheduler()
				driver.ReconcileTasksFn = func(ts []*mesosproto.TaskStatus) (mesosproto.Status, error) {
					return mesosproto.Status_DRIVER_RUNNING, nil
				}
				db.Clean()

				s.Reregistered(driver, &mesosproto.MasterInfo{})

				Convey("The tasks should be reconciled", func() {
					So(driver.ReconcileTasksFnInvoked, ShouldBeTrue)
				})
			})

			Convey("When the scheduler is disconnected", func() {
				s.Disconnected(nil)
			})

			Convey("When an offer rescinded", func() {
				s.OfferRescinded(nil, &mesosproto.OfferID{})
			})

			Convey("When a slave was lost", func() {
				s.SlaveLost(nil, &mesosproto.SlaveID{})
			})

			Convey("When an executor was lost", func() {
				s.ExecutorLost(nil, &mesosproto.ExecutorID{}, &mesosproto.SlaveID{}, 2)
			})

			Convey("When there was an error", func() {
				s.Error(nil, "Error")
			})
		})
	})
	Convey("ResourceOffers", t, func() {
		Convey("Given a scheduler with one task", func() {
			s := &Scheduler{
				tasks:    make(chan string, 1),
				database: db,
			}

			id := "eremetic-task.9999"

			db.PutTask(&eremetic.Task{ID: id})

			driver := mock.NewMesosScheduler()

			Convey("When there are no offers", func() {
				offers := []*mesosproto.Offer{}
				s.ResourceOffers(driver, offers)

				Convey("No offers should be declined", func() {
					So(driver.DeclineOfferFnInvoked, ShouldBeFalse)
				})
				Convey("No tasks should be launched", func() {
					So(driver.LaunchTasksFnInvoked, ShouldBeFalse)
				})
			})
			Convey("When there are no tasks", func() {
				offers := []*mesosproto.Offer{
					offer("1234", 1.0, 128),
				}
				driver.DeclineOfferFn = func(_ *mesosproto.OfferID, _ *mesosproto.Filters) (mesosproto.Status, error) {
					return mesosproto.Status_DRIVER_RUNNING, nil
				}

				s.ResourceOffers(driver, offers)

				Convey("The offer should be declined", func() {
					So(driver.DeclineOfferFnInvoked, ShouldBeTrue)
				})
				Convey("No tasks should be launched", func() {
					So(driver.LaunchTasksFnInvoked, ShouldBeFalse)
				})
			})
			Convey("When a task is able to launch", func() {
				offers := []*mesosproto.Offer{
					offer("1234", 1.0, 128),
				}
				driver.LaunchTasksFn = func(_ []*mesosproto.OfferID, _ []*mesosproto.TaskInfo, _ *mesosproto.Filters) (mesosproto.Status, error) {
					return mesosproto.Status_DRIVER_RUNNING, nil
				}

				taskID, err := s.ScheduleTask(eremetic.Request{
					TaskCPUs:    0.5,
					TaskMem:     22.0,
					DockerImage: "busybox",
					Command:     "echo hello",
				})
				So(err, ShouldBeNil)

				s.ResourceOffers(driver, offers)

				task, err := db.ReadTask(taskID)
				So(err, ShouldBeNil)

				Convey("The task should contain the status history", func() {
					So(task.Status, ShouldHaveLength, 2)
					So(task.Status[0].Status, ShouldEqual, eremetic.TaskQueued)
					So(task.Status[1].Status, ShouldEqual, eremetic.TaskStaging)
				})
				Convey("The offer should not be declined", func() {
					So(driver.DeclineOfferFnInvoked, ShouldBeFalse)
				})
				Convey("The tasks should be launched", func() {
					So(driver.LaunchTasksFnInvoked, ShouldBeTrue)
				})
			})

			Convey("When a task unable to launch", func() {
				offers := []*mesosproto.Offer{
					offer("1234", 1.0, 128),
				}
				driver.DeclineOfferFn = func(_ *mesosproto.OfferID, _ *mesosproto.Filters) (mesosproto.Status, error) {
					return mesosproto.Status_DRIVER_RUNNING, nil
				}

				_, err := s.ScheduleTask(eremetic.Request{
					TaskCPUs:    1.5,
					TaskMem:     22.0,
					DockerImage: "busybox",
					Command:     "echo hello",
				})
				So(err, ShouldBeNil)

				s.ResourceOffers(driver, offers)

				Convey("The offer should be declined", func() {
					So(driver.DeclineOfferFnInvoked, ShouldBeTrue)
				})
				Convey("The tasks should not be launched", func() {
					So(driver.LaunchTasksFnInvoked, ShouldBeFalse)
				})
			})

			Convey("When a task is marked for termination", func() {
				offers := []*mesosproto.Offer{offer("1234", 1.0, 128)}
				driver.DeclineOfferFn = func(_ *mesosproto.OfferID, _ *mesosproto.Filters) (mesosproto.Status, error) {
					return mesosproto.Status_DRIVER_RUNNING, nil
				}

				id, _ := s.ScheduleTask(eremetic.Request{
					TaskCPUs:    1.5,
					TaskMem:     22.0,
					DockerImage: "busybox",
					Command:     "echo hello",
				})
				task, _ := db.ReadTask(id)
				task.UpdateStatus(eremetic.Status{
					Time:   time.Now().Unix(),
					Status: eremetic.TaskTerminating,
				})
				db.PutTask(&task)

				s.ResourceOffers(driver, offers)

				task, _ = db.ReadTask(id)
				So(task.CurrentStatus(), ShouldEqual, eremetic.TaskKilled)
				So(driver.LaunchTasksFnInvoked, ShouldBeFalse)
				So(driver.DeclineOfferFnInvoked, ShouldBeTrue)
			})
		})
	})
	Convey("StatusUpdate", t, func() {
		Convey("Given a scheduler with one task", func() {
			s := &Scheduler{
				tasks:    make(chan string, 1),
				database: db,
			}

			id := "eremetic-task.9999"

			db.PutTask(&eremetic.Task{ID: id})

			Convey("When a running task fails", func() {
				s.StatusUpdate(nil, &mesosproto.TaskStatus{
					TaskId: &mesosproto.TaskID{
						Value: proto.String(id),
					},
					State: mesosproto.TaskState_TASK_RUNNING.Enum(),
				})

				task, err := db.ReadTask(id)
				So(err, ShouldBeNil)

				So(len(task.Status), ShouldEqual, 1)
				So(task.Status[0].Status, ShouldEqual, eremetic.TaskRunning)

				s.StatusUpdate(nil, &mesosproto.TaskStatus{
					TaskId: &mesosproto.TaskID{
						Value: proto.String(id),
					},
					State: mesosproto.TaskState_TASK_FAILED.Enum(),
				})

				task, err = db.ReadTask(id)
				So(err, ShouldBeNil)

				Convey("The task status history should contain the failed status", func() {
					So(len(task.Status), ShouldEqual, 2)
					So(task.Status[0].Status, ShouldEqual, eremetic.TaskRunning)
					So(task.Status[1].Status, ShouldEqual, eremetic.TaskFailed)
				})
			})

			Convey("When a task fails immediately", func() {
				s.tasks = make(chan string, 100)

				s.StatusUpdate(nil, &mesosproto.TaskStatus{
					TaskId: &mesosproto.TaskID{
						Value: proto.String(id),
					},
					State: mesosproto.TaskState_TASK_FAILED.Enum(),
				})

				task, err := db.ReadTask(id)
				So(err, ShouldBeNil)

				Convey("The task should have a queued status", func() {
					So(len(task.Status), ShouldEqual, 2)
					So(task.Status[0].Status, ShouldEqual, eremetic.TaskFailed)
					So(task.Status[1].Status, ShouldEqual, eremetic.TaskQueued)
				})

				Convey("The task should be published on channel", func() {
					c := <-s.tasks

					So(c, ShouldEqual, id)
				})
			})

			Convey("When a task finishes with a callback specified", func() {
				cb, ts := callbackReceiver()
				defer ts.Close()

				id := "eremetic-task.1000"

				db.PutTask(&eremetic.Task{
					ID:          id,
					CallbackURI: ts.URL,
				})

				s.StatusUpdate(nil, &mesosproto.TaskStatus{
					TaskId: &mesosproto.TaskID{
						Value: proto.String(id),
					},
					State: mesosproto.TaskState_TASK_FINISHED.Enum(),
				})

				Convey("The callback data should be available", func() {
					c := <-cb

					So(c.TaskID, ShouldEqual, id)
					So(c.Status, ShouldEqual, "TASK_FINISHED")
				})
			})

			Convey("When a task fails with a callback specified", func() {
				cb, ts := callbackReceiver()
				defer ts.Close()

				id := "eremetic-task.1001"

				db.PutTask(&eremetic.Task{
					ID:          id,
					CallbackURI: ts.URL,
				})

				s.StatusUpdate(nil, &mesosproto.TaskStatus{
					TaskId: &mesosproto.TaskID{
						Value: proto.String(id),
					},
					State: mesosproto.TaskState_TASK_RUNNING.Enum(),
				})

				s.StatusUpdate(nil, &mesosproto.TaskStatus{
					TaskId: &mesosproto.TaskID{
						Value: proto.String(id),
					},
					State: mesosproto.TaskState_TASK_FAILED.Enum(),
				})

				Convey("The callback data should be available", func() {
					c := <-cb

					So(c.TaskID, ShouldEqual, id)
					So(c.Status, ShouldEqual, "TASK_FAILED")
				})
			})

			Convey("When a task retries with a callback specified", func() {
				cb, ts := callbackReceiver()
				defer ts.Close()

				id := "eremetic-task.1002"

				db.PutTask(&eremetic.Task{
					ID:          id,
					CallbackURI: ts.URL,
				})

				s.StatusUpdate(nil, &mesosproto.TaskStatus{
					TaskId: &mesosproto.TaskID{
						Value: proto.String(id),
					},
					State: mesosproto.TaskState_TASK_FAILED.Enum(),
				})

				s.StatusUpdate(nil, &mesosproto.TaskStatus{
					TaskId: &mesosproto.TaskID{
						Value: proto.String(id),
					},
					State: mesosproto.TaskState_TASK_RUNNING.Enum(),
				})

				s.StatusUpdate(nil, &mesosproto.TaskStatus{
					TaskId: &mesosproto.TaskID{
						Value: proto.String(id),
					},
					State: mesosproto.TaskState_TASK_FINISHED.Enum(),
				})

				Convey("The callback data should be available", func() {
					c := <-cb

					So(c.TaskID, ShouldEqual, id)
					So(c.Status, ShouldEqual, "TASK_FINISHED")
				})
			})

			Convey("When the sandbox is updated", func() {
				id := "eremetic-task.1003"

				s.StatusUpdate(nil, &mesosproto.TaskStatus{
					TaskId: &mesosproto.TaskID{
						Value: proto.String(id),
					},
					Data:  []byte(`[{"Mounts":[{"Source":"/tmp/mesos/slaves/<agent_id>/frameworks/<framework_id>/executors/<task_id>/runs/<container_id>","Destination":"/mnt/mesos/sandbox","Mode":"","RW":true}]}]`),
					State: mesosproto.TaskState_TASK_RUNNING.Enum(),
				})

				task, err := db.ReadTask(id)
				So(err, ShouldBeNil)

				Convey("There should be a path to the sandbox", func() {
					So(task.SandboxPath, ShouldNotBeEmpty)
				})
			})
		})
	})
	Convey("FrameworkMessage", t, func() {
		Convey("Given a scheduler with one task", func() {
			s := &Scheduler{
				tasks:    make(chan string, 1),
				database: db,
			}

			id := "eremetic-task.9999"
			db.PutTask(&eremetic.Task{ID: id})

			driver := mock.NewMesosScheduler()
			message := `{"message": "this is a message"}`

			Convey("When receiving a framework message from Eremetic", func() {
				source := "eremetic-executor"
				executor := mesosproto.ExecutorID{
					Value: proto.String(source),
				}
				s.FrameworkMessage(driver, &executor, &mesosproto.SlaveID{}, message)
			})

			Convey("When receiving a framework message from an unknown source", func() {
				source := "other-source"
				executor := mesosproto.ExecutorID{
					Value: proto.String(source),
				}
				s.FrameworkMessage(driver, &executor, &mesosproto.SlaveID{}, message)
			})

			Convey("When the framework message format is invalid", func() {
				source := "eremetic-executor"
				executor := mesosproto.ExecutorID{
					Value: proto.String(source),
				}
				s.FrameworkMessage(driver, &executor, &mesosproto.SlaveID{}, "not a json")
			})
		})
	})
	Convey("ScheduleTask", t, func() {
		Convey("Given a scheduler with no scheduled tasks", func() {
			scheduler := &Scheduler{
				tasks:    make(chan string, 1),
				database: db,
			}

			Convey("When scheduling a task", func() {
				request := eremetic.Request{
					TaskCPUs:    0.5,
					TaskMem:     22.0,
					DockerImage: "busybox",
					Command:     "echo hello",
				}

				taskID, err := scheduler.ScheduleTask(request)
				So(err, ShouldBeNil)

				Convey("It should put a task id on the channel", func() {
					c := <-scheduler.tasks
					So(c, ShouldEqual, taskID)
				})

				Convey("The task should be present in the database", func() {
					task, err := db.ReadTask(taskID)
					So(err, ShouldBeNil)

					So(task.TaskCPUs, ShouldEqual, request.TaskCPUs)
					So(task.TaskMem, ShouldEqual, request.TaskMem)
					So(task.Command, ShouldEqual, request.Command)
					So(task.User, ShouldEqual, "root")
					So(task.Environment, ShouldBeEmpty)
					So(task.Image, ShouldEqual, request.DockerImage)
					So(task.ID, ShouldStartWith, "eremetic-task.")
				})
			})

			Convey("When scheduling a task and the queue is full", func() {
				scheduler.tasks <- "dummy"

				request := eremetic.Request{
					TaskCPUs:    0.5,
					TaskMem:     22.0,
					DockerImage: "busybox",
					Command:     "echo hello",
				}

				_, err := scheduler.ScheduleTask(request)

				Convey("It should return an error", func() {
					So(err, ShouldNotBeNil)
				})
			})
		})
	})
	Convey("nextID", t, func() {
		Convey("Given a scheduler with no scheduled tasks", func() {
			scheduler := &Scheduler{
				tasks:    make(chan string, 100),
				database: db,
			}

			Convey("When generating a random string", func() {
				str := nextID(scheduler)

				Convey("The string should not be empty", func() {
					So(str, ShouldNotBeEmpty)
				})
			})
		})
	})

	Convey("KillTask", t, func() {
		driver := mock.NewMesosScheduler()
		id := "eremetic-task.9999"

		scheduler := &Scheduler{
			tasks:    make(chan string, 1),
			database: db,
			driver:   driver,
		}

		Convey("Given a running task", func() {
			db.PutTask(&eremetic.Task{
				ID: id,
				Status: []eremetic.Status{
					eremetic.Status{
						Time:   123456,
						Status: eremetic.TaskRunning,
					},
				},
			})
			driver.KillTaskFn = func(_ *mesosproto.TaskID) (mesosproto.Status, error) {
				return mesosproto.Status_DRIVER_RUNNING, nil
			}

			err := scheduler.Kill(id)
			So(err, ShouldBeNil)
			So(driver.KillTaskFnInvoked, ShouldBeTrue)

			task, _ := db.ReadTask(id)
			So(task.CurrentStatus(), ShouldEqual, eremetic.TaskTerminating)
		})

		Convey("Given a queued task", func() {
			driver.KillTaskFn = func(_ *mesosproto.TaskID) (mesosproto.Status, error) {
				return mesosproto.Status_DRIVER_RUNNING, nil
			}
			db.PutTask(&eremetic.Task{
				ID: id,
				Status: []eremetic.Status{
					eremetic.Status{
						Time:   123456,
						Status: eremetic.TaskQueued,
					},
				},
			})
			err := scheduler.Kill(id)
			So(err, ShouldBeNil)
			So(driver.KillTaskFnInvoked, ShouldBeFalse)
		})

		Convey("Given that something goes wrong", func() {
			driver.KillTaskFn = func(_ *mesosproto.TaskID) (mesosproto.Status, error) {
				return mesosproto.Status_DRIVER_RUNNING, errors.New("Nope")
			}

			err := scheduler.Kill(id)

			So(driver.KillTaskFnInvoked, ShouldBeTrue)
			So(err, ShouldNotBeNil)
		})

		Convey("Given already terminated task", func() {
			db.PutTask(&eremetic.Task{
				ID: id,
				Status: []eremetic.Status{
					eremetic.Status{
						Time:   123456,
						Status: eremetic.TaskLost,
					},
				},
			})
			driver.KillTaskFn = func(_ *mesosproto.TaskID) (mesosproto.Status, error) {
				return mesosproto.Status_DRIVER_RUNNING, nil
			}

			err := scheduler.Kill(id)

			So(err, ShouldNotBeNil)
			So(driver.KillTaskFnInvoked, ShouldBeFalse)
		})
	})
}