func TestScheduler(t *gotesting.T) {
mockdriver := &MockSchedulerDriver{}
ntasks := 1
chillFactor := 0
testScheduler := NewEtcdScheduler(
ntasks,
chillFactor,
0,
false,
[]*mesos.CommandInfo_URI{},
false,
4096,
1,
256,
)
// Skip initialization logic, tested in TestStartup.
testScheduler.state = Mutable
reconciliation := map[string]string{}
testScheduler.reconciliationInfoFunc = func([]string, string, string) (map[string]string, error) {
return reconciliation, nil
}
testScheduler.updateReconciliationInfoFunc = func(info map[string]string, _ []string, _ string, _ string) error {
reconciliation = info
return nil
}
taskStatus_task_starting := util.NewTaskStatus(
util.NewTaskID("etcd-1 localhost 1 1 1"),
mesos.TaskState_TASK_RUNNING,
)
testScheduler.StatusUpdate(mockdriver, taskStatus_task_starting)
taskStatus_task_running := util.NewTaskStatus(
util.NewTaskID("etcd-1 localhost 1 1 1"),
mesos.TaskState_TASK_RUNNING,
)
testScheduler.StatusUpdate(mockdriver, taskStatus_task_running)
taskStatus_task_failed := util.NewTaskStatus(
util.NewTaskID("etcd-1 localhost 1 1 1"),
mesos.TaskState_TASK_FAILED,
)
testScheduler.StatusUpdate(mockdriver, taskStatus_task_failed)
//assert that mock was invoked
mockdriver.AssertExpectations(t)
}
func (suite *SchedulerTestSuite) TestSchdulerDriverReconcileTasks() {
messenger := messenger.NewMockedMessenger()
messenger.On("Start").Return(nil)
messenger.On("UPID").Return(&upid.UPID{})
messenger.On("Send").Return(nil)
messenger.On("Stop").Return(nil)
messenger.On("Route").Return(nil)
driver, err := newTestSchedulerDriver(NewMockScheduler(), suite.framework, suite.master, nil)
driver.messenger = messenger
suite.NoError(err)
suite.True(driver.Stopped())
driver.Start()
driver.setConnected(true) // simulated
suite.Equal(mesos.Status_DRIVER_RUNNING, driver.Status())
stat, err := driver.ReconcileTasks(
[]*mesos.TaskStatus{
util.NewTaskStatus(util.NewTaskID("test-task-001"), mesos.TaskState_TASK_FINISHED),
},
)
suite.NoError(err)
suite.Equal(mesos.Status_DRIVER_RUNNING, stat)
}
func fakeStatusUpdate(taskId string, state mesos.TaskState) *mesos.TaskStatus {
status := mesosutil.NewTaskStatus(mesosutil.NewTaskID(taskId), state)
status.Data = []byte("{}") // empty json
masterSource := mesos.TaskStatus_SOURCE_MASTER
status.Source = &masterSource
return status
}
func (s *Scheduler) reconcileTasks(force bool) {
if time.Now().Sub(s.reconcileTime) >= reconcileDelay {
if !s.cluster.IsReconciling() {
s.reconciles = 0
}
s.reconciles++
s.reconcileTime = time.Now()
if s.reconciles > reconcileMaxTries {
for _, task := range s.cluster.GetTasksWithState(TaskStateReconciling) {
if task.Data().TaskID != "" {
Logger.Infof("Reconciling exceeded %d tries for task %s, sending killTask for task %s", reconcileMaxTries, task.Data().ID, task.Data().TaskID)
s.driver.KillTask(util.NewTaskID(task.Data().TaskID))
task.Data().ResetTaskInfo()
}
}
} else {
if force {
s.driver.ReconcileTasks(nil)
} else {
statuses := make([]*mesos.TaskStatus, 0)
for _, task := range s.cluster.GetAllTasks() {
if task.Data().TaskID != "" {
task.Data().State = TaskStateReconciling
Logger.Infof("Reconciling %d/%d task state for id %s, task id %s", s.reconciles, reconcileMaxTries, task.Data().ID, task.Data().TaskID)
statuses = append(statuses, util.NewTaskStatus(util.NewTaskID(task.Data().TaskID), mesos.TaskState_TASK_STAGING))
}
}
s.driver.ReconcileTasks(statuses)
}
}
}
}
func (r *Reconciler) reconcile(driver scheduler.SchedulerDriver, implicit bool) {
if time.Now().Sub(r.reconcileTime) >= r.ReconcileDelay {
r.taskLock.Lock()
defer r.taskLock.Unlock()
r.reconciles++
r.reconcileTime = time.Now()
if r.reconciles > r.ReconcileMaxTries {
for task := range r.tasks {
Logger.Info("Reconciling exceeded %d tries, sending killTask for task %s", r.ReconcileMaxTries, task)
driver.KillTask(util.NewTaskID(task))
}
r.reconciles = 0
} else {
if implicit {
driver.ReconcileTasks(nil)
} else {
statuses := make([]*mesos.TaskStatus, 0)
for task := range r.tasks {
Logger.Debug("Reconciling %d/%d task state for task id %s", r.reconciles, r.ReconcileMaxTries, task)
statuses = append(statuses, util.NewTaskStatus(util.NewTaskID(task), mesos.TaskState_TASK_STAGING))
}
driver.ReconcileTasks(statuses)
}
}
}
}
//test we can handle different status updates, TODO check state transitions
func TestStatus_Update(t *testing.T) {
mockdriver := MockSchedulerDriver{}
// setup expectations
mockdriver.On("KillTask", util.NewTaskID("test-task-001")).Return(mesos.Status_DRIVER_RUNNING, nil)
testFramework := &framework{
offers: offers.CreateRegistry(offers.RegistryConfig{
Compat: func(o *mesos.Offer) bool {
return true
},
// remember expired offers so that we can tell if a previously scheduler offer relies on one
LingerTTL: schedcfg.DefaultOfferLingerTTL,
TTL: schedcfg.DefaultOfferTTL,
ListenerDelay: schedcfg.DefaultListenerDelay,
}),
slaveHostNames: newSlaveRegistry(),
driver: &mockdriver,
sched: mockScheduler(),
}
taskStatus_task_starting := util.NewTaskStatus(
util.NewTaskID("test-task-001"),
mesos.TaskState_TASK_RUNNING,
)
testFramework.StatusUpdate(testFramework.driver, taskStatus_task_starting)
taskStatus_task_running := util.NewTaskStatus(
util.NewTaskID("test-task-001"),
mesos.TaskState_TASK_RUNNING,
)
testFramework.StatusUpdate(testFramework.driver, taskStatus_task_running)
taskStatus_task_failed := util.NewTaskStatus(
util.NewTaskID("test-task-001"),
mesos.TaskState_TASK_FAILED,
)
testFramework.StatusUpdate(testFramework.driver, taskStatus_task_failed)
//assert that mock was invoked
mockdriver.AssertExpectations(t)
}
func TestSendGetStatus(t *testing.T) {
task, err := newTask(nil, cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "")
assert.NoError(t, err)
status := mesosutil.NewTaskStatus(nil, mesosproto.TaskState_TASK_RUNNING)
go func() { task.sendStatus(status) }()
s := task.getStatus()
assert.Equal(t, s, status)
}
func TestSendGetStatus(t *testing.T) {
task, err := NewTask(cluster.BuildContainerConfig(containertypes.Config{}, containertypes.HostConfig{}, networktypes.NetworkingConfig{}), "", 5*time.Second)
assert.NoError(t, err)
status := mesosutil.NewTaskStatus(nil, mesosproto.TaskState_TASK_RUNNING)
go func() { task.SendStatus(status) }()
s := task.GetStatus()
assert.Equal(t, s, status)
}
func (suite *SchedulerTestSuite) TestSchdulerDriverReconcileTasks() {
driver := newTestDriver(suite.T(), driverConfigMessenger(mock_scheduler.New(), suite.framework, suite.master, nil, mockedMessenger()))
driver.Start()
driver.SetConnected(true) // simulated
suite.Equal(mesos.Status_DRIVER_RUNNING, driver.Status())
stat, err := driver.ReconcileTasks(
[]*mesos.TaskStatus{
util.NewTaskStatus(util.NewTaskID("test-task-001"), mesos.TaskState_TASK_FINISHED),
},
)
suite.NoError(err)
suite.Equal(mesos.Status_DRIVER_RUNNING, stat)
}
func TestStatusUpdateAckRace_Issue103(t *testing.T) {
driver, _, _ := createTestExecutorDriver(t)
_, err := driver.Start()
assert.NoError(t, err)
msg := &mesosproto.StatusUpdateAcknowledgementMessage{}
go driver.statusUpdateAcknowledgement(nil, msg)
taskStatus := util.NewTaskStatus(
util.NewTaskID("test-task-001"),
mesosproto.TaskState_TASK_STAGING,
)
driver.SendStatusUpdate(taskStatus)
}
func TestExecutorDriverSendStatusUpdateStaging(t *testing.T) {
driver, _ := createTestExecutorDriver(t)
stat, err := driver.Start()
assert.NoError(t, err)
assert.Equal(t, mesosproto.Status_DRIVER_RUNNING, stat)
driver.setConnected(true)
taskStatus := util.NewTaskStatus(
util.NewTaskID("test-task-001"),
mesosproto.TaskState_TASK_STAGING,
)
stat, err = driver.SendStatusUpdate(taskStatus)
assert.Error(t, err)
assert.Equal(t, mesosproto.Status_DRIVER_ABORTED, stat)
}
func TestExecutorDriverSendStatusUpdate(t *testing.T) {
driver, _, _ := createTestExecutorDriver(t)
stat, err := driver.Start()
assert.NoError(t, err)
assert.Equal(t, mesosproto.Status_DRIVER_RUNNING, stat)
driver.connected = true
driver.stopped = false
taskStatus := util.NewTaskStatus(
util.NewTaskID("test-task-001"),
mesosproto.TaskState_TASK_RUNNING,
)
stat, err = driver.SendStatusUpdate(taskStatus)
assert.NoError(t, err)
assert.Equal(t, mesosproto.Status_DRIVER_RUNNING, stat)
}
func (m *MockSchedulerDriver) LaunchTasks(offerIds []*mesos.OfferID, ti []*mesos.TaskInfo, f *mesos.Filters) (mesos.Status, error) {
m.Lock()
defer m.Unlock()
if m.scheduler != nil {
for _, taskInfo := range ti {
status := util.NewTaskStatus(
taskInfo.TaskId,
mesos.TaskState_TASK_RUNNING,
)
// TODO(tyler) use actual executor here to launch a test instance, so we can catch etcd config errors
m.scheduler.StatusUpdate(m, status)
}
}
// Too much dynamic stuff for comparison, just look at Resources.
tasks := []*mesos.TaskInfo{
{
Resources: ti[0].Resources,
},
}
args := m.Called(offerIds, tasks, f)
return status(args, 0), args.Error(1)
}
func TestExecutorDriverSendStatusUpdateStaging(t *testing.T) {
driver, _, _ := createTestExecutorDriver(t)
exec := NewMockedExecutor()
exec.On("Error").Return(nil)
driver.exec = exec
stat, err := driver.Start()
assert.NoError(t, err)
assert.Equal(t, mesosproto.Status_DRIVER_RUNNING, stat)
driver.connected = true
driver.stopped = false
taskStatus := util.NewTaskStatus(
util.NewTaskID("test-task-001"),
mesosproto.TaskState_TASK_STAGING,
)
stat, err = driver.SendStatusUpdate(taskStatus)
assert.Error(t, err)
assert.Equal(t, mesosproto.Status_DRIVER_ABORTED, stat)
}
func (suite *SchedulerIntegrationTestSuite) TestSchedulerDriverStatusUpdatedEvent() {
t := suite.T()
var wg sync.WaitGroup
wg.Add(2)
suite.config = mockServerConfigurator(func(frameworkId *mesos.FrameworkID, suite *SchedulerIntegrationTestSuite) {
defaultMockServerConfigurator(frameworkId, suite)
suite.server.On("/master/mesos.internal.StatusUpdateAcknowledgementMessage").Do(func(rsp http.ResponseWriter, req *http.Request) {
log.Infoln("Master cvd ACK")
data, _ := ioutil.ReadAll(req.Body)
defer req.Body.Close()
assert.NotNil(t, data)
wg.Done()
log.Infof("MockMaster - Done with wait group")
})
suite.sched.wg = &wg
})
ok := suite.configureServerWithRegisteredFramework()
suite.True(ok, "failed to establish running test server and driver")
// Send a event to this SchedulerDriver (via http) to test handlers.
pbMsg := &mesos.StatusUpdateMessage{
Update: util.NewStatusUpdate(
suite.registeredFrameworkId,
util.NewTaskStatus(util.NewTaskID("test-task-001"), mesos.TaskState_TASK_STARTING),
float64(time.Now().Unix()),
[]byte("test-abcd-ef-3455-454-001"),
),
// note: cannot use driver's pid here if we want an ACK
Pid: proto.String("test-slave-001(1)@foo.bar:1234"),
}
pbMsg.Update.SlaveId = &mesos.SlaveID{Value: proto.String("test-slave-001")}
c := suite.newMockClient()
c.SendMessage(suite.driver.UPID(), pbMsg)
wg.Wait()
}
func TestStatus(t *testing.T) {
status := util.NewTaskStatus(util.NewTaskID("task-1"), mesos.TaskState_TASK_FINISHED)
if !strings.Contains(Status(status), "task-1 TASK_FINISHED reason") {
t.Errorf(`Task status should contain "task-1 TASK_FINISHED reason"; actual %s`, Status(status))
}
status.SlaveId = util.NewSlaveID("20150903-065451-84125888-5050-10715-S1")
if !strings.Contains(Status(status), "task-1 TASK_FINISHED slave: #15-S1") {
t.Errorf(`Task status should contain "task-1 TASK_FINISHED slave: #15-S1"; actual %s`, Status(status))
}
status.State = mesos.TaskState_TASK_RUNNING.Enum()
if strings.Contains(Status(status), "reason") {
t.Errorf(`Task status with running state should not contain "reason"; actual %s`, Status(status))
}
status.State = mesos.TaskState_TASK_LOST.Enum()
status.Reason = mesos.TaskStatus_REASON_EXECUTOR_TERMINATED.Enum()
if !strings.Contains(Status(status), "task-1 TASK_LOST slave: #15-S1 reason: REASON_EXECUTOR_TERMINATED") {
t.Errorf(`Task status should contain "task-1 TASK_LOST slave: #15-S1 reason: REASON_EXECUTOR_TERMINATED"; actual %s`, Status(status))
}
status.Message = proto.String("boom!")
if !strings.Contains(Status(status), "message: boom!") {
t.Errorf(`Task status should contain "message: boom!"; actual %s`, Status(status))
}
}
func (r *Reconciler) reconcile(driver scheduler.SchedulerDriver, implicit bool) {
if time.Now().Sub(r.reconcileTime) >= r.ReconcileDelay {
if !r.tasks.IsReconciling() {
r.reconciles = 0
}
r.reconciles++
r.reconcileTime = time.Now()
if r.reconciles > r.ReconcileMaxTries {
for _, task := range r.tasks.GetWithFilter(func(task Task) bool {
return task.Data().State == TaskStateReconciling
}) {
if task.Data().TaskID != "" {
Logger.Info("Reconciling exceeded %d tries for task %s, sending killTask for task %s", r.ReconcileMaxTries, task.Data().ID, task.Data().TaskID)
driver.KillTask(util.NewTaskID(task.Data().TaskID))
task.Data().Reset()
}
}
} else {
if implicit {
driver.ReconcileTasks(nil)
} else {
statuses := make([]*mesos.TaskStatus, 0)
for _, task := range r.tasks.GetAll() {
if task.Data().TaskID != "" {
task.Data().State = TaskStateReconciling
Logger.Info("Reconciling %d/%d task state for id %s, task id %s", r.reconciles, r.ReconcileMaxTries, task.Data().ID, task.Data().TaskID)
statuses = append(statuses, util.NewTaskStatus(util.NewTaskID(task.Data().TaskID), mesos.TaskState_TASK_STAGING))
}
}
driver.ReconcileTasks(statuses)
}
}
}
}
func TestReconciliationOnStartup(t *gotesting.T) {
testScheduler := NewEtcdScheduler(3, 0, 0, true, []*mesos.CommandInfo_URI{}, false, 4096, 1, 256)
mockdriver := &MockSchedulerDriver{
runningStatuses: make(chan *mesos.TaskStatus, 10),
scheduler: testScheduler,
}
reconciliation := map[string]string{
"etcd-1": "slave-1",
"etcd-2": "slave-2",
"etcd-3": "slave-3",
}
testScheduler.reconciliationInfoFunc = func([]string, string, string) (map[string]string, error) {
return reconciliation, nil
}
testScheduler.updateReconciliationInfoFunc = func(info map[string]string, _ []string, _ string, _ string) error {
reconciliation = info
return nil
}
// Valid reconciled tasks should be added to the running list.
for _, taskStatus := range []*mesos.TaskStatus{
util.NewTaskStatus(
util.NewTaskID("etcd-1 localhost 0 0 0"),
mesos.TaskState_TASK_RUNNING,
),
util.NewTaskStatus(
util.NewTaskID("etcd-2 localhost 0 0 0"),
mesos.TaskState_TASK_RUNNING,
),
util.NewTaskStatus(
util.NewTaskID("etcd-3 localhost 0 0 0"),
mesos.TaskState_TASK_RUNNING,
),
} {
mockdriver.runningStatuses <- taskStatus
}
mockdriver.Lock()
mockdriver.On(
"ReconcileTasks",
0,
).Return(mesos.Status_DRIVER_RUNNING, nil).Once()
mockdriver.On(
"ReconcileTasks",
3,
).Return(mesos.Status_DRIVER_RUNNING, nil).Once()
mockdriver.Unlock()
masterInfo := util.NewMasterInfo("master-1", 0, 0)
masterInfo.Hostname = proto.String("test-host")
testScheduler.Registered(
mockdriver,
util.NewFrameworkID("framework-1"),
masterInfo,
)
time.Sleep(50 * time.Millisecond)
mockdriver.Lock()
defer mockdriver.Unlock()
assert.Equal(t, 3, len(testScheduler.running),
"Scheduler should reconcile tasks properly.")
mockdriver.AssertExpectations(t)
}
func TestGrowToDesiredAfterReconciliation(t *gotesting.T) {
testScheduler := NewEtcdScheduler(3, 0, 0, true, []*mesos.CommandInfo_URI{}, false, 4096, 1, 256)
reconciliation := map[string]string{
"etcd-1": "slave-1",
"etcd-2": "slave-2",
}
testScheduler.reconciliationInfoFunc = func([]string, string, string) (map[string]string, error) {
return reconciliation, nil
}
testScheduler.updateReconciliationInfoFunc = func(info map[string]string, _ []string, _ string, _ string) error {
reconciliation = info
return nil
}
testScheduler.masterInfo = util.NewMasterInfo("master-1", 0, 0)
mockdriver := &MockSchedulerDriver{
runningStatuses: make(chan *mesos.TaskStatus, 10),
scheduler: testScheduler,
}
testScheduler.state = Mutable
testScheduler.healthCheck = func(map[string]*config.Node) error {
return nil
}
// Push more than enough offers to shoot self in foot if unchecked.
for _, offer := range []*mesos.Offer{
NewOffer("1"),
NewOffer("2"),
NewOffer("3"),
} {
testScheduler.offerCache.Push(offer)
}
memberList := config.ClusterMemberList{
Members: []httptypes.Member{
{
ID: "1",
Name: "etcd-1",
PeerURLs: nil,
ClientURLs: nil,
},
{
ID: "2",
Name: "etcd-2",
PeerURLs: nil,
ClientURLs: nil,
},
},
}
_, port1, err := emtesting.NewTestEtcdServer(t, memberList)
if err != nil {
t.Fatalf("Failed to create test etcd server: %s", err)
}
_, port2, err := emtesting.NewTestEtcdServer(t, memberList)
if err != nil {
t.Fatalf("Failed to create test etcd server: %s", err)
}
// Valid reconciled tasks should be added to the running list.
mockdriver.On(
"ReconcileTasks",
0,
).Return(mesos.Status_DRIVER_RUNNING, nil).Once()
for _, taskStatus := range []*mesos.TaskStatus{
util.NewTaskStatus(
util.NewTaskID("etcd-1 localhost 0 "+strconv.Itoa(int(port1))+" 0"),
mesos.TaskState_TASK_RUNNING,
),
util.NewTaskStatus(
util.NewTaskID("etcd-2 localhost 0 "+strconv.Itoa(int(port2))+" 0"),
mesos.TaskState_TASK_RUNNING,
),
} {
mockdriver.runningStatuses <- taskStatus
}
// Scheduler should grow cluster to desired number of nodes.
offer := NewOffer("1")
mockdriver.On(
"LaunchTasks",
[]*mesos.OfferID{
offer.Id,
},
[]*mesos.TaskInfo{
{
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", 1),
util.NewScalarResource("mem", 256),
util.NewScalarResource("disk", 4096),
util.NewRangesResource("ports", []*mesos.Value_Range{
util.NewValueRange(uint64(0), uint64(2)),
}),
},
},
},
&mesos.Filters{
RefuseSeconds: proto.Float64(1),
},
).Return(mesos.Status_DRIVER_RUNNING, nil).Once()
// Simulate failover, registration and time passing.
mockdriver.ReconcileTasks([]*mesos.TaskStatus{})
testScheduler.launchOne(mockdriver)
testScheduler.launchOne(mockdriver)
testScheduler.launchOne(mockdriver)
testScheduler.launchOne(mockdriver)
testScheduler.launchOne(mockdriver)
assert.Equal(t, 3, len(testScheduler.running),
"Scheduler should reconcile tasks properly.")
mockdriver.AssertExpectations(t)
}
