func TestScheduler(t *gotesting.T) { mockdriver := &MockSchedulerDriver{} ntasks := 1 chillFactor := 0 testScheduler := NewEtcdScheduler( ntasks, chillFactor, 0, false, []*mesos.CommandInfo_URI{}, false, 4096, 1, 256, ) // Skip initialization logic, tested in TestStartup. testScheduler.state = Mutable reconciliation := map[string]string{} testScheduler.reconciliationInfoFunc = func([]string, string, string) (map[string]string, error) { return reconciliation, nil } testScheduler.updateReconciliationInfoFunc = func(info map[string]string, _ []string, _ string, _ string) error { reconciliation = info return nil } taskStatus_task_starting := util.NewTaskStatus( util.NewTaskID("etcd-1 localhost 1 1 1"), mesos.TaskState_TASK_RUNNING, ) testScheduler.StatusUpdate(mockdriver, taskStatus_task_starting) taskStatus_task_running := util.NewTaskStatus( util.NewTaskID("etcd-1 localhost 1 1 1"), mesos.TaskState_TASK_RUNNING, ) testScheduler.StatusUpdate(mockdriver, taskStatus_task_running) taskStatus_task_failed := util.NewTaskStatus( util.NewTaskID("etcd-1 localhost 1 1 1"), mesos.TaskState_TASK_FAILED, ) testScheduler.StatusUpdate(mockdriver, taskStatus_task_failed) //assert that mock was invoked mockdriver.AssertExpectations(t) }
func (suite *SchedulerTestSuite) TestSchdulerDriverReconcileTasks() { messenger := messenger.NewMockedMessenger() messenger.On("Start").Return(nil) messenger.On("UPID").Return(&upid.UPID{}) messenger.On("Send").Return(nil) messenger.On("Stop").Return(nil) messenger.On("Route").Return(nil) driver, err := newTestSchedulerDriver(NewMockScheduler(), suite.framework, suite.master, nil) driver.messenger = messenger suite.NoError(err) suite.True(driver.Stopped()) driver.Start() driver.setConnected(true) // simulated suite.Equal(mesos.Status_DRIVER_RUNNING, driver.Status()) stat, err := driver.ReconcileTasks( []*mesos.TaskStatus{ util.NewTaskStatus(util.NewTaskID("test-task-001"), mesos.TaskState_TASK_FINISHED), }, ) suite.NoError(err) suite.Equal(mesos.Status_DRIVER_RUNNING, stat) }
func fakeStatusUpdate(taskId string, state mesos.TaskState) *mesos.TaskStatus { status := mesosutil.NewTaskStatus(mesosutil.NewTaskID(taskId), state) status.Data = []byte("{}") // empty json masterSource := mesos.TaskStatus_SOURCE_MASTER status.Source = &masterSource return status }
func (s *Scheduler) reconcileTasks(force bool) { if time.Now().Sub(s.reconcileTime) >= reconcileDelay { if !s.cluster.IsReconciling() { s.reconciles = 0 } s.reconciles++ s.reconcileTime = time.Now() if s.reconciles > reconcileMaxTries { for _, task := range s.cluster.GetTasksWithState(TaskStateReconciling) { if task.Data().TaskID != "" { Logger.Infof("Reconciling exceeded %d tries for task %s, sending killTask for task %s", reconcileMaxTries, task.Data().ID, task.Data().TaskID) s.driver.KillTask(util.NewTaskID(task.Data().TaskID)) task.Data().ResetTaskInfo() } } } else { if force { s.driver.ReconcileTasks(nil) } else { statuses := make([]*mesos.TaskStatus, 0) for _, task := range s.cluster.GetAllTasks() { if task.Data().TaskID != "" { task.Data().State = TaskStateReconciling Logger.Infof("Reconciling %d/%d task state for id %s, task id %s", s.reconciles, reconcileMaxTries, task.Data().ID, task.Data().TaskID) statuses = append(statuses, util.NewTaskStatus(util.NewTaskID(task.Data().TaskID), mesos.TaskState_TASK_STAGING)) } } s.driver.ReconcileTasks(statuses) } } } }
func (r *Reconciler) reconcile(driver scheduler.SchedulerDriver, implicit bool) { if time.Now().Sub(r.reconcileTime) >= r.ReconcileDelay { r.taskLock.Lock() defer r.taskLock.Unlock() r.reconciles++ r.reconcileTime = time.Now() if r.reconciles > r.ReconcileMaxTries { for task := range r.tasks { Logger.Info("Reconciling exceeded %d tries, sending killTask for task %s", r.ReconcileMaxTries, task) driver.KillTask(util.NewTaskID(task)) } r.reconciles = 0 } else { if implicit { driver.ReconcileTasks(nil) } else { statuses := make([]*mesos.TaskStatus, 0) for task := range r.tasks { Logger.Debug("Reconciling %d/%d task state for task id %s", r.reconciles, r.ReconcileMaxTries, task) statuses = append(statuses, util.NewTaskStatus(util.NewTaskID(task), mesos.TaskState_TASK_STAGING)) } driver.ReconcileTasks(statuses) } } } }
//test we can handle different status updates, TODO check state transitions func TestStatus_Update(t *testing.T) { mockdriver := MockSchedulerDriver{} // setup expectations mockdriver.On("KillTask", util.NewTaskID("test-task-001")).Return(mesos.Status_DRIVER_RUNNING, nil) testFramework := &framework{ offers: offers.CreateRegistry(offers.RegistryConfig{ Compat: func(o *mesos.Offer) bool { return true }, // remember expired offers so that we can tell if a previously scheduler offer relies on one LingerTTL: schedcfg.DefaultOfferLingerTTL, TTL: schedcfg.DefaultOfferTTL, ListenerDelay: schedcfg.DefaultListenerDelay, }), slaveHostNames: newSlaveRegistry(), driver: &mockdriver, sched: mockScheduler(), } taskStatus_task_starting := util.NewTaskStatus( util.NewTaskID("test-task-001"), mesos.TaskState_TASK_RUNNING, ) testFramework.StatusUpdate(testFramework.driver, taskStatus_task_starting) taskStatus_task_running := util.NewTaskStatus( util.NewTaskID("test-task-001"), mesos.TaskState_TASK_RUNNING, ) testFramework.StatusUpdate(testFramework.driver, taskStatus_task_running) taskStatus_task_failed := util.NewTaskStatus( util.NewTaskID("test-task-001"), mesos.TaskState_TASK_FAILED, ) testFramework.StatusUpdate(testFramework.driver, taskStatus_task_failed) //assert that mock was invoked mockdriver.AssertExpectations(t) }
func TestSendGetStatus(t *testing.T) { task, err := newTask(nil, cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "") assert.NoError(t, err) status := mesosutil.NewTaskStatus(nil, mesosproto.TaskState_TASK_RUNNING) go func() { task.sendStatus(status) }() s := task.getStatus() assert.Equal(t, s, status) }
func TestSendGetStatus(t *testing.T) { task, err := NewTask(cluster.BuildContainerConfig(containertypes.Config{}, containertypes.HostConfig{}, networktypes.NetworkingConfig{}), "", 5*time.Second) assert.NoError(t, err) status := mesosutil.NewTaskStatus(nil, mesosproto.TaskState_TASK_RUNNING) go func() { task.SendStatus(status) }() s := task.GetStatus() assert.Equal(t, s, status) }
func (suite *SchedulerTestSuite) TestSchdulerDriverReconcileTasks() { driver := newTestDriver(suite.T(), driverConfigMessenger(mock_scheduler.New(), suite.framework, suite.master, nil, mockedMessenger())) driver.Start() driver.SetConnected(true) // simulated suite.Equal(mesos.Status_DRIVER_RUNNING, driver.Status()) stat, err := driver.ReconcileTasks( []*mesos.TaskStatus{ util.NewTaskStatus(util.NewTaskID("test-task-001"), mesos.TaskState_TASK_FINISHED), }, ) suite.NoError(err) suite.Equal(mesos.Status_DRIVER_RUNNING, stat) }
func TestStatusUpdateAckRace_Issue103(t *testing.T) { driver, _, _ := createTestExecutorDriver(t) _, err := driver.Start() assert.NoError(t, err) msg := &mesosproto.StatusUpdateAcknowledgementMessage{} go driver.statusUpdateAcknowledgement(nil, msg) taskStatus := util.NewTaskStatus( util.NewTaskID("test-task-001"), mesosproto.TaskState_TASK_STAGING, ) driver.SendStatusUpdate(taskStatus) }
func TestExecutorDriverSendStatusUpdateStaging(t *testing.T) { driver, _ := createTestExecutorDriver(t) stat, err := driver.Start() assert.NoError(t, err) assert.Equal(t, mesosproto.Status_DRIVER_RUNNING, stat) driver.setConnected(true) taskStatus := util.NewTaskStatus( util.NewTaskID("test-task-001"), mesosproto.TaskState_TASK_STAGING, ) stat, err = driver.SendStatusUpdate(taskStatus) assert.Error(t, err) assert.Equal(t, mesosproto.Status_DRIVER_ABORTED, stat) }
func TestExecutorDriverSendStatusUpdate(t *testing.T) { driver, _, _ := createTestExecutorDriver(t) stat, err := driver.Start() assert.NoError(t, err) assert.Equal(t, mesosproto.Status_DRIVER_RUNNING, stat) driver.connected = true driver.stopped = false taskStatus := util.NewTaskStatus( util.NewTaskID("test-task-001"), mesosproto.TaskState_TASK_RUNNING, ) stat, err = driver.SendStatusUpdate(taskStatus) assert.NoError(t, err) assert.Equal(t, mesosproto.Status_DRIVER_RUNNING, stat) }
func (m *MockSchedulerDriver) LaunchTasks(offerIds []*mesos.OfferID, ti []*mesos.TaskInfo, f *mesos.Filters) (mesos.Status, error) { m.Lock() defer m.Unlock() if m.scheduler != nil { for _, taskInfo := range ti { status := util.NewTaskStatus( taskInfo.TaskId, mesos.TaskState_TASK_RUNNING, ) // TODO(tyler) use actual executor here to launch a test instance, so we can catch etcd config errors m.scheduler.StatusUpdate(m, status) } } // Too much dynamic stuff for comparison, just look at Resources. tasks := []*mesos.TaskInfo{ { Resources: ti[0].Resources, }, } args := m.Called(offerIds, tasks, f) return status(args, 0), args.Error(1) }
func TestExecutorDriverSendStatusUpdateStaging(t *testing.T) { driver, _, _ := createTestExecutorDriver(t) exec := NewMockedExecutor() exec.On("Error").Return(nil) driver.exec = exec stat, err := driver.Start() assert.NoError(t, err) assert.Equal(t, mesosproto.Status_DRIVER_RUNNING, stat) driver.connected = true driver.stopped = false taskStatus := util.NewTaskStatus( util.NewTaskID("test-task-001"), mesosproto.TaskState_TASK_STAGING, ) stat, err = driver.SendStatusUpdate(taskStatus) assert.Error(t, err) assert.Equal(t, mesosproto.Status_DRIVER_ABORTED, stat) }
func (suite *SchedulerIntegrationTestSuite) TestSchedulerDriverStatusUpdatedEvent() { t := suite.T() var wg sync.WaitGroup wg.Add(2) suite.config = mockServerConfigurator(func(frameworkId *mesos.FrameworkID, suite *SchedulerIntegrationTestSuite) { defaultMockServerConfigurator(frameworkId, suite) suite.server.On("/master/mesos.internal.StatusUpdateAcknowledgementMessage").Do(func(rsp http.ResponseWriter, req *http.Request) { log.Infoln("Master cvd ACK") data, _ := ioutil.ReadAll(req.Body) defer req.Body.Close() assert.NotNil(t, data) wg.Done() log.Infof("MockMaster - Done with wait group") }) suite.sched.wg = &wg }) ok := suite.configureServerWithRegisteredFramework() suite.True(ok, "failed to establish running test server and driver") // Send a event to this SchedulerDriver (via http) to test handlers. pbMsg := &mesos.StatusUpdateMessage{ Update: util.NewStatusUpdate( suite.registeredFrameworkId, util.NewTaskStatus(util.NewTaskID("test-task-001"), mesos.TaskState_TASK_STARTING), float64(time.Now().Unix()), []byte("test-abcd-ef-3455-454-001"), ), // note: cannot use driver's pid here if we want an ACK Pid: proto.String("test-slave-001(1)@foo.bar:1234"), } pbMsg.Update.SlaveId = &mesos.SlaveID{Value: proto.String("test-slave-001")} c := suite.newMockClient() c.SendMessage(suite.driver.UPID(), pbMsg) wg.Wait() }
func TestStatus(t *testing.T) { status := util.NewTaskStatus(util.NewTaskID("task-1"), mesos.TaskState_TASK_FINISHED) if !strings.Contains(Status(status), "task-1 TASK_FINISHED reason") { t.Errorf(`Task status should contain "task-1 TASK_FINISHED reason"; actual %s`, Status(status)) } status.SlaveId = util.NewSlaveID("20150903-065451-84125888-5050-10715-S1") if !strings.Contains(Status(status), "task-1 TASK_FINISHED slave: #15-S1") { t.Errorf(`Task status should contain "task-1 TASK_FINISHED slave: #15-S1"; actual %s`, Status(status)) } status.State = mesos.TaskState_TASK_RUNNING.Enum() if strings.Contains(Status(status), "reason") { t.Errorf(`Task status with running state should not contain "reason"; actual %s`, Status(status)) } status.State = mesos.TaskState_TASK_LOST.Enum() status.Reason = mesos.TaskStatus_REASON_EXECUTOR_TERMINATED.Enum() if !strings.Contains(Status(status), "task-1 TASK_LOST slave: #15-S1 reason: REASON_EXECUTOR_TERMINATED") { t.Errorf(`Task status should contain "task-1 TASK_LOST slave: #15-S1 reason: REASON_EXECUTOR_TERMINATED"; actual %s`, Status(status)) } status.Message = proto.String("boom!") if !strings.Contains(Status(status), "message: boom!") { t.Errorf(`Task status should contain "message: boom!"; actual %s`, Status(status)) } }
func (r *Reconciler) reconcile(driver scheduler.SchedulerDriver, implicit bool) { if time.Now().Sub(r.reconcileTime) >= r.ReconcileDelay { if !r.tasks.IsReconciling() { r.reconciles = 0 } r.reconciles++ r.reconcileTime = time.Now() if r.reconciles > r.ReconcileMaxTries { for _, task := range r.tasks.GetWithFilter(func(task Task) bool { return task.Data().State == TaskStateReconciling }) { if task.Data().TaskID != "" { Logger.Info("Reconciling exceeded %d tries for task %s, sending killTask for task %s", r.ReconcileMaxTries, task.Data().ID, task.Data().TaskID) driver.KillTask(util.NewTaskID(task.Data().TaskID)) task.Data().Reset() } } } else { if implicit { driver.ReconcileTasks(nil) } else { statuses := make([]*mesos.TaskStatus, 0) for _, task := range r.tasks.GetAll() { if task.Data().TaskID != "" { task.Data().State = TaskStateReconciling Logger.Info("Reconciling %d/%d task state for id %s, task id %s", r.reconciles, r.ReconcileMaxTries, task.Data().ID, task.Data().TaskID) statuses = append(statuses, util.NewTaskStatus(util.NewTaskID(task.Data().TaskID), mesos.TaskState_TASK_STAGING)) } } driver.ReconcileTasks(statuses) } } } }
func TestReconciliationOnStartup(t *gotesting.T) { testScheduler := NewEtcdScheduler(3, 0, 0, true, []*mesos.CommandInfo_URI{}, false, 4096, 1, 256) mockdriver := &MockSchedulerDriver{ runningStatuses: make(chan *mesos.TaskStatus, 10), scheduler: testScheduler, } reconciliation := map[string]string{ "etcd-1": "slave-1", "etcd-2": "slave-2", "etcd-3": "slave-3", } testScheduler.reconciliationInfoFunc = func([]string, string, string) (map[string]string, error) { return reconciliation, nil } testScheduler.updateReconciliationInfoFunc = func(info map[string]string, _ []string, _ string, _ string) error { reconciliation = info return nil } // Valid reconciled tasks should be added to the running list. for _, taskStatus := range []*mesos.TaskStatus{ util.NewTaskStatus( util.NewTaskID("etcd-1 localhost 0 0 0"), mesos.TaskState_TASK_RUNNING, ), util.NewTaskStatus( util.NewTaskID("etcd-2 localhost 0 0 0"), mesos.TaskState_TASK_RUNNING, ), util.NewTaskStatus( util.NewTaskID("etcd-3 localhost 0 0 0"), mesos.TaskState_TASK_RUNNING, ), } { mockdriver.runningStatuses <- taskStatus } mockdriver.Lock() mockdriver.On( "ReconcileTasks", 0, ).Return(mesos.Status_DRIVER_RUNNING, nil).Once() mockdriver.On( "ReconcileTasks", 3, ).Return(mesos.Status_DRIVER_RUNNING, nil).Once() mockdriver.Unlock() masterInfo := util.NewMasterInfo("master-1", 0, 0) masterInfo.Hostname = proto.String("test-host") testScheduler.Registered( mockdriver, util.NewFrameworkID("framework-1"), masterInfo, ) time.Sleep(50 * time.Millisecond) mockdriver.Lock() defer mockdriver.Unlock() assert.Equal(t, 3, len(testScheduler.running), "Scheduler should reconcile tasks properly.") mockdriver.AssertExpectations(t) }
func TestGrowToDesiredAfterReconciliation(t *gotesting.T) { testScheduler := NewEtcdScheduler(3, 0, 0, true, []*mesos.CommandInfo_URI{}, false, 4096, 1, 256) reconciliation := map[string]string{ "etcd-1": "slave-1", "etcd-2": "slave-2", } testScheduler.reconciliationInfoFunc = func([]string, string, string) (map[string]string, error) { return reconciliation, nil } testScheduler.updateReconciliationInfoFunc = func(info map[string]string, _ []string, _ string, _ string) error { reconciliation = info return nil } testScheduler.masterInfo = util.NewMasterInfo("master-1", 0, 0) mockdriver := &MockSchedulerDriver{ runningStatuses: make(chan *mesos.TaskStatus, 10), scheduler: testScheduler, } testScheduler.state = Mutable testScheduler.healthCheck = func(map[string]*config.Node) error { return nil } // Push more than enough offers to shoot self in foot if unchecked. for _, offer := range []*mesos.Offer{ NewOffer("1"), NewOffer("2"), NewOffer("3"), } { testScheduler.offerCache.Push(offer) } memberList := config.ClusterMemberList{ Members: []httptypes.Member{ { ID: "1", Name: "etcd-1", PeerURLs: nil, ClientURLs: nil, }, { ID: "2", Name: "etcd-2", PeerURLs: nil, ClientURLs: nil, }, }, } _, port1, err := emtesting.NewTestEtcdServer(t, memberList) if err != nil { t.Fatalf("Failed to create test etcd server: %s", err) } _, port2, err := emtesting.NewTestEtcdServer(t, memberList) if err != nil { t.Fatalf("Failed to create test etcd server: %s", err) } // Valid reconciled tasks should be added to the running list. mockdriver.On( "ReconcileTasks", 0, ).Return(mesos.Status_DRIVER_RUNNING, nil).Once() for _, taskStatus := range []*mesos.TaskStatus{ util.NewTaskStatus( util.NewTaskID("etcd-1 localhost 0 "+strconv.Itoa(int(port1))+" 0"), mesos.TaskState_TASK_RUNNING, ), util.NewTaskStatus( util.NewTaskID("etcd-2 localhost 0 "+strconv.Itoa(int(port2))+" 0"), mesos.TaskState_TASK_RUNNING, ), } { mockdriver.runningStatuses <- taskStatus } // Scheduler should grow cluster to desired number of nodes. offer := NewOffer("1") mockdriver.On( "LaunchTasks", []*mesos.OfferID{ offer.Id, }, []*mesos.TaskInfo{ { Resources: []*mesos.Resource{ util.NewScalarResource("cpus", 1), util.NewScalarResource("mem", 256), util.NewScalarResource("disk", 4096), util.NewRangesResource("ports", []*mesos.Value_Range{ util.NewValueRange(uint64(0), uint64(2)), }), }, }, }, &mesos.Filters{ RefuseSeconds: proto.Float64(1), }, ).Return(mesos.Status_DRIVER_RUNNING, nil).Once() // Simulate failover, registration and time passing. mockdriver.ReconcileTasks([]*mesos.TaskStatus{}) testScheduler.launchOne(mockdriver) testScheduler.launchOne(mockdriver) testScheduler.launchOne(mockdriver) testScheduler.launchOne(mockdriver) testScheduler.launchOne(mockdriver) assert.Equal(t, 3, len(testScheduler.running), "Scheduler should reconcile tasks properly.") mockdriver.AssertExpectations(t) }