func TestSchedCachePutSlavePid(t *testing.T) { cache := newSchedCache() pid01, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) pid02, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) pid03, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) cache.putSlavePid(util.NewSlaveID("slave01"), pid01) cache.putSlavePid(util.NewSlaveID("slave02"), pid02) cache.putSlavePid(util.NewSlaveID("slave03"), pid03) assert.Equal(t, len(cache.savedSlavePids), 3) cachedSlavePid1, ok := cache.savedSlavePids["slave01"] assert.True(t, ok) cachedSlavePid2, ok := cache.savedSlavePids["slave02"] assert.True(t, ok) cachedSlavePid3, ok := cache.savedSlavePids["slave03"] assert.True(t, ok) assert.True(t, cachedSlavePid1.Equal(pid01)) assert.True(t, cachedSlavePid2.Equal(pid02)) assert.True(t, cachedSlavePid3.Equal(pid03)) }
func TestSchedCachePutOffer(t *testing.T) { cache := newSchedCache() offer01 := createTestOffer("01") pid01, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) cache.putOffer(offer01, pid01) offer02 := createTestOffer("02") pid02, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) cache.putOffer(offer02, pid02) assert.Equal(t, len(cache.savedOffers), 2) cachedOffer1, ok := cache.savedOffers["test-offer-01"] assert.True(t, ok) cachedOffer2, ok := cache.savedOffers["test-offer-02"] assert.True(t, ok) assert.NotNil(t, cachedOffer1.offer) assert.Equal(t, "test-offer-01", cachedOffer1.offer.Id.GetValue()) assert.NotNil(t, cachedOffer2.offer) assert.Equal(t, "test-offer-02", cachedOffer2.offer.Id.GetValue()) assert.NotNil(t, cachedOffer1.slavePid) assert.Equal(t, "[email protected]:5050", cachedOffer1.slavePid.String()) assert.NotNil(t, cachedOffer2.slavePid) assert.Equal(t, "[email protected]:5050", cachedOffer2.slavePid.String()) }
func getLibprocessFrom(r *http.Request) (*upid.UPID, error) { if r.Method != "POST" { return nil, fmt.Errorf("Not a POST request") } if agent, ok := parseLibprocessAgent(r.Header); ok { return upid.Parse(agent) } lf, ok := r.Header["Libprocess-From"] if ok { // TODO(yifan): Just take the first field for now. return upid.Parse(lf[0]) } return nil, fmt.Errorf("Cannot find 'User-Agent' or 'Libprocess-From'") }
func TestSchedCacheContainsSlavePid(t *testing.T) { cache := newSchedCache() pid01, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) pid02, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) cache.putSlavePid(util.NewSlaveID("slave01"), pid01) cache.putSlavePid(util.NewSlaveID("slave02"), pid02) assert.True(t, cache.containsSlavePid(util.NewSlaveID("slave01"))) assert.True(t, cache.containsSlavePid(util.NewSlaveID("slave02"))) assert.False(t, cache.containsSlavePid(util.NewSlaveID("slave05"))) }
// assumes that address is in host:port format func (s *Standalone) _fetchPid(ctx context.Context, address string) (*upid.UPID, error) { //TODO(jdef) need SSL support uri := fmt.Sprintf("http://%s/state.json", address) req, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } var pid *upid.UPID err = s.httpDo(ctx, req, func(res *http.Response, err error) error { if err != nil { return err } defer res.Body.Close() if res.StatusCode != 200 { return fmt.Errorf("HTTP request failed with code %d: %v", res.StatusCode, res.Status) } blob, err1 := ioutil.ReadAll(res.Body) if err1 != nil { return err1 } log.V(3).Infof("Got mesos state, content length %v", len(blob)) type State struct { Leader string `json:"leader"` // ex: master(1)@10.22.211.18:5050 } state := &State{} err = json.Unmarshal(blob, state) if err != nil { return err } pid, err = upid.Parse(state.Leader) return err }) return pid, err }
func TestMutatedHostUPid(t *testing.T) { serverId := "testserver" // NOTE(tsenart): This static port can cause conflicts if multiple instances // of this test run concurrently or else if this port is already bound by // another socket. serverPort := 12345 serverHost := "127.0.0.1" serverAddr := serverHost + ":" + strconv.Itoa(serverPort) // override the upid.Host with this listener IP addr := net.ParseIP("0.0.0.0") // setup receiver (server) process uPid, err := upid.Parse(fmt.Sprintf("%s@%s", serverId, serverAddr)) assert.NoError(t, err) receiver := NewHTTPTransporter(*uPid, addr) err = receiver.listen() assert.NoError(t, err) if receiver.upid.Host != "127.0.0.1" { t.Fatalf("reciever.upid.Host was expected to return %s, got %s\n", serverHost, receiver.upid.Host) } if receiver.upid.Port != strconv.Itoa(serverPort) { t.Fatalf("receiver.upid.Port was expected to return %d, got %s\n", serverPort, receiver.upid.Port) } }
func TestSlaveHealthCheckerPartitonedSlave(t *testing.T) { s := newPartitionedServer(5, 9) ts := httptest.NewUnstartedServer(s) ts.Start() defer ts.Close() t.Log("test server listening on", ts.Listener.Addr()) upid, err := upid.Parse(fmt.Sprintf("slave@%s", ts.Listener.Addr().String())) assert.NoError(t, err) checker := NewSlaveHealthChecker(upid, 10, time.Millisecond*10, time.Millisecond*10) ch := checker.Start() defer func() { checker.Stop() <-checker.stop }() select { case <-time.After(2 * time.Second): actual := atomic.LoadInt32(&checker.continuousUnhealthyCount) assert.EqualValues(t, 0, actual, "expected 0 unhealthy counts instead of %d", actual) case <-ch: t.Fatal("Shouldn't get unhealthy notification") } }
func TestSchedCacheContainsOffer(t *testing.T) { cache := newSchedCache() offer01 := createTestOffer("01") pid01, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) offer02 := createTestOffer("02") pid02, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) cache.putOffer(offer01, pid01) cache.putOffer(offer02, pid02) assert.True(t, cache.containsOffer(util.NewOfferID("test-offer-01"))) assert.True(t, cache.containsOffer(util.NewOfferID("test-offer-02"))) assert.False(t, cache.containsOffer(util.NewOfferID("test-offer-05"))) }
func (driver *MesosSchedulerDriver) resourcesOffered(from *upid.UPID, pbMsg proto.Message) { log.V(2).Infoln("Handling resource offers.") msg := pbMsg.(*mesos.ResourceOffersMessage) if driver.status == mesos.Status_DRIVER_ABORTED { log.Infoln("Ignoring ResourceOffersMessage, the driver is aborted!") return } if !driver.connected { log.Infoln("Ignoring ResourceOffersMessage, the driver is not connected!") return } pidStrings := msg.GetPids() if len(pidStrings) != len(msg.Offers) { log.Errorln("Ignoring offers, Offer count does not match Slave PID count.") return } for i, offer := range msg.Offers { if pid, err := upid.Parse(pidStrings[i]); err == nil { driver.cache.putOffer(offer, pid) log.V(2).Infof("Cached offer %s from SlavePID %s", offer.Id.GetValue(), pid) } else { log.Warningf("Failed to parse offer PID '%v': %v", pid, err) } } driver.withScheduler(func(s Scheduler) { s.ResourceOffers(driver, msg.Offers) }) }
func TestSlaveHealthCheckerFailedOnBlockedSlave(t *testing.T) { s := newBlockedServer(5) ts := httptest.NewUnstartedServer(s) ts.Start() defer ts.Close() upid, err := upid.Parse(fmt.Sprintf("slave@%s", ts.Listener.Addr().String())) assert.NoError(t, err) checker := NewSlaveHealthChecker(upid, 10, time.Millisecond*10, time.Millisecond*10) ch := checker.Start() defer checker.Stop() select { case <-time.After(time.Second): s.stop() t.Fatal("timeout") case <-ch: s.stop() assert.True(t, atomic.LoadInt32(&s.th.cnt) > 10) } // TODO(jdef) hack: this sucks, but there's a data race in httptest's handler when Close() // and ServeHTTP() are invoked (WaitGroup DATA RACE). Sleeping here to attempt to avoid that. time.Sleep(5 * time.Second) }
func TestTransporter_DiscardedSend(t *testing.T) { serverId := "testserver" // setup mesos client-side protoMsg := testmessage.GenerateSmallMessage() msgName := getMessageName(protoMsg) msg := &Message{ Name: msgName, ProtoMessage: protoMsg, } requestURI := fmt.Sprintf("/%s/%s", serverId, msgName) // setup server-side msgReceived := make(chan struct{}) srv := makeMockServer(requestURI, func(rsp http.ResponseWriter, req *http.Request) { close(msgReceived) time.Sleep(2 * time.Second) // long enough that we should be able to stop it }) defer srv.Close() toUpid, err := upid.Parse(fmt.Sprintf("%s@%s", serverId, srv.Listener.Addr().String())) assert.NoError(t, err) // make transport call. transport := NewHTTPTransporter(upid.UPID{ID: "mesos1", Host: "localhost"}, nil) _, errch := transport.Start() defer transport.Stop(false) msg.UPID = toUpid senderr := make(chan struct{}) go func() { defer close(senderr) err = transport.Send(context.TODO(), msg) assert.NotNil(t, err) assert.Equal(t, discardOnStopError, err) }() // wait for message to be received select { case <-time.After(2 * time.Second): t.Fatalf("timed out waiting for message receipt") return case <-msgReceived: transport.Stop(false) case err := <-errch: if err != nil { t.Fatalf(err.Error()) return } } // wait for send() to process discarded-error select { case <-time.After(5 * time.Second): t.Fatalf("timed out waiting for aborted send") return case <-senderr: // continue } }
func TestSchedulerDriverNew_WithPid(t *testing.T) { masterAddr := "[email protected]:5050" mUpid, err := upid.Parse(masterAddr) assert.NoError(t, err) driver := newTestSchedulerDriver(t, driverConfig(NewMockScheduler(), &mesos.FrameworkInfo{}, masterAddr, nil)) driver.handleMasterChanged(driver.self, &mesos.InternalMasterChangeDetected{Master: &mesos.MasterInfo{Pid: proto.String(mUpid.String())}}) assert.True(t, driver.masterPid.Equal(mUpid), fmt.Sprintf("expected upid %+v instead of %+v", mUpid, driver.masterPid)) assert.NoError(t, err) }
func TestSchedCacheGetOffer(t *testing.T) { cache := newSchedCache() offer01 := createTestOffer("01") pid01, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) offer02 := createTestOffer("02") pid02, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) cache.putOffer(offer01, pid01) cache.putOffer(offer02, pid02) cachedOffer01 := cache.getOffer(util.NewOfferID("test-offer-01")).offer cachedOffer02 := cache.getOffer(util.NewOfferID("test-offer-02")).offer assert.NotEqual(t, offer01, cachedOffer02) assert.Equal(t, offer01, cachedOffer01) assert.Equal(t, offer02, cachedOffer02) }
func TestSchedCacheGetSlavePid(t *testing.T) { cache := newSchedCache() pid01, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) pid02, err := upid.Parse("[email protected]:5050") assert.NoError(t, err) cache.putSlavePid(util.NewSlaveID("slave01"), pid01) cache.putSlavePid(util.NewSlaveID("slave02"), pid02) cachedSlavePid1 := cache.getSlavePid(util.NewSlaveID("slave01")) cachedSlavePid2 := cache.getSlavePid(util.NewSlaveID("slave02")) assert.NotNil(t, cachedSlavePid1) assert.NotNil(t, cachedSlavePid2) assert.True(t, pid01.Equal(cachedSlavePid1)) assert.True(t, pid02.Equal(cachedSlavePid2)) assert.False(t, pid01.Equal(cachedSlavePid2)) }
func NewMockSlaveHttpServer(t *testing.T, handler func(rsp http.ResponseWriter, req *http.Request)) *MockMesosHttpServer { h, lock := guardedHandler(http.HandlerFunc(handler)) server := httptest.NewServer(h) assert.NotNil(t, server) addr := server.Listener.Addr().String() pid, err := upid.Parse("slave(1)@" + addr) assert.NoError(t, err) assert.NotNil(t, pid) assert.NoError(t, os.Setenv("MESOS_SLAVE_PID", pid.String())) assert.NoError(t, os.Setenv("MESOS_SLAVE_ID", "test-slave-001")) log.Infoln("Created test Slave http server with PID", pid.String()) return &MockMesosHttpServer{PID: pid, Addr: addr, server: server, t: t, lock: lock} }
func (suite *SchedulerTestSuite) TestSchdulerDriverLaunchTasksWithError() { sched := NewMockScheduler() sched.On("StatusUpdate").Return(nil) sched.On("Error").Return() msgr := mockedMessenger() driver := newTestSchedulerDriver(suite.T(), driverConfigMessenger(sched, suite.framework, suite.master, nil, msgr)) driver.dispatch = func(_ context.Context, _ *upid.UPID, _ proto.Message) error { return fmt.Errorf("Unable to send message") } go func() { driver.Run() }() <-driver.started driver.setConnected(true) // simulated suite.True(driver.Running()) // setup an offer offer := util.NewOffer( util.NewOfferID("test-offer-001"), suite.framework.Id, util.NewSlaveID("test-slave-001"), "test-slave(1)@localhost:5050", ) pid, err := upid.Parse("test-slave(1)@localhost:5050") suite.NoError(err) driver.cache.putOffer(offer, pid) // launch task task := util.NewTaskInfo( "simple-task", util.NewTaskID("simpe-task-1"), util.NewSlaveID("test-slave-001"), []*mesos.Resource{util.NewScalarResource("mem", 400)}, ) task.Command = util.NewCommandInfo("pwd") task.Executor = util.NewExecutorInfo(util.NewExecutorID("test-exec"), task.Command) tasks := []*mesos.TaskInfo{task} stat, err := driver.LaunchTasks( []*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{}, ) suite.Equal(mesos.Status_DRIVER_RUNNING, stat) suite.Error(err) }
func TestTransporterSend(t *testing.T) { idreg := regexp.MustCompile(`[A-Za-z0-9_\-]+@[A-Za-z0-9_\-\.]+:[0-9]+`) serverId := "testserver" // setup mesos client-side protoMsg := testmessage.GenerateSmallMessage() msgName := getMessageName(protoMsg) msg := &Message{ Name: msgName, ProtoMessage: protoMsg, } requestURI := fmt.Sprintf("/%s/%s", serverId, msgName) // setup server-side msgReceived := make(chan struct{}) srv := makeMockServer(requestURI, func(rsp http.ResponseWriter, req *http.Request) { defer close(msgReceived) from := req.Header.Get("Libprocess-From") assert.NotEmpty(t, from) assert.True(t, idreg.MatchString(from), fmt.Sprintf("regexp failed for '%v'", from)) }) defer srv.Close() toUpid, err := upid.Parse(fmt.Sprintf("%s@%s", serverId, srv.Listener.Addr().String())) assert.NoError(t, err) // make transport call. transport := NewHTTPTransporter(upid.UPID{ID: "mesos1", Host: "localhost"}, nil) _, errch := transport.Start() defer transport.Stop(false) msg.UPID = toUpid err = transport.Send(context.TODO(), msg) assert.NoError(t, err) select { case <-time.After(2 * time.Second): t.Fatalf("timed out waiting for message receipt") case <-msgReceived: case err := <-errch: if err != nil { t.Fatalf(err.Error()) } } }
func TestSlaveHealthCheckerFailedOnErrorStatusSlave(t *testing.T) { s := newErrorStatusServer(5) ts := httptest.NewUnstartedServer(s) ts.Start() defer ts.Close() upid, err := upid.Parse(fmt.Sprintf("slave@%s", ts.Listener.Addr().String())) assert.NoError(t, err) checker := NewSlaveHealthChecker(upid, 10, time.Millisecond*10, time.Millisecond*10) ch := checker.Start() defer checker.Stop() select { case <-time.After(time.Second): t.Fatal("timeout") case <-ch: assert.True(t, atomic.LoadInt32(&s.th.cnt) > 10) } }
func TestSlaveHealthCheckerSucceed(t *testing.T) { s := new(goodServer) ts := httptest.NewUnstartedServer(s) ts.Start() defer ts.Close() upid, err := upid.Parse(fmt.Sprintf("slave@%s", ts.Listener.Addr().String())) assert.NoError(t, err) checker := NewSlaveHealthChecker(upid, 10, time.Millisecond*10, time.Millisecond*10) ch := checker.Start() defer checker.Stop() select { case <-time.After(time.Second): assert.EqualValues(t, 0, atomic.LoadInt32(&checker.continuousUnhealthyCount)) case <-ch: t.Fatal("Shouldn't get unhealthy notification") } }
func (driver *MesosExecutorDriver) parseEnviroments() error { var value string value = os.Getenv("MESOS_LOCAL") if len(value) > 0 { driver.local = true } value = os.Getenv("MESOS_SLAVE_PID") if len(value) == 0 { return fmt.Errorf("Cannot find MESOS_SLAVE_PID in the environment") } upid, err := upid.Parse(value) if err != nil { log.Errorf("Cannot parse UPID %v\n", err) return err } driver.slaveUPID = upid value = os.Getenv("MESOS_SLAVE_ID") driver.slaveID = &mesosproto.SlaveID{Value: proto.String(value)} value = os.Getenv("MESOS_FRAMEWORK_ID") driver.frameworkID = &mesosproto.FrameworkID{Value: proto.String(value)} value = os.Getenv("MESOS_EXECUTOR_ID") driver.executorID = &mesosproto.ExecutorID{Value: proto.String(value)} value = os.Getenv("MESOS_DIRECTORY") if len(value) > 0 { driver.workDir = value } value = os.Getenv("MESOS_CHECKPOINT") if value == "1" { driver.checkpoint = true } // TODO(yifan): Parse the duration. For now just use default. return nil }
func BenchmarkMessengerSendMixedMessage(b *testing.B) { messages := generateMixedMessages(1000) wg := new(sync.WaitGroup) wg.Add(b.N) srv := runTestServer(b, wg) defer srv.Close() upid2, err := upid.Parse(fmt.Sprintf("testserver@%s", srv.Listener.Addr().String())) assert.NoError(b, err) m1 := NewHttp(upid.UPID{ID: "mesos1", Host: "localhost"}) assert.NoError(b, m1.Start()) defer m1.Stop() b.ResetTimer() for i := 0; i < b.N; i++ { m1.Send(context.TODO(), upid2, messages[i%1000]) } wg.Wait() b.StopTimer() time.Sleep(2 * time.Second) // allow time for connection cleanup }
func NewMockMasterHttpServer(t *testing.T, handler func(rsp http.ResponseWriter, req *http.Request)) *MockMesosHttpServer { var server *httptest.Server when := make(map[string]http.HandlerFunc) stateHandler := func(rsp http.ResponseWriter, req *http.Request) { if "/state.json" == req.RequestURI { state := fmt.Sprintf(`{ "leader": "master@%v" }`, server.Listener.Addr()) log.V(1).Infof("returning JSON %v", state) io.WriteString(rsp, state) } else if f, found := when[req.RequestURI]; found { f(rsp, req) } else { handler(rsp, req) } } h, lock := guardedHandler(http.HandlerFunc(stateHandler)) server = httptest.NewServer(h) assert.NotNil(t, server) addr := server.Listener.Addr().String() pid, err := upid.Parse("master@" + addr) assert.NoError(t, err) assert.NotNil(t, pid) log.Infoln("Created test Master http server with PID", pid.String()) return &MockMesosHttpServer{PID: pid, Addr: addr, server: server, t: t, when: when, lock: lock} }
// lead master detection callback. func (driver *MesosSchedulerDriver) handleMasterChanged(from *upid.UPID, pbMsg proto.Message) { if driver.status == mesos.Status_DRIVER_ABORTED { log.Info("Ignoring master change because the driver is aborted.") return } else if !from.Equal(driver.self) { log.Errorf("ignoring master changed message received from upid '%v'", from) return } // Reconnect every time a master is detected. if driver.connected { log.V(3).Info("Disconnecting scheduler.") driver.masterPid = nil driver.withScheduler(func(s Scheduler) { s.Disconnected(driver) }) } msg := pbMsg.(*mesos.InternalMasterChangeDetected) master := msg.Master driver.connected = false driver.authenticated = false if master != nil { log.Infof("New master %s detected\n", master.GetPid()) pid, err := upid.Parse(master.GetPid()) if err != nil { panic("Unable to parse Master's PID value.") // this should not happen. } driver.masterPid = pid // save for downstream ops. driver.tryAuthentication() } else { log.Infoln("No master detected.") } }
log "github.com/golang/glog" ) var ( pluginLock sync.Mutex plugins = map[string]PluginFactory{} EmptySpecError = errors.New("empty master specification") defaultFactory = PluginFactory(func(spec string) (Master, error) { if len(spec) == 0 { return nil, EmptySpecError } if strings.Index(spec, "@") < 0 { spec = "master@" + spec } if pid, err := upid.Parse(spec); err == nil { return NewStandalone(CreateMasterInfo(pid)), nil } else { return nil, err } }) ) type PluginFactory func(string) (Master, error) // associates a plugin implementation with a Master specification prefix. // packages that provide plugins are expected to invoke this func within // their init() implementation. schedulers that wish to support plugins may // anonymously import ("_") a package the auto-registers said plugins. func Register(prefix string, f PluginFactory) error { if prefix == "" {