func TestSchedCachePutSlavePid(t *testing.T) {
cache := newSchedCache()
pid01, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
pid02, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
pid03, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
cache.putSlavePid(util.NewSlaveID("slave01"), pid01)
cache.putSlavePid(util.NewSlaveID("slave02"), pid02)
cache.putSlavePid(util.NewSlaveID("slave03"), pid03)
assert.Equal(t, len(cache.savedSlavePids), 3)
cachedSlavePid1, ok := cache.savedSlavePids["slave01"]
assert.True(t, ok)
cachedSlavePid2, ok := cache.savedSlavePids["slave02"]
assert.True(t, ok)
cachedSlavePid3, ok := cache.savedSlavePids["slave03"]
assert.True(t, ok)
assert.True(t, cachedSlavePid1.Equal(pid01))
assert.True(t, cachedSlavePid2.Equal(pid02))
assert.True(t, cachedSlavePid3.Equal(pid03))
}
func TestSchedCachePutOffer(t *testing.T) {
cache := newSchedCache()
offer01 := createTestOffer("01")
pid01, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
cache.putOffer(offer01, pid01)
offer02 := createTestOffer("02")
pid02, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
cache.putOffer(offer02, pid02)
assert.Equal(t, len(cache.savedOffers), 2)
cachedOffer1, ok := cache.savedOffers["test-offer-01"]
assert.True(t, ok)
cachedOffer2, ok := cache.savedOffers["test-offer-02"]
assert.True(t, ok)
assert.NotNil(t, cachedOffer1.offer)
assert.Equal(t, "test-offer-01", cachedOffer1.offer.Id.GetValue())
assert.NotNil(t, cachedOffer2.offer)
assert.Equal(t, "test-offer-02", cachedOffer2.offer.Id.GetValue())
assert.NotNil(t, cachedOffer1.slavePid)
assert.Equal(t, "[email protected]:5050", cachedOffer1.slavePid.String())
assert.NotNil(t, cachedOffer2.slavePid)
assert.Equal(t, "[email protected]:5050", cachedOffer2.slavePid.String())
}
func getLibprocessFrom(r *http.Request) (*upid.UPID, error) {
if r.Method != "POST" {
return nil, fmt.Errorf("Not a POST request")
}
if agent, ok := parseLibprocessAgent(r.Header); ok {
return upid.Parse(agent)
}
lf, ok := r.Header["Libprocess-From"]
if ok {
// TODO(yifan): Just take the first field for now.
return upid.Parse(lf[0])
}
return nil, fmt.Errorf("Cannot find 'User-Agent' or 'Libprocess-From'")
}
func TestSchedCacheContainsSlavePid(t *testing.T) {
cache := newSchedCache()
pid01, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
pid02, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
cache.putSlavePid(util.NewSlaveID("slave01"), pid01)
cache.putSlavePid(util.NewSlaveID("slave02"), pid02)
assert.True(t, cache.containsSlavePid(util.NewSlaveID("slave01")))
assert.True(t, cache.containsSlavePid(util.NewSlaveID("slave02")))
assert.False(t, cache.containsSlavePid(util.NewSlaveID("slave05")))
}
// assumes that address is in host:port format
func (s *Standalone) _fetchPid(ctx context.Context, address string) (*upid.UPID, error) {
//TODO(jdef) need SSL support
uri := fmt.Sprintf("http://%s/state.json", address)
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return nil, err
}
var pid *upid.UPID
err = s.httpDo(ctx, req, func(res *http.Response, err error) error {
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return fmt.Errorf("HTTP request failed with code %d: %v", res.StatusCode, res.Status)
}
blob, err1 := ioutil.ReadAll(res.Body)
if err1 != nil {
return err1
}
log.V(3).Infof("Got mesos state, content length %v", len(blob))
type State struct {
Leader string `json:"leader"` // ex: master(1)@10.22.211.18:5050
}
state := &State{}
err = json.Unmarshal(blob, state)
if err != nil {
return err
}
pid, err = upid.Parse(state.Leader)
return err
})
return pid, err
}
func TestMutatedHostUPid(t *testing.T) {
serverId := "testserver"
// NOTE(tsenart): This static port can cause conflicts if multiple instances
// of this test run concurrently or else if this port is already bound by
// another socket.
serverPort := 12345
serverHost := "127.0.0.1"
serverAddr := serverHost + ":" + strconv.Itoa(serverPort)
// override the upid.Host with this listener IP
addr := net.ParseIP("0.0.0.0")
// setup receiver (server) process
uPid, err := upid.Parse(fmt.Sprintf("%s@%s", serverId, serverAddr))
assert.NoError(t, err)
receiver := NewHTTPTransporter(*uPid, addr)
err = receiver.listen()
assert.NoError(t, err)
if receiver.upid.Host != "127.0.0.1" {
t.Fatalf("reciever.upid.Host was expected to return %s, got %s\n", serverHost, receiver.upid.Host)
}
if receiver.upid.Port != strconv.Itoa(serverPort) {
t.Fatalf("receiver.upid.Port was expected to return %d, got %s\n", serverPort, receiver.upid.Port)
}
}
func TestSlaveHealthCheckerPartitonedSlave(t *testing.T) {
s := newPartitionedServer(5, 9)
ts := httptest.NewUnstartedServer(s)
ts.Start()
defer ts.Close()
t.Log("test server listening on", ts.Listener.Addr())
upid, err := upid.Parse(fmt.Sprintf("slave@%s", ts.Listener.Addr().String()))
assert.NoError(t, err)
checker := NewSlaveHealthChecker(upid, 10, time.Millisecond*10, time.Millisecond*10)
ch := checker.Start()
defer func() {
checker.Stop()
<-checker.stop
}()
select {
case <-time.After(2 * time.Second):
actual := atomic.LoadInt32(&checker.continuousUnhealthyCount)
assert.EqualValues(t, 0, actual, "expected 0 unhealthy counts instead of %d", actual)
case <-ch:
t.Fatal("Shouldn't get unhealthy notification")
}
}
func TestSchedCacheContainsOffer(t *testing.T) {
cache := newSchedCache()
offer01 := createTestOffer("01")
pid01, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
offer02 := createTestOffer("02")
pid02, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
cache.putOffer(offer01, pid01)
cache.putOffer(offer02, pid02)
assert.True(t, cache.containsOffer(util.NewOfferID("test-offer-01")))
assert.True(t, cache.containsOffer(util.NewOfferID("test-offer-02")))
assert.False(t, cache.containsOffer(util.NewOfferID("test-offer-05")))
}
func (driver *MesosSchedulerDriver) resourcesOffered(from *upid.UPID, pbMsg proto.Message) {
log.V(2).Infoln("Handling resource offers.")
msg := pbMsg.(*mesos.ResourceOffersMessage)
if driver.status == mesos.Status_DRIVER_ABORTED {
log.Infoln("Ignoring ResourceOffersMessage, the driver is aborted!")
return
}
if !driver.connected {
log.Infoln("Ignoring ResourceOffersMessage, the driver is not connected!")
return
}
pidStrings := msg.GetPids()
if len(pidStrings) != len(msg.Offers) {
log.Errorln("Ignoring offers, Offer count does not match Slave PID count.")
return
}
for i, offer := range msg.Offers {
if pid, err := upid.Parse(pidStrings[i]); err == nil {
driver.cache.putOffer(offer, pid)
log.V(2).Infof("Cached offer %s from SlavePID %s", offer.Id.GetValue(), pid)
} else {
log.Warningf("Failed to parse offer PID '%v': %v", pid, err)
}
}
driver.withScheduler(func(s Scheduler) { s.ResourceOffers(driver, msg.Offers) })
}
func TestSlaveHealthCheckerFailedOnBlockedSlave(t *testing.T) {
s := newBlockedServer(5)
ts := httptest.NewUnstartedServer(s)
ts.Start()
defer ts.Close()
upid, err := upid.Parse(fmt.Sprintf("slave@%s", ts.Listener.Addr().String()))
assert.NoError(t, err)
checker := NewSlaveHealthChecker(upid, 10, time.Millisecond*10, time.Millisecond*10)
ch := checker.Start()
defer checker.Stop()
select {
case <-time.After(time.Second):
s.stop()
t.Fatal("timeout")
case <-ch:
s.stop()
assert.True(t, atomic.LoadInt32(&s.th.cnt) > 10)
}
// TODO(jdef) hack: this sucks, but there's a data race in httptest's handler when Close()
// and ServeHTTP() are invoked (WaitGroup DATA RACE). Sleeping here to attempt to avoid that.
time.Sleep(5 * time.Second)
}
func TestTransporter_DiscardedSend(t *testing.T) {
serverId := "testserver"
// setup mesos client-side
protoMsg := testmessage.GenerateSmallMessage()
msgName := getMessageName(protoMsg)
msg := &Message{
Name: msgName,
ProtoMessage: protoMsg,
}
requestURI := fmt.Sprintf("/%s/%s", serverId, msgName)
// setup server-side
msgReceived := make(chan struct{})
srv := makeMockServer(requestURI, func(rsp http.ResponseWriter, req *http.Request) {
close(msgReceived)
time.Sleep(2 * time.Second) // long enough that we should be able to stop it
})
defer srv.Close()
toUpid, err := upid.Parse(fmt.Sprintf("%s@%s", serverId, srv.Listener.Addr().String()))
assert.NoError(t, err)
// make transport call.
transport := NewHTTPTransporter(upid.UPID{ID: "mesos1", Host: "localhost"}, nil)
_, errch := transport.Start()
defer transport.Stop(false)
msg.UPID = toUpid
senderr := make(chan struct{})
go func() {
defer close(senderr)
err = transport.Send(context.TODO(), msg)
assert.NotNil(t, err)
assert.Equal(t, discardOnStopError, err)
}()
// wait for message to be received
select {
case <-time.After(2 * time.Second):
t.Fatalf("timed out waiting for message receipt")
return
case <-msgReceived:
transport.Stop(false)
case err := <-errch:
if err != nil {
t.Fatalf(err.Error())
return
}
}
// wait for send() to process discarded-error
select {
case <-time.After(5 * time.Second):
t.Fatalf("timed out waiting for aborted send")
return
case <-senderr: // continue
}
}
func TestSchedulerDriverNew_WithPid(t *testing.T) {
masterAddr := "[email protected]:5050"
mUpid, err := upid.Parse(masterAddr)
assert.NoError(t, err)
driver := newTestSchedulerDriver(t, driverConfig(NewMockScheduler(), &mesos.FrameworkInfo{}, masterAddr, nil))
driver.handleMasterChanged(driver.self, &mesos.InternalMasterChangeDetected{Master: &mesos.MasterInfo{Pid: proto.String(mUpid.String())}})
assert.True(t, driver.masterPid.Equal(mUpid), fmt.Sprintf("expected upid %+v instead of %+v", mUpid, driver.masterPid))
assert.NoError(t, err)
}
func TestSchedCacheGetOffer(t *testing.T) {
cache := newSchedCache()
offer01 := createTestOffer("01")
pid01, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
offer02 := createTestOffer("02")
pid02, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
cache.putOffer(offer01, pid01)
cache.putOffer(offer02, pid02)
cachedOffer01 := cache.getOffer(util.NewOfferID("test-offer-01")).offer
cachedOffer02 := cache.getOffer(util.NewOfferID("test-offer-02")).offer
assert.NotEqual(t, offer01, cachedOffer02)
assert.Equal(t, offer01, cachedOffer01)
assert.Equal(t, offer02, cachedOffer02)
}
func TestSchedCacheGetSlavePid(t *testing.T) {
cache := newSchedCache()
pid01, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
pid02, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
cache.putSlavePid(util.NewSlaveID("slave01"), pid01)
cache.putSlavePid(util.NewSlaveID("slave02"), pid02)
cachedSlavePid1 := cache.getSlavePid(util.NewSlaveID("slave01"))
cachedSlavePid2 := cache.getSlavePid(util.NewSlaveID("slave02"))
assert.NotNil(t, cachedSlavePid1)
assert.NotNil(t, cachedSlavePid2)
assert.True(t, pid01.Equal(cachedSlavePid1))
assert.True(t, pid02.Equal(cachedSlavePid2))
assert.False(t, pid01.Equal(cachedSlavePid2))
}
func NewMockSlaveHttpServer(t *testing.T, handler func(rsp http.ResponseWriter, req *http.Request)) *MockMesosHttpServer {
h, lock := guardedHandler(http.HandlerFunc(handler))
server := httptest.NewServer(h)
assert.NotNil(t, server)
addr := server.Listener.Addr().String()
pid, err := upid.Parse("slave(1)@" + addr)
assert.NoError(t, err)
assert.NotNil(t, pid)
assert.NoError(t, os.Setenv("MESOS_SLAVE_PID", pid.String()))
assert.NoError(t, os.Setenv("MESOS_SLAVE_ID", "test-slave-001"))
log.Infoln("Created test Slave http server with PID", pid.String())
return &MockMesosHttpServer{PID: pid, Addr: addr, server: server, t: t, lock: lock}
}
func (suite *SchedulerTestSuite) TestSchdulerDriverLaunchTasksWithError() {
sched := NewMockScheduler()
sched.On("StatusUpdate").Return(nil)
sched.On("Error").Return()
msgr := mockedMessenger()
driver := newTestSchedulerDriver(suite.T(), driverConfigMessenger(sched, suite.framework, suite.master, nil, msgr))
driver.dispatch = func(_ context.Context, _ *upid.UPID, _ proto.Message) error {
return fmt.Errorf("Unable to send message")
}
go func() {
driver.Run()
}()
<-driver.started
driver.setConnected(true) // simulated
suite.True(driver.Running())
// setup an offer
offer := util.NewOffer(
util.NewOfferID("test-offer-001"),
suite.framework.Id,
util.NewSlaveID("test-slave-001"),
"test-slave(1)@localhost:5050",
)
pid, err := upid.Parse("test-slave(1)@localhost:5050")
suite.NoError(err)
driver.cache.putOffer(offer, pid)
// launch task
task := util.NewTaskInfo(
"simple-task",
util.NewTaskID("simpe-task-1"),
util.NewSlaveID("test-slave-001"),
[]*mesos.Resource{util.NewScalarResource("mem", 400)},
)
task.Command = util.NewCommandInfo("pwd")
task.Executor = util.NewExecutorInfo(util.NewExecutorID("test-exec"), task.Command)
tasks := []*mesos.TaskInfo{task}
stat, err := driver.LaunchTasks(
[]*mesos.OfferID{offer.Id},
tasks,
&mesos.Filters{},
)
suite.Equal(mesos.Status_DRIVER_RUNNING, stat)
suite.Error(err)
}
func TestTransporterSend(t *testing.T) {
idreg := regexp.MustCompile(`[A-Za-z0-9_\-]+@[A-Za-z0-9_\-\.]+:[0-9]+`)
serverId := "testserver"
// setup mesos client-side
protoMsg := testmessage.GenerateSmallMessage()
msgName := getMessageName(protoMsg)
msg := &Message{
Name: msgName,
ProtoMessage: protoMsg,
}
requestURI := fmt.Sprintf("/%s/%s", serverId, msgName)
// setup server-side
msgReceived := make(chan struct{})
srv := makeMockServer(requestURI, func(rsp http.ResponseWriter, req *http.Request) {
defer close(msgReceived)
from := req.Header.Get("Libprocess-From")
assert.NotEmpty(t, from)
assert.True(t, idreg.MatchString(from), fmt.Sprintf("regexp failed for '%v'", from))
})
defer srv.Close()
toUpid, err := upid.Parse(fmt.Sprintf("%s@%s", serverId, srv.Listener.Addr().String()))
assert.NoError(t, err)
// make transport call.
transport := NewHTTPTransporter(upid.UPID{ID: "mesos1", Host: "localhost"}, nil)
_, errch := transport.Start()
defer transport.Stop(false)
msg.UPID = toUpid
err = transport.Send(context.TODO(), msg)
assert.NoError(t, err)
select {
case <-time.After(2 * time.Second):
t.Fatalf("timed out waiting for message receipt")
case <-msgReceived:
case err := <-errch:
if err != nil {
t.Fatalf(err.Error())
}
}
}
func TestSlaveHealthCheckerFailedOnErrorStatusSlave(t *testing.T) {
s := newErrorStatusServer(5)
ts := httptest.NewUnstartedServer(s)
ts.Start()
defer ts.Close()
upid, err := upid.Parse(fmt.Sprintf("slave@%s", ts.Listener.Addr().String()))
assert.NoError(t, err)
checker := NewSlaveHealthChecker(upid, 10, time.Millisecond*10, time.Millisecond*10)
ch := checker.Start()
defer checker.Stop()
select {
case <-time.After(time.Second):
t.Fatal("timeout")
case <-ch:
assert.True(t, atomic.LoadInt32(&s.th.cnt) > 10)
}
}
func TestSlaveHealthCheckerSucceed(t *testing.T) {
s := new(goodServer)
ts := httptest.NewUnstartedServer(s)
ts.Start()
defer ts.Close()
upid, err := upid.Parse(fmt.Sprintf("slave@%s", ts.Listener.Addr().String()))
assert.NoError(t, err)
checker := NewSlaveHealthChecker(upid, 10, time.Millisecond*10, time.Millisecond*10)
ch := checker.Start()
defer checker.Stop()
select {
case <-time.After(time.Second):
assert.EqualValues(t, 0, atomic.LoadInt32(&checker.continuousUnhealthyCount))
case <-ch:
t.Fatal("Shouldn't get unhealthy notification")
}
}
func (driver *MesosExecutorDriver) parseEnviroments() error {
var value string
value = os.Getenv("MESOS_LOCAL")
if len(value) > 0 {
driver.local = true
}
value = os.Getenv("MESOS_SLAVE_PID")
if len(value) == 0 {
return fmt.Errorf("Cannot find MESOS_SLAVE_PID in the environment")
}
upid, err := upid.Parse(value)
if err != nil {
log.Errorf("Cannot parse UPID %v\n", err)
return err
}
driver.slaveUPID = upid
value = os.Getenv("MESOS_SLAVE_ID")
driver.slaveID = &mesosproto.SlaveID{Value: proto.String(value)}
value = os.Getenv("MESOS_FRAMEWORK_ID")
driver.frameworkID = &mesosproto.FrameworkID{Value: proto.String(value)}
value = os.Getenv("MESOS_EXECUTOR_ID")
driver.executorID = &mesosproto.ExecutorID{Value: proto.String(value)}
value = os.Getenv("MESOS_DIRECTORY")
if len(value) > 0 {
driver.workDir = value
}
value = os.Getenv("MESOS_CHECKPOINT")
if value == "1" {
driver.checkpoint = true
}
// TODO(yifan): Parse the duration. For now just use default.
return nil
}
func BenchmarkMessengerSendMixedMessage(b *testing.B) {
messages := generateMixedMessages(1000)
wg := new(sync.WaitGroup)
wg.Add(b.N)
srv := runTestServer(b, wg)
defer srv.Close()
upid2, err := upid.Parse(fmt.Sprintf("testserver@%s", srv.Listener.Addr().String()))
assert.NoError(b, err)
m1 := NewHttp(upid.UPID{ID: "mesos1", Host: "localhost"})
assert.NoError(b, m1.Start())
defer m1.Stop()
b.ResetTimer()
for i := 0; i < b.N; i++ {
m1.Send(context.TODO(), upid2, messages[i%1000])
}
wg.Wait()
b.StopTimer()
time.Sleep(2 * time.Second) // allow time for connection cleanup
}
func NewMockMasterHttpServer(t *testing.T, handler func(rsp http.ResponseWriter, req *http.Request)) *MockMesosHttpServer {
var server *httptest.Server
when := make(map[string]http.HandlerFunc)
stateHandler := func(rsp http.ResponseWriter, req *http.Request) {
if "/state.json" == req.RequestURI {
state := fmt.Sprintf(`{ "leader": "master@%v" }`, server.Listener.Addr())
log.V(1).Infof("returning JSON %v", state)
io.WriteString(rsp, state)
} else if f, found := when[req.RequestURI]; found {
f(rsp, req)
} else {
handler(rsp, req)
}
}
h, lock := guardedHandler(http.HandlerFunc(stateHandler))
server = httptest.NewServer(h)
assert.NotNil(t, server)
addr := server.Listener.Addr().String()
pid, err := upid.Parse("master@" + addr)
assert.NoError(t, err)
assert.NotNil(t, pid)
log.Infoln("Created test Master http server with PID", pid.String())
return &MockMesosHttpServer{PID: pid, Addr: addr, server: server, t: t, when: when, lock: lock}
}
// lead master detection callback.
func (driver *MesosSchedulerDriver) handleMasterChanged(from *upid.UPID, pbMsg proto.Message) {
if driver.status == mesos.Status_DRIVER_ABORTED {
log.Info("Ignoring master change because the driver is aborted.")
return
} else if !from.Equal(driver.self) {
log.Errorf("ignoring master changed message received from upid '%v'", from)
return
}
// Reconnect every time a master is detected.
if driver.connected {
log.V(3).Info("Disconnecting scheduler.")
driver.masterPid = nil
driver.withScheduler(func(s Scheduler) { s.Disconnected(driver) })
}
msg := pbMsg.(*mesos.InternalMasterChangeDetected)
master := msg.Master
driver.connected = false
driver.authenticated = false
if master != nil {
log.Infof("New master %s detected\n", master.GetPid())
pid, err := upid.Parse(master.GetPid())
if err != nil {
panic("Unable to parse Master's PID value.") // this should not happen.
}
driver.masterPid = pid // save for downstream ops.
driver.tryAuthentication()
} else {
log.Infoln("No master detected.")
}
}
log "github.com/golang/glog"
)
var (
pluginLock sync.Mutex
plugins = map[string]PluginFactory{}
EmptySpecError = errors.New("empty master specification")
defaultFactory = PluginFactory(func(spec string) (Master, error) {
if len(spec) == 0 {
return nil, EmptySpecError
}
if strings.Index(spec, "@") < 0 {
spec = "master@" + spec
}
if pid, err := upid.Parse(spec); err == nil {
return NewStandalone(CreateMasterInfo(pid)), nil
} else {
return nil, err
}
})
)
type PluginFactory func(string) (Master, error)
// associates a plugin implementation with a Master specification prefix.
// packages that provide plugins are expected to invoke this func within
// their init() implementation. schedulers that wish to support plugins may
// anonymously import ("_") a package the auto-registers said plugins.
func Register(prefix string, f PluginFactory) error {
if prefix == "" {