//test that when a slave is lost we remove all offers
func TestSlave_Lost(t *testing.T) {
assert := assert.New(t)
//
testFramework := &framework{
offers: offers.CreateRegistry(offers.RegistryConfig{
Compat: func(o *mesos.Offer) bool {
return true
},
// remember expired offers so that we can tell if a previously scheduler offer relies on one
LingerTTL: schedcfg.DefaultOfferLingerTTL,
TTL: schedcfg.DefaultOfferTTL,
ListenerDelay: schedcfg.DefaultListenerDelay,
}),
slaveHostNames: newSlaveRegistry(),
sched: mockScheduler(),
}
hostname := "h1"
offer1 := &mesos.Offer{Id: util.NewOfferID("test1"), Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)}
offers1 := []*mesos.Offer{offer1}
testFramework.ResourceOffers(nil, offers1)
offer2 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)}
offers2 := []*mesos.Offer{offer2}
testFramework.ResourceOffers(nil, offers2)
//add another offer from different slaveID
hostname2 := "h2"
offer3 := &mesos.Offer{Id: util.NewOfferID("test3"), Hostname: &hostname2, SlaveId: util.NewSlaveID(hostname2)}
offers3 := []*mesos.Offer{offer3}
testFramework.ResourceOffers(nil, offers3)
//test precondition
assert.Equal(3, getNumberOffers(testFramework.offers))
assert.Equal(2, len(testFramework.slaveHostNames.SlaveIDs()))
//remove first slave
testFramework.SlaveLost(nil, util.NewSlaveID(hostname))
//offers should be removed
assert.Equal(1, getNumberOffers(testFramework.offers))
//slave hostnames should still be all present
assert.Equal(2, len(testFramework.slaveHostNames.SlaveIDs()))
//remove second slave
testFramework.SlaveLost(nil, util.NewSlaveID(hostname2))
//offers should be removed
assert.Equal(0, getNumberOffers(testFramework.offers))
//slave hostnames should still be all present
assert.Equal(2, len(testFramework.slaveHostNames.SlaveIDs()))
//try to remove non existing slave
testFramework.SlaveLost(nil, util.NewSlaveID("notExist"))
}
//test adding of ressource offer, should be added to offer registry and slavesf
func TestResourceOffer_Add_Rescind(t *testing.T) {
assert := assert.New(t)
testFramework := &framework{
offers: offers.CreateRegistry(offers.RegistryConfig{
Compat: func(o *mesos.Offer) bool {
return true
},
DeclineOffer: func(offerId string) <-chan error {
return proc.ErrorChan(nil)
},
// remember expired offers so that we can tell if a previously scheduler offer relies on one
LingerTTL: schedcfg.DefaultOfferLingerTTL,
TTL: schedcfg.DefaultOfferTTL,
ListenerDelay: schedcfg.DefaultListenerDelay,
}),
slaveHostNames: newSlaveRegistry(),
sched: mockScheduler(),
}
hostname := "h1"
offerID1 := util.NewOfferID("test1")
offer1 := &mesos.Offer{Id: offerID1, Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)}
offers1 := []*mesos.Offer{offer1}
testFramework.ResourceOffers(nil, offers1)
assert.Equal(1, getNumberOffers(testFramework.offers))
//check slave hostname
assert.Equal(1, len(testFramework.slaveHostNames.SlaveIDs()))
//add another offer
hostname2 := "h2"
offer2 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname2, SlaveId: util.NewSlaveID(hostname2)}
offers2 := []*mesos.Offer{offer2}
testFramework.ResourceOffers(nil, offers2)
assert.Equal(2, getNumberOffers(testFramework.offers))
//check slave hostnames
assert.Equal(2, len(testFramework.slaveHostNames.SlaveIDs()))
//next whether offers can be rescinded
testFramework.OfferRescinded(nil, offerID1)
assert.Equal(1, getNumberOffers(testFramework.offers))
//next whether offers can be rescinded
testFramework.OfferRescinded(nil, util.NewOfferID("test2"))
//walk offers again and check it is removed from registry
assert.Equal(0, getNumberOffers(testFramework.offers))
//remove non existing ID
testFramework.OfferRescinded(nil, util.NewOfferID("notExist"))
}
func TestSchedCacheContainsOffer(t *testing.T) {
cache := newSchedCache()
offer01 := createTestOffer("01")
pid01, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
offer02 := createTestOffer("02")
pid02, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
cache.putOffer(offer01, pid01)
cache.putOffer(offer02, pid02)
assert.True(t, cache.containsOffer(util.NewOfferID("test-offer-01")))
assert.True(t, cache.containsOffer(util.NewOfferID("test-offer-02")))
assert.False(t, cache.containsOffer(util.NewOfferID("test-offer-05")))
}
func TestOffer(t *testing.T) {
offer := util.NewOffer(util.NewOfferID("487c73d8-9951-f23c-34bd-8085bfd30c49"), util.NewFrameworkID("20150903-065451-84125888-5050-10715-0053"),
util.NewSlaveID("20150903-065451-84125888-5050-10715-S1"), "slave0")
if Offer(offer) != "slave0#30c49" {
t.Errorf(`util.NewOffer(util.NewOfferID("487c73d8-9951-f23c-34bd-8085bfd30c49"), util.NewFrameworkID("20150903-065451-84125888-5050-10715-0053"), util.NewSlaveID("20150903-065451-84125888-5050-10715-S1"), "slave0") != "slave0#30c49"; actual %s`, Offer(offer))
}
offer.Resources = []*mesos.Resource{util.NewScalarResource("cpus", 4), util.NewScalarResource("mem", 512), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31000, 32000)})}
if Offer(offer) != "slave0#30c49 cpus:4.00 mem:512.00 ports:[31000..32000]" {
t.Errorf("Expected slave0#30c49 cpus:4.00 mem:512.00 ports:[31000..32000]; actual %s", Offer(offer))
}
offer.Attributes = []*mesos.Attribute{&mesos.Attribute{
Name: proto.String("rack"),
Type: mesos.Value_SCALAR.Enum(),
Scalar: &mesos.Value_Scalar{Value: proto.Float64(2)},
}}
if Offer(offer) != "slave0#30c49 cpus:4.00 mem:512.00 ports:[31000..32000] rack:2.00" {
t.Errorf("Expected slave0#30c49 cpus:4.00 mem:512.00 ports:[31000..32000] rack:2.00; actual %s", Offer(offer))
}
offer.Resources = nil
if Offer(offer) != "slave0#30c49 rack:2.00" {
t.Errorf("Expected slave0#30c49 rack:2.00; actual %s", Offer(offer))
}
}
func TestAddOffer(t *testing.T) {
s := newSlave("SID", nil)
assert.Empty(t, s.offers)
assert.True(t, s.empty())
s.addOffer(mesosutil.NewOffer(mesosutil.NewOfferID("ID1"), nil, nil, "hostname1"))
s.addOffer(mesosutil.NewOffer(mesosutil.NewOfferID("ID2"), nil, nil, "hostname1"))
assert.Equal(t, len(s.offers), 2)
assert.Equal(t, len(s.getOffers()), 2)
assert.False(t, s.empty())
s.addOffer(mesosutil.NewOffer(mesosutil.NewOfferID("ID1"), nil, nil, "hostname1"))
assert.Equal(t, len(s.offers), 2)
assert.Equal(t, len(s.getOffers()), 2)
}
func TestRescind(t *testing.T) {
for i, tt := range []struct {
offers []string
rescinds []string
want int
}{
{[]string{"a"}, []string{"a"}, 0},
{[]string{"a"}, []string{"b"}, 1},
{[]string{}, []string{"a"}, 0},
// with 7 valid offers (2 more than cap), and invalidation of one
// of the non-rejected ones as well as one of the accepted ones,
// we should have 4 offers cached.
{[]string{"a", "b", "c", "d", "e", "f", "g"}, []string{"a", "g"}, 4},
} {
oc := New(5, false)
for _, o := range tt.offers {
oc.Push(newOffer(o, o))
}
for _, r := range tt.rescinds {
oc.Rescind(util.NewOfferID(r))
}
if got := oc.Len(); got != tt.want {
t.Errorf("test #%d: got : %s, want: %s", i, got, tt.want)
}
}
}
func TestRemoveOffer(t *testing.T) {
s := newAgent("SID", nil)
assert.Empty(t, s.offers)
s.addOffer(mesosutil.NewOffer(mesosutil.NewOfferID("ID1"), nil, nil, "hostname1"))
s.addOffer(mesosutil.NewOffer(mesosutil.NewOfferID("ID2"), nil, nil, "hostname1"))
assert.Equal(t, len(s.offers), 2)
assert.Equal(t, len(s.getOffers()), 2)
assert.True(t, s.removeOffer("ID1"))
assert.Equal(t, len(s.offers), 1)
assert.Equal(t, len(s.getOffers()), 1)
assert.False(t, s.removeOffer("ID1"))
}
func Test_gc(t *testing.T) {
oc := New(5, false)
for i := 0; i < 5000; i++ {
oc.Rescind(util.NewOfferID(string(i - 50)))
oc.Push(newOffer(string(i), string(i)))
}
assert.Equal(t, 5, oc.Len())
}
func createTestOffer(idSuffix string) *mesos.Offer {
return util.NewOffer(
util.NewOfferID("test-offer-"+idSuffix),
util.NewFrameworkID("test-framework-"+idSuffix),
util.NewSlaveID("test-slave-"+idSuffix),
"localhost."+idSuffix,
)
}
//test adding of ressource offer, should be added to offer registry and slaves
func TestResourceOffer_Add(t *testing.T) {
assert := assert.New(t)
registrator := &mockRegistrator{cache.NewStore(cache.MetaNamespaceKeyFunc)}
testFramework := &framework{
offers: offers.CreateRegistry(offers.RegistryConfig{
Compat: func(o *mesos.Offer) bool {
return true
},
DeclineOffer: func(offerId string) <-chan error {
return proc.ErrorChan(nil)
},
// remember expired offers so that we can tell if a previously scheduler offer relies on one
LingerTTL: schedcfg.DefaultOfferLingerTTL,
TTL: schedcfg.DefaultOfferTTL,
ListenerDelay: schedcfg.DefaultListenerDelay,
}),
slaveHostNames: newSlaveRegistry(),
nodeRegistrator: registrator,
sched: mockScheduler(),
}
hostname := "h1"
offerID1 := util.NewOfferID("test1")
offer1 := &mesos.Offer{Id: offerID1, Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)}
offers1 := []*mesos.Offer{offer1}
testFramework.ResourceOffers(nil, offers1)
assert.Equal(1, len(registrator.store.List()))
assert.Equal(1, getNumberOffers(testFramework.offers))
//check slave hostname
assert.Equal(1, len(testFramework.slaveHostNames.SlaveIDs()))
//add another offer
hostname2 := "h2"
offer2 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname2, SlaveId: util.NewSlaveID(hostname2)}
offers2 := []*mesos.Offer{offer2}
testFramework.ResourceOffers(nil, offers2)
//check it is stored in registry
assert.Equal(2, getNumberOffers(testFramework.offers))
//check slave hostnames
assert.Equal(2, len(testFramework.slaveHostNames.SlaveIDs()))
}
// New creates a new KubernetesScheduler
func New(config Config) *KubernetesScheduler {
var k *KubernetesScheduler
k = &KubernetesScheduler{
schedcfg: &config.Schedcfg,
RWMutex: new(sync.RWMutex),
executor: config.Executor,
executorGroup: uid.Parse(config.Executor.ExecutorId.GetValue()).Group(),
PodScheduler: config.Scheduler,
client: config.Client,
etcdClient: config.EtcdClient,
failoverTimeout: config.FailoverTimeout,
reconcileInterval: config.ReconcileInterval,
nodeRegistrator: node.NewRegistrator(config.Client, config.LookupNode),
offers: offers.CreateRegistry(offers.RegistryConfig{
Compat: func(o *mesos.Offer) bool {
// the node must be registered and have up-to-date labels
n := config.LookupNode(o.GetHostname())
if n == nil || !node.IsUpToDate(n, node.SlaveAttributesToLabels(o.GetAttributes())) {
return false
}
// the executor IDs must not identify a kubelet-executor with a group that doesn't match ours
for _, eid := range o.GetExecutorIds() {
execuid := uid.Parse(eid.GetValue())
if execuid.Name() == execcfg.DefaultInfoID && execuid.Group() != k.executorGroup {
return false
}
}
return true
},
DeclineOffer: func(id string) <-chan error {
errOnce := proc.NewErrorOnce(k.terminate)
errOuter := k.asRegisteredMaster.Do(func() {
var err error
defer errOnce.Report(err)
offerId := mutil.NewOfferID(id)
filters := &mesos.Filters{}
_, err = k.driver.DeclineOffer(offerId, filters)
})
return errOnce.Send(errOuter).Err()
},
// remember expired offers so that we can tell if a previously scheduler offer relies on one
LingerTTL: config.Schedcfg.OfferLingerTTL.Duration,
TTL: config.Schedcfg.OfferTTL.Duration,
ListenerDelay: config.Schedcfg.ListenerDelay.Duration,
}),
slaveHostNames: slave.NewRegistry(),
taskRegistry: podtask.NewInMemoryRegistry(),
reconcileCooldown: config.ReconcileCooldown,
registration: make(chan struct{}),
asRegisteredMaster: proc.DoerFunc(func(proc.Action) <-chan error {
return proc.ErrorChanf("cannot execute action with unregistered scheduler")
}),
}
return k
}
func TestOffers(t *testing.T) {
offer1 := util.NewOffer(util.NewOfferID("487c73d8-9951-f23c-34bd-8085bfd30c49"), util.NewFrameworkID("20150903-065451-84125888-5050-10715-0053"),
util.NewSlaveID("20150903-065451-84125888-5050-10715-S1"), "slave0")
offer1.Resources = []*mesos.Resource{util.NewScalarResource("cpus", 4), util.NewScalarResource("mem", 512), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31000, 32000)})}
offer2 := util.NewOffer(util.NewOfferID("26d5b34c-ef81-638d-5ad5-32c743c9c033"), util.NewFrameworkID("20150903-065451-84125888-5050-10715-0037"),
util.NewSlaveID("20150903-065451-84125888-5050-10715-S0"), "master")
offer2.Resources = []*mesos.Resource{util.NewScalarResource("cpus", 2), util.NewScalarResource("mem", 1024), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(4000, 7000)})}
offer2.Attributes = []*mesos.Attribute{&mesos.Attribute{
Name: proto.String("rack"),
Type: mesos.Value_SCALAR.Enum(),
Scalar: &mesos.Value_Scalar{Value: proto.Float64(2)},
}}
offers := Offers([]*mesos.Offer{offer1, offer2})
if len(strings.Split(offers, "\n")) != 2 {
t.Errorf("Offers([]*mesos.Offer{offer1, offer2}) should contain two offers split by new line, actual: %s", offers)
}
}
func TestSchedCacheGetOffer(t *testing.T) {
cache := newSchedCache()
offer01 := createTestOffer("01")
pid01, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
offer02 := createTestOffer("02")
pid02, err := upid.Parse("[email protected]:5050")
assert.NoError(t, err)
cache.putOffer(offer01, pid01)
cache.putOffer(offer02, pid02)
cachedOffer01 := cache.getOffer(util.NewOfferID("test-offer-01")).offer
cachedOffer02 := cache.getOffer(util.NewOfferID("test-offer-02")).offer
assert.NotEqual(t, offer01, cachedOffer02)
assert.Equal(t, offer01, cachedOffer01)
assert.Equal(t, offer02, cachedOffer02)
}
//test adding of ressource offer, should be added to offer registry and slavesf
func TestResourceOffer_Add(t *testing.T) {
assert := assert.New(t)
testScheduler := &KubernetesScheduler{
offers: offers.CreateRegistry(offers.RegistryConfig{
Compat: func(o *mesos.Offer) bool {
return true
},
DeclineOffer: func(offerId string) <-chan error {
return proc.ErrorChan(nil)
},
// remember expired offers so that we can tell if a previously scheduler offer relies on one
LingerTTL: schedcfg.DefaultOfferLingerTTL,
TTL: schedcfg.DefaultOfferTTL,
ListenerDelay: schedcfg.DefaultListenerDelay,
}),
slaves: newSlaveStorage(),
}
hostname := "h1"
offerID1 := util.NewOfferID("test1")
offer1 := &mesos.Offer{Id: offerID1, Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)}
offers1 := []*mesos.Offer{offer1}
testScheduler.ResourceOffers(nil, offers1)
assert.Equal(1, getNumberOffers(testScheduler.offers))
//check slave hostname
assert.Equal(1, len(testScheduler.slaves.getSlaveIds()))
//add another offer
hostname2 := "h2"
offer2 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname2, SlaveId: util.NewSlaveID(hostname2)}
offers2 := []*mesos.Offer{offer2}
testScheduler.ResourceOffers(nil, offers2)
//check it is stored in registry
assert.Equal(2, getNumberOffers(testScheduler.offers))
//check slave hostnames
assert.Equal(2, len(testScheduler.slaves.getSlaveIds()))
}
//test when we loose connection to master we invalidate all cached offers
func TestDisconnect(t *testing.T) {
assert := assert.New(t)
//
testScheduler := &KubernetesScheduler{
offers: offers.CreateRegistry(offers.RegistryConfig{
Compat: func(o *mesos.Offer) bool {
return true
},
// remember expired offers so that we can tell if a previously scheduler offer relies on one
LingerTTL: schedcfg.DefaultOfferLingerTTL,
TTL: schedcfg.DefaultOfferTTL,
ListenerDelay: schedcfg.DefaultListenerDelay,
}),
slaveHostNames: slave.NewRegistry(),
}
hostname := "h1"
offer1 := &mesos.Offer{Id: util.NewOfferID("test1"), Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)}
offers1 := []*mesos.Offer{offer1}
testScheduler.ResourceOffers(nil, offers1)
offer2 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)}
offers2 := []*mesos.Offer{offer2}
testScheduler.ResourceOffers(nil, offers2)
//add another offer from different slaveID
hostname2 := "h2"
offer3 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname2, SlaveId: util.NewSlaveID(hostname2)}
offers3 := []*mesos.Offer{offer3}
testScheduler.ResourceOffers(nil, offers3)
//disconnect
testScheduler.Disconnected(nil)
//all offers should be removed
assert.Equal(0, getNumberOffers(testScheduler.offers))
//slave hostnames should still be all present
assert.Equal(2, len(testScheduler.slaveHostNames.SlaveIDs()))
}
func (suite *SchedulerIntegrationTestSuite) TestSchedulerDriverRescindOfferEvent() {
ok := suite.configureServerWithRegisteredFramework()
suite.True(ok, "failed to establish running test server and driver")
// Send a event to this SchedulerDriver (via http) to test handlers.
pbMsg := &mesos.RescindResourceOfferMessage{
OfferId: util.NewOfferID("test-offer-001"),
}
c := suite.newMockClient()
c.SendMessage(suite.driver.UPID(), pbMsg)
suite.sched.waitForCallback(0)
}
func (suite *SchedulerTestSuite) TestSchdulerDriverAcceptOffersWithError() {
sched := mock_scheduler.New()
sched.On("StatusUpdate").Return(nil)
sched.On("Error").Return()
msgr := mockedMessenger()
driver := newTestDriver(suite.T(), driverConfigMessenger(sched, suite.framework, suite.master, nil, msgr))
driver.OnDispatch(func(_ context.Context, _ *upid.UPID, _ proto.Message) error {
return fmt.Errorf("Unable to send message")
})
go func() {
driver.Run()
}()
<-driver.Started()
driver.SetConnected(true) // simulated
suite.True(driver.Running())
// setup an offer
offer := util.NewOffer(
util.NewOfferID("test-offer-001"),
suite.framework.Id,
util.NewSlaveID("test-slave-001"),
"test-slave(1)@localhost:5050",
)
pid, err := upid.Parse("test-slave(1)@localhost:5050")
suite.NoError(err)
driver.CacheOffer(offer, pid)
// launch task
task := util.NewTaskInfo(
"simple-task",
util.NewTaskID("simpe-task-1"),
util.NewSlaveID("test-slave-001"),
[]*mesos.Resource{util.NewScalarResourceWithReservation("mem", 400, "principal", "role")},
)
task.Command = util.NewCommandInfo("pwd")
task.Executor = util.NewExecutorInfo(util.NewExecutorID("test-exec"), task.Command)
tasks := []*mesos.TaskInfo{task}
operations := []*mesos.Offer_Operation{util.NewLaunchOperation(tasks)}
stat, err := driver.AcceptOffers(
[]*mesos.OfferID{offer.Id},
operations,
&mesos.Filters{},
)
suite.Equal(mesos.Status_DRIVER_RUNNING, stat)
suite.Error(err)
}
// New creates a new QingYuanScheduler
func New(config Config) *QingYuanScheduler {
var k *QingYuanScheduler
k = &QingYuanScheduler{
schedcfg: &config.Schedcfg,
RWMutex: new(sync.RWMutex),
executor: config.Executor,
executorGroup: uid.Parse(config.Executor.ExecutorId.GetValue()).Group(),
scheduleFunc: config.ScheduleFunc,
client: config.Client,
etcdClient: config.EtcdClient,
failoverTimeout: config.FailoverTimeout,
reconcileInterval: config.ReconcileInterval,
offers: offers.CreateRegistry(offers.RegistryConfig{
Compat: func(o *mesos.Offer) bool {
// filter the offers: the executor IDs must not identify a qinglet-
// executor with a group that doesn't match ours
for _, eid := range o.GetExecutorIds() {
execuid := uid.Parse(eid.GetValue())
if execuid.Name() == execcfg.DefaultInfoID && execuid.Group() != k.executorGroup {
return false
}
}
return true
},
DeclineOffer: func(id string) <-chan error {
errOnce := proc.NewErrorOnce(k.terminate)
errOuter := k.asRegisteredMaster.Do(func() {
var err error
defer errOnce.Report(err)
offerId := mutil.NewOfferID(id)
filters := &mesos.Filters{}
_, err = k.driver.DeclineOffer(offerId, filters)
})
return errOnce.Send(errOuter).Err()
},
// remember expired offers so that we can tell if a previously scheduler offer relies on one
LingerTTL: config.Schedcfg.OfferLingerTTL.Duration,
TTL: config.Schedcfg.OfferTTL.Duration,
ListenerDelay: config.Schedcfg.ListenerDelay.Duration,
}),
slaves: newSlaveStorage(),
taskRegistry: podtask.NewInMemoryRegistry(),
reconcileCooldown: config.ReconcileCooldown,
registration: make(chan struct{}),
asRegisteredMaster: proc.DoerFunc(func(proc.Action) <-chan error {
return proc.ErrorChanf("cannot execute action with unregistered scheduler")
}),
}
return k
}
func TestFilterResources(t *testing.T) {
rf := ResourceFilter{}
o := util.NewOffer(util.NewOfferID("offerid"), util.NewFrameworkID("frameworkid"), util.NewSlaveID("slaveId"), "hostname")
o.Resources = []*mesos.Resource{
util.NewScalarResource("name", 1.0),
util.NewScalarResource("ub0r-resource", 2.0),
util.NewScalarResource("ub0r-resource", 3.0),
}
res := rf.FilterResources(o, "ub0r-resource")
assert.Equal(t, 2, len(res))
assert.Equal(t, "ub0r-resource", res[0].GetName())
}
func TestListen(t *testing.T) {
ttl := time.Second / 4
config := RegistryConfig{
DeclineOffer: func(offerId string) <-chan error {
return proc.ErrorChan(nil)
},
Compat: func(o *mesos.Offer) bool {
return true
},
TTL: ttl,
ListenerDelay: ttl / 2,
}
storage := CreateRegistry(config)
done := make(chan struct{})
storage.Init(done)
// Create two listeners with a hostname filter
hostname1 := "hostname1"
hostname2 := "hostname2"
listener1 := storage.Listen("listener1", func(offer *mesos.Offer) bool {
return offer.GetHostname() == hostname1
})
listener2 := storage.Listen("listener2", func(offer *mesos.Offer) bool {
return offer.GetHostname() == hostname2
})
// Add hostname1 offer
id := util.NewOfferID("foo")
o := &mesos.Offer{Id: id, Hostname: &hostname1}
storage.Add([]*mesos.Offer{o})
// listener1 is notified by closing channel
select {
case _, more := <-listener1:
if more {
t.Error("listener1 is not closed")
}
}
// listener2 is not notified within ttl
select {
case <-listener2:
t.Error("listener2 is notified")
case <-time.After(ttl):
}
close(done)
} // TestListen
// Offering some cpus and memory and the 8000-9000 port range
func NewTestOffer(id string) *mesos.Offer {
hostname := "some_hostname"
cpus := util.NewScalarResource("cpus", 3.75)
mem := util.NewScalarResource("mem", 940)
var port8000 uint64 = 8000
var port9000 uint64 = 9000
ports8000to9000 := mesos.Value_Range{Begin: &port8000, End: &port9000}
ports := util.NewRangesResource("ports", []*mesos.Value_Range{&ports8000to9000})
return &mesos.Offer{
Id: util.NewOfferID(id),
Hostname: &hostname,
SlaveId: util.NewSlaveID(hostname),
Resources: []*mesos.Resource{cpus, mem, ports},
}
}
func NewOffer(id string) *mesos.Offer {
return &mesos.Offer{
Id: util.NewOfferID(id),
FrameworkId: util.NewFrameworkID("test-etcd-framework"),
SlaveId: util.NewSlaveID("slave-" + id),
Hostname: proto.String("localhost"),
Resources: []*mesos.Resource{
util.NewScalarResource("cpus", 1),
util.NewScalarResource("mem", 256),
util.NewScalarResource("disk", 4096),
util.NewRangesResource("ports", []*mesos.Value_Range{
util.NewValueRange(uint64(0), uint64(65535)),
}),
},
}
}
func TestBlockingPop(t *testing.T) {
for i, tt := range []struct {
offers []string
rescinds []string
want int
}{
// with one valid offer, we should pop once
{[]string{"a"}, []string{"b"}, 1},
// with 7 valid offers (2 more than cap), and invalidation of one
// of the non-rejected ones as well as one of the accepted ones,
// we should be able to pop 4 offers
{[]string{"a", "b", "c", "d", "e", "f", "g"}, []string{"a", "g"}, 4},
} {
oc := New(5, false)
for _, o := range tt.offers {
oc.Push(newOffer(o, o))
}
for _, r := range tt.rescinds {
oc.Rescind(util.NewOfferID(r))
}
got := func() int {
n := 0
for oc.Len() > 0 {
c := make(chan struct{})
go func() {
oc.BlockingPop()
c <- struct{}{}
}()
select {
case <-c:
n += 1
case <-time.After(time.Second):
return n
}
}
return n
}()
if got != tt.want {
t.Errorf("test #%d: got : %s, want: %s", i, got, tt.want)
}
}
}
func (suite *SchedulerIntegrationTestSuite) TestSchedulerDriverResourceOffersEvent() {
ok := suite.configureServerWithRegisteredFramework()
suite.True(ok, "failed to establish running test server and driver")
// Send a event to this SchedulerDriver (via http) to test handlers.
offer := util.NewOffer(
util.NewOfferID("test-offer-001"),
suite.registeredFrameworkId,
util.NewSlaveID("test-slave-001"),
"test-localhost",
)
pbMsg := &mesos.ResourceOffersMessage{
Offers: []*mesos.Offer{offer},
Pids: []string{"test-offer-001@test-slave-001:5051"},
}
c := suite.newMockClient()
c.SendMessage(suite.driver.UPID(), pbMsg)
suite.sched.waitForCallback(0)
}
func TestInMemoryRegistry_Update(t *testing.T) {
assert := assert.New(t)
// create offers registry
ttl := time.Second / 4
config := offers.RegistryConfig{
DeclineOffer: func(offerId string) <-chan error {
return proc.ErrorChan(nil)
},
Compat: func(o *mesos.Offer) bool {
return true
},
TTL: ttl,
LingerTTL: 2 * ttl,
}
storage := offers.CreateRegistry(config)
// Add offer
offerId := mesosutil.NewOfferID("foo")
mesosOffer := &mesos.Offer{Id: offerId}
storage.Add([]*mesos.Offer{mesosOffer})
offer, ok := storage.Get(offerId.GetValue())
assert.True(ok)
// create registry
registry := NewInMemoryRegistry()
a, _ := fakePodTask("a")
registry.Register(a.Clone(), nil) // here clone a because we change it below
// state changes are ignored
a.State = StateRunning
err := registry.Update(a)
assert.NoError(err)
a_clone, _ := registry.Get(a.ID)
assert.Equal(StatePending, a_clone.State)
// offer is updated while pending
a.Offer = offer
err = registry.Update(a)
assert.NoError(err)
a_clone, _ = registry.Get(a.ID)
assert.Equal(offer.Id(), a_clone.Offer.Id())
// spec is updated while pending
a.Spec = Spec{SlaveID: "slave-1"}
err = registry.Update(a)
assert.NoError(err)
a_clone, _ = registry.Get(a.ID)
assert.Equal("slave-1", a_clone.Spec.SlaveID)
// flags are updated while pending
a.Flags[Launched] = struct{}{}
err = registry.Update(a)
assert.NoError(err)
a_clone, _ = registry.Get(a.ID)
_, found_launched := a_clone.Flags[Launched]
assert.True(found_launched)
// flags are updated while running
registry.UpdateStatus(fakeStatusUpdate(a.ID, mesos.TaskState_TASK_RUNNING))
a.Flags[Bound] = struct{}{}
err = registry.Update(a)
assert.NoError(err)
a_clone, _ = registry.Get(a.ID)
_, found_launched = a_clone.Flags[Launched]
assert.True(found_launched)
_, found_bound := a_clone.Flags[Bound]
assert.True(found_bound)
// spec is ignored while running
a.Spec = Spec{SlaveID: "slave-2"}
err = registry.Update(a)
assert.NoError(err)
a_clone, _ = registry.Get(a.ID)
assert.Equal("slave-1", a_clone.Spec.SlaveID)
// error when finished
registry.UpdateStatus(fakeStatusUpdate(a.ID, mesos.TaskState_TASK_FINISHED))
err = registry.Update(a)
assert.Error(err)
// update unknown task
unknown_task, _ := fakePodTask("unknown-task")
err = registry.Update(unknown_task)
assert.Error(err)
// update nil task
err = registry.Update(nil)
assert.Nil(err)
}
func newOffer(offer, slave string) *mesos.Offer {
return &mesos.Offer{
Id: util.NewOfferID(offer),
SlaveId: util.NewSlaveID(slave),
}
}
func TestWalk(t *testing.T) {
t.Parallel()
config := RegistryConfig{
DeclineOffer: func(offerId string) <-chan error {
return proc.ErrorChan(nil)
},
TTL: 0 * time.Second,
LingerTTL: 0 * time.Second,
ListenerDelay: 0 * time.Second,
}
storage := CreateRegistry(config)
acceptedOfferId := ""
walked := 0
walker1 := func(p Perishable) (bool, error) {
walked++
if p.Acquire() {
acceptedOfferId = p.Details().Id.GetValue()
return true, nil
}
return false, nil
}
// sanity check
err := storage.Walk(walker1)
if err != nil {
t.Fatalf("received impossible error %v", err)
}
if walked != 0 {
t.Fatal("walked empty storage")
}
if acceptedOfferId != "" {
t.Fatal("somehow found an offer when registry was empty")
}
impl, ok := storage.(*offerStorage)
if !ok {
t.Fatal("unexpected offer storage impl")
}
// single offer
ttl := 2 * time.Second
now := time.Now()
o := &liveOffer{&mesos.Offer{Id: util.NewOfferID("foo")}, now.Add(ttl), 0}
impl.offers.Add(o)
err = storage.Walk(walker1)
if err != nil {
t.Fatalf("received impossible error %v", err)
}
if walked != 1 {
t.Fatalf("walk count %d", walked)
}
if acceptedOfferId != "foo" {
t.Fatalf("found offer %v", acceptedOfferId)
}
acceptedOfferId = ""
err = storage.Walk(walker1)
if err != nil {
t.Fatalf("received impossible error %v", err)
}
if walked != 2 {
t.Fatalf("walk count %d", walked)
}
if acceptedOfferId != "" {
t.Fatalf("found offer %v", acceptedOfferId)
}
walker2 := func(p Perishable) (bool, error) {
walked++
return true, nil
}
err = storage.Walk(walker2)
if err != nil {
t.Fatalf("received impossible error %v", err)
}
if walked != 3 {
t.Fatalf("walk count %d", walked)
}
if acceptedOfferId != "" {
t.Fatalf("found offer %v", acceptedOfferId)
}
walker3 := func(p Perishable) (bool, error) {
walked++
return true, errors.New("baz")
}
err = storage.Walk(walker3)
if err == nil {
t.Fatal("expected error")
}
if walked != 4 {
t.Fatalf("walk count %d", walked)
}
}
func TestOfferStorage(t *testing.T) {
ttl := time.Second / 4
var declinedNum int32
getDeclinedNum := func() int32 { return atomic.LoadInt32(&declinedNum) }
config := RegistryConfig{
DeclineOffer: func(offerId string) <-chan error {
atomic.AddInt32(&declinedNum, 1)
return proc.ErrorChan(nil)
},
Compat: func(o *mesos.Offer) bool {
return o.Hostname == nil || *o.Hostname != "incompatiblehost"
},
TTL: ttl,
LingerTTL: 2 * ttl,
}
storage := CreateRegistry(config)
done := make(chan struct{})
storage.Init(done)
// Add offer
id := util.NewOfferID("foo")
o := &mesos.Offer{Id: id}
storage.Add([]*mesos.Offer{o})
// Added offer should be in the storage
if obj, ok := storage.Get(id.GetValue()); obj == nil || !ok {
t.Error("offer not added")
}
if obj, _ := storage.Get(id.GetValue()); obj.Details() != o {
t.Error("added offer differs from returned offer")
}
// Not-added offer is not in storage
if obj, ok := storage.Get("bar"); obj != nil || ok {
t.Error("offer bar should not exist in storage")
}
// Deleted offer lingers in storage, is acquired and declined
offer, _ := storage.Get(id.GetValue())
declinedNumBefore := getDeclinedNum()
storage.Delete(id.GetValue(), "deleted for test")
if obj, _ := storage.Get(id.GetValue()); obj == nil {
t.Error("deleted offer is not lingering")
}
if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() {
t.Error("deleted offer is no expired")
}
if ok := offer.Acquire(); ok {
t.Error("deleted offer can be acquired")
}
if getDeclinedNum() <= declinedNumBefore {
t.Error("deleted offer was not declined")
}
// Acquired offer is only declined after 2*ttl
id = util.NewOfferID("foo2")
o = &mesos.Offer{Id: id}
storage.Add([]*mesos.Offer{o})
offer, _ = storage.Get(id.GetValue())
declinedNumBefore = getDeclinedNum()
offer.Acquire()
storage.Delete(id.GetValue(), "deleted for test")
if getDeclinedNum() > declinedNumBefore {
t.Error("acquired offer is declined")
}
offer.Release()
time.Sleep(3 * ttl)
if getDeclinedNum() <= declinedNumBefore {
t.Error("released offer is not declined after 2*ttl")
}
// Added offer should be expired after ttl, but lingering
id = util.NewOfferID("foo3")
o = &mesos.Offer{Id: id}
storage.Add([]*mesos.Offer{o})
time.Sleep(2 * ttl)
obj, ok := storage.Get(id.GetValue())
if obj == nil || !ok {
t.Error("offer not lingering after ttl")
}
if !obj.HasExpired() {
t.Error("offer is not expired after ttl")
}
// Should be deleted when waiting longer than LingerTTL
time.Sleep(2 * ttl)
if obj, ok := storage.Get(id.GetValue()); obj != nil || ok {
t.Error("offer not deleted after LingerTTL")
}
// Incompatible offer is declined
id = util.NewOfferID("foo4")
incompatibleHostname := "incompatiblehost"
o = &mesos.Offer{Id: id, Hostname: &incompatibleHostname}
declinedNumBefore = getDeclinedNum()
storage.Add([]*mesos.Offer{o})
if obj, ok := storage.Get(id.GetValue()); obj != nil || ok {
t.Error("incompatible offer not rejected")
}
if getDeclinedNum() <= declinedNumBefore {
t.Error("incompatible offer is not declined")
}
// Invalidated offer are not declined, but expired
id = util.NewOfferID("foo5")
o = &mesos.Offer{Id: id}
storage.Add([]*mesos.Offer{o})
offer, _ = storage.Get(id.GetValue())
declinedNumBefore = getDeclinedNum()
storage.Invalidate(id.GetValue())
if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() {
t.Error("invalidated offer is not expired")
}
if getDeclinedNum() > declinedNumBefore {
t.Error("invalidated offer is declined")
}
if ok := offer.Acquire(); ok {
t.Error("invalidated offer can be acquired")
}
// Invalidate "" will invalidate all offers
id = util.NewOfferID("foo6")
o = &mesos.Offer{Id: id}
storage.Add([]*mesos.Offer{o})
id2 := util.NewOfferID("foo7")
o2 := &mesos.Offer{Id: id2}
storage.Add([]*mesos.Offer{o2})
storage.Invalidate("")
if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() {
t.Error("invalidated offer is not expired")
}
if obj2, _ := storage.Get(id2.GetValue()); !obj2.HasExpired() {
t.Error("invalidated offer is not expired")
}
// InvalidateForSlave invalides all offers for that slave, but only those
id = util.NewOfferID("foo8")
slaveId := util.NewSlaveID("test-slave")
o = &mesos.Offer{Id: id, SlaveId: slaveId}
storage.Add([]*mesos.Offer{o})
id2 = util.NewOfferID("foo9")
o2 = &mesos.Offer{Id: id2}
storage.Add([]*mesos.Offer{o2})
storage.InvalidateForSlave(slaveId.GetValue())
if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() {
t.Error("invalidated offer for test-slave is not expired")
}
if obj2, _ := storage.Get(id2.GetValue()); obj2.HasExpired() {
t.Error("invalidated offer another slave is expired")
}
close(done)
} // TestOfferStorage
// New creates a new Framework
func New(config Config) Framework {
var k *framework
k = &framework{
schedulerConfig: &config.SchedulerConfig,
RWMutex: new(sync.RWMutex),
client: config.Client,
failoverTimeout: config.FailoverTimeout,
reconcileInterval: config.ReconcileInterval,
nodeRegistrator: node.NewRegistrator(config.Client, config.LookupNode),
executorId: config.ExecutorId,
offers: offers.CreateRegistry(offers.RegistryConfig{
Compat: func(o *mesos.Offer) bool {
// the node must be registered and have up-to-date labels
n := config.LookupNode(o.GetHostname())
if n == nil || !node.IsUpToDate(n, node.SlaveAttributesToLabels(o.GetAttributes())) {
return false
}
eids := len(o.GetExecutorIds())
switch {
case eids > 1:
// at most one executor id expected. More than one means that
// the given node is seriously in trouble.
return false
case eids == 1:
// the executor id must match, otherwise the running executor
// is incompatible with the current scheduler configuration.
if eid := o.GetExecutorIds()[0]; eid.GetValue() != config.ExecutorId.GetValue() {
return false
}
}
return true
},
DeclineOffer: func(id string) <-chan error {
errOnce := proc.NewErrorOnce(k.terminate)
errOuter := k.asRegisteredMaster.Do(func() {
var err error
defer errOnce.Report(err)
offerId := mutil.NewOfferID(id)
filters := &mesos.Filters{}
_, err = k.driver.DeclineOffer(offerId, filters)
})
return errOnce.Send(errOuter).Err()
},
// remember expired offers so that we can tell if a previously scheduler offer relies on one
LingerTTL: config.SchedulerConfig.OfferLingerTTL.Duration,
TTL: config.SchedulerConfig.OfferTTL.Duration,
ListenerDelay: config.SchedulerConfig.ListenerDelay.Duration,
}),
slaveHostNames: newSlaveRegistry(),
reconcileCooldown: config.ReconcileCooldown,
registration: make(chan struct{}),
asRegisteredMaster: proc.DoerFunc(func(proc.Action) <-chan error {
return proc.ErrorChanf("cannot execute action with unregistered scheduler")
}),
storeFrameworkId: config.StoreFrameworkId,
lookupNode: config.LookupNode,
}
return k
}
func (suite *SchedulerTestSuite) TestSchdulerDriverLaunchTasksWithError() {
sched := NewMockScheduler()
sched.On("StatusUpdate").Return(nil)
sched.On("Error").Return()
msgr := messenger.NewMockedMessenger()
msgr.On("Start").Return(nil)
msgr.On("Send").Return(nil)
msgr.On("UPID").Return(&upid.UPID{})
msgr.On("Stop").Return(nil)
msgr.On("Route").Return(nil)
driver := newTestSchedulerDriver(suite.T(), sched, suite.framework, suite.master, nil)
driver.messenger = msgr
suite.True(driver.Stopped())
go func() {
driver.Run()
}()
time.Sleep(time.Millisecond * 1)
driver.setConnected(true) // simulated
suite.False(driver.Stopped())
suite.Equal(mesos.Status_DRIVER_RUNNING, driver.Status())
// to trigger error
msgr2 := messenger.NewMockedMessenger()
msgr2.On("Start").Return(nil)
msgr2.On("UPID").Return(&upid.UPID{})
msgr2.On("Send").Return(fmt.Errorf("Unable to send message"))
msgr2.On("Stop").Return(nil)
msgr.On("Route").Return(nil)
driver.messenger = msgr2
// setup an offer
offer := util.NewOffer(
util.NewOfferID("test-offer-001"),
suite.framework.Id,
util.NewSlaveID("test-slave-001"),
"test-slave(1)@localhost:5050",
)
pid, err := upid.Parse("test-slave(1)@localhost:5050")
suite.NoError(err)
driver.cache.putOffer(offer, pid)
// launch task
task := util.NewTaskInfo(
"simple-task",
util.NewTaskID("simpe-task-1"),
util.NewSlaveID("test-slave-001"),
[]*mesos.Resource{util.NewScalarResource("mem", 400)},
)
task.Command = util.NewCommandInfo("pwd")
task.Executor = util.NewExecutorInfo(util.NewExecutorID("test-exec"), task.Command)
tasks := []*mesos.TaskInfo{task}
stat, err := driver.LaunchTasks(
[]*mesos.OfferID{offer.Id},
tasks,
&mesos.Filters{},
)
suite.Error(err)
suite.Equal(mesos.Status_DRIVER_RUNNING, stat)
}