// reconciler action factory, performs explicit task reconciliation for non-terminal // tasks identified by annotations in the Kubernetes pod registry. func (k *KubernetesScheduler) makePodRegistryReconciler() ReconcilerAction { return ReconcilerAction(func(drv bindings.SchedulerDriver, cancel <-chan struct{}) <-chan error { ctx := api.NewDefaultContext() podList, err := k.client.Pods(api.NamespaceValue(ctx)).List(labels.Everything(), fields.Everything()) if err != nil { return proc.ErrorChanf("failed to reconcile pod registry: %v", err) } taskToSlave := make(map[string]string) for _, pod := range podList.Items { if len(pod.Annotations) == 0 { continue } taskId, found := pod.Annotations[meta.TaskIdKey] if !found { continue } slaveId, found := pod.Annotations[meta.SlaveIdKey] if !found { continue } taskToSlave[taskId] = slaveId } return proc.ErrorChan(k.explicitlyReconcileTasks(drv, taskToSlave, cancel)) }) }
// returns a Process instance that will only execute a proc.Action if the scheduler is the elected master func (self *SchedulerProcess) Master() proc.Process { return &masterProcess{ SchedulerProcess: self, doer: proc.DoWith(self, proc.DoerFunc(func(a proc.Action) <-chan error { return proc.ErrorChan(masterStage.When(self, a)) })), } }
// reconciler action factory, performs explicit task reconciliation for non-terminal // tasks listed in the scheduler's internal taskRegistry. func (k *KubernetesScheduler) makeTaskRegistryReconciler() ReconcilerAction { return ReconcilerAction(func(drv bindings.SchedulerDriver, cancel <-chan struct{}) <-chan error { taskToSlave := make(map[string]string) for _, t := range k.taskRegistry.List(explicitTaskFilter) { if t.Spec.SlaveID != "" { taskToSlave[t.ID] = t.Spec.SlaveID } } return proc.ErrorChan(k.explicitlyReconcileTasks(drv, taskToSlave, cancel)) }) }
//test adding of ressource offer, should be added to offer registry and slavesf func TestResourceOffer_Add_Rescind(t *testing.T) { assert := assert.New(t) testScheduler := &KubernetesScheduler{ offers: offers.CreateRegistry(offers.RegistryConfig{ Compat: func(o *mesos.Offer) bool { return true }, DeclineOffer: func(offerId string) <-chan error { return proc.ErrorChan(nil) }, // remember expired offers so that we can tell if a previously scheduler offer relies on one LingerTTL: schedcfg.DefaultOfferLingerTTL, TTL: schedcfg.DefaultOfferTTL, ListenerDelay: schedcfg.DefaultListenerDelay, }), slaves: newSlaveStorage(), } hostname := "h1" offerID1 := util.NewOfferID("test1") offer1 := &mesos.Offer{Id: offerID1, Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)} offers1 := []*mesos.Offer{offer1} testScheduler.ResourceOffers(nil, offers1) assert.Equal(1, getNumberOffers(testScheduler.offers)) //check slave hostname assert.Equal(1, len(testScheduler.slaves.getSlaveIds())) //add another offer hostname2 := "h2" offer2 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname2, SlaveId: util.NewSlaveID(hostname2)} offers2 := []*mesos.Offer{offer2} testScheduler.ResourceOffers(nil, offers2) assert.Equal(2, getNumberOffers(testScheduler.offers)) //check slave hostnames assert.Equal(2, len(testScheduler.slaves.getSlaveIds())) //next whether offers can be rescinded testScheduler.OfferRescinded(nil, offerID1) assert.Equal(1, getNumberOffers(testScheduler.offers)) //next whether offers can be rescinded testScheduler.OfferRescinded(nil, util.NewOfferID("test2")) //walk offers again and check it is removed from registry assert.Equal(0, getNumberOffers(testScheduler.offers)) //remove non existing ID testScheduler.OfferRescinded(nil, util.NewOfferID("notExist")) }
func TestListen(t *testing.T) { ttl := time.Second / 4 config := RegistryConfig{ DeclineOffer: func(offerId string) <-chan error { return proc.ErrorChan(nil) }, Compat: func(o *mesos.Offer) bool { return true }, TTL: ttl, ListenerDelay: ttl / 2, } storage := CreateRegistry(config) done := make(chan struct{}) storage.Init(done) // Create two listeners with a hostname filter hostname1 := "hostname1" hostname2 := "hostname2" listener1 := storage.Listen("listener1", func(offer *mesos.Offer) bool { return offer.GetHostname() == hostname1 }) listener2 := storage.Listen("listener2", func(offer *mesos.Offer) bool { return offer.GetHostname() == hostname2 }) // Add hostname1 offer id := util.NewOfferID("foo") o := &mesos.Offer{Id: id, Hostname: &hostname1} storage.Add([]*mesos.Offer{o}) // listener1 is notified by closing channel select { case _, more := <-listener1: if more { t.Error("listener1 is not closed") } } // listener2 is not notified within ttl select { case <-listener2: t.Error("listener2 is notified") case <-time.After(ttl): } close(done) } // TestListen
func TestInMemoryRegistry_Update(t *testing.T) { assert := assert.New(t) // create offers registry ttl := time.Second / 4 config := offers.RegistryConfig{ DeclineOffer: func(offerId string) <-chan error { return proc.ErrorChan(nil) }, Compat: func(o *mesos.Offer) bool { return true }, TTL: ttl, LingerTTL: 2 * ttl, } storage := offers.CreateRegistry(config) // Add offer offerId := mesosutil.NewOfferID("foo") mesosOffer := &mesos.Offer{Id: offerId} storage.Add([]*mesos.Offer{mesosOffer}) offer, ok := storage.Get(offerId.GetValue()) assert.True(ok) // create registry registry := NewInMemoryRegistry() a, _ := fakePodTask("a") registry.Register(a.Clone(), nil) // here clone a because we change it below // state changes are ignored a.State = StateRunning err := registry.Update(a) assert.NoError(err) a_clone, _ := registry.Get(a.ID) assert.Equal(StatePending, a_clone.State) // offer is updated while pending a.Offer = offer err = registry.Update(a) assert.NoError(err) a_clone, _ = registry.Get(a.ID) assert.Equal(offer.Id(), a_clone.Offer.Id()) // spec is updated while pending a.Spec = Spec{SlaveID: "slave-1"} err = registry.Update(a) assert.NoError(err) a_clone, _ = registry.Get(a.ID) assert.Equal("slave-1", a_clone.Spec.SlaveID) // flags are updated while pending a.Flags[Launched] = struct{}{} err = registry.Update(a) assert.NoError(err) a_clone, _ = registry.Get(a.ID) _, found_launched := a_clone.Flags[Launched] assert.True(found_launched) // flags are updated while running registry.UpdateStatus(fakeStatusUpdate(a.ID, mesos.TaskState_TASK_RUNNING)) a.Flags[Bound] = struct{}{} err = registry.Update(a) assert.NoError(err) a_clone, _ = registry.Get(a.ID) _, found_launched = a_clone.Flags[Launched] assert.True(found_launched) _, found_bound := a_clone.Flags[Bound] assert.True(found_bound) // spec is ignored while running a.Spec = Spec{SlaveID: "slave-2"} err = registry.Update(a) assert.NoError(err) a_clone, _ = registry.Get(a.ID) assert.Equal("slave-1", a_clone.Spec.SlaveID) // error when finished registry.UpdateStatus(fakeStatusUpdate(a.ID, mesos.TaskState_TASK_FINISHED)) err = registry.Update(a) assert.Error(err) // update unknown task unknown_task, _ := fakePodTask("unknown-task") err = registry.Update(unknown_task) assert.Error(err) // update nil task err = registry.Update(nil) assert.Nil(err) }
func TestOfferStorage(t *testing.T) { ttl := time.Second / 4 var declinedNum int32 getDeclinedNum := func() int32 { return atomic.LoadInt32(&declinedNum) } config := RegistryConfig{ DeclineOffer: func(offerId string) <-chan error { atomic.AddInt32(&declinedNum, 1) return proc.ErrorChan(nil) }, Compat: func(o *mesos.Offer) bool { return o.Hostname == nil || *o.Hostname != "incompatiblehost" }, TTL: ttl, LingerTTL: 2 * ttl, } storage := CreateRegistry(config) done := make(chan struct{}) storage.Init(done) // Add offer id := util.NewOfferID("foo") o := &mesos.Offer{Id: id} storage.Add([]*mesos.Offer{o}) // Added offer should be in the storage if obj, ok := storage.Get(id.GetValue()); obj == nil || !ok { t.Error("offer not added") } if obj, _ := storage.Get(id.GetValue()); obj.Details() != o { t.Error("added offer differs from returned offer") } // Not-added offer is not in storage if obj, ok := storage.Get("bar"); obj != nil || ok { t.Error("offer bar should not exist in storage") } // Deleted offer lingers in storage, is acquired and declined offer, _ := storage.Get(id.GetValue()) declinedNumBefore := getDeclinedNum() storage.Delete(id.GetValue(), "deleted for test") if obj, _ := storage.Get(id.GetValue()); obj == nil { t.Error("deleted offer is not lingering") } if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() { t.Error("deleted offer is no expired") } if ok := offer.Acquire(); ok { t.Error("deleted offer can be acquired") } if getDeclinedNum() <= declinedNumBefore { t.Error("deleted offer was not declined") } // Acquired offer is only declined after 2*ttl id = util.NewOfferID("foo2") o = &mesos.Offer{Id: id} storage.Add([]*mesos.Offer{o}) offer, _ = storage.Get(id.GetValue()) declinedNumBefore = getDeclinedNum() offer.Acquire() storage.Delete(id.GetValue(), "deleted for test") if getDeclinedNum() > declinedNumBefore { t.Error("acquired offer is declined") } offer.Release() time.Sleep(3 * ttl) if getDeclinedNum() <= declinedNumBefore { t.Error("released offer is not declined after 2*ttl") } // Added offer should be expired after ttl, but lingering id = util.NewOfferID("foo3") o = &mesos.Offer{Id: id} storage.Add([]*mesos.Offer{o}) time.Sleep(2 * ttl) obj, ok := storage.Get(id.GetValue()) if obj == nil || !ok { t.Error("offer not lingering after ttl") } if !obj.HasExpired() { t.Error("offer is not expired after ttl") } // Should be deleted when waiting longer than LingerTTL time.Sleep(2 * ttl) if obj, ok := storage.Get(id.GetValue()); obj != nil || ok { t.Error("offer not deleted after LingerTTL") } // Incompatible offer is declined id = util.NewOfferID("foo4") incompatibleHostname := "incompatiblehost" o = &mesos.Offer{Id: id, Hostname: &incompatibleHostname} declinedNumBefore = getDeclinedNum() storage.Add([]*mesos.Offer{o}) if obj, ok := storage.Get(id.GetValue()); obj != nil || ok { t.Error("incompatible offer not rejected") } if getDeclinedNum() <= declinedNumBefore { t.Error("incompatible offer is not declined") } // Invalidated offer are not declined, but expired id = util.NewOfferID("foo5") o = &mesos.Offer{Id: id} storage.Add([]*mesos.Offer{o}) offer, _ = storage.Get(id.GetValue()) declinedNumBefore = getDeclinedNum() storage.Invalidate(id.GetValue()) if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() { t.Error("invalidated offer is not expired") } if getDeclinedNum() > declinedNumBefore { t.Error("invalidated offer is declined") } if ok := offer.Acquire(); ok { t.Error("invalidated offer can be acquired") } // Invalidate "" will invalidate all offers id = util.NewOfferID("foo6") o = &mesos.Offer{Id: id} storage.Add([]*mesos.Offer{o}) id2 := util.NewOfferID("foo7") o2 := &mesos.Offer{Id: id2} storage.Add([]*mesos.Offer{o2}) storage.Invalidate("") if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() { t.Error("invalidated offer is not expired") } if obj2, _ := storage.Get(id2.GetValue()); !obj2.HasExpired() { t.Error("invalidated offer is not expired") } // InvalidateForSlave invalides all offers for that slave, but only those id = util.NewOfferID("foo8") slaveId := util.NewSlaveID("test-slave") o = &mesos.Offer{Id: id, SlaveId: slaveId} storage.Add([]*mesos.Offer{o}) id2 = util.NewOfferID("foo9") o2 = &mesos.Offer{Id: id2} storage.Add([]*mesos.Offer{o2}) storage.InvalidateForSlave(slaveId.GetValue()) if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() { t.Error("invalidated offer for test-slave is not expired") } if obj2, _ := storage.Get(id2.GetValue()); obj2.HasExpired() { t.Error("invalidated offer another slave is expired") } close(done) } // TestOfferStorage
func TestWalk(t *testing.T) { t.Parallel() config := RegistryConfig{ DeclineOffer: func(offerId string) <-chan error { return proc.ErrorChan(nil) }, TTL: 0 * time.Second, LingerTTL: 0 * time.Second, ListenerDelay: 0 * time.Second, } storage := CreateRegistry(config) acceptedOfferId := "" walked := 0 walker1 := func(p Perishable) (bool, error) { walked++ if p.Acquire() { acceptedOfferId = p.Details().Id.GetValue() return true, nil } return false, nil } // sanity check err := storage.Walk(walker1) if err != nil { t.Fatalf("received impossible error %v", err) } if walked != 0 { t.Fatal("walked empty storage") } if acceptedOfferId != "" { t.Fatal("somehow found an offer when registry was empty") } impl, ok := storage.(*offerStorage) if !ok { t.Fatal("unexpected offer storage impl") } // single offer ttl := 2 * time.Second now := time.Now() o := &liveOffer{&mesos.Offer{Id: util.NewOfferID("foo")}, now.Add(ttl), 0} impl.offers.Add(o) err = storage.Walk(walker1) if err != nil { t.Fatalf("received impossible error %v", err) } if walked != 1 { t.Fatalf("walk count %d", walked) } if acceptedOfferId != "foo" { t.Fatalf("found offer %v", acceptedOfferId) } acceptedOfferId = "" err = storage.Walk(walker1) if err != nil { t.Fatalf("received impossible error %v", err) } if walked != 2 { t.Fatalf("walk count %d", walked) } if acceptedOfferId != "" { t.Fatalf("found offer %v", acceptedOfferId) } walker2 := func(p Perishable) (bool, error) { walked++ return true, nil } err = storage.Walk(walker2) if err != nil { t.Fatalf("received impossible error %v", err) } if walked != 3 { t.Fatalf("walk count %d", walked) } if acceptedOfferId != "" { t.Fatalf("found offer %v", acceptedOfferId) } walker3 := func(p Perishable) (bool, error) { walked++ return true, errors.New("baz") } err = storage.Walk(walker3) if err == nil { t.Fatal("expected error") } if walked != 4 { t.Fatalf("walk count %d", walked) } }