func TestUnitStatePublisherRunWithDelays(t *testing.T) { states := make([]string, 0) mu := sync.Mutex{} // protects states from concurrent modification fclock := clockwork.NewFakeClock() var wgs, wgf sync.WaitGroup // track starting and stopping of publishers slowpf := func(name string, us *unit.UnitState) { wgs.Done() // simulate a delay in communication with the registry fclock.Sleep(3 * time.Second) mu.Lock() states = append(states, name) mu.Unlock() wgf.Done() } usp := &UnitStatePublisher{ mach: &machine.FakeMachine{}, ttl: time.Hour, // we never expect to reach this publisher: slowpf, cache: make(map[string]*unit.UnitState), cacheMutex: sync.RWMutex{}, toPublish: make(chan string), toPublishStates: make(map[string]*unit.UnitState), toPublishMutex: sync.RWMutex{}, clock: clockwork.NewFakeClock(), } bc := make(chan *unit.UnitStateHeartbeat) sc := make(chan bool) wgs.Add(numPublishers) wgf.Add(numPublishers) go func() { usp.Run(bc, sc) }() // queue a bunch of unit states for publishing - occupy all publish workers wantPublished := make([]string, numPublishers) for i := 0; i < numPublishers; i++ { name := fmt.Sprintf("%d.service", i) wantPublished[i] = name usp.queueForPublish( name, &unit.UnitState{ UnitName: name, }, ) } // now queue some more unit states for publishing - expect them to hang go usp.queueForPublish("foo.service", &unit.UnitState{UnitName: "foo.service", ActiveState: "active"}) go usp.queueForPublish("bar.service", &unit.UnitState{}) go usp.queueForPublish("baz.service", &unit.UnitState{}) // re-queue one of the states; this should replace the above go usp.queueForPublish("foo.service", &unit.UnitState{UnitName: "foo.service", ActiveState: "inactive"}) // wait for all publish workers to start wgs.Wait() // so far, no states should be published, and the last three should be enqueued ws := []string{} mu.Lock() if !reflect.DeepEqual(states, ws) { t.Errorf("bad UnitStates: got %#v, want %#v", states, ws) } mu.Unlock() wtps := map[string]*unit.UnitState{ "foo.service": &unit.UnitState{ UnitName: "foo.service", ActiveState: "inactive", }, "bar.service": &unit.UnitState{}, "baz.service": &unit.UnitState{}, } usp.toPublishMutex.RLock() if !reflect.DeepEqual(usp.toPublishStates, wtps) { t.Errorf("bad toPublishStates: got %#v, want %#v", usp.toPublishStates, wtps) } usp.toPublishMutex.RUnlock() // end the registry delay by ticking past it just once - // expect three more publishers to start, and block wgs.Add(3) fclock.Advance(3 * time.Second) // wait for the original publishers to finish wgf.Wait() // now, all five original states should have been published ws = wantPublished mu.Lock() sort.Strings(states) if !reflect.DeepEqual(states, ws) { t.Errorf("bad published UnitStates: got %#v, want %#v", states, ws) } mu.Unlock() // wait for the new publishers to start wgs.Wait() // and the other states should be flushed from the toPublish queue wtps = map[string]*unit.UnitState{} usp.toPublishMutex.RLock() if !reflect.DeepEqual(usp.toPublishStates, wtps) { t.Errorf("bad toPublishStates: got %#v, want %#v", usp.toPublishStates, wtps) } usp.toPublishMutex.RUnlock() // but still not published mu.Lock() sort.Strings(states) if !reflect.DeepEqual(states, ws) { t.Errorf("bad published UnitStates: got %#v, want %#v", states, ws) } // clear the published states states = []string{} mu.Unlock() // prepare for the new publishers wgf.Add(3) // tick past the registry delay again so the new publishers continue fclock.Advance(10 * time.Second) // wait for them to complete wgf.Wait() // and now the other states should be published ws = []string{"bar.service", "baz.service", "foo.service"} mu.Lock() sort.Strings(states) if !reflect.DeepEqual(states, ws) { t.Errorf("bad UnitStates: got %#v, want %#v", states, ws) } mu.Unlock() // and toPublish queue should still be empty wtps = map[string]*unit.UnitState{} usp.toPublishMutex.RLock() if !reflect.DeepEqual(usp.toPublishStates, wtps) { t.Errorf("bad toPublishStates: got %#v, want %#v", usp.toPublishStates, wtps) } usp.toPublishMutex.RUnlock() }
func TestUnitStatePublisherRunTiming(t *testing.T) { fclock := clockwork.NewFakeClock() states := make([]*unit.UnitState, 0) mu := sync.Mutex{} // protects states from concurrent modification published := make(chan struct{}) pf := func(name string, us *unit.UnitState) { mu.Lock() states = append(states, us) mu.Unlock() go func() { published <- struct{}{} }() } usp := &UnitStatePublisher{ mach: &machine.FakeMachine{}, ttl: 5 * time.Second, publisher: pf, cache: make(map[string]*unit.UnitState), cacheMutex: sync.RWMutex{}, toPublish: make(chan string), toPublishStates: make(map[string]*unit.UnitState), toPublishMutex: sync.RWMutex{}, clock: fclock, } usp.cache = map[string]*unit.UnitState{ "foo.service": &unit.UnitState{ UnitName: "foo.service", ActiveState: "active", MachineID: "XXX", }, } bc := make(chan *unit.UnitStateHeartbeat) sc := make(chan bool) go func() { usp.Run(bc, sc) }() // yield so Run() is definitely waiting on After() fclock.BlockUntil(1) // tick less than the publish interval fclock.Advance(time.Second) select { case <-published: t.Fatal("UnitState published unexpectedly!") default: } want := []*unit.UnitState{} mu.Lock() if !reflect.DeepEqual(states, want) { t.Errorf("bad UnitStates: got %#v, want %#v", states, want) } mu.Unlock() // now up to the publish interval fclock.Advance(4 * time.Second) want = []*unit.UnitState{ &unit.UnitState{ UnitName: "foo.service", ActiveState: "active", MachineID: "XXX", }, } select { case <-published: mu.Lock() if !reflect.DeepEqual(states, want) { t.Errorf("bad UnitStates: got %#v, want %#v", states, want) } mu.Unlock() case <-time.After(time.Second): t.Fatalf("UnitState not published as expected!") } // reset states mu.Lock() states = []*unit.UnitState{} mu.Unlock() // yield so Run() is definitely waiting on After() fclock.BlockUntil(1) // tick less than the publish interval, again fclock.Advance(4 * time.Second) // no more should be published select { case <-published: t.Fatal("UnitState published unexpectedly!") default: } want = []*unit.UnitState{} mu.Lock() if !reflect.DeepEqual(states, want) { t.Errorf("bad UnitStates: got %#v, want %#v", states, want) } mu.Unlock() // tick way past the publish interval fclock.Advance(time.Hour) want = []*unit.UnitState{ &unit.UnitState{ UnitName: "foo.service", ActiveState: "active", MachineID: "XXX", }, } // state should be published, but just once (since it's still just a single event) select { case <-published: mu.Lock() if !reflect.DeepEqual(states, want) { t.Errorf("bad UnitStates: got %#v, want %#v", states, want) } mu.Unlock() case <-time.After(time.Second): t.Fatalf("UnitState not published as expected!") } // now stop the UnitStatePublisher close(sc) // reset states mu.Lock() states = []*unit.UnitState{} mu.Unlock() // yield so Run() is definitely waiting on After() fclock.BlockUntil(1) // tick way past the publish interval fclock.Advance(time.Hour) // no more states should be published select { case <-published: t.Fatal("UnitState published unexpectedly!") default: } want = []*unit.UnitState{} mu.Lock() if !reflect.DeepEqual(states, want) { t.Errorf("bad UnitStates: got %#v, want %#v", states, want) } mu.Unlock() }
func TestUnitStatePublisherRunQueuing(t *testing.T) { states := make([]string, 0) mu := sync.Mutex{} // protects states from concurrent modification var wg sync.WaitGroup wg.Add(numPublishers) block := make(chan struct{}) pf := func(name string, us *unit.UnitState) { wg.Done() <-block mu.Lock() states = append(states, name) mu.Unlock() } usp := &UnitStatePublisher{ mach: &machine.FakeMachine{ MachineState: machine.MachineState{ ID: "some_id", }, }, ttl: time.Hour, // we never expect to reach this publisher: pf, cache: make(map[string]*unit.UnitState), cacheMutex: sync.RWMutex{}, toPublish: make(chan string), toPublishStates: make(map[string]*unit.UnitState), toPublishMutex: sync.RWMutex{}, clock: clockwork.NewFakeClock(), } bc := make(chan *unit.UnitStateHeartbeat) sc := make(chan bool) go func() { usp.Run(bc, sc) }() // first, fill up the publishers with a bunch of things wcache := make(map[string]*unit.UnitState) for i := 0; i < numPublishers; i++ { name := fmt.Sprintf("%d.service", i) us := &unit.UnitState{ UnitName: name, } wcache[name] = us bc <- &unit.UnitStateHeartbeat{ Name: name, State: us, } } // wait until they're all started wg.Wait() // now the cache should contain them usp.cacheMutex.RLock() got := usp.cache if !reflect.DeepEqual(got, wcache) { t.Errorf("bad cache: got %#v, want %#v", got, wcache) } usp.cacheMutex.RUnlock() // as should the toPublish queue select { case update := <-usp.toPublish: t.Errorf("unexpected item in toPublish queue: %#v", update) default: } usp.cacheMutex.Lock() // flush the cache usp.cache = map[string]*unit.UnitState{ "foo.service": &unit.UnitState{ UnitName: "foo.service", ActiveState: "active", }, } usp.cacheMutex.Unlock() // now send a new UnitStateHeartbeat referencing something already in the cache bc <- &unit.UnitStateHeartbeat{ Name: "foo.service", State: &unit.UnitState{ UnitName: "foo.service", ActiveState: "inactive", }, } // since something changed, queue should be updated select { case got := <-usp.toPublish: want := "foo.service" if !reflect.DeepEqual(got, want) { t.Fatalf("update not as expected: got %#v, want %#v", got, want) } case <-time.After(time.Second): t.Fatalf("change not added to toPublish queue as expected!") } // as should the cache usp.cacheMutex.RLock() got = usp.cache wcache = map[string]*unit.UnitState{ "foo.service": &unit.UnitState{ UnitName: "foo.service", ActiveState: "inactive", MachineID: "some_id", }, } if !reflect.DeepEqual(got, wcache) { t.Errorf("cache not as expected: got %#v, want %#v", got, wcache) } usp.cacheMutex.RUnlock() // finally, send another of the same update bc <- &unit.UnitStateHeartbeat{ Name: "foo.service", State: &unit.UnitState{ UnitName: "foo.service", ActiveState: "inactive", }, } // same state as cache, so queue should _not_ be updated, nor should cache select { case update := <-usp.toPublish: t.Errorf("unexpected change in toPublish queue: %#v", update) default: } usp.cacheMutex.RLock() got = usp.cache if !reflect.DeepEqual(got, wcache) { t.Errorf("cache not as expected: got %#v, want %#v", got, wcache) } usp.cacheMutex.RUnlock() }
// TestPeriodicReconcilerRun attempts to validate the behaviour of the central Run // loop of the PeriodicReconciler func TestPeriodicReconcilerRun(t *testing.T) { ival := 5 * time.Hour fclock := clockwork.NewFakeClock() fes := &fakeEventStream{make(chan Event)} called := make(chan struct{}) rec := func() { go func() { called <- struct{}{} }() } pr := &reconciler{ ival: ival, rFunc: rec, eStream: fes, clock: fclock, } // launch the PeriodicReconciler in the background prDone := make(chan struct{}) stop := make(chan struct{}) go func() { pr.Run(stop) close(prDone) }() // reconcile should have occurred once at start-up select { case <-called: case <-time.After(time.Second): t.Fatalf("rFunc() not called at start-up as expected!") } // no further reconciles yet expected select { case <-called: t.Fatalf("rFunc() called unexpectedly!") default: } // now, send an event on the EventStream and ensure rFunc occurs fes.trigger() select { case <-called: case <-time.After(time.Second): t.Fatalf("rFunc() not called after trigger!") } // assert rFunc was only called once select { case <-called: t.Fatalf("rFunc() called unexpectedly!") default: } // another event should work OK fes.trigger() select { case <-called: case <-time.After(time.Second): t.Fatalf("rFunc() not called after trigger!") } // again, assert rFunc was only called once select { case <-called: t.Fatalf("rFunc() called unexpectedly!") default: } // now check that time changes have the expected effect fclock.Advance(2 * time.Hour) select { case <-called: t.Fatalf("rFunc() called unexpectedly!") default: } fclock.Advance(3 * time.Hour) select { case <-called: case <-time.After(time.Second): t.Fatalf("rFunc() not called after time event!") } // stop the PeriodicReconciler close(stop) // now, sending an event should do nothing fes.trigger() select { case <-called: t.Fatalf("rFunc() called unexpectedly!") default: } // and nor should changes in time fclock.Advance(10 * time.Hour) select { case <-called: t.Fatalf("rFunc() called unexpectedly!") default: } // and the PeriodicReconciler should have shut down select { case <-prDone: case <-time.After(time.Second): t.Fatalf("PeriodicReconciler.Run did not return after stop signal!") } }