func TestUnitStatePublisherRunWithDelays(t *testing.T) {
states := make([]string, 0)
mu := sync.Mutex{} // protects states from concurrent modification
fclock := clockwork.NewFakeClock()
var wgs, wgf sync.WaitGroup // track starting and stopping of publishers
slowpf := func(name string, us *unit.UnitState) {
wgs.Done()
// simulate a delay in communication with the registry
fclock.Sleep(3 * time.Second)
mu.Lock()
states = append(states, name)
mu.Unlock()
wgf.Done()
}
usp := &UnitStatePublisher{
mach: &machine.FakeMachine{},
ttl: time.Hour, // we never expect to reach this
publisher: slowpf,
cache: make(map[string]*unit.UnitState),
cacheMutex: sync.RWMutex{},
toPublish: make(chan string),
toPublishStates: make(map[string]*unit.UnitState),
toPublishMutex: sync.RWMutex{},
clock: clockwork.NewFakeClock(),
}
bc := make(chan *unit.UnitStateHeartbeat)
sc := make(chan bool)
wgs.Add(numPublishers)
wgf.Add(numPublishers)
go func() {
usp.Run(bc, sc)
}()
// queue a bunch of unit states for publishing - occupy all publish workers
wantPublished := make([]string, numPublishers)
for i := 0; i < numPublishers; i++ {
name := fmt.Sprintf("%d.service", i)
wantPublished[i] = name
usp.queueForPublish(
name,
&unit.UnitState{
UnitName: name,
},
)
}
// now queue some more unit states for publishing - expect them to hang
go usp.queueForPublish("foo.service", &unit.UnitState{UnitName: "foo.service", ActiveState: "active"})
go usp.queueForPublish("bar.service", &unit.UnitState{})
go usp.queueForPublish("baz.service", &unit.UnitState{})
// re-queue one of the states; this should replace the above
go usp.queueForPublish("foo.service", &unit.UnitState{UnitName: "foo.service", ActiveState: "inactive"})
// wait for all publish workers to start
wgs.Wait()
// so far, no states should be published, and the last three should be enqueued
ws := []string{}
mu.Lock()
if !reflect.DeepEqual(states, ws) {
t.Errorf("bad UnitStates: got %#v, want %#v", states, ws)
}
mu.Unlock()
wtps := map[string]*unit.UnitState{
"foo.service": &unit.UnitState{
UnitName: "foo.service",
ActiveState: "inactive",
},
"bar.service": &unit.UnitState{},
"baz.service": &unit.UnitState{},
}
usp.toPublishMutex.RLock()
if !reflect.DeepEqual(usp.toPublishStates, wtps) {
t.Errorf("bad toPublishStates: got %#v, want %#v", usp.toPublishStates, wtps)
}
usp.toPublishMutex.RUnlock()
// end the registry delay by ticking past it just once -
// expect three more publishers to start, and block
wgs.Add(3)
fclock.Advance(3 * time.Second)
// wait for the original publishers to finish
wgf.Wait()
// now, all five original states should have been published
ws = wantPublished
mu.Lock()
sort.Strings(states)
if !reflect.DeepEqual(states, ws) {
t.Errorf("bad published UnitStates: got %#v, want %#v", states, ws)
}
mu.Unlock()
// wait for the new publishers to start
wgs.Wait()
// and the other states should be flushed from the toPublish queue
wtps = map[string]*unit.UnitState{}
usp.toPublishMutex.RLock()
if !reflect.DeepEqual(usp.toPublishStates, wtps) {
t.Errorf("bad toPublishStates: got %#v, want %#v", usp.toPublishStates, wtps)
}
usp.toPublishMutex.RUnlock()
// but still not published
mu.Lock()
sort.Strings(states)
if !reflect.DeepEqual(states, ws) {
t.Errorf("bad published UnitStates: got %#v, want %#v", states, ws)
}
// clear the published states
states = []string{}
mu.Unlock()
// prepare for the new publishers
wgf.Add(3)
// tick past the registry delay again so the new publishers continue
fclock.Advance(10 * time.Second)
// wait for them to complete
wgf.Wait()
// and now the other states should be published
ws = []string{"bar.service", "baz.service", "foo.service"}
mu.Lock()
sort.Strings(states)
if !reflect.DeepEqual(states, ws) {
t.Errorf("bad UnitStates: got %#v, want %#v", states, ws)
}
mu.Unlock()
// and toPublish queue should still be empty
wtps = map[string]*unit.UnitState{}
usp.toPublishMutex.RLock()
if !reflect.DeepEqual(usp.toPublishStates, wtps) {
t.Errorf("bad toPublishStates: got %#v, want %#v", usp.toPublishStates, wtps)
}
usp.toPublishMutex.RUnlock()
}
func TestUnitStatePublisherRunTiming(t *testing.T) {
fclock := clockwork.NewFakeClock()
states := make([]*unit.UnitState, 0)
mu := sync.Mutex{} // protects states from concurrent modification
published := make(chan struct{})
pf := func(name string, us *unit.UnitState) {
mu.Lock()
states = append(states, us)
mu.Unlock()
go func() {
published <- struct{}{}
}()
}
usp := &UnitStatePublisher{
mach: &machine.FakeMachine{},
ttl: 5 * time.Second,
publisher: pf,
cache: make(map[string]*unit.UnitState),
cacheMutex: sync.RWMutex{},
toPublish: make(chan string),
toPublishStates: make(map[string]*unit.UnitState),
toPublishMutex: sync.RWMutex{},
clock: fclock,
}
usp.cache = map[string]*unit.UnitState{
"foo.service": &unit.UnitState{
UnitName: "foo.service",
ActiveState: "active",
MachineID: "XXX",
},
}
bc := make(chan *unit.UnitStateHeartbeat)
sc := make(chan bool)
go func() {
usp.Run(bc, sc)
}()
// yield so Run() is definitely waiting on After()
fclock.BlockUntil(1)
// tick less than the publish interval
fclock.Advance(time.Second)
select {
case <-published:
t.Fatal("UnitState published unexpectedly!")
default:
}
want := []*unit.UnitState{}
mu.Lock()
if !reflect.DeepEqual(states, want) {
t.Errorf("bad UnitStates: got %#v, want %#v", states, want)
}
mu.Unlock()
// now up to the publish interval
fclock.Advance(4 * time.Second)
want = []*unit.UnitState{
&unit.UnitState{
UnitName: "foo.service",
ActiveState: "active",
MachineID: "XXX",
},
}
select {
case <-published:
mu.Lock()
if !reflect.DeepEqual(states, want) {
t.Errorf("bad UnitStates: got %#v, want %#v", states, want)
}
mu.Unlock()
case <-time.After(time.Second):
t.Fatalf("UnitState not published as expected!")
}
// reset states
mu.Lock()
states = []*unit.UnitState{}
mu.Unlock()
// yield so Run() is definitely waiting on After()
fclock.BlockUntil(1)
// tick less than the publish interval, again
fclock.Advance(4 * time.Second)
// no more should be published
select {
case <-published:
t.Fatal("UnitState published unexpectedly!")
default:
}
want = []*unit.UnitState{}
mu.Lock()
if !reflect.DeepEqual(states, want) {
t.Errorf("bad UnitStates: got %#v, want %#v", states, want)
}
mu.Unlock()
// tick way past the publish interval
fclock.Advance(time.Hour)
want = []*unit.UnitState{
&unit.UnitState{
UnitName: "foo.service",
ActiveState: "active",
MachineID: "XXX",
},
}
// state should be published, but just once (since it's still just a single event)
select {
case <-published:
mu.Lock()
if !reflect.DeepEqual(states, want) {
t.Errorf("bad UnitStates: got %#v, want %#v", states, want)
}
mu.Unlock()
case <-time.After(time.Second):
t.Fatalf("UnitState not published as expected!")
}
// now stop the UnitStatePublisher
close(sc)
// reset states
mu.Lock()
states = []*unit.UnitState{}
mu.Unlock()
// yield so Run() is definitely waiting on After()
fclock.BlockUntil(1)
// tick way past the publish interval
fclock.Advance(time.Hour)
// no more states should be published
select {
case <-published:
t.Fatal("UnitState published unexpectedly!")
default:
}
want = []*unit.UnitState{}
mu.Lock()
if !reflect.DeepEqual(states, want) {
t.Errorf("bad UnitStates: got %#v, want %#v", states, want)
}
mu.Unlock()
}
func TestUnitStatePublisherRunQueuing(t *testing.T) {
states := make([]string, 0)
mu := sync.Mutex{} // protects states from concurrent modification
var wg sync.WaitGroup
wg.Add(numPublishers)
block := make(chan struct{})
pf := func(name string, us *unit.UnitState) {
wg.Done()
<-block
mu.Lock()
states = append(states, name)
mu.Unlock()
}
usp := &UnitStatePublisher{
mach: &machine.FakeMachine{
MachineState: machine.MachineState{
ID: "some_id",
},
},
ttl: time.Hour, // we never expect to reach this
publisher: pf,
cache: make(map[string]*unit.UnitState),
cacheMutex: sync.RWMutex{},
toPublish: make(chan string),
toPublishStates: make(map[string]*unit.UnitState),
toPublishMutex: sync.RWMutex{},
clock: clockwork.NewFakeClock(),
}
bc := make(chan *unit.UnitStateHeartbeat)
sc := make(chan bool)
go func() {
usp.Run(bc, sc)
}()
// first, fill up the publishers with a bunch of things
wcache := make(map[string]*unit.UnitState)
for i := 0; i < numPublishers; i++ {
name := fmt.Sprintf("%d.service", i)
us := &unit.UnitState{
UnitName: name,
}
wcache[name] = us
bc <- &unit.UnitStateHeartbeat{
Name: name,
State: us,
}
}
// wait until they're all started
wg.Wait()
// now the cache should contain them
usp.cacheMutex.RLock()
got := usp.cache
if !reflect.DeepEqual(got, wcache) {
t.Errorf("bad cache: got %#v, want %#v", got, wcache)
}
usp.cacheMutex.RUnlock()
// as should the toPublish queue
select {
case update := <-usp.toPublish:
t.Errorf("unexpected item in toPublish queue: %#v", update)
default:
}
usp.cacheMutex.Lock()
// flush the cache
usp.cache = map[string]*unit.UnitState{
"foo.service": &unit.UnitState{
UnitName: "foo.service",
ActiveState: "active",
},
}
usp.cacheMutex.Unlock()
// now send a new UnitStateHeartbeat referencing something already in the cache
bc <- &unit.UnitStateHeartbeat{
Name: "foo.service",
State: &unit.UnitState{
UnitName: "foo.service",
ActiveState: "inactive",
},
}
// since something changed, queue should be updated
select {
case got := <-usp.toPublish:
want := "foo.service"
if !reflect.DeepEqual(got, want) {
t.Fatalf("update not as expected: got %#v, want %#v", got, want)
}
case <-time.After(time.Second):
t.Fatalf("change not added to toPublish queue as expected!")
}
// as should the cache
usp.cacheMutex.RLock()
got = usp.cache
wcache = map[string]*unit.UnitState{
"foo.service": &unit.UnitState{
UnitName: "foo.service",
ActiveState: "inactive",
MachineID: "some_id",
},
}
if !reflect.DeepEqual(got, wcache) {
t.Errorf("cache not as expected: got %#v, want %#v", got, wcache)
}
usp.cacheMutex.RUnlock()
// finally, send another of the same update
bc <- &unit.UnitStateHeartbeat{
Name: "foo.service",
State: &unit.UnitState{
UnitName: "foo.service",
ActiveState: "inactive",
},
}
// same state as cache, so queue should _not_ be updated, nor should cache
select {
case update := <-usp.toPublish:
t.Errorf("unexpected change in toPublish queue: %#v", update)
default:
}
usp.cacheMutex.RLock()
got = usp.cache
if !reflect.DeepEqual(got, wcache) {
t.Errorf("cache not as expected: got %#v, want %#v", got, wcache)
}
usp.cacheMutex.RUnlock()
}
// TestPeriodicReconcilerRun attempts to validate the behaviour of the central Run
// loop of the PeriodicReconciler
func TestPeriodicReconcilerRun(t *testing.T) {
ival := 5 * time.Hour
fclock := clockwork.NewFakeClock()
fes := &fakeEventStream{make(chan Event)}
called := make(chan struct{})
rec := func() {
go func() {
called <- struct{}{}
}()
}
pr := &reconciler{
ival: ival,
rFunc: rec,
eStream: fes,
clock: fclock,
}
// launch the PeriodicReconciler in the background
prDone := make(chan struct{})
stop := make(chan struct{})
go func() {
pr.Run(stop)
close(prDone)
}()
// reconcile should have occurred once at start-up
select {
case <-called:
case <-time.After(time.Second):
t.Fatalf("rFunc() not called at start-up as expected!")
}
// no further reconciles yet expected
select {
case <-called:
t.Fatalf("rFunc() called unexpectedly!")
default:
}
// now, send an event on the EventStream and ensure rFunc occurs
fes.trigger()
select {
case <-called:
case <-time.After(time.Second):
t.Fatalf("rFunc() not called after trigger!")
}
// assert rFunc was only called once
select {
case <-called:
t.Fatalf("rFunc() called unexpectedly!")
default:
}
// another event should work OK
fes.trigger()
select {
case <-called:
case <-time.After(time.Second):
t.Fatalf("rFunc() not called after trigger!")
}
// again, assert rFunc was only called once
select {
case <-called:
t.Fatalf("rFunc() called unexpectedly!")
default:
}
// now check that time changes have the expected effect
fclock.Advance(2 * time.Hour)
select {
case <-called:
t.Fatalf("rFunc() called unexpectedly!")
default:
}
fclock.Advance(3 * time.Hour)
select {
case <-called:
case <-time.After(time.Second):
t.Fatalf("rFunc() not called after time event!")
}
// stop the PeriodicReconciler
close(stop)
// now, sending an event should do nothing
fes.trigger()
select {
case <-called:
t.Fatalf("rFunc() called unexpectedly!")
default:
}
// and nor should changes in time
fclock.Advance(10 * time.Hour)
select {
case <-called:
t.Fatalf("rFunc() called unexpectedly!")
default:
}
// and the PeriodicReconciler should have shut down
select {
case <-prDone:
case <-time.After(time.Second):
t.Fatalf("PeriodicReconciler.Run did not return after stop signal!")
}
}