func (s *notifyWorkerSuite) TestStop(c *gc.C) {
err := worker.Stop(s.worker)
c.Assert(err, gc.IsNil)
// After stop, Wait should return right away
err = waitShort(c, s.worker)
c.Assert(err, gc.IsNil)
}
func (s *workerSuite) TestWorkerPublishesInstanceIds(c *gc.C) {
s.PatchValue(&pollInterval, coretesting.LongWait+time.Second)
s.PatchValue(&initialRetryInterval, 5*time.Millisecond)
s.PatchValue(&maxRetryInterval, initialRetryInterval)
publishCh := make(chan []instance.Id, 100)
publish := func(apiServers [][]instance.HostPort, instanceIds []instance.Id) error {
publishCh <- instanceIds
return nil
}
st := newFakeState()
initState(c, st, 3)
w := newWorker(st, publisherFunc(publish))
defer func() {
c.Check(worker.Stop(w), gc.IsNil)
}()
select {
case instanceIds := <-publishCh:
c.Assert(instanceIds, jc.SameContents, []instance.Id{"id-10", "id-11", "id-12"})
case <-time.After(coretesting.LongWait):
c.Errorf("timed out waiting for publish")
}
}
func (s *workerSuite) TestStateServersArePublished(c *gc.C) {
publishCh := make(chan [][]instance.HostPort)
publish := func(apiServers [][]instance.HostPort, instanceIds []instance.Id) error {
publishCh <- apiServers
return nil
}
st := newFakeState()
initState(c, st, 3)
w := newWorker(st, publisherFunc(publish))
defer func() {
c.Check(worker.Stop(w), gc.IsNil)
}()
select {
case servers := <-publishCh:
assertAPIHostPorts(c, servers, expectedAPIHostPorts(3))
case <-time.After(coretesting.LongWait):
c.Fatalf("timed out waiting for publish")
}
// Change one of the servers' API addresses and check that it's published.
newMachine10APIHostPorts := addressesWithPort(apiPort, "0.2.8.124")
st.machine("10").setAPIHostPorts(newMachine10APIHostPorts)
select {
case servers := <-publishCh:
expected := expectedAPIHostPorts(3)
expected[0] = newMachine10APIHostPorts
assertAPIHostPorts(c, servers, expected)
case <-time.After(coretesting.LongWait):
c.Fatalf("timed out waiting for publish")
}
}
func (s *workerJujuConnSuite) TestPublisherSetsAPIHostPorts(c *gc.C) {
st := newFakeState()
initState(c, st, 3)
watcher := s.State.WatchAPIHostPorts()
cwatch := statetesting.NewNotifyWatcherC(c, s.State, watcher)
cwatch.AssertOneChange()
statePublish := newPublisher(s.State)
// Wrap the publisher so that we can call StartSync immediately
// after the publishAPIServers method is called.
publish := func(apiServers [][]instance.HostPort, instanceIds []instance.Id) error {
err := statePublish.publishAPIServers(apiServers, instanceIds)
s.State.StartSync()
return err
}
w := newWorker(st, publisherFunc(publish))
defer func() {
c.Check(worker.Stop(w), gc.IsNil)
}()
cwatch.AssertOneChange()
hps, err := s.State.APIHostPorts()
c.Assert(err, gc.IsNil)
assertAPIHostPorts(c, hps, expectedAPIHostPorts(3))
}
func (s *workerSuite) TestAddressChange(c *gc.C) {
st := newFakeState()
initState(c, st, 3)
memberWatcher := st.session.members.Watch()
mustNext(c, memberWatcher)
assertMembers(c, memberWatcher.Value(), mkMembers("0v"))
logger.Infof("starting worker")
w := newWorker(st, noPublisher{})
defer func() {
c.Check(worker.Stop(w), gc.IsNil)
}()
// Wait for the worker to set the initial members.
mustNext(c, memberWatcher)
assertMembers(c, memberWatcher.Value(), mkMembers("0v 1 2"))
// Change an address and wait for it to be changed in the
// members.
st.machine("11").setStateHostPort("0.1.99.99:9876")
mustNext(c, memberWatcher)
expectMembers := mkMembers("0v 1 2")
expectMembers[1].Address = "0.1.99.99:9876"
assertMembers(c, memberWatcher.Value(), expectMembers)
}
func (s *notifyWorkerSuite) TestChangesTriggerHandler(c *gc.C) {
s.actor.CheckActions(c, "setup")
s.actor.watcher.TriggerChange(c)
waitForHandledNotify(c, s.actor.handled)
s.actor.CheckActions(c, "setup", "handler")
s.actor.watcher.TriggerChange(c)
waitForHandledNotify(c, s.actor.handled)
s.actor.watcher.TriggerChange(c)
waitForHandledNotify(c, s.actor.handled)
s.actor.CheckActions(c, "setup", "handler", "handler", "handler")
c.Assert(worker.Stop(s.worker), gc.IsNil)
s.actor.CheckActions(c, "setup", "handler", "handler", "handler", "teardown")
}
// During teardown we try to stop the worker, but don't hang the test suite if
// Stop never returns
func (s *notifyWorkerSuite) stopWorker(c *gc.C) {
if s.worker == nil {
return
}
done := make(chan error)
go func() {
done <- worker.Stop(s.worker)
}()
err := waitForTimeout(c, done, coretesting.LongWait)
c.Check(err, gc.IsNil)
s.actor = nil
s.worker = nil
}
func (s *minUnitsWorkerSuite) TestMinUnitsWorker(c *gc.C) {
mu := minunitsworker.NewMinUnitsWorker(s.State)
defer func() { c.Assert(worker.Stop(mu), gc.IsNil) }()
// Set up services and units for later use.
wordpress := s.AddTestingService(c, "wordpress", s.AddTestingCharm(c, "wordpress"))
mysql := s.AddTestingService(c, "mysql", s.AddTestingCharm(c, "mysql"))
unit, err := wordpress.AddUnit()
c.Assert(err, gc.IsNil)
_, err = wordpress.AddUnit()
c.Assert(err, gc.IsNil)
// Set up minimum units for services.
err = wordpress.SetMinUnits(3)
c.Assert(err, gc.IsNil)
err = mysql.SetMinUnits(2)
c.Assert(err, gc.IsNil)
// Remove a unit for a service.
err = unit.Destroy()
c.Assert(err, gc.IsNil)
timeout := time.After(coretesting.LongWait)
for {
s.State.StartSync()
select {
case <-time.After(coretesting.ShortWait):
wordpressUnits, err := wordpress.AllUnits()
c.Assert(err, gc.IsNil)
mysqlUnits, err := mysql.AllUnits()
c.Assert(err, gc.IsNil)
wordpressCount := len(wordpressUnits)
mysqlCount := len(mysqlUnits)
if wordpressCount == 3 && mysqlCount == 2 {
return
}
logger.Infof("wordpress units: %d; mysql units: %d", wordpressCount, mysqlCount)
case <-timeout:
c.Fatalf("timed out waiting for minunits events")
}
}
}
func (s *workerSuite) TestSetMembersErrorIsNotFatal(c *gc.C) {
st := newFakeState()
initState(c, st, 3)
st.session.setStatus(mkStatuses("0p 1s 2s"))
var isSet voyeur.Value
count := 0
setErrorFuncFor("Session.Set", func() error {
isSet.Set(count)
count++
return errors.New("sample")
})
s.PatchValue(&initialRetryInterval, 10*time.Microsecond)
s.PatchValue(&maxRetryInterval, coretesting.ShortWait/4)
expectedIterations := 0
for d := initialRetryInterval; d < maxRetryInterval*2; d *= 2 {
expectedIterations++
}
w := newWorker(st, noPublisher{})
defer func() {
c.Check(worker.Stop(w), gc.IsNil)
}()
isSetWatcher := isSet.Watch()
n0 := mustNext(c, isSetWatcher).(int)
time.Sleep(maxRetryInterval * 2)
n1 := mustNext(c, isSetWatcher).(int)
// The worker should have backed off exponentially...
c.Assert(n1-n0, jc.LessThan, expectedIterations+1)
c.Logf("actual iterations %d; expected iterations %d", n1-n0, expectedIterations)
// ... but only up to the maximum retry interval
n0 = mustNext(c, isSetWatcher).(int)
time.Sleep(maxRetryInterval * 2)
n1 = mustNext(c, isSetWatcher).(int)
c.Assert(n1-n0, jc.LessThan, 3)
}
func (s *workerSuite) TestWorkerRetriesOnPublishError(c *gc.C) {
s.PatchValue(&pollInterval, coretesting.LongWait+time.Second)
s.PatchValue(&initialRetryInterval, 5*time.Millisecond)
s.PatchValue(&maxRetryInterval, initialRetryInterval)
publishCh := make(chan [][]instance.HostPort, 100)
count := 0
publish := func(apiServers [][]instance.HostPort, instanceIds []instance.Id) error {
publishCh <- apiServers
count++
if count <= 3 {
return fmt.Errorf("publish error")
}
return nil
}
st := newFakeState()
initState(c, st, 3)
w := newWorker(st, publisherFunc(publish))
defer func() {
c.Check(worker.Stop(w), gc.IsNil)
}()
for i := 0; i < 4; i++ {
select {
case servers := <-publishCh:
assertAPIHostPorts(c, servers, expectedAPIHostPorts(3))
case <-time.After(coretesting.LongWait):
c.Fatalf("timed out waiting for publish #%d", i)
}
}
select {
case <-publishCh:
c.Errorf("unexpected publish event")
case <-time.After(coretesting.ShortWait):
}
}
func (n *notifier) Stop() error {
return worker.Stop(n)
}
func stop(c *gc.C, w worker.Worker) {
c.Assert(worker.Stop(w), gc.IsNil)
}
func (s *workerJujuConnSuite) TestStartStop(c *gc.C) {
w, err := New(s.State)
c.Assert(err, gc.IsNil)
err = worker.Stop(w)
c.Assert(err, gc.IsNil)
}
func (s *workerSuite) TestSetsAndUpdatesMembers(c *gc.C) {
s.PatchValue(&pollInterval, 5*time.Millisecond)
st := newFakeState()
initState(c, st, 3)
memberWatcher := st.session.members.Watch()
mustNext(c, memberWatcher)
assertMembers(c, memberWatcher.Value(), mkMembers("0v"))
logger.Infof("starting worker")
w := newWorker(st, noPublisher{})
defer func() {
c.Check(worker.Stop(w), gc.IsNil)
}()
// Wait for the worker to set the initial members.
mustNext(c, memberWatcher)
assertMembers(c, memberWatcher.Value(), mkMembers("0v 1 2"))
// Update the status of the new members
// and check that they become voting.
c.Logf("updating new member status")
st.session.setStatus(mkStatuses("0p 1s 2s"))
mustNext(c, memberWatcher)
assertMembers(c, memberWatcher.Value(), mkMembers("0v 1v 2v"))
c.Logf("adding another machine")
// Add another machine.
m13 := st.addMachine("13", false)
m13.setStateHostPort(fmt.Sprintf("0.1.2.%d:%d", 13, mongoPort))
st.setStateServers("10", "11", "12", "13")
c.Logf("waiting for new member to be added")
mustNext(c, memberWatcher)
assertMembers(c, memberWatcher.Value(), mkMembers("0v 1v 2v 3"))
// Remove vote from an existing member;
// and give it to the new machine.
// Also set the status of the new machine to
// healthy.
c.Logf("removing vote from machine 10 and adding it to machine 13")
st.machine("10").setWantsVote(false)
st.machine("13").setWantsVote(true)
st.session.setStatus(mkStatuses("0p 1s 2s 3s"))
// Check that the new machine gets the vote and the
// old machine loses it.
c.Logf("waiting for vote switch")
mustNext(c, memberWatcher)
assertMembers(c, memberWatcher.Value(), mkMembers("0 1v 2v 3v"))
c.Logf("removing old machine")
// Remove the old machine.
st.removeMachine("10")
st.setStateServers("11", "12", "13")
// Check that it's removed from the members.
c.Logf("waiting for removal")
mustNext(c, memberWatcher)
assertMembers(c, memberWatcher.Value(), mkMembers("1v 2v 3v"))
}