func (s *suite) TestNothingHappensWhenEnvIsSeenAgain(c *gc.C) {
// This could happen if there's a change to an environment doc but
// it's otherwise still alive (unlikely but possible).
st := newStateWithFakeWatcher(s.State)
uuid := st.EnvironUUID()
otherSt := s.makeEnvironment(c)
otherUUID := otherSt.EnvironUUID()
m := envworkermanager.NewEnvWorkerManager(st, s.startEnvWorker, s.dyingEnvWorker, time.Millisecond)
defer m.Kill()
// First time: runners started
st.sendEnvChange(uuid)
st.sendEnvChange(otherUUID)
s.seeRunnersStart(c, 2)
// Second time: no runners started
st.sendEnvChange(uuid)
s.checkNoRunnersStart(c)
destroyEnvironment(c, otherSt)
st.sendEnvChange(otherUUID)
s.seeRunnersStart(c, 1)
st.sendEnvChange(otherUUID)
s.checkNoRunnersStart(c)
}
func (s *suite) TestLoopExitKillsRunner(c *gc.C) {
// If something causes EnvWorkerManager.loop to exit that isn't Kill() then it should stop the runner.
// Currently the best way to cause this is to make
// m.st.GetEnvironment(tag) fail with any error other than NotFound
otherSt := s.makeEnvironment(c)
st := newStateWithFailingGetEnvironment(s.State)
uuid := st.EnvironUUID()
m := envworkermanager.NewEnvWorkerManager(st, s.startEnvWorker, s.dyingEnvWorker, time.Millisecond)
defer m.Kill()
// First time: runners started
st.sendEnvChange(uuid)
runners := s.seeRunnersStart(c, 1)
c.Assert(runners[0].killed, jc.IsFalse)
destroyEnvironment(c, otherSt)
st.sendEnvChange(otherSt.EnvironUUID())
dyingRunner := s.seeRunnersStart(c, 1)[0]
c.Assert(dyingRunner.killed, jc.IsFalse)
// Now we start failing
st.shouldFail = true
st.sendEnvChange(uuid)
// This should kill the manager
err := waitOrFatal(c, m.Wait)
c.Assert(err, gc.ErrorMatches, "error loading environment .*: unable to GetEnvironment")
// And that should kill all the runners
c.Assert(runners[0].killed, jc.IsTrue)
c.Assert(dyingRunner.killed, jc.IsTrue)
}
func (s *suite) TestStateIsClosedIfStartEnvWorkersFails(c *gc.C) {
// If State is not closed when startEnvWorker errors, MgoSuite's
// dirty socket detection will pick up the leaked socket and
// panic.
s.startErr = worker.ErrTerminateAgent // This will make envWorkerManager exit.
m := envworkermanager.NewEnvWorkerManager(s.State, s.startEnvWorker, s.dyingEnvWorker, time.Millisecond)
waitOrFatal(c, m.Wait)
}
func (s *suite) TestStartsWorkersForNewEnv(c *gc.C) {
m := envworkermanager.NewEnvWorkerManager(s.State, s.startEnvWorker, s.dyingEnvWorker, time.Millisecond)
defer m.Kill()
s.seeRunnersStart(c, 1) // Runner for state server env
// Create another environment and watch a runner be created for it.
st2 := s.makeEnvironment(c)
runner := s.seeRunnersStart(c, 1)[0]
c.Assert(runner.envUUID, gc.Equals, st2.EnvironUUID())
}
func (s *suite) TestNonFatalErrorCausesRunnerRestart(c *gc.C) {
m := envworkermanager.NewEnvWorkerManager(s.State, s.startEnvWorker, s.dyingEnvWorker, time.Millisecond)
defer m.Kill()
runner0 := s.seeRunnersStart(c, 1)[0]
runner0.tomb.Kill(errors.New("trivial"))
runner0.tomb.Done()
s.seeRunnersStart(c, 1)
}
func (s *suite) TestFatalErrorKillsEnvWorkerManager(c *gc.C) {
m := envworkermanager.NewEnvWorkerManager(s.State, s.startEnvWorker, s.dyingEnvWorker, time.Millisecond)
runner := s.seeRunnersStart(c, 1)[0]
runner.tomb.Kill(worker.ErrTerminateAgent)
runner.tomb.Done()
err := waitOrFatal(c, m.Wait)
c.Assert(errors.Cause(err), gc.Equals, worker.ErrTerminateAgent)
}
func (s *suite) TestStartsWorkersForPreExistingEnvs(c *gc.C) {
moreState := s.makeEnvironment(c)
var seenEnvs []string
m := envworkermanager.NewEnvWorkerManager(s.State, s.startEnvWorker)
defer m.Kill()
for _, r := range s.seeRunnersStart(c, 2) {
seenEnvs = append(seenEnvs, r.envUUID)
}
c.Assert(seenEnvs, jc.SameContents,
[]string{s.State.EnvironUUID(), moreState.EnvironUUID()},
)
}
func (s *suite) TestNothingHappensWhenUnknownEnvReported(c *gc.C) {
// This could perhaps happen when an environment is dying just as
// the EnvWorkerManager is coming up (unlikely but possible).
st := newStateWithFakeWatcher(s.State)
m := envworkermanager.NewEnvWorkerManager(st, s.startEnvWorker, s.dyingEnvWorker, time.Millisecond)
defer m.Kill()
st.sendEnvChange("unknown-env-uuid")
s.checkNoRunnersStart(c)
// Existing environment still works.
st.sendEnvChange(st.EnvironUUID())
s.seeRunnersStart(c, 1)
}
func (s *suite) TestKillPropagates(c *gc.C) {
s.makeEnvironment(c)
m := envworkermanager.NewEnvWorkerManager(s.State, s.startEnvWorker)
runners := s.seeRunnersStart(c, 2)
c.Assert(runners[0].killed, jc.IsFalse)
c.Assert(runners[1].killed, jc.IsFalse)
m.Kill()
err := waitOrFatal(c, m.Wait)
c.Assert(err, jc.ErrorIsNil)
c.Assert(runners[0].killed, jc.IsTrue)
c.Assert(runners[1].killed, jc.IsTrue)
}
func (s *suite) TestWorkerErrorIsPropagatedWhenKilled(c *gc.C) {
st := newStateWithFakeWatcher(s.State)
started := make(chan struct{}, 1)
m := envworkermanager.NewEnvWorkerManager(st, func(envworkermanager.InitialState, *state.State) (worker.Worker, error) {
c.Logf("starting worker")
started <- struct{}{}
return &errorWhenKilledWorker{
err: &cmdutil.FatalError{"an error"},
}, nil
}, s.dyingEnvWorker, time.Millisecond)
st.sendEnvChange(st.EnvironUUID())
s.State.StartSync()
<-started
m.Kill()
err := m.Wait()
c.Assert(err, gc.ErrorMatches, "an error")
}
func (s *suite) TestKillPropagates(c *gc.C) {
otherSt := s.makeEnvironment(c)
m := envworkermanager.NewEnvWorkerManager(s.State, s.startEnvWorker, s.dyingEnvWorker, time.Millisecond)
runners := s.seeRunnersStart(c, 2)
c.Assert(runners[0].killed, jc.IsFalse)
c.Assert(runners[1].killed, jc.IsFalse)
destroyEnvironment(c, otherSt)
dyingRunner := s.seeRunnersStart(c, 1)[0]
c.Assert(dyingRunner.killed, jc.IsFalse)
m.Kill()
err := waitOrFatal(c, m.Wait)
c.Assert(err, jc.ErrorIsNil)
c.Assert(runners[0].killed, jc.IsTrue)
c.Assert(runners[1].killed, jc.IsTrue)
c.Assert(dyingRunner.killed, jc.IsTrue)
}
func (s *suite) TestStopsWorkersWhenEnvGoesAway(c *gc.C) {
m := envworkermanager.NewEnvWorkerManager(s.State, s.startEnvWorker, s.dyingEnvWorker, time.Millisecond)
defer m.Kill()
runner0 := s.seeRunnersStart(c, 1)[0]
// Create an environment and grab the runner for it.
otherState := s.makeEnvironment(c)
runner1 := s.seeRunnersStart(c, 1)[0]
// Set environment to dying.
destroyEnvironment(c, otherState)
s.seeRunnersStart(c, 1) // dying env runner
// Set environment to dead.
err := otherState.ProcessDyingEnviron()
c.Assert(err, jc.ErrorIsNil)
// See that the first runner is still running but the runner for
// the new environment is stopped.
s.State.StartSync()
select {
case <-runner0.tomb.Dead():
c.Fatal("first runner should not die here")
case <-runner1.tomb.Dead():
break
case <-time.After(testing.LongWait):
c.Fatal("timed out waiting for runner to die")
}
// Make sure the first runner doesn't get stopped.
s.State.StartSync()
select {
case <-runner0.tomb.Dead():
c.Fatal("first runner should not die here")
case <-time.After(testing.ShortWait):
break
}
}