func (s *suite) TestClosedChangesChannel(c *gc.C) {
s.runDirtyTest(c, func(w worker.Worker, backend *mockBackend) {
backend.sendModelChange("uuid1", "uuid2")
workers := s.waitWorkers(c, 2)
close(backend.envWatcher.changes)
err := workertest.CheckKilled(c, w)
c.Check(err, gc.ErrorMatches, "changes stopped")
for _, worker := range workers {
workertest.CheckKilled(c, worker)
}
s.assertNoWorkers(c)
})
}
func (*WorkerSuite) TestStopKills(c *gc.C) {
w := workertest.NewErrorWorker(nil)
defer workertest.CleanKill(c, w)
worker.Stop(w)
workertest.CheckKilled(c, w)
}
func (s *Suite) TestSUCCESSMinionWaitTimeout(c *gc.C) {
// The SUCCESS phase is special in that even if some minions fail
// to report the migration should continue. There's no turning
// back from SUCCESS.
s.facade.queueStatus(s.makeStatus(coremigration.SUCCESS))
worker, err := migrationmaster.New(s.config)
c.Assert(err, jc.ErrorIsNil)
defer workertest.DirtyKill(c, worker)
select {
case <-s.clock.Alarms():
case <-time.After(coretesting.LongWait):
c.Fatal("timed out waiting for clock.After call")
}
// Move time ahead in order to trigger timeout.
s.clock.Advance(15 * time.Minute)
err = workertest.CheckKilled(c, worker)
c.Assert(err, gc.Equals, migrationmaster.ErrMigrated)
s.stub.CheckCalls(c, joinCalls(
watchStatusLockdownCalls,
[]jujutesting.StubCall{
{"facade.WatchMinionReports", nil},
{"facade.SetPhase", []interface{}{coremigration.LOGTRANSFER}},
{"facade.SetPhase", []interface{}{coremigration.REAP}},
{"facade.Reap", nil},
{"facade.SetPhase", []interface{}{coremigration.DONE}},
},
))
}
func (s *Suite) TestClosedWatcherChannel(c *gc.C) {
close(s.client.watcher.changes)
w, err := migrationminion.New(s.config)
c.Assert(err, jc.ErrorIsNil)
err = workertest.CheckKilled(c, w)
c.Check(err, gc.ErrorMatches, "watcher channel closed")
}
func (*WorkerSuite) TestStopWaits(c *gc.C) {
w := workertest.NewForeverWorker(nil)
defer workertest.CheckKilled(c, w)
defer w.ReallyKill()
done := make(chan struct{})
go func() {
defer close(done)
worker.Stop(w)
}()
select {
case <-time.After(coretesting.ShortWait):
case <-done:
c.Fatalf("Stop returned early")
}
w.ReallyKill()
select {
case <-done:
case <-time.After(coretesting.LongWait):
c.Fatalf("Stop never returned")
}
}
// TestWorstError starts an engine with two manifolds that always error
// with fatal errors. We test that the most important error is the one
// returned by the engine.
//
// This test uses manifolds whose workers ignore kill requests. We want
// this (dangerous!) behaviour so that we don't race over which fatal
// error is seen by the engine first.
func (s *EngineSuite) TestWorstError(c *gc.C) {
worstErr := errors.New("awful error")
callCount := 0
s.fix.worstError = func(err1, err2 error) error {
callCount++
return worstErr
}
s.fix.isFatal = alwaysFatal
s.fix.dirty = true
s.fix.run(c, func(engine dependency.Engine) {
mh1 := newErrorIgnoringManifoldHarness()
err := engine.Install("task", mh1.Manifold())
c.Assert(err, jc.ErrorIsNil)
mh1.AssertOneStart(c)
mh2 := newErrorIgnoringManifoldHarness()
err = engine.Install("another task", mh2.Manifold())
c.Assert(err, jc.ErrorIsNil)
mh2.AssertOneStart(c)
mh1.InjectError(c, errors.New("ping"))
mh2.InjectError(c, errors.New("pong"))
err = workertest.CheckKilled(c, engine)
c.Check(errors.Cause(err), gc.Equals, worstErr)
c.Check(callCount, gc.Equals, 2)
})
}
// cleanup checks that every expected worker has already been stopped by
// the SUT. (i.e.: don't set up more workers than your fixture needs).
func (wl *workerList) cleanup(c *gc.C) {
for _, w := range wl.workers {
if w != nil {
workertest.CheckKilled(c, w)
}
}
}
func (s *UndertakerSuite) TestProcessDyingModelErrorRetried(c *gc.C) {
s.fix.errors = []error{
nil, // ModelInfo
nil, // SetStatus
nil, // WatchModelResources,
errors.New("meh, will retry"), // ProcessDyingModel,
errors.New("will retry again"), // ProcessDyingModel,
nil, // ProcessDyingModel,
nil, // SetStatus
nil, // Destroy,
nil, // RemoveModel
}
stub := s.fix.run(c, func(w worker.Worker) {
workertest.CheckKilled(c, w)
})
stub.CheckCallNames(c,
"ModelInfo",
"SetStatus",
"WatchModelResources",
"ProcessDyingModel",
"ProcessDyingModel",
"ProcessDyingModel",
"SetStatus",
"Destroy",
"RemoveModel",
)
}
func (s *Suite) TestImportFailure(c *gc.C) {
masterClient := newStubMasterClient(s.stub)
worker, err := migrationmaster.New(migrationmaster.Config{
Facade: masterClient,
Guard: newStubGuard(s.stub),
})
c.Assert(err, jc.ErrorIsNil)
s.connection.importErr = errors.New("boom")
s.triggerMigration(masterClient)
err = workertest.CheckKilled(c, worker)
c.Assert(err, gc.Equals, migrationmaster.ErrDoneForNow)
s.stub.CheckCalls(c, []jujutesting.StubCall{
{"masterClient.Watch", nil},
{"masterClient.GetMigrationStatus", nil},
{"guard.Lockdown", nil},
{"masterClient.SetPhase", []interface{}{migration.READONLY}},
{"masterClient.SetPhase", []interface{}{migration.PRECHECK}},
{"masterClient.SetPhase", []interface{}{migration.IMPORT}},
{"masterClient.Export", nil},
apiOpenCall,
importCall,
connCloseCall,
{"masterClient.SetPhase", []interface{}{migration.ABORT}},
apiOpenCall,
abortCall,
connCloseCall,
{"masterClient.SetPhase", []interface{}{migration.ABORTDONE}},
})
}
func (s *UndertakerSuite) TestAlreadyDeadRemoves(c *gc.C) {
s.fix.info.Result.Life = "dead"
stub := s.fix.run(c, func(w worker.Worker) {
workertest.CheckKilled(c, w)
})
stub.CheckCallNames(c, "ModelInfo", "SetStatus", "Destroy", "RemoveModel")
}
func (s *Suite) TestWatchFailure(c *gc.C) {
s.client.watchErr = errors.New("boom")
w, err := migrationminion.New(s.config)
c.Assert(err, jc.ErrorIsNil)
err = workertest.CheckKilled(c, w)
c.Check(err, gc.ErrorMatches, "setting up watcher: boom")
}
func (s *EngineSuite) TestIsFatal(c *gc.C) {
fatalErr := errors.New("KABOOM")
s.fix.isFatal = isFatalIf(fatalErr)
s.fix.dirty = true
s.fix.run(c, func(engine dependency.Engine) {
// Start two independent workers.
mh1 := newManifoldHarness()
err := engine.Install("some-task", mh1.Manifold())
c.Assert(err, jc.ErrorIsNil)
mh1.AssertOneStart(c)
mh2 := newManifoldHarness()
err = engine.Install("other-task", mh2.Manifold())
c.Assert(err, jc.ErrorIsNil)
mh2.AssertOneStart(c)
// Bounce one worker with Just Some Error; check that worker bounces.
mh1.InjectError(c, errors.New("splort"))
mh1.AssertOneStart(c)
mh2.AssertNoStart(c)
// Bounce another worker with the fatal error; check the engine exits with
// the right error.
mh2.InjectError(c, fatalErr)
mh1.AssertNoStart(c)
mh2.AssertNoStart(c)
err = workertest.CheckKilled(c, engine)
c.Assert(err, gc.Equals, fatalErr)
})
}
func (*WorkerSuite) TestWatchErrorNonEmptyRunningActions(c *gc.C) {
stub := &testing.Stub{}
stub.SetErrors(nil, errors.New("ignored"), errors.New("kuso"))
facade := &mockFacade{
stub: stub,
runningActions: fakeRunningActions,
}
config := machineactions.WorkerConfig{
Facade: facade,
MachineTag: fakeTag,
HandleAction: mockHandleAction(stub),
}
worker, err := machineactions.NewMachineActionsWorker(config)
c.Assert(err, jc.ErrorIsNil)
err = workertest.CheckKilled(c, worker)
c.Check(err, gc.ErrorMatches, "kuso")
stub.CheckCalls(c, []testing.StubCall{{
FuncName: "RunningActions",
Args: []interface{}{fakeTag},
}, {
FuncName: "ActionFinish",
Args: []interface{}{thirdActionTag, params.ActionFailed, "action cancelled"},
}, {
FuncName: "WatchActionNotifications",
Args: []interface{}{fakeTag},
}})
}
func (s *Suite) TestReportKeysError(c *gc.C) {
s.facade.reportErr = errors.New("blam")
w, err := hostkeyreporter.New(s.config)
c.Assert(err, jc.ErrorIsNil)
err = workertest.CheckKilled(c, w)
c.Check(err, gc.ErrorMatches, "blam")
}
func (*ResumerSuite) TestImmediateFailure(c *gc.C) {
fix := newFixture(errors.New("zap"))
stub := fix.Run(c, func(_ *testing.Clock, worker *resumer.Resumer) {
err := workertest.CheckKilled(c, worker)
c.Check(err, gc.ErrorMatches, "cannot resume transactions: zap")
})
stub.CheckCallNames(c, "ResumeTransactions")
}
func (s *Suite) TestCheckKilledTimeout(c *gc.C) {
w := workertest.NewErrorWorker(nil)
defer workertest.CleanKill(c, w)
err := workertest.CheckKilled(c, w)
s.CheckFailed(c)
c.Check(err, gc.ErrorMatches, "workertest: worker not stopping")
}
func (context *context) checkCleanedUp() {
context.c.Logf("checking no active current pinger")
context.mu.Lock()
defer context.mu.Unlock()
if context.current != nil {
workertest.CheckKilled(context.c, context.current)
}
}
func (s *Suite) TestCheckKilledSuccess(c *gc.C) {
expect := errors.New("snifplog")
w := workertest.NewErrorWorker(expect)
defer workertest.DirtyKill(c, w)
w.Kill()
err := workertest.CheckKilled(c, w)
c.Check(err, gc.Equals, expect)
}
func (s *UndertakerSuite) TestAlreadyDeadTimeMissingFinishes(c *gc.C) {
s.fix.info.Result.Life = "dead"
stub := s.fix.run(c, func(w worker.Worker, clock *coretesting.Clock) {
waitAlarm(c, clock)
clock.Advance(RIPTime)
workertest.CheckKilled(c, w)
})
stub.CheckCallNames(c, "ModelInfo", "Destroy", "RemoveModel")
}
func (s *undertakerSuite) TestErrorGettingRemovals(c *gc.C) {
api := s.makeAPIWithWatcher()
api.SetErrors(nil, errors.New("explodo"))
w, err := machineundertaker.NewWorker(api, &fakeEnviron{})
c.Assert(err, jc.ErrorIsNil)
err = workertest.CheckKilled(c, w)
c.Check(err, gc.ErrorMatches, "explodo")
api.CheckCallNames(c, "WatchMachineRemovals", "AllMachineRemovals")
}
func (s *Suite) TestSuccess(c *gc.C) {
w, err := hostkeyreporter.New(s.config)
c.Assert(err, jc.ErrorIsNil)
err = workertest.CheckKilled(c, w)
c.Check(err, gc.Equals, dependency.ErrUninstall)
s.stub.CheckCalls(c, []jujutesting.StubCall{{
"ReportKeys", []interface{}{"42", []string{"dsa", "ecdsa", "rsa"}},
}})
}
func (s *Suite) TestNoSSHDir(c *gc.C) {
// No /etc/ssh at all
s.config.RootDir = c.MkDir()
w, err := hostkeyreporter.New(s.config)
c.Assert(err, jc.ErrorIsNil)
err = workertest.CheckKilled(c, w)
c.Check(errors.Cause(err), gc.Equals, dependency.ErrUninstall)
}
func (s *UndertakerSuite) TestModelInfoErrorFatal(c *gc.C) {
s.fix.errors = []error{errors.New("pow")}
s.fix.dirty = true
stub := s.fix.run(c, func(w worker.Worker) {
err := workertest.CheckKilled(c, w)
c.Check(err, gc.ErrorMatches, "pow")
})
stub.CheckCallNames(c, "ModelInfo")
}
func (s *UndertakerSuite) TestAliveError(c *gc.C) {
s.fix.info.Result.Life = "alive"
s.fix.dirty = true
stub := s.fix.run(c, func(w worker.Worker) {
err := workertest.CheckKilled(c, w)
c.Check(err, gc.ErrorMatches, "model still alive")
})
stub.CheckCallNames(c, "ModelInfo")
}
func (s *UndertakerSuite) TestWatchModelResourcesErrorFatal(c *gc.C) {
s.fix.errors = []error{nil, nil, errors.New("pow")}
s.fix.dirty = true
stub := s.fix.run(c, func(w worker.Worker) {
err := workertest.CheckKilled(c, w)
c.Check(err, gc.ErrorMatches, "pow")
})
stub.CheckCallNames(c, "ModelInfo", "SetStatus", "WatchModelResources")
}
func (s *FlagSuite) TestClaimFailureWaitSuccess(c *gc.C) {
fix := newFixture(c, errClaimDenied, nil)
fix.Run(c, func(flag *singular.FlagWorker, _ *coretesting.Clock, unblock func()) {
c.Check(flag.Check(), jc.IsFalse)
unblock()
err := workertest.CheckKilled(c, flag)
c.Check(errors.Cause(err), gc.Equals, singular.ErrRefresh)
})
fix.CheckClaimWait(c)
}
func (s *FlagSuite) TestClaimFailureWaitError(c *gc.C) {
fix := newFixture(c, errClaimDenied, errors.New("glug"))
fix.Run(c, func(flag *singular.FlagWorker, _ *coretesting.Clock, unblock func()) {
c.Check(flag.Check(), jc.IsFalse)
unblock()
err := workertest.CheckKilled(c, flag)
c.Check(err, gc.ErrorMatches, "glug")
})
fix.CheckClaimWait(c)
}
func (s *UndertakerSuite) TestRemoveModelErrorFatal(c *gc.C) {
s.fix.errors = []error{nil, nil, nil, errors.New("pow")}
s.fix.info.Result.Life = "dead"
s.fix.dirty = true
stub := s.fix.run(c, func(w worker.Worker) {
err := workertest.CheckKilled(c, w)
c.Check(err, gc.ErrorMatches, "cannot remove model: pow")
})
stub.CheckCallNames(c, "ModelInfo", "SetStatus", "Destroy", "RemoveModel")
}
func (s *FlagSuite) TestClaimSuccessThenFailure(c *gc.C) {
fix := newFixture(c, nil, errClaimDenied)
fix.Run(c, func(flag *singular.FlagWorker, clock *coretesting.Clock, unblock func()) {
<-clock.Alarms()
clock.Advance(30 * time.Second)
err := workertest.CheckKilled(c, flag)
c.Check(errors.Cause(err), gc.Equals, singular.ErrRefresh)
})
fix.CheckClaims(c, 2)
}
func (s *UndertakerSuite) TestDestroyErrorFatal(c *gc.C) {
s.fix.errors = []error{nil, errors.New("pow")}
s.fix.info.Result.Life = "dead"
s.fix.dirty = true
stub := s.fix.run(c, func(w worker.Worker, clock *coretesting.Clock) {
err := workertest.CheckKilled(c, w)
c.Check(err, gc.ErrorMatches, "pow")
})
stub.CheckCallNames(c, "ModelInfo", "Destroy")
}