func (s *destroyEnvironmentSuite) TestDestroyEnvironmentManual(c *gc.C) {
_, nonManager := s.setUpManual(c)
// If there are any non-manager manual machines in state, DestroyEnvironment will
// error. It will not set the Dying flag on the environment.
err := common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, gc.ErrorMatches, fmt.Sprintf("failed to destroy environment: manually provisioned machines must first be destroyed with `juju destroy-machine %s`", nonManager.Id()))
env, err := s.State.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.Life(), gc.Equals, state.Alive)
// If we remove the non-manager machine, it should pass.
// Manager machines will remain.
err = nonManager.EnsureDead()
c.Assert(err, jc.ErrorIsNil)
err = nonManager.Remove()
c.Assert(err, jc.ErrorIsNil)
err = common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
err = env.Refresh()
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.Life(), gc.Equals, state.Dying)
s.metricSender.CheckCalls(c, []jtesting.StubCall{{FuncName: "SendMetrics"}})
}
func (s *destroyTwoEnvironmentsSuite) TestDestroyStateServerAfterNonStateServerIsDestroyed(c *gc.C) {
err := common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, gc.ErrorMatches, "failed to destroy environment: hosting 1 other environments")
err = common.DestroyEnvironment(s.State, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
err = common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
}
func (s *destroyTwoEnvironmentsSuite) TestDestroyStateServerAfterNonStateServerIsDestroyed(c *gc.C) {
err := common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, gc.ErrorMatches, "failed to destroy environment: hosting 1 other environments")
err = common.DestroyEnvironment(s.State, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
err = common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
s.metricSender.CheckCalls(c, []jtesting.StubCall{{FuncName: "SendMetrics"}, {FuncName: "SendMetrics"}})
}
func (s *destroyTwoEnvironmentsSuite) TestCanDestroyNonBlockedEnv(c *gc.C) {
bh := commontesting.NewBlockHelper(s.APIState)
defer bh.Close()
bh.BlockDestroyEnvironment(c, "TestBlockDestroyDestroyEnvironment")
err := common.DestroyEnvironment(s.State, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
err = common.DestroyEnvironment(s.State, s.State.EnvironTag())
bh.AssertBlocked(c, err, "TestBlockDestroyDestroyEnvironment")
}
// DestroySystem will attempt to destroy the system. If the args specify the
// removal of blocks or the destruction of the environments, this method will
// attempt to do so.
func (s *SystemManagerAPI) DestroySystem(args params.DestroySystemArgs) error {
// Get list of all environments in the system.
allEnvs, err := s.state.AllEnvironments()
if err != nil {
return errors.Trace(err)
}
// If there are hosted environments and DestroyEnvironments was not
// specified, don't bother trying to destroy the system, as it will fail.
if len(allEnvs) > 1 && !args.DestroyEnvironments {
return errors.Errorf("state server environment cannot be destroyed before all other environments are destroyed")
}
// If there are blocks, and we aren't being told to ignore them, let the
// user know.
blocks, err := s.state.AllBlocksForSystem()
if err != nil {
logger.Debugf("Unable to get blocks for system: %s", err)
if !args.IgnoreBlocks {
return errors.Trace(err)
}
}
if len(blocks) > 0 {
if !args.IgnoreBlocks {
return common.ErrOperationBlocked("found blocks in system environments")
}
err := s.state.RemoveAllBlocksForSystem()
if err != nil {
return errors.Trace(err)
}
}
systemEnv, err := s.state.StateServerEnvironment()
if err != nil {
return errors.Trace(err)
}
systemTag := systemEnv.EnvironTag()
if args.DestroyEnvironments {
for _, env := range allEnvs {
environTag := env.EnvironTag()
if environTag != systemTag {
if err := common.DestroyEnvironment(s.state, environTag); err != nil {
logger.Errorf("unable to destroy environment %q: %s", env.UUID(), err)
}
}
}
}
return errors.Trace(common.DestroyEnvironment(s.state, systemTag))
}
func (s *destroyTwoEnvironmentsSuite) TestCanDestroyNonBlockedEnv(c *gc.C) {
bh := commontesting.NewBlockHelper(s.APIState)
defer bh.Close()
bh.BlockDestroyEnvironment(c, "TestBlockDestroyDestroyEnvironment")
err := common.DestroyEnvironment(s.State, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
err = common.DestroyEnvironment(s.State, s.State.EnvironTag())
bh.AssertBlocked(c, err, "TestBlockDestroyDestroyEnvironment")
s.metricSender.CheckCalls(c, []jtesting.StubCall{{FuncName: "SendMetrics"}})
}
func (s *destroyEnvironmentSuite) TestBlockRemoveDestroyEnvironment(c *gc.C) {
// Setup environment
s.setUpInstances(c)
s.BlockRemoveObject(c, "TestBlockRemoveDestroyEnvironment")
err := common.DestroyEnvironment(s.State, s.State.EnvironTag())
s.AssertBlocked(c, err, "TestBlockRemoveDestroyEnvironment")
}
func (s *destroyEnvironmentSuite) TestBlockChangesDestroyEnvironment(c *gc.C) {
// Setup environment
s.setUpInstances(c)
// lock environment: can't destroy locked environment
s.BlockAllChanges(c, "TestBlockChangesDestroyEnvironment")
err := common.DestroyEnvironment(s.State, s.State.EnvironTag())
s.AssertBlocked(c, err, "TestBlockChangesDestroyEnvironment")
}
func (s *destroyEnvironmentSuite) TestBlockRemoveDestroyEnvironment(c *gc.C) {
// Setup environment
s.setUpInstances(c)
s.BlockRemoveObject(c, "TestBlockRemoveDestroyEnvironment")
err := common.DestroyEnvironment(s.State, s.State.EnvironTag())
s.AssertBlocked(c, err, "TestBlockRemoveDestroyEnvironment")
s.metricSender.CheckCalls(c, []jtesting.StubCall{})
}
// DestroyEnvironment will try to destroy the current environment.
// If there is a block on destruction, this method will return an error.
func (c *Client) DestroyEnvironment() (err error) {
if err := c.check.DestroyAllowed(); err != nil {
return errors.Trace(err)
}
environTag := c.api.state.EnvironTag()
return errors.Trace(common.DestroyEnvironment(c.api.state, environTag))
}
func (s *destroyEnvironmentSuite) TestMetrics(c *gc.C) {
metricSender := &testMetricSender{}
s.PatchValue(common.SendMetrics, metricSender.SendMetrics)
err := common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
metricSender.CheckCalls(c, []jtesting.StubCall{{FuncName: "SendMetrics"}})
}
func (s *destroyEnvironmentSuite) TestBlockChangesDestroyEnvironment(c *gc.C) {
// Setup environment
s.setUpInstances(c)
// lock environment: can't destroy locked environment
s.BlockAllChanges(c, "TestBlockChangesDestroyEnvironment")
err := common.DestroyEnvironment(s.State, s.State.EnvironTag())
s.AssertBlocked(c, err, "TestBlockChangesDestroyEnvironment")
s.metricSender.CheckCalls(c, []jtesting.StubCall{})
}
func (s *destroySystemSuite) TestDestroySystemNoHostedEnvs(c *gc.C) {
err := common.DestroyEnvironment(s.State, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
err = s.systemManager.DestroySystem(params.DestroySystemArgs{})
c.Assert(err, jc.ErrorIsNil)
env, err := s.State.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.Life(), gc.Equals, state.Dying)
}
func (s *destroyControllerSuite) TestDestroyControllerErrsOnNoHostedEnvsWithBlock(c *gc.C) {
err := common.DestroyEnvironment(s.State, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
s.BlockDestroyEnvironment(c, "TestBlockDestroyEnvironment")
s.BlockRemoveObject(c, "TestBlockRemoveObject")
err = s.controller.DestroyController(params.DestroyControllerArgs{})
c.Assert(err, gc.ErrorMatches, "found blocks in controller environments")
env, err := s.State.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.Life(), gc.Equals, state.Alive)
}
func (s *destroySystemSuite) TestDestroySystemNoHostedEnvsWithBlockFail(c *gc.C) {
err := common.DestroyEnvironment(s.State, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
s.BlockDestroyEnvironment(c, "TestBlockDestroyEnvironment")
s.BlockRemoveObject(c, "TestBlockRemoveObject")
err = s.systemManager.DestroySystem(params.DestroySystemArgs{})
c.Assert(params.IsCodeOperationBlocked(err), jc.IsTrue)
numBlocks, err := s.State.AllBlocksForSystem()
c.Assert(err, jc.ErrorIsNil)
c.Assert(len(numBlocks), gc.Equals, 2)
}
func (s *destroyTwoEnvironmentsSuite) TestDestroyStateServerAfterNonStateServerIsDestroyed(c *gc.C) {
otherFactory := factory.NewFactory(s.otherState)
otherFactory.MakeMachine(c, nil)
m := otherFactory.MakeMachine(c, nil)
otherFactory.MakeMachineNested(c, m.Id(), nil)
err := common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, gc.ErrorMatches, "failed to destroy environment: hosting 1 other environments")
needsCleanup, err := s.State.NeedsCleanup()
c.Assert(err, jc.ErrorIsNil)
c.Assert(needsCleanup, jc.IsFalse)
err = common.DestroyEnvironment(s.State, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
err = common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
// Make sure we can continue to take the hosted environ down while the
// controller environ is dying.
runAllCleanups(c, s.otherState)
assertAllMachinesDeadAndRemove(c, s.otherState)
c.Assert(s.otherState.ProcessDyingEnviron(), jc.ErrorIsNil)
otherEnv, err := s.otherState.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(otherEnv.Life(), gc.Equals, state.Dead)
env, err := s.State.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.Life(), gc.Equals, state.Dying)
c.Assert(s.State.ProcessDyingEnviron(), jc.ErrorIsNil)
c.Assert(env.Refresh(), jc.ErrorIsNil)
c.Assert(env.Life(), gc.Equals, state.Dead)
}
func (s *destroyTwoEnvironmentsSuite) TestCleanupEnvironDocs(c *gc.C) {
otherFactory := factory.NewFactory(s.otherState)
otherFactory.MakeMachine(c, nil)
m := otherFactory.MakeMachine(c, nil)
otherFactory.MakeMachineNested(c, m.Id(), nil)
err := common.DestroyEnvironment(s.otherState, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
_, err = s.otherState.Environment()
c.Assert(errors.IsNotFound(err), jc.IsTrue)
_, err = s.State.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(s.otherState.EnsureEnvironmentRemoved(), jc.ErrorIsNil)
}
func (s *destroySystemSuite) TestDestroySystemNoHostedEnvsWithBlock(c *gc.C) {
err := common.DestroyEnvironment(s.State, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
s.BlockDestroyEnvironment(c, "TestBlockDestroyEnvironment")
s.BlockRemoveObject(c, "TestBlockRemoveObject")
err = s.systemManager.DestroySystem(params.DestroySystemArgs{
IgnoreBlocks: true,
})
c.Assert(err, jc.ErrorIsNil)
env, err := s.State.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.Life(), gc.Equals, state.Dying)
}
func (s *destroyEnvironmentSuite) TestDestroyEnvironmentWithContainers(c *gc.C) {
ops := make(chan dummy.Operation, 500)
dummy.Listen(ops)
_, nonManager, _ := s.setUpInstances(c)
nonManagerId, _ := nonManager.InstanceId()
err := common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
for op := range ops {
if op, ok := op.(dummy.OpStopInstances); ok {
c.Assert(op.Ids, jc.SameContents, []instance.Id{nonManagerId})
break
}
}
}
func (s *destroyTwoEnvironmentsSuite) TestCleanupEnvironDocs(c *gc.C) {
otherFactory := factory.NewFactory(s.otherState)
otherFactory.MakeMachine(c, nil)
m := otherFactory.MakeMachine(c, nil)
otherFactory.MakeMachineNested(c, m.Id(), nil)
err := common.DestroyEnvironment(s.otherState, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
_, err = s.otherState.Environment()
c.Assert(errors.IsNotFound(err), jc.IsTrue)
_, err = s.State.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(s.otherState.EnsureEnvironmentRemoved(), jc.ErrorIsNil)
s.metricSender.CheckCalls(c, []jtesting.StubCall{{FuncName: "SendMetrics"}})
}
func (s *destroyEnvironmentSuite) TestDestroyEnvironment(c *gc.C) {
manager, nonManager, _ := s.setUpInstances(c)
managerId, _ := manager.InstanceId()
nonManagerId, _ := nonManager.InstanceId()
instances, err := s.Environ.Instances([]instance.Id{managerId, nonManagerId})
c.Assert(err, jc.ErrorIsNil)
for _, inst := range instances {
c.Assert(inst, gc.NotNil)
}
services, err := s.State.AllServices()
c.Assert(err, jc.ErrorIsNil)
err = common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
s.metricSender.CheckCalls(c, []jtesting.StubCall{{FuncName: "SendMetrics"}})
// After DestroyEnvironment returns, we should have:
// - all non-manager instances stopped
instances, err = s.Environ.Instances([]instance.Id{managerId, nonManagerId})
c.Assert(err, gc.Equals, environs.ErrPartialInstances)
c.Assert(instances[0], gc.NotNil)
c.Assert(instances[1], jc.ErrorIsNil)
// - all services in state are Dying or Dead (or removed altogether),
// after running the state Cleanups.
needsCleanup, err := s.State.NeedsCleanup()
c.Assert(err, jc.ErrorIsNil)
c.Assert(needsCleanup, jc.IsTrue)
err = s.State.Cleanup()
c.Assert(err, jc.ErrorIsNil)
for _, s := range services {
err = s.Refresh()
if err != nil {
c.Assert(err, jc.Satisfies, errors.IsNotFound)
} else {
c.Assert(s.Life(), gc.Not(gc.Equals), state.Alive)
}
}
// - environment is Dying
env, err := s.State.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.Life(), gc.Equals, state.Dying)
}
func (s *destroyTwoEnvironmentsSuite) TestDifferentStateEnv(c *gc.C) {
otherFactory := factory.NewFactory(s.otherState)
otherFactory.MakeMachine(c, nil)
m := otherFactory.MakeMachine(c, nil)
otherFactory.MakeMachineNested(c, m.Id(), nil)
// NOTE: pass in the main test State instance, which is 'bound'
// to the state server environment.
err := common.DestroyEnvironment(s.State, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
_, err = s.otherState.Environment()
c.Assert(errors.IsNotFound(err), jc.IsTrue)
_, err = s.State.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(s.otherState.EnsureEnvironmentRemoved(), jc.ErrorIsNil)
}
func (s *destroyEnvironmentSuite) TestDestroyEnvironment(c *gc.C) {
manager, nonManager, _ := s.setUpInstances(c)
managerId, _ := manager.InstanceId()
nonManagerId, _ := nonManager.InstanceId()
instances, err := s.Environ.Instances([]instance.Id{managerId, nonManagerId})
c.Assert(err, jc.ErrorIsNil)
for _, inst := range instances {
c.Assert(inst, gc.NotNil)
}
services, err := s.State.AllServices()
c.Assert(err, jc.ErrorIsNil)
err = common.DestroyEnvironment(s.State, s.State.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
runAllCleanups(c, s.State)
// After DestroyEnvironment returns and all cleanup jobs have run, we should have:
// - all non-manager machines dying
assertLife(c, manager, state.Alive)
// Note: we leave the machine in a dead state and rely on the provisioner
// to stop the backing instances, remove the dead machines and finally
// remove all environment docs from state.
assertLife(c, nonManager, state.Dead)
// - all services in state are Dying or Dead (or removed altogether),
// after running the state Cleanups.
for _, s := range services {
err = s.Refresh()
if err != nil {
c.Assert(err, jc.Satisfies, errors.IsNotFound)
} else {
c.Assert(s.Life(), gc.Not(gc.Equals), state.Alive)
}
}
// - environment is Dying or Dead.
env, err := s.State.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.Life(), gc.Not(gc.Equals), state.Alive)
}
func (s *destroyTwoEnvironmentsSuite) TestCleanupEnvironResources(c *gc.C) {
otherFactory := factory.NewFactory(s.otherState)
m := otherFactory.MakeMachine(c, nil)
otherFactory.MakeMachineNested(c, m.Id(), nil)
err := common.DestroyEnvironment(s.otherState, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
// Assert that the machines are not removed until the cleanup runs.
c.Assert(m.Refresh(), jc.ErrorIsNil)
assertMachineCount(c, s.otherState, 2)
runAllCleanups(c, s.otherState)
assertAllMachinesDeadAndRemove(c, s.otherState)
otherEnv, err := s.otherState.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(otherEnv.Life(), gc.Equals, state.Dying)
c.Assert(s.otherState.ProcessDyingEnviron(), jc.ErrorIsNil)
c.Assert(otherEnv.Refresh(), jc.ErrorIsNil)
c.Assert(otherEnv.Life(), gc.Equals, state.Dead)
}
// DestroyController will attempt to destroy the controller. If the args
// specify the removal of blocks or the destruction of the environments, this
// method will attempt to do so.
func (s *ControllerAPI) DestroyController(args params.DestroyControllerArgs) error {
controllerEnv, err := s.state.ControllerEnvironment()
if err != nil {
return errors.Trace(err)
}
systemTag := controllerEnv.EnvironTag()
if err = s.ensureNotBlocked(args); err != nil {
return errors.Trace(err)
}
// If we are destroying environments, we need to tolerate living
// environments but set the controller to dying to prevent new
// environments sneaking in. If we are not destroying hosted environments,
// this will fail if any hosted environments are found.
if args.DestroyEnvironments {
return errors.Trace(common.DestroyEnvironmentIncludingHosted(s.state, systemTag))
}
if err = common.DestroyEnvironment(s.state, systemTag); state.IsHasHostedEnvironsError(err) {
err = errors.New("controller environment cannot be destroyed before all other environments are destroyed")
}
return errors.Trace(err)
}
func (s *destroyTwoEnvironmentsSuite) TestDifferentStateEnv(c *gc.C) {
otherFactory := factory.NewFactory(s.otherState)
otherFactory.MakeMachine(c, nil)
m := otherFactory.MakeMachine(c, nil)
otherFactory.MakeMachineNested(c, m.Id(), nil)
// NOTE: pass in the main test State instance, which is 'bound'
// to the state server environment.
err := common.DestroyEnvironment(s.State, s.otherState.EnvironTag())
c.Assert(err, jc.ErrorIsNil)
runAllCleanups(c, s.otherState)
assertAllMachinesDeadAndRemove(c, s.otherState)
otherEnv, err := s.otherState.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(s.otherState.ProcessDyingEnviron(), jc.ErrorIsNil)
c.Assert(otherEnv.Refresh(), jc.ErrorIsNil)
c.Assert(otherEnv.Life(), gc.Equals, state.Dead)
env, err := s.State.Environment()
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.Life(), gc.Equals, state.Alive)
}
