func (s *UpgradeSuite) TestDowngradeOnMasterWhenOtherStateServerDoesntStartUpgrade(c *gc.C) {
coretesting.SkipIfWindowsBug(c, "lp:1446885")
// This test checks that the master triggers a downgrade if one of
// the other state server fails to signal it is ready for upgrade.
//
// This test is functional, ensuring that the upgrader worker
// terminates the machine agent with the UpgradeReadyError which
// makes the downgrade happen.
// Speed up the watcher frequency to make the test much faster.
s.PatchValue(&watcher.Period, 200*time.Millisecond)
// Provide (fake) tools so that the upgrader has something to downgrade to.
envtesting.AssertUploadFakeToolsVersions(
c, s.DefaultToolsStorage, s.Environ.Config().AgentStream(), s.Environ.Config().AgentStream(), s.oldVersion)
// Only the first machine is going to be ready for upgrade.
machineIdA, machineIdB, _ := s.createUpgradingStateServers(c)
// One of the other state servers is ready for upgrade (but machine C doesn't).
info, err := s.State.EnsureUpgradeInfo(machineIdB, s.oldVersion.Number, version.Current)
c.Assert(err, jc.ErrorIsNil)
agent := s.newAgentFromMachineId(c, machineIdA)
defer agent.Stop()
s.machineIsMaster = true
var agentErr error
agentDone := make(chan bool)
go func() {
agentErr = agent.Run(nil)
close(agentDone)
}()
select {
case <-agentDone:
upgradeReadyErr, ok := agentErr.(*upgrader.UpgradeReadyError)
if !ok {
c.Fatalf("didn't see UpgradeReadyError, instead got: %v", agentErr)
}
// Confirm that the downgrade is back to the previous version.
current := version.Binary{
Number: version.Current,
Arch: arch.HostArch(),
Series: series.HostSeries(),
}
c.Assert(upgradeReadyErr.OldTools, gc.Equals, current)
c.Assert(upgradeReadyErr.NewTools, gc.Equals, s.oldVersion)
case <-time.After(coretesting.LongWait):
c.Fatal("machine agent did not exit as expected")
}
// UpgradeInfo doc should now be archived.
err = info.Refresh()
c.Assert(err, gc.ErrorMatches, "current upgrade info not found")
}
func (s *upgradeSuite) TestLoginsDuringUpgrade(c *gc.C) {
coretesting.SkipIfWindowsBug(c, "lp:1446885")
// Create machine agent to upgrade
machine, machine0Conf := s.makeStateAgentVersion(c, s.oldVersion)
// Set up a second machine to log in as. API logins are tested
// manually so there's no need to actually start this machine.
machine1, password := s.Factory.MakeMachineReturningPassword(c, &factory.MachineParams{
Nonce: agent.BootstrapNonce,
})
machine1Conf, _ := s.PrimeAgent(c, machine1.Tag(), password)
// Mock out upgrade logic, using a channel so that the test knows
// when upgrades have started and can control when upgrades
// should finish.
upgradeCh := make(chan bool)
abort := make(chan bool)
fakePerformUpgrade := func(version.Number, []upgrades.Target, upgrades.Context) error {
// Signal that upgrade has started.
select {
case upgradeCh <- true:
case <-abort:
return nil
}
// Wait for signal that upgrades should finish.
select {
case <-upgradeCh:
case <-abort:
return nil
}
return nil
}
s.PatchValue(&upgradesteps.PerformUpgrade, fakePerformUpgrade)
a := s.newAgent(c, machine)
go func() { c.Check(a.Run(nil), jc.ErrorIsNil) }()
defer func() { c.Check(a.Stop(), jc.ErrorIsNil) }()
c.Assert(waitForUpgradeToStart(upgradeCh), jc.IsTrue)
// Only user and local logins are allowed during upgrade. Users get a restricted API.
s.checkLoginToAPIAsUser(c, machine0Conf, RestrictedAPIExposed)
c.Assert(canLoginToAPIAsMachine(c, machine0Conf, machine0Conf), jc.IsTrue)
c.Assert(canLoginToAPIAsMachine(c, machine1Conf, machine0Conf), jc.IsFalse)
close(upgradeCh) // Allow upgrade to complete
waitForUpgradeToFinish(c, machine0Conf)
// All logins are allowed after upgrade
s.checkLoginToAPIAsUser(c, machine0Conf, FullAPIExposed)
c.Assert(canLoginToAPIAsMachine(c, machine0Conf, machine0Conf), jc.IsTrue)
c.Assert(canLoginToAPIAsMachine(c, machine1Conf, machine0Conf), jc.IsTrue)
}
func (s *UpgradeSuite) TestUpgradeStepsHostMachine(c *gc.C) {
coretesting.SkipIfPPC64EL(c, "lp:1444576")
coretesting.SkipIfWindowsBug(c, "lp:1446885")
s.setInstantRetryStrategy(c)
// We need to first start up a state server that thinks it has already been upgraded.
ss, _, _ := s.primeAgent(c, state.JobManageEnviron)
a := s.newAgent(c, ss)
go func() { c.Check(a.Run(nil), gc.IsNil) }()
defer func() { c.Check(a.Stop(), gc.IsNil) }()
// Now run the test.
s.assertUpgradeSteps(c, state.JobHostUnits)
s.assertHostUpgrades(c)
}
func (s *UpgradeSuite) TestUpgradeStepsStateServer(c *gc.C) {
coretesting.SkipIfI386(c, "lp:1444576")
coretesting.SkipIfPPC64EL(c, "lp:1444576")
coretesting.SkipIfWindowsBug(c, "lp:1446885")
s.setInstantRetryStrategy(c)
// Upload tools to provider storage, so they can be migrated to environment storage.
stor, err := environs.LegacyStorage(s.State)
if !errors.IsNotSupported(err) {
c.Assert(err, jc.ErrorIsNil)
envtesting.AssertUploadFakeToolsVersions(
c, stor, "releases", s.Environ.Config().AgentStream(), s.oldVersion)
}
s.assertUpgradeSteps(c, state.JobManageEnviron)
s.assertStateServerUpgrades(c)
}
func (s *upgradeSuite) TestDowngradeOnMasterWhenOtherControllerDoesntStartUpgrade(c *gc.C) {
coretesting.SkipIfWindowsBug(c, "lp:1446885")
// This test checks that the master triggers a downgrade if one of
// the other controller fails to signal it is ready for upgrade.
//
// This test is functional, ensuring that the upgrader worker
// terminates the machine agent with the UpgradeReadyError which
// makes the downgrade happen.
// Speed up the watcher frequency to make the test much faster.
s.PatchValue(&watcher.Period, 200*time.Millisecond)
// Provide (fake) tools so that the upgrader has something to downgrade to.
envtesting.AssertUploadFakeToolsVersions(
c, s.DefaultToolsStorage, s.Environ.Config().AgentStream(), s.Environ.Config().AgentStream(), s.oldVersion)
// Create 3 controllers
machineA, _ := s.makeStateAgentVersion(c, s.oldVersion)
changes, err := s.State.EnableHA(3, constraints.Value{}, "quantal", nil)
c.Assert(err, jc.ErrorIsNil)
c.Assert(len(changes.Added), gc.Equals, 2)
machineB, _, _ := s.configureMachine(c, changes.Added[0], s.oldVersion)
s.configureMachine(c, changes.Added[1], s.oldVersion)
// One of the other controllers is ready for upgrade (but machine C isn't).
info, err := s.State.EnsureUpgradeInfo(machineB.Id(), s.oldVersion.Number, jujuversion.Current)
c.Assert(err, jc.ErrorIsNil)
// Ensure the agent will think it's the master controller.
fakeIsMachineMaster := func(*state.State, string) (bool, error) {
return true, nil
}
s.PatchValue(&upgradesteps.IsMachineMaster, fakeIsMachineMaster)
// Start the agent
agent := s.newAgent(c, machineA)
defer agent.Stop()
agentDone := make(chan error)
go func() {
agentDone <- agent.Run(nil)
}()
select {
case agentErr := <-agentDone:
upgradeReadyErr, ok := agentErr.(*upgrader.UpgradeReadyError)
if !ok {
c.Fatalf("didn't see UpgradeReadyError, instead got: %v", agentErr)
}
// Confirm that the downgrade is back to the previous version.
current := version.Binary{
Number: jujuversion.Current,
Arch: arch.HostArch(),
Series: series.HostSeries(),
}
c.Assert(upgradeReadyErr.OldTools, gc.Equals, current)
c.Assert(upgradeReadyErr.NewTools, gc.Equals, s.oldVersion)
case <-time.After(coretesting.LongWait):
c.Fatal("machine agent did not exit as expected")
}
// UpgradeInfo doc should now be archived.
err = info.Refresh()
c.Assert(err, gc.ErrorMatches, "current upgrade info not found")
}