func (s *UpgradeSuite) TestDowngradeOnMasterWhenOtherStateServerDoesntStartUpgrade(c *gc.C) {
coretesting.SkipIfWindowsBug(c, "lp:1446885")
// This test checks that the master triggers a downgrade if one of
// the other state server fails to signal it is ready for upgrade.
//
// This test is functional, ensuring that the upgrader worker
// terminates the machine agent with the UpgradeReadyError which
// makes the downgrade happen.
// Speed up the watcher frequency to make the test much faster.
s.PatchValue(&watcher.Period, 200*time.Millisecond)
// Provide (fake) tools so that the upgrader has something to downgrade to.
envtesting.AssertUploadFakeToolsVersions(
c, s.DefaultToolsStorage, s.Environ.Config().AgentStream(), s.Environ.Config().AgentStream(), s.oldVersion)
// Only the first machine is going to be ready for upgrade.
machineIdA, machineIdB, _ := s.createUpgradingStateServers(c)
// One of the other state servers is ready for upgrade (but machine C doesn't).
info, err := s.State.EnsureUpgradeInfo(machineIdB, s.oldVersion.Number, version.Current)
c.Assert(err, jc.ErrorIsNil)
agent := s.newAgentFromMachineId(c, machineIdA)
defer agent.Stop()
s.machineIsMaster = true
var agentErr error
agentDone := make(chan bool)
go func() {
agentErr = agent.Run(nil)
close(agentDone)
}()
select {
case <-agentDone:
upgradeReadyErr, ok := agentErr.(*upgrader.UpgradeReadyError)
if !ok {
c.Fatalf("didn't see UpgradeReadyError, instead got: %v", agentErr)
}
// Confirm that the downgrade is back to the previous version.
current := version.Binary{
Number: version.Current,
Arch: arch.HostArch(),
Series: series.HostSeries(),
}
c.Assert(upgradeReadyErr.OldTools, gc.Equals, current)
c.Assert(upgradeReadyErr.NewTools, gc.Equals, s.oldVersion)
case <-time.After(coretesting.LongWait):
c.Fatal("machine agent did not exit as expected")
}
// UpgradeInfo doc should now be archived.
err = info.Refresh()
c.Assert(err, gc.ErrorMatches, "current upgrade info not found")
}
func (s *UpgradeSuite) TestDowngradeOnMasterWhenOtherStateServerDoesntStartUpgrade(c *gc.C) {
// This test checks that the master triggers a downgrade if one of
// the other state server fails to signal it is ready for upgrade.
//
// This test is functional, ensuring that the upgrader worker
// terminates the machine agent with the UpgradeReadyError which
// makes the downgrade happen.
// Speed up the watcher frequency to make the test much faster.
s.PatchValue(&watcher.Period, 200*time.Millisecond)
s.machineIsMaster = true
// Signal that some state servers didn't come up for upgrade. This
// should trigger a rollback to the previous agent version.
s.waitForOtherStateServersErr = errors.New("boom")
// Provide (fake) tools so that the upgrader has something to downgrade to.
envtesting.AssertUploadFakeToolsVersions(c, s.Environ.Storage(), s.oldVersion)
agent := s.createUpgradingAgent(c, state.JobManageEnviron)
defer agent.Stop()
var agentErr error
agentDone := make(chan bool)
go func() {
agentErr = agent.Run(nil)
close(agentDone)
}()
select {
case <-agentDone:
upgradeReadyErr, ok := agentErr.(*upgrader.UpgradeReadyError)
if !ok {
c.Fatalf("didn't see UpgradeReadyError, instead got: %v", agentErr)
}
// Confirm that the downgrade is back to the previous version.
c.Assert(upgradeReadyErr.OldTools, gc.Equals, s.upgradeToVersion)
c.Assert(upgradeReadyErr.NewTools, gc.Equals, s.oldVersion)
case <-time.After(coretesting.LongWait):
c.Fatal("machine agent did not exit as expected")
}
}
func (s *upgradeSuite) TestDowngradeOnMasterWhenOtherControllerDoesntStartUpgrade(c *gc.C) {
coretesting.SkipIfWindowsBug(c, "lp:1446885")
// This test checks that the master triggers a downgrade if one of
// the other controller fails to signal it is ready for upgrade.
//
// This test is functional, ensuring that the upgrader worker
// terminates the machine agent with the UpgradeReadyError which
// makes the downgrade happen.
// Speed up the watcher frequency to make the test much faster.
s.PatchValue(&watcher.Period, 200*time.Millisecond)
// Provide (fake) tools so that the upgrader has something to downgrade to.
envtesting.AssertUploadFakeToolsVersions(
c, s.DefaultToolsStorage, s.Environ.Config().AgentStream(), s.Environ.Config().AgentStream(), s.oldVersion)
// Create 3 controllers
machineA, _ := s.makeStateAgentVersion(c, s.oldVersion)
changes, err := s.State.EnableHA(3, constraints.Value{}, "quantal", nil)
c.Assert(err, jc.ErrorIsNil)
c.Assert(len(changes.Added), gc.Equals, 2)
machineB, _, _ := s.configureMachine(c, changes.Added[0], s.oldVersion)
s.configureMachine(c, changes.Added[1], s.oldVersion)
// One of the other controllers is ready for upgrade (but machine C isn't).
info, err := s.State.EnsureUpgradeInfo(machineB.Id(), s.oldVersion.Number, jujuversion.Current)
c.Assert(err, jc.ErrorIsNil)
// Ensure the agent will think it's the master controller.
fakeIsMachineMaster := func(*state.State, string) (bool, error) {
return true, nil
}
s.PatchValue(&upgradesteps.IsMachineMaster, fakeIsMachineMaster)
// Start the agent
agent := s.newAgent(c, machineA)
defer agent.Stop()
agentDone := make(chan error)
go func() {
agentDone <- agent.Run(nil)
}()
select {
case agentErr := <-agentDone:
upgradeReadyErr, ok := agentErr.(*upgrader.UpgradeReadyError)
if !ok {
c.Fatalf("didn't see UpgradeReadyError, instead got: %v", agentErr)
}
// Confirm that the downgrade is back to the previous version.
current := version.Binary{
Number: jujuversion.Current,
Arch: arch.HostArch(),
Series: series.HostSeries(),
}
c.Assert(upgradeReadyErr.OldTools, gc.Equals, current)
c.Assert(upgradeReadyErr.NewTools, gc.Equals, s.oldVersion)
case <-time.After(coretesting.LongWait):
c.Fatal("machine agent did not exit as expected")
}
// UpgradeInfo doc should now be archived.
err = info.Refresh()
c.Assert(err, gc.ErrorMatches, "current upgrade info not found")
}