func (c *upgradeWorkerContext) finaliseUpgrade(info *state.UpgradeInfo) error {
if !c.isStateServer {
return nil
}
if c.isMaster {
// Tell other state servers that the master has completed its
// upgrade steps.
if err := info.SetStatus(state.UpgradeFinishing); err != nil {
return errors.Annotate(err, "upgrade done but")
}
}
if err := info.SetStateServerDone(c.machineId); err != nil {
return errors.Annotate(err, "upgrade done but failed to synchronise")
}
return nil
}
func (w *upgradesteps) finaliseUpgrade(info *state.UpgradeInfo) error {
if !w.isController {
return nil
}
if w.isMaster {
// Tell other controllers that the master has completed its
// upgrade steps.
if err := info.SetStatus(state.UpgradeFinishing); err != nil {
return errors.Annotate(err, "upgrade done but")
}
}
if err := info.SetControllerDone(w.tag.Id()); err != nil {
return errors.Annotate(err, "upgrade done but failed to synchronise")
}
return nil
}
func (s *UpgradeSuite) checkUpgradeInfoArchived(
c *gc.C,
initialInfo *state.UpgradeInfo,
expectedStatus state.UpgradeStatus,
expectedControllers int,
) {
info := s.getOneUpgradeInfo(c)
c.Assert(info.Status(), gc.Equals, expectedStatus)
c.Assert(info.PreviousVersion(), gc.Equals, initialInfo.PreviousVersion())
c.Assert(info.TargetVersion(), gc.Equals, initialInfo.TargetVersion())
// Truncate because mongo only stores times down to millisecond resolution.
c.Assert(info.Started().Equal(initialInfo.Started().Truncate(time.Millisecond)), jc.IsTrue)
c.Assert(len(info.ControllersDone()), gc.Equals, expectedControllers)
if expectedControllers > 0 {
c.Assert(info.ControllersDone(), jc.SameContents, info.ControllersReady())
}
}
func (c *upgradeWorkerContext) waitForOtherStateServers(info *state.UpgradeInfo) error {
watcher := info.Watch()
defer watcher.Stop()
maxWait := getUpgradeStartTimeout(c.isMaster)
timeout := time.After(maxWait)
for {
select {
case <-watcher.Changes():
if err := info.Refresh(); err != nil {
return errors.Trace(err)
}
if c.isMaster {
if ready, err := info.AllProvisionedStateServersReady(); err != nil {
return errors.Trace(err)
} else if ready {
// All state servers ready to start upgrade
err := info.SetStatus(state.UpgradeRunning)
return errors.Trace(err)
}
} else {
if info.Status() == state.UpgradeFinishing {
// Master is done, ok to proceed
return nil
}
}
case <-timeout:
if c.isMaster {
if err := info.Abort(); err != nil {
return errors.Annotate(err, "unable to abort upgrade")
}
}
return errors.Errorf("timed out after %s", maxWait)
case <-c.agent.Dying():
return agentTerminating
}
}
}
func (s *UpgradeSuite) setToFinishing(c *gc.C, info *state.UpgradeInfo) {
err := info.SetStatus(state.UpgradeRunning)
c.Assert(err, jc.ErrorIsNil)
err = info.SetStatus(state.UpgradeFinishing)
c.Assert(err, jc.ErrorIsNil)
}
// runUpgrades runs the upgrade operations for each job type and
// updates the updatedToVersion on success.
func (c *upgradeWorkerContext) runUpgrades() error {
from := version.Current
from.Number = c.agentConfig.UpgradedToVersion()
if from == version.Current {
logger.Infof("upgrade to %v already completed.", version.Current)
return nil
}
a := c.agent
tag, ok := c.agentConfig.Tag().(names.MachineTag)
if !ok {
return errors.New("machine agent's tag is not a MachineTag")
}
machineId := tag.Id()
isMaster, err := isMachineMaster(c.st, machineId)
if err != nil {
return errors.Trace(err)
}
var upgradeInfo *state.UpgradeInfo
if c.isStateServer {
upgradeInfo, err = c.st.EnsureUpgradeInfo(machineId, from.Number, version.Current.Number)
if err != nil {
return errors.Trace(err)
}
// State servers need to wait for other state servers to be
// ready to run the upgrade.
logger.Infof("waiting for other state servers to be ready for upgrade")
if err := c.waitForOtherStateServers(upgradeInfo, isMaster); err != nil {
if err == agentTerminating {
logger.Warningf(`stopped waiting for other state servers: %v`, err)
} else {
logger.Errorf(`other state servers failed to come up for upgrade `+
`to %s - aborting: %v`, version.Current, err)
a.setMachineStatus(c.apiState, params.StatusError,
fmt.Sprintf("upgrade to %v aborted while waiting for other "+
"state servers: %v", version.Current, err))
// If master, trigger a rollback to the previous agent version.
if isMaster {
logger.Errorf("downgrading environment agent version to %v due to aborted upgrade",
from.Number)
if rollbackErr := c.st.SetEnvironAgentVersion(from.Number); rollbackErr != nil {
logger.Errorf("rollback failed: %v", rollbackErr)
return errors.Annotate(rollbackErr, "failed to roll back desired agent version")
}
}
}
return err
}
}
err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
var upgradeErr error
a.setMachineStatus(c.apiState, params.StatusStarted,
fmt.Sprintf("upgrading to %v", version.Current))
context := upgrades.NewContext(agentConfig, c.apiState, c.st)
for _, job := range c.jobs {
target := upgradeTarget(job, isMaster)
if target == "" {
continue
}
logger.Infof("starting upgrade from %v to %v for %v %q",
from, version.Current, target, tag)
attempts := getUpgradeRetryStrategy()
for attempt := attempts.Start(); attempt.Next(); {
upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
if upgradeErr == nil {
break
}
if connectionIsDead(c.apiState) {
// API connection has gone away - abort!
return &apiLostDuringUpgrade{upgradeErr}
}
retryText := "will retry"
if !attempt.HasNext() {
retryText = "giving up"
}
logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
from, version.Current, target, tag, retryText, upgradeErr)
a.setMachineStatus(c.apiState, params.StatusError,
fmt.Sprintf("upgrade to %v failed (%s): %v",
version.Current, retryText, upgradeErr))
}
}
if upgradeErr != nil {
return upgradeErr
}
agentConfig.SetUpgradedToVersion(version.Current.Number)
return nil
})
if err != nil {
logger.Errorf("upgrade to %v failed: %v", version.Current, err)
return err
}
if c.isStateServer {
if isMaster {
if err := upgradeInfo.SetStatus(state.UpgradeFinishing); err != nil {
logger.Errorf("upgrade done but failed to set status: %v", err)
return err
}
}
if err := upgradeInfo.SetStateServerDone(machineId); err != nil {
logger.Errorf("upgrade done but failed to synchronise: %v", err)
return err
}
}
logger.Infof("upgrade to %v completed successfully.", version.Current)
a.setMachineStatus(c.apiState, params.StatusStarted, "")
return nil
}