// runUpgradeSteps runs the required upgrade steps for the machine
// agent, retrying on failure. The agent's UpgradedToVersion is set
// once the upgrade is complete.
//
// This function conforms to the agent.ConfigMutator type and is
// designed to be called via a machine agent's ChangeConfig method.
func (c *upgradeWorkerContext) runUpgradeSteps(agentConfig agent.ConfigSetter) error {
var upgradeErr error
a := c.agent
a.setMachineStatus(c.apiState, params.StatusStarted, fmt.Sprintf("upgrading to %v", c.toVersion))
context := upgrades.NewContext(agentConfig, c.apiState, c.st)
logger.Infof("starting upgrade from %v to %v for %q", c.fromVersion, c.toVersion, c.tag)
targets := jobsToTargets(c.jobs, c.isMaster)
attempts := getUpgradeRetryStrategy()
for attempt := attempts.Start(); attempt.Next(); {
upgradeErr = upgradesPerformUpgrade(c.fromVersion, targets, context)
if upgradeErr == nil {
break
}
if cmdutil.ConnectionIsDead(logger, c.apiState) {
// API connection has gone away - abort!
return &apiLostDuringUpgrade{upgradeErr}
}
if attempt.HasNext() {
c.reportUpgradeFailure(upgradeErr, true)
}
}
if upgradeErr != nil {
return upgradeErr
}
agentConfig.SetUpgradedToVersion(c.toVersion)
return nil
}
// runUpgrades runs the upgrade operations for each job type and updates the updatedToVersion on success.
func (a *MachineAgent) runUpgrades(
st *state.State,
apiState *api.State,
jobs []params.MachineJob,
agentConfig agent.Config,
) error {
from := version.Current
from.Number = agentConfig.UpgradedToVersion()
if from == version.Current {
logger.Infof("upgrade to %v already completed.", version.Current)
return nil
}
var err error
writeErr := a.ChangeConfig(func(agentConfig agent.ConfigSetter) {
context := upgrades.NewContext(agentConfig, apiState, st)
for _, job := range jobs {
target := upgradeTarget(job)
if target == "" {
continue
}
logger.Infof("starting upgrade from %v to %v for %v %q", from, version.Current, target, a.Tag())
if err = upgradesPerformUpgrade(from.Number, target, context); err != nil {
err = fmt.Errorf("cannot perform upgrade from %v to %v for %v %q: %v", from, version.Current, target, a.Tag(), err)
return
}
}
agentConfig.SetUpgradedToVersion(version.Current.Number)
})
if writeErr != nil {
return fmt.Errorf("cannot write updated agent configuration: %v", writeErr)
}
return nil
}
// runUpgradeSteps runs the required upgrade steps for the machine
// agent, retrying on failure. The agent's UpgradedToVersion is set
// once the upgrade is complete.
//
// This function conforms to the agent.ConfigMutator type and is
// designed to be called via a machine agent's ChangeConfig method.
func (w *upgradesteps) runUpgradeSteps(agentConfig agent.ConfigSetter) error {
var upgradeErr error
w.machine.SetStatus(status.Started, fmt.Sprintf("upgrading to %v", w.toVersion), nil)
context := upgrades.NewContext(agentConfig, w.apiConn, w.st)
logger.Infof("starting upgrade from %v to %v for %q", w.fromVersion, w.toVersion, w.tag)
targets := jobsToTargets(w.jobs, w.isMaster)
attempts := getUpgradeRetryStrategy()
for attempt := attempts.Start(); attempt.Next(); {
upgradeErr = PerformUpgrade(w.fromVersion, targets, context)
if upgradeErr == nil {
break
}
if cmdutil.ConnectionIsDead(logger, w.apiConn) {
// API connection has gone away - abort!
return &apiLostDuringUpgrade{upgradeErr}
}
if attempt.HasNext() {
w.reportUpgradeFailure(upgradeErr, true)
}
}
if upgradeErr != nil {
return upgradeErr
}
agentConfig.SetUpgradedToVersion(w.toVersion)
return nil
}
// runUpgrades runs the upgrade operations for each job type and updates the updatedToVersion on success.
func (a *MachineAgent) runUpgrades(
st *state.State,
apiState *api.State,
jobs []params.MachineJob,
agentConfig agent.Config,
) error {
from := version.Current
from.Number = agentConfig.UpgradedToVersion()
if from == version.Current {
logger.Infof("upgrade to %v already completed.", version.Current)
return nil
}
isMaster, err := a.isMaster(st, agentConfig)
if err != nil {
return errors.Trace(err)
}
err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
var upgradeErr error
a.setMachineStatus(apiState, params.StatusStarted,
fmt.Sprintf("upgrading to %v", version.Current))
context := upgrades.NewContext(agentConfig, apiState, st)
for _, job := range jobs {
target := upgradeTarget(job, isMaster)
if target == "" {
continue
}
logger.Infof("starting upgrade from %v to %v for %v %q", from, version.Current, target, a.Tag())
attempts := getUpgradeRetryStrategy()
for attempt := attempts.Start(); attempt.Next(); {
upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
if upgradeErr == nil {
break
} else {
retryText := "will retry"
if !attempt.HasNext() {
retryText = "giving up"
}
logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
from, version.Current, target, a.Tag(), retryText, upgradeErr)
a.setMachineStatus(apiState, params.StatusError,
fmt.Sprintf("upgrade to %v failed (%s): %v", version.Current, retryText, upgradeErr))
}
}
}
if upgradeErr != nil {
return upgradeErr
}
agentConfig.SetUpgradedToVersion(version.Current.Number)
return nil
})
if err != nil {
logger.Errorf("upgrade to %v failed: %v", version.Current, err)
return &fatalError{err.Error()}
}
logger.Infof("upgrade to %v completed successfully.", version.Current)
a.setMachineStatus(apiState, params.StatusStarted, "")
return nil
}
func (s *upgradeSuite) checkContextRestriction(c *gc.C, expectedPanic string) {
fromVersion := version.MustParse("1.20.0")
type fakeAgentConfigSetter struct{ agent.ConfigSetter }
ctx := upgrades.NewContext(fakeAgentConfigSetter{}, nil, new(state.State))
c.Assert(
func() { upgrades.PerformUpgrade(fromVersion, targets(upgrades.Controller), ctx) },
gc.PanicMatches, expectedPanic,
)
}
// runUpgrades runs the upgrade operations for each job type and
// updates the updatedToVersion on success.
func (c *upgradeWorkerContext) runUpgrades() error {
from := version.Current
from.Number = c.agentConfig.UpgradedToVersion()
if from == version.Current {
logger.Infof("upgrade to %v already completed.", version.Current)
return nil
}
a := c.agent
tag := c.agentConfig.Tag().(names.MachineTag)
isMaster, err := isMachineMaster(c.st, tag)
if err != nil {
return errors.Trace(err)
}
if c.isStateServer {
// State servers need to wait for other state servers to be
// ready to run the upgrade.
if err := waitForOtherStateServers(c.st, isMaster); err != nil {
logger.Errorf(`other state servers failed to come up for upgrade `+
`to %s - aborting: %v`, version.Current, err)
a.setMachineStatus(c.apiState, params.StatusError,
fmt.Sprintf("upgrade to %v aborted while waiting for other "+
"state servers: %v", version.Current, err))
// TODO(menn0): if master, roll agent-version back to
// previous version, once the upgrader allows downgrades
// to previous version.
return err
}
}
err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
var upgradeErr error
a.setMachineStatus(c.apiState, params.StatusStarted,
fmt.Sprintf("upgrading to %v", version.Current))
context := upgrades.NewContext(agentConfig, c.apiState, c.st)
for _, job := range c.jobs {
target := upgradeTarget(job, isMaster)
if target == "" {
continue
}
logger.Infof("starting upgrade from %v to %v for %v %q",
from, version.Current, target, tag)
attempts := getUpgradeRetryStrategy()
for attempt := attempts.Start(); attempt.Next(); {
upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
if upgradeErr == nil {
break
}
if connectionIsDead(c.apiState) {
// API connection has gone away - abort!
return &apiLostDuringUpgrade{upgradeErr}
}
retryText := "will retry"
if !attempt.HasNext() {
retryText = "giving up"
}
logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
from, version.Current, target, tag, retryText, upgradeErr)
a.setMachineStatus(c.apiState, params.StatusError,
fmt.Sprintf("upgrade to %v failed (%s): %v",
version.Current, retryText, upgradeErr))
}
}
if upgradeErr != nil {
return upgradeErr
}
agentConfig.SetUpgradedToVersion(version.Current.Number)
return nil
})
if err != nil {
logger.Errorf("upgrade to %v failed: %v", version.Current, err)
return err
}
logger.Infof("upgrade to %v completed successfully.", version.Current)
a.setMachineStatus(c.apiState, params.StatusStarted, "")
return nil
}
// runUpgrades runs the upgrade operations for each job type and
// updates the updatedToVersion on success.
func (c *upgradeWorkerContext) runUpgrades(st *state.State, agentConfig agent.Config) error {
from := version.Current
from.Number = agentConfig.UpgradedToVersion()
if from == version.Current {
logger.Infof("upgrade to %v already completed.", version.Current)
return nil
}
a := c.agent
tag := agentConfig.Tag().(names.MachineTag)
isMaster, err := isMachineMaster(st, tag)
if err != nil {
return errors.Trace(err)
}
err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
var upgradeErr error
a.setMachineStatus(c.apiState, params.StatusStarted,
fmt.Sprintf("upgrading to %v", version.Current))
context := upgrades.NewContext(agentConfig, c.apiState, st)
for _, job := range c.jobs {
target := upgradeTarget(job, isMaster)
if target == "" {
continue
}
logger.Infof("starting upgrade from %v to %v for %v %q",
from, version.Current, target, tag)
attempts := getUpgradeRetryStrategy()
for attempt := attempts.Start(); attempt.Next(); {
upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
if upgradeErr == nil {
break
}
if connectionIsDead(c.apiState) {
// API connection has gone away - abort!
return &apiLostDuringUpgrade{upgradeErr}
}
retryText := "will retry"
if !attempt.HasNext() {
retryText = "giving up"
}
logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
from, version.Current, target, tag, retryText, upgradeErr)
a.setMachineStatus(c.apiState, params.StatusError,
fmt.Sprintf("upgrade to %v failed (%s): %v",
version.Current, retryText, upgradeErr))
}
}
if upgradeErr != nil {
return upgradeErr
}
agentConfig.SetUpgradedToVersion(version.Current.Number)
return nil
})
if err != nil {
logger.Errorf("upgrade to %v failed: %v", version.Current, err)
return err
}
logger.Infof("upgrade to %v completed successfully.", version.Current)
a.setMachineStatus(c.apiState, params.StatusStarted, "")
return nil
}
// runUpgrades runs the upgrade operations for each job type and
// updates the updatedToVersion on success.
func (c *upgradeWorkerContext) runUpgrades() error {
from := version.Current
from.Number = c.agentConfig.UpgradedToVersion()
if from == version.Current {
logger.Infof("upgrade to %v already completed.", version.Current)
return nil
}
a := c.agent
tag, ok := c.agentConfig.Tag().(names.MachineTag)
if !ok {
return errors.New("machine agent's tag is not a MachineTag")
}
machineId := tag.Id()
isMaster, err := isMachineMaster(c.st, machineId)
if err != nil {
return errors.Trace(err)
}
var upgradeInfo *state.UpgradeInfo
if c.isStateServer {
upgradeInfo, err = c.st.EnsureUpgradeInfo(machineId, from.Number, version.Current.Number)
if err != nil {
return errors.Trace(err)
}
// State servers need to wait for other state servers to be
// ready to run the upgrade.
logger.Infof("waiting for other state servers to be ready for upgrade")
if err := c.waitForOtherStateServers(upgradeInfo, isMaster); err != nil {
if err == agentTerminating {
logger.Warningf(`stopped waiting for other state servers: %v`, err)
} else {
logger.Errorf(`other state servers failed to come up for upgrade `+
`to %s - aborting: %v`, version.Current, err)
a.setMachineStatus(c.apiState, params.StatusError,
fmt.Sprintf("upgrade to %v aborted while waiting for other "+
"state servers: %v", version.Current, err))
// If master, trigger a rollback to the previous agent version.
if isMaster {
logger.Errorf("downgrading environment agent version to %v due to aborted upgrade",
from.Number)
if rollbackErr := c.st.SetEnvironAgentVersion(from.Number); rollbackErr != nil {
logger.Errorf("rollback failed: %v", rollbackErr)
return errors.Annotate(rollbackErr, "failed to roll back desired agent version")
}
}
}
return err
}
}
err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
var upgradeErr error
a.setMachineStatus(c.apiState, params.StatusStarted,
fmt.Sprintf("upgrading to %v", version.Current))
context := upgrades.NewContext(agentConfig, c.apiState, c.st)
for _, job := range c.jobs {
target := upgradeTarget(job, isMaster)
if target == "" {
continue
}
logger.Infof("starting upgrade from %v to %v for %v %q",
from, version.Current, target, tag)
attempts := getUpgradeRetryStrategy()
for attempt := attempts.Start(); attempt.Next(); {
upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
if upgradeErr == nil {
break
}
if connectionIsDead(c.apiState) {
// API connection has gone away - abort!
return &apiLostDuringUpgrade{upgradeErr}
}
retryText := "will retry"
if !attempt.HasNext() {
retryText = "giving up"
}
logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
from, version.Current, target, tag, retryText, upgradeErr)
a.setMachineStatus(c.apiState, params.StatusError,
fmt.Sprintf("upgrade to %v failed (%s): %v",
version.Current, retryText, upgradeErr))
}
}
if upgradeErr != nil {
return upgradeErr
}
agentConfig.SetUpgradedToVersion(version.Current.Number)
return nil
})
if err != nil {
logger.Errorf("upgrade to %v failed: %v", version.Current, err)
return err
}
if c.isStateServer {
if isMaster {
if err := upgradeInfo.SetStatus(state.UpgradeFinishing); err != nil {
logger.Errorf("upgrade done but failed to set status: %v", err)
return err
}
}
if err := upgradeInfo.SetStateServerDone(machineId); err != nil {
logger.Errorf("upgrade done but failed to synchronise: %v", err)
return err
}
}
logger.Infof("upgrade to %v completed successfully.", version.Current)
a.setMachineStatus(c.apiState, params.StatusStarted, "")
return nil
}