Ejemplo n.º 1
0
func (w *upgradesteps) finaliseUpgrade(info *state.UpgradeInfo) error {
	if !w.isStateServer {
		return nil
	}

	if w.isMaster {
		// Tell other state servers that the master has completed its
		// upgrade steps.
		if err := info.SetStatus(state.UpgradeFinishing); err != nil {
			return errors.Annotate(err, "upgrade done but")
		}
	}

	if err := info.SetStateServerDone(w.tag.Id()); err != nil {
		return errors.Annotate(err, "upgrade done but failed to synchronise")
	}

	return nil
}
Ejemplo n.º 2
0
func (c *upgradeWorkerContext) finaliseUpgrade(info *state.UpgradeInfo) error {
	if !c.isStateServer {
		return nil
	}

	if c.isMaster {
		// Tell other state servers that the master has completed its
		// upgrade steps.
		if err := info.SetStatus(state.UpgradeFinishing); err != nil {
			return errors.Annotate(err, "upgrade done but")
		}
	}

	if err := info.SetStateServerDone(c.machineId); err != nil {
		return errors.Annotate(err, "upgrade done but failed to synchronise")
	}

	return nil
}
Ejemplo n.º 3
0
// runUpgrades runs the upgrade operations for each job type and
// updates the updatedToVersion on success.
func (c *upgradeWorkerContext) runUpgrades() error {
	from := version.Current
	from.Number = c.agentConfig.UpgradedToVersion()
	if from == version.Current {
		logger.Infof("upgrade to %v already completed.", version.Current)
		return nil
	}

	a := c.agent
	tag, ok := c.agentConfig.Tag().(names.MachineTag)
	if !ok {
		return errors.New("machine agent's tag is not a MachineTag")
	}
	machineId := tag.Id()

	isMaster, err := isMachineMaster(c.st, machineId)
	if err != nil {
		return errors.Trace(err)
	}

	var upgradeInfo *state.UpgradeInfo
	if c.isStateServer {
		upgradeInfo, err = c.st.EnsureUpgradeInfo(machineId, from.Number, version.Current.Number)
		if err != nil {
			return errors.Trace(err)
		}

		// State servers need to wait for other state servers to be
		// ready to run the upgrade.
		logger.Infof("waiting for other state servers to be ready for upgrade")
		if err := c.waitForOtherStateServers(upgradeInfo, isMaster); err != nil {
			if err == agentTerminating {
				logger.Warningf(`stopped waiting for other state servers: %v`, err)
			} else {
				logger.Errorf(`other state servers failed to come up for upgrade `+
					`to %s - aborting: %v`, version.Current, err)
				a.setMachineStatus(c.apiState, params.StatusError,
					fmt.Sprintf("upgrade to %v aborted while waiting for other "+
						"state servers: %v", version.Current, err))
				// If master, trigger a rollback to the previous agent version.
				if isMaster {
					logger.Errorf("downgrading environment agent version to %v due to aborted upgrade",
						from.Number)
					if rollbackErr := c.st.SetEnvironAgentVersion(from.Number); rollbackErr != nil {
						logger.Errorf("rollback failed: %v", rollbackErr)
						return errors.Annotate(rollbackErr, "failed to roll back desired agent version")
					}
				}
			}
			return err
		}
	}

	err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
		var upgradeErr error
		a.setMachineStatus(c.apiState, params.StatusStarted,
			fmt.Sprintf("upgrading to %v", version.Current))

		context := upgrades.NewContext(agentConfig, c.apiState, c.st)
		for _, job := range c.jobs {
			target := upgradeTarget(job, isMaster)
			if target == "" {
				continue
			}
			logger.Infof("starting upgrade from %v to %v for %v %q",
				from, version.Current, target, tag)

			attempts := getUpgradeRetryStrategy()
			for attempt := attempts.Start(); attempt.Next(); {
				upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
				if upgradeErr == nil {
					break
				}
				if connectionIsDead(c.apiState) {
					// API connection has gone away - abort!
					return &apiLostDuringUpgrade{upgradeErr}
				}
				retryText := "will retry"
				if !attempt.HasNext() {
					retryText = "giving up"
				}
				logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
					from, version.Current, target, tag, retryText, upgradeErr)
				a.setMachineStatus(c.apiState, params.StatusError,
					fmt.Sprintf("upgrade to %v failed (%s): %v",
						version.Current, retryText, upgradeErr))
			}
		}
		if upgradeErr != nil {
			return upgradeErr
		}
		agentConfig.SetUpgradedToVersion(version.Current.Number)
		return nil
	})
	if err != nil {
		logger.Errorf("upgrade to %v failed: %v", version.Current, err)
		return err
	}

	if c.isStateServer {
		if isMaster {
			if err := upgradeInfo.SetStatus(state.UpgradeFinishing); err != nil {
				logger.Errorf("upgrade done but failed to set status: %v", err)
				return err
			}
		}
		if err := upgradeInfo.SetStateServerDone(machineId); err != nil {
			logger.Errorf("upgrade done but failed to synchronise: %v", err)
			return err
		}
	}

	logger.Infof("upgrade to %v completed successfully.", version.Current)
	a.setMachineStatus(c.apiState, params.StatusStarted, "")
	return nil
}