Ejemplo n.º 1
0
// runUpgradeSteps runs the required upgrade steps for the machine
// agent, retrying on failure. The agent's UpgradedToVersion is set
// once the upgrade is complete.
//
// This function conforms to the agent.ConfigMutator type and is
// designed to be called via a machine agent's ChangeConfig method.
func (c *upgradeWorkerContext) runUpgradeSteps(agentConfig agent.ConfigSetter) error {
	var upgradeErr error
	a := c.agent
	a.setMachineStatus(c.apiState, params.StatusStarted, fmt.Sprintf("upgrading to %v", c.toVersion))

	context := upgrades.NewContext(agentConfig, c.apiState, c.st)
	logger.Infof("starting upgrade from %v to %v for %q", c.fromVersion, c.toVersion, c.tag)

	targets := jobsToTargets(c.jobs, c.isMaster)
	attempts := getUpgradeRetryStrategy()
	for attempt := attempts.Start(); attempt.Next(); {
		upgradeErr = upgradesPerformUpgrade(c.fromVersion, targets, context)
		if upgradeErr == nil {
			break
		}
		if cmdutil.ConnectionIsDead(logger, c.apiState) {
			// API connection has gone away - abort!
			return &apiLostDuringUpgrade{upgradeErr}
		}
		if attempt.HasNext() {
			c.reportUpgradeFailure(upgradeErr, true)
		}
	}
	if upgradeErr != nil {
		return upgradeErr
	}
	agentConfig.SetUpgradedToVersion(c.toVersion)
	return nil
}
Ejemplo n.º 2
0
// runUpgrades runs the upgrade operations for each job type and updates the updatedToVersion on success.
func (a *MachineAgent) runUpgrades(
	st *state.State,
	apiState *api.State,
	jobs []params.MachineJob,
	agentConfig agent.Config,
) error {
	from := version.Current
	from.Number = agentConfig.UpgradedToVersion()
	if from == version.Current {
		logger.Infof("upgrade to %v already completed.", version.Current)
		return nil
	}
	var err error
	writeErr := a.ChangeConfig(func(agentConfig agent.ConfigSetter) {
		context := upgrades.NewContext(agentConfig, apiState, st)
		for _, job := range jobs {
			target := upgradeTarget(job)
			if target == "" {
				continue
			}
			logger.Infof("starting upgrade from %v to %v for %v %q", from, version.Current, target, a.Tag())
			if err = upgradesPerformUpgrade(from.Number, target, context); err != nil {
				err = fmt.Errorf("cannot perform upgrade from %v to %v for %v %q: %v", from, version.Current, target, a.Tag(), err)
				return
			}
		}
		agentConfig.SetUpgradedToVersion(version.Current.Number)
	})
	if writeErr != nil {
		return fmt.Errorf("cannot write updated agent configuration: %v", writeErr)
	}
	return nil
}
Ejemplo n.º 3
0
Archivo: worker.go Proyecto: bac/juju
// runUpgradeSteps runs the required upgrade steps for the machine
// agent, retrying on failure. The agent's UpgradedToVersion is set
// once the upgrade is complete.
//
// This function conforms to the agent.ConfigMutator type and is
// designed to be called via a machine agent's ChangeConfig method.
func (w *upgradesteps) runUpgradeSteps(agentConfig agent.ConfigSetter) error {
	var upgradeErr error
	w.machine.SetStatus(status.Started, fmt.Sprintf("upgrading to %v", w.toVersion), nil)

	context := upgrades.NewContext(agentConfig, w.apiConn, w.st)
	logger.Infof("starting upgrade from %v to %v for %q", w.fromVersion, w.toVersion, w.tag)

	targets := jobsToTargets(w.jobs, w.isMaster)
	attempts := getUpgradeRetryStrategy()
	for attempt := attempts.Start(); attempt.Next(); {
		upgradeErr = PerformUpgrade(w.fromVersion, targets, context)
		if upgradeErr == nil {
			break
		}
		if cmdutil.ConnectionIsDead(logger, w.apiConn) {
			// API connection has gone away - abort!
			return &apiLostDuringUpgrade{upgradeErr}
		}
		if attempt.HasNext() {
			w.reportUpgradeFailure(upgradeErr, true)
		}
	}
	if upgradeErr != nil {
		return upgradeErr
	}
	agentConfig.SetUpgradedToVersion(w.toVersion)
	return nil
}
Ejemplo n.º 4
0
// runUpgrades runs the upgrade operations for each job type and updates the updatedToVersion on success.
func (a *MachineAgent) runUpgrades(
	st *state.State,
	apiState *api.State,
	jobs []params.MachineJob,
	agentConfig agent.Config,
) error {
	from := version.Current
	from.Number = agentConfig.UpgradedToVersion()
	if from == version.Current {
		logger.Infof("upgrade to %v already completed.", version.Current)
		return nil
	}
	isMaster, err := a.isMaster(st, agentConfig)
	if err != nil {
		return errors.Trace(err)
	}
	err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
		var upgradeErr error
		a.setMachineStatus(apiState, params.StatusStarted,
			fmt.Sprintf("upgrading to %v", version.Current))
		context := upgrades.NewContext(agentConfig, apiState, st)
		for _, job := range jobs {
			target := upgradeTarget(job, isMaster)
			if target == "" {
				continue
			}
			logger.Infof("starting upgrade from %v to %v for %v %q", from, version.Current, target, a.Tag())

			attempts := getUpgradeRetryStrategy()
			for attempt := attempts.Start(); attempt.Next(); {
				upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
				if upgradeErr == nil {
					break
				} else {
					retryText := "will retry"
					if !attempt.HasNext() {
						retryText = "giving up"
					}
					logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
						from, version.Current, target, a.Tag(), retryText, upgradeErr)
					a.setMachineStatus(apiState, params.StatusError,
						fmt.Sprintf("upgrade to %v failed (%s): %v", version.Current, retryText, upgradeErr))
				}
			}
		}
		if upgradeErr != nil {
			return upgradeErr
		}
		agentConfig.SetUpgradedToVersion(version.Current.Number)
		return nil
	})
	if err != nil {
		logger.Errorf("upgrade to %v failed: %v", version.Current, err)
		return &fatalError{err.Error()}
	}

	logger.Infof("upgrade to %v completed successfully.", version.Current)
	a.setMachineStatus(apiState, params.StatusStarted, "")
	return nil
}
Ejemplo n.º 5
0
func (s *upgradeSuite) checkContextRestriction(c *gc.C, expectedPanic string) {
	fromVersion := version.MustParse("1.20.0")
	type fakeAgentConfigSetter struct{ agent.ConfigSetter }
	ctx := upgrades.NewContext(fakeAgentConfigSetter{}, nil, new(state.State))
	c.Assert(
		func() { upgrades.PerformUpgrade(fromVersion, targets(upgrades.Controller), ctx) },
		gc.PanicMatches, expectedPanic,
	)
}
Ejemplo n.º 6
0
// runUpgrades runs the upgrade operations for each job type and
// updates the updatedToVersion on success.
func (c *upgradeWorkerContext) runUpgrades() error {
	from := version.Current
	from.Number = c.agentConfig.UpgradedToVersion()
	if from == version.Current {
		logger.Infof("upgrade to %v already completed.", version.Current)
		return nil
	}

	a := c.agent
	tag := c.agentConfig.Tag().(names.MachineTag)

	isMaster, err := isMachineMaster(c.st, tag)
	if err != nil {
		return errors.Trace(err)
	}

	if c.isStateServer {
		// State servers need to wait for other state servers to be
		// ready to run the upgrade.
		if err := waitForOtherStateServers(c.st, isMaster); err != nil {
			logger.Errorf(`other state servers failed to come up for upgrade `+
				`to %s - aborting: %v`, version.Current, err)

			a.setMachineStatus(c.apiState, params.StatusError,
				fmt.Sprintf("upgrade to %v aborted while waiting for other "+
					"state servers: %v", version.Current, err))

			// TODO(menn0): if master, roll agent-version back to
			// previous version, once the upgrader allows downgrades
			// to previous version.

			return err
		}
	}

	err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
		var upgradeErr error
		a.setMachineStatus(c.apiState, params.StatusStarted,
			fmt.Sprintf("upgrading to %v", version.Current))

		context := upgrades.NewContext(agentConfig, c.apiState, c.st)
		for _, job := range c.jobs {
			target := upgradeTarget(job, isMaster)
			if target == "" {
				continue
			}
			logger.Infof("starting upgrade from %v to %v for %v %q",
				from, version.Current, target, tag)

			attempts := getUpgradeRetryStrategy()
			for attempt := attempts.Start(); attempt.Next(); {
				upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
				if upgradeErr == nil {
					break
				}
				if connectionIsDead(c.apiState) {
					// API connection has gone away - abort!
					return &apiLostDuringUpgrade{upgradeErr}
				}
				retryText := "will retry"
				if !attempt.HasNext() {
					retryText = "giving up"
				}
				logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
					from, version.Current, target, tag, retryText, upgradeErr)
				a.setMachineStatus(c.apiState, params.StatusError,
					fmt.Sprintf("upgrade to %v failed (%s): %v",
						version.Current, retryText, upgradeErr))
			}
		}
		if upgradeErr != nil {
			return upgradeErr
		}
		agentConfig.SetUpgradedToVersion(version.Current.Number)
		return nil
	})
	if err != nil {
		logger.Errorf("upgrade to %v failed: %v", version.Current, err)
		return err
	}

	logger.Infof("upgrade to %v completed successfully.", version.Current)
	a.setMachineStatus(c.apiState, params.StatusStarted, "")
	return nil
}
Ejemplo n.º 7
0
// runUpgrades runs the upgrade operations for each job type and
// updates the updatedToVersion on success.
func (c *upgradeWorkerContext) runUpgrades(st *state.State, agentConfig agent.Config) error {
	from := version.Current
	from.Number = agentConfig.UpgradedToVersion()
	if from == version.Current {
		logger.Infof("upgrade to %v already completed.", version.Current)
		return nil
	}

	a := c.agent
	tag := agentConfig.Tag().(names.MachineTag)

	isMaster, err := isMachineMaster(st, tag)
	if err != nil {
		return errors.Trace(err)
	}

	err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
		var upgradeErr error
		a.setMachineStatus(c.apiState, params.StatusStarted,
			fmt.Sprintf("upgrading to %v", version.Current))
		context := upgrades.NewContext(agentConfig, c.apiState, st)
		for _, job := range c.jobs {
			target := upgradeTarget(job, isMaster)
			if target == "" {
				continue
			}
			logger.Infof("starting upgrade from %v to %v for %v %q",
				from, version.Current, target, tag)

			attempts := getUpgradeRetryStrategy()
			for attempt := attempts.Start(); attempt.Next(); {
				upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
				if upgradeErr == nil {
					break
				}
				if connectionIsDead(c.apiState) {
					// API connection has gone away - abort!
					return &apiLostDuringUpgrade{upgradeErr}
				}
				retryText := "will retry"
				if !attempt.HasNext() {
					retryText = "giving up"
				}
				logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
					from, version.Current, target, tag, retryText, upgradeErr)
				a.setMachineStatus(c.apiState, params.StatusError,
					fmt.Sprintf("upgrade to %v failed (%s): %v",
						version.Current, retryText, upgradeErr))
			}
		}
		if upgradeErr != nil {
			return upgradeErr
		}
		agentConfig.SetUpgradedToVersion(version.Current.Number)
		return nil
	})
	if err != nil {
		logger.Errorf("upgrade to %v failed: %v", version.Current, err)
		return err
	}

	logger.Infof("upgrade to %v completed successfully.", version.Current)
	a.setMachineStatus(c.apiState, params.StatusStarted, "")
	return nil
}
Ejemplo n.º 8
0
// runUpgrades runs the upgrade operations for each job type and
// updates the updatedToVersion on success.
func (c *upgradeWorkerContext) runUpgrades() error {
	from := version.Current
	from.Number = c.agentConfig.UpgradedToVersion()
	if from == version.Current {
		logger.Infof("upgrade to %v already completed.", version.Current)
		return nil
	}

	a := c.agent
	tag, ok := c.agentConfig.Tag().(names.MachineTag)
	if !ok {
		return errors.New("machine agent's tag is not a MachineTag")
	}
	machineId := tag.Id()

	isMaster, err := isMachineMaster(c.st, machineId)
	if err != nil {
		return errors.Trace(err)
	}

	var upgradeInfo *state.UpgradeInfo
	if c.isStateServer {
		upgradeInfo, err = c.st.EnsureUpgradeInfo(machineId, from.Number, version.Current.Number)
		if err != nil {
			return errors.Trace(err)
		}

		// State servers need to wait for other state servers to be
		// ready to run the upgrade.
		logger.Infof("waiting for other state servers to be ready for upgrade")
		if err := c.waitForOtherStateServers(upgradeInfo, isMaster); err != nil {
			if err == agentTerminating {
				logger.Warningf(`stopped waiting for other state servers: %v`, err)
			} else {
				logger.Errorf(`other state servers failed to come up for upgrade `+
					`to %s - aborting: %v`, version.Current, err)
				a.setMachineStatus(c.apiState, params.StatusError,
					fmt.Sprintf("upgrade to %v aborted while waiting for other "+
						"state servers: %v", version.Current, err))
				// If master, trigger a rollback to the previous agent version.
				if isMaster {
					logger.Errorf("downgrading environment agent version to %v due to aborted upgrade",
						from.Number)
					if rollbackErr := c.st.SetEnvironAgentVersion(from.Number); rollbackErr != nil {
						logger.Errorf("rollback failed: %v", rollbackErr)
						return errors.Annotate(rollbackErr, "failed to roll back desired agent version")
					}
				}
			}
			return err
		}
	}

	err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
		var upgradeErr error
		a.setMachineStatus(c.apiState, params.StatusStarted,
			fmt.Sprintf("upgrading to %v", version.Current))

		context := upgrades.NewContext(agentConfig, c.apiState, c.st)
		for _, job := range c.jobs {
			target := upgradeTarget(job, isMaster)
			if target == "" {
				continue
			}
			logger.Infof("starting upgrade from %v to %v for %v %q",
				from, version.Current, target, tag)

			attempts := getUpgradeRetryStrategy()
			for attempt := attempts.Start(); attempt.Next(); {
				upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
				if upgradeErr == nil {
					break
				}
				if connectionIsDead(c.apiState) {
					// API connection has gone away - abort!
					return &apiLostDuringUpgrade{upgradeErr}
				}
				retryText := "will retry"
				if !attempt.HasNext() {
					retryText = "giving up"
				}
				logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
					from, version.Current, target, tag, retryText, upgradeErr)
				a.setMachineStatus(c.apiState, params.StatusError,
					fmt.Sprintf("upgrade to %v failed (%s): %v",
						version.Current, retryText, upgradeErr))
			}
		}
		if upgradeErr != nil {
			return upgradeErr
		}
		agentConfig.SetUpgradedToVersion(version.Current.Number)
		return nil
	})
	if err != nil {
		logger.Errorf("upgrade to %v failed: %v", version.Current, err)
		return err
	}

	if c.isStateServer {
		if isMaster {
			if err := upgradeInfo.SetStatus(state.UpgradeFinishing); err != nil {
				logger.Errorf("upgrade done but failed to set status: %v", err)
				return err
			}
		}
		if err := upgradeInfo.SetStateServerDone(machineId); err != nil {
			logger.Errorf("upgrade done but failed to synchronise: %v", err)
			return err
		}
	}

	logger.Infof("upgrade to %v completed successfully.", version.Current)
	a.setMachineStatus(c.apiState, params.StatusStarted, "")
	return nil
}