// runUpgrades runs the upgrade operations for each job type and updates the updatedToVersion on success.
func (a *MachineAgent) runUpgrades(
st *state.State,
apiState *api.State,
jobs []params.MachineJob,
agentConfig agent.Config,
) error {
from := version.Current
from.Number = agentConfig.UpgradedToVersion()
if from == version.Current {
logger.Infof("upgrade to %v already completed.", version.Current)
return nil
}
var err error
writeErr := a.ChangeConfig(func(agentConfig agent.ConfigSetter) {
context := upgrades.NewContext(agentConfig, apiState, st)
for _, job := range jobs {
target := upgradeTarget(job)
if target == "" {
continue
}
logger.Infof("starting upgrade from %v to %v for %v %q", from, version.Current, target, a.Tag())
if err = upgradesPerformUpgrade(from.Number, target, context); err != nil {
err = fmt.Errorf("cannot perform upgrade from %v to %v for %v %q: %v", from, version.Current, target, a.Tag(), err)
return
}
}
agentConfig.SetUpgradedToVersion(version.Current.Number)
})
if writeErr != nil {
return fmt.Errorf("cannot write updated agent configuration: %v", writeErr)
}
return nil
}
// runUpgrades runs the upgrade operations for each job type and updates the updatedToVersion on success.
func (a *MachineAgent) runUpgrades(
st *state.State,
apiState *api.State,
jobs []params.MachineJob,
agentConfig agent.Config,
) error {
from := version.Current
from.Number = agentConfig.UpgradedToVersion()
if from == version.Current {
logger.Infof("upgrade to %v already completed.", version.Current)
return nil
}
isMaster, err := a.isMaster(st, agentConfig)
if err != nil {
return errors.Trace(err)
}
err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
var upgradeErr error
a.setMachineStatus(apiState, params.StatusStarted,
fmt.Sprintf("upgrading to %v", version.Current))
context := upgrades.NewContext(agentConfig, apiState, st)
for _, job := range jobs {
target := upgradeTarget(job, isMaster)
if target == "" {
continue
}
logger.Infof("starting upgrade from %v to %v for %v %q", from, version.Current, target, a.Tag())
attempts := getUpgradeRetryStrategy()
for attempt := attempts.Start(); attempt.Next(); {
upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
if upgradeErr == nil {
break
} else {
retryText := "will retry"
if !attempt.HasNext() {
retryText = "giving up"
}
logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
from, version.Current, target, a.Tag(), retryText, upgradeErr)
a.setMachineStatus(apiState, params.StatusError,
fmt.Sprintf("upgrade to %v failed (%s): %v", version.Current, retryText, upgradeErr))
}
}
}
if upgradeErr != nil {
return upgradeErr
}
agentConfig.SetUpgradedToVersion(version.Current.Number)
return nil
})
if err != nil {
logger.Errorf("upgrade to %v failed: %v", version.Current, err)
return &fatalError{err.Error()}
}
logger.Infof("upgrade to %v completed successfully.", version.Current)
a.setMachineStatus(apiState, params.StatusStarted, "")
return nil
}
// ensureMongoServer ensures that mongo is installed and running,
// and ready for opening a state connection.
func (a *MachineAgent) ensureMongoServer(agentConfig agent.Config) error {
a.mongoInitMutex.Lock()
defer a.mongoInitMutex.Unlock()
if a.mongoInitialized {
logger.Debugf("mongo is already initialized")
return nil
}
servingInfo, ok := agentConfig.StateServingInfo()
if !ok {
return fmt.Errorf("state worker was started with no state serving info")
}
namespace := agentConfig.Value(agent.Namespace)
// When upgrading from a pre-HA-capable environment,
// we must add machine-0 to the admin database and
// initiate its replicaset.
//
// TODO(axw) remove this when we no longer need
// to upgrade from pre-HA-capable environments.
var shouldInitiateMongoServer bool
var addrs []network.Address
if isPreHAVersion(agentConfig.UpgradedToVersion()) {
_, err := a.ensureMongoAdminUser(agentConfig)
if err != nil {
return err
}
if servingInfo.SharedSecret == "" {
servingInfo.SharedSecret, err = mongo.GenerateSharedSecret()
if err != nil {
return err
}
if err = a.ChangeConfig(func(config agent.ConfigSetter) {
config.SetStateServingInfo(servingInfo)
}); err != nil {
return err
}
agentConfig = a.CurrentConfig()
}
// Note: we set Direct=true in the mongo options because it's
// possible that we've previously upgraded the mongo server's
// configuration to form a replicaset, but failed to initiate it.
st, m, err := openState(agentConfig, mongo.DialOpts{Direct: true})
if err != nil {
return err
}
if err := st.SetStateServingInfo(servingInfo); err != nil {
st.Close()
return fmt.Errorf("cannot set state serving info: %v", err)
}
st.Close()
addrs = m.Addresses()
shouldInitiateMongoServer = true
}
// ensureMongoServer installs/upgrades the upstart config as necessary.
if err := ensureMongoServer(
agentConfig.DataDir(),
namespace,
servingInfo,
); err != nil {
return err
}
if !shouldInitiateMongoServer {
return nil
}
// Initiate the replicaset for upgraded environments.
//
// TODO(axw) remove this when we no longer need
// to upgrade from pre-HA-capable environments.
stateInfo, ok := agentConfig.StateInfo()
if !ok {
return fmt.Errorf("state worker was started with no state serving info")
}
dialInfo, err := mongo.DialInfo(stateInfo.Info, mongo.DefaultDialOpts())
if err != nil {
return err
}
peerAddr := mongo.SelectPeerAddress(addrs)
if peerAddr == "" {
return fmt.Errorf("no appropriate peer address found in %q", addrs)
}
if err := maybeInitiateMongoServer(peergrouper.InitiateMongoParams{
DialInfo: dialInfo,
MemberHostPort: net.JoinHostPort(peerAddr, fmt.Sprint(servingInfo.StatePort)),
User: stateInfo.Tag,
Password: stateInfo.Password,
}); err != nil {
return err
}
a.mongoInitialized = true
return nil
}
// runUpgrades runs the upgrade operations for each job type and
// updates the updatedToVersion on success.
func (c *upgradeWorkerContext) runUpgrades(st *state.State, agentConfig agent.Config) error {
from := version.Current
from.Number = agentConfig.UpgradedToVersion()
if from == version.Current {
logger.Infof("upgrade to %v already completed.", version.Current)
return nil
}
a := c.agent
tag := agentConfig.Tag().(names.MachineTag)
isMaster, err := isMachineMaster(st, tag)
if err != nil {
return errors.Trace(err)
}
err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
var upgradeErr error
a.setMachineStatus(c.apiState, params.StatusStarted,
fmt.Sprintf("upgrading to %v", version.Current))
context := upgrades.NewContext(agentConfig, c.apiState, st)
for _, job := range c.jobs {
target := upgradeTarget(job, isMaster)
if target == "" {
continue
}
logger.Infof("starting upgrade from %v to %v for %v %q",
from, version.Current, target, tag)
attempts := getUpgradeRetryStrategy()
for attempt := attempts.Start(); attempt.Next(); {
upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
if upgradeErr == nil {
break
}
if connectionIsDead(c.apiState) {
// API connection has gone away - abort!
return &apiLostDuringUpgrade{upgradeErr}
}
retryText := "will retry"
if !attempt.HasNext() {
retryText = "giving up"
}
logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
from, version.Current, target, tag, retryText, upgradeErr)
a.setMachineStatus(c.apiState, params.StatusError,
fmt.Sprintf("upgrade to %v failed (%s): %v",
version.Current, retryText, upgradeErr))
}
}
if upgradeErr != nil {
return upgradeErr
}
agentConfig.SetUpgradedToVersion(version.Current.Number)
return nil
})
if err != nil {
logger.Errorf("upgrade to %v failed: %v", version.Current, err)
return err
}
logger.Infof("upgrade to %v completed successfully.", version.Current)
a.setMachineStatus(c.apiState, params.StatusStarted, "")
return nil
}