// runUpgrades runs the upgrade operations for each job type and updates the updatedToVersion on success. func (a *MachineAgent) runUpgrades( st *state.State, apiState *api.State, jobs []params.MachineJob, agentConfig agent.Config, ) error { from := version.Current from.Number = agentConfig.UpgradedToVersion() if from == version.Current { logger.Infof("upgrade to %v already completed.", version.Current) return nil } var err error writeErr := a.ChangeConfig(func(agentConfig agent.ConfigSetter) { context := upgrades.NewContext(agentConfig, apiState, st) for _, job := range jobs { target := upgradeTarget(job) if target == "" { continue } logger.Infof("starting upgrade from %v to %v for %v %q", from, version.Current, target, a.Tag()) if err = upgradesPerformUpgrade(from.Number, target, context); err != nil { err = fmt.Errorf("cannot perform upgrade from %v to %v for %v %q: %v", from, version.Current, target, a.Tag(), err) return } } agentConfig.SetUpgradedToVersion(version.Current.Number) }) if writeErr != nil { return fmt.Errorf("cannot write updated agent configuration: %v", writeErr) } return nil }
// runUpgrades runs the upgrade operations for each job type and updates the updatedToVersion on success. func (a *MachineAgent) runUpgrades( st *state.State, apiState *api.State, jobs []params.MachineJob, agentConfig agent.Config, ) error { from := version.Current from.Number = agentConfig.UpgradedToVersion() if from == version.Current { logger.Infof("upgrade to %v already completed.", version.Current) return nil } isMaster, err := a.isMaster(st, agentConfig) if err != nil { return errors.Trace(err) } err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error { var upgradeErr error a.setMachineStatus(apiState, params.StatusStarted, fmt.Sprintf("upgrading to %v", version.Current)) context := upgrades.NewContext(agentConfig, apiState, st) for _, job := range jobs { target := upgradeTarget(job, isMaster) if target == "" { continue } logger.Infof("starting upgrade from %v to %v for %v %q", from, version.Current, target, a.Tag()) attempts := getUpgradeRetryStrategy() for attempt := attempts.Start(); attempt.Next(); { upgradeErr = upgradesPerformUpgrade(from.Number, target, context) if upgradeErr == nil { break } else { retryText := "will retry" if !attempt.HasNext() { retryText = "giving up" } logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v", from, version.Current, target, a.Tag(), retryText, upgradeErr) a.setMachineStatus(apiState, params.StatusError, fmt.Sprintf("upgrade to %v failed (%s): %v", version.Current, retryText, upgradeErr)) } } } if upgradeErr != nil { return upgradeErr } agentConfig.SetUpgradedToVersion(version.Current.Number) return nil }) if err != nil { logger.Errorf("upgrade to %v failed: %v", version.Current, err) return &fatalError{err.Error()} } logger.Infof("upgrade to %v completed successfully.", version.Current) a.setMachineStatus(apiState, params.StatusStarted, "") return nil }
// ensureMongoServer ensures that mongo is installed and running, // and ready for opening a state connection. func (a *MachineAgent) ensureMongoServer(agentConfig agent.Config) error { a.mongoInitMutex.Lock() defer a.mongoInitMutex.Unlock() if a.mongoInitialized { logger.Debugf("mongo is already initialized") return nil } servingInfo, ok := agentConfig.StateServingInfo() if !ok { return fmt.Errorf("state worker was started with no state serving info") } namespace := agentConfig.Value(agent.Namespace) // When upgrading from a pre-HA-capable environment, // we must add machine-0 to the admin database and // initiate its replicaset. // // TODO(axw) remove this when we no longer need // to upgrade from pre-HA-capable environments. var shouldInitiateMongoServer bool var addrs []network.Address if isPreHAVersion(agentConfig.UpgradedToVersion()) { _, err := a.ensureMongoAdminUser(agentConfig) if err != nil { return err } if servingInfo.SharedSecret == "" { servingInfo.SharedSecret, err = mongo.GenerateSharedSecret() if err != nil { return err } if err = a.ChangeConfig(func(config agent.ConfigSetter) { config.SetStateServingInfo(servingInfo) }); err != nil { return err } agentConfig = a.CurrentConfig() } // Note: we set Direct=true in the mongo options because it's // possible that we've previously upgraded the mongo server's // configuration to form a replicaset, but failed to initiate it. st, m, err := openState(agentConfig, mongo.DialOpts{Direct: true}) if err != nil { return err } if err := st.SetStateServingInfo(servingInfo); err != nil { st.Close() return fmt.Errorf("cannot set state serving info: %v", err) } st.Close() addrs = m.Addresses() shouldInitiateMongoServer = true } // ensureMongoServer installs/upgrades the upstart config as necessary. if err := ensureMongoServer( agentConfig.DataDir(), namespace, servingInfo, ); err != nil { return err } if !shouldInitiateMongoServer { return nil } // Initiate the replicaset for upgraded environments. // // TODO(axw) remove this when we no longer need // to upgrade from pre-HA-capable environments. stateInfo, ok := agentConfig.StateInfo() if !ok { return fmt.Errorf("state worker was started with no state serving info") } dialInfo, err := mongo.DialInfo(stateInfo.Info, mongo.DefaultDialOpts()) if err != nil { return err } peerAddr := mongo.SelectPeerAddress(addrs) if peerAddr == "" { return fmt.Errorf("no appropriate peer address found in %q", addrs) } if err := maybeInitiateMongoServer(peergrouper.InitiateMongoParams{ DialInfo: dialInfo, MemberHostPort: net.JoinHostPort(peerAddr, fmt.Sprint(servingInfo.StatePort)), User: stateInfo.Tag, Password: stateInfo.Password, }); err != nil { return err } a.mongoInitialized = true return nil }
// runUpgrades runs the upgrade operations for each job type and // updates the updatedToVersion on success. func (c *upgradeWorkerContext) runUpgrades(st *state.State, agentConfig agent.Config) error { from := version.Current from.Number = agentConfig.UpgradedToVersion() if from == version.Current { logger.Infof("upgrade to %v already completed.", version.Current) return nil } a := c.agent tag := agentConfig.Tag().(names.MachineTag) isMaster, err := isMachineMaster(st, tag) if err != nil { return errors.Trace(err) } err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error { var upgradeErr error a.setMachineStatus(c.apiState, params.StatusStarted, fmt.Sprintf("upgrading to %v", version.Current)) context := upgrades.NewContext(agentConfig, c.apiState, st) for _, job := range c.jobs { target := upgradeTarget(job, isMaster) if target == "" { continue } logger.Infof("starting upgrade from %v to %v for %v %q", from, version.Current, target, tag) attempts := getUpgradeRetryStrategy() for attempt := attempts.Start(); attempt.Next(); { upgradeErr = upgradesPerformUpgrade(from.Number, target, context) if upgradeErr == nil { break } if connectionIsDead(c.apiState) { // API connection has gone away - abort! return &apiLostDuringUpgrade{upgradeErr} } retryText := "will retry" if !attempt.HasNext() { retryText = "giving up" } logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v", from, version.Current, target, tag, retryText, upgradeErr) a.setMachineStatus(c.apiState, params.StatusError, fmt.Sprintf("upgrade to %v failed (%s): %v", version.Current, retryText, upgradeErr)) } } if upgradeErr != nil { return upgradeErr } agentConfig.SetUpgradedToVersion(version.Current.Number) return nil }) if err != nil { logger.Errorf("upgrade to %v failed: %v", version.Current, err) return err } logger.Infof("upgrade to %v completed successfully.", version.Current) a.setMachineStatus(c.apiState, params.StatusStarted, "") return nil }