// runUpgradeSteps runs the required upgrade steps for the machine
// agent, retrying on failure. The agent's UpgradedToVersion is set
// once the upgrade is complete.
//
// This function conforms to the agent.ConfigMutator type and is
// designed to be called via a machine agent's ChangeConfig method.
func (c *upgradeWorkerContext) runUpgradeSteps(agentConfig agent.ConfigSetter) error {
var upgradeErr error
a := c.agent
a.setMachineStatus(c.apiState, params.StatusStarted, fmt.Sprintf("upgrading to %v", c.toVersion))
context := upgrades.NewContext(agentConfig, c.apiState, c.st)
logger.Infof("starting upgrade from %v to %v for %q", c.fromVersion, c.toVersion, c.tag)
targets := jobsToTargets(c.jobs, c.isMaster)
attempts := getUpgradeRetryStrategy()
for attempt := attempts.Start(); attempt.Next(); {
upgradeErr = upgradesPerformUpgrade(c.fromVersion, targets, context)
if upgradeErr == nil {
break
}
if cmdutil.ConnectionIsDead(logger, c.apiState) {
// API connection has gone away - abort!
return &apiLostDuringUpgrade{upgradeErr}
}
if attempt.HasNext() {
c.reportUpgradeFailure(upgradeErr, true)
}
}
if upgradeErr != nil {
return upgradeErr
}
agentConfig.SetUpgradedToVersion(c.toVersion)
return nil
}
// runUpgradeSteps runs the required upgrade steps for the machine
// agent, retrying on failure. The agent's UpgradedToVersion is set
// once the upgrade is complete.
//
// This function conforms to the agent.ConfigMutator type and is
// designed to be called via a machine agent's ChangeConfig method.
func (w *upgradesteps) runUpgradeSteps(agentConfig agent.ConfigSetter) error {
var upgradeErr error
w.machine.SetStatus(status.Started, fmt.Sprintf("upgrading to %v", w.toVersion), nil)
context := upgrades.NewContext(agentConfig, w.apiConn, w.st)
logger.Infof("starting upgrade from %v to %v for %q", w.fromVersion, w.toVersion, w.tag)
targets := jobsToTargets(w.jobs, w.isMaster)
attempts := getUpgradeRetryStrategy()
for attempt := attempts.Start(); attempt.Next(); {
upgradeErr = PerformUpgrade(w.fromVersion, targets, context)
if upgradeErr == nil {
break
}
if cmdutil.ConnectionIsDead(logger, w.apiConn) {
// API connection has gone away - abort!
return &apiLostDuringUpgrade{upgradeErr}
}
if attempt.HasNext() {
w.reportUpgradeFailure(upgradeErr, true)
}
}
if upgradeErr != nil {
return upgradeErr
}
agentConfig.SetUpgradedToVersion(w.toVersion)
return nil
}
// initBootstrapMachine initializes the initial bootstrap machine in state.
func initBootstrapMachine(c agent.ConfigSetter, st *state.State, args InitializeStateParams) (*state.Machine, error) {
logger.Infof("initialising bootstrap machine with config: %+v", args)
jobs := make([]state.MachineJob, len(args.BootstrapMachineJobs))
for i, job := range args.BootstrapMachineJobs {
machineJob, err := machineJobFromParams(job)
if err != nil {
return nil, errors.Errorf("invalid bootstrap machine job %q: %v", job, err)
}
jobs[i] = machineJob
}
var hardware instance.HardwareCharacteristics
if args.BootstrapMachineHardwareCharacteristics != nil {
hardware = *args.BootstrapMachineHardwareCharacteristics
}
m, err := st.AddOneMachine(state.MachineTemplate{
Addresses: args.BootstrapMachineAddresses,
Series: series.HostSeries(),
Nonce: agent.BootstrapNonce,
Constraints: args.BootstrapMachineConstraints,
InstanceId: args.BootstrapMachineInstanceId,
HardwareCharacteristics: hardware,
Jobs: jobs,
})
if err != nil {
return nil, errors.Errorf("cannot create bootstrap machine in state: %v", err)
}
if m.Id() != agent.BootstrapMachineId {
return nil, errors.Errorf("bootstrap machine expected id 0, got %q", m.Id())
}
// Read the machine agent's password and change it to
// a new password (other agents will change their password
// via the API connection).
logger.Debugf("create new random password for machine %v", m.Id())
newPassword, err := utils.RandomPassword()
if err != nil {
return nil, err
}
if err := m.SetPassword(newPassword); err != nil {
return nil, err
}
if err := m.SetMongoPassword(newPassword); err != nil {
return nil, err
}
c.SetPassword(newPassword)
return m, nil
}
// upgradeCertificateDNSNames ensure that the controller certificate
// recorded in the agent config and also mongo server.pem contains the
// DNSNames entries required by Juju.
func upgradeCertificateDNSNames(config agent.ConfigSetter) error {
si, ok := config.StateServingInfo()
if !ok || si.CAPrivateKey == "" {
// No certificate information exists yet, nothing to do.
return nil
}
// Validate the current certificate and private key pair, and then
// extract the current DNS names from the certificate. If the
// certificate validation fails, or it does not contain the DNS
// names we require, we will generate a new one.
var dnsNames set.Strings
serverCert, _, err := cert.ParseCertAndKey(si.Cert, si.PrivateKey)
if err != nil {
// The certificate is invalid, so create a new one.
logger.Infof("parsing certificate/key failed, will generate a new one: %v", err)
dnsNames = set.NewStrings()
} else {
dnsNames = set.NewStrings(serverCert.DNSNames...)
}
update := false
requiredDNSNames := []string{"local", "juju-apiserver", "juju-mongodb"}
for _, dnsName := range requiredDNSNames {
if dnsNames.Contains(dnsName) {
continue
}
dnsNames.Add(dnsName)
update = true
}
if !update {
return nil
}
// Write a new certificate to the mongo pem and agent config files.
si.Cert, si.PrivateKey, err = cert.NewDefaultServer(config.CACert(), si.CAPrivateKey, dnsNames.Values())
if err != nil {
return err
}
if err := mongo.UpdateSSLKey(config.DataDir(), si.Cert, si.PrivateKey); err != nil {
return err
}
config.SetStateServingInfo(si)
return nil
}
// InitializeState should be called on the bootstrap machine's agent
// configuration. It uses that information to create the controller, dial the
// controller, and initialize it. It also generates a new password for the
// bootstrap machine and calls Write to save the the configuration.
//
// The cfg values will be stored in the state's ModelConfig; the
// machineCfg values will be used to configure the bootstrap Machine,
// and its constraints will be also be used for the model-level
// constraints. The connection to the controller will respect the
// given timeout parameter.
//
// InitializeState returns the newly initialized state and bootstrap
// machine. If it fails, the state may well be irredeemably compromised.
func InitializeState(
adminUser names.UserTag,
c agent.ConfigSetter,
args InitializeStateParams,
dialOpts mongo.DialOpts,
newPolicy state.NewPolicyFunc,
) (_ *state.State, _ *state.Machine, resultErr error) {
if c.Tag() != names.NewMachineTag(agent.BootstrapMachineId) {
return nil, nil, errors.Errorf("InitializeState not called with bootstrap machine's configuration")
}
servingInfo, ok := c.StateServingInfo()
if !ok {
return nil, nil, errors.Errorf("state serving information not available")
}
// N.B. no users are set up when we're initializing the state,
// so don't use any tag or password when opening it.
info, ok := c.MongoInfo()
if !ok {
return nil, nil, errors.Errorf("stateinfo not available")
}
info.Tag = nil
info.Password = c.OldPassword()
if err := initMongoAdminUser(info.Info, dialOpts, info.Password); err != nil {
return nil, nil, errors.Annotate(err, "failed to initialize mongo admin user")
}
cloudCredentials := make(map[names.CloudCredentialTag]cloud.Credential)
var cloudCredentialTag names.CloudCredentialTag
if args.ControllerCloudCredential != nil && args.ControllerCloudCredentialName != "" {
cloudCredentialTag = names.NewCloudCredentialTag(fmt.Sprintf(
"%s/%s/%s",
args.ControllerCloudName,
adminUser.Canonical(),
args.ControllerCloudCredentialName,
))
cloudCredentials[cloudCredentialTag] = *args.ControllerCloudCredential
}
logger.Debugf("initializing address %v", info.Addrs)
st, err := state.Initialize(state.InitializeParams{
Clock: clock.WallClock,
ControllerModelArgs: state.ModelArgs{
Owner: adminUser,
Config: args.ControllerModelConfig,
Constraints: args.ModelConstraints,
CloudName: args.ControllerCloudName,
CloudRegion: args.ControllerCloudRegion,
CloudCredential: cloudCredentialTag,
StorageProviderRegistry: args.StorageProviderRegistry,
},
CloudName: args.ControllerCloudName,
Cloud: args.ControllerCloud,
CloudCredentials: cloudCredentials,
ControllerConfig: args.ControllerConfig,
ControllerInheritedConfig: args.ControllerInheritedConfig,
RegionInheritedConfig: args.RegionInheritedConfig,
MongoInfo: info,
MongoDialOpts: dialOpts,
NewPolicy: newPolicy,
})
if err != nil {
return nil, nil, errors.Errorf("failed to initialize state: %v", err)
}
logger.Debugf("connected to initial state")
defer func() {
if resultErr != nil {
st.Close()
}
}()
servingInfo.SharedSecret = args.SharedSecret
c.SetStateServingInfo(servingInfo)
// Filter out any LXC or LXD bridge addresses from the machine addresses.
args.BootstrapMachineAddresses = network.FilterBridgeAddresses(args.BootstrapMachineAddresses)
if err = initAPIHostPorts(c, st, args.BootstrapMachineAddresses, servingInfo.APIPort); err != nil {
return nil, nil, err
}
ssi := paramsStateServingInfoToStateStateServingInfo(servingInfo)
if err := st.SetStateServingInfo(ssi); err != nil {
return nil, nil, errors.Errorf("cannot set state serving info: %v", err)
}
m, err := initBootstrapMachine(c, st, args)
if err != nil {
return nil, nil, errors.Annotate(err, "cannot initialize bootstrap machine")
}
// Create the initial hosted model, with the model config passed to
// bootstrap, which contains the UUID, name for the hosted model,
// and any user supplied config. We also copy the authorized-keys
// from the controller model.
attrs := make(map[string]interface{})
for k, v := range args.HostedModelConfig {
attrs[k] = v
}
attrs[config.AuthorizedKeysKey] = args.ControllerModelConfig.AuthorizedKeys()
// Construct a CloudSpec to pass on to NewModelConfig below.
cloudSpec, err := environs.MakeCloudSpec(
args.ControllerCloud,
args.ControllerCloudName,
args.ControllerCloudRegion,
args.ControllerCloudCredential,
)
if err != nil {
return nil, nil, errors.Trace(err)
}
controllerUUID := args.ControllerConfig.ControllerUUID()
creator := modelmanager.ModelConfigCreator{Provider: args.Provider}
hostedModelConfig, err := creator.NewModelConfig(
cloudSpec, args.ControllerModelConfig, attrs,
)
if err != nil {
return nil, nil, errors.Annotate(err, "creating hosted model config")
}
provider, err := args.Provider(cloudSpec.Type)
if err != nil {
return nil, nil, errors.Annotate(err, "getting environ provider")
}
hostedModelEnv, err := provider.Open(environs.OpenParams{
Cloud: cloudSpec,
Config: hostedModelConfig,
})
if err != nil {
return nil, nil, errors.Annotate(err, "opening hosted model environment")
}
if err := hostedModelEnv.Create(environs.CreateParams{
ControllerUUID: controllerUUID,
}); err != nil {
return nil, nil, errors.Annotate(err, "creating hosted model environment")
}
_, hostedModelState, err := st.NewModel(state.ModelArgs{
Owner: adminUser,
Config: hostedModelConfig,
Constraints: args.ModelConstraints,
CloudName: args.ControllerCloudName,
CloudRegion: args.ControllerCloudRegion,
CloudCredential: cloudCredentialTag,
StorageProviderRegistry: args.StorageProviderRegistry,
})
if err != nil {
return nil, nil, errors.Annotate(err, "creating hosted model")
}
hostedModelState.Close()
return st, m, nil
}
// InitializeState should be called on the bootstrap machine's agent
// configuration. It uses that information to create the controller, dial the
// controller, and initialize it. It also generates a new password for the
// bootstrap machine and calls Write to save the the configuration.
//
// The cfg values will be stored in the state's ModelConfig; the
// machineCfg values will be used to configure the bootstrap Machine,
// and its constraints will be also be used for the model-level
// constraints. The connection to the controller will respect the
// given timeout parameter.
//
// InitializeState returns the newly initialized state and bootstrap
// machine. If it fails, the state may well be irredeemably compromised.
func InitializeState(
adminUser names.UserTag,
c agent.ConfigSetter,
cfg *config.Config,
hostedModelConfigAttrs map[string]interface{},
machineCfg BootstrapMachineConfig,
dialOpts mongo.DialOpts,
policy state.Policy,
) (_ *state.State, _ *state.Machine, resultErr error) {
if c.Tag() != names.NewMachineTag(agent.BootstrapMachineId) {
return nil, nil, errors.Errorf("InitializeState not called with bootstrap machine's configuration")
}
servingInfo, ok := c.StateServingInfo()
if !ok {
return nil, nil, errors.Errorf("state serving information not available")
}
// N.B. no users are set up when we're initializing the state,
// so don't use any tag or password when opening it.
info, ok := c.MongoInfo()
if !ok {
return nil, nil, errors.Errorf("stateinfo not available")
}
info.Tag = nil
info.Password = c.OldPassword()
if err := initMongoAdminUser(info.Info, dialOpts, info.Password); err != nil {
return nil, nil, errors.Annotate(err, "failed to initialize mongo admin user")
}
logger.Debugf("initializing address %v", info.Addrs)
st, err := state.Initialize(adminUser, info, cfg, dialOpts, policy)
if err != nil {
return nil, nil, errors.Errorf("failed to initialize state: %v", err)
}
logger.Debugf("connected to initial state")
defer func() {
if resultErr != nil {
st.Close()
}
}()
servingInfo.SharedSecret = machineCfg.SharedSecret
c.SetStateServingInfo(servingInfo)
// Filter out any LXC bridge addresses from the machine addresses.
machineCfg.Addresses = network.FilterLXCAddresses(machineCfg.Addresses)
if err = initAPIHostPorts(c, st, machineCfg.Addresses, servingInfo.APIPort); err != nil {
return nil, nil, err
}
ssi := paramsStateServingInfoToStateStateServingInfo(servingInfo)
if err := st.SetStateServingInfo(ssi); err != nil {
return nil, nil, errors.Errorf("cannot set state serving info: %v", err)
}
m, err := initConstraintsAndBootstrapMachine(c, st, machineCfg)
if err != nil {
return nil, nil, err
}
// Create the initial hosted model, with the model config passed to
// bootstrap, which contains the UUID, name for the hosted model,
// and any user supplied config.
attrs := make(map[string]interface{})
for k, v := range hostedModelConfigAttrs {
attrs[k] = v
}
hostedModelConfig, err := modelmanager.ModelConfigCreator{}.NewModelConfig(modelmanager.IsAdmin, cfg, attrs)
if err != nil {
return nil, nil, errors.Annotate(err, "creating hosted model config")
}
_, hostedModelState, err := st.NewModel(hostedModelConfig, adminUser)
if err != nil {
return nil, nil, errors.Annotate(err, "creating hosted model")
}
if err := hostedModelState.SetModelConstraints(machineCfg.ModelConstraints); err != nil {
return nil, nil, errors.Annotate(err, "cannot set initial hosted model constraints")
}
hostedModelState.Close()
return st, m, nil
}
