func destroyEnvironment(st *state.State, environTag names.EnvironTag, destroyHostedEnvirons bool) error {
var err error
if environTag != st.EnvironTag() {
if st, err = st.ForEnviron(environTag); err != nil {
return errors.Trace(err)
}
defer st.Close()
}
if destroyHostedEnvirons {
envs, err := st.AllEnvironments()
if err != nil {
return errors.Trace(err)
}
for _, env := range envs {
envSt, err := st.ForEnviron(env.EnvironTag())
defer envSt.Close()
if err != nil {
return errors.Trace(err)
}
check := NewBlockChecker(envSt)
if err = check.DestroyAllowed(); err != nil {
return errors.Trace(err)
}
}
} else {
check := NewBlockChecker(st)
if err = check.DestroyAllowed(); err != nil {
return errors.Trace(err)
}
}
env, err := st.Environment()
if err != nil {
return errors.Trace(err)
}
if destroyHostedEnvirons {
if err := env.DestroyIncludingHosted(); err != nil {
return err
}
} else {
if err = env.Destroy(); err != nil {
return errors.Trace(err)
}
}
err = sendMetrics(st)
if err != nil {
logger.Warningf("failed to send leftover metrics: %v", err)
}
// Return to the caller. If it's the CLI, it will finish up by calling the
// provider's Destroy method, which will destroy the state servers, any
// straggler instances, and other provider-specific resources. Once all
// resources are torn down, the Undertaker worker handles the removal of
// the environment.
return nil
}
func (c *upgradeWorkerContext) run(stop <-chan struct{}) error {
select {
case <-c.UpgradeComplete:
// Our work is already done (we're probably being restarted
// because the API connection has gone down), so do nothing.
return nil
default:
}
agentConfig := c.agent.CurrentConfig()
// If the machine agent is a state server, flag that state
// needs to be opened before running upgrade steps
needsState := false
for _, job := range c.jobs {
if job == params.JobManageEnviron {
needsState = true
}
}
// We need a *state.State for upgrades. We open it independently
// of StateWorker, because we have no guarantees about when
// and how often StateWorker might run.
var st *state.State
if needsState {
if err := c.agent.ensureMongoServer(agentConfig); err != nil {
return err
}
var err error
info, ok := agentConfig.MongoInfo()
if !ok {
return fmt.Errorf("no state info available")
}
st, err = state.Open(info, mongo.DialOpts{}, environs.NewStatePolicy())
if err != nil {
return err
}
defer st.Close()
}
if err := c.runUpgrades(st, agentConfig); err != nil {
// Only return an error from the worker if the connection to
// state went away (possible mongo master change). Returning
// an error when the connection is lost will cause the agent
// to restart.
//
// For other errors, the error is not returned because we want
// the machine agent to stay running in an error state waiting
// for user intervention.
if isAPILostDuringUpgrade(err) {
return err
}
} else {
// Upgrade succeeded - signal that the upgrade is complete.
close(c.UpgradeComplete)
}
return nil
}
// DestroyEnvironment destroys all services and non-manager machine
// instances in the specified environment. This function assumes that all
// necessary authentication checks have been done.
func DestroyEnvironment(st *state.State, environTag names.EnvironTag) error {
var err error
if environTag != st.EnvironTag() {
if st, err = st.ForEnviron(environTag); err != nil {
return errors.Trace(err)
}
defer st.Close()
}
check := NewBlockChecker(st)
if err = check.DestroyAllowed(); err != nil {
return errors.Trace(err)
}
env, err := st.Environment()
if err != nil {
return errors.Trace(err)
}
if err = env.Destroy(); err != nil {
return errors.Trace(err)
}
machines, err := st.AllMachines()
if err != nil {
return errors.Trace(err)
}
err = sendMetrics(st)
if err != nil {
logger.Warningf("failed to send leftover metrics: %v", err)
}
// We must destroy instances server-side to support JES (Juju Environment
// Server), as there's no CLI to fall back on. In that case, we only ever
// destroy non-state machines; we leave destroying state servers in non-
// hosted environments to the CLI, as otherwise the API server may get cut
// off.
if err := destroyNonManagerMachines(st, machines); err != nil {
return errors.Trace(err)
}
// If this is not the state server environment, remove all documents from
// state associated with the environment.
if env.EnvironTag() != env.ServerTag() {
return errors.Trace(st.RemoveAllEnvironDocs())
}
// Return to the caller. If it's the CLI, it will finish up
// by calling the provider's Destroy method, which will
// destroy the state servers, any straggler instances, and
// other provider-specific resources.
return nil
}
// upgradeWorker runs the required upgrade operations to upgrade to the current Juju version.
func (a *MachineAgent) upgradeWorker(
apiState *api.State,
jobs []params.MachineJob,
agentConfig agent.Config,
) worker.Worker {
return worker.NewSimpleWorker(func(stop <-chan struct{}) error {
select {
case <-a.upgradeComplete:
// Our work is already done (we're probably being restarted
// because the API connection has gone down), so do nothing.
<-stop
return nil
default:
}
// If the machine agent is a state server, flag that state
// needs to be opened before running upgrade steps
needsState := false
for _, job := range jobs {
if job == params.JobManageEnviron {
needsState = true
}
}
// We need a *state.State for upgrades. We open it independently
// of StateWorker, because we have no guarantees about when
// and how often StateWorker might run.
var st *state.State
if needsState {
if err := a.ensureMongoServer(agentConfig); err != nil {
return err
}
var err error
info, ok := agentConfig.StateInfo()
if !ok {
return fmt.Errorf("no state info available")
}
st, err = state.Open(info, mongo.DialOpts{}, environs.NewStatePolicy())
if err != nil {
return err
}
defer st.Close()
}
err := a.runUpgrades(st, apiState, jobs, agentConfig)
if err != nil {
return err
}
logger.Infof("upgrade to %v completed.", version.Current)
close(a.upgradeComplete)
<-stop
return nil
})
}
func assertStateBecomesClosed(c *gc.C, st *state.State) {
// This is gross but I can't see any other way to check for
// closedness outside the state package.
checkModel := func() {
attempt := utils.AttemptStrategy{
Total: coretesting.LongWait,
Delay: coretesting.ShortWait,
}
for a := attempt.Start(); a.Next(); {
// This will panic once the state is closed.
_, _ = st.Model()
}
// If we got here then st is still open.
st.Close()
}
c.Assert(checkModel, gc.PanicMatches, "Session already closed")
}
// Run initializes state for an environment.
func (c *BootstrapCommand) Run(_ *cmd.Context) error {
envCfg, err := config.New(config.NoDefaults, c.ControllerModelConfig)
if err != nil {
return err
}
err = c.ReadConfig("machine-0")
if err != nil {
return err
}
agentConfig := c.CurrentConfig()
network.SetPreferIPv6(agentConfig.PreferIPv6())
// agent.Jobs is an optional field in the agent config, and was
// introduced after 1.17.2. We default to allowing units on
// machine-0 if missing.
jobs := agentConfig.Jobs()
if len(jobs) == 0 {
jobs = []multiwatcher.MachineJob{
multiwatcher.JobManageModel,
multiwatcher.JobHostUnits,
multiwatcher.JobManageNetworking,
}
}
// Get the bootstrap machine's addresses from the provider.
env, err := environs.New(envCfg)
if err != nil {
return err
}
newConfigAttrs := make(map[string]interface{})
// Check to see if a newer agent version has been requested
// by the bootstrap client.
desiredVersion, ok := envCfg.AgentVersion()
if ok && desiredVersion != jujuversion.Current {
// If we have been asked for a newer version, ensure the newer
// tools can actually be found, or else bootstrap won't complete.
stream := envtools.PreferredStream(&desiredVersion, envCfg.Development(), envCfg.AgentStream())
logger.Infof("newer tools requested, looking for %v in stream %v", desiredVersion, stream)
filter := tools.Filter{
Number: desiredVersion,
Arch: arch.HostArch(),
Series: series.HostSeries(),
}
_, toolsErr := envtools.FindTools(env, -1, -1, stream, filter)
if toolsErr == nil {
logger.Infof("tools are available, upgrade will occur after bootstrap")
}
if errors.IsNotFound(toolsErr) {
// Newer tools not available, so revert to using the tools
// matching the current agent version.
logger.Warningf("newer tools for %q not available, sticking with version %q", desiredVersion, jujuversion.Current)
newConfigAttrs["agent-version"] = jujuversion.Current.String()
} else if toolsErr != nil {
logger.Errorf("cannot find newer tools: %v", toolsErr)
return toolsErr
}
}
instanceId := instance.Id(c.InstanceId)
instances, err := env.Instances([]instance.Id{instanceId})
if err != nil {
return err
}
addrs, err := instances[0].Addresses()
if err != nil {
return err
}
// When machine addresses are reported from state, they have
// duplicates removed. We should do the same here so that
// there is not unnecessary churn in the mongo replicaset.
// TODO (cherylj) Add explicit unit tests for this - tracked
// by bug #1544158.
addrs = network.MergedAddresses([]network.Address{}, addrs)
// Generate a private SSH key for the controllers, and add
// the public key to the environment config. We'll add the
// private key to StateServingInfo below.
privateKey, publicKey, err := sshGenerateKey(config.JujuSystemKey)
if err != nil {
return errors.Annotate(err, "failed to generate system key")
}
authorizedKeys := config.ConcatAuthKeys(envCfg.AuthorizedKeys(), publicKey)
newConfigAttrs[config.AuthKeysConfig] = authorizedKeys
// Generate a shared secret for the Mongo replica set, and write it out.
sharedSecret, err := mongo.GenerateSharedSecret()
if err != nil {
return err
}
info, ok := agentConfig.StateServingInfo()
if !ok {
return fmt.Errorf("bootstrap machine config has no state serving info")
}
info.SharedSecret = sharedSecret
info.SystemIdentity = privateKey
err = c.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
agentConfig.SetStateServingInfo(info)
return nil
})
if err != nil {
return fmt.Errorf("cannot write agent config: %v", err)
}
err = c.ChangeConfig(func(config agent.ConfigSetter) error {
// We'll try for mongo 3.2 first and fallback to
// mongo 2.4 if the newer binaries are not available.
if mongo.BinariesAvailable(mongo.Mongo32wt) {
config.SetMongoVersion(mongo.Mongo32wt)
} else {
config.SetMongoVersion(mongo.Mongo24)
}
return nil
})
if err != nil {
return errors.Annotate(err, "cannot set mongo version")
}
agentConfig = c.CurrentConfig()
// Create system-identity file
if err := agent.WriteSystemIdentityFile(agentConfig); err != nil {
return err
}
if err := c.startMongo(addrs, agentConfig); err != nil {
return err
}
logger.Infof("started mongo")
// Initialise state, and store any agent config (e.g. password) changes.
envCfg, err = env.Config().Apply(newConfigAttrs)
if err != nil {
return errors.Annotate(err, "failed to update model config")
}
var st *state.State
var m *state.Machine
err = c.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
var stateErr error
dialOpts := mongo.DefaultDialOpts()
// Set a longer socket timeout than usual, as the machine
// will be starting up and disk I/O slower than usual. This
// has been known to cause timeouts in queries.
timeouts := envCfg.BootstrapSSHOpts()
dialOpts.SocketTimeout = timeouts.Timeout
if dialOpts.SocketTimeout < minSocketTimeout {
dialOpts.SocketTimeout = minSocketTimeout
}
// We shouldn't attempt to dial peers until we have some.
dialOpts.Direct = true
adminTag := names.NewLocalUserTag(c.AdminUsername)
st, m, stateErr = agentInitializeState(
adminTag,
agentConfig,
envCfg, c.HostedModelConfig,
agentbootstrap.BootstrapMachineConfig{
Addresses: addrs,
BootstrapConstraints: c.BootstrapConstraints,
ModelConstraints: c.ModelConstraints,
Jobs: jobs,
InstanceId: instanceId,
Characteristics: c.Hardware,
SharedSecret: sharedSecret,
},
dialOpts,
environs.NewStatePolicy(),
)
return stateErr
})
if err != nil {
return err
}
defer st.Close()
// Populate the tools catalogue.
if err := c.populateTools(st, env); err != nil {
return err
}
// Populate the GUI archive catalogue.
if err := c.populateGUIArchive(st, env); err != nil {
// Do not stop the bootstrapping process for Juju GUI archive errors.
logger.Warningf("cannot set up Juju GUI: %s", err)
} else {
logger.Debugf("Juju GUI successfully set up")
}
// Add custom image metadata to environment storage.
if c.ImageMetadataDir != "" {
if err := c.saveCustomImageMetadata(st, env); err != nil {
return err
}
stor := newStateStorage(st.ModelUUID(), st.MongoSession())
if err := c.storeCustomImageMetadata(stor); err != nil {
return err
}
}
// Populate the storage pools.
if err = c.populateDefaultStoragePools(st); err != nil {
return err
}
// bootstrap machine always gets the vote
return m.SetHasVote(true)
}
// Run initializes state for an environment.
func (c *BootstrapCommand) Run(_ *cmd.Context) error {
envCfg, err := config.New(config.NoDefaults, c.EnvConfig)
if err != nil {
return err
}
err = c.ReadConfig("machine-0")
if err != nil {
return err
}
agentConfig := c.CurrentConfig()
network.InitializeFromConfig(agentConfig)
// agent.Jobs is an optional field in the agent config, and was
// introduced after 1.17.2. We default to allowing units on
// machine-0 if missing.
jobs := agentConfig.Jobs()
if len(jobs) == 0 {
jobs = []multiwatcher.MachineJob{
multiwatcher.JobManageEnviron,
multiwatcher.JobHostUnits,
multiwatcher.JobManageNetworking,
}
}
// Get the bootstrap machine's addresses from the provider.
env, err := environs.New(envCfg)
if err != nil {
return err
}
instanceId := instance.Id(c.InstanceId)
instances, err := env.Instances([]instance.Id{instanceId})
if err != nil {
return err
}
addrs, err := instances[0].Addresses()
if err != nil {
return err
}
// Generate a private SSH key for the state servers, and add
// the public key to the environment config. We'll add the
// private key to StateServingInfo below.
privateKey, publicKey, err := sshGenerateKey(config.JujuSystemKey)
if err != nil {
return errors.Annotate(err, "failed to generate system key")
}
authorizedKeys := config.ConcatAuthKeys(envCfg.AuthorizedKeys(), publicKey)
envCfg, err = env.Config().Apply(map[string]interface{}{
config.AuthKeysConfig: authorizedKeys,
})
if err != nil {
return errors.Annotate(err, "failed to add public key to environment config")
}
// Generate a shared secret for the Mongo replica set, and write it out.
sharedSecret, err := mongo.GenerateSharedSecret()
if err != nil {
return err
}
info, ok := agentConfig.StateServingInfo()
if !ok {
return fmt.Errorf("bootstrap machine config has no state serving info")
}
info.SharedSecret = sharedSecret
info.SystemIdentity = privateKey
err = c.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
agentConfig.SetStateServingInfo(info)
return nil
})
if err != nil {
return fmt.Errorf("cannot write agent config: %v", err)
}
agentConfig = c.CurrentConfig()
// Create system-identity file
if err := agent.WriteSystemIdentityFile(agentConfig); err != nil {
return err
}
if err := c.startMongo(addrs, agentConfig); err != nil {
return err
}
logger.Infof("started mongo")
// Initialise state, and store any agent config (e.g. password) changes.
var st *state.State
var m *state.Machine
err = c.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
var stateErr error
dialOpts := mongo.DefaultDialOpts()
// Set a longer socket timeout than usual, as the machine
// will be starting up and disk I/O slower than usual. This
// has been known to cause timeouts in queries.
timeouts := envCfg.BootstrapSSHOpts()
dialOpts.SocketTimeout = timeouts.Timeout
if dialOpts.SocketTimeout < minSocketTimeout {
dialOpts.SocketTimeout = minSocketTimeout
}
// We shouldn't attempt to dial peers until we have some.
dialOpts.Direct = true
adminTag := names.NewLocalUserTag(c.AdminUsername)
st, m, stateErr = agentInitializeState(
adminTag,
agentConfig,
envCfg,
agent.BootstrapMachineConfig{
Addresses: addrs,
Constraints: c.Constraints,
Jobs: jobs,
InstanceId: instanceId,
Characteristics: c.Hardware,
SharedSecret: sharedSecret,
},
dialOpts,
environs.NewStatePolicy(),
)
return stateErr
})
if err != nil {
return err
}
defer st.Close()
// Populate the tools catalogue.
if err := c.populateTools(st, env); err != nil {
return err
}
// Add custom image metadata to environment storage.
if c.ImageMetadataDir != "" {
if err := c.saveCustomImageMetadata(st); err != nil {
return err
}
// TODO (anastasiamac 2015-09-24) Remove this once search path is updated..
stor := newStateStorage(st.EnvironUUID(), st.MongoSession())
if err := c.storeCustomImageMetadata(stor); err != nil {
return err
}
}
// Populate the storage pools.
if err := c.populateDefaultStoragePools(st); err != nil {
return err
}
// bootstrap machine always gets the vote
return m.SetHasVote(true)
}
// Run initializes state for an environment.
func (c *BootstrapCommand) Run(_ *cmd.Context) error {
envCfg, err := config.New(config.NoDefaults, c.EnvConfig)
if err != nil {
return err
}
err = c.ReadConfig("machine-0")
if err != nil {
return err
}
agentConfig := c.CurrentConfig()
// agent.Jobs is an optional field in the agent config, and was
// introduced after 1.17.2. We default to allowing units on
// machine-0 if missing.
jobs := agentConfig.Jobs()
if len(jobs) == 0 {
jobs = []params.MachineJob{
params.JobManageEnviron,
params.JobHostUnits,
}
}
// Get the bootstrap machine's addresses from the provider.
env, err := environs.New(envCfg)
if err != nil {
return err
}
instanceId := instance.Id(c.InstanceId)
instances, err := env.Instances([]instance.Id{instanceId})
if err != nil {
return err
}
addrs, err := instances[0].Addresses()
if err != nil {
return err
}
// Create system-identity file
if err := agent.WriteSystemIdentityFile(agentConfig); err != nil {
return err
}
// Generate a shared secret for the Mongo replica set, and write it out.
sharedSecret, err := mongo.GenerateSharedSecret()
if err != nil {
return err
}
info, ok := agentConfig.StateServingInfo()
if !ok {
return fmt.Errorf("bootstrap machine config has no state serving info")
}
info.SharedSecret = sharedSecret
err = c.ChangeConfig(func(agentConfig agent.ConfigSetter) {
agentConfig.SetStateServingInfo(info)
})
if err != nil {
return fmt.Errorf("cannot write agent config: %v", err)
}
agentConfig = c.CurrentConfig()
if err := c.startMongo(addrs, agentConfig); err != nil {
return err
}
logger.Infof("started mongo")
// Initialise state, and store any agent config (e.g. password) changes.
var st *state.State
var m *state.Machine
err = nil
writeErr := c.ChangeConfig(func(agentConfig agent.ConfigSetter) {
st, m, err = agent.InitializeState(
agentConfig,
envCfg,
agent.BootstrapMachineConfig{
Addresses: addrs,
Constraints: c.Constraints,
Jobs: jobs,
InstanceId: instanceId,
Characteristics: c.Hardware,
SharedSecret: sharedSecret,
},
mongo.DefaultDialOpts(),
environs.NewStatePolicy(),
)
})
if writeErr != nil {
return fmt.Errorf("cannot write initial configuration: %v", err)
}
if err != nil {
return err
}
defer st.Close()
// bootstrap machine always gets the vote
return m.SetHasVote(true)
}
// upgradeWorker runs the required upgrade operations to upgrade to the current Juju version.
func (a *MachineAgent) upgradeWorker(
apiState *api.State,
jobs []params.MachineJob,
agentConfig agent.Config,
) worker.Worker {
return worker.NewSimpleWorker(func(stop <-chan struct{}) error {
select {
case <-a.upgradeComplete:
// Our work is already done (we're probably being restarted
// because the API connection has gone down), so do nothing.
<-stop
return nil
default:
}
// If the machine agent is a state server, flag that state
// needs to be opened before running upgrade steps
needsState := false
for _, job := range jobs {
if job == params.JobManageEnviron {
needsState = true
}
}
// We need a *state.State for upgrades. We open it independently
// of StateWorker, because we have no guarantees about when
// and how often StateWorker might run.
var st *state.State
if needsState {
if err := a.ensureMongoServer(agentConfig); err != nil {
return err
}
var err error
info, ok := agentConfig.MongoInfo()
if !ok {
return fmt.Errorf("no state info available")
}
st, err = state.Open(info, mongo.DialOpts{}, environs.NewStatePolicy())
if err != nil {
return err
}
defer st.Close()
}
err := a.runUpgrades(st, apiState, jobs, agentConfig)
if err == nil {
// Only signal that the upgrade is complete if no error
// was returned.
close(a.upgradeComplete)
} else if !isFatal(err) {
// Only non-fatal errors are returned (this will trigger
// the worker to be restarted).
//
// Fatal upgrade errors are not returned because user
// intervention is required at that point. We don't want
// the upgrade worker or the agent to restart. Status
// output and the logs will report that the upgrade has
// failed.
return err
}
<-stop
return nil
})
}