func (s *SimpleStreamsToolsSuite) TestFindTools(c *gc.C) {
for i, test := range findToolsTests {
c.Logf("\ntest %d: %s", i, test.info)
s.reset(c, nil)
custom := s.uploadCustom(c, test.custom...)
public := s.uploadPublic(c, test.public...)
stream := envtools.PreferredStream(&version.Current, s.env.Config().Development(), s.env.Config().AgentStream())
actual, err := envtools.FindTools(s.env, test.major, test.minor, stream, coretools.Filter{})
if test.err != nil {
if len(actual) > 0 {
c.Logf(actual.String())
}
c.Check(err, jc.Satisfies, errors.IsNotFound)
continue
}
expect := map[version.Binary]string{}
for _, expected := range test.expect {
// If the tools exist in custom, that's preferred.
var ok bool
if expect[expected], ok = custom[expected]; !ok {
expect[expected] = public[expected]
}
}
c.Check(actual.URLs(), gc.DeepEquals, expect)
}
}
func checkToolsAvailability(cfg *config.Config, finder toolsFinder) (version.Number, error) {
currentVersion, ok := cfg.AgentVersion()
if !ok || currentVersion == version.Zero {
return version.Zero, nil
}
env, err := newEnvirons(cfg)
if err != nil {
return version.Zero, errors.Annotatef(err, "cannot make environ")
}
// finder receives major and minor as parameters as it uses them to filter versions and
// only return patches for the passed major.minor (from major.minor.patch).
// We'll try the released stream first, then fall back to the current configured stream
// if no released tools are found.
vers, err := finder(env, currentVersion.Major, currentVersion.Minor, tools.ReleasedStream, coretools.Filter{})
preferredStream := tools.PreferredStream(¤tVersion, cfg.Development(), cfg.AgentStream())
if preferredStream != tools.ReleasedStream && errors.Cause(err) == coretools.ErrNoMatches {
vers, err = finder(env, currentVersion.Major, currentVersion.Minor, preferredStream, coretools.Filter{})
}
if err != nil {
return version.Zero, errors.Annotatef(err, "cannot find available tools")
}
// Newest also returns a list of the items in this list matching with the
// newest version.
newest, _ := vers.Newest()
return newest, nil
}
// StartInstanceWithParams is a test helper function that starts an instance
// with the given parameters, and a plausible but invalid configuration, and
// returns the result of Environ.StartInstance. The provided params's
// InstanceConfig and Tools field values will be ignored.
func StartInstanceWithParams(
env environs.Environ, machineId string,
params environs.StartInstanceParams,
networks []string,
) (
*environs.StartInstanceResult, error,
) {
preferredSeries := config.PreferredSeries(env.Config())
agentVersion, ok := env.Config().AgentVersion()
if !ok {
return nil, errors.New("missing agent version in model config")
}
filter := coretools.Filter{
Number: agentVersion,
Series: preferredSeries,
}
if params.Constraints.Arch != nil {
filter.Arch = *params.Constraints.Arch
}
stream := tools.PreferredStream(&agentVersion, env.Config().Development(), env.Config().AgentStream())
possibleTools, err := tools.FindTools(env, -1, -1, stream, filter)
if err != nil {
return nil, errors.Trace(err)
}
if params.ImageMetadata == nil {
if err := SetImageMetadata(
env,
possibleTools.AllSeries(),
possibleTools.Arches(),
¶ms.ImageMetadata,
); err != nil {
return nil, errors.Trace(err)
}
}
machineNonce := "fake_nonce"
stateInfo := FakeStateInfo(machineId)
apiInfo := FakeAPIInfo(machineId)
instanceConfig, err := instancecfg.NewInstanceConfig(
machineId,
machineNonce,
imagemetadata.ReleasedStream,
preferredSeries,
"",
true,
networks,
stateInfo,
apiInfo,
)
if err != nil {
return nil, errors.Trace(err)
}
eUUID, _ := env.Config().UUID()
instanceConfig.Tags[tags.JujuModel] = eUUID
params.Tools = possibleTools
params.InstanceConfig = instanceConfig
return env.StartInstance(params)
}
// findAvailableTools returns a list of available tools,
// including tools that may be locally built and then
// uploaded. Tools that need to be built will have an
// empty URL.
func findAvailableTools(env environs.Environ, vers *version.Number, arch *string, upload bool) (coretools.List, error) {
if upload {
// We're forcing an upload: ensure we can do so.
if err := validateUploadAllowed(env, arch); err != nil {
return nil, err
}
return locallyBuildableTools(), nil
}
// We're not forcing an upload, so look for tools
// in the environment's simplestreams search paths
// for existing tools.
// If the user hasn't asked for a specified tools version, see if
// one is configured in the environment.
if vers == nil {
if agentVersion, ok := env.Config().AgentVersion(); ok {
vers = &agentVersion
}
}
logger.Infof("looking for bootstrap tools: version=%v", vers)
toolsList, findToolsErr := findBootstrapTools(env, vers, arch)
if findToolsErr != nil && !errors.IsNotFound(findToolsErr) {
return nil, findToolsErr
}
preferredStream := envtools.PreferredStream(vers, env.Config().Development(), env.Config().AgentStream())
if preferredStream == envtools.ReleasedStream || vers != nil {
// We are not running a development build, or agent-version
// was specified; the only tools available are the ones we've
// just found.
return toolsList, findToolsErr
}
// The tools located may not include the ones that the
// provider requires. We are running a development build,
// so augment the list of tools with those that we can build
// locally.
// Collate the set of arch+series that are externally available
// so we can see if we need to build any locally. If we need
// to, only then do we validate that we can upload (which
// involves a potentially expensive SupportedArchitectures call).
archSeries := make(set.Strings)
for _, tools := range toolsList {
archSeries.Add(tools.Version.Arch + tools.Version.Series)
}
var localToolsList coretools.List
for _, tools := range locallyBuildableTools() {
if !archSeries.Contains(tools.Version.Arch + tools.Version.Series) {
localToolsList = append(localToolsList, tools)
}
}
if len(localToolsList) == 0 || validateUploadAllowed(env, arch) != nil {
return toolsList, findToolsErr
}
return append(toolsList, localToolsList...), nil
}
// findBootstrapTools returns a tools.List containing only those tools with
// which it would be reasonable to launch an environment's first machine,
// given the supplied constraints. If a specific agent version is not requested,
// all tools matching the current major.minor version are chosen.
func findBootstrapTools(env environs.Environ, vers *version.Number, arch *string) (list coretools.List, err error) {
// Construct a tools filter.
cliVersion := version.Current.Number
var filter coretools.Filter
if arch != nil {
filter.Arch = *arch
}
if vers != nil {
filter.Number = *vers
}
stream := envtools.PreferredStream(vers, env.Config().Development(), env.Config().AgentStream())
return findTools(env, cliVersion.Major, cliVersion.Minor, stream, filter)
}
func (s *SimpleStreamsToolsSuite) TestPreferredStream(c *gc.C) {
for i, test := range preferredStreamTests {
c.Logf("\ntest %d", i)
s.PatchValue(&version.Current, version.MustParse(test.currentVers))
var vers *version.Number
if test.explicitVers != "" {
v := version.MustParse(test.explicitVers)
vers = &v
}
obtained := envtools.PreferredStream(vers, test.forceDevel, test.streamInConfig)
c.Check(obtained, gc.Equals, test.expected)
}
}
// findMatchingTools searches toolstorage and simplestreams for tools matching the
// given parameters. If an exact match is specified (number, series and arch)
// and is found in toolstorage, then simplestreams will not be searched.
func (f *ToolsFinder) findMatchingTools(args params.FindToolsParams) (coretools.List, error) {
exactMatch := args.Number != version.Zero && args.Series != "" && args.Arch != ""
storageList, err := f.matchingStorageTools(args)
if err == nil && exactMatch {
return storageList, nil
} else if err != nil && err != coretools.ErrNoMatches {
return nil, err
}
// Look for tools in simplestreams too, but don't replace
// any versions found in storage.
cfg, err := f.configGetter.ModelConfig()
if err != nil {
return nil, err
}
env, err := environs.New(cfg)
if err != nil {
return nil, err
}
filter := toolsFilter(args)
stream := envtools.PreferredStream(&args.Number, cfg.Development(), cfg.AgentStream())
simplestreamsList, err := envtoolsFindTools(
env, args.MajorVersion, args.MinorVersion, stream, filter,
)
if len(storageList) == 0 && err != nil {
return nil, err
}
list := storageList
found := make(map[version.Binary]bool)
for _, tools := range storageList {
found[tools.Version] = true
}
for _, tools := range simplestreamsList {
if !found[tools.Version] {
list = append(list, tools)
}
}
sort.Sort(list)
return list, nil
}
// Run initializes state for an environment.
func (c *BootstrapCommand) Run(_ *cmd.Context) error {
envCfg, err := config.New(config.NoDefaults, c.ControllerModelConfig)
if err != nil {
return err
}
err = c.ReadConfig("machine-0")
if err != nil {
return err
}
agentConfig := c.CurrentConfig()
network.SetPreferIPv6(agentConfig.PreferIPv6())
// agent.Jobs is an optional field in the agent config, and was
// introduced after 1.17.2. We default to allowing units on
// machine-0 if missing.
jobs := agentConfig.Jobs()
if len(jobs) == 0 {
jobs = []multiwatcher.MachineJob{
multiwatcher.JobManageModel,
multiwatcher.JobHostUnits,
multiwatcher.JobManageNetworking,
}
}
// Get the bootstrap machine's addresses from the provider.
env, err := environs.New(envCfg)
if err != nil {
return err
}
newConfigAttrs := make(map[string]interface{})
// Check to see if a newer agent version has been requested
// by the bootstrap client.
desiredVersion, ok := envCfg.AgentVersion()
if ok && desiredVersion != jujuversion.Current {
// If we have been asked for a newer version, ensure the newer
// tools can actually be found, or else bootstrap won't complete.
stream := envtools.PreferredStream(&desiredVersion, envCfg.Development(), envCfg.AgentStream())
logger.Infof("newer tools requested, looking for %v in stream %v", desiredVersion, stream)
filter := tools.Filter{
Number: desiredVersion,
Arch: arch.HostArch(),
Series: series.HostSeries(),
}
_, toolsErr := envtools.FindTools(env, -1, -1, stream, filter)
if toolsErr == nil {
logger.Infof("tools are available, upgrade will occur after bootstrap")
}
if errors.IsNotFound(toolsErr) {
// Newer tools not available, so revert to using the tools
// matching the current agent version.
logger.Warningf("newer tools for %q not available, sticking with version %q", desiredVersion, jujuversion.Current)
newConfigAttrs["agent-version"] = jujuversion.Current.String()
} else if toolsErr != nil {
logger.Errorf("cannot find newer tools: %v", toolsErr)
return toolsErr
}
}
instanceId := instance.Id(c.InstanceId)
instances, err := env.Instances([]instance.Id{instanceId})
if err != nil {
return err
}
addrs, err := instances[0].Addresses()
if err != nil {
return err
}
// When machine addresses are reported from state, they have
// duplicates removed. We should do the same here so that
// there is not unnecessary churn in the mongo replicaset.
// TODO (cherylj) Add explicit unit tests for this - tracked
// by bug #1544158.
addrs = network.MergedAddresses([]network.Address{}, addrs)
// Generate a private SSH key for the controllers, and add
// the public key to the environment config. We'll add the
// private key to StateServingInfo below.
privateKey, publicKey, err := sshGenerateKey(config.JujuSystemKey)
if err != nil {
return errors.Annotate(err, "failed to generate system key")
}
authorizedKeys := config.ConcatAuthKeys(envCfg.AuthorizedKeys(), publicKey)
newConfigAttrs[config.AuthKeysConfig] = authorizedKeys
// Generate a shared secret for the Mongo replica set, and write it out.
sharedSecret, err := mongo.GenerateSharedSecret()
if err != nil {
return err
}
info, ok := agentConfig.StateServingInfo()
if !ok {
return fmt.Errorf("bootstrap machine config has no state serving info")
}
info.SharedSecret = sharedSecret
info.SystemIdentity = privateKey
err = c.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
agentConfig.SetStateServingInfo(info)
return nil
})
if err != nil {
return fmt.Errorf("cannot write agent config: %v", err)
}
err = c.ChangeConfig(func(config agent.ConfigSetter) error {
// We'll try for mongo 3.2 first and fallback to
// mongo 2.4 if the newer binaries are not available.
if mongo.BinariesAvailable(mongo.Mongo32wt) {
config.SetMongoVersion(mongo.Mongo32wt)
} else {
config.SetMongoVersion(mongo.Mongo24)
}
return nil
})
if err != nil {
return errors.Annotate(err, "cannot set mongo version")
}
agentConfig = c.CurrentConfig()
// Create system-identity file
if err := agent.WriteSystemIdentityFile(agentConfig); err != nil {
return err
}
if err := c.startMongo(addrs, agentConfig); err != nil {
return err
}
logger.Infof("started mongo")
// Initialise state, and store any agent config (e.g. password) changes.
envCfg, err = env.Config().Apply(newConfigAttrs)
if err != nil {
return errors.Annotate(err, "failed to update model config")
}
var st *state.State
var m *state.Machine
err = c.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
var stateErr error
dialOpts := mongo.DefaultDialOpts()
// Set a longer socket timeout than usual, as the machine
// will be starting up and disk I/O slower than usual. This
// has been known to cause timeouts in queries.
timeouts := envCfg.BootstrapSSHOpts()
dialOpts.SocketTimeout = timeouts.Timeout
if dialOpts.SocketTimeout < minSocketTimeout {
dialOpts.SocketTimeout = minSocketTimeout
}
// We shouldn't attempt to dial peers until we have some.
dialOpts.Direct = true
adminTag := names.NewLocalUserTag(c.AdminUsername)
st, m, stateErr = agentInitializeState(
adminTag,
agentConfig,
envCfg, c.HostedModelConfig,
agentbootstrap.BootstrapMachineConfig{
Addresses: addrs,
BootstrapConstraints: c.BootstrapConstraints,
ModelConstraints: c.ModelConstraints,
Jobs: jobs,
InstanceId: instanceId,
Characteristics: c.Hardware,
SharedSecret: sharedSecret,
},
dialOpts,
environs.NewStatePolicy(),
)
return stateErr
})
if err != nil {
return err
}
defer st.Close()
// Populate the tools catalogue.
if err := c.populateTools(st, env); err != nil {
return err
}
// Populate the GUI archive catalogue.
if err := c.populateGUIArchive(st, env); err != nil {
// Do not stop the bootstrapping process for Juju GUI archive errors.
logger.Warningf("cannot set up Juju GUI: %s", err)
} else {
logger.Debugf("Juju GUI successfully set up")
}
// Add custom image metadata to environment storage.
if c.ImageMetadataDir != "" {
if err := c.saveCustomImageMetadata(st, env); err != nil {
return err
}
stor := newStateStorage(st.ModelUUID(), st.MongoSession())
if err := c.storeCustomImageMetadata(stor); err != nil {
return err
}
}
// Populate the storage pools.
if err = c.populateDefaultStoragePools(st); err != nil {
return err
}
// bootstrap machine always gets the vote
return m.SetHasVote(true)
}
// SyncTools copies the Juju tools tarball from the official bucket
// or a specified source directory into the user's environment.
func SyncTools(syncContext *SyncContext) error {
sourceDataSource, err := selectSourceDatasource(syncContext)
if err != nil {
return err
}
logger.Infof("listing available tools")
if syncContext.MajorVersion == 0 && syncContext.MinorVersion == 0 {
syncContext.MajorVersion = jujuversion.Current.Major
syncContext.MinorVersion = -1
if !syncContext.AllVersions {
syncContext.MinorVersion = jujuversion.Current.Minor
}
}
toolsDir := syncContext.Stream
// If no stream has been specified, assume "released" for non-devel versions of Juju.
if syncContext.Stream == "" {
// We now store the tools in a directory named after their stream, but the
// legacy behaviour is to store all tools in a single "releases" directory.
toolsDir = envtools.LegacyReleaseDirectory
syncContext.Stream = envtools.PreferredStream(&jujuversion.Current, false, syncContext.Stream)
}
sourceTools, err := envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
syncContext.Stream, syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{})
// For backwards compatibility with cloud storage, if there are no tools in the specified stream,
// double check the release stream.
// TODO - remove this when we no longer need to support cloud storage upgrades.
if err == envtools.ErrNoTools {
sourceTools, err = envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
envtools.ReleasedStream, syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{})
}
if err != nil {
return err
}
logger.Infof("found %d tools", len(sourceTools))
if !syncContext.AllVersions {
var latest version.Number
latest, sourceTools = sourceTools.Newest()
logger.Infof("found %d recent tools (version %s)", len(sourceTools), latest)
}
for _, tool := range sourceTools {
logger.Debugf("found source tool: %v", tool)
}
logger.Infof("listing target tools storage")
targetTools, err := syncContext.TargetToolsFinder.FindTools(syncContext.MajorVersion, syncContext.Stream)
switch err {
case nil, coretools.ErrNoMatches, envtools.ErrNoTools:
default:
return err
}
for _, tool := range targetTools {
logger.Debugf("found target tool: %v", tool)
}
missing := sourceTools.Exclude(targetTools)
logger.Infof("found %d tools in target; %d tools to be copied", len(targetTools), len(missing))
if syncContext.DryRun {
for _, tools := range missing {
logger.Infof("copying %s from %s", tools.Version, tools.URL)
}
return nil
}
err = copyTools(toolsDir, syncContext.Stream, missing, syncContext.TargetToolsUploader)
if err != nil {
return err
}
logger.Infof("copied %d tools", len(missing))
return nil
}
func fillinStartInstanceParams(env environs.Environ, machineId string, isController bool, params *environs.StartInstanceParams) error {
if params.ControllerUUID == "" {
return errors.New("missing controller UUID in start instance parameters")
}
preferredSeries := config.PreferredSeries(env.Config())
agentVersion, ok := env.Config().AgentVersion()
if !ok {
return errors.New("missing agent version in model config")
}
filter := coretools.Filter{
Number: agentVersion,
Series: preferredSeries,
}
if params.Constraints.Arch != nil {
filter.Arch = *params.Constraints.Arch
}
stream := tools.PreferredStream(&agentVersion, env.Config().Development(), env.Config().AgentStream())
possibleTools, err := tools.FindTools(env, -1, -1, stream, filter)
if err != nil {
return errors.Trace(err)
}
if params.ImageMetadata == nil {
if err := SetImageMetadata(
env,
possibleTools.AllSeries(),
possibleTools.Arches(),
¶ms.ImageMetadata,
); err != nil {
return errors.Trace(err)
}
}
machineNonce := "fake_nonce"
apiInfo := FakeAPIInfo(machineId)
instanceConfig, err := instancecfg.NewInstanceConfig(
testing.ControllerTag,
machineId,
machineNonce,
imagemetadata.ReleasedStream,
preferredSeries,
apiInfo,
)
if err != nil {
return errors.Trace(err)
}
if isController {
instanceConfig.Controller = &instancecfg.ControllerConfig{
Config: testing.FakeControllerConfig(),
MongoInfo: &mongo.MongoInfo{
Info: mongo.Info{
Addrs: []string{"127.0.0.1:1234"},
CACert: "CA CERT\n" + testing.CACert,
},
Password: "******",
Tag: names.NewMachineTag(machineId),
},
}
instanceConfig.Jobs = []multiwatcher.MachineJob{multiwatcher.JobHostUnits, multiwatcher.JobManageModel}
}
cfg := env.Config()
instanceConfig.Tags = instancecfg.InstanceTags(env.Config().UUID(), params.ControllerUUID, cfg, nil)
params.Tools = possibleTools
params.InstanceConfig = instanceConfig
return nil
}