func (s *bootstrapSuite) TestBootstrapMetadataImagesMissing(c *gc.C) {
environs.UnregisterImageDataSourceFunc("bootstrap metadata")
noImagesDir := c.MkDir()
stor, err := filestorage.NewFileStorageWriter(noImagesDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, "released", "released")
env := newEnviron("foo", useDefaultKeys, nil)
s.setDummyStorage(c, env)
err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
ControllerConfig: coretesting.FakeControllerConfig(),
AdminSecret: "admin-secret",
CAPrivateKey: coretesting.CAKey,
MetadataDir: noImagesDir,
})
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.bootstrapCount, gc.Equals, 1)
datasources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
c.Assert(datasources, gc.HasLen, 2)
c.Assert(datasources[0].Description(), gc.Equals, "default cloud images")
c.Assert(datasources[1].Description(), gc.Equals, "default ubuntu cloud images")
}
func (s *ImageMetadataSuite) TestImageMetadataURLsRegisteredFuncs(c *gc.C) {
environs.RegisterImageDataSourceFunc("id0", func(environs.Environ) (simplestreams.DataSource, error) {
return simplestreams.NewURLDataSource("id0", "betwixt/releases", utils.NoVerifySSLHostnames), nil
})
environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
return simplestreams.NewURLDataSource("id1", "yoink", utils.NoVerifySSLHostnames), nil
})
// overwrite the one previously registered against id1
environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
return nil, errors.NewNotSupported(nil, "oyvey")
})
environs.RegisterUserImageDataSourceFunc("id2", func(environs.Environ) (simplestreams.DataSource, error) {
return simplestreams.NewURLDataSource("id2", "foobar", utils.NoVerifySSLHostnames), nil
})
defer environs.UnregisterImageDataSourceFunc("id0")
defer environs.UnregisterImageDataSourceFunc("id1")
defer environs.UnregisterImageDataSourceFunc("id2")
env := s.env(c, "config-image-metadata-url", "")
sources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
sstesting.AssertExpectedSources(c, sources, []string{
"config-image-metadata-url/",
"foobar/",
"betwixt/releases/",
"http://cloud-images.ubuntu.com/releases/",
})
}
func findImageMetadata(env *environ, args environs.StartInstanceParams) (*OvaFileMetadata, error) {
arches := args.Tools.Arches()
series := args.Tools.OneSeries()
ic := &imagemetadata.ImageConstraint{
LookupParams: simplestreams.LookupParams{
Series: []string{series},
Arches: arches,
Stream: env.ecfg.ImageStream(),
},
}
sources, err := environs.ImageMetadataSources(env)
if err != nil {
return nil, errors.Trace(err)
}
matchingImages, err := imageMetadataFetch(sources, ic)
if err != nil {
return nil, errors.Trace(err)
}
if len(matchingImages) == 0 {
return nil, errors.Errorf("no matching images found for given constraints: %v", ic)
}
return matchingImages[0], nil
}
// TestBootstrapImageMetadataFromAllSources tests that we are looking for
// image metadata in all data sources available to environment.
// Abandoning look up too soon led to misleading bootstrap failures:
// Juju reported no images available for a particular configuration,
// despite image metadata in other data sources compatible with the same configuration as well.
// Related to bug#1560625.
func (s *bootstrapSuite) TestBootstrapImageMetadataFromAllSources(c *gc.C) {
s.PatchValue(&series.HostSeries, func() string { return "raring" })
s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })
// Ensure that we can find at least one image metadata
// early on in the image metadata lookup.
// We should continue looking despite it.
metadataDir, _ := createImageMetadata(c)
stor, err := filestorage.NewFileStorageWriter(metadataDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, "released", "released")
env := bootstrapEnvironWithRegion{
newEnviron("foo", useDefaultKeys, nil),
simplestreams.CloudSpec{
Region: "region",
Endpoint: "endpoint",
},
}
s.setDummyStorage(c, env.bootstrapEnviron)
bootstrapCons := constraints.MustParse("arch=amd64")
err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
BootstrapConstraints: bootstrapCons,
MetadataDir: metadataDir,
})
c.Assert(err, jc.ErrorIsNil)
datasources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
for _, source := range datasources {
// make sure we looked in each and all...
c.Assert(c.GetTestLog(), jc.Contains, fmt.Sprintf("image metadata in %s", source.Description()))
}
}
func (s *bootstrapSuite) TestBootstrapMetadata(c *gc.C) {
environs.UnregisterImageDataSourceFunc("bootstrap metadata")
metadataDir, metadata := createImageMetadata(c)
stor, err := filestorage.NewFileStorageWriter(metadataDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, "released", "released")
env := newEnviron("foo", useDefaultKeys, nil)
s.setDummyStorage(c, env)
err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
MetadataDir: metadataDir,
})
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.bootstrapCount, gc.Equals, 1)
c.Assert(envtools.DefaultBaseURL, gc.Equals, metadataDir)
datasources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
c.Assert(datasources, gc.HasLen, 3)
c.Assert(datasources[0].Description(), gc.Equals, "bootstrap metadata")
// This data source does not require to contain signed data.
// However, it may still contain it.
// Since we will always try to read signed data first,
// we want to be able to try to read this signed data
// with a user provided key.
// for this test, user provided key is empty.
// Bugs #1542127, #1542131
c.Assert(datasources[0].PublicSigningKey(), gc.Equals, "")
c.Assert(env.instanceConfig, gc.NotNil)
c.Assert(env.instanceConfig.CustomImageMetadata, gc.HasLen, 1)
c.Assert(env.instanceConfig.CustomImageMetadata[0], gc.DeepEquals, metadata[0])
}
func (s *ImageMetadataSuite) TestImageMetadataURLsRegisteredFuncs(c *gc.C) {
environs.RegisterImageDataSourceFunc("id0", func(environs.Environ) (simplestreams.DataSource, error) {
return simplestreams.NewURLDataSource("id0", "betwixt/releases", utils.NoVerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, false), nil
})
environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
return simplestreams.NewURLDataSource("id1", "yoink", utils.NoVerifySSLHostnames, simplestreams.SPECIFIC_CLOUD_DATA, false), nil
})
// overwrite the one previously registered against id1
environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
return nil, errors.NewNotSupported(nil, "oyvey")
})
environs.RegisterUserImageDataSourceFunc("id2", func(environs.Environ) (simplestreams.DataSource, error) {
return simplestreams.NewURLDataSource("id2", "foobar", utils.NoVerifySSLHostnames, simplestreams.CUSTOM_CLOUD_DATA, false), nil
})
defer environs.UnregisterImageDataSourceFunc("id0")
defer environs.UnregisterImageDataSourceFunc("id1")
defer environs.UnregisterImageDataSourceFunc("id2")
env := s.env(c, "config-image-metadata-url", "")
sources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
sstesting.AssertExpectedSources(c, sources, []sstesting.SourceDetails{
{"config-image-metadata-url/", ""},
{"foobar/", ""},
{"betwixt/releases/", ""},
{"https://streams.canonical.com/juju/images/releases/", imagemetadata.SimplestreamsImagesPublicKey},
{"http://cloud-images.ubuntu.com/releases/", imagemetadata.SimplestreamsImagesPublicKey},
})
}
func (s *bootstrapSuite) TestBootstrapMetadata(c *gc.C) {
environs.UnregisterImageDataSourceFunc("bootstrap metadata")
metadataDir, metadata := createImageMetadata(c)
stor, err := filestorage.NewFileStorageWriter(metadataDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, "released", "released")
env := newEnviron("foo", useDefaultKeys, nil)
s.setDummyStorage(c, env)
err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
MetadataDir: metadataDir,
})
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.bootstrapCount, gc.Equals, 1)
c.Assert(envtools.DefaultBaseURL, gc.Equals, metadataDir)
datasources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
c.Assert(datasources, gc.HasLen, 3)
c.Assert(datasources[0].Description(), gc.Equals, "bootstrap metadata")
c.Assert(env.instanceConfig, gc.NotNil)
c.Assert(env.instanceConfig.CustomImageMetadata, gc.HasLen, 1)
c.Assert(env.instanceConfig.CustomImageMetadata[0], gc.DeepEquals, metadata[0])
}
func SetImageMetadata(env environs.Environ, series, arches []string, out *[]*imagemetadata.ImageMetadata) error {
hasRegion, ok := env.(simplestreams.HasRegion)
if !ok {
return nil
}
sources, err := environs.ImageMetadataSources(env)
if err != nil {
return errors.Trace(err)
}
region, err := hasRegion.Region()
if err != nil {
return errors.Trace(err)
}
imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
CloudSpec: region,
Series: series,
Arches: arches,
Stream: env.Config().ImageStream(),
})
imageMetadata, _, err := imagemetadata.Fetch(sources, imageConstraint)
if err != nil {
return errors.Trace(err)
}
*out = imageMetadata
return nil
}
// FindInstanceSpec returns an InstanceSpec satisfying the supplied instanceConstraint.
func (env *joyentEnviron) FindInstanceSpec(ic *instances.InstanceConstraint) (*instances.InstanceSpec, error) {
// Require at least one VCPU so we get KVM rather than smart package.
if ic.Constraints.CpuCores == nil {
ic.Constraints.CpuCores = &defaultCpuCores
}
allInstanceTypes, err := env.listInstanceTypes()
if err != nil {
return nil, err
}
imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
CloudSpec: simplestreams.CloudSpec{ic.Region, env.Ecfg().SdcUrl()},
Series: []string{ic.Series},
Arches: ic.Arches,
})
sources, err := environs.ImageMetadataSources(env)
if err != nil {
return nil, err
}
matchingImages, _, err := imagemetadata.Fetch(sources, imageConstraint, signedImageDataOnly)
if err != nil {
return nil, err
}
images := instances.ImageMetadataToImages(matchingImages)
spec, err := instances.FindInstanceSpec(images, ic, allInstanceTypes)
if err != nil {
return nil, err
}
return spec, nil
}
func (s *ImageMetadataSuite) TestImageMetadataURLs(c *gc.C) {
env := s.env(c, "config-image-metadata-url", "")
sources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
sstesting.AssertExpectedSources(c, sources, []string{
"config-image-metadata-url/", "http://cloud-images.ubuntu.com/releases/",
})
}
func (s *ImageMetadataSuite) TestImageMetadataURLsNonReleaseStream(c *gc.C) {
env := s.env(c, "", "daily")
sources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
sstesting.AssertExpectedSources(c, sources, []string{
"http://cloud-images.ubuntu.com/daily/",
})
}
func (s *ImageMetadataSuite) TestImageMetadataURLsNoConfigURL(c *gc.C) {
env := s.env(c, "", "")
sources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
sstesting.AssertExpectedSources(c, sources, []string{
"https://streams.canonical.com/juju/images/releases/",
"http://cloud-images.ubuntu.com/releases/",
})
}
func (s *ImageMetadataSuite) TestImageMetadataURLsNonReleaseStream(c *gc.C) {
env := s.env(c, "", "daily")
sources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
sstesting.AssertExpectedSources(c, sources, []sstesting.SourceDetails{
{"https://streams.canonical.com/juju/images/daily/", imagemetadata.SimplestreamsImagesPublicKey},
{"http://cloud-images.ubuntu.com/daily/", imagemetadata.SimplestreamsImagesPublicKey},
})
}
func (s *ImageMetadataSuite) TestImageMetadataURLs(c *gc.C) {
env := s.env(c, "config-image-metadata-url", "")
sources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
sstesting.AssertExpectedSources(c, sources, []sstesting.SourceDetails{
{"config-image-metadata-url/", ""},
{"https://streams.canonical.com/juju/images/releases/", keys.JujuPublicKey},
{"http://cloud-images.ubuntu.com/releases/", imagemetadata.SimplestreamsImagesPublicKey},
})
}
func (s *ImageMetadataSuite) TestImageMetadataURLsRegisteredFuncsError(c *gc.C) {
environs.RegisterImageDataSourceFunc("id0", func(environs.Environ) (simplestreams.DataSource, error) {
return nil, errors.New("oyvey!")
})
defer environs.UnregisterImageDataSourceFunc("id0")
env := s.env(c, "config-image-metadata-url", "")
_, err := environs.ImageMetadataSources(env)
c.Assert(err, gc.ErrorMatches, "oyvey!")
}
func (c *validateImageMetadataCommand) createLookupParams(context *cmd.Context) (*simplestreams.MetadataLookupParams, error) {
params := &simplestreams.MetadataLookupParams{Stream: c.stream}
if c.providerType == "" {
environ, err := c.prepare(context)
if err != nil {
return nil, err
}
mdLookup, ok := environ.(simplestreams.MetadataValidator)
if !ok {
return nil, fmt.Errorf("%s provider does not support image metadata validation", environ.Config().Type())
}
params, err = mdLookup.MetadataLookupParams(c.region)
if err != nil {
return nil, err
}
oes := &overrideEnvStream{environ, c.stream}
params.Sources, err = environs.ImageMetadataSources(oes)
if err != nil {
return nil, err
}
} else {
prov, err := environs.Provider(c.providerType)
if err != nil {
return nil, err
}
mdLookup, ok := prov.(simplestreams.MetadataValidator)
if !ok {
return nil, fmt.Errorf("%s provider does not support image metadata validation", c.providerType)
}
params, err = mdLookup.MetadataLookupParams(c.region)
if err != nil {
return nil, err
}
}
if c.series != "" {
params.Series = c.series
}
if c.region != "" {
params.Region = c.region
}
if c.endpoint != "" {
params.Endpoint = c.endpoint
}
if c.metadataDir != "" {
dir := filepath.Join(c.metadataDir, "images")
if _, err := os.Stat(dir); err != nil {
return nil, err
}
params.Sources = imagesDataSources(dir)
}
return params, nil
}
func (s *localServerSuite) TestValidateImageMetadata(c *gc.C) {
env := s.Prepare(c)
params, err := env.(simplestreams.MetadataValidator).MetadataLookupParams("some-region")
c.Assert(err, jc.ErrorIsNil)
params.Sources, err = environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
params.Series = "raring"
image_ids, _, err := imagemetadata.ValidateImageMetadata(params)
c.Assert(err, jc.ErrorIsNil)
c.Assert(image_ids, gc.DeepEquals, []string{"11223344-0a0a-dd77-33cd-abcd1234e5f6"})
}
func (t *localServerSuite) TestValidateImageMetadata(c *gc.C) {
env := t.Prepare(c)
params, err := env.(simplestreams.MetadataValidator).MetadataLookupParams("test")
c.Assert(err, jc.ErrorIsNil)
params.Series = coretesting.FakeDefaultSeries
params.Endpoint = "https://ec2.endpoint.com"
params.Sources, err = environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
image_ids, _, err := imagemetadata.ValidateImageMetadata(params)
c.Assert(err, jc.ErrorIsNil)
sort.Strings(image_ids)
c.Assert(image_ids, gc.DeepEquals, []string{"ami-00000033", "ami-00000034", "ami-00000035", "ami-00000039"})
}
// SupportedArchitectures returns all the image architectures for env matching the constraints.
func SupportedArchitectures(env environs.Environ, imageConstraint *imagemetadata.ImageConstraint) ([]string, error) {
sources, err := environs.ImageMetadataSources(env)
if err != nil {
return nil, err
}
matchingImages, _, err := imagemetadata.Fetch(sources, imageConstraint, false)
if err != nil {
return nil, err
}
var arches = set.NewStrings()
for _, im := range matchingImages {
arches.Add(im.Arch)
}
return arches.Values(), nil
}
func (api *API) retrievePublished() error {
envCfg, err := api.metadata.ModelConfig()
if err != nil {
return errors.Annotatef(err, "getting environ config")
}
env, err := environs.New(envCfg)
if err != nil {
return errors.Annotatef(err, "getting environ")
}
sources, err := environs.ImageMetadataSources(env)
if err != nil {
return errors.Annotatef(err, "getting cloud specific image metadata sources")
}
cons := envmetadata.NewImageConstraint(simplestreams.LookupParams{})
if inst, ok := env.(simplestreams.HasRegion); !ok {
// Published image metadata for some providers are in simple streams.
// Providers that do not rely on simplestreams, don't need to do anything here.
return nil
} else {
// If we can determine current region,
// we want only metadata specific to this region.
cloud, err := inst.Region()
if err != nil {
return errors.Annotatef(err, "getting cloud specific region information")
}
cons.CloudSpec = cloud
}
// We want all relevant metadata from all data sources.
for _, source := range sources {
logger.Debugf("looking in data source %v", source.Description())
metadata, info, err := envmetadata.Fetch([]simplestreams.DataSource{source}, cons)
if err != nil {
// Do not stop looking in other data sources if there is an issue here.
logger.Errorf("encountered %v while getting published images metadata from %v", err, source.Description())
continue
}
err = api.saveAll(info, source.Priority(), metadata)
if err != nil {
// Do not stop looking in other data sources if there is an issue here.
logger.Errorf("encountered %v while saving published images metadata from %v", err, source.Description())
}
}
return nil
}
// findInstanceSpec returns an image and instance type satisfying the constraint.
// The instance type comes from querying the flavors supported by the deployment.
func findInstanceSpec(e *environ, ic *instances.InstanceConstraint) (*instances.InstanceSpec, error) {
// first construct all available instance types from the supported flavors.
nova := e.nova()
flavors, err := nova.ListFlavorsDetail()
if err != nil {
return nil, err
}
allInstanceTypes := []instances.InstanceType{}
for _, flavor := range flavors {
instanceType := instances.InstanceType{
Id: flavor.Id,
Name: flavor.Name,
Arches: ic.Arches,
Mem: uint64(flavor.RAM),
CpuCores: uint64(flavor.VCPUs),
RootDisk: uint64(flavor.Disk * 1024),
// tags not currently supported on openstack
}
allInstanceTypes = append(allInstanceTypes, instanceType)
}
imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
CloudSpec: simplestreams.CloudSpec{ic.Region, e.ecfg().authURL()},
Series: []string{ic.Series},
Arches: ic.Arches,
Stream: e.Config().ImageStream(),
})
sources, err := environs.ImageMetadataSources(e)
if err != nil {
return nil, err
}
// TODO (wallyworld): use an env parameter (default true) to mandate use of only signed image metadata.
matchingImages, _, err := imagemetadata.Fetch(sources, imageConstraint, false)
if err != nil {
return nil, err
}
images := instances.ImageMetadataToImages(matchingImages)
spec, err := instances.FindInstanceSpec(images, ic, allInstanceTypes)
if err != nil {
return nil, err
}
return spec, nil
}
// findMatchingImages queries simplestreams for OS images that match the given
// requirements.
//
// If it finds no matching images, that's an error.
func findMatchingImages(e *azureEnviron, location, series string, arches []string) ([]*imagemetadata.ImageMetadata, error) {
endpoint := getEndpoint(location)
constraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
CloudSpec: simplestreams.CloudSpec{location, endpoint},
Series: []string{series},
Arches: arches,
Stream: e.Config().ImageStream(),
})
sources, err := environs.ImageMetadataSources(e)
if err != nil {
return nil, err
}
images, _, err := imagemetadata.Fetch(sources, constraint, signedImageDataOnly)
if len(images) == 0 || errors.IsNotFound(err) {
return nil, fmt.Errorf("no OS images found for location %q, series %q, architectures %q (and endpoint: %q)", location, series, arches, endpoint)
} else if err != nil {
return nil, err
}
return images, nil
}
func (env *environ) lookupArchitectures() ([]string, error) {
// Create a filter to get all images for the
// correct stream.
imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
Stream: env.Config().ImageStream(),
})
sources, err := environs.ImageMetadataSources(env)
if err != nil {
return nil, errors.Trace(err)
}
matchingImages, err := imageMetadataFetch(sources, imageConstraint)
if err != nil {
return nil, errors.Trace(err)
}
var arches = set.NewStrings()
for _, im := range matchingImages {
arches.Add(im.Arch)
}
return arches.Values(), nil
}
func (api *API) retrievePublished() ([]*envmetadata.ImageMetadata, error) {
// Get environ
envCfg, err := api.metadata.EnvironConfig()
env, err := environs.New(envCfg)
if err != nil {
return nil, errors.Trace(err)
}
// Get all images metadata sources for this environ.
sources, err := environs.ImageMetadataSources(env)
if err != nil {
return nil, err
}
// We want all metadata.
cons := envmetadata.NewImageConstraint(simplestreams.LookupParams{})
metadata, _, err := envmetadata.Fetch(sources, cons, false)
if err != nil {
return nil, err
}
return metadata, nil
}
func (env *environ) getImageSources() ([]lxdclient.Remote, error) {
metadataSources, err := environs.ImageMetadataSources(env)
if err != nil {
return nil, errors.Trace(err)
}
remotes := make([]lxdclient.Remote, 0)
for _, source := range metadataSources {
url, err := source.URL("")
if err != nil {
logger.Debugf("failed to get the URL for metadataSource: %s", err)
continue
}
// NOTE(jam) LXD only allows you to pass HTTPS URLs. So strip
// off http:// and replace it with https://
// Arguably we could give the user a direct error if
// env.ImageMetadataURL is http instead of https, but we also
// get http from the DefaultImageSources, which is why we
// replace it.
// TODO(jam) Maybe we could add a Validate step that ensures
// image-metadata-url is an "https://" URL, so that Users get a
// "your configuration is wrong" error, rather than silently
// changing it and having them get confused.
// https://github.com/lxc/lxd/issues/1763
if strings.HasPrefix(url, "http://") {
url = strings.TrimPrefix(url, "http://")
url = "https://" + url
logger.Debugf("LXD requires https://, using: %s", url)
}
remotes = append(remotes, lxdclient.Remote{
Name: source.Description(),
Host: url,
Protocol: lxdclient.SimplestreamsProtocol,
Cert: nil,
ServerPEMCert: "",
})
}
return remotes, nil
}
// findInstanceSpec initializes a new instance spec for the given stream
// (and constraints) and returns it. This only covers populating the
// initial data for the spec. However, it does include fetching the
// correct simplestreams image data.
func (env *environ) findInstanceSpec(stream string, ic *instances.InstanceConstraint) (*instances.InstanceSpec, error) {
sources, err := environs.ImageMetadataSources(env)
if err != nil {
return nil, errors.Trace(err)
}
imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
CloudSpec: env.cloudSpec(ic.Region),
Series: []string{ic.Series},
Arches: ic.Arches,
Stream: stream,
})
signedImageDataOnly := false
matchingImages, _, err := imageMetadataFetch(sources, imageConstraint, signedImageDataOnly)
if err != nil {
return nil, errors.Trace(err)
}
images := instances.ImageMetadataToImages(matchingImages)
spec, err := instances.FindInstanceSpec(images, ic, allInstanceTypes)
return spec, errors.Trace(err)
}
// bootstrapImageMetadata returns the image metadata to use for bootstrapping
// the given environment. If the environment provider does not make use of
// simplestreams, no metadata will be returned.
//
// If a bootstrap image ID is specified, image metadata will be synthesised
// using that image ID, and the architecture and series specified by the
// initiator. In addition, the custom image metadata that is saved into the
// state database will have the synthesised image metadata added to it.
func bootstrapImageMetadata(
environ environs.Environ,
availableTools coretools.List,
bootstrapImageId string,
customImageMetadata *[]*imagemetadata.ImageMetadata,
) ([]*imagemetadata.ImageMetadata, error) {
hasRegion, ok := environ.(simplestreams.HasRegion)
if !ok {
if bootstrapImageId != "" {
// We only support specifying image IDs for providers
// that use simplestreams for now.
return nil, errors.NotSupportedf(
"specifying bootstrap image for %q provider",
environ.Config().Type(),
)
}
// No region, no metadata.
return nil, nil
}
region, err := hasRegion.Region()
if err != nil {
return nil, errors.Trace(err)
}
if bootstrapImageId != "" {
arches := availableTools.Arches()
if len(arches) != 1 {
return nil, errors.NotValidf("multiple architectures with bootstrap image")
}
allSeries := availableTools.AllSeries()
if len(allSeries) != 1 {
return nil, errors.NotValidf("multiple series with bootstrap image")
}
seriesVersion, err := series.SeriesVersion(allSeries[0])
if err != nil {
return nil, errors.Trace(err)
}
// The returned metadata does not have information about the
// storage or virtualisation type. Any provider that wants to
// filter on those properties should allow for empty values.
meta := &imagemetadata.ImageMetadata{
Id: bootstrapImageId,
Arch: arches[0],
Version: seriesVersion,
RegionName: region.Region,
Endpoint: region.Endpoint,
Stream: environ.Config().ImageStream(),
}
*customImageMetadata = append(*customImageMetadata, meta)
return []*imagemetadata.ImageMetadata{meta}, nil
}
// For providers that support making use of simplestreams
// image metadata, search public image metadata. We need
// to pass this onto Bootstrap for selecting images.
sources, err := environs.ImageMetadataSources(environ)
if err != nil {
return nil, errors.Trace(err)
}
imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
CloudSpec: region,
Series: availableTools.AllSeries(),
Arches: availableTools.Arches(),
Stream: environ.Config().ImageStream(),
})
logger.Debugf("constraints for image metadata lookup %v", imageConstraint)
// Get image metadata from all data sources.
// Since order of data source matters, order of image metadata matters too. Append is important here.
var publicImageMetadata []*imagemetadata.ImageMetadata
for _, source := range sources {
sourceMetadata, _, err := imagemetadata.Fetch([]simplestreams.DataSource{source}, imageConstraint)
if err != nil {
logger.Debugf("ignoring image metadata in %s: %v", source.Description(), err)
// Just keep looking...
continue
}
logger.Debugf("found %d image metadata in %s", len(sourceMetadata), source.Description())
publicImageMetadata = append(publicImageMetadata, sourceMetadata...)
}
logger.Debugf("found %d image metadata from all image data sources", len(publicImageMetadata))
if len(publicImageMetadata) == 0 {
return nil, errors.New("no image metadata found")
}
return publicImageMetadata, nil
}
// bootstrapImageMetadata returns the image metadata to use for bootstrapping
// the given environment. If the environment provider does not make use of
// simplestreams, no metadata will be returned.
//
// If a bootstrap image ID is specified, image metadat will be synthesised
// using that image ID, and the architecture and series specified by the
// initiator. In addition, the custom image metadat that is saved into the
// state database will have the synthesised image metadata added to it.
func bootstrapImageMetadata(
environ environs.Environ,
availableTools coretools.List,
bootstrapImageId string,
customImageMetadata *[]*imagemetadata.ImageMetadata,
) ([]*imagemetadata.ImageMetadata, error) {
hasRegion, ok := environ.(simplestreams.HasRegion)
if !ok {
if bootstrapImageId != "" {
// We only support specifying image IDs for providers
// that use simplestreams for now.
return nil, errors.NotSupportedf(
"specifying bootstrap image for %q provider",
environ.Config().Type(),
)
}
// No region, no metadata.
return nil, nil
}
region, err := hasRegion.Region()
if err != nil {
return nil, errors.Trace(err)
}
if bootstrapImageId != "" {
arches := availableTools.Arches()
if len(arches) != 1 {
return nil, errors.NotValidf("multiple architectures with bootstrap image")
}
allSeries := availableTools.AllSeries()
if len(allSeries) != 1 {
return nil, errors.NotValidf("multiple series with bootstrap image")
}
seriesVersion, err := series.SeriesVersion(allSeries[0])
if err != nil {
return nil, errors.Trace(err)
}
// The returned metadata does not have information about the
// storage or virtualisation type. Any provider that wants to
// filter on those properties should allow for empty values.
meta := &imagemetadata.ImageMetadata{
Id: bootstrapImageId,
Arch: arches[0],
Version: seriesVersion,
RegionName: region.Region,
Endpoint: region.Endpoint,
Stream: environ.Config().ImageStream(),
}
*customImageMetadata = append(*customImageMetadata, meta)
return []*imagemetadata.ImageMetadata{meta}, nil
}
// For providers that support making use of simplestreams
// image metadata, search public image metadata. We need
// to pass this onto Bootstrap for selecting images.
sources, err := environs.ImageMetadataSources(environ)
if err != nil {
return nil, errors.Trace(err)
}
imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
CloudSpec: region,
Series: availableTools.AllSeries(),
Arches: availableTools.Arches(),
Stream: environ.Config().ImageStream(),
})
publicImageMetadata, _, err := imagemetadata.Fetch(sources, imageConstraint)
if err != nil {
return nil, errors.Annotate(err, "searching image metadata")
}
return publicImageMetadata, nil
}
"github.com/juju/juju/cloudconfig/instancecfg"
"github.com/juju/juju/cloudconfig/providerinit"
"github.com/juju/juju/environs"
"github.com/juju/juju/environs/imagemetadata"
"github.com/juju/juju/environs/simplestreams"
"github.com/juju/juju/instance"
"github.com/juju/juju/tools"
)
//
// Imlementation of InstanceBroker: methods for starting and stopping instances.
//
var findInstanceImage = func(env *environ, ic *imagemetadata.ImageConstraint) (*imagemetadata.ImageMetadata, error) {
sources, err := environs.ImageMetadataSources(env)
if err != nil {
return nil, err
}
matchingImages, _, err := imagemetadata.Fetch(sources, ic, false)
if err != nil {
return nil, err
}
if len(matchingImages) == 0 {
return nil, errors.New("no matching image meta data")
}
return matchingImages[0], nil
}
// imageMetadataFromDataSources finds image metadata that match specified criteria in existing data sources.
func (p *ProvisionerAPI) imageMetadataFromDataSources(env environs.Environ, constraint *imagemetadata.ImageConstraint) ([]params.CloudImageMetadata, error) {
sources, err := environs.ImageMetadataSources(env)
if err != nil {
return nil, errors.Trace(err)
}
cfg := env.Config()
toModel := func(m *imagemetadata.ImageMetadata, mSeries string, source string, priority int) cloudimagemetadata.Metadata {
result := cloudimagemetadata.Metadata{
MetadataAttributes: cloudimagemetadata.MetadataAttributes{
Region: m.RegionName,
Arch: m.Arch,
VirtType: m.VirtType,
RootStorageType: m.Storage,
Source: source,
Series: mSeries,
Stream: m.Stream,
Version: m.Version,
},
Priority: priority,
ImageId: m.Id,
}
// TODO (anastasiamac 2016-08-24) This is a band-aid solution.
// Once correct value is read from simplestreams, this needs to go.
// Bug# 1616295
if result.Stream == "" {
result.Stream = constraint.Stream
}
if result.Stream == "" {
result.Stream = cfg.ImageStream()
}
return result
}
var metadataState []cloudimagemetadata.Metadata
for _, source := range sources {
logger.Debugf("looking in data source %v", source.Description())
found, info, err := imagemetadata.Fetch([]simplestreams.DataSource{source}, constraint)
if err != nil {
// Do not stop looking in other data sources if there is an issue here.
logger.Warningf("encountered %v while getting published images metadata from %v", err, source.Description())
continue
}
for _, m := range found {
mSeries, err := series.VersionSeries(m.Version)
if err != nil {
logger.Warningf("could not determine series for image id %s: %v", m.Id, err)
continue
}
metadataState = append(metadataState, toModel(m, mSeries, info.Source, source.Priority()))
}
}
if len(metadataState) > 0 {
if err := p.st.CloudImageMetadataStorage.SaveMetadata(metadataState); err != nil {
// No need to react here, just take note
logger.Warningf("failed to save published image metadata: %v", err)
}
}
// Since we've fallen through to data sources search and have saved all needed images into controller,
// let's try to get them from controller to avoid duplication of conversion logic here.
all, err := p.imageMetadataFromState(constraint)
if err != nil {
return nil, errors.Annotate(err, "could not read metadata from controller after saving it there from data sources")
}
if len(all) == 0 {
return nil, errors.NotFoundf("image metadata for series %v, arch %v", constraint.Series, constraint.Arches)
}
return all, nil
}