func (s *localServerSuite) assertGetImageMetadataSources(c *gc.C, stream, officialSourcePath string) {
// Create a config that matches s.TestConfig but with the specified stream.
envAttrs := s.TestConfig
if stream != "" {
envAttrs = envAttrs.Merge(coretesting.Attrs{"image-stream": stream})
}
cfg, err := config.New(config.NoDefaults, envAttrs)
c.Assert(err, gc.IsNil)
env, err := environs.New(cfg)
c.Assert(err, gc.IsNil)
sources, err := imagemetadata.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
c.Assert(sources, gc.HasLen, 4)
var urls = make([]string, len(sources))
for i, source := range sources {
url, err := source.URL("")
c.Assert(err, gc.IsNil)
urls[i] = url
}
// The image-metadata-url ends with "/juju-dist-test/".
c.Check(strings.HasSuffix(urls[0], "/juju-dist-test/"), jc.IsTrue)
// The control bucket URL contains the bucket name.
c.Check(strings.Contains(urls[1], openstack.ControlBucketName(env)+"/images"), jc.IsTrue)
// The product-streams URL ends with "/imagemetadata".
c.Check(strings.HasSuffix(urls[2], "/imagemetadata/"), jc.IsTrue)
c.Assert(urls[3], gc.Equals, fmt.Sprintf("http://cloud-images.ubuntu.com/%s/", officialSourcePath))
}
func (t *ProviderSuite) assertGetImageMetadataSources(c *gc.C, stream, officialSourcePath string) {
// Make an env configured with the stream.
envAttrs := localConfigAttrs
if stream != "" {
envAttrs = envAttrs.Merge(coretesting.Attrs{
"image-stream": stream,
})
}
cfg, err := config.New(config.NoDefaults, envAttrs)
c.Assert(err, gc.IsNil)
env, err := environs.Prepare(cfg, coretesting.Context(c), configstore.NewMem())
c.Assert(err, gc.IsNil)
c.Assert(env, gc.NotNil)
sources, err := imagemetadata.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
c.Assert(len(sources), gc.Equals, 2)
var urls = make([]string, len(sources))
for i, source := range sources {
url, err := source.URL("")
c.Assert(err, gc.IsNil)
urls[i] = url
}
// The control bucket URL contains the bucket name.
c.Check(strings.Contains(urls[0], ec2.ControlBucketName(env)+"/images"), jc.IsTrue)
c.Assert(urls[1], gc.Equals, fmt.Sprintf("http://cloud-images.ubuntu.com/%s/", officialSourcePath))
}
func (suite *environSuite) assertGetImageMetadataSources(c *gc.C, stream, officialSourcePath string) {
// Make an env configured with the stream.
testAttrs := maasEnvAttrs
testAttrs = testAttrs.Merge(coretesting.Attrs{
"maas-server": suite.testMAASObject.TestServer.URL,
})
if stream != "" {
testAttrs = testAttrs.Merge(coretesting.Attrs{
"image-stream": stream,
})
}
attrs := coretesting.FakeConfig().Merge(testAttrs)
cfg, err := config.New(config.NoDefaults, attrs)
c.Assert(err, gc.IsNil)
env, err := NewEnviron(cfg)
c.Assert(err, gc.IsNil)
// Add a dummy file to storage so we can use that to check the
// obtained source later.
data := makeRandomBytes(10)
stor := NewStorage(env)
err = stor.Put("images/filename", bytes.NewBuffer([]byte(data)), int64(len(data)))
c.Assert(err, gc.IsNil)
sources, err := imagemetadata.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
c.Assert(len(sources), gc.Equals, 2)
assertSourceContents(c, sources[0], "filename", data)
url, err := sources[1].URL("")
c.Assert(err, gc.IsNil)
c.Assert(url, gc.Equals, fmt.Sprintf("http://cloud-images.ubuntu.com/%s/", officialSourcePath))
}
func (s *URLsSuite) TestImageMetadataURLsNonReleaseStream(c *gc.C) {
env := s.env(c, "", "daily")
sources, err := imagemetadata.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
privateStorageURL, err := env.Storage().URL("images")
c.Assert(err, gc.IsNil)
sstesting.AssertExpectedSources(c, sources, []string{
privateStorageURL, "http://cloud-images.ubuntu.com/daily/"})
}
func (s *URLsSuite) TestImageMetadataURLs(c *gc.C) {
env := s.env(c, "config-image-metadata-url", "")
sources, err := imagemetadata.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
privateStorageURL, err := env.Storage().URL("images")
c.Assert(err, gc.IsNil)
sstesting.AssertExpectedSources(c, sources, []string{
"config-image-metadata-url/", privateStorageURL, "http://cloud-images.ubuntu.com/releases/"})
}
func (s *localServerSuite) TestValidateImageMetadata(c *gc.C) {
env := s.Open(c)
params, err := env.(simplestreams.MetadataValidator).MetadataLookupParams("some-region")
c.Assert(err, gc.IsNil)
params.Sources, err = imagemetadata.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
params.Series = "raring"
image_ids, _, err := imagemetadata.ValidateImageMetadata(params)
c.Assert(err, gc.IsNil)
c.Assert(image_ids, jc.SameContents, []string{"id-y"})
}
func (s *localServerSuite) TestValidateImageMetadata(c *gc.C) {
env := s.Prepare(c)
params, err := env.(simplestreams.MetadataValidator).MetadataLookupParams("some-region")
c.Assert(err, gc.IsNil)
params.Sources, err = imagemetadata.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
params.Series = "raring"
image_ids, _, err := imagemetadata.ValidateImageMetadata(params)
c.Assert(err, gc.IsNil)
c.Assert(image_ids, gc.DeepEquals, []string{"11223344-0a0a-dd77-33cd-abcd1234e5f6"})
}
func (t *localServerSuite) TestValidateImageMetadata(c *gc.C) {
env := t.Prepare(c)
params, err := env.(simplestreams.MetadataValidator).MetadataLookupParams("test")
c.Assert(err, gc.IsNil)
params.Series = coretesting.FakeDefaultSeries
params.Endpoint = "https://ec2.endpoint.com"
params.Sources, err = imagemetadata.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
image_ids, _, err := imagemetadata.ValidateImageMetadata(params)
c.Assert(err, gc.IsNil)
sort.Strings(image_ids)
c.Assert(image_ids, gc.DeepEquals, []string{"ami-00000033", "ami-00000034", "ami-00000035"})
}
func (s *localServerSuite) TestGetImageMetadataSources(c *gc.C) {
env := s.Prepare(c)
sources, err := imagemetadata.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
c.Assert(len(sources), gc.Equals, 2)
var urls = make([]string, len(sources))
for i, source := range sources {
url, err := source.URL("")
c.Assert(err, gc.IsNil)
urls[i] = url
}
// The control bucket URL contains the bucket name.
c.Assert(strings.Contains(urls[0], joyent.ControlBucketName(env)+"/images"), jc.IsTrue)
}
// SupportedArchitectures returns all the image architectures for env matching the constraints.
func SupportedArchitectures(env environs.Environ, imageConstraint *imagemetadata.ImageConstraint) ([]string, error) {
sources, err := imagemetadata.GetMetadataSources(env)
if err != nil {
return nil, err
}
matchingImages, _, err := imagemetadata.Fetch(sources, simplestreams.DefaultIndexPath, imageConstraint, false)
if err != nil {
return nil, err
}
var arches = set.NewStrings()
for _, im := range matchingImages {
arches.Add(im.Arch)
}
return arches.Values(), nil
}
// findInstanceSpec returns an image and instance type satisfying the constraint.
// The instance type comes from querying the flavors supported by the deployment.
func findInstanceSpec(e *environ, ic *instances.InstanceConstraint) (*instances.InstanceSpec, error) {
// first construct all available instance types from the supported flavors.
nova := e.nova()
flavors, err := nova.ListFlavorsDetail()
if err != nil {
return nil, err
}
allInstanceTypes := []instances.InstanceType{}
for _, flavor := range flavors {
instanceType := instances.InstanceType{
Id: flavor.Id,
Name: flavor.Name,
Arches: ic.Arches,
Mem: uint64(flavor.RAM),
CpuCores: uint64(flavor.VCPUs),
RootDisk: uint64(flavor.Disk * 1024),
// tags not currently supported on openstack
}
allInstanceTypes = append(allInstanceTypes, instanceType)
}
imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
CloudSpec: simplestreams.CloudSpec{ic.Region, e.ecfg().authURL()},
Series: []string{ic.Series},
Arches: ic.Arches,
Stream: e.Config().ImageStream(),
})
sources, err := imagemetadata.GetMetadataSources(e)
if err != nil {
return nil, err
}
// TODO (wallyworld): use an env parameter (default true) to mandate use of only signed image metadata.
matchingImages, _, err := imagemetadata.Fetch(sources, simplestreams.DefaultIndexPath, imageConstraint, false)
if err != nil {
return nil, err
}
images := instances.ImageMetadataToImages(matchingImages)
spec, err := instances.FindInstanceSpec(images, ic, allInstanceTypes)
if err != nil {
return nil, err
}
return spec, nil
}
// findMatchingImages queries simplestreams for OS images that match the given
// requirements.
//
// If it finds no matching images, that's an error.
func findMatchingImages(e *azureEnviron, location, series string, arches []string) ([]*imagemetadata.ImageMetadata, error) {
endpoint := getEndpoint(location)
constraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
CloudSpec: simplestreams.CloudSpec{location, endpoint},
Series: []string{series},
Arches: arches,
Stream: e.Config().ImageStream(),
})
sources, err := imagemetadata.GetMetadataSources(e)
if err != nil {
return nil, err
}
indexPath := simplestreams.DefaultIndexPath
images, _, err := imagemetadata.Fetch(sources, indexPath, constraint, signedImageDataOnly)
if len(images) == 0 || errors.IsNotFound(err) {
return nil, fmt.Errorf("no OS images found for location %q, series %q, architectures %q (and endpoint: %q)", location, series, arches, endpoint)
} else if err != nil {
return nil, err
}
return images, nil
}
func (s *localHTTPSServerSuite) TestFetchFromImageMetadataSources(c *gc.C) {
// Setup a custom URL for image metadata
customStorage := openstack.CreateCustomStorage(s.env, "custom-metadata")
customURL, err := customStorage.URL("")
c.Assert(err, gc.IsNil)
c.Check(customURL[:8], gc.Equals, "https://")
config, err := s.env.Config().Apply(
map[string]interface{}{"image-metadata-url": customURL},
)
c.Assert(err, gc.IsNil)
err = s.env.SetConfig(config)
c.Assert(err, gc.IsNil)
sources, err := imagemetadata.GetMetadataSources(s.env)
c.Assert(err, gc.IsNil)
c.Assert(sources, gc.HasLen, 4)
// Make sure there is something to download from each location
private := "private-content"
err = s.env.Storage().Put("images/"+private, bytes.NewBufferString(private), int64(len(private)))
c.Assert(err, gc.IsNil)
metadata := "metadata-content"
metadataStorage := openstack.ImageMetadataStorage(s.env)
err = metadataStorage.Put(metadata, bytes.NewBufferString(metadata), int64(len(metadata)))
c.Assert(err, gc.IsNil)
custom := "custom-content"
err = customStorage.Put(custom, bytes.NewBufferString(custom), int64(len(custom)))
c.Assert(err, gc.IsNil)
// Read from the Config entry's image-metadata-url
contentReader, url, err := sources[0].Fetch(custom)
c.Assert(err, gc.IsNil)
defer contentReader.Close()
content, err := ioutil.ReadAll(contentReader)
c.Assert(err, gc.IsNil)
c.Assert(string(content), gc.Equals, custom)
c.Check(url[:8], gc.Equals, "https://")
// Read from the private bucket
contentReader, url, err = sources[1].Fetch(private)
c.Assert(err, gc.IsNil)
defer contentReader.Close()
content, err = ioutil.ReadAll(contentReader)
c.Assert(err, gc.IsNil)
c.Check(string(content), gc.Equals, private)
c.Check(url[:8], gc.Equals, "https://")
// Check the entry we got from keystone
contentReader, url, err = sources[2].Fetch(metadata)
c.Assert(err, gc.IsNil)
defer contentReader.Close()
content, err = ioutil.ReadAll(contentReader)
c.Assert(err, gc.IsNil)
c.Assert(string(content), gc.Equals, metadata)
c.Check(url[:8], gc.Equals, "https://")
// Verify that we are pointing at exactly where metadataStorage thinks we are
metaURL, err := metadataStorage.URL(metadata)
c.Assert(err, gc.IsNil)
c.Check(url, gc.Equals, metaURL)
}
func (c *ValidateImageMetadataCommand) Run(context *cmd.Context) error {
var params *simplestreams.MetadataLookupParams
if c.providerType == "" {
store, err := configstore.Default()
if err != nil {
return err
}
environ, err := c.prepare(context, store)
if err != nil {
return err
}
mdLookup, ok := environ.(simplestreams.MetadataValidator)
if !ok {
return fmt.Errorf("%s provider does not support image metadata validation", environ.Config().Type())
}
params, err = mdLookup.MetadataLookupParams(c.region)
if err != nil {
return err
}
oes := &overrideEnvStream{environ, c.stream}
params.Sources, err = imagemetadata.GetMetadataSources(oes)
if err != nil {
return err
}
} else {
prov, err := environs.Provider(c.providerType)
if err != nil {
return err
}
mdLookup, ok := prov.(simplestreams.MetadataValidator)
if !ok {
return fmt.Errorf("%s provider does not support image metadata validation", c.providerType)
}
params, err = mdLookup.MetadataLookupParams(c.region)
if err != nil {
return err
}
}
if c.series != "" {
params.Series = c.series
}
if c.region != "" {
params.Region = c.region
}
if c.endpoint != "" {
params.Endpoint = c.endpoint
}
if c.metadataDir != "" {
dir := filepath.Join(c.metadataDir, "images")
if _, err := os.Stat(dir); err != nil {
return err
}
params.Sources = []simplestreams.DataSource{
simplestreams.NewURLDataSource(
"local metadata directory", "file://"+dir, utils.VerifySSLHostnames),
}
}
params.Stream = c.stream
image_ids, resolveInfo, err := imagemetadata.ValidateImageMetadata(params)
if err != nil {
if resolveInfo != nil {
metadata := map[string]interface{}{
"Resolve Metadata": *resolveInfo,
}
if metadataYaml, yamlErr := cmd.FormatYaml(metadata); yamlErr == nil {
err = fmt.Errorf("%v\n%v", err, string(metadataYaml))
}
}
return err
}
if len(image_ids) > 0 {
metadata := map[string]interface{}{
"ImageIds": image_ids,
"Region": params.Region,
"Resolve Metadata": *resolveInfo,
}
c.out.Write(context, metadata)
} else {
var sources []string
for _, s := range params.Sources {
url, err := s.URL("")
if err == nil {
sources = append(sources, fmt.Sprintf("- %s (%s)", s.Description(), url))
}
}
return fmt.Errorf(
"no matching image ids for region %s using sources:\n%s",
params.Region, strings.Join(sources, "\n"))
}
return nil
}