Example #1
0
func (s *bootstrapSuite) TestBootstrapMetadataImagesMissing(c *gc.C) {
	environs.UnregisterImageDataSourceFunc("bootstrap metadata")

	noImagesDir := c.MkDir()
	stor, err := filestorage.NewFileStorageWriter(noImagesDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, "released", "released")

	env := newEnviron("foo", useDefaultKeys, nil)
	s.setDummyStorage(c, env)
	err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
		ControllerConfig: coretesting.FakeControllerConfig(),
		AdminSecret:      "admin-secret",
		CAPrivateKey:     coretesting.CAKey,
		MetadataDir:      noImagesDir,
	})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(env.bootstrapCount, gc.Equals, 1)

	datasources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(datasources, gc.HasLen, 2)
	c.Assert(datasources[0].Description(), gc.Equals, "default cloud images")
	c.Assert(datasources[1].Description(), gc.Equals, "default ubuntu cloud images")
}
func (s *ImageMetadataSuite) TestImageMetadataURLsRegisteredFuncs(c *gc.C) {
	environs.RegisterImageDataSourceFunc("id0", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id0", "betwixt/releases", utils.NoVerifySSLHostnames), nil
	})
	environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id1", "yoink", utils.NoVerifySSLHostnames), nil
	})
	// overwrite the one previously registered against id1
	environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
		return nil, errors.NewNotSupported(nil, "oyvey")
	})
	environs.RegisterUserImageDataSourceFunc("id2", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id2", "foobar", utils.NoVerifySSLHostnames), nil
	})
	defer environs.UnregisterImageDataSourceFunc("id0")
	defer environs.UnregisterImageDataSourceFunc("id1")
	defer environs.UnregisterImageDataSourceFunc("id2")

	env := s.env(c, "config-image-metadata-url", "")
	sources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	sstesting.AssertExpectedSources(c, sources, []string{
		"config-image-metadata-url/",
		"foobar/",
		"betwixt/releases/",
		"http://cloud-images.ubuntu.com/releases/",
	})
}
Example #3
0
func findImageMetadata(env *environ, args environs.StartInstanceParams) (*OvaFileMetadata, error) {
	arches := args.Tools.Arches()
	series := args.Tools.OneSeries()
	ic := &imagemetadata.ImageConstraint{
		LookupParams: simplestreams.LookupParams{
			Series: []string{series},
			Arches: arches,
			Stream: env.ecfg.ImageStream(),
		},
	}
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, errors.Trace(err)
	}

	matchingImages, err := imageMetadataFetch(sources, ic)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if len(matchingImages) == 0 {
		return nil, errors.Errorf("no matching images found for given constraints: %v", ic)
	}

	return matchingImages[0], nil
}
Example #4
0
// TestBootstrapImageMetadataFromAllSources tests that we are looking for
// image metadata in all data sources available to environment.
// Abandoning look up too soon led to misleading bootstrap failures:
// Juju reported no images available for a particular configuration,
// despite image metadata in other data sources compatible with the same configuration as well.
// Related to bug#1560625.
func (s *bootstrapSuite) TestBootstrapImageMetadataFromAllSources(c *gc.C) {
	s.PatchValue(&series.HostSeries, func() string { return "raring" })
	s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })

	// Ensure that we can find at least one image metadata
	// early on in the image metadata lookup.
	// We should continue looking despite it.
	metadataDir, _ := createImageMetadata(c)
	stor, err := filestorage.NewFileStorageWriter(metadataDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, "released", "released")

	env := bootstrapEnvironWithRegion{
		newEnviron("foo", useDefaultKeys, nil),
		simplestreams.CloudSpec{
			Region:   "region",
			Endpoint: "endpoint",
		},
	}
	s.setDummyStorage(c, env.bootstrapEnviron)

	bootstrapCons := constraints.MustParse("arch=amd64")
	err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
		BootstrapConstraints: bootstrapCons,
		MetadataDir:          metadataDir,
	})
	c.Assert(err, jc.ErrorIsNil)

	datasources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	for _, source := range datasources {
		// make sure we looked in each and all...
		c.Assert(c.GetTestLog(), jc.Contains, fmt.Sprintf("image metadata in %s", source.Description()))
	}
}
Example #5
0
func (s *bootstrapSuite) TestBootstrapMetadata(c *gc.C) {
	environs.UnregisterImageDataSourceFunc("bootstrap metadata")

	metadataDir, metadata := createImageMetadata(c)
	stor, err := filestorage.NewFileStorageWriter(metadataDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, "released", "released")

	env := newEnviron("foo", useDefaultKeys, nil)
	s.setDummyStorage(c, env)
	err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
		MetadataDir: metadataDir,
	})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(env.bootstrapCount, gc.Equals, 1)
	c.Assert(envtools.DefaultBaseURL, gc.Equals, metadataDir)

	datasources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(datasources, gc.HasLen, 3)
	c.Assert(datasources[0].Description(), gc.Equals, "bootstrap metadata")
	// This data source does not require to contain signed data.
	// However, it may still contain it.
	// Since we will always try to read signed data first,
	// we want to be able to try to read this signed data
	// with a user provided key.
	// for this test, user provided key is empty.
	// Bugs #1542127, #1542131
	c.Assert(datasources[0].PublicSigningKey(), gc.Equals, "")
	c.Assert(env.instanceConfig, gc.NotNil)
	c.Assert(env.instanceConfig.CustomImageMetadata, gc.HasLen, 1)
	c.Assert(env.instanceConfig.CustomImageMetadata[0], gc.DeepEquals, metadata[0])
}
func (s *ImageMetadataSuite) TestImageMetadataURLsRegisteredFuncs(c *gc.C) {
	environs.RegisterImageDataSourceFunc("id0", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id0", "betwixt/releases", utils.NoVerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, false), nil
	})
	environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id1", "yoink", utils.NoVerifySSLHostnames, simplestreams.SPECIFIC_CLOUD_DATA, false), nil
	})
	// overwrite the one previously registered against id1
	environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
		return nil, errors.NewNotSupported(nil, "oyvey")
	})
	environs.RegisterUserImageDataSourceFunc("id2", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id2", "foobar", utils.NoVerifySSLHostnames, simplestreams.CUSTOM_CLOUD_DATA, false), nil
	})
	defer environs.UnregisterImageDataSourceFunc("id0")
	defer environs.UnregisterImageDataSourceFunc("id1")
	defer environs.UnregisterImageDataSourceFunc("id2")

	env := s.env(c, "config-image-metadata-url", "")
	sources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	sstesting.AssertExpectedSources(c, sources, []sstesting.SourceDetails{
		{"config-image-metadata-url/", ""},
		{"foobar/", ""},
		{"betwixt/releases/", ""},
		{"https://streams.canonical.com/juju/images/releases/", imagemetadata.SimplestreamsImagesPublicKey},
		{"http://cloud-images.ubuntu.com/releases/", imagemetadata.SimplestreamsImagesPublicKey},
	})
}
Example #7
0
func (s *bootstrapSuite) TestBootstrapMetadata(c *gc.C) {
	environs.UnregisterImageDataSourceFunc("bootstrap metadata")

	metadataDir, metadata := createImageMetadata(c)
	stor, err := filestorage.NewFileStorageWriter(metadataDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, "released", "released")

	env := newEnviron("foo", useDefaultKeys, nil)
	s.setDummyStorage(c, env)
	err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
		MetadataDir: metadataDir,
	})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(env.bootstrapCount, gc.Equals, 1)
	c.Assert(envtools.DefaultBaseURL, gc.Equals, metadataDir)

	datasources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(datasources, gc.HasLen, 3)
	c.Assert(datasources[0].Description(), gc.Equals, "bootstrap metadata")
	c.Assert(env.instanceConfig, gc.NotNil)
	c.Assert(env.instanceConfig.CustomImageMetadata, gc.HasLen, 1)
	c.Assert(env.instanceConfig.CustomImageMetadata[0], gc.DeepEquals, metadata[0])
}
Example #8
0
func SetImageMetadata(env environs.Environ, series, arches []string, out *[]*imagemetadata.ImageMetadata) error {
	hasRegion, ok := env.(simplestreams.HasRegion)
	if !ok {
		return nil
	}
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return errors.Trace(err)
	}
	region, err := hasRegion.Region()
	if err != nil {
		return errors.Trace(err)
	}
	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: region,
		Series:    series,
		Arches:    arches,
		Stream:    env.Config().ImageStream(),
	})
	imageMetadata, _, err := imagemetadata.Fetch(sources, imageConstraint)
	if err != nil {
		return errors.Trace(err)
	}
	*out = imageMetadata
	return nil
}
Example #9
0
// FindInstanceSpec returns an InstanceSpec satisfying the supplied instanceConstraint.
func (env *joyentEnviron) FindInstanceSpec(ic *instances.InstanceConstraint) (*instances.InstanceSpec, error) {
	// Require at least one VCPU so we get KVM rather than smart package.
	if ic.Constraints.CpuCores == nil {
		ic.Constraints.CpuCores = &defaultCpuCores
	}
	allInstanceTypes, err := env.listInstanceTypes()
	if err != nil {
		return nil, err
	}
	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: simplestreams.CloudSpec{ic.Region, env.Ecfg().SdcUrl()},
		Series:    []string{ic.Series},
		Arches:    ic.Arches,
	})
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, err
	}

	matchingImages, _, err := imagemetadata.Fetch(sources, imageConstraint, signedImageDataOnly)
	if err != nil {
		return nil, err
	}
	images := instances.ImageMetadataToImages(matchingImages)
	spec, err := instances.FindInstanceSpec(images, ic, allInstanceTypes)
	if err != nil {
		return nil, err
	}
	return spec, nil
}
Example #10
0
func (s *ImageMetadataSuite) TestImageMetadataURLs(c *gc.C) {
	env := s.env(c, "config-image-metadata-url", "")
	sources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	sstesting.AssertExpectedSources(c, sources, []string{
		"config-image-metadata-url/", "http://cloud-images.ubuntu.com/releases/",
	})
}
Example #11
0
func (s *ImageMetadataSuite) TestImageMetadataURLsNonReleaseStream(c *gc.C) {
	env := s.env(c, "", "daily")
	sources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	sstesting.AssertExpectedSources(c, sources, []string{
		"http://cloud-images.ubuntu.com/daily/",
	})
}
Example #12
0
func (s *ImageMetadataSuite) TestImageMetadataURLsNoConfigURL(c *gc.C) {
	env := s.env(c, "", "")
	sources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	sstesting.AssertExpectedSources(c, sources, []string{
		"https://streams.canonical.com/juju/images/releases/",
		"http://cloud-images.ubuntu.com/releases/",
	})
}
Example #13
0
func (s *ImageMetadataSuite) TestImageMetadataURLsNonReleaseStream(c *gc.C) {
	env := s.env(c, "", "daily")
	sources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	sstesting.AssertExpectedSources(c, sources, []sstesting.SourceDetails{
		{"https://streams.canonical.com/juju/images/daily/", imagemetadata.SimplestreamsImagesPublicKey},
		{"http://cloud-images.ubuntu.com/daily/", imagemetadata.SimplestreamsImagesPublicKey},
	})
}
Example #14
0
func (s *ImageMetadataSuite) TestImageMetadataURLs(c *gc.C) {
	env := s.env(c, "config-image-metadata-url", "")
	sources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	sstesting.AssertExpectedSources(c, sources, []sstesting.SourceDetails{
		{"config-image-metadata-url/", ""},
		{"https://streams.canonical.com/juju/images/releases/", keys.JujuPublicKey},
		{"http://cloud-images.ubuntu.com/releases/", imagemetadata.SimplestreamsImagesPublicKey},
	})
}
Example #15
0
func (s *ImageMetadataSuite) TestImageMetadataURLsRegisteredFuncsError(c *gc.C) {
	environs.RegisterImageDataSourceFunc("id0", func(environs.Environ) (simplestreams.DataSource, error) {
		return nil, errors.New("oyvey!")
	})
	defer environs.UnregisterImageDataSourceFunc("id0")

	env := s.env(c, "config-image-metadata-url", "")
	_, err := environs.ImageMetadataSources(env)
	c.Assert(err, gc.ErrorMatches, "oyvey!")
}
func (c *validateImageMetadataCommand) createLookupParams(context *cmd.Context) (*simplestreams.MetadataLookupParams, error) {
	params := &simplestreams.MetadataLookupParams{Stream: c.stream}

	if c.providerType == "" {
		environ, err := c.prepare(context)
		if err != nil {
			return nil, err
		}
		mdLookup, ok := environ.(simplestreams.MetadataValidator)
		if !ok {
			return nil, fmt.Errorf("%s provider does not support image metadata validation", environ.Config().Type())
		}
		params, err = mdLookup.MetadataLookupParams(c.region)
		if err != nil {
			return nil, err
		}
		oes := &overrideEnvStream{environ, c.stream}
		params.Sources, err = environs.ImageMetadataSources(oes)
		if err != nil {
			return nil, err
		}
	} else {
		prov, err := environs.Provider(c.providerType)
		if err != nil {
			return nil, err
		}
		mdLookup, ok := prov.(simplestreams.MetadataValidator)
		if !ok {
			return nil, fmt.Errorf("%s provider does not support image metadata validation", c.providerType)
		}
		params, err = mdLookup.MetadataLookupParams(c.region)
		if err != nil {
			return nil, err
		}
	}

	if c.series != "" {
		params.Series = c.series
	}
	if c.region != "" {
		params.Region = c.region
	}
	if c.endpoint != "" {
		params.Endpoint = c.endpoint
	}
	if c.metadataDir != "" {
		dir := filepath.Join(c.metadataDir, "images")
		if _, err := os.Stat(dir); err != nil {
			return nil, err
		}
		params.Sources = imagesDataSources(dir)
	}
	return params, nil
}
Example #17
0
func (s *localServerSuite) TestValidateImageMetadata(c *gc.C) {
	env := s.Prepare(c)
	params, err := env.(simplestreams.MetadataValidator).MetadataLookupParams("some-region")
	c.Assert(err, jc.ErrorIsNil)
	params.Sources, err = environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	params.Series = "raring"
	image_ids, _, err := imagemetadata.ValidateImageMetadata(params)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(image_ids, gc.DeepEquals, []string{"11223344-0a0a-dd77-33cd-abcd1234e5f6"})
}
Example #18
0
func (t *localServerSuite) TestValidateImageMetadata(c *gc.C) {
	env := t.Prepare(c)
	params, err := env.(simplestreams.MetadataValidator).MetadataLookupParams("test")
	c.Assert(err, jc.ErrorIsNil)
	params.Series = coretesting.FakeDefaultSeries
	params.Endpoint = "https://ec2.endpoint.com"
	params.Sources, err = environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	image_ids, _, err := imagemetadata.ValidateImageMetadata(params)
	c.Assert(err, jc.ErrorIsNil)
	sort.Strings(image_ids)
	c.Assert(image_ids, gc.DeepEquals, []string{"ami-00000033", "ami-00000034", "ami-00000035", "ami-00000039"})
}
Example #19
0
// SupportedArchitectures returns all the image architectures for env matching the constraints.
func SupportedArchitectures(env environs.Environ, imageConstraint *imagemetadata.ImageConstraint) ([]string, error) {
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, err
	}
	matchingImages, _, err := imagemetadata.Fetch(sources, imageConstraint, false)
	if err != nil {
		return nil, err
	}
	var arches = set.NewStrings()
	for _, im := range matchingImages {
		arches.Add(im.Arch)
	}
	return arches.Values(), nil
}
Example #20
0
func (api *API) retrievePublished() error {
	envCfg, err := api.metadata.ModelConfig()
	if err != nil {
		return errors.Annotatef(err, "getting environ config")
	}
	env, err := environs.New(envCfg)
	if err != nil {
		return errors.Annotatef(err, "getting environ")
	}

	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return errors.Annotatef(err, "getting cloud specific image metadata sources")
	}

	cons := envmetadata.NewImageConstraint(simplestreams.LookupParams{})
	if inst, ok := env.(simplestreams.HasRegion); !ok {
		// Published image metadata for some providers are in simple streams.
		// Providers that do not rely on simplestreams, don't need to do anything here.
		return nil
	} else {
		// If we can determine current region,
		// we want only metadata specific to this region.
		cloud, err := inst.Region()
		if err != nil {
			return errors.Annotatef(err, "getting cloud specific region information")
		}
		cons.CloudSpec = cloud
	}

	// We want all relevant metadata from all data sources.
	for _, source := range sources {
		logger.Debugf("looking in data source %v", source.Description())
		metadata, info, err := envmetadata.Fetch([]simplestreams.DataSource{source}, cons)
		if err != nil {
			// Do not stop looking in other data sources if there is an issue here.
			logger.Errorf("encountered %v while getting published images metadata from %v", err, source.Description())
			continue
		}
		err = api.saveAll(info, source.Priority(), metadata)
		if err != nil {
			// Do not stop looking in other data sources if there is an issue here.
			logger.Errorf("encountered %v while saving published images metadata from %v", err, source.Description())
		}
	}
	return nil
}
Example #21
0
// findInstanceSpec returns an image and instance type satisfying the constraint.
// The instance type comes from querying the flavors supported by the deployment.
func findInstanceSpec(e *environ, ic *instances.InstanceConstraint) (*instances.InstanceSpec, error) {
	// first construct all available instance types from the supported flavors.
	nova := e.nova()
	flavors, err := nova.ListFlavorsDetail()
	if err != nil {
		return nil, err
	}
	allInstanceTypes := []instances.InstanceType{}
	for _, flavor := range flavors {
		instanceType := instances.InstanceType{
			Id:       flavor.Id,
			Name:     flavor.Name,
			Arches:   ic.Arches,
			Mem:      uint64(flavor.RAM),
			CpuCores: uint64(flavor.VCPUs),
			RootDisk: uint64(flavor.Disk * 1024),
			// tags not currently supported on openstack
		}
		allInstanceTypes = append(allInstanceTypes, instanceType)
	}

	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: simplestreams.CloudSpec{ic.Region, e.ecfg().authURL()},
		Series:    []string{ic.Series},
		Arches:    ic.Arches,
		Stream:    e.Config().ImageStream(),
	})
	sources, err := environs.ImageMetadataSources(e)
	if err != nil {
		return nil, err
	}
	// TODO (wallyworld): use an env parameter (default true) to mandate use of only signed image metadata.
	matchingImages, _, err := imagemetadata.Fetch(sources, imageConstraint, false)
	if err != nil {
		return nil, err
	}
	images := instances.ImageMetadataToImages(matchingImages)
	spec, err := instances.FindInstanceSpec(images, ic, allInstanceTypes)
	if err != nil {
		return nil, err
	}
	return spec, nil
}
Example #22
0
// findMatchingImages queries simplestreams for OS images that match the given
// requirements.
//
// If it finds no matching images, that's an error.
func findMatchingImages(e *azureEnviron, location, series string, arches []string) ([]*imagemetadata.ImageMetadata, error) {
	endpoint := getEndpoint(location)
	constraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: simplestreams.CloudSpec{location, endpoint},
		Series:    []string{series},
		Arches:    arches,
		Stream:    e.Config().ImageStream(),
	})
	sources, err := environs.ImageMetadataSources(e)
	if err != nil {
		return nil, err
	}
	images, _, err := imagemetadata.Fetch(sources, constraint, signedImageDataOnly)
	if len(images) == 0 || errors.IsNotFound(err) {
		return nil, fmt.Errorf("no OS images found for location %q, series %q, architectures %q (and endpoint: %q)", location, series, arches, endpoint)
	} else if err != nil {
		return nil, err
	}
	return images, nil
}
Example #23
0
func (env *environ) lookupArchitectures() ([]string, error) {
	// Create a filter to get all images for the
	// correct stream.
	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		Stream: env.Config().ImageStream(),
	})
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, errors.Trace(err)
	}
	matchingImages, err := imageMetadataFetch(sources, imageConstraint)
	if err != nil {
		return nil, errors.Trace(err)
	}

	var arches = set.NewStrings()
	for _, im := range matchingImages {
		arches.Add(im.Arch)
	}

	return arches.Values(), nil
}
Example #24
0
func (api *API) retrievePublished() ([]*envmetadata.ImageMetadata, error) {
	// Get environ
	envCfg, err := api.metadata.EnvironConfig()
	env, err := environs.New(envCfg)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// Get all images metadata sources for this environ.
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, err
	}

	// We want all metadata.
	cons := envmetadata.NewImageConstraint(simplestreams.LookupParams{})
	metadata, _, err := envmetadata.Fetch(sources, cons, false)
	if err != nil {
		return nil, err
	}
	return metadata, nil
}
Example #25
0
func (env *environ) getImageSources() ([]lxdclient.Remote, error) {
	metadataSources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, errors.Trace(err)
	}
	remotes := make([]lxdclient.Remote, 0)
	for _, source := range metadataSources {
		url, err := source.URL("")
		if err != nil {
			logger.Debugf("failed to get the URL for metadataSource: %s", err)
			continue
		}
		// NOTE(jam) LXD only allows you to pass HTTPS URLs. So strip
		// off http:// and replace it with https://
		// Arguably we could give the user a direct error if
		// env.ImageMetadataURL is http instead of https, but we also
		// get http from the DefaultImageSources, which is why we
		// replace it.
		// TODO(jam) Maybe we could add a Validate step that ensures
		// image-metadata-url is an "https://" URL, so that Users get a
		// "your configuration is wrong" error, rather than silently
		// changing it and having them get confused.
		// https://github.com/lxc/lxd/issues/1763
		if strings.HasPrefix(url, "http://") {
			url = strings.TrimPrefix(url, "http://")
			url = "https://" + url
			logger.Debugf("LXD requires https://, using: %s", url)
		}
		remotes = append(remotes, lxdclient.Remote{
			Name:          source.Description(),
			Host:          url,
			Protocol:      lxdclient.SimplestreamsProtocol,
			Cert:          nil,
			ServerPEMCert: "",
		})
	}
	return remotes, nil
}
Example #26
0
// findInstanceSpec initializes a new instance spec for the given stream
// (and constraints) and returns it. This only covers populating the
// initial data for the spec. However, it does include fetching the
// correct simplestreams image data.
func (env *environ) findInstanceSpec(stream string, ic *instances.InstanceConstraint) (*instances.InstanceSpec, error) {
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, errors.Trace(err)
	}

	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: env.cloudSpec(ic.Region),
		Series:    []string{ic.Series},
		Arches:    ic.Arches,
		Stream:    stream,
	})

	signedImageDataOnly := false
	matchingImages, _, err := imageMetadataFetch(sources, imageConstraint, signedImageDataOnly)
	if err != nil {
		return nil, errors.Trace(err)
	}

	images := instances.ImageMetadataToImages(matchingImages)
	spec, err := instances.FindInstanceSpec(images, ic, allInstanceTypes)
	return spec, errors.Trace(err)
}
Example #27
0
// bootstrapImageMetadata returns the image metadata to use for bootstrapping
// the given environment. If the environment provider does not make use of
// simplestreams, no metadata will be returned.
//
// If a bootstrap image ID is specified, image metadata will be synthesised
// using that image ID, and the architecture and series specified by the
// initiator. In addition, the custom image metadata that is saved into the
// state database will have the synthesised image metadata added to it.
func bootstrapImageMetadata(
	environ environs.Environ,
	availableTools coretools.List,
	bootstrapImageId string,
	customImageMetadata *[]*imagemetadata.ImageMetadata,
) ([]*imagemetadata.ImageMetadata, error) {

	hasRegion, ok := environ.(simplestreams.HasRegion)
	if !ok {
		if bootstrapImageId != "" {
			// We only support specifying image IDs for providers
			// that use simplestreams for now.
			return nil, errors.NotSupportedf(
				"specifying bootstrap image for %q provider",
				environ.Config().Type(),
			)
		}
		// No region, no metadata.
		return nil, nil
	}
	region, err := hasRegion.Region()
	if err != nil {
		return nil, errors.Trace(err)
	}

	if bootstrapImageId != "" {
		arches := availableTools.Arches()
		if len(arches) != 1 {
			return nil, errors.NotValidf("multiple architectures with bootstrap image")
		}
		allSeries := availableTools.AllSeries()
		if len(allSeries) != 1 {
			return nil, errors.NotValidf("multiple series with bootstrap image")
		}
		seriesVersion, err := series.SeriesVersion(allSeries[0])
		if err != nil {
			return nil, errors.Trace(err)
		}
		// The returned metadata does not have information about the
		// storage or virtualisation type. Any provider that wants to
		// filter on those properties should allow for empty values.
		meta := &imagemetadata.ImageMetadata{
			Id:         bootstrapImageId,
			Arch:       arches[0],
			Version:    seriesVersion,
			RegionName: region.Region,
			Endpoint:   region.Endpoint,
			Stream:     environ.Config().ImageStream(),
		}
		*customImageMetadata = append(*customImageMetadata, meta)
		return []*imagemetadata.ImageMetadata{meta}, nil
	}

	// For providers that support making use of simplestreams
	// image metadata, search public image metadata. We need
	// to pass this onto Bootstrap for selecting images.
	sources, err := environs.ImageMetadataSources(environ)
	if err != nil {
		return nil, errors.Trace(err)
	}
	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: region,
		Series:    availableTools.AllSeries(),
		Arches:    availableTools.Arches(),
		Stream:    environ.Config().ImageStream(),
	})
	logger.Debugf("constraints for image metadata lookup %v", imageConstraint)

	// Get image metadata from all data sources.
	// Since order of data source matters, order of image metadata matters too. Append is important here.
	var publicImageMetadata []*imagemetadata.ImageMetadata
	for _, source := range sources {
		sourceMetadata, _, err := imagemetadata.Fetch([]simplestreams.DataSource{source}, imageConstraint)
		if err != nil {
			logger.Debugf("ignoring image metadata in %s: %v", source.Description(), err)
			// Just keep looking...
			continue
		}
		logger.Debugf("found %d image metadata in %s", len(sourceMetadata), source.Description())
		publicImageMetadata = append(publicImageMetadata, sourceMetadata...)
	}

	logger.Debugf("found %d image metadata from all image data sources", len(publicImageMetadata))
	if len(publicImageMetadata) == 0 {
		return nil, errors.New("no image metadata found")
	}
	return publicImageMetadata, nil
}
Example #28
0
// bootstrapImageMetadata returns the image metadata to use for bootstrapping
// the given environment. If the environment provider does not make use of
// simplestreams, no metadata will be returned.
//
// If a bootstrap image ID is specified, image metadat will be synthesised
// using that image ID, and the architecture and series specified by the
// initiator. In addition, the custom image metadat that is saved into the
// state database will have the synthesised image metadata added to it.
func bootstrapImageMetadata(
	environ environs.Environ,
	availableTools coretools.List,
	bootstrapImageId string,
	customImageMetadata *[]*imagemetadata.ImageMetadata,
) ([]*imagemetadata.ImageMetadata, error) {

	hasRegion, ok := environ.(simplestreams.HasRegion)
	if !ok {
		if bootstrapImageId != "" {
			// We only support specifying image IDs for providers
			// that use simplestreams for now.
			return nil, errors.NotSupportedf(
				"specifying bootstrap image for %q provider",
				environ.Config().Type(),
			)
		}
		// No region, no metadata.
		return nil, nil
	}
	region, err := hasRegion.Region()
	if err != nil {
		return nil, errors.Trace(err)
	}

	if bootstrapImageId != "" {
		arches := availableTools.Arches()
		if len(arches) != 1 {
			return nil, errors.NotValidf("multiple architectures with bootstrap image")
		}
		allSeries := availableTools.AllSeries()
		if len(allSeries) != 1 {
			return nil, errors.NotValidf("multiple series with bootstrap image")
		}
		seriesVersion, err := series.SeriesVersion(allSeries[0])
		if err != nil {
			return nil, errors.Trace(err)
		}
		// The returned metadata does not have information about the
		// storage or virtualisation type. Any provider that wants to
		// filter on those properties should allow for empty values.
		meta := &imagemetadata.ImageMetadata{
			Id:         bootstrapImageId,
			Arch:       arches[0],
			Version:    seriesVersion,
			RegionName: region.Region,
			Endpoint:   region.Endpoint,
			Stream:     environ.Config().ImageStream(),
		}
		*customImageMetadata = append(*customImageMetadata, meta)
		return []*imagemetadata.ImageMetadata{meta}, nil
	}

	// For providers that support making use of simplestreams
	// image metadata, search public image metadata. We need
	// to pass this onto Bootstrap for selecting images.
	sources, err := environs.ImageMetadataSources(environ)
	if err != nil {
		return nil, errors.Trace(err)
	}
	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: region,
		Series:    availableTools.AllSeries(),
		Arches:    availableTools.Arches(),
		Stream:    environ.Config().ImageStream(),
	})
	publicImageMetadata, _, err := imagemetadata.Fetch(sources, imageConstraint)
	if err != nil {
		return nil, errors.Annotate(err, "searching image metadata")
	}
	return publicImageMetadata, nil
}
Example #29
0
	"github.com/juju/juju/cloudconfig/instancecfg"
	"github.com/juju/juju/cloudconfig/providerinit"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/environs/simplestreams"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/tools"
)

//
// Imlementation of InstanceBroker: methods for starting and stopping instances.
//

var findInstanceImage = func(env *environ, ic *imagemetadata.ImageConstraint) (*imagemetadata.ImageMetadata, error) {

	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, err
	}

	matchingImages, _, err := imagemetadata.Fetch(sources, ic, false)
	if err != nil {
		return nil, err
	}
	if len(matchingImages) == 0 {
		return nil, errors.New("no matching image meta data")
	}

	return matchingImages[0], nil
}
Example #30
0
// imageMetadataFromDataSources finds image metadata that match specified criteria in existing data sources.
func (p *ProvisionerAPI) imageMetadataFromDataSources(env environs.Environ, constraint *imagemetadata.ImageConstraint) ([]params.CloudImageMetadata, error) {
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, errors.Trace(err)
	}

	cfg := env.Config()
	toModel := func(m *imagemetadata.ImageMetadata, mSeries string, source string, priority int) cloudimagemetadata.Metadata {
		result := cloudimagemetadata.Metadata{
			MetadataAttributes: cloudimagemetadata.MetadataAttributes{
				Region:          m.RegionName,
				Arch:            m.Arch,
				VirtType:        m.VirtType,
				RootStorageType: m.Storage,
				Source:          source,
				Series:          mSeries,
				Stream:          m.Stream,
				Version:         m.Version,
			},
			Priority: priority,
			ImageId:  m.Id,
		}
		// TODO (anastasiamac 2016-08-24) This is a band-aid solution.
		// Once correct value is read from simplestreams, this needs to go.
		// Bug# 1616295
		if result.Stream == "" {
			result.Stream = constraint.Stream
		}
		if result.Stream == "" {
			result.Stream = cfg.ImageStream()
		}
		return result
	}

	var metadataState []cloudimagemetadata.Metadata
	for _, source := range sources {
		logger.Debugf("looking in data source %v", source.Description())
		found, info, err := imagemetadata.Fetch([]simplestreams.DataSource{source}, constraint)
		if err != nil {
			// Do not stop looking in other data sources if there is an issue here.
			logger.Warningf("encountered %v while getting published images metadata from %v", err, source.Description())
			continue
		}
		for _, m := range found {
			mSeries, err := series.VersionSeries(m.Version)
			if err != nil {
				logger.Warningf("could not determine series for image id %s: %v", m.Id, err)
				continue
			}
			metadataState = append(metadataState, toModel(m, mSeries, info.Source, source.Priority()))
		}
	}
	if len(metadataState) > 0 {
		if err := p.st.CloudImageMetadataStorage.SaveMetadata(metadataState); err != nil {
			// No need to react here, just take note
			logger.Warningf("failed to save published image metadata: %v", err)
		}
	}

	// Since we've fallen through to data sources search and have saved all needed images into controller,
	// let's try to get them from controller to avoid duplication of conversion logic here.
	all, err := p.imageMetadataFromState(constraint)
	if err != nil {
		return nil, errors.Annotate(err, "could not read metadata from controller after saving it there from data sources")
	}

	if len(all) == 0 {
		return nil, errors.NotFoundf("image metadata for series %v, arch %v", constraint.Series, constraint.Arches)
	}

	return all, nil
}