// GetMetadataSourcesWithRetries returns the sources to use when looking for
// simplestreams tools metadata. If env implements SupportsCustomSurces,
// the sources returned from that method will also be considered.
// The sources are configured to use retries according to the value of allowRetry.
func GetMetadataSourcesWithRetries(env environs.ConfigGetter, allowRetry bool) ([]simplestreams.DataSource, error) {
var sources []simplestreams.DataSource
config := env.Config()
if userURL, ok := config.ToolsURL(); ok {
verify := utils.VerifySSLHostnames
if !config.SSLHostnameVerification() {
verify = utils.NoVerifySSLHostnames
}
sources = append(sources, simplestreams.NewURLDataSource("tools-metadata-url", userURL, verify))
}
if custom, ok := env.(SupportsCustomSources); ok {
customSources, err := custom.GetToolsSources()
if err != nil {
return nil, err
}
sources = append(sources, customSources...)
}
defaultURL, err := ToolsURL(DefaultBaseURL)
if err != nil {
return nil, err
}
if defaultURL != "" {
sources = append(sources,
simplestreams.NewURLDataSource("default simplestreams", defaultURL, utils.VerifySSLHostnames))
}
for _, source := range sources {
source.SetAllowRetry(allowRetry)
}
return sources, nil
}
// GetMetadataSources returns the sources to use when looking for
// simplestreams image id metadata for the given stream. If env implements
// SupportsCustomSources, the sources returned from that method will also
// be considered.
func GetMetadataSources(env environs.ConfigGetter) ([]simplestreams.DataSource, error) {
var sources []simplestreams.DataSource
config := env.Config()
if userURL, ok := config.ImageMetadataURL(); ok {
verify := utils.VerifySSLHostnames
if !config.SSLHostnameVerification() {
verify = utils.NoVerifySSLHostnames
}
sources = append(sources, simplestreams.NewURLDataSource("image-metadata-url", userURL, verify))
}
if custom, ok := env.(SupportsCustomSources); ok {
customSources, err := custom.GetImageSources()
if err != nil {
return nil, err
}
sources = append(sources, customSources...)
}
defaultURL, err := ImageMetadataURL(DefaultBaseURL, config.ImageStream())
if err != nil {
return nil, err
}
if defaultURL != "" {
sources = append(sources,
simplestreams.NewURLDataSource("default cloud images", defaultURL, utils.VerifySSLHostnames))
}
return sources, nil
}
func (s *datasourceHTTPSSuite) TestNormalClientFails(c *gc.C) {
ds := simplestreams.NewURLDataSource("test", s.Server.URL, utils.VerifySSLHostnames)
url, err := ds.URL("bar")
c.Assert(err, gc.IsNil)
c.Check(url, gc.Equals, s.Server.URL+"/bar")
reader, _, err := ds.Fetch("bar")
// The underlying failure is a x509: certificate signed by unknown authority
// However, the urlDataSource abstraction hides that as a simple NotFound
c.Assert(err, gc.ErrorMatches, "invalid URL \".*/bar\" not found")
c.Check(reader, gc.IsNil)
}
func (s *datasourceSuite) TestFetch(c *gc.C) {
ds := simplestreams.NewURLDataSource("test", "test:", utils.VerifySSLHostnames)
rc, url, err := ds.Fetch("streams/v1/tools_metadata.json")
c.Assert(err, gc.IsNil)
defer rc.Close()
c.Assert(url, gc.Equals, "test:/streams/v1/tools_metadata.json")
data, err := ioutil.ReadAll(rc)
cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, imagemetadata.ImageMetadata{})
c.Assert(err, gc.IsNil)
c.Assert(len(cloudMetadata.Products), jc.GreaterThan, 0)
}
func (s *ValidateSuite) assertNoMatch(c *gc.C, stream string) {
s.makeLocalMetadata(c, "1234", "region-2", "raring", "some-auth-url", stream)
params := &simplestreams.MetadataLookupParams{
Region: "region-2",
Series: "precise",
Architectures: []string{"amd64"},
Endpoint: "some-auth-url",
Stream: stream,
Sources: []simplestreams.DataSource{
simplestreams.NewURLDataSource("test", "file://"+s.metadataDir, utils.VerifySSLHostnames)},
}
_, _, err := imagemetadata.ValidateImageMetadata(params)
c.Assert(err, gc.Not(gc.IsNil))
}
func (s *datasourceHTTPSSuite) TestNonVerifyingClientSucceeds(c *gc.C) {
ds := simplestreams.NewURLDataSource("test", s.Server.URL, utils.NoVerifySSLHostnames)
url, err := ds.URL("bar")
c.Assert(err, gc.IsNil)
c.Check(url, gc.Equals, s.Server.URL+"/bar")
reader, _, err := ds.Fetch("bar")
// The underlying failure is a x509: certificate signed by unknown authority
// However, the urlDataSource abstraction hides that as a simple NotFound
c.Assert(err, gc.IsNil)
defer reader.Close()
byteContent, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(string(byteContent), gc.Equals, "Greetings!\n")
}
func registerSimpleStreamsTests() {
gc.Suite(&simplestreamsSuite{
LocalLiveSimplestreamsSuite: sstesting.LocalLiveSimplestreamsSuite{
Source: simplestreams.NewURLDataSource("test", "test:", utils.VerifySSLHostnames),
RequireSigned: false,
DataType: "image-ids",
ValidConstraint: sstesting.NewTestConstraint(simplestreams.LookupParams{
CloudSpec: simplestreams.CloudSpec{
Region: "us-east-1",
Endpoint: "https://ec2.us-east-1.amazonaws.com",
},
Series: []string{"precise"},
Arches: []string{"amd64", "arm"},
}),
},
})
}
func (s *ValidateSuite) assertMatch(c *gc.C, stream string) {
s.makeLocalMetadata(c, "1234", "region-2", "raring", "some-auth-url", stream)
metadataPath := filepath.Join(s.metadataDir, "images")
params := &simplestreams.MetadataLookupParams{
Region: "region-2",
Series: "raring",
Architectures: []string{"amd64"},
Endpoint: "some-auth-url",
Stream: stream,
Sources: []simplestreams.DataSource{
simplestreams.NewURLDataSource("test", "file://"+metadataPath, utils.VerifySSLHostnames)},
}
imageIds, resolveInfo, err := imagemetadata.ValidateImageMetadata(params)
c.Assert(err, gc.IsNil)
c.Assert(imageIds, gc.DeepEquals, []string{"1234"})
c.Check(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{
Source: "test",
Signed: false,
IndexURL: "file://" + path.Join(metadataPath, "streams/v1/index.json"),
MirrorURL: "",
})
}
func (s *simplestreamsSuite) TestGetMetadataNoMatching(c *gc.C) {
source := &countingSource{
DataSource: simplestreams.NewURLDataSource(
"test", "test:/daily", utils.VerifySSLHostnames,
),
}
sources := []simplestreams.DataSource{source, source, source}
params := simplestreams.ValueParams{DataType: "image-ids"}
constraint := sstesting.NewTestConstraint(simplestreams.LookupParams{
CloudSpec: simplestreams.CloudSpec{
Region: "us-east-1",
Endpoint: "https://ec2.us-east-1.amazonaws.com",
},
Series: []string{"precise"},
Arches: []string{"not-a-real-arch"}, // never matches
})
items, resolveInfo, err := simplestreams.GetMetadata(
sources,
simplestreams.DefaultIndexPath,
constraint,
false,
params,
)
c.Assert(err, gc.IsNil)
c.Assert(items, gc.HasLen, 0)
c.Assert(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{
Source: "test",
Signed: false,
IndexURL: "test:/daily/streams/v1/index.json",
MirrorURL: "",
})
// There should be 2 calls to each data-source:
// one for .sjson, one for .json.
c.Assert(source.count, gc.Equals, 2*len(sources))
}
func (c *ValidateImageMetadataCommand) Run(context *cmd.Context) error {
var params *simplestreams.MetadataLookupParams
if c.providerType == "" {
store, err := configstore.Default()
if err != nil {
return err
}
environ, err := environs.PrepareFromName(c.EnvName, context, store)
if err != nil {
return err
}
mdLookup, ok := environ.(simplestreams.MetadataValidator)
if !ok {
return fmt.Errorf("%s provider does not support image metadata validation", environ.Config().Type())
}
params, err = mdLookup.MetadataLookupParams(c.region)
if err != nil {
return err
}
oes := &overrideEnvStream{environ, c.stream}
params.Sources, err = imagemetadata.GetMetadataSources(oes)
if err != nil {
return err
}
} else {
prov, err := environs.Provider(c.providerType)
if err != nil {
return err
}
mdLookup, ok := prov.(simplestreams.MetadataValidator)
if !ok {
return fmt.Errorf("%s provider does not support image metadata validation", c.providerType)
}
params, err = mdLookup.MetadataLookupParams(c.region)
if err != nil {
return err
}
}
if c.series != "" {
params.Series = c.series
}
if c.region != "" {
params.Region = c.region
}
if c.endpoint != "" {
params.Endpoint = c.endpoint
}
if c.metadataDir != "" {
dir := filepath.Join(c.metadataDir, "images")
if _, err := os.Stat(dir); err != nil {
return err
}
params.Sources = []simplestreams.DataSource{
simplestreams.NewURLDataSource(
"local metadata directory", "file://"+dir, utils.VerifySSLHostnames),
}
}
params.Stream = c.stream
image_ids, resolveInfo, err := imagemetadata.ValidateImageMetadata(params)
if err != nil {
if resolveInfo != nil {
metadata := map[string]interface{}{
"Resolve Metadata": *resolveInfo,
}
if metadataYaml, yamlErr := cmd.FormatYaml(metadata); yamlErr == nil {
err = fmt.Errorf("%v\n%v", err, string(metadataYaml))
}
}
return err
}
if len(image_ids) > 0 {
metadata := map[string]interface{}{
"ImageIds": image_ids,
"Region": params.Region,
"Resolve Metadata": *resolveInfo,
}
c.out.Write(context, metadata)
} else {
var sources []string
for _, s := range params.Sources {
url, err := s.URL("")
if err == nil {
sources = append(sources, fmt.Sprintf("- %s (%s)", s.Description(), url))
}
}
return fmt.Errorf(
"no matching image ids for region %s using sources:\n%s",
params.Region, strings.Join(sources, "\n"))
}
return nil
}
func (s *datasourceSuite) TestURL(c *gc.C) {
ds := simplestreams.NewURLDataSource("test", "foo", utils.VerifySSLHostnames)
url, err := ds.URL("bar")
c.Assert(err, gc.IsNil)
c.Assert(url, gc.Equals, "foo/bar")
}