// ParseMetadataFromStorage loads ImageMetadata from the specified storage reader.
func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader) []*imagemetadata.ImageMetadata {
source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "images")
// Find the simplestreams index file.
params := simplestreams.ValueParams{
DataType: "image-ids",
ValueTemplate: imagemetadata.ImageMetadata{},
}
const requireSigned = false
indexPath := simplestreams.UnsignedIndex
indexRef, err := simplestreams.GetIndexWithFormat(
source, indexPath, "index:1.0", requireSigned, simplestreams.CloudSpec{}, params)
c.Assert(err, gc.IsNil)
c.Assert(indexRef.Indexes, gc.HasLen, 1)
imageIndexMetadata := indexRef.Indexes["com.ubuntu.cloud:custom"]
c.Assert(imageIndexMetadata, gc.NotNil)
// Read the products file contents.
r, err := stor.Get(path.Join("images", imageIndexMetadata.ProductsFilePath))
defer r.Close()
c.Assert(err, gc.IsNil)
data, err := ioutil.ReadAll(r)
c.Assert(err, gc.IsNil)
// Parse the products file metadata.
url, err := source.URL(imageIndexMetadata.ProductsFilePath)
c.Assert(err, gc.IsNil)
cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, imagemetadata.ImageMetadata{})
c.Assert(err, gc.IsNil)
// Collate the metadata.
imageMetadataMap := make(map[string]*imagemetadata.ImageMetadata)
var expectedProductIds set.Strings
var imageVersions set.Strings
for _, mc := range cloudMetadata.Products {
for _, items := range mc.Items {
for key, item := range items.Items {
imageMetadata := item.(*imagemetadata.ImageMetadata)
imageMetadataMap[key] = imageMetadata
imageVersions.Add(key)
productId := fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", mc.Version, imageMetadata.Arch)
expectedProductIds.Add(productId)
}
}
}
// Make sure index's product IDs are all represented in the products metadata.
sort.Strings(imageIndexMetadata.ProductIds)
c.Assert(imageIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues())
imageMetadata := make([]*imagemetadata.ImageMetadata, len(imageMetadataMap))
for i, key := range imageVersions.SortedValues() {
imageMetadata[i] = imageMetadataMap[key]
}
return imageMetadata
}
func (s *datasourceSuite) TestFetch(c *gc.C) {
ds := simplestreams.NewURLDataSource("test", "test:", utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, false)
rc, url, err := ds.Fetch("streams/v1/tools_metadata.json")
c.Assert(err, jc.ErrorIsNil)
defer rc.Close()
c.Assert(url, gc.Equals, "test:/streams/v1/tools_metadata.json")
data, err := ioutil.ReadAll(rc)
cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, imagemetadata.ImageMetadata{})
c.Assert(err, jc.ErrorIsNil)
c.Assert(len(cloudMetadata.Products), jc.GreaterThan, 0)
}
// GetLatestImageIdMetadata is provided so it can be call by tests outside the imagemetadata package.
func GetLatestImageIdMetadata(data []byte, source simplestreams.DataSource, cons *ImageConstraint) ([]*ImageMetadata, error) {
metadata, err := simplestreams.ParseCloudMetadata(data, simplestreams.ProductFormat, "<unknown>", ImageMetadata{})
if err != nil {
return nil, err
}
items, err := simplestreams.GetLatestMetadata(metadata, cons, source, appendMatchingImages)
if err != nil {
return nil, err
}
result := make([]*ImageMetadata, len(items))
for i, md := range items {
result[i] = md.(*ImageMetadata)
}
return result, nil
}
// ParseMetadataFromStorage loads ImageMetadata from the specified storage reader.
func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader) []*imagemetadata.ImageMetadata {
imageIndexMetadata, source := ParseIndexMetadataFromStorage(c, stor)
c.Assert(imageIndexMetadata, gc.NotNil)
// Read the products file contents.
r, err := stor.Get(path.Join("images", imageIndexMetadata.ProductsFilePath))
defer r.Close()
c.Assert(err, jc.ErrorIsNil)
data, err := ioutil.ReadAll(r)
c.Assert(err, jc.ErrorIsNil)
// Parse the products file metadata.
url, err := source.URL(imageIndexMetadata.ProductsFilePath)
c.Assert(err, jc.ErrorIsNil)
cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, imagemetadata.ImageMetadata{})
c.Assert(err, jc.ErrorIsNil)
// Collate the metadata.
imageMetadataMap := make(map[string]*imagemetadata.ImageMetadata)
expectedProductIds, imageVersions := make(set.Strings), make(set.Strings)
for _, mc := range cloudMetadata.Products {
for _, items := range mc.Items {
for key, item := range items.Items {
imageMetadata := item.(*imagemetadata.ImageMetadata)
imageMetadataMap[key] = imageMetadata
imageVersions.Add(key)
productId := fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", mc.Version, imageMetadata.Arch)
expectedProductIds.Add(productId)
}
}
}
// Make sure index's product IDs are all represented in the products metadata.
sort.Strings(imageIndexMetadata.ProductIds)
c.Assert(imageIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues())
imageMetadata := make([]*imagemetadata.ImageMetadata, len(imageMetadataMap))
for i, key := range imageVersions.SortedValues() {
imageMetadata[i] = imageMetadataMap[key]
}
return imageMetadata
}
func (s *productSpecSuite) TestLargeNumber(c *gc.C) {
json := `{
"updated": "Fri, 30 Aug 2013 16:12:58 +0800",
"format": "products:1.0",
"products": {
"com.ubuntu.juju:1.10.0:amd64": {
"version": "1.10.0",
"arch": "amd64",
"versions": {
"20133008": {
"items": {
"1.10.0-precise-amd64": {
"release": "precise",
"version": "1.10.0",
"arch": "amd64",
"size": 9223372036854775807,
"path": "releases/juju-1.10.0-precise-amd64.tgz",
"ftype": "tar.gz",
"sha256": ""
}
}
}
}
}
}
}`
cloudMetadata, err := simplestreams.ParseCloudMetadata([]byte(json), "products:1.0", "", tools.ToolsMetadata{})
c.Assert(err, jc.ErrorIsNil)
c.Assert(cloudMetadata.Products, gc.HasLen, 1)
product := cloudMetadata.Products["com.ubuntu.juju:1.10.0:amd64"]
c.Assert(product, gc.NotNil)
c.Assert(product.Items, gc.HasLen, 1)
version := product.Items["20133008"]
c.Assert(version, gc.NotNil)
c.Assert(version.Items, gc.HasLen, 1)
item := version.Items["1.10.0-precise-amd64"]
c.Assert(item, gc.NotNil)
c.Assert(item, gc.FitsTypeOf, &tools.ToolsMetadata{})
c.Assert(item.(*tools.ToolsMetadata).Size, gc.Equals, int64(9223372036854775807))
}
// ParseMetadataFromStorage loads ToolsMetadata from the specified storage reader.
func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader, stream string, expectMirrors bool) []*tools.ToolsMetadata {
source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "tools")
params := simplestreams.ValueParams{
DataType: tools.ContentDownload,
ValueTemplate: tools.ToolsMetadata{},
}
const requireSigned = false
indexPath := simplestreams.UnsignedIndex("v1", 2)
mirrorsPath := simplestreams.MirrorsPath("v1")
indexRef, err := simplestreams.GetIndexWithFormat(
source, indexPath, "index:1.0", mirrorsPath, requireSigned, simplestreams.CloudSpec{}, params)
c.Assert(err, jc.ErrorIsNil)
toolsIndexMetadata := indexRef.Indexes[tools.ToolsContentId(stream)]
c.Assert(toolsIndexMetadata, gc.NotNil)
// Read the products file contents.
r, err := stor.Get(path.Join("tools", toolsIndexMetadata.ProductsFilePath))
defer r.Close()
c.Assert(err, jc.ErrorIsNil)
data, err := ioutil.ReadAll(r)
c.Assert(err, jc.ErrorIsNil)
url, err := source.URL(toolsIndexMetadata.ProductsFilePath)
c.Assert(err, jc.ErrorIsNil)
cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, tools.ToolsMetadata{})
c.Assert(err, jc.ErrorIsNil)
toolsMetadataMap := make(map[string]*tools.ToolsMetadata)
expectedProductIds := make(set.Strings)
toolsVersions := make(set.Strings)
for _, mc := range cloudMetadata.Products {
for _, items := range mc.Items {
for key, item := range items.Items {
toolsMetadata := item.(*tools.ToolsMetadata)
toolsMetadataMap[key] = toolsMetadata
toolsVersions.Add(key)
seriesVersion, err := series.SeriesVersion(toolsMetadata.Release)
if err != nil {
c.Assert(err, jc.Satisfies, series.IsUnknownSeriesVersionError)
}
productId := fmt.Sprintf("com.ubuntu.juju:%s:%s", seriesVersion, toolsMetadata.Arch)
expectedProductIds.Add(productId)
}
}
}
// Make sure index's product IDs are all represented in the products metadata.
sort.Strings(toolsIndexMetadata.ProductIds)
c.Assert(toolsIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues())
toolsMetadata := make([]*tools.ToolsMetadata, len(toolsMetadataMap))
for i, key := range toolsVersions.SortedValues() {
toolsMetadata[i] = toolsMetadataMap[key]
}
if expectMirrors {
r, err = stor.Get(path.Join("tools", simplestreams.UnsignedMirror("v1")))
c.Assert(err, jc.ErrorIsNil)
defer r.Close()
data, err = ioutil.ReadAll(r)
c.Assert(err, jc.ErrorIsNil)
c.Assert(string(data), jc.Contains, `"mirrors":`)
c.Assert(string(data), jc.Contains, tools.ToolsContentId(stream))
c.Assert(err, jc.ErrorIsNil)
}
return toolsMetadata
}