func generateMetadata(c *gc.C, stream string, versions ...version.Binary) []metadataFile { var metadata = make([]*tools.ToolsMetadata, len(versions)) for i, vers := range versions { basePath := fmt.Sprintf("%s/tools-%s.tar.gz", stream, vers.String()) metadata[i] = &tools.ToolsMetadata{ Release: vers.Series, Version: vers.Number.String(), Arch: vers.Arch, Path: basePath, } } var streamMetadata = map[string][]*tools.ToolsMetadata{ stream: metadata, } index, legacyIndex, products, err := tools.MarshalToolsMetadataJSON(streamMetadata, time.Now()) c.Assert(err, jc.ErrorIsNil) objects := []metadataFile{ {simplestreams.UnsignedIndex("v1", 2), index}, } if stream == "released" { objects = append(objects, metadataFile{simplestreams.UnsignedIndex("v1", 1), legacyIndex}) } for stream, metadata := range products { objects = append(objects, metadataFile{tools.ProductMetadataPath(stream), metadata}) } return objects }
// WriteMetadata writes the given tools metadata for the specified streams to the given storage. // streamMetadata contains all known metadata so that the correct index files can be written. // Only product files for the specified streams are written. func WriteMetadata(stor storage.Storage, streamMetadata map[string][]*ToolsMetadata, streams []string, writeMirrors ShouldWriteMirrors) error { // TODO(perrito666) 2016-05-02 lp:1558657 updated := time.Now() index, legacyIndex, products, err := MarshalToolsMetadataJSON(streamMetadata, updated) if err != nil { return err } metadataInfo := []MetadataFile{ {simplestreams.UnsignedIndex(currentStreamsVersion, IndexFileVersion), index}, } if legacyIndex != nil { metadataInfo = append(metadataInfo, MetadataFile{ simplestreams.UnsignedIndex(currentStreamsVersion, 1), legacyIndex, }) } for _, stream := range streams { if metadata, ok := products[stream]; ok { // If metadata hasn't changed, do not overwrite. unchanged, err := metadataUnchanged(stor, stream, metadata) if err != nil { return err } if unchanged { logger.Infof("Metadata for stream %q unchanged", stream) continue } // Metadata is different, so include it. metadataInfo = append(metadataInfo, MetadataFile{ProductMetadataPath(stream), metadata}) } } if writeMirrors { streamsMirrorsMetadata := make(map[string][]simplestreams.MirrorReference) for stream := range streamMetadata { streamsMirrorsMetadata[ToolsContentId(stream)] = []simplestreams.MirrorReference{{ Updated: updated.Format("20060102"), // YYYYMMDD DataType: ContentDownload, Format: simplestreams.MirrorFormat, Path: simplestreams.MirrorFile, }} } mirrorsMetadata := map[string]map[string][]simplestreams.MirrorReference{ "mirrors": streamsMirrorsMetadata, } mirrorsInfo, err := json.MarshalIndent(&mirrorsMetadata, "", " ") if err != nil { return err } metadataInfo = append( metadataInfo, MetadataFile{simplestreams.UnsignedMirror(currentStreamsVersion), mirrorsInfo}) } return writeMetadataFiles(stor, metadataInfo) }
// ParseMetadataFromStorage loads ImageMetadata from the specified storage reader. func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader) []*imagemetadata.ImageMetadata { source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "images") // Find the simplestreams index file. params := simplestreams.ValueParams{ DataType: "image-ids", ValueTemplate: imagemetadata.ImageMetadata{}, } const requireSigned = false indexPath := simplestreams.UnsignedIndex("v1", 1) mirrorsPath := simplestreams.MirrorsPath("v1") indexRef, err := simplestreams.GetIndexWithFormat( source, indexPath, "index:1.0", mirrorsPath, requireSigned, simplestreams.CloudSpec{}, params) c.Assert(err, jc.ErrorIsNil) c.Assert(indexRef.Indexes, gc.HasLen, 1) imageIndexMetadata := indexRef.Indexes["com.ubuntu.cloud:custom"] c.Assert(imageIndexMetadata, gc.NotNil) // Read the products file contents. r, err := stor.Get(path.Join("images", imageIndexMetadata.ProductsFilePath)) defer r.Close() c.Assert(err, jc.ErrorIsNil) data, err := ioutil.ReadAll(r) c.Assert(err, jc.ErrorIsNil) // Parse the products file metadata. url, err := source.URL(imageIndexMetadata.ProductsFilePath) c.Assert(err, jc.ErrorIsNil) cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, imagemetadata.ImageMetadata{}) c.Assert(err, jc.ErrorIsNil) // Collate the metadata. imageMetadataMap := make(map[string]*imagemetadata.ImageMetadata) expectedProductIds, imageVersions := make(set.Strings), make(set.Strings) for _, mc := range cloudMetadata.Products { for _, items := range mc.Items { for key, item := range items.Items { imageMetadata := item.(*imagemetadata.ImageMetadata) imageMetadataMap[key] = imageMetadata imageVersions.Add(key) productId := fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", mc.Version, imageMetadata.Arch) expectedProductIds.Add(productId) } } } // Make sure index's product IDs are all represented in the products metadata. sort.Strings(imageIndexMetadata.ProductIds) c.Assert(imageIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues()) imageMetadata := make([]*imagemetadata.ImageMetadata, len(imageMetadataMap)) for i, key := range imageVersions.SortedValues() { imageMetadata[i] = imageMetadataMap[key] } return imageMetadata }
func UseTestImageData(stor envstorage.Storage, cred *identity.Credentials) { // Put some image metadata files into the public storage. t := template.Must(template.New("").Parse(indexData)) var metadata bytes.Buffer if err := t.Execute(&metadata, cred); err != nil { panic(fmt.Errorf("cannot generate index metdata: %v", err)) } data := metadata.Bytes() stor.Put(simplestreams.UnsignedIndex("v1", 1), bytes.NewReader(data), int64(len(data))) stor.Put( productMetadatafile, strings.NewReader(imagesData), int64(len(imagesData))) }
func generateMetadata(c *gc.C, stream string, versions ...version.Binary) []metadataFile { var metadata = make([]*tools.ToolsMetadata, len(versions)) for i, vers := range versions { basePath := fmt.Sprintf("%s/tools-%s.tar.gz", stream, vers.String()) metadata[i] = &tools.ToolsMetadata{ Release: vers.Series, Version: vers.Number.String(), Arch: vers.Arch, Path: basePath, } } var streamMetadata = map[string][]*tools.ToolsMetadata{ stream: metadata, } // TODO(perrito666) 2016-05-02 lp:1558657 index, legacyIndex, products, err := tools.MarshalToolsMetadataJSON(streamMetadata, time.Now()) c.Assert(err, jc.ErrorIsNil) objects := []metadataFile{} addTools := func(fileName string, content []byte) { // add unsigned objects = append(objects, metadataFile{fileName, content}) signedFilename, signedContent, err := sstesting.SignMetadata(fileName, content) c.Assert(err, jc.ErrorIsNil) // add signed objects = append(objects, metadataFile{signedFilename, signedContent}) } addTools(simplestreams.UnsignedIndex("v1", 2), index) if stream == "released" { addTools(simplestreams.UnsignedIndex("v1", 1), legacyIndex) } for stream, metadata := range products { addTools(tools.ProductMetadataPath(stream), metadata) } return objects }
// ParseIndexMetadataFromStorage loads Indices from the specified storage reader. func ParseIndexMetadataFromStorage(c *gc.C, stor storage.StorageReader) (*simplestreams.IndexMetadata, simplestreams.DataSource) { source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "images", simplestreams.DEFAULT_CLOUD_DATA, false) // Find the simplestreams index file. params := simplestreams.ValueParams{ DataType: "image-ids", ValueTemplate: imagemetadata.ImageMetadata{}, } const requireSigned = false indexPath := simplestreams.UnsignedIndex("v1", 1) mirrorsPath := simplestreams.MirrorsPath("v1") indexRef, err := simplestreams.GetIndexWithFormat( source, indexPath, "index:1.0", mirrorsPath, requireSigned, simplestreams.CloudSpec{}, params) c.Assert(err, jc.ErrorIsNil) c.Assert(indexRef.Indexes, gc.HasLen, 1) imageIndexMetadata := indexRef.Indexes["com.ubuntu.cloud:custom"] c.Assert(imageIndexMetadata, gc.NotNil) return imageIndexMetadata, source }
// ParseMetadataFromStorage loads ToolsMetadata from the specified storage reader. func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader, stream string, expectMirrors bool) []*tools.ToolsMetadata { source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "tools") params := simplestreams.ValueParams{ DataType: tools.ContentDownload, ValueTemplate: tools.ToolsMetadata{}, } const requireSigned = false indexPath := simplestreams.UnsignedIndex("v1", 2) mirrorsPath := simplestreams.MirrorsPath("v1") indexRef, err := simplestreams.GetIndexWithFormat( source, indexPath, "index:1.0", mirrorsPath, requireSigned, simplestreams.CloudSpec{}, params) c.Assert(err, jc.ErrorIsNil) toolsIndexMetadata := indexRef.Indexes[tools.ToolsContentId(stream)] c.Assert(toolsIndexMetadata, gc.NotNil) // Read the products file contents. r, err := stor.Get(path.Join("tools", toolsIndexMetadata.ProductsFilePath)) defer r.Close() c.Assert(err, jc.ErrorIsNil) data, err := ioutil.ReadAll(r) c.Assert(err, jc.ErrorIsNil) url, err := source.URL(toolsIndexMetadata.ProductsFilePath) c.Assert(err, jc.ErrorIsNil) cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, tools.ToolsMetadata{}) c.Assert(err, jc.ErrorIsNil) toolsMetadataMap := make(map[string]*tools.ToolsMetadata) expectedProductIds := make(set.Strings) toolsVersions := make(set.Strings) for _, mc := range cloudMetadata.Products { for _, items := range mc.Items { for key, item := range items.Items { toolsMetadata := item.(*tools.ToolsMetadata) toolsMetadataMap[key] = toolsMetadata toolsVersions.Add(key) seriesVersion, err := series.SeriesVersion(toolsMetadata.Release) if err != nil { c.Assert(err, jc.Satisfies, series.IsUnknownSeriesVersionError) } productId := fmt.Sprintf("com.ubuntu.juju:%s:%s", seriesVersion, toolsMetadata.Arch) expectedProductIds.Add(productId) } } } // Make sure index's product IDs are all represented in the products metadata. sort.Strings(toolsIndexMetadata.ProductIds) c.Assert(toolsIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues()) toolsMetadata := make([]*tools.ToolsMetadata, len(toolsMetadataMap)) for i, key := range toolsVersions.SortedValues() { toolsMetadata[i] = toolsMetadataMap[key] } if expectMirrors { r, err = stor.Get(path.Join("tools", simplestreams.UnsignedMirror("v1"))) c.Assert(err, jc.ErrorIsNil) defer r.Close() data, err = ioutil.ReadAll(r) c.Assert(err, jc.ErrorIsNil) c.Assert(string(data), jc.Contains, `"mirrors":`) c.Assert(string(data), jc.Contains, tools.ToolsContentId(stream)) c.Assert(err, jc.ErrorIsNil) } return toolsMetadata }
func RemoveTestImageData(stor envstorage.Storage) { stor.Remove(simplestreams.UnsignedIndex("v1", 1)) stor.Remove(productMetadatafile) }
// IndexStoragePath returns the storage path for the image metadata index file. func IndexStoragePath() string { return path.Join(storage.BaseImagesPath, simplestreams.UnsignedIndex(currentStreamsVersion, IndexFileVersion)) }