func checkFileHasContents(c *gc.C, stor storage.StorageReader, name string, contents []byte, attempt utils.AttemptStrategy) { r, err := storage.GetWithRetry(stor, name, attempt) c.Assert(err, gc.IsNil) c.Check(r, gc.NotNil) defer r.Close() data, err := ioutil.ReadAll(r) c.Check(err, gc.IsNil) c.Check(data, gc.DeepEquals, contents) url, err := stor.URL(name) c.Assert(err, gc.IsNil) var resp *http.Response for a := attempt.Start(); a.Next(); { resp, err = utils.GetValidatingHTTPClient().Get(url) c.Assert(err, gc.IsNil) if resp.StatusCode != 404 { break } c.Logf("get retrying after earlier get succeeded. *sigh*.") } c.Assert(err, gc.IsNil) data, err = ioutil.ReadAll(resp.Body) c.Assert(err, gc.IsNil) defer resp.Body.Close() c.Assert(resp.StatusCode, gc.Equals, 200, gc.Commentf("error response: %s", data)) c.Check(data, gc.DeepEquals, contents) }
func readImageMetadata(stor storage.StorageReader, path string) ([]byte, error) { r, err := stor.Get(path) if err != nil { return nil, err } defer r.Close() return ioutil.ReadAll(r) }
// ParseMetadataFromStorage loads ImageMetadata from the specified storage reader. func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader) []*imagemetadata.ImageMetadata { source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "images") // Find the simplestreams index file. params := simplestreams.ValueParams{ DataType: "image-ids", ValueTemplate: imagemetadata.ImageMetadata{}, } const requireSigned = false indexPath := simplestreams.UnsignedIndex indexRef, err := simplestreams.GetIndexWithFormat( source, indexPath, "index:1.0", requireSigned, simplestreams.CloudSpec{}, params) c.Assert(err, gc.IsNil) c.Assert(indexRef.Indexes, gc.HasLen, 1) imageIndexMetadata := indexRef.Indexes["com.ubuntu.cloud:custom"] c.Assert(imageIndexMetadata, gc.NotNil) // Read the products file contents. r, err := stor.Get(path.Join("images", imageIndexMetadata.ProductsFilePath)) defer r.Close() c.Assert(err, gc.IsNil) data, err := ioutil.ReadAll(r) c.Assert(err, gc.IsNil) // Parse the products file metadata. url, err := source.URL(imageIndexMetadata.ProductsFilePath) c.Assert(err, gc.IsNil) cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, imagemetadata.ImageMetadata{}) c.Assert(err, gc.IsNil) // Collate the metadata. imageMetadataMap := make(map[string]*imagemetadata.ImageMetadata) var expectedProductIds set.Strings var imageVersions set.Strings for _, mc := range cloudMetadata.Products { for _, items := range mc.Items { for key, item := range items.Items { imageMetadata := item.(*imagemetadata.ImageMetadata) imageMetadataMap[key] = imageMetadata imageVersions.Add(key) productId := fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", mc.Version, imageMetadata.Arch) expectedProductIds.Add(productId) } } } // Make sure index's product IDs are all represented in the products metadata. sort.Strings(imageIndexMetadata.ProductIds) c.Assert(imageIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues()) imageMetadata := make([]*imagemetadata.ImageMetadata, len(imageMetadataMap)) for i, key := range imageVersions.SortedValues() { imageMetadata[i] = imageMetadataMap[key] } return imageMetadata }
// ReadList returns a List of the tools in store with the given major.minor version. // If minorVersion = -1, then only majorVersion is considered. // If store contains no such tools, it returns ErrNoMatches. func ReadList(stor storage.StorageReader, toolsDir string, majorVersion, minorVersion int) (coretools.List, error) { if minorVersion >= 0 { logger.Debugf("reading v%d.%d tools", majorVersion, minorVersion) } else { logger.Debugf("reading v%d.* tools", majorVersion) } storagePrefix := storagePrefix(toolsDir) names, err := storage.List(stor, storagePrefix) if err != nil { return nil, err } var list coretools.List var foundAnyTools bool for _, name := range names { name = filepath.ToSlash(name) if !strings.HasPrefix(name, storagePrefix) || !strings.HasSuffix(name, toolSuffix) { continue } var t coretools.Tools vers := name[len(storagePrefix) : len(name)-len(toolSuffix)] if t.Version, err = version.ParseBinary(vers); err != nil { logger.Debugf("failed to parse version %q: %v", vers, err) continue } foundAnyTools = true // Major version must match specified value. if t.Version.Major != majorVersion { continue } // If specified minor version value supplied, minor version must match. if minorVersion >= 0 && t.Version.Minor != minorVersion { continue } logger.Debugf("found %s", vers) if t.URL, err = stor.URL(name); err != nil { return nil, err } list = append(list, &t) // Older versions of Juju only know about ppc64, so add metadata for that arch. if t.Version.Arch == arch.PPC64EL { legacyPPC64Tools := t legacyPPC64Tools.Version.Arch = arch.LEGACY_PPC64 list = append(list, &legacyPPC64Tools) } } if len(list) == 0 { if foundAnyTools { return nil, coretools.ErrNoMatches } return nil, ErrNoTools } return list, nil }
// ReadList returns a List of the tools in store with the given major.minor version. // If minorVersion = -1, then only majorVersion is considered. // If store contains no such tools, it returns ErrNoMatches. func ReadList(stor storage.StorageReader, majorVersion, minorVersion int) (coretools.List, error) { if minorVersion >= 0 { logger.Debugf("reading v%d.%d tools", majorVersion, minorVersion) } else { logger.Debugf("reading v%d.* tools", majorVersion) } names, err := storage.List(stor, toolPrefix) if err != nil { return nil, err } var list coretools.List var foundAnyTools bool for _, name := range names { if !strings.HasPrefix(name, toolPrefix) || !strings.HasSuffix(name, toolSuffix) { continue } var t coretools.Tools vers := name[len(toolPrefix) : len(name)-len(toolSuffix)] if t.Version, err = version.ParseBinary(vers); err != nil { logger.Debugf("failed to parse version %q: %v", vers, err) continue } foundAnyTools = true // Major version must match specified value. if t.Version.Major != majorVersion { continue } // If specified minor version value supplied, minor version must match. if minorVersion >= 0 && t.Version.Minor != minorVersion { continue } logger.Debugf("found %s", vers) if t.URL, err = stor.URL(name); err != nil { return nil, err } list = append(list, &t) } if len(list) == 0 { if foundAnyTools { return nil, coretools.ErrNoMatches } return nil, ErrNoTools } return list, nil }
func checkFileHasContents(c *gc.C, stor storage.StorageReader, name string, contents []byte) { r, err := storage.Get(stor, name) c.Assert(err, gc.IsNil) c.Check(r, gc.NotNil) defer r.Close() data, err := ioutil.ReadAll(r) c.Check(err, gc.IsNil) c.Check(data, gc.DeepEquals, contents) url, err := stor.URL(name) c.Assert(err, gc.IsNil) resp, err := http.Get(url) c.Assert(err, gc.IsNil) data, err = ioutil.ReadAll(resp.Body) c.Assert(err, gc.IsNil) defer resp.Body.Close() c.Assert(resp.StatusCode, gc.Equals, http.StatusOK, gc.Commentf("error response: %s", data)) c.Check(data, gc.DeepEquals, contents) }
// migrateCustomImageMetadata copies uploaded image metadata from provider // storage to environment storage, preserving paths. func migrateCustomImageMetadata(st *state.State, agentConfig agent.Config) error { logger.Debugf("migrating custom image metadata to environment storage") estor := newStateStorage(st.EnvironUUID(), st.MongoSession()) // Local and manual provider host storage on the state server's // filesystem, and serve via HTTP storage. The storage worker // doesn't run yet, so we just open the files directly. var pstor storage.StorageReader providerType := agentConfig.Value(agent.ProviderType) if providerType == provider.Local || provider.IsManual(providerType) { storageDir := agentConfig.Value(agent.StorageDir) var err error pstor, err = filestorage.NewFileStorageReader(storageDir) if err != nil { return errors.Annotate(err, "cannot get local filesystem storage reader") } } else { var err error pstor, err = environs.LegacyStorage(st) if errors.IsNotSupported(err) { return nil } else if err != nil { return errors.Annotate(err, "cannot get provider storage") } } paths, err := pstor.List(storage.BaseImagesPath) if err != nil { return err } for _, path := range paths { logger.Infof("migrating image metadata at path %q", path) data, err := readImageMetadata(pstor, path) if err != nil { return errors.Annotate(err, "failed to read image metadata") } err = estor.Put(path, bytes.NewReader(data), int64(len(data))) if err != nil { return errors.Annotate(err, "failed to write image metadata") } } return nil }
// ParseMetadataFromStorage loads ImageMetadata from the specified storage reader. func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader) []*imagemetadata.ImageMetadata { imageIndexMetadata, source := ParseIndexMetadataFromStorage(c, stor) c.Assert(imageIndexMetadata, gc.NotNil) // Read the products file contents. r, err := stor.Get(path.Join("images", imageIndexMetadata.ProductsFilePath)) defer r.Close() c.Assert(err, jc.ErrorIsNil) data, err := ioutil.ReadAll(r) c.Assert(err, jc.ErrorIsNil) // Parse the products file metadata. url, err := source.URL(imageIndexMetadata.ProductsFilePath) c.Assert(err, jc.ErrorIsNil) cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, imagemetadata.ImageMetadata{}) c.Assert(err, jc.ErrorIsNil) // Collate the metadata. imageMetadataMap := make(map[string]*imagemetadata.ImageMetadata) expectedProductIds, imageVersions := make(set.Strings), make(set.Strings) for _, mc := range cloudMetadata.Products { for _, items := range mc.Items { for key, item := range items.Items { imageMetadata := item.(*imagemetadata.ImageMetadata) imageMetadataMap[key] = imageMetadata imageVersions.Add(key) productId := fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", mc.Version, imageMetadata.Arch) expectedProductIds.Add(productId) } } } // Make sure index's product IDs are all represented in the products metadata. sort.Strings(imageIndexMetadata.ProductIds) c.Assert(imageIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues()) imageMetadata := make([]*imagemetadata.ImageMetadata, len(imageMetadataMap)) for i, key := range imageVersions.SortedValues() { imageMetadata[i] = imageMetadataMap[key] } return imageMetadata }
func fetchToolsArchive(stor storage.StorageReader, toolsDir string, agentTools *tools.Tools) ([]byte, error) { r, err := stor.Get(envtools.StorageName(agentTools.Version, toolsDir)) if err != nil { return nil, err } defer r.Close() var buf bytes.Buffer hash, size, err := utils.ReadSHA256(io.TeeReader(r, &buf)) if err != nil { return nil, err } if hash != agentTools.SHA256 { return nil, errors.New("hash mismatch") } if size != agentTools.Size { return nil, errors.New("size mismatch") } return buf.Bytes(), nil }
// ParseMetadataFromStorage loads ToolsMetadata from the specified storage reader. func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader, stream string, expectMirrors bool) []*tools.ToolsMetadata { source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "tools") params := simplestreams.ValueParams{ DataType: tools.ContentDownload, ValueTemplate: tools.ToolsMetadata{}, } const requireSigned = false indexPath := simplestreams.UnsignedIndex("v1", 2) mirrorsPath := simplestreams.MirrorsPath("v1") indexRef, err := simplestreams.GetIndexWithFormat( source, indexPath, "index:1.0", mirrorsPath, requireSigned, simplestreams.CloudSpec{}, params) c.Assert(err, jc.ErrorIsNil) toolsIndexMetadata := indexRef.Indexes[tools.ToolsContentId(stream)] c.Assert(toolsIndexMetadata, gc.NotNil) // Read the products file contents. r, err := stor.Get(path.Join("tools", toolsIndexMetadata.ProductsFilePath)) defer r.Close() c.Assert(err, jc.ErrorIsNil) data, err := ioutil.ReadAll(r) c.Assert(err, jc.ErrorIsNil) url, err := source.URL(toolsIndexMetadata.ProductsFilePath) c.Assert(err, jc.ErrorIsNil) cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, tools.ToolsMetadata{}) c.Assert(err, jc.ErrorIsNil) toolsMetadataMap := make(map[string]*tools.ToolsMetadata) expectedProductIds := make(set.Strings) toolsVersions := make(set.Strings) for _, mc := range cloudMetadata.Products { for _, items := range mc.Items { for key, item := range items.Items { toolsMetadata := item.(*tools.ToolsMetadata) toolsMetadataMap[key] = toolsMetadata toolsVersions.Add(key) seriesVersion, err := series.SeriesVersion(toolsMetadata.Release) if err != nil { c.Assert(err, jc.Satisfies, series.IsUnknownSeriesVersionError) } productId := fmt.Sprintf("com.ubuntu.juju:%s:%s", seriesVersion, toolsMetadata.Arch) expectedProductIds.Add(productId) } } } // Make sure index's product IDs are all represented in the products metadata. sort.Strings(toolsIndexMetadata.ProductIds) c.Assert(toolsIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues()) toolsMetadata := make([]*tools.ToolsMetadata, len(toolsMetadataMap)) for i, key := range toolsVersions.SortedValues() { toolsMetadata[i] = toolsMetadataMap[key] } if expectMirrors { r, err = stor.Get(path.Join("tools", simplestreams.UnsignedMirror("v1"))) c.Assert(err, jc.ErrorIsNil) defer r.Close() data, err = ioutil.ReadAll(r) c.Assert(err, jc.ErrorIsNil) c.Assert(string(data), jc.Contains, `"mirrors":`) c.Assert(string(data), jc.Contains, tools.ToolsContentId(stream)) c.Assert(err, jc.ErrorIsNil) } return toolsMetadata }