func (env *mockEnviron) GetToolsSources() ([]simplestreams.DataSource, error) { if env.getToolsSources != nil { return env.getToolsSources() } datasource := storage.NewStorageSimpleStreamsDataSource("test cloud storage", env.Storage(), storage.BaseToolsPath) return []simplestreams.DataSource{datasource}, nil }
func (s *bootstrapSuite) setupBootstrapSpecificVersion(c *gc.C, clientMajor, clientMinor int, toolsVersion *version.Number) (error, int, version.Number) { currentVersion := version.Current currentVersion.Major = clientMajor currentVersion.Minor = clientMinor currentVersion.Tag = "" s.PatchValue(&version.Current, currentVersion) s.PatchValue(&series.HostSeries, func() string { return "trusty" }) s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 }) env := newEnviron("foo", useDefaultKeys, nil) s.setDummyStorage(c, env) envtools.RegisterToolsDataSourceFunc("local storage", func(environs.Environ) (simplestreams.DataSource, error) { return storage.NewStorageSimpleStreamsDataSource("test datasource", env.storage, "tools", simplestreams.CUSTOM_CLOUD_DATA, false), nil }) defer envtools.UnregisterToolsDataSourceFunc("local storage") toolsBinaries := []version.Binary{ version.MustParseBinary("10.11.12-trusty-amd64"), version.MustParseBinary("10.11.13-trusty-amd64"), version.MustParseBinary("10.11-beta1-trusty-amd64"), } stream := "released" if toolsVersion != nil && toolsVersion.Tag != "" { stream = "devel" currentVersion.Tag = toolsVersion.Tag } _, err := envtesting.UploadFakeToolsVersions(env.storage, stream, stream, toolsBinaries...) c.Assert(err, jc.ErrorIsNil) err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{ AgentVersion: toolsVersion, }) vers, _ := env.cfg.AgentVersion() return err, env.bootstrapCount, vers }
func (s *bootstrapSuite) setupBootstrapSpecificVersion( c *gc.C, clientMajor, clientMinor int, toolsVersion *version.Number, ) (error, int, version.Number) { currentVersion := version.Current currentVersion.Major = clientMajor currentVersion.Minor = clientMinor currentVersion.Series = "trusty" currentVersion.Arch = "amd64" s.PatchValue(&version.Current, currentVersion) env := newEnviron("foo", useDefaultKeys, nil) s.setDummyStorage(c, env) envtools.RegisterToolsDataSourceFunc("local storage", func(environs.Environ) (simplestreams.DataSource, error) { return storage.NewStorageSimpleStreamsDataSource("test datasource", env.storage, "tools"), nil }) defer envtools.UnregisterToolsDataSourceFunc("local storage") toolsBinaries := []version.Binary{ version.MustParseBinary("10.11.12-trusty-amd64"), version.MustParseBinary("10.11.13-trusty-amd64"), } _, err := envtesting.UploadFakeToolsVersions(env.storage, "released", "released", toolsBinaries...) c.Assert(err, jc.ErrorIsNil) err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{ AgentVersion: toolsVersion, }) vers, _ := env.cfg.AgentVersion() return err, env.bootstrapCount, vers }
func (s *datasourceSuite) TestURLWithBasePath(c *gc.C) { sampleData := "hello world" s.stor.Put("base/bar/data.txt", bytes.NewReader([]byte(sampleData)), int64(len(sampleData))) ds := storage.NewStorageSimpleStreamsDataSource("test datasource", s.stor, "base") url, err := ds.URL("bar") c.Assert(err, gc.IsNil) expectedURL, _ := s.stor.URL("base/bar") c.Assert(url, gc.Equals, expectedURL) }
// ParseMetadataFromStorage loads ImageMetadata from the specified storage reader. func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader) []*imagemetadata.ImageMetadata { source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "images") // Find the simplestreams index file. params := simplestreams.ValueParams{ DataType: "image-ids", ValueTemplate: imagemetadata.ImageMetadata{}, } const requireSigned = false indexPath := simplestreams.UnsignedIndex indexRef, err := simplestreams.GetIndexWithFormat( source, indexPath, "index:1.0", requireSigned, simplestreams.CloudSpec{}, params) c.Assert(err, gc.IsNil) c.Assert(indexRef.Indexes, gc.HasLen, 1) imageIndexMetadata := indexRef.Indexes["com.ubuntu.cloud:custom"] c.Assert(imageIndexMetadata, gc.NotNil) // Read the products file contents. r, err := stor.Get(path.Join("images", imageIndexMetadata.ProductsFilePath)) defer r.Close() c.Assert(err, gc.IsNil) data, err := ioutil.ReadAll(r) c.Assert(err, gc.IsNil) // Parse the products file metadata. url, err := source.URL(imageIndexMetadata.ProductsFilePath) c.Assert(err, gc.IsNil) cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, imagemetadata.ImageMetadata{}) c.Assert(err, gc.IsNil) // Collate the metadata. imageMetadataMap := make(map[string]*imagemetadata.ImageMetadata) var expectedProductIds set.Strings var imageVersions set.Strings for _, mc := range cloudMetadata.Products { for _, items := range mc.Items { for key, item := range items.Items { imageMetadata := item.(*imagemetadata.ImageMetadata) imageMetadataMap[key] = imageMetadata imageVersions.Add(key) productId := fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", mc.Version, imageMetadata.Arch) expectedProductIds.Add(productId) } } } // Make sure index's product IDs are all represented in the products metadata. sort.Strings(imageIndexMetadata.ProductIds) c.Assert(imageIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues()) imageMetadata := make([]*imagemetadata.ImageMetadata, len(imageMetadataMap)) for i, key := range imageVersions.SortedValues() { imageMetadata[i] = imageMetadataMap[key] } return imageMetadata }
func (s *datasourceSuite) TestFetchWithRetry(c *gc.C) { stor := &fakeStorage{shouldRetry: true} ds := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "base", simplestreams.DEFAULT_CLOUD_DATA, false) ds.SetAllowRetry(true) _, _, err := ds.Fetch("foo/bar/data.txt") c.Assert(err, gc.ErrorMatches, "an error") c.Assert(stor.getName, gc.Equals, "base/foo/bar/data.txt") c.Assert(stor.invokeCount, gc.Equals, 10) }
func (s *datasourceSuite) TestURL(c *gc.C) { sampleData := "hello world" s.stor.Put("bar/data.txt", bytes.NewReader([]byte(sampleData)), int64(len(sampleData))) ds := storage.NewStorageSimpleStreamsDataSource("test datasource", s.stor, "", simplestreams.DEFAULT_CLOUD_DATA, false) url, err := ds.URL("bar") c.Assert(err, jc.ErrorIsNil) expectedURL, _ := s.stor.URL("bar") c.Assert(url, gc.Equals, expectedURL) }
func (s *datasourceSuite) TestFetchWithNoRetry(c *gc.C) { // NB shouldRetry below is true indicating the fake storage is capable of // retrying, not that it will retry. stor := &fakeStorage{shouldRetry: true} ds := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "base", simplestreams.DEFAULT_CLOUD_DATA, false) _, _, err := ds.Fetch("foo/bar/data.txt") c.Assert(err, gc.ErrorMatches, "an error") c.Assert(stor.getName, gc.Equals, "base/foo/bar/data.txt") c.Assert(stor.invokeCount, gc.Equals, 1) }
// readMetadata reads the image metadata from metadataStore. func readMetadata(metadataStore storage.Storage) ([]*ImageMetadata, error) { // Read any existing metadata so we can merge the new tools metadata with what's there. dataSource := storage.NewStorageSimpleStreamsDataSource("existing metadata", metadataStore, storage.BaseImagesPath) imageConstraint := NewImageConstraint(simplestreams.LookupParams{}) existingMetadata, _, err := Fetch([]simplestreams.DataSource{dataSource}, imageConstraint, false) if err != nil && !errors.IsNotFound(err) { return nil, err } return existingMetadata, nil }
func (s *datasourceSuite) TestFetchWithBasePath(c *gc.C) { sampleData := "hello world" s.stor.Put("base/foo/bar/data.txt", bytes.NewReader([]byte(sampleData)), int64(len(sampleData))) ds := storage.NewStorageSimpleStreamsDataSource("test datasource", s.stor, "base", simplestreams.DEFAULT_CLOUD_DATA, false) rc, url, err := ds.Fetch("foo/bar/data.txt") c.Assert(err, jc.ErrorIsNil) defer rc.Close() c.Assert(url, gc.Equals, s.baseURL+"/base/foo/bar/data.txt") data, err := ioutil.ReadAll(rc) c.Assert(data, gc.DeepEquals, []byte(sampleData)) }
// ReadMetadata returns the tools metadata from the given storage for the specified stream. func ReadMetadata(store storage.StorageReader, stream string) ([]*ToolsMetadata, error) { dataSource := storage.NewStorageSimpleStreamsDataSource("existing metadata", store, storage.BaseToolsPath, simplestreams.EXISTING_CLOUD_DATA, false) toolsConstraint, err := makeToolsConstraint(simplestreams.CloudSpec{}, stream, -1, -1, coretools.Filter{}) if err != nil { return nil, err } metadata, _, err := Fetch([]simplestreams.DataSource{dataSource}, toolsConstraint) if err != nil && !errors.IsNotFound(err) { return nil, err } return metadata, nil }
func assertFetch(c *gc.C, stor storage.Storage, series, arch, region, endpoint, id string) { cons := imagemetadata.NewImageConstraint(simplestreams.LookupParams{ CloudSpec: simplestreams.CloudSpec{region, endpoint}, Series: []string{series}, Arches: []string{arch}, }) dataSource := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "images") metadata, _, err := imagemetadata.Fetch([]simplestreams.DataSource{dataSource}, cons, false) c.Assert(err, jc.ErrorIsNil) c.Assert(metadata, gc.HasLen, 1) c.Assert(metadata[0].Id, gc.Equals, id) }
func assertFetch(c *gc.C, stor storage.Storage, series, arch, region, endpoint string, ids ...string) { cons := imagemetadata.NewImageConstraint(simplestreams.LookupParams{ CloudSpec: simplestreams.CloudSpec{region, endpoint}, Series: []string{series}, Arches: []string{arch}, }) dataSource := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "images", simplestreams.DEFAULT_CLOUD_DATA, false) metadata, _, err := imagemetadata.Fetch([]simplestreams.DataSource{dataSource}, cons) c.Assert(err, jc.ErrorIsNil) c.Assert(metadata, gc.HasLen, len(ids)) for i, id := range ids { c.Assert(metadata[i].Id, gc.Equals, id) } }
func (s *bootstrapSuite) setupBootstrapSpecificVersion(c *gc.C, clientMajor, clientMinor int, toolsVersion *version.Number) (error, int, version.Number) { currentVersion := jujuversion.Current currentVersion.Major = clientMajor currentVersion.Minor = clientMinor currentVersion.Tag = "" s.PatchValue(&jujuversion.Current, currentVersion) s.PatchValue(&series.HostSeries, func() string { return "trusty" }) s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 }) env := newEnviron("foo", useDefaultKeys, nil) s.setDummyStorage(c, env) envtools.RegisterToolsDataSourceFunc("local storage", func(environs.Environ) (simplestreams.DataSource, error) { return storage.NewStorageSimpleStreamsDataSource("test datasource", env.storage, "tools", simplestreams.CUSTOM_CLOUD_DATA, false), nil }) defer envtools.UnregisterToolsDataSourceFunc("local storage") toolsBinaries := []version.Binary{ version.MustParseBinary("10.11.12-trusty-amd64"), version.MustParseBinary("10.11.13-trusty-amd64"), version.MustParseBinary("10.11-beta1-trusty-amd64"), } stream := "released" if toolsVersion != nil && toolsVersion.Tag != "" { stream = "devel" currentVersion.Tag = toolsVersion.Tag } _, err := envtesting.UploadFakeToolsVersions(env.storage, stream, stream, toolsBinaries...) c.Assert(err, jc.ErrorIsNil) err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{ ControllerConfig: coretesting.FakeControllerConfig(), AdminSecret: "admin-secret", CAPrivateKey: coretesting.CAKey, AgentVersion: toolsVersion, BuildAgentTarball: func(build bool, ver *version.Number, _ string) (*sync.BuiltAgent, error) { c.Logf("BuildAgentTarball version %s", ver) c.Assert(build, jc.IsFalse) return &sync.BuiltAgent{Dir: c.MkDir()}, nil }, }) vers, _ := env.cfg.AgentVersion() return err, env.bootstrapCount, vers }
// ParseIndexMetadataFromStorage loads Indices from the specified storage reader. func ParseIndexMetadataFromStorage(c *gc.C, stor storage.StorageReader) (*simplestreams.IndexMetadata, simplestreams.DataSource) { source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "images", simplestreams.DEFAULT_CLOUD_DATA, false) // Find the simplestreams index file. params := simplestreams.ValueParams{ DataType: "image-ids", ValueTemplate: imagemetadata.ImageMetadata{}, } const requireSigned = false indexPath := simplestreams.UnsignedIndex("v1", 1) mirrorsPath := simplestreams.MirrorsPath("v1") indexRef, err := simplestreams.GetIndexWithFormat( source, indexPath, "index:1.0", mirrorsPath, requireSigned, simplestreams.CloudSpec{}, params) c.Assert(err, jc.ErrorIsNil) c.Assert(indexRef.Indexes, gc.HasLen, 1) imageIndexMetadata := indexRef.Indexes["com.ubuntu.cloud:custom"] c.Assert(imageIndexMetadata, gc.NotNil) return imageIndexMetadata, source }
// GetToolsSources returns a list of sources which are used to search for simplestreams tools metadata. func (e *environ) GetToolsSources() ([]simplestreams.DataSource, error) { return []simplestreams.DataSource{ storage.NewStorageSimpleStreamsDataSource("cloud storage", e.Storage(), storage.BaseToolsPath)}, nil }
// GetToolsSources returns a list of sources which are used to search for simplestreams tools metadata. func (e *bootstrapEnviron) GetToolsSources() ([]simplestreams.DataSource, error) { // Add the simplestreams source off the control bucket. return []simplestreams.DataSource{ storage.NewStorageSimpleStreamsDataSource("cloud storage", e.Storage(), storage.BaseToolsPath)}, nil }
// ParseMetadataFromStorage loads ToolsMetadata from the specified storage reader. func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader, stream string, expectMirrors bool) []*tools.ToolsMetadata { source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "tools") params := simplestreams.ValueParams{ DataType: tools.ContentDownload, ValueTemplate: tools.ToolsMetadata{}, } const requireSigned = false indexPath := simplestreams.UnsignedIndex("v1", 2) mirrorsPath := simplestreams.MirrorsPath("v1") indexRef, err := simplestreams.GetIndexWithFormat( source, indexPath, "index:1.0", mirrorsPath, requireSigned, simplestreams.CloudSpec{}, params) c.Assert(err, jc.ErrorIsNil) toolsIndexMetadata := indexRef.Indexes[tools.ToolsContentId(stream)] c.Assert(toolsIndexMetadata, gc.NotNil) // Read the products file contents. r, err := stor.Get(path.Join("tools", toolsIndexMetadata.ProductsFilePath)) defer r.Close() c.Assert(err, jc.ErrorIsNil) data, err := ioutil.ReadAll(r) c.Assert(err, jc.ErrorIsNil) url, err := source.URL(toolsIndexMetadata.ProductsFilePath) c.Assert(err, jc.ErrorIsNil) cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, tools.ToolsMetadata{}) c.Assert(err, jc.ErrorIsNil) toolsMetadataMap := make(map[string]*tools.ToolsMetadata) expectedProductIds := make(set.Strings) toolsVersions := make(set.Strings) for _, mc := range cloudMetadata.Products { for _, items := range mc.Items { for key, item := range items.Items { toolsMetadata := item.(*tools.ToolsMetadata) toolsMetadataMap[key] = toolsMetadata toolsVersions.Add(key) seriesVersion, err := series.SeriesVersion(toolsMetadata.Release) if err != nil { c.Assert(err, jc.Satisfies, series.IsUnknownSeriesVersionError) } productId := fmt.Sprintf("com.ubuntu.juju:%s:%s", seriesVersion, toolsMetadata.Arch) expectedProductIds.Add(productId) } } } // Make sure index's product IDs are all represented in the products metadata. sort.Strings(toolsIndexMetadata.ProductIds) c.Assert(toolsIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues()) toolsMetadata := make([]*tools.ToolsMetadata, len(toolsMetadataMap)) for i, key := range toolsVersions.SortedValues() { toolsMetadata[i] = toolsMetadataMap[key] } if expectMirrors { r, err = stor.Get(path.Join("tools", simplestreams.UnsignedMirror("v1"))) c.Assert(err, jc.ErrorIsNil) defer r.Close() data, err = ioutil.ReadAll(r) c.Assert(err, jc.ErrorIsNil) c.Assert(string(data), jc.Contains, `"mirrors":`) c.Assert(string(data), jc.Contains, tools.ToolsContentId(stream)) c.Assert(err, jc.ErrorIsNil) } return toolsMetadata }
// GetToolsSources returns a list of sources which are used to search for simplestreams tools metadata. func (env *joyentEnviron) GetToolsSources() ([]simplestreams.DataSource, error) { // Add the simplestreams source off the control bucket. sources := []simplestreams.DataSource{ storage.NewStorageSimpleStreamsDataSource("cloud storage", env.Storage(), storage.BaseToolsPath)} return sources, nil }
func (env *mockEnviron) GetImageSources() ([]simplestreams.DataSource, error) { datasource := storage.NewStorageSimpleStreamsDataSource("test cloud storage", env.Storage(), storage.BaseImagesPath) return []simplestreams.DataSource{datasource}, nil }
// migrateToolsStorage copies tools from provider storage to // environment storage. func migrateToolsStorage(st *state.State, agentConfig agent.Config) error { logger.Debugf("migrating tools to environment storage") tstor, err := stateToolsStorage(st) if err != nil { return errors.Annotate(err, "cannot get tools storage") } defer tstor.Close() // Local and manual provider host storage on the state server's // filesystem, and serve via HTTP storage. The storage worker // doesn't run yet, so we just open the files directly. var stor storage.StorageReader providerType := agentConfig.Value(agent.ProviderType) if providerType == provider.Local || provider.IsManual(providerType) { storageDir := agentConfig.Value(agent.StorageDir) var err error stor, err = filestorage.NewFileStorageReader(storageDir) if err != nil { return errors.Annotate(err, "cannot get local filesystem storage reader") } } else { var err error stor, err = environs.LegacyStorage(st) if errors.IsNotSupported(err) { return nil } else if err != nil { return errors.Annotate(err, "cannot get provider storage") } } // Search provider storage for tools. datasource := storage.NewStorageSimpleStreamsDataSource("provider storage", stor, storage.BaseToolsPath) toolsList, err := envtools.FindToolsForCloud( []simplestreams.DataSource{datasource}, simplestreams.CloudSpec{}, envtools.ReleasedStream, -1, -1, tools.Filter{}) switch err { case nil: break case tools.ErrNoMatches, envtools.ErrNoTools: // No tools in provider storage: nothing to do. return nil default: return errors.Annotate(err, "cannot find tools in provider storage") } for _, agentTools := range toolsList { logger.Infof("migrating %v tools to environment storage", agentTools.Version) data, err := fetchToolsArchive(stor, envtools.LegacyReleaseDirectory, agentTools) if err != nil { return errors.Annotatef(err, "failed to fetch %v tools", agentTools.Version) } err = tstor.AddTools(bytes.NewReader(data), toolstorage.Metadata{ Version: agentTools.Version, Size: agentTools.Size, SHA256: agentTools.SHA256, }) if err != nil { return errors.Annotatef(err, "failed to add %v tools to environment storage", agentTools.Version) } } return nil }