func (env *mockEnviron) GetToolsSources() ([]simplestreams.DataSource, error) {
if env.getToolsSources != nil {
return env.getToolsSources()
}
datasource := storage.NewStorageSimpleStreamsDataSource("test cloud storage", env.Storage(), storage.BaseToolsPath)
return []simplestreams.DataSource{datasource}, nil
}
func (s *bootstrapSuite) setupBootstrapSpecificVersion(c *gc.C, clientMajor, clientMinor int, toolsVersion *version.Number) (error, int, version.Number) {
currentVersion := version.Current
currentVersion.Major = clientMajor
currentVersion.Minor = clientMinor
currentVersion.Tag = ""
s.PatchValue(&version.Current, currentVersion)
s.PatchValue(&series.HostSeries, func() string { return "trusty" })
s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })
env := newEnviron("foo", useDefaultKeys, nil)
s.setDummyStorage(c, env)
envtools.RegisterToolsDataSourceFunc("local storage", func(environs.Environ) (simplestreams.DataSource, error) {
return storage.NewStorageSimpleStreamsDataSource("test datasource", env.storage, "tools", simplestreams.CUSTOM_CLOUD_DATA, false), nil
})
defer envtools.UnregisterToolsDataSourceFunc("local storage")
toolsBinaries := []version.Binary{
version.MustParseBinary("10.11.12-trusty-amd64"),
version.MustParseBinary("10.11.13-trusty-amd64"),
version.MustParseBinary("10.11-beta1-trusty-amd64"),
}
stream := "released"
if toolsVersion != nil && toolsVersion.Tag != "" {
stream = "devel"
currentVersion.Tag = toolsVersion.Tag
}
_, err := envtesting.UploadFakeToolsVersions(env.storage, stream, stream, toolsBinaries...)
c.Assert(err, jc.ErrorIsNil)
err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
AgentVersion: toolsVersion,
})
vers, _ := env.cfg.AgentVersion()
return err, env.bootstrapCount, vers
}
func (s *bootstrapSuite) setupBootstrapSpecificVersion(
c *gc.C, clientMajor, clientMinor int, toolsVersion *version.Number,
) (error, int, version.Number) {
currentVersion := version.Current
currentVersion.Major = clientMajor
currentVersion.Minor = clientMinor
currentVersion.Series = "trusty"
currentVersion.Arch = "amd64"
s.PatchValue(&version.Current, currentVersion)
env := newEnviron("foo", useDefaultKeys, nil)
s.setDummyStorage(c, env)
envtools.RegisterToolsDataSourceFunc("local storage", func(environs.Environ) (simplestreams.DataSource, error) {
return storage.NewStorageSimpleStreamsDataSource("test datasource", env.storage, "tools"), nil
})
defer envtools.UnregisterToolsDataSourceFunc("local storage")
toolsBinaries := []version.Binary{
version.MustParseBinary("10.11.12-trusty-amd64"),
version.MustParseBinary("10.11.13-trusty-amd64"),
}
_, err := envtesting.UploadFakeToolsVersions(env.storage, "released", "released", toolsBinaries...)
c.Assert(err, jc.ErrorIsNil)
err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
AgentVersion: toolsVersion,
})
vers, _ := env.cfg.AgentVersion()
return err, env.bootstrapCount, vers
}
func (s *datasourceSuite) TestURLWithBasePath(c *gc.C) {
sampleData := "hello world"
s.stor.Put("base/bar/data.txt", bytes.NewReader([]byte(sampleData)), int64(len(sampleData)))
ds := storage.NewStorageSimpleStreamsDataSource("test datasource", s.stor, "base")
url, err := ds.URL("bar")
c.Assert(err, gc.IsNil)
expectedURL, _ := s.stor.URL("base/bar")
c.Assert(url, gc.Equals, expectedURL)
}
// ParseMetadataFromStorage loads ImageMetadata from the specified storage reader.
func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader) []*imagemetadata.ImageMetadata {
source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "images")
// Find the simplestreams index file.
params := simplestreams.ValueParams{
DataType: "image-ids",
ValueTemplate: imagemetadata.ImageMetadata{},
}
const requireSigned = false
indexPath := simplestreams.UnsignedIndex
indexRef, err := simplestreams.GetIndexWithFormat(
source, indexPath, "index:1.0", requireSigned, simplestreams.CloudSpec{}, params)
c.Assert(err, gc.IsNil)
c.Assert(indexRef.Indexes, gc.HasLen, 1)
imageIndexMetadata := indexRef.Indexes["com.ubuntu.cloud:custom"]
c.Assert(imageIndexMetadata, gc.NotNil)
// Read the products file contents.
r, err := stor.Get(path.Join("images", imageIndexMetadata.ProductsFilePath))
defer r.Close()
c.Assert(err, gc.IsNil)
data, err := ioutil.ReadAll(r)
c.Assert(err, gc.IsNil)
// Parse the products file metadata.
url, err := source.URL(imageIndexMetadata.ProductsFilePath)
c.Assert(err, gc.IsNil)
cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, imagemetadata.ImageMetadata{})
c.Assert(err, gc.IsNil)
// Collate the metadata.
imageMetadataMap := make(map[string]*imagemetadata.ImageMetadata)
var expectedProductIds set.Strings
var imageVersions set.Strings
for _, mc := range cloudMetadata.Products {
for _, items := range mc.Items {
for key, item := range items.Items {
imageMetadata := item.(*imagemetadata.ImageMetadata)
imageMetadataMap[key] = imageMetadata
imageVersions.Add(key)
productId := fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", mc.Version, imageMetadata.Arch)
expectedProductIds.Add(productId)
}
}
}
// Make sure index's product IDs are all represented in the products metadata.
sort.Strings(imageIndexMetadata.ProductIds)
c.Assert(imageIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues())
imageMetadata := make([]*imagemetadata.ImageMetadata, len(imageMetadataMap))
for i, key := range imageVersions.SortedValues() {
imageMetadata[i] = imageMetadataMap[key]
}
return imageMetadata
}
func (s *datasourceSuite) TestFetchWithRetry(c *gc.C) {
stor := &fakeStorage{shouldRetry: true}
ds := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "base", simplestreams.DEFAULT_CLOUD_DATA, false)
ds.SetAllowRetry(true)
_, _, err := ds.Fetch("foo/bar/data.txt")
c.Assert(err, gc.ErrorMatches, "an error")
c.Assert(stor.getName, gc.Equals, "base/foo/bar/data.txt")
c.Assert(stor.invokeCount, gc.Equals, 10)
}
func (s *datasourceSuite) TestURL(c *gc.C) {
sampleData := "hello world"
s.stor.Put("bar/data.txt", bytes.NewReader([]byte(sampleData)), int64(len(sampleData)))
ds := storage.NewStorageSimpleStreamsDataSource("test datasource", s.stor, "", simplestreams.DEFAULT_CLOUD_DATA, false)
url, err := ds.URL("bar")
c.Assert(err, jc.ErrorIsNil)
expectedURL, _ := s.stor.URL("bar")
c.Assert(url, gc.Equals, expectedURL)
}
func (s *datasourceSuite) TestFetchWithNoRetry(c *gc.C) {
// NB shouldRetry below is true indicating the fake storage is capable of
// retrying, not that it will retry.
stor := &fakeStorage{shouldRetry: true}
ds := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "base", simplestreams.DEFAULT_CLOUD_DATA, false)
_, _, err := ds.Fetch("foo/bar/data.txt")
c.Assert(err, gc.ErrorMatches, "an error")
c.Assert(stor.getName, gc.Equals, "base/foo/bar/data.txt")
c.Assert(stor.invokeCount, gc.Equals, 1)
}
// readMetadata reads the image metadata from metadataStore.
func readMetadata(metadataStore storage.Storage) ([]*ImageMetadata, error) {
// Read any existing metadata so we can merge the new tools metadata with what's there.
dataSource := storage.NewStorageSimpleStreamsDataSource("existing metadata", metadataStore, storage.BaseImagesPath)
imageConstraint := NewImageConstraint(simplestreams.LookupParams{})
existingMetadata, _, err := Fetch([]simplestreams.DataSource{dataSource}, imageConstraint, false)
if err != nil && !errors.IsNotFound(err) {
return nil, err
}
return existingMetadata, nil
}
func (s *datasourceSuite) TestFetchWithBasePath(c *gc.C) {
sampleData := "hello world"
s.stor.Put("base/foo/bar/data.txt", bytes.NewReader([]byte(sampleData)), int64(len(sampleData)))
ds := storage.NewStorageSimpleStreamsDataSource("test datasource", s.stor, "base", simplestreams.DEFAULT_CLOUD_DATA, false)
rc, url, err := ds.Fetch("foo/bar/data.txt")
c.Assert(err, jc.ErrorIsNil)
defer rc.Close()
c.Assert(url, gc.Equals, s.baseURL+"/base/foo/bar/data.txt")
data, err := ioutil.ReadAll(rc)
c.Assert(data, gc.DeepEquals, []byte(sampleData))
}
// ReadMetadata returns the tools metadata from the given storage for the specified stream.
func ReadMetadata(store storage.StorageReader, stream string) ([]*ToolsMetadata, error) {
dataSource := storage.NewStorageSimpleStreamsDataSource("existing metadata", store, storage.BaseToolsPath, simplestreams.EXISTING_CLOUD_DATA, false)
toolsConstraint, err := makeToolsConstraint(simplestreams.CloudSpec{}, stream, -1, -1, coretools.Filter{})
if err != nil {
return nil, err
}
metadata, _, err := Fetch([]simplestreams.DataSource{dataSource}, toolsConstraint)
if err != nil && !errors.IsNotFound(err) {
return nil, err
}
return metadata, nil
}
func assertFetch(c *gc.C, stor storage.Storage, series, arch, region, endpoint, id string) {
cons := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
CloudSpec: simplestreams.CloudSpec{region, endpoint},
Series: []string{series},
Arches: []string{arch},
})
dataSource := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "images")
metadata, _, err := imagemetadata.Fetch([]simplestreams.DataSource{dataSource}, cons, false)
c.Assert(err, jc.ErrorIsNil)
c.Assert(metadata, gc.HasLen, 1)
c.Assert(metadata[0].Id, gc.Equals, id)
}
func assertFetch(c *gc.C, stor storage.Storage, series, arch, region, endpoint string, ids ...string) {
cons := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
CloudSpec: simplestreams.CloudSpec{region, endpoint},
Series: []string{series},
Arches: []string{arch},
})
dataSource := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "images", simplestreams.DEFAULT_CLOUD_DATA, false)
metadata, _, err := imagemetadata.Fetch([]simplestreams.DataSource{dataSource}, cons)
c.Assert(err, jc.ErrorIsNil)
c.Assert(metadata, gc.HasLen, len(ids))
for i, id := range ids {
c.Assert(metadata[i].Id, gc.Equals, id)
}
}
func (s *bootstrapSuite) setupBootstrapSpecificVersion(c *gc.C, clientMajor, clientMinor int, toolsVersion *version.Number) (error, int, version.Number) {
currentVersion := jujuversion.Current
currentVersion.Major = clientMajor
currentVersion.Minor = clientMinor
currentVersion.Tag = ""
s.PatchValue(&jujuversion.Current, currentVersion)
s.PatchValue(&series.HostSeries, func() string { return "trusty" })
s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })
env := newEnviron("foo", useDefaultKeys, nil)
s.setDummyStorage(c, env)
envtools.RegisterToolsDataSourceFunc("local storage", func(environs.Environ) (simplestreams.DataSource, error) {
return storage.NewStorageSimpleStreamsDataSource("test datasource", env.storage, "tools", simplestreams.CUSTOM_CLOUD_DATA, false), nil
})
defer envtools.UnregisterToolsDataSourceFunc("local storage")
toolsBinaries := []version.Binary{
version.MustParseBinary("10.11.12-trusty-amd64"),
version.MustParseBinary("10.11.13-trusty-amd64"),
version.MustParseBinary("10.11-beta1-trusty-amd64"),
}
stream := "released"
if toolsVersion != nil && toolsVersion.Tag != "" {
stream = "devel"
currentVersion.Tag = toolsVersion.Tag
}
_, err := envtesting.UploadFakeToolsVersions(env.storage, stream, stream, toolsBinaries...)
c.Assert(err, jc.ErrorIsNil)
err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
ControllerConfig: coretesting.FakeControllerConfig(),
AdminSecret: "admin-secret",
CAPrivateKey: coretesting.CAKey,
AgentVersion: toolsVersion,
BuildAgentTarball: func(build bool, ver *version.Number, _ string) (*sync.BuiltAgent, error) {
c.Logf("BuildAgentTarball version %s", ver)
c.Assert(build, jc.IsFalse)
return &sync.BuiltAgent{Dir: c.MkDir()}, nil
},
})
vers, _ := env.cfg.AgentVersion()
return err, env.bootstrapCount, vers
}
// ParseIndexMetadataFromStorage loads Indices from the specified storage reader.
func ParseIndexMetadataFromStorage(c *gc.C, stor storage.StorageReader) (*simplestreams.IndexMetadata, simplestreams.DataSource) {
source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "images", simplestreams.DEFAULT_CLOUD_DATA, false)
// Find the simplestreams index file.
params := simplestreams.ValueParams{
DataType: "image-ids",
ValueTemplate: imagemetadata.ImageMetadata{},
}
const requireSigned = false
indexPath := simplestreams.UnsignedIndex("v1", 1)
mirrorsPath := simplestreams.MirrorsPath("v1")
indexRef, err := simplestreams.GetIndexWithFormat(
source, indexPath, "index:1.0", mirrorsPath, requireSigned, simplestreams.CloudSpec{}, params)
c.Assert(err, jc.ErrorIsNil)
c.Assert(indexRef.Indexes, gc.HasLen, 1)
imageIndexMetadata := indexRef.Indexes["com.ubuntu.cloud:custom"]
c.Assert(imageIndexMetadata, gc.NotNil)
return imageIndexMetadata, source
}
// GetToolsSources returns a list of sources which are used to search for simplestreams tools metadata.
func (e *environ) GetToolsSources() ([]simplestreams.DataSource, error) {
return []simplestreams.DataSource{
storage.NewStorageSimpleStreamsDataSource("cloud storage", e.Storage(), storage.BaseToolsPath)}, nil
}
// GetToolsSources returns a list of sources which are used to search for simplestreams tools metadata.
func (e *bootstrapEnviron) GetToolsSources() ([]simplestreams.DataSource, error) {
// Add the simplestreams source off the control bucket.
return []simplestreams.DataSource{
storage.NewStorageSimpleStreamsDataSource("cloud storage", e.Storage(), storage.BaseToolsPath)}, nil
}
// ParseMetadataFromStorage loads ToolsMetadata from the specified storage reader.
func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader, stream string, expectMirrors bool) []*tools.ToolsMetadata {
source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "tools")
params := simplestreams.ValueParams{
DataType: tools.ContentDownload,
ValueTemplate: tools.ToolsMetadata{},
}
const requireSigned = false
indexPath := simplestreams.UnsignedIndex("v1", 2)
mirrorsPath := simplestreams.MirrorsPath("v1")
indexRef, err := simplestreams.GetIndexWithFormat(
source, indexPath, "index:1.0", mirrorsPath, requireSigned, simplestreams.CloudSpec{}, params)
c.Assert(err, jc.ErrorIsNil)
toolsIndexMetadata := indexRef.Indexes[tools.ToolsContentId(stream)]
c.Assert(toolsIndexMetadata, gc.NotNil)
// Read the products file contents.
r, err := stor.Get(path.Join("tools", toolsIndexMetadata.ProductsFilePath))
defer r.Close()
c.Assert(err, jc.ErrorIsNil)
data, err := ioutil.ReadAll(r)
c.Assert(err, jc.ErrorIsNil)
url, err := source.URL(toolsIndexMetadata.ProductsFilePath)
c.Assert(err, jc.ErrorIsNil)
cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, tools.ToolsMetadata{})
c.Assert(err, jc.ErrorIsNil)
toolsMetadataMap := make(map[string]*tools.ToolsMetadata)
expectedProductIds := make(set.Strings)
toolsVersions := make(set.Strings)
for _, mc := range cloudMetadata.Products {
for _, items := range mc.Items {
for key, item := range items.Items {
toolsMetadata := item.(*tools.ToolsMetadata)
toolsMetadataMap[key] = toolsMetadata
toolsVersions.Add(key)
seriesVersion, err := series.SeriesVersion(toolsMetadata.Release)
if err != nil {
c.Assert(err, jc.Satisfies, series.IsUnknownSeriesVersionError)
}
productId := fmt.Sprintf("com.ubuntu.juju:%s:%s", seriesVersion, toolsMetadata.Arch)
expectedProductIds.Add(productId)
}
}
}
// Make sure index's product IDs are all represented in the products metadata.
sort.Strings(toolsIndexMetadata.ProductIds)
c.Assert(toolsIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues())
toolsMetadata := make([]*tools.ToolsMetadata, len(toolsMetadataMap))
for i, key := range toolsVersions.SortedValues() {
toolsMetadata[i] = toolsMetadataMap[key]
}
if expectMirrors {
r, err = stor.Get(path.Join("tools", simplestreams.UnsignedMirror("v1")))
c.Assert(err, jc.ErrorIsNil)
defer r.Close()
data, err = ioutil.ReadAll(r)
c.Assert(err, jc.ErrorIsNil)
c.Assert(string(data), jc.Contains, `"mirrors":`)
c.Assert(string(data), jc.Contains, tools.ToolsContentId(stream))
c.Assert(err, jc.ErrorIsNil)
}
return toolsMetadata
}
// GetToolsSources returns a list of sources which are used to search for simplestreams tools metadata.
func (env *joyentEnviron) GetToolsSources() ([]simplestreams.DataSource, error) {
// Add the simplestreams source off the control bucket.
sources := []simplestreams.DataSource{
storage.NewStorageSimpleStreamsDataSource("cloud storage", env.Storage(), storage.BaseToolsPath)}
return sources, nil
}
func (env *mockEnviron) GetImageSources() ([]simplestreams.DataSource, error) {
datasource := storage.NewStorageSimpleStreamsDataSource("test cloud storage", env.Storage(), storage.BaseImagesPath)
return []simplestreams.DataSource{datasource}, nil
}
// migrateToolsStorage copies tools from provider storage to
// environment storage.
func migrateToolsStorage(st *state.State, agentConfig agent.Config) error {
logger.Debugf("migrating tools to environment storage")
tstor, err := stateToolsStorage(st)
if err != nil {
return errors.Annotate(err, "cannot get tools storage")
}
defer tstor.Close()
// Local and manual provider host storage on the state server's
// filesystem, and serve via HTTP storage. The storage worker
// doesn't run yet, so we just open the files directly.
var stor storage.StorageReader
providerType := agentConfig.Value(agent.ProviderType)
if providerType == provider.Local || provider.IsManual(providerType) {
storageDir := agentConfig.Value(agent.StorageDir)
var err error
stor, err = filestorage.NewFileStorageReader(storageDir)
if err != nil {
return errors.Annotate(err, "cannot get local filesystem storage reader")
}
} else {
var err error
stor, err = environs.LegacyStorage(st)
if errors.IsNotSupported(err) {
return nil
} else if err != nil {
return errors.Annotate(err, "cannot get provider storage")
}
}
// Search provider storage for tools.
datasource := storage.NewStorageSimpleStreamsDataSource("provider storage", stor, storage.BaseToolsPath)
toolsList, err := envtools.FindToolsForCloud(
[]simplestreams.DataSource{datasource},
simplestreams.CloudSpec{},
envtools.ReleasedStream,
-1, -1, tools.Filter{})
switch err {
case nil:
break
case tools.ErrNoMatches, envtools.ErrNoTools:
// No tools in provider storage: nothing to do.
return nil
default:
return errors.Annotate(err, "cannot find tools in provider storage")
}
for _, agentTools := range toolsList {
logger.Infof("migrating %v tools to environment storage", agentTools.Version)
data, err := fetchToolsArchive(stor, envtools.LegacyReleaseDirectory, agentTools)
if err != nil {
return errors.Annotatef(err, "failed to fetch %v tools", agentTools.Version)
}
err = tstor.AddTools(bytes.NewReader(data), toolstorage.Metadata{
Version: agentTools.Version,
Size: agentTools.Size,
SHA256: agentTools.SHA256,
})
if err != nil {
return errors.Annotatef(err, "failed to add %v tools to environment storage", agentTools.Version)
}
}
return nil
}