func (*metadataHelperSuite) TestResolveMetadata(c *gc.C) {
var versionStrings = []string{"1.2.3-precise-amd64"}
dir := c.MkDir()
toolstesting.MakeTools(c, dir, "released", versionStrings)
toolsList := coretools.List{{
Version: version.MustParseBinary(versionStrings[0]),
Size: 123,
SHA256: "abc",
}}
stor, err := filestorage.NewFileStorageReader(dir)
c.Assert(err, jc.ErrorIsNil)
err = tools.ResolveMetadata(stor, "released", nil)
c.Assert(err, jc.ErrorIsNil)
// We already have size/sha256, so ensure that storage isn't consulted.
countingStorage := &countingStorage{StorageReader: stor}
metadata := tools.MetadataFromTools(toolsList, "released")
err = tools.ResolveMetadata(countingStorage, "released", metadata)
c.Assert(err, jc.ErrorIsNil)
c.Assert(countingStorage.counter, gc.Equals, 0)
// Now clear size/sha256, and check that it is called, and
// the size/sha256 sum are updated.
metadata[0].Size = 0
metadata[0].SHA256 = ""
err = tools.ResolveMetadata(countingStorage, "released", metadata)
c.Assert(err, jc.ErrorIsNil)
c.Assert(countingStorage.counter, gc.Equals, 1)
c.Assert(metadata[0].Size, gc.Not(gc.Equals), 0)
c.Assert(metadata[0].SHA256, gc.Not(gc.Equals), "")
}
func (s *filestorageSuite) SetUpTest(c *gc.C) {
s.dir = c.MkDir()
var err error
s.reader, err = filestorage.NewFileStorageReader(s.dir)
c.Assert(err, jc.ErrorIsNil)
s.writer, err = filestorage.NewFileStorageWriter(s.dir)
c.Assert(err, jc.ErrorIsNil)
}
func (s *filestorageSuite) TestPathRelativeToHome(c *gc.C) {
homeDir := utils.Home()
tempDir, err := ioutil.TempDir(homeDir, "")
c.Assert(err, jc.ErrorIsNil)
defer os.RemoveAll(tempDir)
dirName := strings.Replace(tempDir, homeDir, "", -1)
reader, err := filestorage.NewFileStorageReader(filepath.Join(utils.Home(), dirName))
c.Assert(err, jc.ErrorIsNil)
url, err := reader.URL("")
c.Assert(err, jc.ErrorIsNil)
c.Assert(url, gc.Equals, utils.MakeFileURL(filepath.Join(homeDir, dirName)))
}
func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuHome()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return err
}
// We now store the tools in a directory named after their stream, but the
// legacy behaviour is to store all tools in a single "releases" directory.
toolsDir := c.stream
if c.stream == "" {
fmt.Fprintf(context.Stdout, "No stream specified, defaulting to released tools in the releases directory.\n")
c.stream = envtools.ReleasedStream
toolsDir = envtools.LegacyReleaseDirectory
}
fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
const minorVersion = -1
toolsList, err := envtools.ReadList(sourceStorage, toolsDir, version.Current.Major, minorVersion)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return err
}
sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames)
toolsList, err = envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{}, c.stream,
version.Current.Major, minorVersion, coretools.Filter{})
}
if err != nil {
return err
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return err
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return mergeAndWriteMetadata(targetStorage, toolsDir, c.stream, c.clean, toolsList, writeMirrors)
}
func (*metadataHelperSuite) TestResolveMetadataLegacyPPC64(c *gc.C) {
var versionStrings = []string{"1.2.3-precise-amd64", "1.2.3-precise-ppc64el"}
dir := c.MkDir()
toolstesting.MakeTools(c, dir, "released", versionStrings)
toolsList := coretools.List{
{
Version: version.MustParseBinary(versionStrings[0]),
}, {
Version: version.MustParseBinary(versionStrings[1]),
}, {
Version: version.MustParseBinary("1.2.3-precise-ppc64"),
},
}
toolsMetadata := tools.MetadataFromTools(toolsList, dir)
stor, err := filestorage.NewFileStorageReader(dir)
c.Assert(err, jc.ErrorIsNil)
err = tools.ResolveMetadata(stor, "released", toolsMetadata)
c.Assert(err, jc.ErrorIsNil)
c.Assert(toolsMetadata, gc.DeepEquals, []*tools.ToolsMetadata{
{
Release: "precise",
Version: "1.2.3",
Arch: "amd64",
Size: 19,
FileType: "tar.gz",
SHA256: "dcdd65b962b804a3d63b108d670290ee95a867a97fe9b9f99b2b77b5c7173e59",
Path: fmt.Sprintf("%s/juju-1.2.3-precise-amd64.tgz", dir),
},
{
Release: "precise",
Version: "1.2.3",
Arch: "ppc64el",
Size: 21,
FileType: "tar.gz",
SHA256: "a3460ed45eb07a69adfcd541413a495f988c5842d715c6a40353075c3ad47af2",
Path: fmt.Sprintf("%s/juju-1.2.3-precise-ppc64el.tgz", dir),
},
{
Release: "precise",
Version: "1.2.3",
Arch: "ppc64",
Size: 21,
FileType: "tar.gz",
SHA256: "a3460ed45eb07a69adfcd541413a495f988c5842d715c6a40353075c3ad47af2",
Path: fmt.Sprintf("%s/juju-1.2.3-precise-ppc64el.tgz", dir),
},
})
}
func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
writer := loggo.NewMinimumLevelWriter(
cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr),
loggo.INFO)
loggo.RegisterWriter("toolsmetadata", writer)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuXDGDataHomeDir()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return err
}
// We now store the tools in a directory named after their stream, but the
// legacy behaviour is to store all tools in a single "releases" directory.
toolsDir := c.stream
if c.stream == "" {
fmt.Fprintln(context.Stdout, "No stream specified, defaulting to released tools in the releases directory.")
c.stream = envtools.ReleasedStream
toolsDir = envtools.LegacyReleaseDirectory
}
fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
toolsList, err := envtools.ReadList(sourceStorage, toolsDir, -1, -1)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return err
}
toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, c.stream, -1, -1, coretools.Filter{})
}
if err != nil {
return err
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return err
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return mergeAndWriteMetadata(targetStorage, toolsDir, c.stream, c.clean, toolsList, writeMirrors)
}
func (s *filestorageSuite) TestRelativePath(c *gc.C) {
dir := c.MkDir()
err := os.MkdirAll(filepath.Join(dir, "a", "b", "c"), os.ModePerm)
c.Assert(err, jc.ErrorIsNil)
cwd, err := os.Getwd()
c.Assert(err, jc.ErrorIsNil)
err = os.Chdir(filepath.Join(dir, "a", "b", "c"))
c.Assert(err, jc.ErrorIsNil)
defer os.Chdir(cwd)
reader, err := filestorage.NewFileStorageReader("../..")
c.Assert(err, jc.ErrorIsNil)
url, err := reader.URL("")
c.Assert(err, jc.ErrorIsNil)
c.Assert(url, gc.Equals, utils.MakeFileURL(dir)+"/a")
}
// migrateCustomImageMetadata copies uploaded image metadata from provider
// storage to environment storage, preserving paths.
func migrateCustomImageMetadata(st *state.State, agentConfig agent.Config) error {
logger.Debugf("migrating custom image metadata to environment storage")
estor := newStateStorage(st.EnvironUUID(), st.MongoSession())
// Local and manual provider host storage on the state server's
// filesystem, and serve via HTTP storage. The storage worker
// doesn't run yet, so we just open the files directly.
var pstor storage.StorageReader
providerType := agentConfig.Value(agent.ProviderType)
if providerType == provider.Local || provider.IsManual(providerType) {
storageDir := agentConfig.Value(agent.StorageDir)
var err error
pstor, err = filestorage.NewFileStorageReader(storageDir)
if err != nil {
return errors.Annotate(err, "cannot get local filesystem storage reader")
}
} else {
var err error
pstor, err = environs.LegacyStorage(st)
if errors.IsNotSupported(err) {
return nil
} else if err != nil {
return errors.Annotate(err, "cannot get provider storage")
}
}
paths, err := pstor.List(storage.BaseImagesPath)
if err != nil {
return err
}
for _, path := range paths {
logger.Infof("migrating image metadata at path %q", path)
data, err := readImageMetadata(pstor, path)
if err != nil {
return errors.Annotate(err, "failed to read image metadata")
}
err = estor.Put(path, bytes.NewReader(data), int64(len(data)))
if err != nil {
return errors.Annotate(err, "failed to write image metadata")
}
}
return nil
}
func (c *ToolsMetadataCommand) Run(context *cmd.Context) error {
loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuHome()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return err
}
fmt.Fprintf(context.Stdout, "Finding tools in %s\n", c.metadataDir)
const minorVersion = -1
toolsList, err := envtools.ReadList(sourceStorage, version.Current.Major, minorVersion)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return err
}
sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames)
toolsList, err = envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
version.Current.Major, minorVersion, coretools.Filter{})
}
if err != nil {
return err
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return err
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return mergeAndWriteMetadata(targetStorage, toolsList, writeMirrors)
}
func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
writer := loggo.NewMinimumLevelWriter(
cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr),
loggo.INFO)
loggo.RegisterWriter("toolsmetadata", writer)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuXDGDataHomeDir()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return errors.Trace(err)
}
fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
toolsList, err := envtools.ReadList(sourceStorage, c.stream, -1, -1)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return errors.Trace(err)
}
toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, c.stream, -1, -1, coretools.Filter{})
}
if err != nil {
return errors.Trace(err)
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return errors.Trace(err)
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return errors.Trace(mergeAndWriteMetadata(targetStorage, c.stream, c.stream, c.clean, toolsList, writeMirrors))
}
// ParseMetadataFromDir loads ImageMetadata from the specified directory.
func ParseMetadataFromDir(c *gc.C, metadataDir string) []*imagemetadata.ImageMetadata {
stor, err := filestorage.NewFileStorageReader(metadataDir)
c.Assert(err, gc.IsNil)
return ParseMetadataFromStorage(c, stor)
}
// ParseMetadataFromDir loads ToolsMetadata from the specified directory.
func ParseMetadataFromDir(c *gc.C, metadataDir, stream string, expectMirrors bool) []*tools.ToolsMetadata {
stor, err := filestorage.NewFileStorageReader(metadataDir)
c.Assert(err, jc.ErrorIsNil)
return ParseMetadataFromStorage(c, stor, stream, expectMirrors)
}
// migrateToolsStorage copies tools from provider storage to
// environment storage.
func migrateToolsStorage(st *state.State, agentConfig agent.Config) error {
logger.Debugf("migrating tools to environment storage")
tstor, err := stateToolsStorage(st)
if err != nil {
return errors.Annotate(err, "cannot get tools storage")
}
defer tstor.Close()
// Local and manual provider host storage on the state server's
// filesystem, and serve via HTTP storage. The storage worker
// doesn't run yet, so we just open the files directly.
var stor storage.StorageReader
providerType := agentConfig.Value(agent.ProviderType)
if providerType == provider.Local || provider.IsManual(providerType) {
storageDir := agentConfig.Value(agent.StorageDir)
var err error
stor, err = filestorage.NewFileStorageReader(storageDir)
if err != nil {
return errors.Annotate(err, "cannot get local filesystem storage reader")
}
} else {
var err error
stor, err = environs.LegacyStorage(st)
if errors.IsNotSupported(err) {
return nil
} else if err != nil {
return errors.Annotate(err, "cannot get provider storage")
}
}
// Search provider storage for tools.
datasource := storage.NewStorageSimpleStreamsDataSource("provider storage", stor, storage.BaseToolsPath)
toolsList, err := envtools.FindToolsForCloud(
[]simplestreams.DataSource{datasource},
simplestreams.CloudSpec{},
envtools.ReleasedStream,
-1, -1, tools.Filter{})
switch err {
case nil:
break
case tools.ErrNoMatches, envtools.ErrNoTools:
// No tools in provider storage: nothing to do.
return nil
default:
return errors.Annotate(err, "cannot find tools in provider storage")
}
for _, agentTools := range toolsList {
logger.Infof("migrating %v tools to environment storage", agentTools.Version)
data, err := fetchToolsArchive(stor, envtools.LegacyReleaseDirectory, agentTools)
if err != nil {
return errors.Annotatef(err, "failed to fetch %v tools", agentTools.Version)
}
err = tstor.AddTools(bytes.NewReader(data), toolstorage.Metadata{
Version: agentTools.Version,
Size: agentTools.Size,
SHA256: agentTools.SHA256,
})
if err != nil {
return errors.Annotatef(err, "failed to add %v tools to environment storage", agentTools.Version)
}
}
return nil
}