// UploadFakeToolsVersions puts fake tools in the supplied storage for the supplied versions.
func UploadFakeToolsVersions(stor storage.Storage, toolsDir, stream string, versions ...version.Binary) ([]*coretools.Tools, error) {
// Leave existing tools alone.
existingTools := make(map[version.Binary]*coretools.Tools)
existing, _ := envtools.ReadList(stor, toolsDir, 1, -1)
for _, tools := range existing {
existingTools[tools.Version] = tools
}
var agentTools coretools.List = make(coretools.List, len(versions))
for i, version := range versions {
if tools, ok := existingTools[version]; ok {
agentTools[i] = tools
} else {
t, err := uploadFakeToolsVersion(stor, toolsDir, version)
if err != nil {
return nil, err
}
agentTools[i] = t
}
}
if err := envtools.MergeAndWriteMetadata(stor, toolsDir, stream, agentTools, envtools.DoNotWriteMirrors); err != nil {
return nil, err
}
err := SignTestTools(stor)
if err != nil {
return nil, err
}
return agentTools, nil
}
func assertNoUnexpectedTools(c *gc.C, stor storage.StorageReader) {
// We only expect v1.x tools, no v2.x tools.
list, err := envtools.ReadList(stor, 2, 0)
if len(list) > 0 {
c.Logf("got unexpected tools: %s", list)
}
c.Assert(err, gc.Equals, coretools.ErrNoMatches)
}
func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuHome()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return err
}
// We now store the tools in a directory named after their stream, but the
// legacy behaviour is to store all tools in a single "releases" directory.
toolsDir := c.stream
if c.stream == "" {
fmt.Fprintf(context.Stdout, "No stream specified, defaulting to released tools in the releases directory.\n")
c.stream = envtools.ReleasedStream
toolsDir = envtools.LegacyReleaseDirectory
}
fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
const minorVersion = -1
toolsList, err := envtools.ReadList(sourceStorage, toolsDir, version.Current.Major, minorVersion)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return err
}
sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames)
toolsList, err = envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{}, c.stream,
version.Current.Major, minorVersion, coretools.Filter{})
}
if err != nil {
return err
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return err
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return mergeAndWriteMetadata(targetStorage, toolsDir, c.stream, c.clean, toolsList, writeMirrors)
}
func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
writer := loggo.NewMinimumLevelWriter(
cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr),
loggo.INFO)
loggo.RegisterWriter("toolsmetadata", writer)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuXDGDataHomeDir()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return err
}
// We now store the tools in a directory named after their stream, but the
// legacy behaviour is to store all tools in a single "releases" directory.
toolsDir := c.stream
if c.stream == "" {
fmt.Fprintln(context.Stdout, "No stream specified, defaulting to released tools in the releases directory.")
c.stream = envtools.ReleasedStream
toolsDir = envtools.LegacyReleaseDirectory
}
fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
toolsList, err := envtools.ReadList(sourceStorage, toolsDir, -1, -1)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return err
}
toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, c.stream, -1, -1, coretools.Filter{})
}
if err != nil {
return err
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return err
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return mergeAndWriteMetadata(targetStorage, toolsDir, c.stream, c.clean, toolsList, writeMirrors)
}
func (s *StorageSuite) TestReadList(c *gc.C) {
stor, err := filestorage.NewFileStorageWriter(c.MkDir())
c.Assert(err, jc.ErrorIsNil)
v100 := version.MustParseBinary("1.0.0-precise-amd64")
v101 := version.MustParseBinary("1.0.1-precise-amd64")
v111 := version.MustParseBinary("1.1.1-precise-amd64")
v201 := version.MustParseBinary("2.0.1-precise-amd64")
agentTools := envtesting.AssertUploadFakeToolsVersions(c, stor, "proposed", "proposed", v100, v101, v111, v201)
t100 := agentTools[0]
t101 := agentTools[1]
t111 := agentTools[2]
t201 := agentTools[3]
for i, t := range []struct {
majorVersion,
minorVersion int
list coretools.List
}{{
-1, -1, coretools.List{t100, t101, t111, t201},
}, {
1, 0, coretools.List{t100, t101},
}, {
1, 1, coretools.List{t111},
}, {
1, -1, coretools.List{t100, t101, t111},
}, {
1, 2, nil,
}, {
3, 0, nil,
}} {
c.Logf("test %d", i)
list, err := envtools.ReadList(stor, "proposed", t.majorVersion, t.minorVersion)
if t.list != nil {
c.Assert(err, jc.ErrorIsNil)
// ReadList doesn't set the Size or SHA256, so blank out those attributes.
for _, tool := range t.list {
tool.Size = 0
tool.SHA256 = ""
}
c.Assert(list, gc.DeepEquals, t.list)
} else {
c.Assert(err, gc.Equals, coretools.ErrNoMatches)
}
}
}
func (s *StorageSuite) TestReadList(c *gc.C) {
store := s.env.Storage()
v001 := version.MustParseBinary("0.0.1-precise-amd64")
v100 := version.MustParseBinary("1.0.0-precise-amd64")
v101 := version.MustParseBinary("1.0.1-precise-amd64")
v111 := version.MustParseBinary("1.1.1-precise-amd64")
agentTools := envtesting.AssertUploadFakeToolsVersions(c, store, v001, v100, v101, v111)
t001 := agentTools[0]
t100 := agentTools[1]
t101 := agentTools[2]
t111 := agentTools[3]
for i, t := range []struct {
majorVersion,
minorVersion int
list coretools.List
}{{
0, 0, coretools.List{t001},
}, {
1, 0, coretools.List{t100, t101},
}, {
1, 1, coretools.List{t111},
}, {
1, -1, coretools.List{t100, t101, t111},
}, {
1, 2, nil,
}, {
2, 0, nil,
}} {
c.Logf("test %d", i)
list, err := envtools.ReadList(store, t.majorVersion, t.minorVersion)
if t.list != nil {
c.Assert(err, gc.IsNil)
// ReadList doesn't set the Size of SHA256, so blank out those attributes.
for _, tool := range t.list {
tool.Size = 0
tool.SHA256 = ""
}
c.Assert(list, gc.DeepEquals, t.list)
} else {
c.Assert(err, gc.Equals, coretools.ErrNoMatches)
}
}
}
func (s *uploadSuite) assertUploadedTools(c *gc.C, t *coretools.Tools, expectSeries []string, stream string) {
s.assertEqualsCurrentVersion(c, t.Version)
expectRaw := downloadToolsRaw(c, t)
list, err := envtools.ReadList(s.targetStorage, stream, jujuversion.Current.Major, jujuversion.Current.Minor)
c.Assert(err, jc.ErrorIsNil)
c.Assert(list.AllSeries(), jc.SameContents, expectSeries)
sort.Strings(expectSeries)
c.Assert(list.AllSeries(), gc.DeepEquals, expectSeries)
for _, t := range list {
c.Logf("checking %s", t.URL)
c.Assert(t.Version.Number, gc.Equals, jujuversion.Current)
actualRaw := downloadToolsRaw(c, t)
c.Assert(string(actualRaw), gc.Equals, string(expectRaw))
}
metadata, err := envtools.ReadMetadata(s.targetStorage, stream)
c.Assert(err, jc.ErrorIsNil)
c.Assert(metadata, gc.HasLen, 0)
}
func (s *uploadSuite) assertUploadedTools(c *gc.C, t *coretools.Tools, uploadedSeries string) {
c.Assert(t.Version, gc.Equals, version.Current)
expectRaw := downloadToolsRaw(c, t)
list, err := envtools.ReadList(s.env.Storage(), version.Current.Major, version.Current.Minor)
c.Assert(err, gc.IsNil)
c.Assert(list, gc.HasLen, 3)
expectSeries := []string{"quantal", uploadedSeries, version.Current.Series}
sort.Strings(expectSeries)
c.Assert(list.AllSeries(), gc.DeepEquals, expectSeries)
for _, t := range list {
c.Logf("checking %s", t.URL)
c.Assert(t.Version.Number, gc.Equals, version.Current.Number)
actualRaw := downloadToolsRaw(c, t)
c.Assert(string(actualRaw), gc.Equals, string(expectRaw))
}
metadata, err := envtools.ReadMetadata(s.env.Storage())
c.Assert(err, gc.IsNil)
c.Assert(metadata, gc.HasLen, 0)
}
func (c *ToolsMetadataCommand) Run(context *cmd.Context) error {
loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuHome()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return err
}
fmt.Fprintf(context.Stdout, "Finding tools in %s\n", c.metadataDir)
const minorVersion = -1
toolsList, err := envtools.ReadList(sourceStorage, version.Current.Major, minorVersion)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return err
}
sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames)
toolsList, err = envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
version.Current.Major, minorVersion, coretools.Filter{})
}
if err != nil {
return err
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return err
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return mergeAndWriteMetadata(targetStorage, toolsList, writeMirrors)
}
func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
writer := loggo.NewMinimumLevelWriter(
cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr),
loggo.INFO)
loggo.RegisterWriter("toolsmetadata", writer)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuXDGDataHomeDir()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return errors.Trace(err)
}
fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
toolsList, err := envtools.ReadList(sourceStorage, c.stream, -1, -1)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return errors.Trace(err)
}
toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, c.stream, -1, -1, coretools.Filter{})
}
if err != nil {
return errors.Trace(err)
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return errors.Trace(err)
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return errors.Trace(mergeAndWriteMetadata(targetStorage, c.stream, c.stream, c.clean, toolsList, writeMirrors))
}
func (s *StorageSuite) TestReadListLegacyPPC64(c *gc.C) {
stor, err := filestorage.NewFileStorageWriter(c.MkDir())
c.Assert(err, jc.ErrorIsNil)
v100 := version.MustParseBinary("1.0.0-precise-amd64")
v101 := version.MustParseBinary("1.0.1-precise-ppc64el")
agentTools := envtesting.AssertUploadFakeToolsVersions(c, stor, "proposed", "proposed", v100, v101)
amd64Tools := agentTools[0]
ppc64elTools := agentTools[1]
// We also expect metadata for ppc64 to be added.
ppc64Tools := *ppc64elTools
ppc64Tools.Version.Arch = "ppc64"
expected := coretools.List{amd64Tools, ppc64elTools, &ppc64Tools}
list, err := envtools.ReadList(stor, "proposed", 1, 0)
c.Assert(err, jc.ErrorIsNil)
// ReadList doesn't set the Size or SHA256, so blank out those attributes.
for _, tool := range expected {
tool.Size = 0
tool.SHA256 = ""
}
c.Assert(list, gc.DeepEquals, expected)
}
func (s *StorageSuite) TestReadListEmpty(c *gc.C) {
stor, err := filestorage.NewFileStorageWriter(c.MkDir())
c.Assert(err, jc.ErrorIsNil)
_, err = envtools.ReadList(stor, "released", 2, 0)
c.Assert(err, gc.Equals, envtools.ErrNoTools)
}
// SyncTools copies the Juju tools tarball from the official bucket
// or a specified source directory into the user's environment.
func SyncTools(syncContext *SyncContext) error {
sourceDataSource, err := selectSourceDatasource(syncContext)
if err != nil {
return err
}
logger.Infof("listing available tools")
if syncContext.MajorVersion == 0 && syncContext.MinorVersion == 0 {
syncContext.MajorVersion = version.Current.Major
syncContext.MinorVersion = -1
if !syncContext.AllVersions {
syncContext.MinorVersion = version.Current.Minor
}
} else if !syncContext.Dev && syncContext.MinorVersion != -1 {
// If a major.minor version is specified, we allow dev versions.
// If Dev is already true, leave it alone.
syncContext.Dev = true
}
released := !syncContext.Dev && !version.Current.IsDev()
sourceTools, err := envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{Released: released})
if err != nil {
return err
}
logger.Infof("found %d tools", len(sourceTools))
if !syncContext.AllVersions {
var latest version.Number
latest, sourceTools = sourceTools.Newest()
logger.Infof("found %d recent tools (version %s)", len(sourceTools), latest)
}
for _, tool := range sourceTools {
logger.Debugf("found source tool: %v", tool)
}
logger.Infof("listing target tools storage")
targetStorage := syncContext.Target
targetTools, err := envtools.ReadList(targetStorage, syncContext.MajorVersion, -1)
switch err {
case nil, coretools.ErrNoMatches, envtools.ErrNoTools:
default:
return err
}
for _, tool := range targetTools {
logger.Debugf("found target tool: %v", tool)
}
missing := sourceTools.Exclude(targetTools)
logger.Infof("found %d tools in target; %d tools to be copied", len(targetTools), len(missing))
err = copyTools(missing, syncContext, targetStorage)
if err != nil {
return err
}
logger.Infof("copied %d tools", len(missing))
logger.Infof("generating tools metadata")
if !syncContext.DryRun {
targetTools = append(targetTools, missing...)
writeMirrors := envtools.DoNotWriteMirrors
if syncContext.Public {
writeMirrors = envtools.WriteMirrors
}
err = envtools.MergeAndWriteMetadata(targetStorage, targetTools, writeMirrors)
if err != nil {
return err
}
}
logger.Infof("tools metadata written")
return nil
}
func (f StorageToolsFinder) FindTools(major int) (coretools.List, error) {
return envtools.ReadList(f.Storage, major, -1)
}
func (s *StorageSuite) TestReadListEmpty(c *gc.C) {
store := s.env.Storage()
_, err := envtools.ReadList(store, 2, 0)
c.Assert(err, gc.Equals, envtools.ErrNoTools)
}