func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuHome()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return err
}
// We now store the tools in a directory named after their stream, but the
// legacy behaviour is to store all tools in a single "releases" directory.
toolsDir := c.stream
if c.stream == "" {
fmt.Fprintf(context.Stdout, "No stream specified, defaulting to released tools in the releases directory.\n")
c.stream = envtools.ReleasedStream
toolsDir = envtools.LegacyReleaseDirectory
}
fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
const minorVersion = -1
toolsList, err := envtools.ReadList(sourceStorage, toolsDir, version.Current.Major, minorVersion)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return err
}
sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames)
toolsList, err = envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{}, c.stream,
version.Current.Major, minorVersion, coretools.Filter{})
}
if err != nil {
return err
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return err
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return mergeAndWriteMetadata(targetStorage, toolsDir, c.stream, c.clean, toolsList, writeMirrors)
}
func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
writer := loggo.NewMinimumLevelWriter(
cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr),
loggo.INFO)
loggo.RegisterWriter("toolsmetadata", writer)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuXDGDataHomeDir()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return err
}
// We now store the tools in a directory named after their stream, but the
// legacy behaviour is to store all tools in a single "releases" directory.
toolsDir := c.stream
if c.stream == "" {
fmt.Fprintln(context.Stdout, "No stream specified, defaulting to released tools in the releases directory.")
c.stream = envtools.ReleasedStream
toolsDir = envtools.LegacyReleaseDirectory
}
fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
toolsList, err := envtools.ReadList(sourceStorage, toolsDir, -1, -1)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return err
}
toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, c.stream, -1, -1, coretools.Filter{})
}
if err != nil {
return err
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return err
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return mergeAndWriteMetadata(targetStorage, toolsDir, c.stream, c.clean, toolsList, writeMirrors)
}
func (c *ToolsMetadataCommand) Run(context *cmd.Context) error {
loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuHome()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return err
}
fmt.Fprintf(context.Stdout, "Finding tools in %s\n", c.metadataDir)
const minorVersion = -1
toolsList, err := envtools.ReadList(sourceStorage, version.Current.Major, minorVersion)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return err
}
sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames)
toolsList, err = envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
version.Current.Major, minorVersion, coretools.Filter{})
}
if err != nil {
return err
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return err
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return mergeAndWriteMetadata(targetStorage, toolsList, writeMirrors)
}
func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
writer := loggo.NewMinimumLevelWriter(
cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr),
loggo.INFO)
loggo.RegisterWriter("toolsmetadata", writer)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuXDGDataHomeDir()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return errors.Trace(err)
}
fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
toolsList, err := envtools.ReadList(sourceStorage, c.stream, -1, -1)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return errors.Trace(err)
}
toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, c.stream, -1, -1, coretools.Filter{})
}
if err != nil {
return errors.Trace(err)
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return errors.Trace(err)
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return errors.Trace(mergeAndWriteMetadata(targetStorage, c.stream, c.stream, c.clean, toolsList, writeMirrors))
}
// SyncTools copies the Juju tools tarball from the official bucket
// or a specified source directory into the user's environment.
func SyncTools(syncContext *SyncContext) error {
sourceDataSource, err := selectSourceDatasource(syncContext)
if err != nil {
return err
}
logger.Infof("listing available tools")
if syncContext.MajorVersion == 0 && syncContext.MinorVersion == 0 {
syncContext.MajorVersion = version.Current.Major
syncContext.MinorVersion = -1
if !syncContext.AllVersions {
syncContext.MinorVersion = version.Current.Minor
}
} else if !syncContext.Dev && syncContext.MinorVersion != -1 {
// If a major.minor version is specified, we allow dev versions.
// If Dev is already true, leave it alone.
syncContext.Dev = true
}
released := !syncContext.Dev && !version.Current.IsDev()
sourceTools, err := envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{Released: released})
if err != nil {
return err
}
logger.Infof("found %d tools", len(sourceTools))
if !syncContext.AllVersions {
var latest version.Number
latest, sourceTools = sourceTools.Newest()
logger.Infof("found %d recent tools (version %s)", len(sourceTools), latest)
}
for _, tool := range sourceTools {
logger.Debugf("found source tool: %v", tool)
}
logger.Infof("listing target tools storage")
targetStorage := syncContext.Target
targetTools, err := envtools.ReadList(targetStorage, syncContext.MajorVersion, -1)
switch err {
case nil, coretools.ErrNoMatches, envtools.ErrNoTools:
default:
return err
}
for _, tool := range targetTools {
logger.Debugf("found target tool: %v", tool)
}
missing := sourceTools.Exclude(targetTools)
logger.Infof("found %d tools in target; %d tools to be copied", len(targetTools), len(missing))
err = copyTools(missing, syncContext, targetStorage)
if err != nil {
return err
}
logger.Infof("copied %d tools", len(missing))
logger.Infof("generating tools metadata")
if !syncContext.DryRun {
targetTools = append(targetTools, missing...)
writeMirrors := envtools.DoNotWriteMirrors
if syncContext.Public {
writeMirrors = envtools.WriteMirrors
}
err = envtools.MergeAndWriteMetadata(targetStorage, targetTools, writeMirrors)
if err != nil {
return err
}
}
logger.Infof("tools metadata written")
return nil
}
// SyncTools copies the Juju tools tarball from the official bucket
// or a specified source directory into the user's environment.
func SyncTools(syncContext *SyncContext) error {
sourceDataSource, err := selectSourceDatasource(syncContext)
if err != nil {
return err
}
logger.Infof("listing available tools")
if syncContext.MajorVersion == 0 && syncContext.MinorVersion == 0 {
syncContext.MajorVersion = version.Current.Major
syncContext.MinorVersion = -1
if !syncContext.AllVersions {
syncContext.MinorVersion = version.Current.Minor
}
} else if !syncContext.Dev && syncContext.MinorVersion != -1 {
// If a major.minor version is specified, we allow dev versions.
// If Dev is already true, leave it alone.
syncContext.Dev = true
}
released := !syncContext.Dev && !version.Current.IsDev()
sourceTools, err := envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{Released: released})
if err != nil {
return err
}
logger.Infof("found %d tools", len(sourceTools))
if !syncContext.AllVersions {
var latest version.Number
latest, sourceTools = sourceTools.Newest()
logger.Infof("found %d recent tools (version %s)", len(sourceTools), latest)
}
for _, tool := range sourceTools {
logger.Debugf("found source tool: %v", tool)
}
logger.Infof("listing target tools storage")
targetTools, err := syncContext.TargetToolsFinder.FindTools(syncContext.MajorVersion)
switch err {
case nil, coretools.ErrNoMatches, envtools.ErrNoTools:
default:
return err
}
for _, tool := range targetTools {
logger.Debugf("found target tool: %v", tool)
}
missing := sourceTools.Exclude(targetTools)
logger.Infof("found %d tools in target; %d tools to be copied", len(targetTools), len(missing))
if syncContext.DryRun {
for _, tools := range missing {
logger.Infof("copying %s from %s", tools.Version, tools.URL)
}
return nil
}
err = copyTools(missing, syncContext.TargetToolsUploader)
if err != nil {
return err
}
logger.Infof("copied %d tools", len(missing))
return nil
}
// SyncTools copies the Juju tools tarball from the official bucket
// or a specified source directory into the user's environment.
func SyncTools(syncContext *SyncContext) error {
sourceDataSource, err := selectSourceDatasource(syncContext)
if err != nil {
return err
}
logger.Infof("listing available tools")
if syncContext.MajorVersion == 0 && syncContext.MinorVersion == 0 {
syncContext.MajorVersion = jujuversion.Current.Major
syncContext.MinorVersion = -1
if !syncContext.AllVersions {
syncContext.MinorVersion = jujuversion.Current.Minor
}
}
toolsDir := syncContext.Stream
// If no stream has been specified, assume "released" for non-devel versions of Juju.
if syncContext.Stream == "" {
// We now store the tools in a directory named after their stream, but the
// legacy behaviour is to store all tools in a single "releases" directory.
toolsDir = envtools.LegacyReleaseDirectory
syncContext.Stream = envtools.PreferredStream(&jujuversion.Current, false, syncContext.Stream)
}
sourceTools, err := envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
syncContext.Stream, syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{})
// For backwards compatibility with cloud storage, if there are no tools in the specified stream,
// double check the release stream.
// TODO - remove this when we no longer need to support cloud storage upgrades.
if err == envtools.ErrNoTools {
sourceTools, err = envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
envtools.ReleasedStream, syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{})
}
if err != nil {
return err
}
logger.Infof("found %d tools", len(sourceTools))
if !syncContext.AllVersions {
var latest version.Number
latest, sourceTools = sourceTools.Newest()
logger.Infof("found %d recent tools (version %s)", len(sourceTools), latest)
}
for _, tool := range sourceTools {
logger.Debugf("found source tool: %v", tool)
}
logger.Infof("listing target tools storage")
targetTools, err := syncContext.TargetToolsFinder.FindTools(syncContext.MajorVersion, syncContext.Stream)
switch err {
case nil, coretools.ErrNoMatches, envtools.ErrNoTools:
default:
return err
}
for _, tool := range targetTools {
logger.Debugf("found target tool: %v", tool)
}
missing := sourceTools.Exclude(targetTools)
logger.Infof("found %d tools in target; %d tools to be copied", len(targetTools), len(missing))
if syncContext.DryRun {
for _, tools := range missing {
logger.Infof("copying %s from %s", tools.Version, tools.URL)
}
return nil
}
err = copyTools(toolsDir, syncContext.Stream, missing, syncContext.TargetToolsUploader)
if err != nil {
return err
}
logger.Infof("copied %d tools", len(missing))
return nil
}
// migrateToolsStorage copies tools from provider storage to
// environment storage.
func migrateToolsStorage(st *state.State, agentConfig agent.Config) error {
logger.Debugf("migrating tools to environment storage")
tstor, err := stateToolsStorage(st)
if err != nil {
return errors.Annotate(err, "cannot get tools storage")
}
defer tstor.Close()
// Local and manual provider host storage on the state server's
// filesystem, and serve via HTTP storage. The storage worker
// doesn't run yet, so we just open the files directly.
var stor storage.StorageReader
providerType := agentConfig.Value(agent.ProviderType)
if providerType == provider.Local || provider.IsManual(providerType) {
storageDir := agentConfig.Value(agent.StorageDir)
var err error
stor, err = filestorage.NewFileStorageReader(storageDir)
if err != nil {
return errors.Annotate(err, "cannot get local filesystem storage reader")
}
} else {
var err error
stor, err = environs.LegacyStorage(st)
if errors.IsNotSupported(err) {
return nil
} else if err != nil {
return errors.Annotate(err, "cannot get provider storage")
}
}
// Search provider storage for tools.
datasource := storage.NewStorageSimpleStreamsDataSource("provider storage", stor, storage.BaseToolsPath)
toolsList, err := envtools.FindToolsForCloud(
[]simplestreams.DataSource{datasource},
simplestreams.CloudSpec{},
envtools.ReleasedStream,
-1, -1, tools.Filter{})
switch err {
case nil:
break
case tools.ErrNoMatches, envtools.ErrNoTools:
// No tools in provider storage: nothing to do.
return nil
default:
return errors.Annotate(err, "cannot find tools in provider storage")
}
for _, agentTools := range toolsList {
logger.Infof("migrating %v tools to environment storage", agentTools.Version)
data, err := fetchToolsArchive(stor, envtools.LegacyReleaseDirectory, agentTools)
if err != nil {
return errors.Annotatef(err, "failed to fetch %v tools", agentTools.Version)
}
err = tstor.AddTools(bytes.NewReader(data), toolstorage.Metadata{
Version: agentTools.Version,
Size: agentTools.Size,
SHA256: agentTools.SHA256,
})
if err != nil {
return errors.Annotatef(err, "failed to add %v tools to environment storage", agentTools.Version)
}
}
return nil
}
