func (s *simplestreamsSuite) TestWriteMetadataMergeWithExisting(c *gc.C) {
dir := c.MkDir()
existingToolsList := coretools.List{
{
Version: version.MustParseBinary("1.2.3-precise-amd64"),
Size: 123,
SHA256: "abc",
}, {
Version: version.MustParseBinary("2.0.1-raring-amd64"),
Size: 456,
SHA256: "xyz",
},
}
writer, err := filestorage.NewFileStorageWriter(dir)
c.Assert(err, gc.IsNil)
err = tools.MergeAndWriteMetadata(writer, existingToolsList, tools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
newToolsList := coretools.List{
existingToolsList[0],
{
Version: version.MustParseBinary("2.1.0-raring-amd64"),
Size: 789,
SHA256: "def",
},
}
err = tools.MergeAndWriteMetadata(writer, newToolsList, tools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
requiredToolsList := append(existingToolsList, newToolsList[1])
metadata := toolstesting.ParseMetadataFromDir(c, dir, false)
assertMetadataMatches(c, dir, requiredToolsList, metadata)
}
func (s *simplestreamsSuite) TestWriteMetadataNoFetch(c *gc.C) {
toolsList := coretools.List{
{
Version: version.MustParseBinary("1.2.3-precise-amd64"),
Size: 123,
SHA256: "abcd",
}, {
Version: version.MustParseBinary("2.0.1-raring-amd64"),
Size: 456,
SHA256: "xyz",
},
}
expected := toolsList
// Add tools with an unknown series. Do not add an entry in the
// expected list as these tools should be ignored.
vers, err := version.ParseBinary("3.2.1-xuanhuaceratops-amd64")
c.Assert(err, jc.Satisfies, series.IsUnknownOSForSeriesError)
toolsList = append(toolsList, &coretools.Tools{
Version: vers,
Size: 456,
SHA256: "wqe",
})
dir := c.MkDir()
writer, err := filestorage.NewFileStorageWriter(dir)
c.Assert(err, jc.ErrorIsNil)
err = tools.MergeAndWriteMetadata(writer, "proposed", "proposed", toolsList, tools.DoNotWriteMirrors)
c.Assert(err, jc.ErrorIsNil)
metadata := toolstesting.ParseMetadataFromDir(c, dir, "proposed", false)
assertMetadataMatches(c, dir, "proposed", expected, metadata)
}
// PrimeAgentVersion writes the configuration file and tools with version
// vers for an agent with the given entity name. It returns the agent's
// configuration and the current tools.
func (s *AgentSuite) PrimeAgentVersion(c *gc.C, tag names.Tag, password string, vers version.Binary) (agent.ConfigSetterWriter, *coretools.Tools) {
c.Logf("priming agent %s", tag.String())
stor, err := filestorage.NewFileStorageWriter(c.MkDir())
c.Assert(err, jc.ErrorIsNil)
agentTools := envtesting.PrimeTools(c, stor, s.DataDir(), "released", vers)
err = envtools.MergeAndWriteMetadata(stor, "released", "released", coretools.List{agentTools}, envtools.DoNotWriteMirrors)
tools1, err := agenttools.ChangeAgentTools(s.DataDir(), tag.String(), vers)
c.Assert(err, jc.ErrorIsNil)
c.Assert(tools1, gc.DeepEquals, agentTools)
stateInfo := s.MongoInfo(c)
apiInfo := s.APIInfo(c)
paths := agent.DefaultPaths
paths.DataDir = s.DataDir()
conf, err := agent.NewAgentConfig(
agent.AgentConfigParams{
Paths: paths,
Tag: tag,
UpgradedToVersion: vers.Number,
Password: password,
Nonce: agent.BootstrapNonce,
StateAddresses: stateInfo.Addrs,
APIAddresses: apiInfo.Addrs,
CACert: stateInfo.CACert,
Model: apiInfo.ModelTag,
})
c.Assert(err, jc.ErrorIsNil)
conf.SetPassword(password)
c.Assert(conf.Write(), gc.IsNil)
s.primeAPIHostPorts(c)
return conf, agentTools
}
// UploadFakeToolsVersions puts fake tools in the supplied storage for the supplied versions.
func UploadFakeToolsVersions(stor storage.Storage, toolsDir, stream string, versions ...version.Binary) ([]*coretools.Tools, error) {
// Leave existing tools alone.
existingTools := make(map[version.Binary]*coretools.Tools)
existing, _ := envtools.ReadList(stor, toolsDir, 1, -1)
for _, tools := range existing {
existingTools[tools.Version] = tools
}
var agentTools coretools.List = make(coretools.List, len(versions))
for i, version := range versions {
if tools, ok := existingTools[version]; ok {
agentTools[i] = tools
} else {
t, err := uploadFakeToolsVersion(stor, toolsDir, version)
if err != nil {
return nil, err
}
agentTools[i] = t
}
}
if err := envtools.MergeAndWriteMetadata(stor, toolsDir, stream, agentTools, envtools.DoNotWriteMirrors); err != nil {
return nil, err
}
err := SignTestTools(stor)
if err != nil {
return nil, err
}
return agentTools, nil
}
// AssertUploadFakeToolsVersions puts fake tools in the supplied storage for the supplied versions.
func AssertUploadFakeToolsVersions(c *gc.C, stor storage.Storage, toolsDir, stream string, versions ...version.Binary) []*coretools.Tools {
agentTools, err := UploadFakeToolsVersions(stor, toolsDir, stream, versions...)
c.Assert(err, jc.ErrorIsNil)
err = envtools.MergeAndWriteMetadata(stor, toolsDir, stream, agentTools, envtools.DoNotWriteMirrors)
c.Assert(err, jc.ErrorIsNil)
return agentTools
}
func makeTools(c *gc.C, metadataDir, stream string, versionStrings []string, withCheckSum bool) coretools.List {
toolsDir := filepath.Join(metadataDir, storage.BaseToolsPath, stream)
c.Assert(os.MkdirAll(toolsDir, 0755), gc.IsNil)
var toolsList coretools.List
for _, versionString := range versionStrings {
binary, err := version.ParseBinary(versionString)
if err != nil {
c.Assert(err, jc.Satisfies, series.IsUnknownOSForSeriesError)
}
path := filepath.Join(toolsDir, fmt.Sprintf("juju-%s.tgz", binary))
data := binary.String()
err = ioutil.WriteFile(path, []byte(data), 0644)
c.Assert(err, jc.ErrorIsNil)
tool := &coretools.Tools{
Version: binary,
URL: path,
}
if withCheckSum {
tool.Size, tool.SHA256 = SHA256sum(c, path)
}
toolsList = append(toolsList, tool)
}
// Write the tools metadata.
stor, err := filestorage.NewFileStorageWriter(metadataDir)
c.Assert(err, jc.ErrorIsNil)
err = tools.MergeAndWriteMetadata(stor, stream, stream, toolsList, false)
c.Assert(err, jc.ErrorIsNil)
return toolsList
}
func (s *simplestreamsSuite) assertWriteMetadata(c *gc.C, withMirrors bool) {
var versionStrings = []string{
"1.2.3-precise-amd64",
"2.0.1-raring-amd64",
}
dir := c.MkDir()
toolstesting.MakeTools(c, dir, "proposed", versionStrings)
toolsList := coretools.List{
{
// If sha256/size is already known, do not recalculate
Version: version.MustParseBinary("1.2.3-precise-amd64"),
Size: 123,
SHA256: "abcd",
}, {
Version: version.MustParseBinary("2.0.1-raring-amd64"),
// The URL is not used for generating metadata.
URL: "bogus://",
},
}
writer, err := filestorage.NewFileStorageWriter(dir)
c.Assert(err, jc.ErrorIsNil)
writeMirrors := tools.DoNotWriteMirrors
if withMirrors {
writeMirrors = tools.WriteMirrors
}
err = tools.MergeAndWriteMetadata(writer, "proposed", "proposed", toolsList, writeMirrors)
c.Assert(err, jc.ErrorIsNil)
metadata := toolstesting.ParseMetadataFromDir(c, dir, "proposed", withMirrors)
assertMetadataMatches(c, dir, "proposed", toolsList, metadata)
// No release stream generated so there will not be a legacy index file created.
_, err = writer.Get("tools/streams/v1/index.json")
c.Assert(err, gc.NotNil)
}
// primeAgent writes the configuration file and tools with version vers
// for an agent with the given entity name. It returns the agent's
// configuration and the current tools.
func (s *agentSuite) primeAgent(c *gc.C, tag, password string, vers version.Binary) (agent.ConfigSetterWriter, *coretools.Tools) {
stor := s.Conn.Environ.Storage()
agentTools := envtesting.PrimeTools(c, stor, s.DataDir(), vers)
err := envtools.MergeAndWriteMetadata(stor, coretools.List{agentTools}, envtools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
tools1, err := agenttools.ChangeAgentTools(s.DataDir(), tag, vers)
c.Assert(err, gc.IsNil)
c.Assert(tools1, gc.DeepEquals, agentTools)
stateInfo := s.StateInfo(c)
apiInfo := s.APIInfo(c)
conf, err := agent.NewAgentConfig(
agent.AgentConfigParams{
DataDir: s.DataDir(),
Tag: tag,
UpgradedToVersion: vers.Number,
Password: password,
Nonce: state.BootstrapNonce,
StateAddresses: stateInfo.Addrs,
APIAddresses: apiInfo.Addrs,
CACert: stateInfo.CACert,
})
conf.SetPassword(password)
c.Assert(conf.Write(), gc.IsNil)
s.primeAPIHostPorts(c)
return conf, agentTools
}
func (u StorageToolsUploader) UploadTools(tools *coretools.Tools, data []byte) error {
toolsName := envtools.StorageName(tools.Version)
if err := u.Storage.Put(toolsName, bytes.NewReader(data), int64(len(data))); err != nil {
return err
}
err := envtools.MergeAndWriteMetadata(u.Storage, coretools.List{tools}, u.WriteMirrors)
if err != nil {
logger.Errorf("error writing tools metadata: %v", err)
return err
}
return nil
}
// cloneToolsForSeries copies the built tools tarball into a tarball for the specified
// stream and series and generates corresponding metadata.
func cloneToolsForSeries(toolsInfo *BuiltTools, stream string, series ...string) error {
// Copy the tools to the target storage, recording a Tools struct for each one.
var targetTools coretools.List
targetTools = append(targetTools, &coretools.Tools{
Version: toolsInfo.Version,
Size: toolsInfo.Size,
SHA256: toolsInfo.Sha256Hash,
})
putTools := func(vers version.Binary) (string, error) {
name := envtools.StorageName(vers, stream)
src := filepath.Join(toolsInfo.Dir, toolsInfo.StorageName)
dest := filepath.Join(toolsInfo.Dir, name)
destDir := filepath.Dir(dest)
if err := os.MkdirAll(destDir, 0755); err != nil {
return "", err
}
if err := utils.CopyFile(dest, src); err != nil {
return "", err
}
// Append to targetTools the attributes required to write out tools metadata.
targetTools = append(targetTools, &coretools.Tools{
Version: vers,
Size: toolsInfo.Size,
SHA256: toolsInfo.Sha256Hash,
})
return name, nil
}
logger.Debugf("generating tarballs for %v", series)
for _, series := range series {
_, err := jujuseries.SeriesVersion(series)
if err != nil {
return err
}
if series != toolsInfo.Version.Series {
fakeVersion := toolsInfo.Version
fakeVersion.Series = series
if _, err := putTools(fakeVersion); err != nil {
return err
}
}
}
// The tools have been copied to a temp location from which they will be uploaded,
// now write out the matching simplestreams metadata so that SyncTools can find them.
metadataStore, err := filestorage.NewFileStorageWriter(toolsInfo.Dir)
if err != nil {
return err
}
logger.Debugf("generating tools metadata")
return envtools.MergeAndWriteMetadata(metadataStore, stream, stream, targetTools, false)
}
// uploadToStorage uploads the tools from the specified directory to environment storage.
func (h *toolsHandler) uploadToStorage(uploadedTools *tools.Tools, toolsDir,
toolsFilename string, fakeSeries ...string) (*tools.Tools, bool, error) {
// SyncTools requires simplestreams metadata to find the tools to upload.
stor, err := filestorage.NewFileStorageWriter(toolsDir)
if err != nil {
return nil, false, fmt.Errorf("cannot create metadata storage: %v", err)
}
// Generate metadata for the fake series. The URL for each fake series
// record points to the same tools tarball.
allToolsMetadata := []*tools.Tools{uploadedTools}
for _, series := range fakeSeries {
vers := uploadedTools.Version
vers.Series = series
allToolsMetadata = append(allToolsMetadata, &tools.Tools{
Version: vers,
URL: uploadedTools.URL,
Size: uploadedTools.Size,
SHA256: uploadedTools.SHA256,
})
}
err = envtools.MergeAndWriteMetadata(stor, allToolsMetadata, false)
if err != nil {
return nil, false, fmt.Errorf("cannot get environment config: %v", err)
}
// Create the environment so we can get the storage to which we upload the tools.
envConfig, err := h.state.EnvironConfig()
if err != nil {
return nil, false, fmt.Errorf("cannot get environment config: %v", err)
}
env, err := environs.New(envConfig)
if err != nil {
return nil, false, fmt.Errorf("cannot access environment: %v", err)
}
// Now perform the upload.
builtTools := &sync.BuiltTools{
Version: uploadedTools.Version,
Dir: toolsDir,
StorageName: toolsFilename,
Size: uploadedTools.Size,
Sha256Hash: uploadedTools.SHA256,
}
uploadedTools, err = sync.SyncBuiltTools(env.Storage(), builtTools, fakeSeries...)
if err != nil {
return nil, false, err
}
return uploadedTools, !envConfig.SSLHostnameVerification(), nil
}
// MustUploadFakeToolsVersions acts as UploadFakeToolsVersions, but panics on failure.
func MustUploadFakeToolsVersions(stor storage.Storage, stream string, versions ...version.Binary) []*coretools.Tools {
var agentTools coretools.List = make(coretools.List, len(versions))
for i, version := range versions {
t, err := uploadFakeToolsVersion(stor, stream, version)
if err != nil {
panic(err)
}
agentTools[i] = t
}
err := envtools.MergeAndWriteMetadata(stor, stream, stream, agentTools, envtools.DoNotWriteMirrors)
if err != nil {
panic(err)
}
return agentTools
}
func (s *UpgraderSuite) TestUpgraderSetsTools(c *gc.C) {
vers := version.MustParseBinary("5.4.3-precise-amd64")
err := statetesting.SetAgentVersion(s.State, vers.Number)
c.Assert(err, gc.IsNil)
stor := s.Environ.Storage()
agentTools := envtesting.PrimeTools(c, stor, s.DataDir(), vers)
s.PatchValue(&version.Current, agentTools.Version)
err = envtools.MergeAndWriteMetadata(stor, coretools.List{agentTools}, envtools.DoNotWriteMirrors)
_, err = s.machine.AgentTools()
c.Assert(err, jc.Satisfies, errors.IsNotFound)
u := s.makeUpgrader()
statetesting.AssertStop(c, u)
s.machine.Refresh()
gotTools, err := s.machine.AgentTools()
c.Assert(err, gc.IsNil)
envtesting.CheckTools(c, gotTools, agentTools)
}
func (s *UpgraderSuite) TestUpgraderSetsTools(c *gc.C) {
vers := version.MustParseBinary("5.4.3-precise-amd64")
err := statetesting.SetAgentVersion(s.State, vers.Number)
c.Assert(err, jc.ErrorIsNil)
stor := s.DefaultToolsStorage
agentTools := envtesting.PrimeTools(c, stor, s.DataDir(), s.Environ.Config().AgentStream(), vers)
s.patchVersion(agentTools.Version)
err = envtools.MergeAndWriteMetadata(stor, "released", "released", coretools.List{agentTools}, envtools.DoNotWriteMirrors)
_, err = s.machine.AgentTools()
c.Assert(err, jc.Satisfies, errors.IsNotFound)
u := s.makeUpgrader(c)
statetesting.AssertStop(c, u)
s.expectInitialUpgradeCheckDone(c)
s.machine.Refresh()
gotTools, err := s.machine.AgentTools()
c.Assert(err, jc.ErrorIsNil)
envtesting.CheckTools(c, gotTools, agentTools)
}
func (s *simplestreamsSuite) TestWriteMetadataNoFetch(c *gc.C) {
toolsList := coretools.List{
{
Version: version.MustParseBinary("1.2.3-precise-amd64"),
Size: 123,
SHA256: "abcd",
}, {
Version: version.MustParseBinary("2.0.1-raring-amd64"),
Size: 456,
SHA256: "xyz",
},
}
dir := c.MkDir()
writer, err := filestorage.NewFileStorageWriter(dir)
c.Assert(err, gc.IsNil)
err = tools.MergeAndWriteMetadata(writer, toolsList, tools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
metadata := toolstesting.ParseMetadataFromDir(c, dir, false)
assertMetadataMatches(c, dir, toolsList, metadata)
}
func (s *UpgraderSuite) TestChangeAgentTools(c *gc.C) {
oldTools := &coretools.Tools{
Version: version.MustParseBinary("1.2.3-quantal-amd64"),
}
stor := s.Conn.Environ.Storage()
newTools := envtesting.PrimeTools(c, stor, s.DataDir(), version.MustParseBinary("5.4.3-precise-amd64"))
s.PatchValue(&version.Current, newTools.Version)
err := envtools.MergeAndWriteMetadata(stor, coretools.List{newTools}, envtools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
ugErr := &upgrader.UpgradeReadyError{
AgentName: "anAgent",
OldTools: oldTools.Version,
NewTools: newTools.Version,
DataDir: s.DataDir(),
}
err = ugErr.ChangeAgentTools()
c.Assert(err, gc.IsNil)
link, err := symlink.Read(agenttools.ToolsDir(s.DataDir(), "anAgent"))
c.Assert(err, gc.IsNil)
c.Assert(link, gc.Equals, newTools.Version.String())
}
func (s *UpgraderSuite) TestChangeAgentTools(c *gc.C) {
oldTools := &coretools.Tools{
Version: version.MustParseBinary("1.2.3-quantal-amd64"),
}
stor := s.DefaultToolsStorage
newToolsBinary := "5.4.3-precise-amd64"
newTools := envtesting.PrimeTools(c, stor, s.DataDir(), s.Environ.Config().AgentStream(), version.MustParseBinary(newToolsBinary))
s.PatchValue(&version.Current, newTools.Version)
err := envtools.MergeAndWriteMetadata(stor, "released", "released", coretools.List{newTools}, envtools.DoNotWriteMirrors)
c.Assert(err, jc.ErrorIsNil)
ugErr := &upgrader.UpgradeReadyError{
AgentName: "anAgent",
OldTools: oldTools.Version,
NewTools: newTools.Version,
DataDir: s.DataDir(),
}
err = ugErr.ChangeAgentTools()
c.Assert(err, jc.ErrorIsNil)
target := agenttools.ToolsDir(s.DataDir(), newToolsBinary)
link, err := symlink.Read(agenttools.ToolsDir(s.DataDir(), "anAgent"))
c.Assert(err, jc.ErrorIsNil)
c.Assert(link, jc.SamePath, target)
}
// SyncTools copies the Juju tools tarball from the official bucket
// or a specified source directory into the user's environment.
func SyncTools(syncContext *SyncContext) error {
sourceDataSource, err := selectSourceDatasource(syncContext)
if err != nil {
return err
}
logger.Infof("listing available tools")
if syncContext.MajorVersion == 0 && syncContext.MinorVersion == 0 {
syncContext.MajorVersion = version.Current.Major
syncContext.MinorVersion = -1
if !syncContext.AllVersions {
syncContext.MinorVersion = version.Current.Minor
}
} else if !syncContext.Dev && syncContext.MinorVersion != -1 {
// If a major.minor version is specified, we allow dev versions.
// If Dev is already true, leave it alone.
syncContext.Dev = true
}
released := !syncContext.Dev && !version.Current.IsDev()
sourceTools, err := envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{Released: released})
if err != nil {
return err
}
logger.Infof("found %d tools", len(sourceTools))
if !syncContext.AllVersions {
var latest version.Number
latest, sourceTools = sourceTools.Newest()
logger.Infof("found %d recent tools (version %s)", len(sourceTools), latest)
}
for _, tool := range sourceTools {
logger.Debugf("found source tool: %v", tool)
}
logger.Infof("listing target tools storage")
targetStorage := syncContext.Target
targetTools, err := envtools.ReadList(targetStorage, syncContext.MajorVersion, -1)
switch err {
case nil, coretools.ErrNoMatches, envtools.ErrNoTools:
default:
return err
}
for _, tool := range targetTools {
logger.Debugf("found target tool: %v", tool)
}
missing := sourceTools.Exclude(targetTools)
logger.Infof("found %d tools in target; %d tools to be copied", len(targetTools), len(missing))
err = copyTools(missing, syncContext, targetStorage)
if err != nil {
return err
}
logger.Infof("copied %d tools", len(missing))
logger.Infof("generating tools metadata")
if !syncContext.DryRun {
targetTools = append(targetTools, missing...)
writeMirrors := envtools.DoNotWriteMirrors
if syncContext.Public {
writeMirrors = envtools.WriteMirrors
}
err = envtools.MergeAndWriteMetadata(targetStorage, targetTools, writeMirrors)
if err != nil {
return err
}
}
logger.Infof("tools metadata written")
return nil
}