func (s *toolsSuite) TestUploadSeriesExpanded(c *gc.C) {
// Make some fake tools.
expectedTools, vers, toolPath := s.setupToolsForUpload(c)
// Now try uploading them. The "series" parameter is accepted
// but ignored; the API server will expand the tools for all
// supported series.
params := "?binaryVersion=" + vers.String() + "&series=nonsense"
resp, err := s.uploadRequest(c, s.toolsURI(c, params), true, toolPath)
c.Assert(err, gc.IsNil)
// Check the response.
stor := s.Environ.Storage()
toolsURL, err := stor.URL(tools.StorageName(vers))
c.Assert(err, gc.IsNil)
expectedTools[0].URL = toolsURL
s.assertUploadResponse(c, resp, expectedTools[0])
// Check the contents.
for _, series := range version.OSSupportedSeries(version.Ubuntu) {
toolsVersion := vers
toolsVersion.Series = series
r, err := stor.Get(tools.StorageName(toolsVersion))
c.Assert(err, gc.IsNil)
uploadedData, err := ioutil.ReadAll(r)
c.Assert(err, gc.IsNil)
expectedData, err := ioutil.ReadFile(toolPath)
c.Assert(err, gc.IsNil)
c.Assert(uploadedData, gc.DeepEquals, expectedData)
}
}
func (s *toolsSuite) TestUploadFakeSeries(c *gc.C) {
// Make some fake tools.
expectedTools, vers, toolPath := s.setupToolsForUpload(c)
// Now try uploading them.
params := "?binaryVersion=" + vers.String() + "&series=precise,trusty"
resp, err := s.uploadRequest(c, s.toolsURI(c, params), true, toolPath)
c.Assert(err, gc.IsNil)
// Check the response.
stor := s.Conn.Environ.Storage()
toolsURL, err := stor.URL(tools.StorageName(vers))
c.Assert(err, gc.IsNil)
expectedTools[0].URL = toolsURL
s.assertUploadResponse(c, resp, expectedTools[0])
// Check the contents.
for _, series := range []string{"precise", "quantal", "trusty"} {
toolsVersion := vers
toolsVersion.Series = series
r, err := stor.Get(tools.StorageName(toolsVersion))
c.Assert(err, gc.IsNil)
uploadedData, err := ioutil.ReadAll(r)
c.Assert(err, gc.IsNil)
expectedData, err := ioutil.ReadFile(toolPath)
c.Assert(err, gc.IsNil)
c.Assert(uploadedData, gc.DeepEquals, expectedData)
}
}
func (t *LiveTests) TestBootstrapWithDefaultSeries(c *gc.C) {
if !t.HasProvisioner {
c.Skip("HasProvisioner is false; cannot test deployment")
}
current := version.Current
other := current
other.Series = "quantal"
if current == other {
other.Series = "precise"
}
dummyCfg, err := config.New(config.NoDefaults, dummy.SampleConfig().Merge(coretesting.Attrs{
"state-server": false,
"name": "dummy storage",
}))
dummyenv, err := environs.Prepare(dummyCfg, coretesting.Context(c), configstore.NewMem())
c.Assert(err, gc.IsNil)
defer dummyenv.Destroy()
t.Destroy(c)
attrs := t.TestConfig.Merge(coretesting.Attrs{"default-series": other.Series})
cfg, err := config.New(config.NoDefaults, attrs)
c.Assert(err, gc.IsNil)
env, err := environs.Prepare(cfg, coretesting.Context(c), t.ConfigStore)
c.Assert(err, gc.IsNil)
defer environs.Destroy(env, t.ConfigStore)
currentName := envtools.StorageName(current)
otherName := envtools.StorageName(other)
envStorage := env.Storage()
dummyStorage := dummyenv.Storage()
defer envStorage.Remove(otherName)
_, err = sync.Upload(dummyStorage, ¤t.Number)
c.Assert(err, gc.IsNil)
// This will only work while cross-compiling across releases is safe,
// which depends on external elements. Tends to be safe for the last
// few releases, but we may have to refactor some day.
err = storageCopy(dummyStorage, currentName, envStorage, otherName)
c.Assert(err, gc.IsNil)
err = bootstrap.Bootstrap(coretesting.Context(c), env, environs.BootstrapParams{})
c.Assert(err, gc.IsNil)
st := t.Env.(testing.GetStater).GetStateInAPIServer()
// Wait for machine agent to come up on the bootstrap
// machine and ensure it deployed the proper series.
m0, err := st.Machine("0")
c.Assert(err, gc.IsNil)
mw0 := newMachineToolWaiter(m0)
defer mw0.Stop()
waitAgentTools(c, mw0, other)
}
// populateTools stores uploaded tools in provider storage
// and updates the tools metadata.
//
// TODO(axw) store tools in gridfs, catalogue in state.
func (c *BootstrapCommand) populateTools(env environs.Environ) error {
agentConfig := c.CurrentConfig()
dataDir := agentConfig.DataDir()
tools, err := agenttools.ReadTools(dataDir, version.Current)
if err != nil {
return err
}
if !strings.HasPrefix(tools.URL, "file://") {
// Nothing to do since the tools were not uploaded.
return nil
}
// This is a hack: providers using localstorage (local, manual)
// can't use storage during bootstrap as the localstorage worker
// isn't running. Use filestorage instead.
var stor storage.Storage
storageDir := agentConfig.Value(agent.StorageDir)
if storageDir != "" {
stor, err = filestorage.NewFileStorageWriter(storageDir)
if err != nil {
return err
}
} else {
stor = env.Storage()
}
// Create a temporary directory to contain source and cloned tools.
tempDir, err := ioutil.TempDir("", "juju-sync-tools")
if err != nil {
return err
}
defer os.RemoveAll(tempDir)
destTools := filepath.Join(tempDir, filepath.FromSlash(envtools.StorageName(tools.Version)))
if err := os.MkdirAll(filepath.Dir(destTools), 0700); err != nil {
return err
}
srcTools := filepath.Join(
agenttools.SharedToolsDir(dataDir, version.Current),
"tools.tar.gz",
)
if err := utils.CopyFile(destTools, srcTools); err != nil {
return err
}
// Until we catalogue tools in state, we clone the tools
// for each of the supported series of the same OS.
otherSeries := version.OSSupportedSeries(version.Current.OS)
_, err = sync.SyncBuiltTools(stor, &sync.BuiltTools{
Version: tools.Version,
Dir: tempDir,
StorageName: envtools.StorageName(tools.Version),
Sha256Hash: tools.SHA256,
Size: tools.Size,
}, otherSeries...)
return err
}
// RemoveFakeTools deletes the fake tools from the supplied storage.
func RemoveFakeTools(c *gc.C, stor storage.Storage, toolsDir string) {
c.Logf("removing fake tools")
toolsVersion := version.Current
name := envtools.StorageName(toolsVersion, toolsDir)
err := stor.Remove(name)
c.Check(err, jc.ErrorIsNil)
defaultSeries := coretesting.FakeDefaultSeries
if series.HostSeries() != defaultSeries {
toolsVersion.Series = defaultSeries
name := envtools.StorageName(toolsVersion, toolsDir)
err := stor.Remove(name)
c.Check(err, jc.ErrorIsNil)
}
RemoveFakeToolsMetadata(c, stor)
}
// RemoveFakeTools deletes the fake tools from the supplied storage.
func RemoveFakeTools(c *gc.C, stor storage.Storage) {
c.Logf("removing fake tools")
toolsVersion := version.Current
name := envtools.StorageName(toolsVersion)
err := stor.Remove(name)
c.Check(err, gc.IsNil)
defaultSeries := coretesting.FakeDefaultSeries
if version.Current.Series != defaultSeries {
toolsVersion.Series = defaultSeries
name := envtools.StorageName(toolsVersion)
err := stor.Remove(name)
c.Check(err, gc.IsNil)
}
RemoveFakeToolsMetadata(c, stor)
}
func (*metadataHelperSuite) TestMetadataFromTools(c *gc.C) {
metadata := tools.MetadataFromTools(nil, "proposed")
c.Assert(metadata, gc.HasLen, 0)
toolsList := coretools.List{{
Version: version.MustParseBinary("1.2.3-precise-amd64"),
Size: 123,
SHA256: "abc",
}, {
Version: version.MustParseBinary("2.0.1-raring-amd64"),
URL: "file:///tmp/proposed/juju-2.0.1-raring-amd64.tgz",
Size: 456,
SHA256: "xyz",
}}
metadata = tools.MetadataFromTools(toolsList, "proposed")
c.Assert(metadata, gc.HasLen, len(toolsList))
for i, t := range toolsList {
md := metadata[i]
c.Assert(md.Release, gc.Equals, t.Version.Series)
c.Assert(md.Version, gc.Equals, t.Version.Number.String())
c.Assert(md.Arch, gc.Equals, t.Version.Arch)
// FullPath is only filled out when reading tools using simplestreams.
// It's not needed elsewhere and requires a URL() call.
c.Assert(md.FullPath, gc.Equals, "")
c.Assert(md.Path, gc.Equals, tools.StorageName(t.Version, "proposed")[len("tools/"):])
c.Assert(md.FileType, gc.Equals, "tar.gz")
c.Assert(md.Size, gc.Equals, t.Size)
c.Assert(md.SHA256, gc.Equals, t.SHA256)
}
}
func (s *bootstrapSuite) TestBootstrapToolsFileURL(c *gc.C) {
storageName := tools.StorageName(version.Current)
sftpURL, err := s.env.Storage().URL(storageName)
c.Assert(err, gc.IsNil)
fileURL := fmt.Sprintf("file://%s/%s", s.env.storageDir, storageName)
s.testBootstrapToolsURL(c, sftpURL, fileURL)
}
func (s *toolsSuite) setupToolsForUpload(c *gc.C) (coretools.List, version.Binary, string) {
localStorage := c.MkDir()
vers := version.MustParseBinary("1.9.0-quantal-amd64")
versionStrings := []string{vers.String()}
expectedTools := toolstesting.MakeToolsWithCheckSum(c, localStorage, "releases", versionStrings)
toolsFile := tools.StorageName(vers)
return expectedTools, vers, path.Join(localStorage, toolsFile)
}
// RemoveFakeTools deletes the fake tools from the supplied storage.
func RemoveFakeTools(c *gc.C, stor storage.Storage, toolsDir string) {
c.Logf("removing fake tools")
toolsVersion := version.Binary{
Number: jujuversion.Current,
Arch: arch.HostArch(),
Series: series.HostSeries(),
}
name := envtools.StorageName(toolsVersion, toolsDir)
err := stor.Remove(name)
c.Check(err, jc.ErrorIsNil)
defaultSeries := series.LatestLts()
if series.HostSeries() != defaultSeries {
toolsVersion.Series = defaultSeries
name := envtools.StorageName(toolsVersion, toolsDir)
err := stor.Remove(name)
c.Check(err, jc.ErrorIsNil)
}
RemoveFakeToolsMetadata(c, stor)
}
func (s *UpgraderSuite) TestUpgraderRetryAndChanged(c *gc.C) {
stor := s.DefaultToolsStorage
oldTools := envtesting.PrimeTools(c, stor, s.DataDir(), s.Environ.Config().AgentStream(), version.MustParseBinary("5.4.3-precise-amd64"))
s.PatchValue(&version.Current, oldTools.Version)
newTools := envtesting.AssertUploadFakeToolsVersions(
c, stor, s.Environ.Config().AgentStream(), s.Environ.Config().AgentStream(), version.MustParseBinary("5.4.5-precise-amd64"))[0]
err := statetesting.SetAgentVersion(s.State, newTools.Version.Number)
c.Assert(err, jc.ErrorIsNil)
retryc := make(chan time.Time)
*upgrader.RetryAfter = func() <-chan time.Time {
c.Logf("replacement retry after")
return retryc
}
err = stor.Remove(envtools.StorageName(newTools.Version, "released"))
c.Assert(err, jc.ErrorIsNil)
u := s.makeUpgrader(c)
defer u.Stop()
s.expectUpgradeChannelNotClosed(c)
for i := 0; i < 3; i++ {
select {
case retryc <- time.Now():
case <-time.After(coretesting.LongWait):
c.Fatalf("upgrader did not retry (attempt %d)", i)
}
}
// Make it upgrade to some newer tools that can be
// downloaded ok; it should stop retrying, download
// the newer tools and exit.
newerTools := envtesting.AssertUploadFakeToolsVersions(
c, stor, s.Environ.Config().AgentStream(), s.Environ.Config().AgentStream(), version.MustParseBinary("5.4.6-precise-amd64"))[0]
err = statetesting.SetAgentVersion(s.State, newerTools.Version.Number)
c.Assert(err, jc.ErrorIsNil)
s.BackingState.StartSync()
done := make(chan error)
go func() {
done <- u.Wait()
}()
select {
case err := <-done:
envtesting.CheckUpgraderReadyError(c, err, &upgrader.UpgradeReadyError{
AgentName: s.machine.Tag().String(),
OldTools: oldTools.Version,
NewTools: newerTools.Version,
DataDir: s.DataDir(),
})
case <-time.After(coretesting.LongWait):
c.Fatalf("upgrader did not quit after upgrading")
}
}
func (u StorageToolsUploader) UploadTools(tools *coretools.Tools, data []byte) error {
toolsName := envtools.StorageName(tools.Version)
if err := u.Storage.Put(toolsName, bytes.NewReader(data), int64(len(data))); err != nil {
return err
}
err := envtools.MergeAndWriteMetadata(u.Storage, coretools.List{tools}, u.WriteMirrors)
if err != nil {
logger.Errorf("error writing tools metadata: %v", err)
return err
}
return nil
}
func (s *toolsSuite) TestDownloadFetchesAndVerifiesSize(c *gc.C) {
// Upload fake tools, then upload over the top so the SHA256 hash does not match.
stor := s.Environ.Storage()
envtesting.RemoveTools(c, stor)
tools := envtesting.AssertUploadFakeToolsVersions(c, stor, version.Current)[0]
err := stor.Put(envtools.StorageName(tools.Version), strings.NewReader("!"), 1)
resp, err := s.downloadRequest(c, tools.Version, "")
c.Assert(err, gc.IsNil)
s.assertErrorResponse(c, resp, http.StatusBadRequest, "error fetching tools: size mismatch for .*")
s.assertToolsNotStored(c, tools.Version)
}
// buildToolsTarball bundles a tools tarball and places it in a temp directory in
// the expected tools path.
func buildToolsTarball(forceVersion *version.Number) (builtTools *BuiltTools, err error) {
// TODO(rog) find binaries from $PATH when not using a development
// version of juju within a $GOPATH.
logger.Debugf("Building tools")
// We create the entire archive before asking the environment to
// start uploading so that we can be sure we have archived
// correctly.
f, err := ioutil.TempFile("", "juju-tgz")
if err != nil {
return nil, err
}
defer f.Close()
defer os.Remove(f.Name())
toolsVersion, sha256Hash, err := envtools.BundleTools(f, forceVersion)
if err != nil {
return nil, err
}
fileInfo, err := f.Stat()
if err != nil {
return nil, fmt.Errorf("cannot stat newly made tools archive: %v", err)
}
size := fileInfo.Size()
logger.Infof("built tools %v (%dkB)", toolsVersion, (size+512)/1024)
baseToolsDir, err := ioutil.TempDir("", "juju-tools")
if err != nil {
return nil, err
}
// If we exit with an error, clean up the built tools directory.
defer func() {
if err != nil {
os.RemoveAll(baseToolsDir)
}
}()
err = os.MkdirAll(filepath.Join(baseToolsDir, storage.BaseToolsPath, "releases"), 0755)
if err != nil {
return nil, err
}
storageName := envtools.StorageName(toolsVersion)
err = utils.CopyFile(filepath.Join(baseToolsDir, storageName), f.Name())
if err != nil {
return nil, err
}
return &BuiltTools{
Version: toolsVersion,
Dir: baseToolsDir,
StorageName: storageName,
Size: size,
Sha256Hash: sha256Hash,
}, nil
}
func (s *toolsSuite) TestDownloadFetchesAndVerifiesSize(c *gc.C) {
// Upload fake tools, then upload over the top so the SHA256 hash does not match.
s.PatchValue(&version.Current.Number, testing.FakeVersionNumber)
stor := s.DefaultToolsStorage
envtesting.RemoveTools(c, stor, "released")
tools := envtesting.AssertUploadFakeToolsVersions(c, stor, "released", "released", version.Current)[0]
err := stor.Put(envtools.StorageName(tools.Version, "released"), strings.NewReader("!"), 1)
c.Assert(err, jc.ErrorIsNil)
resp, err := s.downloadRequest(c, tools.Version, "")
c.Assert(err, jc.ErrorIsNil)
s.assertErrorResponse(c, resp, http.StatusBadRequest, "error fetching tools: size mismatch for .*")
s.assertToolsNotStored(c, tools.Version)
}
// cloneToolsForSeries copies the built tools tarball into a tarball for the specified
// stream and series and generates corresponding metadata.
func cloneToolsForSeries(toolsInfo *BuiltTools, stream string, series ...string) error {
// Copy the tools to the target storage, recording a Tools struct for each one.
var targetTools coretools.List
targetTools = append(targetTools, &coretools.Tools{
Version: toolsInfo.Version,
Size: toolsInfo.Size,
SHA256: toolsInfo.Sha256Hash,
})
putTools := func(vers version.Binary) (string, error) {
name := envtools.StorageName(vers, stream)
src := filepath.Join(toolsInfo.Dir, toolsInfo.StorageName)
dest := filepath.Join(toolsInfo.Dir, name)
destDir := filepath.Dir(dest)
if err := os.MkdirAll(destDir, 0755); err != nil {
return "", err
}
if err := utils.CopyFile(dest, src); err != nil {
return "", err
}
// Append to targetTools the attributes required to write out tools metadata.
targetTools = append(targetTools, &coretools.Tools{
Version: vers,
Size: toolsInfo.Size,
SHA256: toolsInfo.Sha256Hash,
})
return name, nil
}
logger.Debugf("generating tarballs for %v", series)
for _, series := range series {
_, err := jujuseries.SeriesVersion(series)
if err != nil {
return err
}
if series != toolsInfo.Version.Series {
fakeVersion := toolsInfo.Version
fakeVersion.Series = series
if _, err := putTools(fakeVersion); err != nil {
return err
}
}
}
// The tools have been copied to a temp location from which they will be uploaded,
// now write out the matching simplestreams metadata so that SyncTools can find them.
metadataStore, err := filestorage.NewFileStorageWriter(toolsInfo.Dir)
if err != nil {
return err
}
logger.Debugf("generating tools metadata")
return envtools.MergeAndWriteMetadata(metadataStore, stream, stream, targetTools, false)
}
func uploadFakeToolsVersion(stor storage.Storage, toolsDir string, vers version.Binary) (*coretools.Tools, error) {
logger.Infof("uploading FAKE tools %s", vers)
tgz, checksum := makeFakeTools(vers)
size := int64(len(tgz))
name := envtools.StorageName(vers, toolsDir)
if err := stor.Put(name, bytes.NewReader(tgz), size); err != nil {
return nil, err
}
url, err := stor.URL(name)
if err != nil {
return nil, err
}
return &coretools.Tools{URL: url, Version: vers, Size: size, SHA256: checksum}, nil
}
func uploadFakeToolsVersion(stor storage.Storage, vers version.Binary) (*coretools.Tools, error) {
logger.Infof("uploading FAKE tools %s", vers)
tgz, checksum := coretesting.TarGz(
coretesting.NewTarFile("jujud", 0777, "jujud contents "+vers.String()))
size := int64(len(tgz))
name := envtools.StorageName(vers)
if err := stor.Put(name, bytes.NewReader(tgz), size); err != nil {
return nil, err
}
url, err := stor.URL(name)
if err != nil {
return nil, err
}
return &coretools.Tools{URL: url, Version: vers, Size: size, SHA256: checksum}, nil
}
func (s *toolsSuite) TestUploadAllowsTopLevelPath(c *gc.C) {
// Backwards compatibility check, that we can upload tools to
// https://host:port/tools
expectedTools, vers, toolPath := s.setupToolsForUpload(c)
url := s.toolsURL(c, "binaryVersion="+vers.String())
url.Path = "/tools"
resp, err := s.uploadRequest(c, url.String(), true, toolPath)
c.Assert(err, gc.IsNil)
// Check the response.
stor := s.Environ.Storage()
toolsURL, err := stor.URL(tools.StorageName(vers))
c.Assert(err, gc.IsNil)
expectedTools[0].URL = toolsURL
s.assertUploadResponse(c, resp, expectedTools[0])
}
func (s *toolsSuite) TestUpload(c *gc.C) {
// Make some fake tools.
expectedTools, vers, toolPath := s.setupToolsForUpload(c)
// Now try uploading them.
resp, err := s.uploadRequest(
c, s.toolsURI(c, "?binaryVersion="+vers.String()), true, toolPath)
c.Assert(err, gc.IsNil)
// Check the response.
stor := s.Environ.Storage()
toolsURL, err := stor.URL(tools.StorageName(vers))
c.Assert(err, gc.IsNil)
expectedTools[0].URL = toolsURL
s.assertUploadResponse(c, resp, expectedTools[0])
// Check the contents.
r, err := stor.Get(tools.StorageName(vers))
c.Assert(err, gc.IsNil)
uploadedData, err := ioutil.ReadAll(r)
c.Assert(err, gc.IsNil)
expectedData, err := ioutil.ReadFile(toolPath)
c.Assert(err, gc.IsNil)
c.Assert(uploadedData, gc.DeepEquals, expectedData)
}
func (s *toolsSuite) TestUploadAllowsEnvUUIDPath(c *gc.C) {
// Check that we can upload tools to https://host:port/ENVUUID/tools
environ, err := s.State.Environment()
c.Assert(err, gc.IsNil)
expectedTools, vers, toolPath := s.setupToolsForUpload(c)
url := s.toolsURL(c, "binaryVersion="+vers.String())
url.Path = fmt.Sprintf("/environment/%s/tools", environ.UUID())
resp, err := s.uploadRequest(c, url.String(), true, toolPath)
c.Assert(err, gc.IsNil)
// Check the response.
stor := s.Environ.Storage()
toolsURL, err := stor.URL(tools.StorageName(vers))
c.Assert(err, gc.IsNil)
expectedTools[0].URL = toolsURL
s.assertUploadResponse(c, resp, expectedTools[0])
}
// handleUpload uploads the tools data from the reader to env storage as the specified version.
func (h *toolsUploadHandler) handleUpload(r io.Reader, toolsVersion version.Binary) (*tools.Tools, bool, error) {
// Set up a local temp directory for the tools tarball.
tmpDir, err := ioutil.TempDir("", "juju-upload-tools-")
if err != nil {
return nil, false, errors.Annotate(err, "cannot create temp dir")
}
defer os.RemoveAll(tmpDir)
toolsFilename := envtools.StorageName(toolsVersion)
toolsDir := path.Dir(toolsFilename)
fullToolsDir := path.Join(tmpDir, toolsDir)
err = os.MkdirAll(fullToolsDir, 0700)
if err != nil {
return nil, false, errors.Annotatef(err, "cannot create tools dir %s", toolsDir)
}
// Read the tools tarball from the request, calculating the sha256 along the way.
fullToolsFilename := path.Join(tmpDir, toolsFilename)
toolsFile, err := os.Create(fullToolsFilename)
if err != nil {
return nil, false, errors.Annotatef(err, "cannot create tools file %s", fullToolsFilename)
}
logger.Debugf("saving uploaded tools to temp file: %s", fullToolsFilename)
defer toolsFile.Close()
sha256hash := sha256.New()
var size int64
if size, err = io.Copy(toolsFile, io.TeeReader(r, sha256hash)); err != nil {
return nil, false, errors.Annotate(err, "error processing file upload")
}
if size == 0 {
return nil, false, errors.New("no tools uploaded")
}
// TODO(wallyworld): check integrity of tools tarball.
// Create a tools record and sync to storage.
uploadedTools := &tools.Tools{
Version: toolsVersion,
Size: size,
SHA256: fmt.Sprintf("%x", sha256hash.Sum(nil)),
}
logger.Debugf("about to upload tools %+v to storage", uploadedTools)
return h.uploadToStorage(uploadedTools, tmpDir, toolsFilename)
}
func (s *toolsSuite) TestDownloadFetchesAndVerifiesHash(c *gc.C) {
// Upload fake tools, then upload over the top so the SHA256 hash does not match.
s.PatchValue(&jujuversion.Current, testing.FakeVersionNumber)
stor := s.DefaultToolsStorage
envtesting.RemoveTools(c, stor, "released")
current := version.Binary{
Number: jujuversion.Current,
Arch: arch.HostArch(),
Series: series.HostSeries(),
}
tools := envtesting.AssertUploadFakeToolsVersions(c, stor, "released", "released", current)[0]
sameSize := strings.Repeat("!", int(tools.Size))
err := stor.Put(envtools.StorageName(tools.Version, "released"), strings.NewReader(sameSize), tools.Size)
c.Assert(err, jc.ErrorIsNil)
resp := s.downloadRequest(c, tools.Version, "")
s.assertErrorResponse(c, resp, http.StatusBadRequest, "error fetching tools: hash mismatch for .*")
s.assertToolsNotStored(c, tools.Version.String())
}
// copyOneToolsPackage copies one tool from the source to the target.
func copyOneToolsPackage(tool *coretools.Tools, dest storage.Storage) error {
toolsName := envtools.StorageName(tool.Version)
logger.Infof("copying %v", toolsName)
resp, err := utils.GetValidatingHTTPClient().Get(tool.URL)
if err != nil {
return err
}
buf := &bytes.Buffer{}
srcFile := resp.Body
defer srcFile.Close()
tool.SHA256, tool.Size, err = utils.ReadSHA256(io.TeeReader(srcFile, buf))
if err != nil {
return err
}
sizeInKB := (tool.Size + 512) / 1024
logger.Infof("downloaded %v (%dkB), uploading", toolsName, sizeInKB)
logger.Infof("download %dkB, uploading", sizeInKB)
return dest.Put(toolsName, buf, tool.Size)
}
func fetchToolsArchive(stor storage.StorageReader, toolsDir string, agentTools *tools.Tools) ([]byte, error) {
r, err := stor.Get(envtools.StorageName(agentTools.Version, toolsDir))
if err != nil {
return nil, err
}
defer r.Close()
var buf bytes.Buffer
hash, size, err := utils.ReadSHA256(io.TeeReader(r, &buf))
if err != nil {
return nil, err
}
if hash != agentTools.SHA256 {
return nil, errors.New("hash mismatch")
}
if size != agentTools.Size {
return nil, errors.New("size mismatch")
}
return buf.Bytes(), nil
}
func (s *configureSuite) getCloudConfig(c *gc.C, stateServer bool, vers version.Binary) *cloudinit.Config {
var mcfg *envcloudinit.MachineConfig
if stateServer {
mcfg = environs.NewBootstrapMachineConfig("private-key")
mcfg.InstanceId = "instance-id"
mcfg.Jobs = []params.MachineJob{params.JobManageEnviron, params.JobHostUnits}
} else {
mcfg = environs.NewMachineConfig("0", "ya", nil, nil, nil)
mcfg.Jobs = []params.MachineJob{params.JobHostUnits}
}
mcfg.Tools = &tools.Tools{
Version: vers,
URL: "file:///var/lib/juju/storage/" + envtools.StorageName(vers),
}
environConfig := testConfig(c, stateServer, vers)
err := environs.FinishMachineConfig(mcfg, environConfig, constraints.Value{})
c.Assert(err, gc.IsNil)
cloudcfg := cloudinit.New()
err = envcloudinit.Configure(mcfg, cloudcfg)
c.Assert(err, gc.IsNil)
return cloudcfg
}
// copyOneToolsPackage copies one tool from the source to the target.
func copyOneToolsPackage(tools *coretools.Tools, u ToolsUploader) error {
toolsName := envtools.StorageName(tools.Version)
logger.Infof("downloading %v (%v)", toolsName, tools.URL)
resp, err := utils.GetValidatingHTTPClient().Get(tools.URL)
if err != nil {
return err
}
defer resp.Body.Close()
// Verify SHA-256 hash.
var buf bytes.Buffer
sha256, size, err := utils.ReadSHA256(io.TeeReader(resp.Body, &buf))
if err != nil {
return err
}
if tools.SHA256 == "" {
logger.Warningf("no SHA-256 hash for %v", tools.SHA256)
} else if sha256 != tools.SHA256 {
return errors.Errorf("SHA-256 hash mismatch (%v/%v)", sha256, tools.SHA256)
}
sizeInKB := (size + 512) / 1024
logger.Infof("uploading %v (%dkB) to environment", toolsName, sizeInKB)
return u.UploadTools(tools, buf.Bytes())
}
func (s *UpgradeJujuSuite) TestUpgradeJuju(c *gc.C) {
oldVersion := version.Current
defer func() {
version.Current = oldVersion
}()
for i, test := range upgradeJujuTests {
c.Logf("\ntest %d: %s", i, test.about)
s.Reset(c)
// Set up apparent CLI version and initialize the command.
version.Current = version.MustParseBinary(test.currentVersion)
com := &UpgradeJujuCommand{}
if err := coretesting.InitCommand(envcmd.Wrap(com), test.args); err != nil {
if test.expectInitErr != "" {
c.Check(err, gc.ErrorMatches, test.expectInitErr)
} else {
c.Check(err, gc.IsNil)
}
continue
}
// Set up state and environ, and run the command.
toolsDir := c.MkDir()
updateAttrs := map[string]interface{}{
"agent-version": test.agentVersion,
"tools-metadata-url": "file://" + toolsDir,
}
err := s.State.UpdateEnvironConfig(updateAttrs, nil, nil)
c.Assert(err, gc.IsNil)
versions := make([]version.Binary, len(test.tools))
for i, v := range test.tools {
versions[i] = version.MustParseBinary(v)
}
if len(versions) > 0 {
envtesting.MustUploadFakeToolsVersions(s.Conn.Environ.Storage(), versions...)
stor, err := filestorage.NewFileStorageWriter(toolsDir)
c.Assert(err, gc.IsNil)
envtesting.MustUploadFakeToolsVersions(stor, versions...)
}
err = com.Run(coretesting.Context(c))
if test.expectErr != "" {
c.Check(err, gc.ErrorMatches, test.expectErr)
continue
} else if !c.Check(err, gc.IsNil) {
continue
}
// Check expected changes to environ/state.
cfg, err := s.State.EnvironConfig()
c.Check(err, gc.IsNil)
agentVersion, ok := cfg.AgentVersion()
c.Check(ok, gc.Equals, true)
c.Check(agentVersion, gc.Equals, version.MustParse(test.expectVersion))
for _, uploaded := range test.expectUploaded {
// Substitute latest LTS for placeholder in expected series for uploaded tools
uploaded = strings.Replace(uploaded, "%LTS%", config.LatestLtsSeries(), 1)
vers := version.MustParseBinary(uploaded)
r, err := storage.Get(s.Conn.Environ.Storage(), envtools.StorageName(vers))
if !c.Check(err, gc.IsNil) {
continue
}
data, err := ioutil.ReadAll(r)
r.Close()
c.Check(err, gc.IsNil)
expectContent := version.Current
expectContent.Number = agentVersion
checkToolsContent(c, data, "jujud contents "+expectContent.String())
}
}
}
// NewManualBootstrapEnviron wraps a LocalStorageEnviron with another which
// overrides the Bootstrap method; when Bootstrap is invoked, the specified
// host will be manually bootstrapped.
//
// InitUbuntuUser is expected to have been executed successfully against
// the host being bootstrapped.
func Bootstrap(args BootstrapArgs) (err error) {
if args.Host == "" {
return errors.New("host argument is empty")
}
if args.Environ == nil {
return errors.New("environ argument is nil")
}
if args.DataDir == "" {
return errors.New("data-dir argument is empty")
}
if args.Series == "" {
return errors.New("series argument is empty")
}
if args.HardwareCharacteristics == nil {
return errors.New("hardware characteristics argument is empty")
}
if len(args.PossibleTools) == 0 {
return errors.New("possible tools is empty")
}
provisioned, err := checkProvisioned(args.Host)
if err != nil {
return fmt.Errorf("failed to check provisioned status: %v", err)
}
if provisioned {
return ErrProvisioned
}
// Filter tools based on detected series/arch.
logger.Infof("Filtering possible tools: %v", args.PossibleTools)
possibleTools, err := args.PossibleTools.Match(tools.Filter{
Arch: *args.HardwareCharacteristics.Arch,
Series: args.Series,
})
if err != nil {
return err
}
// Store the state file. If provisioning fails, we'll remove the file.
logger.Infof("Saving bootstrap state file to bootstrap storage")
bootstrapStorage := args.Environ.Storage()
err = bootstrap.SaveState(
bootstrapStorage,
&bootstrap.BootstrapState{
StateInstances: []instance.Id{BootstrapInstanceId},
},
)
if err != nil {
return err
}
defer func() {
if err != nil {
logger.Errorf("bootstrapping failed, removing state file: %v", err)
bootstrapStorage.Remove(bootstrap.StateFile)
}
}()
// If the tools are on the machine already, get a file:// scheme tools URL.
tools := *possibleTools[0]
storageDir := args.Environ.StorageDir()
toolsStorageName := envtools.StorageName(tools.Version)
if url, _ := bootstrapStorage.URL(toolsStorageName); url == tools.URL {
tools.URL = fmt.Sprintf("file://%s/%s", storageDir, toolsStorageName)
}
// Add the local storage configuration.
agentEnv, err := localstorage.StoreConfig(args.Environ)
if err != nil {
return err
}
privateKey, err := common.GenerateSystemSSHKey(args.Environ)
if err != nil {
return err
}
// Finally, provision the machine agent.
mcfg := environs.NewBootstrapMachineConfig(privateKey)
mcfg.InstanceId = BootstrapInstanceId
mcfg.HardwareCharacteristics = args.HardwareCharacteristics
if args.DataDir != "" {
mcfg.DataDir = args.DataDir
}
mcfg.Tools = &tools
err = environs.FinishMachineConfig(mcfg, args.Environ.Config(), constraints.Value{})
if err != nil {
return err
}
for k, v := range agentEnv {
mcfg.AgentEnvironment[k] = v
}
return provisionMachineAgent(args.Host, mcfg, args.Context.GetStderr())
}
func (s *StorageSuite) TestStorageName(c *gc.C) {
vers := version.MustParseBinary("1.2.3-precise-amd64")
path := envtools.StorageName(vers, "proposed")
c.Assert(path, gc.Equals, "tools/proposed/juju-1.2.3-precise-amd64.tgz")
}