Пример #1
0
func (s *toolsSuite) TestUploadSeriesExpanded(c *gc.C) {
	// Make some fake tools.
	expectedTools, vers, toolPath := s.setupToolsForUpload(c)
	// Now try uploading them. The "series" parameter is accepted
	// but ignored; the API server will expand the tools for all
	// supported series.
	params := "?binaryVersion=" + vers.String() + "&series=nonsense"
	resp, err := s.uploadRequest(c, s.toolsURI(c, params), true, toolPath)
	c.Assert(err, gc.IsNil)

	// Check the response.
	stor := s.Environ.Storage()
	toolsURL, err := stor.URL(tools.StorageName(vers))
	c.Assert(err, gc.IsNil)
	expectedTools[0].URL = toolsURL
	s.assertUploadResponse(c, resp, expectedTools[0])

	// Check the contents.
	for _, series := range version.OSSupportedSeries(version.Ubuntu) {
		toolsVersion := vers
		toolsVersion.Series = series
		r, err := stor.Get(tools.StorageName(toolsVersion))
		c.Assert(err, gc.IsNil)
		uploadedData, err := ioutil.ReadAll(r)
		c.Assert(err, gc.IsNil)
		expectedData, err := ioutil.ReadFile(toolPath)
		c.Assert(err, gc.IsNil)
		c.Assert(uploadedData, gc.DeepEquals, expectedData)
	}
}
Пример #2
0
func (s *toolsSuite) TestUploadFakeSeries(c *gc.C) {
	// Make some fake tools.
	expectedTools, vers, toolPath := s.setupToolsForUpload(c)
	// Now try uploading them.
	params := "?binaryVersion=" + vers.String() + "&series=precise,trusty"
	resp, err := s.uploadRequest(c, s.toolsURI(c, params), true, toolPath)
	c.Assert(err, gc.IsNil)

	// Check the response.
	stor := s.Conn.Environ.Storage()
	toolsURL, err := stor.URL(tools.StorageName(vers))
	c.Assert(err, gc.IsNil)
	expectedTools[0].URL = toolsURL
	s.assertUploadResponse(c, resp, expectedTools[0])

	// Check the contents.
	for _, series := range []string{"precise", "quantal", "trusty"} {
		toolsVersion := vers
		toolsVersion.Series = series
		r, err := stor.Get(tools.StorageName(toolsVersion))
		c.Assert(err, gc.IsNil)
		uploadedData, err := ioutil.ReadAll(r)
		c.Assert(err, gc.IsNil)
		expectedData, err := ioutil.ReadFile(toolPath)
		c.Assert(err, gc.IsNil)
		c.Assert(uploadedData, gc.DeepEquals, expectedData)
	}
}
Пример #3
0
func (t *LiveTests) TestBootstrapWithDefaultSeries(c *gc.C) {
	if !t.HasProvisioner {
		c.Skip("HasProvisioner is false; cannot test deployment")
	}

	current := version.Current
	other := current
	other.Series = "quantal"
	if current == other {
		other.Series = "precise"
	}

	dummyCfg, err := config.New(config.NoDefaults, dummy.SampleConfig().Merge(coretesting.Attrs{
		"state-server": false,
		"name":         "dummy storage",
	}))
	dummyenv, err := environs.Prepare(dummyCfg, coretesting.Context(c), configstore.NewMem())
	c.Assert(err, gc.IsNil)
	defer dummyenv.Destroy()

	t.Destroy(c)

	attrs := t.TestConfig.Merge(coretesting.Attrs{"default-series": other.Series})
	cfg, err := config.New(config.NoDefaults, attrs)
	c.Assert(err, gc.IsNil)
	env, err := environs.Prepare(cfg, coretesting.Context(c), t.ConfigStore)
	c.Assert(err, gc.IsNil)
	defer environs.Destroy(env, t.ConfigStore)

	currentName := envtools.StorageName(current)
	otherName := envtools.StorageName(other)
	envStorage := env.Storage()
	dummyStorage := dummyenv.Storage()

	defer envStorage.Remove(otherName)

	_, err = sync.Upload(dummyStorage, &current.Number)
	c.Assert(err, gc.IsNil)

	// This will only work while cross-compiling across releases is safe,
	// which depends on external elements. Tends to be safe for the last
	// few releases, but we may have to refactor some day.
	err = storageCopy(dummyStorage, currentName, envStorage, otherName)
	c.Assert(err, gc.IsNil)

	err = bootstrap.Bootstrap(coretesting.Context(c), env, environs.BootstrapParams{})
	c.Assert(err, gc.IsNil)

	st := t.Env.(testing.GetStater).GetStateInAPIServer()
	// Wait for machine agent to come up on the bootstrap
	// machine and ensure it deployed the proper series.
	m0, err := st.Machine("0")
	c.Assert(err, gc.IsNil)
	mw0 := newMachineToolWaiter(m0)
	defer mw0.Stop()

	waitAgentTools(c, mw0, other)
}
Пример #4
0
// populateTools stores uploaded tools in provider storage
// and updates the tools metadata.
//
// TODO(axw) store tools in gridfs, catalogue in state.
func (c *BootstrapCommand) populateTools(env environs.Environ) error {
	agentConfig := c.CurrentConfig()
	dataDir := agentConfig.DataDir()
	tools, err := agenttools.ReadTools(dataDir, version.Current)
	if err != nil {
		return err
	}
	if !strings.HasPrefix(tools.URL, "file://") {
		// Nothing to do since the tools were not uploaded.
		return nil
	}

	// This is a hack: providers using localstorage (local, manual)
	// can't use storage during bootstrap as the localstorage worker
	// isn't running. Use filestorage instead.
	var stor storage.Storage
	storageDir := agentConfig.Value(agent.StorageDir)
	if storageDir != "" {
		stor, err = filestorage.NewFileStorageWriter(storageDir)
		if err != nil {
			return err
		}
	} else {
		stor = env.Storage()
	}

	// Create a temporary directory to contain source and cloned tools.
	tempDir, err := ioutil.TempDir("", "juju-sync-tools")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempDir)
	destTools := filepath.Join(tempDir, filepath.FromSlash(envtools.StorageName(tools.Version)))
	if err := os.MkdirAll(filepath.Dir(destTools), 0700); err != nil {
		return err
	}
	srcTools := filepath.Join(
		agenttools.SharedToolsDir(dataDir, version.Current),
		"tools.tar.gz",
	)
	if err := utils.CopyFile(destTools, srcTools); err != nil {
		return err
	}

	// Until we catalogue tools in state, we clone the tools
	// for each of the supported series of the same OS.
	otherSeries := version.OSSupportedSeries(version.Current.OS)
	_, err = sync.SyncBuiltTools(stor, &sync.BuiltTools{
		Version:     tools.Version,
		Dir:         tempDir,
		StorageName: envtools.StorageName(tools.Version),
		Sha256Hash:  tools.SHA256,
		Size:        tools.Size,
	}, otherSeries...)
	return err
}
Пример #5
0
// RemoveFakeTools deletes the fake tools from the supplied storage.
func RemoveFakeTools(c *gc.C, stor storage.Storage, toolsDir string) {
	c.Logf("removing fake tools")
	toolsVersion := version.Current
	name := envtools.StorageName(toolsVersion, toolsDir)
	err := stor.Remove(name)
	c.Check(err, jc.ErrorIsNil)
	defaultSeries := coretesting.FakeDefaultSeries
	if series.HostSeries() != defaultSeries {
		toolsVersion.Series = defaultSeries
		name := envtools.StorageName(toolsVersion, toolsDir)
		err := stor.Remove(name)
		c.Check(err, jc.ErrorIsNil)
	}
	RemoveFakeToolsMetadata(c, stor)
}
Пример #6
0
// RemoveFakeTools deletes the fake tools from the supplied storage.
func RemoveFakeTools(c *gc.C, stor storage.Storage) {
	c.Logf("removing fake tools")
	toolsVersion := version.Current
	name := envtools.StorageName(toolsVersion)
	err := stor.Remove(name)
	c.Check(err, gc.IsNil)
	defaultSeries := coretesting.FakeDefaultSeries
	if version.Current.Series != defaultSeries {
		toolsVersion.Series = defaultSeries
		name := envtools.StorageName(toolsVersion)
		err := stor.Remove(name)
		c.Check(err, gc.IsNil)
	}
	RemoveFakeToolsMetadata(c, stor)
}
Пример #7
0
func (*metadataHelperSuite) TestMetadataFromTools(c *gc.C) {
	metadata := tools.MetadataFromTools(nil, "proposed")
	c.Assert(metadata, gc.HasLen, 0)

	toolsList := coretools.List{{
		Version: version.MustParseBinary("1.2.3-precise-amd64"),
		Size:    123,
		SHA256:  "abc",
	}, {
		Version: version.MustParseBinary("2.0.1-raring-amd64"),
		URL:     "file:///tmp/proposed/juju-2.0.1-raring-amd64.tgz",
		Size:    456,
		SHA256:  "xyz",
	}}
	metadata = tools.MetadataFromTools(toolsList, "proposed")
	c.Assert(metadata, gc.HasLen, len(toolsList))
	for i, t := range toolsList {
		md := metadata[i]
		c.Assert(md.Release, gc.Equals, t.Version.Series)
		c.Assert(md.Version, gc.Equals, t.Version.Number.String())
		c.Assert(md.Arch, gc.Equals, t.Version.Arch)
		// FullPath is only filled out when reading tools using simplestreams.
		// It's not needed elsewhere and requires a URL() call.
		c.Assert(md.FullPath, gc.Equals, "")
		c.Assert(md.Path, gc.Equals, tools.StorageName(t.Version, "proposed")[len("tools/"):])
		c.Assert(md.FileType, gc.Equals, "tar.gz")
		c.Assert(md.Size, gc.Equals, t.Size)
		c.Assert(md.SHA256, gc.Equals, t.SHA256)
	}
}
Пример #8
0
func (s *bootstrapSuite) TestBootstrapToolsFileURL(c *gc.C) {
	storageName := tools.StorageName(version.Current)
	sftpURL, err := s.env.Storage().URL(storageName)
	c.Assert(err, gc.IsNil)
	fileURL := fmt.Sprintf("file://%s/%s", s.env.storageDir, storageName)
	s.testBootstrapToolsURL(c, sftpURL, fileURL)
}
Пример #9
0
func (s *toolsSuite) setupToolsForUpload(c *gc.C) (coretools.List, version.Binary, string) {
	localStorage := c.MkDir()
	vers := version.MustParseBinary("1.9.0-quantal-amd64")
	versionStrings := []string{vers.String()}
	expectedTools := toolstesting.MakeToolsWithCheckSum(c, localStorage, "releases", versionStrings)
	toolsFile := tools.StorageName(vers)
	return expectedTools, vers, path.Join(localStorage, toolsFile)
}
Пример #10
0
Файл: tools.go Проект: bac/juju
// RemoveFakeTools deletes the fake tools from the supplied storage.
func RemoveFakeTools(c *gc.C, stor storage.Storage, toolsDir string) {
	c.Logf("removing fake tools")
	toolsVersion := version.Binary{
		Number: jujuversion.Current,
		Arch:   arch.HostArch(),
		Series: series.HostSeries(),
	}
	name := envtools.StorageName(toolsVersion, toolsDir)
	err := stor.Remove(name)
	c.Check(err, jc.ErrorIsNil)
	defaultSeries := series.LatestLts()
	if series.HostSeries() != defaultSeries {
		toolsVersion.Series = defaultSeries
		name := envtools.StorageName(toolsVersion, toolsDir)
		err := stor.Remove(name)
		c.Check(err, jc.ErrorIsNil)
	}
	RemoveFakeToolsMetadata(c, stor)
}
Пример #11
0
func (s *UpgraderSuite) TestUpgraderRetryAndChanged(c *gc.C) {
	stor := s.DefaultToolsStorage
	oldTools := envtesting.PrimeTools(c, stor, s.DataDir(), s.Environ.Config().AgentStream(), version.MustParseBinary("5.4.3-precise-amd64"))
	s.PatchValue(&version.Current, oldTools.Version)
	newTools := envtesting.AssertUploadFakeToolsVersions(
		c, stor, s.Environ.Config().AgentStream(), s.Environ.Config().AgentStream(), version.MustParseBinary("5.4.5-precise-amd64"))[0]
	err := statetesting.SetAgentVersion(s.State, newTools.Version.Number)
	c.Assert(err, jc.ErrorIsNil)

	retryc := make(chan time.Time)
	*upgrader.RetryAfter = func() <-chan time.Time {
		c.Logf("replacement retry after")
		return retryc
	}
	err = stor.Remove(envtools.StorageName(newTools.Version, "released"))
	c.Assert(err, jc.ErrorIsNil)
	u := s.makeUpgrader(c)
	defer u.Stop()
	s.expectUpgradeChannelNotClosed(c)

	for i := 0; i < 3; i++ {
		select {
		case retryc <- time.Now():
		case <-time.After(coretesting.LongWait):
			c.Fatalf("upgrader did not retry (attempt %d)", i)
		}
	}

	// Make it upgrade to some newer tools that can be
	// downloaded ok; it should stop retrying, download
	// the newer tools and exit.
	newerTools := envtesting.AssertUploadFakeToolsVersions(
		c, stor, s.Environ.Config().AgentStream(), s.Environ.Config().AgentStream(), version.MustParseBinary("5.4.6-precise-amd64"))[0]

	err = statetesting.SetAgentVersion(s.State, newerTools.Version.Number)
	c.Assert(err, jc.ErrorIsNil)

	s.BackingState.StartSync()
	done := make(chan error)
	go func() {
		done <- u.Wait()
	}()
	select {
	case err := <-done:
		envtesting.CheckUpgraderReadyError(c, err, &upgrader.UpgradeReadyError{
			AgentName: s.machine.Tag().String(),
			OldTools:  oldTools.Version,
			NewTools:  newerTools.Version,
			DataDir:   s.DataDir(),
		})
	case <-time.After(coretesting.LongWait):
		c.Fatalf("upgrader did not quit after upgrading")
	}
}
Пример #12
0
func (u StorageToolsUploader) UploadTools(tools *coretools.Tools, data []byte) error {
	toolsName := envtools.StorageName(tools.Version)
	if err := u.Storage.Put(toolsName, bytes.NewReader(data), int64(len(data))); err != nil {
		return err
	}
	err := envtools.MergeAndWriteMetadata(u.Storage, coretools.List{tools}, u.WriteMirrors)
	if err != nil {
		logger.Errorf("error writing tools metadata: %v", err)
		return err
	}
	return nil
}
Пример #13
0
func (s *toolsSuite) TestDownloadFetchesAndVerifiesSize(c *gc.C) {
	// Upload fake tools, then upload over the top so the SHA256 hash does not match.
	stor := s.Environ.Storage()
	envtesting.RemoveTools(c, stor)
	tools := envtesting.AssertUploadFakeToolsVersions(c, stor, version.Current)[0]
	err := stor.Put(envtools.StorageName(tools.Version), strings.NewReader("!"), 1)

	resp, err := s.downloadRequest(c, tools.Version, "")
	c.Assert(err, gc.IsNil)
	s.assertErrorResponse(c, resp, http.StatusBadRequest, "error fetching tools: size mismatch for .*")
	s.assertToolsNotStored(c, tools.Version)
}
Пример #14
0
// buildToolsTarball bundles a tools tarball and places it in a temp directory in
// the expected tools path.
func buildToolsTarball(forceVersion *version.Number) (builtTools *BuiltTools, err error) {
	// TODO(rog) find binaries from $PATH when not using a development
	// version of juju within a $GOPATH.

	logger.Debugf("Building tools")
	// We create the entire archive before asking the environment to
	// start uploading so that we can be sure we have archived
	// correctly.
	f, err := ioutil.TempFile("", "juju-tgz")
	if err != nil {
		return nil, err
	}
	defer f.Close()
	defer os.Remove(f.Name())
	toolsVersion, sha256Hash, err := envtools.BundleTools(f, forceVersion)
	if err != nil {
		return nil, err
	}
	fileInfo, err := f.Stat()
	if err != nil {
		return nil, fmt.Errorf("cannot stat newly made tools archive: %v", err)
	}
	size := fileInfo.Size()
	logger.Infof("built tools %v (%dkB)", toolsVersion, (size+512)/1024)
	baseToolsDir, err := ioutil.TempDir("", "juju-tools")
	if err != nil {
		return nil, err
	}

	// If we exit with an error, clean up the built tools directory.
	defer func() {
		if err != nil {
			os.RemoveAll(baseToolsDir)
		}
	}()

	err = os.MkdirAll(filepath.Join(baseToolsDir, storage.BaseToolsPath, "releases"), 0755)
	if err != nil {
		return nil, err
	}
	storageName := envtools.StorageName(toolsVersion)
	err = utils.CopyFile(filepath.Join(baseToolsDir, storageName), f.Name())
	if err != nil {
		return nil, err
	}
	return &BuiltTools{
		Version:     toolsVersion,
		Dir:         baseToolsDir,
		StorageName: storageName,
		Size:        size,
		Sha256Hash:  sha256Hash,
	}, nil
}
Пример #15
0
func (s *toolsSuite) TestDownloadFetchesAndVerifiesSize(c *gc.C) {
	// Upload fake tools, then upload over the top so the SHA256 hash does not match.
	s.PatchValue(&version.Current.Number, testing.FakeVersionNumber)
	stor := s.DefaultToolsStorage
	envtesting.RemoveTools(c, stor, "released")
	tools := envtesting.AssertUploadFakeToolsVersions(c, stor, "released", "released", version.Current)[0]
	err := stor.Put(envtools.StorageName(tools.Version, "released"), strings.NewReader("!"), 1)
	c.Assert(err, jc.ErrorIsNil)

	resp, err := s.downloadRequest(c, tools.Version, "")
	c.Assert(err, jc.ErrorIsNil)
	s.assertErrorResponse(c, resp, http.StatusBadRequest, "error fetching tools: size mismatch for .*")
	s.assertToolsNotStored(c, tools.Version)
}
Пример #16
0
// cloneToolsForSeries copies the built tools tarball into a tarball for the specified
// stream and series and generates corresponding metadata.
func cloneToolsForSeries(toolsInfo *BuiltTools, stream string, series ...string) error {
	// Copy the tools to the target storage, recording a Tools struct for each one.
	var targetTools coretools.List
	targetTools = append(targetTools, &coretools.Tools{
		Version: toolsInfo.Version,
		Size:    toolsInfo.Size,
		SHA256:  toolsInfo.Sha256Hash,
	})
	putTools := func(vers version.Binary) (string, error) {
		name := envtools.StorageName(vers, stream)
		src := filepath.Join(toolsInfo.Dir, toolsInfo.StorageName)
		dest := filepath.Join(toolsInfo.Dir, name)
		destDir := filepath.Dir(dest)
		if err := os.MkdirAll(destDir, 0755); err != nil {
			return "", err
		}
		if err := utils.CopyFile(dest, src); err != nil {
			return "", err
		}
		// Append to targetTools the attributes required to write out tools metadata.
		targetTools = append(targetTools, &coretools.Tools{
			Version: vers,
			Size:    toolsInfo.Size,
			SHA256:  toolsInfo.Sha256Hash,
		})
		return name, nil
	}
	logger.Debugf("generating tarballs for %v", series)
	for _, series := range series {
		_, err := jujuseries.SeriesVersion(series)
		if err != nil {
			return err
		}
		if series != toolsInfo.Version.Series {
			fakeVersion := toolsInfo.Version
			fakeVersion.Series = series
			if _, err := putTools(fakeVersion); err != nil {
				return err
			}
		}
	}
	// The tools have been copied to a temp location from which they will be uploaded,
	// now write out the matching simplestreams metadata so that SyncTools can find them.
	metadataStore, err := filestorage.NewFileStorageWriter(toolsInfo.Dir)
	if err != nil {
		return err
	}
	logger.Debugf("generating tools metadata")
	return envtools.MergeAndWriteMetadata(metadataStore, stream, stream, targetTools, false)
}
Пример #17
0
func uploadFakeToolsVersion(stor storage.Storage, toolsDir string, vers version.Binary) (*coretools.Tools, error) {
	logger.Infof("uploading FAKE tools %s", vers)
	tgz, checksum := makeFakeTools(vers)
	size := int64(len(tgz))
	name := envtools.StorageName(vers, toolsDir)
	if err := stor.Put(name, bytes.NewReader(tgz), size); err != nil {
		return nil, err
	}
	url, err := stor.URL(name)
	if err != nil {
		return nil, err
	}
	return &coretools.Tools{URL: url, Version: vers, Size: size, SHA256: checksum}, nil
}
Пример #18
0
func uploadFakeToolsVersion(stor storage.Storage, vers version.Binary) (*coretools.Tools, error) {
	logger.Infof("uploading FAKE tools %s", vers)
	tgz, checksum := coretesting.TarGz(
		coretesting.NewTarFile("jujud", 0777, "jujud contents "+vers.String()))
	size := int64(len(tgz))
	name := envtools.StorageName(vers)
	if err := stor.Put(name, bytes.NewReader(tgz), size); err != nil {
		return nil, err
	}
	url, err := stor.URL(name)
	if err != nil {
		return nil, err
	}
	return &coretools.Tools{URL: url, Version: vers, Size: size, SHA256: checksum}, nil
}
Пример #19
0
func (s *toolsSuite) TestUploadAllowsTopLevelPath(c *gc.C) {
	// Backwards compatibility check, that we can upload tools to
	// https://host:port/tools
	expectedTools, vers, toolPath := s.setupToolsForUpload(c)
	url := s.toolsURL(c, "binaryVersion="+vers.String())
	url.Path = "/tools"
	resp, err := s.uploadRequest(c, url.String(), true, toolPath)
	c.Assert(err, gc.IsNil)
	// Check the response.
	stor := s.Environ.Storage()
	toolsURL, err := stor.URL(tools.StorageName(vers))
	c.Assert(err, gc.IsNil)
	expectedTools[0].URL = toolsURL
	s.assertUploadResponse(c, resp, expectedTools[0])
}
Пример #20
0
func (s *toolsSuite) TestUpload(c *gc.C) {
	// Make some fake tools.
	expectedTools, vers, toolPath := s.setupToolsForUpload(c)
	// Now try uploading them.
	resp, err := s.uploadRequest(
		c, s.toolsURI(c, "?binaryVersion="+vers.String()), true, toolPath)
	c.Assert(err, gc.IsNil)

	// Check the response.
	stor := s.Environ.Storage()
	toolsURL, err := stor.URL(tools.StorageName(vers))
	c.Assert(err, gc.IsNil)
	expectedTools[0].URL = toolsURL
	s.assertUploadResponse(c, resp, expectedTools[0])

	// Check the contents.
	r, err := stor.Get(tools.StorageName(vers))
	c.Assert(err, gc.IsNil)
	uploadedData, err := ioutil.ReadAll(r)
	c.Assert(err, gc.IsNil)
	expectedData, err := ioutil.ReadFile(toolPath)
	c.Assert(err, gc.IsNil)
	c.Assert(uploadedData, gc.DeepEquals, expectedData)
}
Пример #21
0
func (s *toolsSuite) TestUploadAllowsEnvUUIDPath(c *gc.C) {
	// Check that we can upload tools to https://host:port/ENVUUID/tools
	environ, err := s.State.Environment()
	c.Assert(err, gc.IsNil)
	expectedTools, vers, toolPath := s.setupToolsForUpload(c)
	url := s.toolsURL(c, "binaryVersion="+vers.String())
	url.Path = fmt.Sprintf("/environment/%s/tools", environ.UUID())
	resp, err := s.uploadRequest(c, url.String(), true, toolPath)
	c.Assert(err, gc.IsNil)
	// Check the response.
	stor := s.Environ.Storage()
	toolsURL, err := stor.URL(tools.StorageName(vers))
	c.Assert(err, gc.IsNil)
	expectedTools[0].URL = toolsURL
	s.assertUploadResponse(c, resp, expectedTools[0])
}
Пример #22
0
// handleUpload uploads the tools data from the reader to env storage as the specified version.
func (h *toolsUploadHandler) handleUpload(r io.Reader, toolsVersion version.Binary) (*tools.Tools, bool, error) {
	// Set up a local temp directory for the tools tarball.
	tmpDir, err := ioutil.TempDir("", "juju-upload-tools-")
	if err != nil {
		return nil, false, errors.Annotate(err, "cannot create temp dir")
	}
	defer os.RemoveAll(tmpDir)
	toolsFilename := envtools.StorageName(toolsVersion)
	toolsDir := path.Dir(toolsFilename)
	fullToolsDir := path.Join(tmpDir, toolsDir)
	err = os.MkdirAll(fullToolsDir, 0700)
	if err != nil {
		return nil, false, errors.Annotatef(err, "cannot create tools dir %s", toolsDir)
	}

	// Read the tools tarball from the request, calculating the sha256 along the way.
	fullToolsFilename := path.Join(tmpDir, toolsFilename)
	toolsFile, err := os.Create(fullToolsFilename)
	if err != nil {
		return nil, false, errors.Annotatef(err, "cannot create tools file %s", fullToolsFilename)
	}
	logger.Debugf("saving uploaded tools to temp file: %s", fullToolsFilename)
	defer toolsFile.Close()
	sha256hash := sha256.New()
	var size int64
	if size, err = io.Copy(toolsFile, io.TeeReader(r, sha256hash)); err != nil {
		return nil, false, errors.Annotate(err, "error processing file upload")
	}
	if size == 0 {
		return nil, false, errors.New("no tools uploaded")
	}

	// TODO(wallyworld): check integrity of tools tarball.

	// Create a tools record and sync to storage.
	uploadedTools := &tools.Tools{
		Version: toolsVersion,
		Size:    size,
		SHA256:  fmt.Sprintf("%x", sha256hash.Sum(nil)),
	}
	logger.Debugf("about to upload tools %+v to storage", uploadedTools)
	return h.uploadToStorage(uploadedTools, tmpDir, toolsFilename)
}
Пример #23
0
func (s *toolsSuite) TestDownloadFetchesAndVerifiesHash(c *gc.C) {
	// Upload fake tools, then upload over the top so the SHA256 hash does not match.
	s.PatchValue(&jujuversion.Current, testing.FakeVersionNumber)
	stor := s.DefaultToolsStorage
	envtesting.RemoveTools(c, stor, "released")
	current := version.Binary{
		Number: jujuversion.Current,
		Arch:   arch.HostArch(),
		Series: series.HostSeries(),
	}
	tools := envtesting.AssertUploadFakeToolsVersions(c, stor, "released", "released", current)[0]
	sameSize := strings.Repeat("!", int(tools.Size))
	err := stor.Put(envtools.StorageName(tools.Version, "released"), strings.NewReader(sameSize), tools.Size)
	c.Assert(err, jc.ErrorIsNil)

	resp := s.downloadRequest(c, tools.Version, "")
	s.assertErrorResponse(c, resp, http.StatusBadRequest, "error fetching tools: hash mismatch for .*")
	s.assertToolsNotStored(c, tools.Version.String())
}
Пример #24
0
// copyOneToolsPackage copies one tool from the source to the target.
func copyOneToolsPackage(tool *coretools.Tools, dest storage.Storage) error {
	toolsName := envtools.StorageName(tool.Version)
	logger.Infof("copying %v", toolsName)
	resp, err := utils.GetValidatingHTTPClient().Get(tool.URL)
	if err != nil {
		return err
	}
	buf := &bytes.Buffer{}
	srcFile := resp.Body
	defer srcFile.Close()
	tool.SHA256, tool.Size, err = utils.ReadSHA256(io.TeeReader(srcFile, buf))
	if err != nil {
		return err
	}
	sizeInKB := (tool.Size + 512) / 1024
	logger.Infof("downloaded %v (%dkB), uploading", toolsName, sizeInKB)
	logger.Infof("download %dkB, uploading", sizeInKB)
	return dest.Put(toolsName, buf, tool.Size)
}
Пример #25
0
func fetchToolsArchive(stor storage.StorageReader, toolsDir string, agentTools *tools.Tools) ([]byte, error) {
	r, err := stor.Get(envtools.StorageName(agentTools.Version, toolsDir))
	if err != nil {
		return nil, err
	}
	defer r.Close()

	var buf bytes.Buffer
	hash, size, err := utils.ReadSHA256(io.TeeReader(r, &buf))
	if err != nil {
		return nil, err
	}
	if hash != agentTools.SHA256 {
		return nil, errors.New("hash mismatch")
	}
	if size != agentTools.Size {
		return nil, errors.New("size mismatch")
	}
	return buf.Bytes(), nil
}
Пример #26
0
func (s *configureSuite) getCloudConfig(c *gc.C, stateServer bool, vers version.Binary) *cloudinit.Config {
	var mcfg *envcloudinit.MachineConfig
	if stateServer {
		mcfg = environs.NewBootstrapMachineConfig("private-key")
		mcfg.InstanceId = "instance-id"
		mcfg.Jobs = []params.MachineJob{params.JobManageEnviron, params.JobHostUnits}
	} else {
		mcfg = environs.NewMachineConfig("0", "ya", nil, nil, nil)
		mcfg.Jobs = []params.MachineJob{params.JobHostUnits}
	}
	mcfg.Tools = &tools.Tools{
		Version: vers,
		URL:     "file:///var/lib/juju/storage/" + envtools.StorageName(vers),
	}
	environConfig := testConfig(c, stateServer, vers)
	err := environs.FinishMachineConfig(mcfg, environConfig, constraints.Value{})
	c.Assert(err, gc.IsNil)
	cloudcfg := cloudinit.New()
	err = envcloudinit.Configure(mcfg, cloudcfg)
	c.Assert(err, gc.IsNil)
	return cloudcfg
}
Пример #27
0
// copyOneToolsPackage copies one tool from the source to the target.
func copyOneToolsPackage(tools *coretools.Tools, u ToolsUploader) error {
	toolsName := envtools.StorageName(tools.Version)
	logger.Infof("downloading %v (%v)", toolsName, tools.URL)
	resp, err := utils.GetValidatingHTTPClient().Get(tools.URL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	// Verify SHA-256 hash.
	var buf bytes.Buffer
	sha256, size, err := utils.ReadSHA256(io.TeeReader(resp.Body, &buf))
	if err != nil {
		return err
	}
	if tools.SHA256 == "" {
		logger.Warningf("no SHA-256 hash for %v", tools.SHA256)
	} else if sha256 != tools.SHA256 {
		return errors.Errorf("SHA-256 hash mismatch (%v/%v)", sha256, tools.SHA256)
	}
	sizeInKB := (size + 512) / 1024
	logger.Infof("uploading %v (%dkB) to environment", toolsName, sizeInKB)
	return u.UploadTools(tools, buf.Bytes())
}
Пример #28
0
func (s *UpgradeJujuSuite) TestUpgradeJuju(c *gc.C) {
	oldVersion := version.Current
	defer func() {
		version.Current = oldVersion
	}()

	for i, test := range upgradeJujuTests {
		c.Logf("\ntest %d: %s", i, test.about)
		s.Reset(c)

		// Set up apparent CLI version and initialize the command.
		version.Current = version.MustParseBinary(test.currentVersion)
		com := &UpgradeJujuCommand{}
		if err := coretesting.InitCommand(envcmd.Wrap(com), test.args); err != nil {
			if test.expectInitErr != "" {
				c.Check(err, gc.ErrorMatches, test.expectInitErr)
			} else {
				c.Check(err, gc.IsNil)
			}
			continue
		}

		// Set up state and environ, and run the command.
		toolsDir := c.MkDir()
		updateAttrs := map[string]interface{}{
			"agent-version":      test.agentVersion,
			"tools-metadata-url": "file://" + toolsDir,
		}
		err := s.State.UpdateEnvironConfig(updateAttrs, nil, nil)
		c.Assert(err, gc.IsNil)
		versions := make([]version.Binary, len(test.tools))
		for i, v := range test.tools {
			versions[i] = version.MustParseBinary(v)
		}
		if len(versions) > 0 {
			envtesting.MustUploadFakeToolsVersions(s.Conn.Environ.Storage(), versions...)
			stor, err := filestorage.NewFileStorageWriter(toolsDir)
			c.Assert(err, gc.IsNil)
			envtesting.MustUploadFakeToolsVersions(stor, versions...)
		}

		err = com.Run(coretesting.Context(c))
		if test.expectErr != "" {
			c.Check(err, gc.ErrorMatches, test.expectErr)
			continue
		} else if !c.Check(err, gc.IsNil) {
			continue
		}

		// Check expected changes to environ/state.
		cfg, err := s.State.EnvironConfig()
		c.Check(err, gc.IsNil)
		agentVersion, ok := cfg.AgentVersion()
		c.Check(ok, gc.Equals, true)
		c.Check(agentVersion, gc.Equals, version.MustParse(test.expectVersion))

		for _, uploaded := range test.expectUploaded {
			// Substitute latest LTS for placeholder in expected series for uploaded tools
			uploaded = strings.Replace(uploaded, "%LTS%", config.LatestLtsSeries(), 1)

			vers := version.MustParseBinary(uploaded)
			r, err := storage.Get(s.Conn.Environ.Storage(), envtools.StorageName(vers))
			if !c.Check(err, gc.IsNil) {
				continue
			}
			data, err := ioutil.ReadAll(r)
			r.Close()
			c.Check(err, gc.IsNil)
			expectContent := version.Current
			expectContent.Number = agentVersion
			checkToolsContent(c, data, "jujud contents "+expectContent.String())
		}
	}
}
Пример #29
0
// NewManualBootstrapEnviron wraps a LocalStorageEnviron with another which
// overrides the Bootstrap method; when Bootstrap is invoked, the specified
// host will be manually bootstrapped.
//
// InitUbuntuUser is expected to have been executed successfully against
// the host being bootstrapped.
func Bootstrap(args BootstrapArgs) (err error) {
	if args.Host == "" {
		return errors.New("host argument is empty")
	}
	if args.Environ == nil {
		return errors.New("environ argument is nil")
	}
	if args.DataDir == "" {
		return errors.New("data-dir argument is empty")
	}
	if args.Series == "" {
		return errors.New("series argument is empty")
	}
	if args.HardwareCharacteristics == nil {
		return errors.New("hardware characteristics argument is empty")
	}
	if len(args.PossibleTools) == 0 {
		return errors.New("possible tools is empty")
	}

	provisioned, err := checkProvisioned(args.Host)
	if err != nil {
		return fmt.Errorf("failed to check provisioned status: %v", err)
	}
	if provisioned {
		return ErrProvisioned
	}

	// Filter tools based on detected series/arch.
	logger.Infof("Filtering possible tools: %v", args.PossibleTools)
	possibleTools, err := args.PossibleTools.Match(tools.Filter{
		Arch:   *args.HardwareCharacteristics.Arch,
		Series: args.Series,
	})
	if err != nil {
		return err
	}

	// Store the state file. If provisioning fails, we'll remove the file.
	logger.Infof("Saving bootstrap state file to bootstrap storage")
	bootstrapStorage := args.Environ.Storage()
	err = bootstrap.SaveState(
		bootstrapStorage,
		&bootstrap.BootstrapState{
			StateInstances: []instance.Id{BootstrapInstanceId},
		},
	)
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			logger.Errorf("bootstrapping failed, removing state file: %v", err)
			bootstrapStorage.Remove(bootstrap.StateFile)
		}
	}()

	// If the tools are on the machine already, get a file:// scheme tools URL.
	tools := *possibleTools[0]
	storageDir := args.Environ.StorageDir()
	toolsStorageName := envtools.StorageName(tools.Version)
	if url, _ := bootstrapStorage.URL(toolsStorageName); url == tools.URL {
		tools.URL = fmt.Sprintf("file://%s/%s", storageDir, toolsStorageName)
	}

	// Add the local storage configuration.
	agentEnv, err := localstorage.StoreConfig(args.Environ)
	if err != nil {
		return err
	}

	privateKey, err := common.GenerateSystemSSHKey(args.Environ)
	if err != nil {
		return err
	}

	// Finally, provision the machine agent.
	mcfg := environs.NewBootstrapMachineConfig(privateKey)
	mcfg.InstanceId = BootstrapInstanceId
	mcfg.HardwareCharacteristics = args.HardwareCharacteristics
	if args.DataDir != "" {
		mcfg.DataDir = args.DataDir
	}
	mcfg.Tools = &tools
	err = environs.FinishMachineConfig(mcfg, args.Environ.Config(), constraints.Value{})
	if err != nil {
		return err
	}
	for k, v := range agentEnv {
		mcfg.AgentEnvironment[k] = v
	}
	return provisionMachineAgent(args.Host, mcfg, args.Context.GetStderr())
}
Пример #30
0
func (s *StorageSuite) TestStorageName(c *gc.C) {
	vers := version.MustParseBinary("1.2.3-precise-amd64")
	path := envtools.StorageName(vers, "proposed")
	c.Assert(path, gc.Equals, "tools/proposed/juju-1.2.3-precise-amd64.tgz")
}