Beispiel #1
0
func (s *toolsSuite) TestUploadFakeSeries(c *gc.C) {
	// Make some fake tools.
	localStorage := c.MkDir()
	vers := version.MustParseBinary("1.9.0-quantal-amd64")
	versionStrings := []string{vers.String()}
	expectedTools := ttesting.MakeToolsWithCheckSum(c, localStorage, "releases", versionStrings)

	// Now try uploading them.
	toolsFile := tools.StorageName(vers)
	params := "?binaryVersion=" + vers.String() + "&series=precise,trusty"
	resp, err := s.uploadRequest(c, s.toolsURI(c, params), true, path.Join(localStorage, toolsFile))
	c.Assert(err, gc.IsNil)

	// Check the response.
	stor := s.Conn.Environ.Storage()
	toolsURL, err := stor.URL(tools.StorageName(vers))
	c.Assert(err, gc.IsNil)
	expectedTools[0].URL = toolsURL
	s.assertUploadResponse(c, resp, expectedTools[0])

	// Check the contents.
	for _, series := range []string{"precise", "quantal", "trusty"} {
		toolsVersion := vers
		toolsVersion.Series = series
		r, err := stor.Get(tools.StorageName(toolsVersion))
		c.Assert(err, gc.IsNil)
		uploadedData, err := ioutil.ReadAll(r)
		c.Assert(err, gc.IsNil)
		expectedData, err := ioutil.ReadFile(filepath.Join(localStorage, tools.StorageName(vers)))
		c.Assert(err, gc.IsNil)
		c.Assert(uploadedData, gc.DeepEquals, expectedData)
	}
}
Beispiel #2
0
// RemoveFakeTools deletes the fake tools from the supplied storage.
func RemoveFakeTools(c *gc.C, stor storage.Storage) {
	c.Logf("removing fake tools")
	toolsVersion := version.Current
	name := envtools.StorageName(toolsVersion)
	err := stor.Remove(name)
	c.Check(err, gc.IsNil)
	defaultSeries := coretesting.FakeDefaultSeries
	if version.Current.Series != defaultSeries {
		toolsVersion.Series = defaultSeries
		name := envtools.StorageName(toolsVersion)
		err := stor.Remove(name)
		c.Check(err, gc.IsNil)
	}
	RemoveFakeToolsMetadata(c, stor)
}
Beispiel #3
0
func (*metadataHelperSuite) TestMetadataFromTools(c *gc.C) {
	metadata := tools.MetadataFromTools(nil)
	c.Assert(metadata, gc.HasLen, 0)

	toolsList := coretools.List{{
		Version: version.MustParseBinary("1.2.3-precise-amd64"),
		Size:    123,
		SHA256:  "abc",
	}, {
		Version: version.MustParseBinary("2.0.1-raring-amd64"),
		URL:     "file:///tmp/releases/juju-2.0.1-raring-amd64.tgz",
		Size:    456,
		SHA256:  "xyz",
	}}
	metadata = tools.MetadataFromTools(toolsList)
	c.Assert(metadata, gc.HasLen, len(toolsList))
	for i, t := range toolsList {
		md := metadata[i]
		c.Assert(md.Release, gc.Equals, t.Version.Series)
		c.Assert(md.Version, gc.Equals, t.Version.Number.String())
		c.Assert(md.Arch, gc.Equals, t.Version.Arch)
		// FullPath is only filled out when reading tools using simplestreams.
		// It's not needed elsewhere and requires a URL() call.
		c.Assert(md.FullPath, gc.Equals, "")
		c.Assert(md.Path, gc.Equals, tools.StorageName(t.Version)[len("tools/"):])
		c.Assert(md.FileType, gc.Equals, "tar.gz")
		c.Assert(md.Size, gc.Equals, t.Size)
		c.Assert(md.SHA256, gc.Equals, t.SHA256)
	}
}
Beispiel #4
0
func (s *bootstrapSuite) TestBootstrapToolsFileURL(c *gc.C) {
	storageName := tools.StorageName(version.Current)
	sftpURL, err := s.env.Storage().URL(storageName)
	c.Assert(err, gc.IsNil)
	fileURL := fmt.Sprintf("file://%s/%s", s.env.storageDir, storageName)
	s.testBootstrapToolsURL(c, sftpURL, fileURL)
}
Beispiel #5
0
// buildToolsTarball bundles a tools tarball and places it in a temp directory in
// the expected tools path.
func buildToolsTarball(forceVersion *version.Number) (builtTools *BuiltTools, err error) {
	// TODO(rog) find binaries from $PATH when not using a development
	// version of juju within a $GOPATH.

	logger.Debugf("Building tools")
	// We create the entire archive before asking the environment to
	// start uploading so that we can be sure we have archived
	// correctly.
	f, err := ioutil.TempFile("", "juju-tgz")
	if err != nil {
		return nil, err
	}
	defer f.Close()
	defer os.Remove(f.Name())
	toolsVersion, sha256Hash, err := envtools.BundleTools(f, forceVersion)
	if err != nil {
		return nil, err
	}
	fileInfo, err := f.Stat()
	if err != nil {
		return nil, fmt.Errorf("cannot stat newly made tools archive: %v", err)
	}
	size := fileInfo.Size()
	logger.Infof("built tools %v (%dkB)", toolsVersion, (size+512)/1024)
	baseToolsDir, err := ioutil.TempDir("", "juju-tools")
	if err != nil {
		return nil, err
	}

	// If we exit with an error, clean up the built tools directory.
	defer func() {
		if err != nil {
			os.RemoveAll(baseToolsDir)
		}
	}()

	err = os.MkdirAll(filepath.Join(baseToolsDir, storage.BaseToolsPath, "releases"), 0755)
	if err != nil {
		return nil, err
	}
	storageName := envtools.StorageName(toolsVersion)
	err = utils.CopyFile(filepath.Join(baseToolsDir, storageName), f.Name())
	if err != nil {
		return nil, err
	}
	return &BuiltTools{
		Version:     toolsVersion,
		Dir:         baseToolsDir,
		StorageName: storageName,
		Size:        size,
		Sha256Hash:  sha256Hash,
	}, nil
}
Beispiel #6
0
func (s *UpgraderSuite) TestUpgraderRetryAndChanged(c *gc.C) {
	stor := s.Conn.Environ.Storage()
	oldTools := envtesting.PrimeTools(c, stor, s.DataDir(), version.MustParseBinary("5.4.3-precise-amd64"))
	s.PatchValue(&version.Current, oldTools.Version)
	newTools := envtesting.AssertUploadFakeToolsVersions(
		c, stor, version.MustParseBinary("5.4.5-precise-amd64"))[0]
	err := statetesting.SetAgentVersion(s.State, newTools.Version.Number)
	c.Assert(err, gc.IsNil)

	retryc := make(chan time.Time)
	*upgrader.RetryAfter = func() <-chan time.Time {
		c.Logf("replacement retry after")
		return retryc
	}
	dummy.Poison(s.Conn.Environ.Storage(), envtools.StorageName(newTools.Version), fmt.Errorf("a non-fatal dose"))
	u := s.makeUpgrader()
	defer u.Stop()

	for i := 0; i < 3; i++ {
		select {
		case retryc <- time.Now():
		case <-time.After(coretesting.LongWait):
			c.Fatalf("upgrader did not retry (attempt %d)", i)
		}
	}

	// Make it upgrade to some newer tools that can be
	// downloaded ok; it should stop retrying, download
	// the newer tools and exit.
	newerTools := envtesting.AssertUploadFakeToolsVersions(
		c, s.Conn.Environ.Storage(), version.MustParseBinary("5.4.6-precise-amd64"))[0]

	err = statetesting.SetAgentVersion(s.State, newerTools.Version.Number)
	c.Assert(err, gc.IsNil)

	s.BackingState.StartSync()
	done := make(chan error)
	go func() {
		done <- u.Wait()
	}()
	select {
	case err := <-done:
		envtesting.CheckUpgraderReadyError(c, err, &upgrader.UpgradeReadyError{
			AgentName: s.machine.Tag(),
			OldTools:  oldTools.Version,
			NewTools:  newerTools.Version,
			DataDir:   s.DataDir(),
		})
	case <-time.After(coretesting.LongWait):
		c.Fatalf("upgrader did not quit after upgrading")
	}
}
Beispiel #7
0
func uploadFakeToolsVersion(stor storage.Storage, vers version.Binary) (*coretools.Tools, error) {
	logger.Infof("uploading FAKE tools %s", vers)
	tgz, checksum := coretesting.TarGz(
		coretesting.NewTarFile("jujud", 0777, "jujud contents "+vers.String()))
	size := int64(len(tgz))
	name := envtools.StorageName(vers)
	if err := stor.Put(name, bytes.NewReader(tgz), size); err != nil {
		return nil, err
	}
	url, err := stor.URL(name)
	if err != nil {
		return nil, err
	}
	return &coretools.Tools{URL: url, Version: vers, Size: size, SHA256: checksum}, nil
}
Beispiel #8
0
// cloneToolsForSeries copies the built tools tarball into a tarball for the specified
// series and generates corresponding metadata.
func cloneToolsForSeries(toolsInfo *BuiltTools, series ...string) error {
	// Copy the tools to the target storage, recording a Tools struct for each one.
	var targetTools coretools.List
	targetTools = append(targetTools, &coretools.Tools{
		Version: toolsInfo.Version,
		Size:    toolsInfo.Size,
		SHA256:  toolsInfo.Sha256Hash,
	})
	putTools := func(vers version.Binary) (string, error) {
		name := envtools.StorageName(vers)
		src := filepath.Join(toolsInfo.Dir, toolsInfo.StorageName)
		dest := filepath.Join(toolsInfo.Dir, name)
		err := utils.CopyFile(dest, src)
		if err != nil {
			return "", err
		}
		// Append to targetTools the attributes required to write out tools metadata.
		targetTools = append(targetTools, &coretools.Tools{
			Version: vers,
			Size:    toolsInfo.Size,
			SHA256:  toolsInfo.Sha256Hash,
		})
		return name, nil
	}
	logger.Debugf("generating tarballs for %v", series)
	for _, series := range series {
		_, err := ubuntu.SeriesVersion(series)
		if err != nil {
			return err
		}
		if series != toolsInfo.Version.Series {
			fakeVersion := toolsInfo.Version
			fakeVersion.Series = series
			if _, err := putTools(fakeVersion); err != nil {
				return err
			}
		}
	}
	// The tools have been copied to a temp location from which they will be uploaded,
	// now write out the matching simplestreams metadata so that SyncTools can find them.
	metadataStore, err := filestorage.NewFileStorageWriter(toolsInfo.Dir)
	if err != nil {
		return err
	}
	logger.Debugf("generating tools metadata")
	return envtools.MergeAndWriteMetadata(metadataStore, targetTools, false)
}
Beispiel #9
0
// handleUpload uploads the tools data from the reader to env storage as the specified version.
func (h *toolsHandler) handleUpload(r io.Reader, toolsVersion version.Binary, fakeSeries ...string) (*tools.Tools, bool, error) {
	// Set up a local temp directory for the tools tarball.
	tmpDir, err := ioutil.TempDir("", "juju-upload-tools-")
	if err != nil {
		return nil, false, fmt.Errorf("cannot create temp dir: %v", err)
	}
	defer os.RemoveAll(tmpDir)
	toolsFilename := envtools.StorageName(toolsVersion)
	toolsDir := path.Dir(toolsFilename)
	fullToolsDir := path.Join(tmpDir, toolsDir)
	err = os.MkdirAll(fullToolsDir, 0700)
	if err != nil {
		return nil, false, fmt.Errorf("cannot create tools dir %s: %v", toolsDir, err)
	}

	// Read the tools tarball from the request, calculating the sha256 along the way.
	fullToolsFilename := path.Join(tmpDir, toolsFilename)
	toolsFile, err := os.Create(fullToolsFilename)
	if err != nil {
		return nil, false, fmt.Errorf("cannot create tools file %s: %v", fullToolsFilename, err)
	}
	logger.Debugf("saving uploaded tools to temp file: %s", fullToolsFilename)
	defer toolsFile.Close()
	sha256hash := sha256.New()
	var size int64
	if size, err = io.Copy(toolsFile, io.TeeReader(r, sha256hash)); err != nil {
		return nil, false, fmt.Errorf("error processing file upload: %v", err)
	}
	if size == 0 {
		return nil, false, fmt.Errorf("no tools uploaded")
	}

	// TODO(wallyworld): check integrity of tools tarball.

	// Create a tools record and sync to storage.
	uploadedTools := &tools.Tools{
		Version: toolsVersion,
		Size:    size,
		SHA256:  fmt.Sprintf("%x", sha256hash.Sum(nil)),
	}
	logger.Debugf("about to upload tools %+v to storage", uploadedTools)
	return h.uploadToStorage(uploadedTools, tmpDir, toolsFilename, fakeSeries...)
}
Beispiel #10
0
// copyOneToolsPackage copies one tool from the source to the target.
func copyOneToolsPackage(tool *coretools.Tools, dest storage.Storage) error {
	toolsName := envtools.StorageName(tool.Version)
	logger.Infof("copying %v", toolsName)
	resp, err := utils.GetValidatingHTTPClient().Get(tool.URL)
	if err != nil {
		return err
	}
	buf := &bytes.Buffer{}
	srcFile := resp.Body
	defer srcFile.Close()
	tool.SHA256, tool.Size, err = utils.ReadSHA256(io.TeeReader(srcFile, buf))
	if err != nil {
		return err
	}
	sizeInKB := (tool.Size + 512) / 1024
	logger.Infof("downloaded %v (%dkB), uploading", toolsName, sizeInKB)
	logger.Infof("download %dkB, uploading", sizeInKB)
	return dest.Put(toolsName, buf, tool.Size)
}
Beispiel #11
0
func (s *configureSuite) getCloudConfig(c *gc.C, stateServer bool, vers version.Binary) *cloudinit.Config {
	var mcfg *envcloudinit.MachineConfig
	if stateServer {
		mcfg = environs.NewBootstrapMachineConfig("private-key")
		mcfg.InstanceId = "instance-id"
		mcfg.Jobs = []params.MachineJob{params.JobManageEnviron, params.JobHostUnits}
	} else {
		mcfg = environs.NewMachineConfig("0", "ya", nil, nil, nil, nil)
		mcfg.Jobs = []params.MachineJob{params.JobHostUnits}
	}
	mcfg.Tools = &tools.Tools{
		Version: vers,
		URL:     "file:///var/lib/juju/storage/" + envtools.StorageName(vers),
	}
	environConfig := testConfig(c, stateServer, vers)
	err := environs.FinishMachineConfig(mcfg, environConfig, constraints.Value{})
	c.Assert(err, gc.IsNil)
	cloudcfg := cloudinit.New()
	err = envcloudinit.Configure(mcfg, cloudcfg)
	c.Assert(err, gc.IsNil)
	return cloudcfg
}
Beispiel #12
0
// getMockBuildTools returns a sync.BuildToolsTarballFunc implementation which generates
// a fake tools tarball.
func (s *UpgradeJujuSuite) getMockBuildTools(c *gc.C) sync.BuildToolsTarballFunc {
	return func(forceVersion *version.Number) (*sync.BuiltTools, error) {
		// UploadFakeToolsVersions requires a storage to write to.
		stor, err := filestorage.NewFileStorageWriter(s.toolsDir)
		c.Assert(err, gc.IsNil)
		vers := version.Current
		if forceVersion != nil {
			vers.Number = *forceVersion
		}
		versions := []version.Binary{vers}
		uploadedTools, err := envtesting.UploadFakeToolsVersions(stor, versions...)
		c.Assert(err, gc.IsNil)
		agentTools := uploadedTools[0]
		return &sync.BuiltTools{
			Dir:         s.toolsDir,
			StorageName: envtools.StorageName(vers),
			Version:     vers,
			Size:        agentTools.Size,
			Sha256Hash:  agentTools.SHA256,
		}, nil
	}
}
Beispiel #13
0
func (t *LiveTests) TestBootstrapWithDefaultSeries(c *gc.C) {
	if !t.HasProvisioner {
		c.Skip("HasProvisioner is false; cannot test deployment")
	}

	current := version.Current
	other := current
	other.Series = "quantal"
	if current == other {
		other.Series = "precise"
	}

	dummyCfg, err := config.New(config.NoDefaults, dummy.SampleConfig().Merge(coretesting.Attrs{
		"state-server": false,
		"name":         "dummy storage",
	}))
	dummyenv, err := environs.Prepare(dummyCfg, coretesting.Context(c), configstore.NewMem())
	c.Assert(err, gc.IsNil)
	defer dummyenv.Destroy()

	t.Destroy(c)

	attrs := t.TestConfig.Merge(coretesting.Attrs{"default-series": other.Series})
	cfg, err := config.New(config.NoDefaults, attrs)
	c.Assert(err, gc.IsNil)
	env, err := environs.Prepare(cfg, coretesting.Context(c), t.ConfigStore)
	c.Assert(err, gc.IsNil)
	defer environs.Destroy(env, t.ConfigStore)

	currentName := envtools.StorageName(current)
	otherName := envtools.StorageName(other)
	envStorage := env.Storage()
	dummyStorage := dummyenv.Storage()

	defer envStorage.Remove(otherName)

	_, err = sync.Upload(dummyStorage, &current.Number)
	c.Assert(err, gc.IsNil)

	// This will only work while cross-compiling across releases is safe,
	// which depends on external elements. Tends to be safe for the last
	// few releases, but we may have to refactor some day.
	err = storageCopy(dummyStorage, currentName, envStorage, otherName)
	c.Assert(err, gc.IsNil)

	err = bootstrap.Bootstrap(coretesting.Context(c), env, environs.BootstrapParams{})
	c.Assert(err, gc.IsNil)

	conn, err := juju.NewConn(env)
	c.Assert(err, gc.IsNil)
	defer conn.Close()

	// Wait for machine agent to come up on the bootstrap
	// machine and ensure it deployed the proper series.
	m0, err := conn.State.Machine("0")
	c.Assert(err, gc.IsNil)
	mw0 := newMachineToolWaiter(m0)
	defer mw0.Stop()

	waitAgentTools(c, mw0, other)
}
Beispiel #14
0
// NewManualBootstrapEnviron wraps a LocalStorageEnviron with another which
// overrides the Bootstrap method; when Bootstrap is invoked, the specified
// host will be manually bootstrapped.
//
// InitUbuntuUser is expected to have been executed successfully against
// the host being bootstrapped.
func Bootstrap(args BootstrapArgs) (err error) {
	if args.Host == "" {
		return errors.New("host argument is empty")
	}
	if args.Environ == nil {
		return errors.New("environ argument is nil")
	}
	if args.DataDir == "" {
		return errors.New("data-dir argument is empty")
	}
	if args.Series == "" {
		return errors.New("series argument is empty")
	}
	if args.HardwareCharacteristics == nil {
		return errors.New("hardware characteristics argument is empty")
	}
	if len(args.PossibleTools) == 0 {
		return errors.New("possible tools is empty")
	}

	provisioned, err := checkProvisioned(args.Host)
	if err != nil {
		return fmt.Errorf("failed to check provisioned status: %v", err)
	}
	if provisioned {
		return ErrProvisioned
	}

	// Filter tools based on detected series/arch.
	logger.Infof("Filtering possible tools: %v", args.PossibleTools)
	possibleTools, err := args.PossibleTools.Match(tools.Filter{
		Arch:   *args.HardwareCharacteristics.Arch,
		Series: args.Series,
	})
	if err != nil {
		return err
	}

	// Store the state file. If provisioning fails, we'll remove the file.
	logger.Infof("Saving bootstrap state file to bootstrap storage")
	bootstrapStorage := args.Environ.Storage()
	err = bootstrap.SaveState(
		bootstrapStorage,
		&bootstrap.BootstrapState{
			StateInstances: []instance.Id{BootstrapInstanceId},
		},
	)
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			logger.Errorf("bootstrapping failed, removing state file: %v", err)
			bootstrapStorage.Remove(bootstrap.StateFile)
		}
	}()

	// If the tools are on the machine already, get a file:// scheme tools URL.
	tools := *possibleTools[0]
	storageDir := args.Environ.StorageDir()
	toolsStorageName := envtools.StorageName(tools.Version)
	if url, _ := bootstrapStorage.URL(toolsStorageName); url == tools.URL {
		tools.URL = fmt.Sprintf("file://%s/%s", storageDir, toolsStorageName)
	}

	// Add the local storage configuration.
	agentEnv, err := localstorage.StoreConfig(args.Environ)
	if err != nil {
		return err
	}

	privateKey, err := common.GenerateSystemSSHKey(args.Environ)
	if err != nil {
		return err
	}

	// Finally, provision the machine agent.
	mcfg := environs.NewBootstrapMachineConfig(privateKey)
	mcfg.InstanceId = BootstrapInstanceId
	mcfg.HardwareCharacteristics = args.HardwareCharacteristics
	if args.DataDir != "" {
		mcfg.DataDir = args.DataDir
	}
	mcfg.Tools = &tools
	err = environs.FinishMachineConfig(mcfg, args.Environ.Config(), constraints.Value{})
	if err != nil {
		return err
	}
	for k, v := range agentEnv {
		mcfg.AgentEnvironment[k] = v
	}
	return provisionMachineAgent(args.Host, mcfg, args.Context.GetStderr())
}
Beispiel #15
0
func (s *StorageSuite) TestStorageName(c *gc.C) {
	vers := version.MustParseBinary("1.2.3-precise-amd64")
	path := envtools.StorageName(vers)
	c.Assert(path, gc.Equals, "tools/releases/juju-1.2.3-precise-amd64.tgz")
}
Beispiel #16
0
func (s *UpgradeJujuSuite) TestUpgradeJuju(c *gc.C) {
	s.PatchValue(&sync.BuildToolsTarball, s.getMockBuildTools(c))
	oldVersion := version.Current
	defer func() {
		version.Current = oldVersion
	}()

	for i, test := range upgradeJujuTests {
		c.Logf("\ntest %d: %s", i, test.about)
		s.Reset(c)

		// Set up apparent CLI version and initialize the command.
		version.Current = version.MustParseBinary(test.currentVersion)
		com := &UpgradeJujuCommand{}
		if err := coretesting.InitCommand(com, test.args); err != nil {
			if test.expectInitErr != "" {
				c.Check(err, gc.ErrorMatches, test.expectInitErr)
			} else {
				c.Check(err, gc.IsNil)
			}
			continue
		}

		// Set up state and environ, and run the command.
		toolsDir := c.MkDir()
		updateAttrs := map[string]interface{}{
			"agent-version":      test.agentVersion,
			"tools-metadata-url": "file://" + toolsDir,
		}
		err := s.State.UpdateEnvironConfig(updateAttrs, nil, nil)
		c.Assert(err, gc.IsNil)
		versions := make([]version.Binary, len(test.tools))
		for i, v := range test.tools {
			versions[i] = version.MustParseBinary(v)
		}
		if len(versions) > 0 {
			envtesting.MustUploadFakeToolsVersions(s.Conn.Environ.Storage(), versions...)
			stor, err := filestorage.NewFileStorageWriter(toolsDir)
			c.Assert(err, gc.IsNil)
			envtesting.MustUploadFakeToolsVersions(stor, versions...)
		}

		err = com.Run(coretesting.Context(c))
		if test.expectErr != "" {
			c.Check(err, gc.ErrorMatches, test.expectErr)
			continue
		} else if !c.Check(err, gc.IsNil) {
			continue
		}

		// Check expected changes to environ/state.
		cfg, err := s.State.EnvironConfig()
		c.Check(err, gc.IsNil)
		agentVersion, ok := cfg.AgentVersion()
		c.Check(ok, gc.Equals, true)
		c.Check(agentVersion, gc.Equals, version.MustParse(test.expectVersion))

		for _, uploaded := range test.expectUploaded {
			// Substitute latest LTS for placeholder in expected series for uploaded tools
			uploaded = strings.Replace(uploaded, "%LTS%", config.LatestLtsSeries(), 1)

			vers := version.MustParseBinary(uploaded)
			r, err := storage.Get(s.Conn.Environ.Storage(), envtools.StorageName(vers))
			if !c.Check(err, gc.IsNil) {
				continue
			}
			data, err := ioutil.ReadAll(r)
			r.Close()
			c.Check(err, gc.IsNil)
			expectContent := version.Current
			expectContent.Number = agentVersion
			checkToolsContent(c, data, "jujud contents "+expectContent.String())
		}
	}
}