func (s *toolsSuite) TestUploadFakeSeries(c *gc.C) { // Make some fake tools. localStorage := c.MkDir() vers := version.MustParseBinary("1.9.0-quantal-amd64") versionStrings := []string{vers.String()} expectedTools := ttesting.MakeToolsWithCheckSum(c, localStorage, "releases", versionStrings) // Now try uploading them. toolsFile := tools.StorageName(vers) params := "?binaryVersion=" + vers.String() + "&series=precise,trusty" resp, err := s.uploadRequest(c, s.toolsURI(c, params), true, path.Join(localStorage, toolsFile)) c.Assert(err, gc.IsNil) // Check the response. stor := s.Conn.Environ.Storage() toolsURL, err := stor.URL(tools.StorageName(vers)) c.Assert(err, gc.IsNil) expectedTools[0].URL = toolsURL s.assertUploadResponse(c, resp, expectedTools[0]) // Check the contents. for _, series := range []string{"precise", "quantal", "trusty"} { toolsVersion := vers toolsVersion.Series = series r, err := stor.Get(tools.StorageName(toolsVersion)) c.Assert(err, gc.IsNil) uploadedData, err := ioutil.ReadAll(r) c.Assert(err, gc.IsNil) expectedData, err := ioutil.ReadFile(filepath.Join(localStorage, tools.StorageName(vers))) c.Assert(err, gc.IsNil) c.Assert(uploadedData, gc.DeepEquals, expectedData) } }
// RemoveFakeTools deletes the fake tools from the supplied storage. func RemoveFakeTools(c *gc.C, stor storage.Storage) { c.Logf("removing fake tools") toolsVersion := version.Current name := envtools.StorageName(toolsVersion) err := stor.Remove(name) c.Check(err, gc.IsNil) defaultSeries := coretesting.FakeDefaultSeries if version.Current.Series != defaultSeries { toolsVersion.Series = defaultSeries name := envtools.StorageName(toolsVersion) err := stor.Remove(name) c.Check(err, gc.IsNil) } RemoveFakeToolsMetadata(c, stor) }
func (*metadataHelperSuite) TestMetadataFromTools(c *gc.C) { metadata := tools.MetadataFromTools(nil) c.Assert(metadata, gc.HasLen, 0) toolsList := coretools.List{{ Version: version.MustParseBinary("1.2.3-precise-amd64"), Size: 123, SHA256: "abc", }, { Version: version.MustParseBinary("2.0.1-raring-amd64"), URL: "file:///tmp/releases/juju-2.0.1-raring-amd64.tgz", Size: 456, SHA256: "xyz", }} metadata = tools.MetadataFromTools(toolsList) c.Assert(metadata, gc.HasLen, len(toolsList)) for i, t := range toolsList { md := metadata[i] c.Assert(md.Release, gc.Equals, t.Version.Series) c.Assert(md.Version, gc.Equals, t.Version.Number.String()) c.Assert(md.Arch, gc.Equals, t.Version.Arch) // FullPath is only filled out when reading tools using simplestreams. // It's not needed elsewhere and requires a URL() call. c.Assert(md.FullPath, gc.Equals, "") c.Assert(md.Path, gc.Equals, tools.StorageName(t.Version)[len("tools/"):]) c.Assert(md.FileType, gc.Equals, "tar.gz") c.Assert(md.Size, gc.Equals, t.Size) c.Assert(md.SHA256, gc.Equals, t.SHA256) } }
func (s *bootstrapSuite) TestBootstrapToolsFileURL(c *gc.C) { storageName := tools.StorageName(version.Current) sftpURL, err := s.env.Storage().URL(storageName) c.Assert(err, gc.IsNil) fileURL := fmt.Sprintf("file://%s/%s", s.env.storageDir, storageName) s.testBootstrapToolsURL(c, sftpURL, fileURL) }
// buildToolsTarball bundles a tools tarball and places it in a temp directory in // the expected tools path. func buildToolsTarball(forceVersion *version.Number) (builtTools *BuiltTools, err error) { // TODO(rog) find binaries from $PATH when not using a development // version of juju within a $GOPATH. logger.Debugf("Building tools") // We create the entire archive before asking the environment to // start uploading so that we can be sure we have archived // correctly. f, err := ioutil.TempFile("", "juju-tgz") if err != nil { return nil, err } defer f.Close() defer os.Remove(f.Name()) toolsVersion, sha256Hash, err := envtools.BundleTools(f, forceVersion) if err != nil { return nil, err } fileInfo, err := f.Stat() if err != nil { return nil, fmt.Errorf("cannot stat newly made tools archive: %v", err) } size := fileInfo.Size() logger.Infof("built tools %v (%dkB)", toolsVersion, (size+512)/1024) baseToolsDir, err := ioutil.TempDir("", "juju-tools") if err != nil { return nil, err } // If we exit with an error, clean up the built tools directory. defer func() { if err != nil { os.RemoveAll(baseToolsDir) } }() err = os.MkdirAll(filepath.Join(baseToolsDir, storage.BaseToolsPath, "releases"), 0755) if err != nil { return nil, err } storageName := envtools.StorageName(toolsVersion) err = utils.CopyFile(filepath.Join(baseToolsDir, storageName), f.Name()) if err != nil { return nil, err } return &BuiltTools{ Version: toolsVersion, Dir: baseToolsDir, StorageName: storageName, Size: size, Sha256Hash: sha256Hash, }, nil }
func (s *UpgraderSuite) TestUpgraderRetryAndChanged(c *gc.C) { stor := s.Conn.Environ.Storage() oldTools := envtesting.PrimeTools(c, stor, s.DataDir(), version.MustParseBinary("5.4.3-precise-amd64")) s.PatchValue(&version.Current, oldTools.Version) newTools := envtesting.AssertUploadFakeToolsVersions( c, stor, version.MustParseBinary("5.4.5-precise-amd64"))[0] err := statetesting.SetAgentVersion(s.State, newTools.Version.Number) c.Assert(err, gc.IsNil) retryc := make(chan time.Time) *upgrader.RetryAfter = func() <-chan time.Time { c.Logf("replacement retry after") return retryc } dummy.Poison(s.Conn.Environ.Storage(), envtools.StorageName(newTools.Version), fmt.Errorf("a non-fatal dose")) u := s.makeUpgrader() defer u.Stop() for i := 0; i < 3; i++ { select { case retryc <- time.Now(): case <-time.After(coretesting.LongWait): c.Fatalf("upgrader did not retry (attempt %d)", i) } } // Make it upgrade to some newer tools that can be // downloaded ok; it should stop retrying, download // the newer tools and exit. newerTools := envtesting.AssertUploadFakeToolsVersions( c, s.Conn.Environ.Storage(), version.MustParseBinary("5.4.6-precise-amd64"))[0] err = statetesting.SetAgentVersion(s.State, newerTools.Version.Number) c.Assert(err, gc.IsNil) s.BackingState.StartSync() done := make(chan error) go func() { done <- u.Wait() }() select { case err := <-done: envtesting.CheckUpgraderReadyError(c, err, &upgrader.UpgradeReadyError{ AgentName: s.machine.Tag(), OldTools: oldTools.Version, NewTools: newerTools.Version, DataDir: s.DataDir(), }) case <-time.After(coretesting.LongWait): c.Fatalf("upgrader did not quit after upgrading") } }
func uploadFakeToolsVersion(stor storage.Storage, vers version.Binary) (*coretools.Tools, error) { logger.Infof("uploading FAKE tools %s", vers) tgz, checksum := coretesting.TarGz( coretesting.NewTarFile("jujud", 0777, "jujud contents "+vers.String())) size := int64(len(tgz)) name := envtools.StorageName(vers) if err := stor.Put(name, bytes.NewReader(tgz), size); err != nil { return nil, err } url, err := stor.URL(name) if err != nil { return nil, err } return &coretools.Tools{URL: url, Version: vers, Size: size, SHA256: checksum}, nil }
// cloneToolsForSeries copies the built tools tarball into a tarball for the specified // series and generates corresponding metadata. func cloneToolsForSeries(toolsInfo *BuiltTools, series ...string) error { // Copy the tools to the target storage, recording a Tools struct for each one. var targetTools coretools.List targetTools = append(targetTools, &coretools.Tools{ Version: toolsInfo.Version, Size: toolsInfo.Size, SHA256: toolsInfo.Sha256Hash, }) putTools := func(vers version.Binary) (string, error) { name := envtools.StorageName(vers) src := filepath.Join(toolsInfo.Dir, toolsInfo.StorageName) dest := filepath.Join(toolsInfo.Dir, name) err := utils.CopyFile(dest, src) if err != nil { return "", err } // Append to targetTools the attributes required to write out tools metadata. targetTools = append(targetTools, &coretools.Tools{ Version: vers, Size: toolsInfo.Size, SHA256: toolsInfo.Sha256Hash, }) return name, nil } logger.Debugf("generating tarballs for %v", series) for _, series := range series { _, err := ubuntu.SeriesVersion(series) if err != nil { return err } if series != toolsInfo.Version.Series { fakeVersion := toolsInfo.Version fakeVersion.Series = series if _, err := putTools(fakeVersion); err != nil { return err } } } // The tools have been copied to a temp location from which they will be uploaded, // now write out the matching simplestreams metadata so that SyncTools can find them. metadataStore, err := filestorage.NewFileStorageWriter(toolsInfo.Dir) if err != nil { return err } logger.Debugf("generating tools metadata") return envtools.MergeAndWriteMetadata(metadataStore, targetTools, false) }
// handleUpload uploads the tools data from the reader to env storage as the specified version. func (h *toolsHandler) handleUpload(r io.Reader, toolsVersion version.Binary, fakeSeries ...string) (*tools.Tools, bool, error) { // Set up a local temp directory for the tools tarball. tmpDir, err := ioutil.TempDir("", "juju-upload-tools-") if err != nil { return nil, false, fmt.Errorf("cannot create temp dir: %v", err) } defer os.RemoveAll(tmpDir) toolsFilename := envtools.StorageName(toolsVersion) toolsDir := path.Dir(toolsFilename) fullToolsDir := path.Join(tmpDir, toolsDir) err = os.MkdirAll(fullToolsDir, 0700) if err != nil { return nil, false, fmt.Errorf("cannot create tools dir %s: %v", toolsDir, err) } // Read the tools tarball from the request, calculating the sha256 along the way. fullToolsFilename := path.Join(tmpDir, toolsFilename) toolsFile, err := os.Create(fullToolsFilename) if err != nil { return nil, false, fmt.Errorf("cannot create tools file %s: %v", fullToolsFilename, err) } logger.Debugf("saving uploaded tools to temp file: %s", fullToolsFilename) defer toolsFile.Close() sha256hash := sha256.New() var size int64 if size, err = io.Copy(toolsFile, io.TeeReader(r, sha256hash)); err != nil { return nil, false, fmt.Errorf("error processing file upload: %v", err) } if size == 0 { return nil, false, fmt.Errorf("no tools uploaded") } // TODO(wallyworld): check integrity of tools tarball. // Create a tools record and sync to storage. uploadedTools := &tools.Tools{ Version: toolsVersion, Size: size, SHA256: fmt.Sprintf("%x", sha256hash.Sum(nil)), } logger.Debugf("about to upload tools %+v to storage", uploadedTools) return h.uploadToStorage(uploadedTools, tmpDir, toolsFilename, fakeSeries...) }
// copyOneToolsPackage copies one tool from the source to the target. func copyOneToolsPackage(tool *coretools.Tools, dest storage.Storage) error { toolsName := envtools.StorageName(tool.Version) logger.Infof("copying %v", toolsName) resp, err := utils.GetValidatingHTTPClient().Get(tool.URL) if err != nil { return err } buf := &bytes.Buffer{} srcFile := resp.Body defer srcFile.Close() tool.SHA256, tool.Size, err = utils.ReadSHA256(io.TeeReader(srcFile, buf)) if err != nil { return err } sizeInKB := (tool.Size + 512) / 1024 logger.Infof("downloaded %v (%dkB), uploading", toolsName, sizeInKB) logger.Infof("download %dkB, uploading", sizeInKB) return dest.Put(toolsName, buf, tool.Size) }
func (s *configureSuite) getCloudConfig(c *gc.C, stateServer bool, vers version.Binary) *cloudinit.Config { var mcfg *envcloudinit.MachineConfig if stateServer { mcfg = environs.NewBootstrapMachineConfig("private-key") mcfg.InstanceId = "instance-id" mcfg.Jobs = []params.MachineJob{params.JobManageEnviron, params.JobHostUnits} } else { mcfg = environs.NewMachineConfig("0", "ya", nil, nil, nil, nil) mcfg.Jobs = []params.MachineJob{params.JobHostUnits} } mcfg.Tools = &tools.Tools{ Version: vers, URL: "file:///var/lib/juju/storage/" + envtools.StorageName(vers), } environConfig := testConfig(c, stateServer, vers) err := environs.FinishMachineConfig(mcfg, environConfig, constraints.Value{}) c.Assert(err, gc.IsNil) cloudcfg := cloudinit.New() err = envcloudinit.Configure(mcfg, cloudcfg) c.Assert(err, gc.IsNil) return cloudcfg }
// getMockBuildTools returns a sync.BuildToolsTarballFunc implementation which generates // a fake tools tarball. func (s *UpgradeJujuSuite) getMockBuildTools(c *gc.C) sync.BuildToolsTarballFunc { return func(forceVersion *version.Number) (*sync.BuiltTools, error) { // UploadFakeToolsVersions requires a storage to write to. stor, err := filestorage.NewFileStorageWriter(s.toolsDir) c.Assert(err, gc.IsNil) vers := version.Current if forceVersion != nil { vers.Number = *forceVersion } versions := []version.Binary{vers} uploadedTools, err := envtesting.UploadFakeToolsVersions(stor, versions...) c.Assert(err, gc.IsNil) agentTools := uploadedTools[0] return &sync.BuiltTools{ Dir: s.toolsDir, StorageName: envtools.StorageName(vers), Version: vers, Size: agentTools.Size, Sha256Hash: agentTools.SHA256, }, nil } }
func (t *LiveTests) TestBootstrapWithDefaultSeries(c *gc.C) { if !t.HasProvisioner { c.Skip("HasProvisioner is false; cannot test deployment") } current := version.Current other := current other.Series = "quantal" if current == other { other.Series = "precise" } dummyCfg, err := config.New(config.NoDefaults, dummy.SampleConfig().Merge(coretesting.Attrs{ "state-server": false, "name": "dummy storage", })) dummyenv, err := environs.Prepare(dummyCfg, coretesting.Context(c), configstore.NewMem()) c.Assert(err, gc.IsNil) defer dummyenv.Destroy() t.Destroy(c) attrs := t.TestConfig.Merge(coretesting.Attrs{"default-series": other.Series}) cfg, err := config.New(config.NoDefaults, attrs) c.Assert(err, gc.IsNil) env, err := environs.Prepare(cfg, coretesting.Context(c), t.ConfigStore) c.Assert(err, gc.IsNil) defer environs.Destroy(env, t.ConfigStore) currentName := envtools.StorageName(current) otherName := envtools.StorageName(other) envStorage := env.Storage() dummyStorage := dummyenv.Storage() defer envStorage.Remove(otherName) _, err = sync.Upload(dummyStorage, ¤t.Number) c.Assert(err, gc.IsNil) // This will only work while cross-compiling across releases is safe, // which depends on external elements. Tends to be safe for the last // few releases, but we may have to refactor some day. err = storageCopy(dummyStorage, currentName, envStorage, otherName) c.Assert(err, gc.IsNil) err = bootstrap.Bootstrap(coretesting.Context(c), env, environs.BootstrapParams{}) c.Assert(err, gc.IsNil) conn, err := juju.NewConn(env) c.Assert(err, gc.IsNil) defer conn.Close() // Wait for machine agent to come up on the bootstrap // machine and ensure it deployed the proper series. m0, err := conn.State.Machine("0") c.Assert(err, gc.IsNil) mw0 := newMachineToolWaiter(m0) defer mw0.Stop() waitAgentTools(c, mw0, other) }
// NewManualBootstrapEnviron wraps a LocalStorageEnviron with another which // overrides the Bootstrap method; when Bootstrap is invoked, the specified // host will be manually bootstrapped. // // InitUbuntuUser is expected to have been executed successfully against // the host being bootstrapped. func Bootstrap(args BootstrapArgs) (err error) { if args.Host == "" { return errors.New("host argument is empty") } if args.Environ == nil { return errors.New("environ argument is nil") } if args.DataDir == "" { return errors.New("data-dir argument is empty") } if args.Series == "" { return errors.New("series argument is empty") } if args.HardwareCharacteristics == nil { return errors.New("hardware characteristics argument is empty") } if len(args.PossibleTools) == 0 { return errors.New("possible tools is empty") } provisioned, err := checkProvisioned(args.Host) if err != nil { return fmt.Errorf("failed to check provisioned status: %v", err) } if provisioned { return ErrProvisioned } // Filter tools based on detected series/arch. logger.Infof("Filtering possible tools: %v", args.PossibleTools) possibleTools, err := args.PossibleTools.Match(tools.Filter{ Arch: *args.HardwareCharacteristics.Arch, Series: args.Series, }) if err != nil { return err } // Store the state file. If provisioning fails, we'll remove the file. logger.Infof("Saving bootstrap state file to bootstrap storage") bootstrapStorage := args.Environ.Storage() err = bootstrap.SaveState( bootstrapStorage, &bootstrap.BootstrapState{ StateInstances: []instance.Id{BootstrapInstanceId}, }, ) if err != nil { return err } defer func() { if err != nil { logger.Errorf("bootstrapping failed, removing state file: %v", err) bootstrapStorage.Remove(bootstrap.StateFile) } }() // If the tools are on the machine already, get a file:// scheme tools URL. tools := *possibleTools[0] storageDir := args.Environ.StorageDir() toolsStorageName := envtools.StorageName(tools.Version) if url, _ := bootstrapStorage.URL(toolsStorageName); url == tools.URL { tools.URL = fmt.Sprintf("file://%s/%s", storageDir, toolsStorageName) } // Add the local storage configuration. agentEnv, err := localstorage.StoreConfig(args.Environ) if err != nil { return err } privateKey, err := common.GenerateSystemSSHKey(args.Environ) if err != nil { return err } // Finally, provision the machine agent. mcfg := environs.NewBootstrapMachineConfig(privateKey) mcfg.InstanceId = BootstrapInstanceId mcfg.HardwareCharacteristics = args.HardwareCharacteristics if args.DataDir != "" { mcfg.DataDir = args.DataDir } mcfg.Tools = &tools err = environs.FinishMachineConfig(mcfg, args.Environ.Config(), constraints.Value{}) if err != nil { return err } for k, v := range agentEnv { mcfg.AgentEnvironment[k] = v } return provisionMachineAgent(args.Host, mcfg, args.Context.GetStderr()) }
func (s *StorageSuite) TestStorageName(c *gc.C) { vers := version.MustParseBinary("1.2.3-precise-amd64") path := envtools.StorageName(vers) c.Assert(path, gc.Equals, "tools/releases/juju-1.2.3-precise-amd64.tgz") }
func (s *UpgradeJujuSuite) TestUpgradeJuju(c *gc.C) { s.PatchValue(&sync.BuildToolsTarball, s.getMockBuildTools(c)) oldVersion := version.Current defer func() { version.Current = oldVersion }() for i, test := range upgradeJujuTests { c.Logf("\ntest %d: %s", i, test.about) s.Reset(c) // Set up apparent CLI version and initialize the command. version.Current = version.MustParseBinary(test.currentVersion) com := &UpgradeJujuCommand{} if err := coretesting.InitCommand(com, test.args); err != nil { if test.expectInitErr != "" { c.Check(err, gc.ErrorMatches, test.expectInitErr) } else { c.Check(err, gc.IsNil) } continue } // Set up state and environ, and run the command. toolsDir := c.MkDir() updateAttrs := map[string]interface{}{ "agent-version": test.agentVersion, "tools-metadata-url": "file://" + toolsDir, } err := s.State.UpdateEnvironConfig(updateAttrs, nil, nil) c.Assert(err, gc.IsNil) versions := make([]version.Binary, len(test.tools)) for i, v := range test.tools { versions[i] = version.MustParseBinary(v) } if len(versions) > 0 { envtesting.MustUploadFakeToolsVersions(s.Conn.Environ.Storage(), versions...) stor, err := filestorage.NewFileStorageWriter(toolsDir) c.Assert(err, gc.IsNil) envtesting.MustUploadFakeToolsVersions(stor, versions...) } err = com.Run(coretesting.Context(c)) if test.expectErr != "" { c.Check(err, gc.ErrorMatches, test.expectErr) continue } else if !c.Check(err, gc.IsNil) { continue } // Check expected changes to environ/state. cfg, err := s.State.EnvironConfig() c.Check(err, gc.IsNil) agentVersion, ok := cfg.AgentVersion() c.Check(ok, gc.Equals, true) c.Check(agentVersion, gc.Equals, version.MustParse(test.expectVersion)) for _, uploaded := range test.expectUploaded { // Substitute latest LTS for placeholder in expected series for uploaded tools uploaded = strings.Replace(uploaded, "%LTS%", config.LatestLtsSeries(), 1) vers := version.MustParseBinary(uploaded) r, err := storage.Get(s.Conn.Environ.Storage(), envtools.StorageName(vers)) if !c.Check(err, gc.IsNil) { continue } data, err := ioutil.ReadAll(r) r.Close() c.Check(err, gc.IsNil) expectContent := version.Current expectContent.Number = agentVersion checkToolsContent(c, data, "jujud contents "+expectContent.String()) } } }