func (s *DirSuite) TestDirRevisionFile(c *gc.C) { charmDir := charmtesting.Charms.ClonedDirPath(c.MkDir(), "dummy") revPath := filepath.Join(charmDir, "revision") // Missing revision file err := os.Remove(revPath) c.Assert(err, gc.IsNil) dir, err := charm.ReadDir(charmDir) c.Assert(err, gc.IsNil) c.Assert(dir.Revision(), gc.Equals, 0) // Missing revision file with old revision in metadata file, err := os.OpenFile(filepath.Join(charmDir, "metadata.yaml"), os.O_WRONLY|os.O_APPEND, 0) c.Assert(err, gc.IsNil) _, err = file.Write([]byte("\nrevision: 1234\n")) c.Assert(err, gc.IsNil) dir, err = charm.ReadDir(charmDir) c.Assert(err, gc.IsNil) c.Assert(dir.Revision(), gc.Equals, 1234) // Revision file with bad content err = ioutil.WriteFile(revPath, []byte("garbage"), 0666) c.Assert(err, gc.IsNil) dir, err = charm.ReadDir(charmDir) c.Assert(err, gc.ErrorMatches, "invalid revision file") c.Assert(dir, gc.IsNil) }
func (s *DirSuite) TestDirSetDiskRevision(c *gc.C) { charmDir := charmtesting.Charms.ClonedDirPath(c.MkDir(), "dummy") dir, err := charm.ReadDir(charmDir) c.Assert(err, gc.IsNil) c.Assert(dir.Revision(), gc.Equals, 1) dir.SetDiskRevision(42) c.Assert(dir.Revision(), gc.Equals, 42) dir, err = charm.ReadDir(charmDir) c.Assert(err, gc.IsNil) c.Assert(dir.Revision(), gc.Equals, 42) }
func (s *DirSuite) TestReadDirWithoutConfig(c *gc.C) { path := charmtesting.Charms.DirPath("varnish") dir, err := charm.ReadDir(path) c.Assert(err, gc.IsNil) // A lacking config.yaml file still causes a proper // Config value to be returned. c.Assert(dir.Config().Options, gc.HasLen, 0) }
func (s *DirSuite) TestReadDirWithoutActions(c *gc.C) { path := charmtesting.Charms.DirPath("wordpress") dir, err := charm.ReadDir(path) c.Assert(err, gc.IsNil) // A lacking actions.yaml file still causes a proper // Actions value to be returned. c.Assert(dir.Actions().ActionSpecs, gc.HasLen, 0) }
func bundleDir(c *gc.C, dirpath string) *charm.Bundle { dir, err := charm.ReadDir(dirpath) c.Assert(err, gc.IsNil) buf := new(bytes.Buffer) err = dir.BundleTo(buf) c.Assert(err, gc.IsNil) bundle, err := charm.ReadBundleBytes(buf.Bytes()) c.Assert(err, gc.IsNil) return bundle }
func (s *UpgradeCharmSuccessSuite) TestRespectsLocalRevisionWhenPossible(c *gc.C) { dir, err := charm.ReadDir(s.path) c.Assert(err, gc.IsNil) err = dir.SetDiskRevision(42) c.Assert(err, gc.IsNil) err = runUpgradeCharm(c, "riak") c.Assert(err, gc.IsNil) s.assertUpgraded(c, 42, false) s.assertLocalRevision(c, 42, s.path) }
func (s *BundleSuite) TestExpandTo(c *gc.C) { bundle, err := charm.ReadBundle(s.bundlePath) c.Assert(err, gc.IsNil) path := filepath.Join(c.MkDir(), "charm") err = bundle.ExpandTo(path) c.Assert(err, gc.IsNil) dir, err := charm.ReadDir(path) c.Assert(err, gc.IsNil) checkDummy(c, dir, path) }
// processUploadedArchive opens the given charm archive from path, // inspects it to see if it has all files at the root of the archive // or it has subdirs. It repackages the archive so it has all the // files at the root dir, if necessary, replacing the original archive // at path. func (h *charmsHandler) processUploadedArchive(path string) error { // Open the archive as a zip. f, err := os.OpenFile(path, os.O_RDWR, 0644) if err != nil { return err } defer f.Close() fi, err := f.Stat() if err != nil { return err } zipr, err := zip.NewReader(f, fi.Size()) if err != nil { return errors.Annotate(err, "cannot open charm archive") } // Find out the root dir prefix from the archive. rootDir, err := h.findArchiveRootDir(zipr) if err != nil { return errors.Annotate(err, "cannot read charm archive") } if rootDir == "." { // Normal charm, just use charm.ReadBundle(). return nil } // There is one or more subdirs, so we need extract it to a temp // dir and then read it as a charm dir. tempDir, err := ioutil.TempDir("", "charm-extract") if err != nil { return errors.Annotate(err, "cannot create temp directory") } defer os.RemoveAll(tempDir) if err := ziputil.Extract(zipr, tempDir, rootDir); err != nil { return errors.Annotate(err, "cannot extract charm archive") } dir, err := charm.ReadDir(tempDir) if err != nil { return errors.Annotate(err, "cannot read extracted archive") } // Now repackage the dir as a bundle at the original path. if err := f.Truncate(0); err != nil { return err } if err := dir.BundleTo(f); err != nil { return err } return nil }
func AddCustomCharm(c *gc.C, st *State, name, filename, content, series string, revision int) *Charm { path := charmtesting.Charms.ClonedDirPath(c.MkDir(), name) if filename != "" { config := filepath.Join(path, filename) err := ioutil.WriteFile(config, []byte(content), 0644) c.Assert(err, gc.IsNil) } ch, err := charm.ReadDir(path) c.Assert(err, gc.IsNil) if revision != -1 { ch.SetRevision(revision) } return addCharm(c, st, series, ch) }
func (s *DirSuite) TestBundleToWithBadType(c *gc.C) { charmDir := charmtesting.Charms.ClonedDirPath(c.MkDir(), "dummy") badFile := filepath.Join(charmDir, "hooks", "badfile") // Symlink targeting a path outside of the charm. err := os.Symlink("../../target", badFile) c.Assert(err, gc.IsNil) dir, err := charm.ReadDir(charmDir) c.Assert(err, gc.IsNil) err = dir.BundleTo(&bytes.Buffer{}) c.Assert(err, gc.ErrorMatches, `symlink "hooks/badfile" links out of charm: "../../target"`) // Symlink targeting an absolute path. os.Remove(badFile) err = os.Symlink("/target", badFile) c.Assert(err, gc.IsNil) dir, err = charm.ReadDir(charmDir) c.Assert(err, gc.IsNil) err = dir.BundleTo(&bytes.Buffer{}) c.Assert(err, gc.ErrorMatches, `symlink "hooks/badfile" is absolute: "/target"`) // Can't bundle special files either. os.Remove(badFile) err = syscall.Mkfifo(badFile, 0644) c.Assert(err, gc.IsNil) dir, err = charm.ReadDir(charmDir) c.Assert(err, gc.IsNil) err = dir.BundleTo(&bytes.Buffer{}) c.Assert(err, gc.ErrorMatches, `file is a named pipe: "hooks/badfile"`) }
func (s *DeploySuite) TestUpgradeCharmDir(c *gc.C) { // Add the charm, so the url will exist and a new revision will be // picked in ServiceDeploy. dummyCharm := s.AddTestingCharm(c, "dummy") dirPath := charmtesting.Charms.ClonedDirPath(s.SeriesPath, "dummy") err := runDeploy(c, "local:quantal/dummy") c.Assert(err, gc.IsNil) upgradedRev := dummyCharm.Revision() + 1 curl := dummyCharm.URL().WithRevision(upgradedRev) s.AssertService(c, "dummy", curl, 1, 0) // Check the charm dir was left untouched. ch, err := charm.ReadDir(dirPath) c.Assert(err, gc.IsNil) c.Assert(ch.Revision(), gc.Equals, 1) }
func (s *UpgradeCharmSuccessSuite) TestUpgradesWithBundle(c *gc.C) { dir, err := charm.ReadDir(s.path) c.Assert(err, gc.IsNil) dir.SetRevision(42) buf := &bytes.Buffer{} err = dir.BundleTo(buf) c.Assert(err, gc.IsNil) bundlePath := path.Join(s.SeriesPath, "riak.charm") err = ioutil.WriteFile(bundlePath, buf.Bytes(), 0644) c.Assert(err, gc.IsNil) err = runUpgradeCharm(c, "riak") c.Assert(err, gc.IsNil) s.assertUpgraded(c, 42, false) s.assertLocalRevision(c, 7, s.path) }
func (s *BundleSuite) TestBundleSetRevision(c *gc.C) { bundle, err := charm.ReadBundle(s.bundlePath) c.Assert(err, gc.IsNil) c.Assert(bundle.Revision(), gc.Equals, 1) bundle.SetRevision(42) c.Assert(bundle.Revision(), gc.Equals, 42) path := filepath.Join(c.MkDir(), "charm") err = bundle.ExpandTo(path) c.Assert(err, gc.IsNil) dir, err := charm.ReadDir(path) c.Assert(err, gc.IsNil) c.Assert(dir.Revision(), gc.Equals, 42) }
func (br *bundleReader) AddCustomBundle(c *gc.C, url *corecharm.URL, customize func(path string)) charm.BundleInfo { base := c.MkDir() dirpath := charmtesting.Charms.ClonedDirPath(base, "dummy") if customize != nil { customize(dirpath) } dir, err := corecharm.ReadDir(dirpath) c.Assert(err, gc.IsNil) err = dir.SetDiskRevision(url.Revision) c.Assert(err, gc.IsNil) bunpath := filepath.Join(base, "bundle") file, err := os.Create(bunpath) c.Assert(err, gc.IsNil) defer file.Close() err = dir.BundleTo(file) c.Assert(err, gc.IsNil) bundle, err := corecharm.ReadBundle(bunpath) c.Assert(err, gc.IsNil) return br.AddBundle(c, url, bundle) }
func (s *BundleSuite) TestExpandToSetsHooksExecutable(c *gc.C) { charmDir := charmtesting.Charms.ClonedDir(c.MkDir(), "all-hooks") // Bundle manually, so we can check ExpandTo(), unaffected // by BundleTo()'s behavior bundlePath := filepath.Join(c.MkDir(), "bundle.charm") s.prepareBundle(c, charmDir, bundlePath) bundle, err := charm.ReadBundle(bundlePath) c.Assert(err, gc.IsNil) path := filepath.Join(c.MkDir(), "charm") err = bundle.ExpandTo(path) c.Assert(err, gc.IsNil) _, err = charm.ReadDir(path) c.Assert(err, gc.IsNil) for name := range bundle.Meta().Hooks() { hookName := string(name) info, err := os.Stat(filepath.Join(path, "hooks", hookName)) c.Assert(err, gc.IsNil) perm := info.Mode() & 0777 c.Assert(perm&0100 != 0, gc.Equals, true, gc.Commentf("hook %q is not executable", hookName)) } }
func (s *DirSuite) assertBundleTo(c *gc.C, baseDir, charmDir string) { var haveSymlinks = true if err := os.Symlink("../target", filepath.Join(charmDir, "hooks/symlink")); err != nil { haveSymlinks = false } dir, err := charm.ReadDir(charmDir) c.Assert(err, gc.IsNil) path := filepath.Join(baseDir, "bundle.charm") file, err := os.Create(path) c.Assert(err, gc.IsNil) err = dir.BundleTo(file) file.Close() c.Assert(err, gc.IsNil) zipr, err := zip.OpenReader(path) c.Assert(err, gc.IsNil) defer zipr.Close() var metaf, instf, emptyf, revf, symf *zip.File for _, f := range zipr.File { c.Logf("Bundled file: %s", f.Name) switch f.Name { case "revision": revf = f case "metadata.yaml": metaf = f case "hooks/install": instf = f case "hooks/symlink": symf = f case "empty/": emptyf = f case "build/ignored": c.Errorf("bundle includes build/*: %s", f.Name) case ".ignored", ".dir/ignored": c.Errorf("bundle includes .* entries: %s", f.Name) } } c.Assert(revf, gc.NotNil) reader, err := revf.Open() c.Assert(err, gc.IsNil) data, err := ioutil.ReadAll(reader) reader.Close() c.Assert(err, gc.IsNil) c.Assert(string(data), gc.Equals, "1") c.Assert(metaf, gc.NotNil) reader, err = metaf.Open() c.Assert(err, gc.IsNil) meta, err := charm.ReadMeta(reader) reader.Close() c.Assert(err, gc.IsNil) c.Assert(meta.Name, gc.Equals, "dummy") c.Assert(instf, gc.NotNil) // Despite it being 0751, we pack and unpack it as 0755. c.Assert(instf.Mode()&0777, gc.Equals, os.FileMode(0755)) if haveSymlinks { c.Assert(symf, gc.NotNil) c.Assert(symf.Mode()&0777, gc.Equals, os.FileMode(0777)) reader, err = symf.Open() c.Assert(err, gc.IsNil) data, err = ioutil.ReadAll(reader) reader.Close() c.Assert(err, gc.IsNil) c.Assert(string(data), gc.Equals, "../target") } else { c.Assert(symf, gc.IsNil) } c.Assert(emptyf, gc.NotNil) c.Assert(emptyf.Mode()&os.ModeType, gc.Equals, os.ModeDir) // Despite it being 0750, we pack and unpack it as 0755. c.Assert(emptyf.Mode()&0777, gc.Equals, os.FileMode(0755)) }
func (c *PublishCommand) Run(ctx *cmd.Context) (err error) { branch := bzr.New(ctx.AbsPath(c.CharmPath)) if _, err := os.Stat(branch.Join(".bzr")); err != nil { return fmt.Errorf("not a charm branch: %s", branch.Location()) } if err := branch.CheckClean(); err != nil { return err } var curl *charm.URL if c.URL == "" { if err == nil { loc, err := branch.PushLocation() if err != nil { return fmt.Errorf("no charm URL provided and cannot infer from current directory (no push location)") } curl, err = charm.Store.CharmURL(loc) if err != nil { return fmt.Errorf("cannot infer charm URL from branch location: %q", loc) } } } else { curl, err = charm.InferURL(c.URL, "") if err != nil { return err } } pushLocation := charm.Store.BranchLocation(curl) if c.changePushLocation != nil { pushLocation = c.changePushLocation(pushLocation) } repo, err := charm.InferRepository(curl.Reference, "/not/important") if err != nil { return err } if repo != charm.Store { return fmt.Errorf("charm URL must reference the juju charm store") } localDigest, err := branch.RevisionId() if err != nil { return fmt.Errorf("cannot obtain local digest: %v", err) } logger.Infof("local digest is %s", localDigest) ch, err := charm.ReadDir(branch.Location()) if err != nil { return err } if ch.Meta().Name != curl.Name { return fmt.Errorf("charm name in metadata must match name in URL: %q != %q", ch.Meta().Name, curl.Name) } oldEvent, err := charm.Store.Event(curl, localDigest) if _, ok := err.(*charm.NotFoundError); ok { oldEvent, err = charm.Store.Event(curl, "") if _, ok := err.(*charm.NotFoundError); ok { logger.Infof("charm %s is not yet in the store", curl) err = nil } } if err != nil { return fmt.Errorf("cannot obtain event details from the store: %s", err) } if oldEvent != nil && oldEvent.Digest == localDigest { return handleEvent(ctx, curl, oldEvent) } logger.Infof("sending charm to the charm store...") err = branch.Push(&bzr.PushAttr{Location: pushLocation, Remember: true}) if err != nil { return err } logger.Infof("charm sent; waiting for it to be published...") for { time.Sleep(c.pollDelay) newEvent, err := charm.Store.Event(curl, "") if _, ok := err.(*charm.NotFoundError); ok { continue } if err != nil { return fmt.Errorf("cannot obtain event details from the store: %s", err) } if oldEvent != nil && oldEvent.Digest == newEvent.Digest { continue } if newEvent.Digest != localDigest { // TODO Check if the published digest is in the local history. return fmt.Errorf("charm changed but not to local charm digest; publishing race?") } return handleEvent(ctx, curl, newEvent) } }
// PublishBazaarBranch checks out the Bazaar branch from burl and // publishes its latest revision at urls in the given store. // The digest parameter must be the most recent known Bazaar // revision id for the branch tip. If publishing this specific digest // for these URLs has been attempted already, the publishing // procedure may abort early. The published digest is the Bazaar // revision id of the checked out branch's tip, though, which may // differ from the digest parameter. func PublishBazaarBranch(store *Store, urls []*charm.URL, burl string, digest string) error { // Prevent other publishers from updating these specific URLs // concurrently. lock, err := store.LockUpdates(urls) if err != nil { return err } defer lock.Unlock() var branchDir string NewTip: // Prepare the charm publisher. This will compute the revision // to be assigned to the charm, and it will also fail if the // operation is unnecessary because charms are up-to-date. pub, err := store.CharmPublisher(urls, digest) if err != nil { return err } // Figure if publishing this charm was already attempted before and // failed. We won't try again endlessly if so. In the future we may // retry automatically in certain circumstances. event, err := store.CharmEvent(urls[0], digest) if err == nil && event.Kind != EventPublished { return fmt.Errorf("charm publishing previously failed: %s", strings.Join(event.Errors, "; ")) } else if err != nil && err != ErrNotFound { return err } if branchDir == "" { // Retrieve the branch with a lightweight checkout, so that it // builds a working tree as cheaply as possible. History // doesn't matter here. tempDir, err := ioutil.TempDir("", "publish-branch-") if err != nil { return err } defer os.RemoveAll(tempDir) branchDir = filepath.Join(tempDir, "branch") output, err := exec.Command("bzr", "checkout", "--lightweight", burl, branchDir).CombinedOutput() if err != nil { return outputErr(output, err) } // Pick actual digest from tip. Publishing the real tip // revision rather than the revision for the digest provided is // strictly necessary to prevent a race condition. If the // provided digest was published instead, there's a chance // another publisher concurrently running could have found a // newer revision and published that first, and the digest // parameter provided is in fact an old version that would // overwrite the new version. tipDigest, err := bzrRevisionId(branchDir) if err != nil { return err } if tipDigest != digest { digest = tipDigest goto NewTip } } ch, err := charm.ReadDir(branchDir) if err == nil { // Hand over the charm to the store for bundling and // streaming its content into the database. err = pub.Publish(ch) if err == ErrUpdateConflict { // A conflict may happen in edge cases if the whole // locking mechanism fails due to an expiration event, // and then the expired concurrent publisher revives // for whatever reason and attempts to finish // publishing. The state of the system is still // consistent in that case, and the error isn't logged // since the revision was properly published before. return err } } // Publishing is done. Log failure or error. event = &CharmEvent{ URLs: urls, Digest: digest, } if err == nil { event.Kind = EventPublished event.Revision = pub.Revision() } else { event.Kind = EventPublishError event.Errors = []string{err.Error()} } if logerr := store.LogCharmEvent(event); logerr != nil { if err == nil { err = logerr } else { err = fmt.Errorf("%v; %v", err, logerr) } } return err }
// updateRelations responds to changes in the life states of the relations // with the supplied ids. If any id corresponds to an alive relation not // known to the unit, the uniter will join that relation and return its // relationer in the added list. func (u *Uniter) updateRelations(ids []int) (added []*Relationer, err error) { for _, id := range ids { if r, found := u.relationers[id]; found { rel := r.ru.Relation() if err := rel.Refresh(); err != nil { return nil, fmt.Errorf("cannot update relation %q: %v", rel, err) } if rel.Life() == params.Dying { if err := r.SetDying(); err != nil { return nil, err } else if r.IsImplicit() { delete(u.relationers, id) } } continue } // Relations that are not alive are simply skipped, because they // were not previously known anyway. rel, err := u.st.RelationById(id) if err != nil { if params.IsCodeNotFoundOrCodeUnauthorized(err) { continue } return nil, err } if rel.Life() != params.Alive { continue } // Make sure we ignore relations not implemented by the unit's charm. ch, err := corecharm.ReadDir(u.charmPath) if err != nil { return nil, err } if ep, err := rel.Endpoint(); err != nil { return nil, err } else if !ep.ImplementedBy(ch) { logger.Warningf("skipping relation with unknown endpoint %q", ep.Name) continue } dir, err := relation.ReadStateDir(u.relationsDir, id) if err != nil { return nil, err } err = u.addRelation(rel, dir) if err == nil { added = append(added, u.relationers[id]) continue } e := dir.Remove() if !params.IsCodeCannotEnterScope(err) { return nil, err } if e != nil { return nil, e } } if ok, err := u.unit.IsPrincipal(); err != nil { return nil, err } else if ok { return added, nil } // If no Alive relations remain between a subordinate unit's service // and its principal's service, the subordinate must become Dying. keepAlive := false for _, r := range u.relationers { scope := r.ru.Endpoint().Scope if scope == corecharm.ScopeContainer && !r.dying { keepAlive = true break } } if !keepAlive { if err := u.unit.Destroy(); err != nil { return nil, err } } return added, nil }
// repackageAndUploadCharm expands the given charm archive to a // temporary directoy, repackages it with the given curl's revision, // then uploads it to providr storage, and finally updates the state. func (h *charmsHandler) repackageAndUploadCharm(archive *charm.Bundle, curl *charm.URL) error { // Create a temp dir to contain the extracted charm // dir and the repackaged archive. tempDir, err := ioutil.TempDir("", "charm-download") if err != nil { return errors.Annotate(err, "cannot create temp directory") } defer os.RemoveAll(tempDir) extractPath := filepath.Join(tempDir, "extracted") repackagedPath := filepath.Join(tempDir, "repackaged.zip") repackagedArchive, err := os.Create(repackagedPath) if err != nil { return errors.Annotate(err, "cannot repackage uploaded charm") } defer repackagedArchive.Close() // Expand and repack it with the revision specified by curl. archive.SetRevision(curl.Revision) if err := archive.ExpandTo(extractPath); err != nil { return errors.Annotate(err, "cannot extract uploaded charm") } charmDir, err := charm.ReadDir(extractPath) if err != nil { return errors.Annotate(err, "cannot read extracted charm") } // Bundle the charm and calculate its sha256 hash at the // same time. hash := sha256.New() err = charmDir.BundleTo(io.MultiWriter(hash, repackagedArchive)) if err != nil { return errors.Annotate(err, "cannot repackage uploaded charm") } bundleSHA256 := hex.EncodeToString(hash.Sum(nil)) size, err := repackagedArchive.Seek(0, 2) if err != nil { return errors.Annotate(err, "cannot get charm file size") } // Now upload to provider storage. if _, err := repackagedArchive.Seek(0, 0); err != nil { return errors.Annotate(err, "cannot rewind the charm file reader") } storage, err := environs.GetStorage(h.state) if err != nil { return errors.Annotate(err, "cannot access provider storage") } name := charm.Quote(curl.String()) if err := storage.Put(name, repackagedArchive, size); err != nil { return errors.Annotate(err, "cannot upload charm to provider storage") } storageURL, err := storage.URL(name) if err != nil { return errors.Annotate(err, "cannot get storage URL for charm") } bundleURL, err := url.Parse(storageURL) if err != nil { return errors.Annotate(err, "cannot parse storage URL") } // And finally, update state. _, err = h.state.UpdateUploadedCharm(archive, curl, bundleURL, bundleSHA256) if err != nil { return errors.Annotate(err, "cannot update uploaded charm in state") } return nil }
func (s *DirSuite) TestReadDir(c *gc.C) { path := charmtesting.Charms.DirPath("dummy") dir, err := charm.ReadDir(path) c.Assert(err, gc.IsNil) checkDummy(c, dir, path) }
func (s *UpgradeCharmSuccessSuite) assertLocalRevision(c *gc.C, revision int, path string) { dir, err := charm.ReadDir(path) c.Assert(err, gc.IsNil) c.Assert(dir.Revision(), gc.Equals, revision) }