// processGet handles a charm file GET request after authentication.
// It returns the bundle path, the requested file path (if any), whether the
// default charm icon has been requested and an error.
func (h *charmsHandler) processGet(r *http.Request, st *state.State) (
archivePath string,
fileArg string,
serveIcon bool,
err error,
) {
errRet := func(err error) (string, string, bool, error) {
return "", "", false, err
}
query := r.URL.Query()
// Retrieve and validate query parameters.
curlString := query.Get("url")
if curlString == "" {
return errRet(errors.Errorf("expected url=CharmURL query argument"))
}
curl, err := charm.ParseURL(curlString)
if err != nil {
return errRet(errors.Trace(err))
}
fileArg = query.Get("file")
if fileArg != "" {
fileArg = path.Clean(fileArg)
} else if query.Get("icon") == "1" {
serveIcon = true
fileArg = "icon.svg"
}
// Ensure the working directory exists.
tmpDir := filepath.Join(h.dataDir, "charm-get-tmp")
if err = os.MkdirAll(tmpDir, 0755); err != nil {
return errRet(errors.Annotate(err, "cannot create charms tmp directory"))
}
// Use the storage to retrieve and save the charm archive.
storage := storage.NewStorage(st.ModelUUID(), st.MongoSession())
ch, err := st.Charm(curl)
if err != nil {
return errRet(errors.Annotate(err, "cannot get charm from state"))
}
reader, _, err := storage.Get(ch.StoragePath())
if err != nil {
return errRet(errors.Annotate(err, "cannot get charm from model storage"))
}
defer reader.Close()
charmFile, err := ioutil.TempFile(tmpDir, "charm")
if err != nil {
return errRet(errors.Annotate(err, "cannot create charm archive file"))
}
if _, err = io.Copy(charmFile, reader); err != nil {
cleanupFile(charmFile)
return errRet(errors.Annotate(err, "error processing charm archive download"))
}
charmFile.Close()
return charmFile.Name(), fileArg, serveIcon, nil
}
func (s *migrateCharmStorageSuite) testMigrateCharmStorage(c *gc.C, curl *charm.URL, agentConfig agent.Config) {
curlPlaceholder := charm.MustParseURL("cs:quantal/dummy-1")
err := s.State.AddStoreCharmPlaceholder(curlPlaceholder)
c.Assert(err, jc.ErrorIsNil)
curlPending := charm.MustParseURL("cs:quantal/missing-123")
_, err = s.State.PrepareStoreCharmUpload(curlPending)
c.Assert(err, jc.ErrorIsNil)
var storagePath string
var called bool
s.PatchValue(upgrades.StateAddCharmStoragePaths, func(st *state.State, storagePaths map[*charm.URL]string) error {
c.Assert(storagePaths, gc.HasLen, 1)
for k, v := range storagePaths {
c.Assert(k.String(), gc.Equals, curl.String())
storagePath = v
}
called = true
return nil
})
err = upgrades.MigrateCharmStorage(s.State, agentConfig)
c.Assert(err, jc.ErrorIsNil)
c.Assert(called, jc.IsTrue)
storage := storage.NewStorage(s.State.EnvironUUID(), s.State.MongoSession())
r, length, err := storage.Get(storagePath)
c.Assert(err, jc.ErrorIsNil)
c.Assert(r, gc.NotNil)
defer r.Close()
c.Assert(length, gc.Equals, int64(3))
data, err := ioutil.ReadAll(r)
c.Assert(err, jc.ErrorIsNil)
c.Assert(string(data), gc.Equals, "abc")
}
func (s *charmsSuite) TestUploadRepackagesNestedArchives(c *gc.C) {
// Make a clone of the dummy charm in a nested directory.
rootDir := c.MkDir()
dirPath := filepath.Join(rootDir, "subdir1", "subdir2")
err := os.MkdirAll(dirPath, 0755)
c.Assert(err, jc.ErrorIsNil)
dir := testcharms.Repo.ClonedDir(dirPath, "dummy")
// Now tweak the path the dir thinks it is in and bundle it.
dir.Path = rootDir
tempFile, err := ioutil.TempFile(c.MkDir(), "charm")
c.Assert(err, jc.ErrorIsNil)
defer tempFile.Close()
defer os.Remove(tempFile.Name())
err = dir.ArchiveTo(tempFile)
c.Assert(err, jc.ErrorIsNil)
// Try reading it as a bundle - should fail due to nested dirs.
_, err = charm.ReadCharmArchive(tempFile.Name())
c.Assert(err, gc.ErrorMatches, `archive file "metadata.yaml" not found`)
// Now try uploading it - should succeeed and be repackaged.
resp, err := s.uploadRequest(c, s.charmsURI(c, "?series=quantal"), true, tempFile.Name())
c.Assert(err, jc.ErrorIsNil)
expectedURL := charm.MustParseURL("local:quantal/dummy-1")
s.assertUploadResponse(c, resp, expectedURL.String())
sch, err := s.State.Charm(expectedURL)
c.Assert(err, jc.ErrorIsNil)
c.Assert(sch.URL(), gc.DeepEquals, expectedURL)
c.Assert(sch.Revision(), gc.Equals, 1)
c.Assert(sch.IsUploaded(), jc.IsTrue)
// Get it from the storage and try to read it as a bundle - it
// should succeed, because it was repackaged during upload to
// strip nested dirs.
storage := storage.NewStorage(s.State.EnvironUUID(), s.State.MongoSession())
reader, _, err := storage.Get(sch.StoragePath())
c.Assert(err, jc.ErrorIsNil)
defer reader.Close()
data, err := ioutil.ReadAll(reader)
c.Assert(err, jc.ErrorIsNil)
downloadedFile, err := ioutil.TempFile(c.MkDir(), "downloaded")
c.Assert(err, jc.ErrorIsNil)
defer downloadedFile.Close()
defer os.Remove(downloadedFile.Name())
err = ioutil.WriteFile(downloadedFile.Name(), data, 0644)
c.Assert(err, jc.ErrorIsNil)
bundle, err := charm.ReadCharmArchive(downloadedFile.Name())
c.Assert(err, jc.ErrorIsNil)
c.Assert(bundle.Revision(), jc.DeepEquals, sch.Revision())
c.Assert(bundle.Meta(), jc.DeepEquals, sch.Meta())
c.Assert(bundle.Config(), jc.DeepEquals, sch.Config())
}
func (s *RepoSuite) AssertCharmUploaded(c *gc.C, curl *charm.URL) {
ch, err := s.State.Charm(curl)
c.Assert(err, jc.ErrorIsNil)
storage := storage.NewStorage(s.State.EnvironUUID(), s.State.MongoSession())
r, _, err := storage.Get(ch.StoragePath())
c.Assert(err, jc.ErrorIsNil)
defer r.Close()
digest, _, err := utils.ReadSHA256(r)
c.Assert(err, jc.ErrorIsNil)
c.Assert(ch.BundleSha256(), gc.Equals, digest)
}
func (s *charmsSuite) TestUploadRespectsLocalRevision(c *gc.C) {
// Make a dummy charm dir with revision 123.
dir := testcharms.Repo.ClonedDir(c.MkDir(), "dummy")
dir.SetDiskRevision(123)
// Now bundle the dir.
tempFile, err := ioutil.TempFile(c.MkDir(), "charm")
c.Assert(err, jc.ErrorIsNil)
defer tempFile.Close()
defer os.Remove(tempFile.Name())
err = dir.ArchiveTo(tempFile)
c.Assert(err, jc.ErrorIsNil)
// Now try uploading it and ensure the revision persists.
resp, err := s.uploadRequest(c, s.charmsURI(c, "?series=quantal"), true, tempFile.Name())
c.Assert(err, jc.ErrorIsNil)
expectedURL := charm.MustParseURL("local:quantal/dummy-123")
s.assertUploadResponse(c, resp, expectedURL.String())
sch, err := s.State.Charm(expectedURL)
c.Assert(err, jc.ErrorIsNil)
c.Assert(sch.URL(), gc.DeepEquals, expectedURL)
c.Assert(sch.Revision(), gc.Equals, 123)
c.Assert(sch.IsUploaded(), jc.IsTrue)
// First rewind the reader, which was reset but BundleTo() above.
_, err = tempFile.Seek(0, 0)
c.Assert(err, jc.ErrorIsNil)
// Finally, verify the SHA256.
expectedSHA256, _, err := utils.ReadSHA256(tempFile)
c.Assert(err, jc.ErrorIsNil)
c.Assert(sch.BundleSha256(), gc.Equals, expectedSHA256)
storage := storage.NewStorage(s.State.EnvironUUID(), s.State.MongoSession())
reader, _, err := storage.Get(sch.StoragePath())
c.Assert(err, jc.ErrorIsNil)
defer reader.Close()
downloadedSHA256, _, err := utils.ReadSHA256(reader)
c.Assert(err, jc.ErrorIsNil)
c.Assert(downloadedSHA256, gc.Equals, expectedSHA256)
}
// downloadCharm downloads the given charm name from the provider storage and
// saves the corresponding zip archive to the given charmArchivePath.
func (h *charmsHandler) downloadCharm(st *state.State, curl *charm.URL, charmArchivePath string) error {
storage := storage.NewStorage(st.ModelUUID(), st.MongoSession())
ch, err := st.Charm(curl)
if err != nil {
return errors.Annotate(err, "cannot get charm from state")
}
// In order to avoid races, the archive is saved in a temporary file which
// is then atomically renamed. The temporary file is created in the
// charm cache directory so that we can safely assume the rename source and
// target live in the same file system.
cacheDir := filepath.Dir(charmArchivePath)
if err = os.MkdirAll(cacheDir, 0755); err != nil {
return errors.Annotate(err, "cannot create the charms cache")
}
tempCharmArchive, err := ioutil.TempFile(cacheDir, "charm")
if err != nil {
return errors.Annotate(err, "cannot create charm archive temp file")
}
defer tempCharmArchive.Close()
// Use the storage to retrieve and save the charm archive.
reader, _, err := storage.Get(ch.StoragePath())
if err != nil {
defer cleanupFile(tempCharmArchive)
return errors.Annotate(err, "cannot get charm from model storage")
}
defer reader.Close()
if _, err = io.Copy(tempCharmArchive, reader); err != nil {
defer cleanupFile(tempCharmArchive)
return errors.Annotate(err, "error processing charm archive download")
}
tempCharmArchive.Close()
if err = os.Rename(tempCharmArchive.Name(), charmArchivePath); err != nil {
defer cleanupFile(tempCharmArchive)
return errors.Annotate(err, "error renaming the charm archive")
}
return nil
}
func uploadCharms(config UploadBinariesConfig) error {
storage := config.GetStateStorage(config.State)
usedCharms := getUsedCharms(config.Model)
charmUploader := config.GetCharmUploader(config.Target)
for _, charmUrl := range usedCharms.Values() {
logger.Debugf("send charm %s to target", charmUrl)
curl, err := charm.ParseURL(charmUrl)
if err != nil {
return errors.Annotate(err, "bad charm URL")
}
path, err := config.GetCharmStoragePath(config.State, curl)
if err != nil {
return errors.Trace(err)
}
reader, _, err := storage.Get(path)
if err != nil {
return errors.Annotate(err, "cannot get charm from storage")
}
defer reader.Close()
content, cleanup, err := streamThroughTempFile(reader)
if err != nil {
return errors.Trace(err)
}
defer cleanup()
if _, err := charmUploader.UploadCharm(curl, content); err != nil {
return errors.Annotate(err, "cannot upload charm")
}
}
return nil
}