func withFakeHome(callback func(dirPath string)) {
fileutils.TempDir("test-config", func(dir string, err error) {
if err != nil {
Fail("Couldn't create tmp file")
}
callback(filepath.Join(dir, ".cf", "config.json"))
})
}
func (b *bloblet) Write(w io.Writer) {
fileutils.TempDir("bloblet-zipdir", func(zipDir string, err error) {
cliutil.Check(err)
zipPath := filepath.Join(zipDir, BlobletFileName)
cliutil.Check(b.Compress(zipPath))
cliutil.Check(fileutils.CopyPathToWriter(zipPath, w))
})
}
func (repo CloudControllerApplicationBitsRepository) UploadApp(appGuid string, appDir string, cb func(path string, zipSize, fileCount uint64)) (apiErr error) {
fileutils.TempDir("apps", func(uploadDir string, err error) {
if err != nil {
apiErr = err
return
}
var presentFiles []resources.AppFileResource
repo.sourceDir(appDir, func(sourceDir string, sourceErr error) {
if sourceErr != nil {
err = sourceErr
return
}
presentFiles, err = repo.copyUploadableFiles(sourceDir, uploadDir)
})
if err != nil {
apiErr = err
return
}
fileutils.TempFile("uploads", func(zipFile *os.File, err error) {
if err != nil {
apiErr = err
return
}
zipFileSize := uint64(0)
zipFileCount := uint64(0)
err = repo.zipper.Zip(uploadDir, zipFile)
switch err := err.(type) {
case nil:
stat, err := zipFile.Stat()
if err != nil {
apiErr = errors.NewWithError("Error zipping application", err)
return
}
zipFileSize = uint64(stat.Size())
zipFileCount = app_files.CountFiles(uploadDir)
case *errors.EmptyDirError:
zipFile = nil
default:
apiErr = errors.NewWithError("Error zipping application", err)
return
}
cb(appDir, zipFileSize, zipFileCount)
apiErr = repo.uploadBits(appGuid, zipFile, presentFiles)
if apiErr != nil {
return
}
})
})
return
}
func (repo CloudControllerApplicationBitsRepository) sourceDir(appDir string, cb func(sourceDir string, err error)) {
// If appDir is a zip, first extract it to a temporary directory
if repo.zipper.IsZipFile(appDir) {
fileutils.TempDir("unzipped-app", func(tmpDir string, err error) {
err = repo.extractZip(appDir, tmpDir)
cb(tmpDir, err)
})
} else {
cb(appDir, nil)
}
}
func (actor PushActorImpl) GatherFiles(appDir string, uploadDir string) (presentFiles []resources.AppFileResource, apiErr error) {
if actor.zipper.IsZipFile(appDir) {
fileutils.TempDir("unzipped-app", func(tmpDir string, err error) {
err = actor.zipper.Unzip(appDir, tmpDir)
if err != nil {
presentFiles = nil
apiErr = err
return
}
presentFiles, apiErr = actor.copyUploadableFiles(tmpDir, uploadDir)
})
} else {
presentFiles, apiErr = actor.copyUploadableFiles(appDir, uploadDir)
}
return presentFiles, apiErr
}
func (actor PushActorImpl) GatherFiles(appDir string, uploadDir string) ([]resources.AppFileResource, bool, error) {
var processAppDir = func(source string) ([]resources.AppFileResource, bool, error) {
files, hasFileToUpload, err := actor.copyUploadableFiles(source, uploadDir)
if err != nil {
return []resources.AppFileResource{}, false, err
}
files, err = actor.PopulateFileMode(source, files)
if err != nil {
return []resources.AppFileResource{}, false, err
}
return files, hasFileToUpload, nil
}
var (
processAppDirErr error
presentFiles []resources.AppFileResource
hasFileToUpload bool
)
if actor.zipper.IsZipFile(appDir) {
var handleZipErr error
fileutils.TempDir("unzipped-app", func(tmpDir string, err error) {
if err != nil {
handleZipErr = err
return
}
err = actor.zipper.Unzip(appDir, tmpDir)
if err != nil {
handleZipErr = err
return
}
presentFiles, hasFileToUpload, processAppDirErr = processAppDir(tmpDir)
})
if handleZipErr != nil {
return []resources.AppFileResource{}, false, handleZipErr
}
} else {
presentFiles, hasFileToUpload, processAppDirErr = processAppDir(appDir)
}
return presentFiles, hasFileToUpload, processAppDirErr
}
})
It("returns an error when zipping fails", func() {
fileutils.TempFile("zip_test", func(zipFile *os.File, err error) {
zipper := ApplicationZipper{}
err = zipper.Zip("/a/bogus/directory", zipFile)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("open /a/bogus/directory"))
})
})
It("returns an error when the directory is empty", func() {
fileutils.TempFile("zip_test", func(zipFile *os.File, err error) {
fileutils.TempDir("zip_test", func(emptyDir string, err error) {
zipper := ApplicationZipper{}
err = zipper.Zip(emptyDir, zipFile)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("is empty"))
})
})
})
Describe(".Unzip", func() {
It("extracts the zip file", func() {
filez := []string{
"example-app/.cfignore",
"example-app/app.rb",
"example-app/config.ru",
"example-app/Gemfile",
"example-app/Gemfile.lock",
"example-app/ignore-me",
"example-app/manifest.yml",
"dir2/child-dir2",
}))
})
// NB: on windows, you can never rely on the size of a directory being zero
// see: http://msdn.microsoft.com/en-us/library/windows/desktop/aa364946(v=vs.85).aspx
// and: https://www.pivotaltracker.com/story/show/70470232
It("always sets the size of directories to zero bytes", func() {
fileutils.TempDir("something", func(tempdir string, err error) {
Expect(err).ToNot(HaveOccurred())
err = os.Mkdir(filepath.Join(tempdir, "nothing"), 0600)
Expect(err).ToNot(HaveOccurred())
files, err := AppFilesInDir(tempdir)
Expect(err).ToNot(HaveOccurred())
sizes := []int64{}
for _, file := range files {
sizes = append(sizes, file.Size)
}
Expect(sizes).To(Equal([]int64{0}))
})
})
})
Describe("CopyFiles", func() {
It("copies only the files specified", func() {
copyDir := filepath.Join(fixturePath, "app-copy-test")
filesToCopy := []models.AppFileFields{
runCurlWithInputs(deps, []string{"--output", tempFile.Name(), "/foo"})
contents, err := ioutil.ReadAll(tempFile)
Expect(err).ToNot(HaveOccurred())
Expect(string(contents)).To(Equal("hai"))
})
})
It("saves the body of the response to the given filepath if it doesn't exists", func() {
fileutils.TempDir("poor-mans-dir", func(tmpDir string, err error) {
Expect(err).ToNot(HaveOccurred())
deps.curlRepo.ResponseBody = "hai"
filePath := filepath.Join(tmpDir, "subdir1", "banana.txt")
runCurlWithInputs(deps, []string{"--output", filePath, "/foo"})
file, err := os.Open(filePath)
Expect(err).ToNot(HaveOccurred())
contents, err := ioutil.ReadAll(file)
Expect(err).ToNot(HaveOccurred())
Expect(string(contents)).To(Equal("hai"))
})
})
})
It("makes a post request given -X", func() {
runCurlWithInputs(deps, []string{"-X", "post", "/foo"})
Expect(deps.curlRepo.Method).To(Equal("post"))
Expect(deps.ui.Outputs).ToNot(ContainSubstrings([]string{"FAILED"}))
})
"dir2/child-dir2",
}))
})
// NB: on windows, you can never rely on the size of a directory being zero
// see: http://msdn.microsoft.com/en-us/library/windows/desktop/aa364946(v=vs.85).aspx
// and: https://www.pivotaltracker.com/story/show/70470232
It("always sets the size of directories to zero bytes", func() {
fileutils.TempDir("something", func(tempdir string, err error) {
Expect(err).ToNot(HaveOccurred())
err = os.Mkdir(filepath.Join(tempdir, "nothing"), 0600)
Expect(err).ToNot(HaveOccurred())
files, err := appFiles.AppFilesInDir(tempdir)
Expect(err).ToNot(HaveOccurred())
sizes := []int64{}
for _, file := range files {
sizes = append(sizes, file.Size)
}
Expect(sizes).To(Equal([]int64{0}))
})
})
})
Describe("CopyFiles", func() {
It("copies only the files specified", func() {
copyDir := filepath.Join(fixturePath, "app-copy-test")
filesToCopy := []models.AppFileFields{
info, err := os.Lstat(filepath.Join(dest, "example-app/ignore-me"))
Expect(err).NotTo(HaveOccurred())
expectedFileMode = fmt.Sprintf("%#o", info.Mode())
return nil
}
})
It("extracts the zip", func() {
fileutils.TempDir("gather-files", func(tmpDir string, err error) {
Expect(err).NotTo(HaveOccurred())
_, _, err = actor.GatherFiles(appDir, tmpDir)
Expect(err).NotTo(HaveOccurred())
Expect(zipper.UnzipCallCount()).To(Equal(1))
})
})
It("returns files list with file mode populated", func() {
fileutils.TempDir("gather-files", func(tmpDir string, err error) {
actualFiles, _, err := actor.GatherFiles(appDir, tmpDir)
Expect(err).NotTo(HaveOccurred())
expectedFiles := []resources.AppFileResource{
resources.AppFileResource{
Path: "example-app/ignore-me",
Mode: expectedFileMode,
},
zippedFile, err := os.Open(fixture)
Expect(err).NotTo(HaveOccurred())
Expect(readFile(zipFile)).To(Equal(readFile(zippedFile)))
})
It("returns an error when zipping fails", func() {
zipper := ApplicationZipper{}
err := zipper.Zip("/a/bogus/directory", zipFile)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("open /a/bogus/directory"))
})
It("returns an error when the directory is empty", func() {
fileutils.TempDir("zip_test", func(emptyDir string, err error) {
zipper := ApplicationZipper{}
err = zipper.Zip(emptyDir, zipFile)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("is empty"))
})
})
})
Describe("IsZipFile", func() {
var (
inDir, outDir string
zipper ApplicationZipper
)
AfterEach(func() {
os.RemoveAll(inDir)
os.RemoveAll(outDir)
})
appFiles.AppFilesInDirReturns(allFiles, nil)
appBitsRepo.GetApplicationFilesReturns(presentFiles, nil)
})
AfterEach(func() {
})
Context("when the input is a zipfile", func() {
BeforeEach(func() {
zipper.IsZipFileReturns(true)
})
It("extracts the zip", func() {
fileutils.TempDir("gather-files", func(tmpDir string, err error) {
files, _, err := actor.GatherFiles(appDir, tmpDir)
Expect(zipper.UnzipCallCount()).To(Equal(1))
Expect(err).NotTo(HaveOccurred())
Expect(files).To(Equal(presentFiles))
})
})
})
Context("when the input is a directory full of files", func() {
BeforeEach(func() {
zipper.IsZipFileReturns(false)
})
It("does not try to unzip the directory", func() {
fileutils.TempDir("gather-files", func(tmpDir string, err error) {
files, _, err := actor.GatherFiles(appDir, tmpDir)
Expect(zipper.UnzipCallCount()).To(Equal(0))
func AppHandler(w http.ResponseWriter, r *http.Request) {
log.Println("AppHandler entered")
appDir, err := ioutil.TempDir("", "recomposed-application")
if err != nil {
servutil.Fail(w, "creating recomposed application directory failed: %s", err)
return
}
defer os.RemoveAll(appDir)
r.ParseMultipartForm(100 * 1024 * 1024)
mpForm := r.MultipartForm
// Unzip any uploaded portion of the application.
appHdrs, ok := mpForm.File["application"]
if ok {
log.Println("AppHandler unzipping uploaded portion of application")
application := appHdrs[0]
zipFile, err := application.Open()
if err != nil {
servutil.Fail(w, "failed to open application zip: %s", err)
return
}
cl := application.Header.Get("Content-Length")
if cl == "" {
servutil.Fail(w, "Content-Length header not supplied for application zip: %s", err)
return
}
zipLen, err := strconv.ParseInt(cl, 10, 64)
if err != nil {
servutil.Fail(w, "Invalid Content-Length %s: %s", cl, err)
return
}
zr, err := zip.NewReader(zipFile, zipLen)
var blobUploadTime time.Duration = 0
for _, f := range zr.File {
destPath := filepath.Join(appDir, f.Name)
destDir, _ := filepath.Split(destPath)
err := os.MkdirAll(destDir[:len(destDir)-1], 0755)
if err != nil {
servutil.Fail(w, "Failed to create destination directory: %s", err)
return
}
if f.FileInfo().IsDir() {
continue
}
dest, err := os.OpenFile(destPath, os.O_CREATE|os.O_RDWR, 0755)
if err != nil {
servutil.Fail(w, "Failed to create destination file: %s", err)
return
}
of, err := f.Open()
if err != nil {
servutil.Fail(w, "Failed to open application file: %s", err)
return
}
io.Copy(dest, of)
of.Close()
fi, err := dest.Stat()
if err != nil {
servutil.Fail(w, "Failed to stat application file: %s", err)
return
}
dest.Close()
fn := dest.Name()
if !fi.IsDir() && fi.Size() > 65535 {
uploadStart := time.Now()
sha := sha(fn)
// log.Printf("AppHandler adding file %s to the blob store: %s\n", f.Name, sha)
blobstore.Add(sha, fn)
blobUploadTime += time.Now().Sub(uploadStart)
}
}
log.Printf("AppHandler unzipped uploaded portion of application and spent %d seconds uploading blobs to the blobstore\n", blobUploadTime/time.Second)
}
// Process any uploaded bloblets.
log.Println("AppHandler unzipping uploaded bloblets")
var blobletUploadTime time.Duration = 0
for i := 0; true; i++ {
hdrs, ok := mpForm.File[fmt.Sprintf("bloblet-%d", i)]
if !ok {
break
}
blobletHeader := hdrs[0]
cd := blobletHeader.Header.Get("Content-Disposition")
var (
j int
path, bfn, hash string
)
k := strings.Replace(cd, `form-data; name="bloblet-`, "", 1)
k = strings.Replace(k, `"; path="`, " ", 1)
k = strings.Replace(k, `"; filename="`, " ", 1)
k = strings.Replace(k, `"; hash="`, " ", 1)
k = strings.Replace(k, `"`, "", 1)
_, err := fmt.Sscanf(k, `%d %s %s %s`, &j, &path, &bfn, &hash)
if err != nil {
servutil.Fail(w, "Malformed Content-Disposition %s: %s", cd, err)
return
}
if j != i || bfn != bloblet.BlobletFileName {
servutil.Fail(w, "Invalid Content-Disposition %d %s: %s", i, cd, err)
return
}
// Read bloblet zip file.
blobletZipFile, err := blobletHeader.Open()
if err != nil {
servutil.Fail(w, "failed to open bloblet zip: %s", err)
return
}
tmpFile, err := ioutil.TempFile("", "bloblet-zip")
if err != nil {
servutil.Fail(w, "failed to create bloblet temporary zip file: %s", err)
return
}
_, err = io.Copy(tmpFile, blobletZipFile)
if err != nil {
servutil.Fail(w, "failed to copy bloblet zip file: %s", err)
return
}
tmpFilename := tmpFile.Name()
err = tmpFile.Close()
if err != nil {
servutil.Fail(w, "failed to close bloblet temporary zip file: %s", err)
return
}
err = blobletZipFile.Close()
if err != nil {
servutil.Fail(w, "failed to close bloblet zip file: %s", err)
return
}
// Store bloblet zip file in blob store.
uploadStart := time.Now()
blobstore.Add(hash, tmpFilename)
blobletUploadTime += time.Now().Sub(uploadStart)
// Unzip bloblet into the application.
dest := filepath.Join(appDir, path)
destDir, _ := filepath.Split(dest)
err = os.MkdirAll(destDir, 0755)
if err != nil {
servutil.Fail(w, "creating destination directory for bloblet failed: %s", err)
return
}
err = unzip(tmpFilename, destDir, w)
if err != nil {
return
}
err = os.RemoveAll(tmpFilename)
if err != nil {
servutil.Fail(w, "failed to delete bloblet temporary zip file: %s", err)
return
}
}
log.Printf("AppHandler unzipped uploaded bloblets and spent %d seconds uploading them to the blobstore\n", blobletUploadTime/time.Second)
// Add blobs to the application
log.Println("AppHandler demarshalling resources")
res := mpForm.Value["resources"]
presentFiles := []resources.AppFileResource{}
err = json.Unmarshal([]byte(res[0]), &presentFiles)
if err != nil {
servutil.Fail(w, "demarshalling resources failed: %s", err)
return
}
log.Println("AppHandler downloading & unzipping resources from the blobstore")
var blobletDownloadTime time.Duration = 0
for _, pf := range presentFiles {
dest := filepath.Join(appDir, pf.Path)
destDir, destFile := filepath.Split(dest)
err = os.MkdirAll(destDir, 0755)
if err != nil {
servutil.Fail(w, "creating destination directory for downloaded file failed: %s", err)
return
}
if destFile == bloblet.BlobletFileName {
fileutils.TempDir("bloblet-download-dir", func(downloadDir string, err error) {
if err != nil {
servutil.Fail(w, "bloblet download directory error: %s", err)
return
}
z := filepath.Join(downloadDir, "bloblet.zip")
downloadStart := time.Now()
blobstore.Get(pf.Sha1, z)
blobletDownloadTime += time.Now().Sub(downloadStart)
err = unzip(z, destDir, w)
if err != nil {
return
}
})
} else {
downloadStart := time.Now()
blobstore.Get(pf.Sha1, dest)
blobletDownloadTime += time.Now().Sub(downloadStart)
}
}
log.Printf("AppHandler downloaded & unzipped resources from the blobstore and spent %d seconds downloading them from the blobstore\n", blobletDownloadTime/time.Second)
log.Println("AppHandler exiting")
}
