func withFakeHome(callback func(dirPath string)) {
	fileutils.TempDir("test-config", func(dir string, err error) {
		if err != nil {
			Fail("Couldn't create tmp file")
		}
		callback(filepath.Join(dir, ".cf", "config.json"))
	})
}
Example #2
0
func (b *bloblet) Write(w io.Writer) {
	fileutils.TempDir("bloblet-zipdir", func(zipDir string, err error) {
		cliutil.Check(err)
		zipPath := filepath.Join(zipDir, BlobletFileName)
		cliutil.Check(b.Compress(zipPath))
		cliutil.Check(fileutils.CopyPathToWriter(zipPath, w))
	})
}
Example #3
0
func (repo CloudControllerApplicationBitsRepository) UploadApp(appGuid string, appDir string, cb func(path string, zipSize, fileCount uint64)) (apiErr error) {
	fileutils.TempDir("apps", func(uploadDir string, err error) {
		if err != nil {
			apiErr = err
			return
		}

		var presentFiles []resources.AppFileResource
		repo.sourceDir(appDir, func(sourceDir string, sourceErr error) {
			if sourceErr != nil {
				err = sourceErr
				return
			}
			presentFiles, err = repo.copyUploadableFiles(sourceDir, uploadDir)
		})

		if err != nil {
			apiErr = err
			return
		}

		fileutils.TempFile("uploads", func(zipFile *os.File, err error) {
			if err != nil {
				apiErr = err
				return
			}

			zipFileSize := uint64(0)
			zipFileCount := uint64(0)

			err = repo.zipper.Zip(uploadDir, zipFile)
			switch err := err.(type) {
			case nil:
				stat, err := zipFile.Stat()
				if err != nil {
					apiErr = errors.NewWithError("Error zipping application", err)
					return
				}

				zipFileSize = uint64(stat.Size())
				zipFileCount = app_files.CountFiles(uploadDir)
			case *errors.EmptyDirError:
				zipFile = nil
			default:
				apiErr = errors.NewWithError("Error zipping application", err)
				return
			}

			cb(appDir, zipFileSize, zipFileCount)

			apiErr = repo.uploadBits(appGuid, zipFile, presentFiles)
			if apiErr != nil {
				return
			}
		})
	})
	return
}
Example #4
0
func (repo CloudControllerApplicationBitsRepository) sourceDir(appDir string, cb func(sourceDir string, err error)) {
	// If appDir is a zip, first extract it to a temporary directory
	if repo.zipper.IsZipFile(appDir) {
		fileutils.TempDir("unzipped-app", func(tmpDir string, err error) {
			err = repo.extractZip(appDir, tmpDir)
			cb(tmpDir, err)
		})
	} else {
		cb(appDir, nil)
	}
}
Example #5
0
File: push.go Project: matanzit/cli
func (actor PushActorImpl) GatherFiles(appDir string, uploadDir string) (presentFiles []resources.AppFileResource, apiErr error) {
	if actor.zipper.IsZipFile(appDir) {
		fileutils.TempDir("unzipped-app", func(tmpDir string, err error) {
			err = actor.zipper.Unzip(appDir, tmpDir)
			if err != nil {
				presentFiles = nil
				apiErr = err
				return
			}
			presentFiles, apiErr = actor.copyUploadableFiles(tmpDir, uploadDir)
		})
	} else {
		presentFiles, apiErr = actor.copyUploadableFiles(appDir, uploadDir)
	}
	return presentFiles, apiErr
}
Example #6
0
func (actor PushActorImpl) GatherFiles(appDir string, uploadDir string) ([]resources.AppFileResource, bool, error) {
	var processAppDir = func(source string) ([]resources.AppFileResource, bool, error) {
		files, hasFileToUpload, err := actor.copyUploadableFiles(source, uploadDir)
		if err != nil {
			return []resources.AppFileResource{}, false, err
		}

		files, err = actor.PopulateFileMode(source, files)
		if err != nil {
			return []resources.AppFileResource{}, false, err
		}

		return files, hasFileToUpload, nil
	}

	var (
		processAppDirErr error
		presentFiles     []resources.AppFileResource
		hasFileToUpload  bool
	)

	if actor.zipper.IsZipFile(appDir) {
		var handleZipErr error
		fileutils.TempDir("unzipped-app", func(tmpDir string, err error) {
			if err != nil {
				handleZipErr = err
				return
			}

			err = actor.zipper.Unzip(appDir, tmpDir)
			if err != nil {
				handleZipErr = err
				return
			}

			presentFiles, hasFileToUpload, processAppDirErr = processAppDir(tmpDir)
		})

		if handleZipErr != nil {
			return []resources.AppFileResource{}, false, handleZipErr
		}
	} else {
		presentFiles, hasFileToUpload, processAppDirErr = processAppDir(appDir)
	}

	return presentFiles, hasFileToUpload, processAppDirErr
}
Example #7
0
	})

	It("returns an error when zipping fails", func() {
		fileutils.TempFile("zip_test", func(zipFile *os.File, err error) {
			zipper := ApplicationZipper{}
			err = zipper.Zip("/a/bogus/directory", zipFile)
			Expect(err).To(HaveOccurred())
			Expect(err.Error()).To(ContainSubstring("open /a/bogus/directory"))
		})
	})

	It("returns an error when the directory is empty", func() {
		fileutils.TempFile("zip_test", func(zipFile *os.File, err error) {
			fileutils.TempDir("zip_test", func(emptyDir string, err error) {
				zipper := ApplicationZipper{}
				err = zipper.Zip(emptyDir, zipFile)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("is empty"))
			})
		})
	})

	Describe(".Unzip", func() {
		It("extracts the zip file", func() {
			filez := []string{
				"example-app/.cfignore",
				"example-app/app.rb",
				"example-app/config.ru",
				"example-app/Gemfile",
				"example-app/Gemfile.lock",
				"example-app/ignore-me",
				"example-app/manifest.yml",
Example #8
0
				"dir2/child-dir2",
			}))
		})

		// NB: on windows, you can never rely on the size of a directory being zero
		// see: http://msdn.microsoft.com/en-us/library/windows/desktop/aa364946(v=vs.85).aspx
		// and: https://www.pivotaltracker.com/story/show/70470232
		It("always sets the size of directories to zero bytes", func() {
			fileutils.TempDir("something", func(tempdir string, err error) {
				Expect(err).ToNot(HaveOccurred())

				err = os.Mkdir(filepath.Join(tempdir, "nothing"), 0600)
				Expect(err).ToNot(HaveOccurred())

				files, err := AppFilesInDir(tempdir)
				Expect(err).ToNot(HaveOccurred())

				sizes := []int64{}
				for _, file := range files {
					sizes = append(sizes, file.Size)
				}

				Expect(sizes).To(Equal([]int64{0}))
			})
		})
	})

	Describe("CopyFiles", func() {
		It("copies only the files specified", func() {
			copyDir := filepath.Join(fixturePath, "app-copy-test")

			filesToCopy := []models.AppFileFields{
Example #9
0
					runCurlWithInputs(deps, []string{"--output", tempFile.Name(), "/foo"})
					contents, err := ioutil.ReadAll(tempFile)
					Expect(err).ToNot(HaveOccurred())
					Expect(string(contents)).To(Equal("hai"))
				})
			})

			It("saves the body of the response to the given filepath if it doesn't exists", func() {
				fileutils.TempDir("poor-mans-dir", func(tmpDir string, err error) {
					Expect(err).ToNot(HaveOccurred())
					deps.curlRepo.ResponseBody = "hai"

					filePath := filepath.Join(tmpDir, "subdir1", "banana.txt")
					runCurlWithInputs(deps, []string{"--output", filePath, "/foo"})

					file, err := os.Open(filePath)
					Expect(err).ToNot(HaveOccurred())

					contents, err := ioutil.ReadAll(file)
					Expect(err).ToNot(HaveOccurred())
					Expect(string(contents)).To(Equal("hai"))
				})
			})
		})

		It("makes a post request given -X", func() {
			runCurlWithInputs(deps, []string{"-X", "post", "/foo"})

			Expect(deps.curlRepo.Method).To(Equal("post"))
			Expect(deps.ui.Outputs).ToNot(ContainSubstrings([]string{"FAILED"}))
		})
Example #10
0
				"dir2/child-dir2",
			}))
		})

		// NB: on windows, you can never rely on the size of a directory being zero
		// see: http://msdn.microsoft.com/en-us/library/windows/desktop/aa364946(v=vs.85).aspx
		// and: https://www.pivotaltracker.com/story/show/70470232
		It("always sets the size of directories to zero bytes", func() {
			fileutils.TempDir("something", func(tempdir string, err error) {
				Expect(err).ToNot(HaveOccurred())

				err = os.Mkdir(filepath.Join(tempdir, "nothing"), 0600)
				Expect(err).ToNot(HaveOccurred())

				files, err := appFiles.AppFilesInDir(tempdir)
				Expect(err).ToNot(HaveOccurred())

				sizes := []int64{}
				for _, file := range files {
					sizes = append(sizes, file.Size)
				}

				Expect(sizes).To(Equal([]int64{0}))
			})
		})
	})

	Describe("CopyFiles", func() {
		It("copies only the files specified", func() {
			copyDir := filepath.Join(fixturePath, "app-copy-test")

			filesToCopy := []models.AppFileFields{
Example #11
0
					info, err := os.Lstat(filepath.Join(dest, "example-app/ignore-me"))
					Expect(err).NotTo(HaveOccurred())

					expectedFileMode = fmt.Sprintf("%#o", info.Mode())

					return nil
				}

			})

			It("extracts the zip", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					Expect(err).NotTo(HaveOccurred())

					_, _, err = actor.GatherFiles(appDir, tmpDir)
					Expect(err).NotTo(HaveOccurred())
					Expect(zipper.UnzipCallCount()).To(Equal(1))
				})
			})

			It("returns files list with file mode populated", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					actualFiles, _, err := actor.GatherFiles(appDir, tmpDir)
					Expect(err).NotTo(HaveOccurred())

					expectedFiles := []resources.AppFileResource{
						resources.AppFileResource{
							Path: "example-app/ignore-me",
							Mode: expectedFileMode,
						},
Example #12
0
			zippedFile, err := os.Open(fixture)
			Expect(err).NotTo(HaveOccurred())
			Expect(readFile(zipFile)).To(Equal(readFile(zippedFile)))
		})

		It("returns an error when zipping fails", func() {
			zipper := ApplicationZipper{}
			err := zipper.Zip("/a/bogus/directory", zipFile)
			Expect(err).To(HaveOccurred())
			Expect(err.Error()).To(ContainSubstring("open /a/bogus/directory"))
		})

		It("returns an error when the directory is empty", func() {
			fileutils.TempDir("zip_test", func(emptyDir string, err error) {
				zipper := ApplicationZipper{}
				err = zipper.Zip(emptyDir, zipFile)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("is empty"))
			})
		})
	})

	Describe("IsZipFile", func() {
		var (
			inDir, outDir string
			zipper        ApplicationZipper
		)

		AfterEach(func() {
			os.RemoveAll(inDir)
			os.RemoveAll(outDir)
		})
Example #13
0
			appFiles.AppFilesInDirReturns(allFiles, nil)
			appBitsRepo.GetApplicationFilesReturns(presentFiles, nil)
		})

		AfterEach(func() {
		})

		Context("when the input is a zipfile", func() {
			BeforeEach(func() {
				zipper.IsZipFileReturns(true)
			})

			It("extracts the zip", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					files, _, err := actor.GatherFiles(appDir, tmpDir)
					Expect(zipper.UnzipCallCount()).To(Equal(1))
					Expect(err).NotTo(HaveOccurred())
					Expect(files).To(Equal(presentFiles))
				})
			})

		})

		Context("when the input is a directory full of files", func() {
			BeforeEach(func() {
				zipper.IsZipFileReturns(false)
			})

			It("does not try to unzip the directory", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					files, _, err := actor.GatherFiles(appDir, tmpDir)
					Expect(zipper.UnzipCallCount()).To(Equal(0))
Example #14
0
func AppHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("AppHandler entered")
	appDir, err := ioutil.TempDir("", "recomposed-application")
	if err != nil {
		servutil.Fail(w, "creating recomposed application directory failed: %s", err)
		return
	}
	defer os.RemoveAll(appDir)

	r.ParseMultipartForm(100 * 1024 * 1024)
	mpForm := r.MultipartForm

	// Unzip any uploaded portion of the application.
	appHdrs, ok := mpForm.File["application"]
	if ok {
		log.Println("AppHandler unzipping uploaded portion of application")
		application := appHdrs[0]
		zipFile, err := application.Open()
		if err != nil {
			servutil.Fail(w, "failed to open application zip: %s", err)
			return
		}
		cl := application.Header.Get("Content-Length")
		if cl == "" {
			servutil.Fail(w, "Content-Length header not supplied for application zip: %s", err)
			return
		}
		zipLen, err := strconv.ParseInt(cl, 10, 64)
		if err != nil {
			servutil.Fail(w, "Invalid Content-Length %s: %s", cl, err)
			return
		}

		zr, err := zip.NewReader(zipFile, zipLen)

		var blobUploadTime time.Duration = 0
		for _, f := range zr.File {
			destPath := filepath.Join(appDir, f.Name)
			destDir, _ := filepath.Split(destPath)
			err := os.MkdirAll(destDir[:len(destDir)-1], 0755)
			if err != nil {
				servutil.Fail(w, "Failed to create destination directory: %s", err)
				return
			}

			if f.FileInfo().IsDir() {
				continue
			}

			dest, err := os.OpenFile(destPath, os.O_CREATE|os.O_RDWR, 0755)
			if err != nil {
				servutil.Fail(w, "Failed to create destination file: %s", err)
				return
			}

			of, err := f.Open()
			if err != nil {
				servutil.Fail(w, "Failed to open application file: %s", err)
				return
			}

			io.Copy(dest, of)
			of.Close()
			fi, err := dest.Stat()
			if err != nil {
				servutil.Fail(w, "Failed to stat application file: %s", err)
				return
			}
			dest.Close()
			fn := dest.Name()
			if !fi.IsDir() && fi.Size() > 65535 {
				uploadStart := time.Now()
				sha := sha(fn)
				//			log.Printf("AppHandler adding file %s to the blob store: %s\n", f.Name, sha)
				blobstore.Add(sha, fn)
				blobUploadTime += time.Now().Sub(uploadStart)
			}
		}
		log.Printf("AppHandler unzipped uploaded portion of application and spent %d seconds uploading blobs to the blobstore\n", blobUploadTime/time.Second)
	}

	// Process any uploaded bloblets.
	log.Println("AppHandler unzipping uploaded bloblets")
	var blobletUploadTime time.Duration = 0
	for i := 0; true; i++ {
		hdrs, ok := mpForm.File[fmt.Sprintf("bloblet-%d", i)]
		if !ok {
			break
		}
		blobletHeader := hdrs[0]
		cd := blobletHeader.Header.Get("Content-Disposition")
		var (
			j               int
			path, bfn, hash string
		)

		k := strings.Replace(cd, `form-data; name="bloblet-`, "", 1)
		k = strings.Replace(k, `"; path="`, " ", 1)
		k = strings.Replace(k, `"; filename="`, " ", 1)
		k = strings.Replace(k, `"; hash="`, " ", 1)
		k = strings.Replace(k, `"`, "", 1)
		_, err := fmt.Sscanf(k, `%d %s %s %s`, &j, &path, &bfn, &hash)
		if err != nil {
			servutil.Fail(w, "Malformed Content-Disposition %s: %s", cd, err)
			return
		}
		if j != i || bfn != bloblet.BlobletFileName {
			servutil.Fail(w, "Invalid Content-Disposition %d %s: %s", i, cd, err)
			return
		}

		// Read bloblet zip file.
		blobletZipFile, err := blobletHeader.Open()
		if err != nil {
			servutil.Fail(w, "failed to open bloblet zip: %s", err)
			return
		}
		tmpFile, err := ioutil.TempFile("", "bloblet-zip")
		if err != nil {
			servutil.Fail(w, "failed to create bloblet temporary zip file: %s", err)
			return
		}
		_, err = io.Copy(tmpFile, blobletZipFile)
		if err != nil {
			servutil.Fail(w, "failed to copy bloblet zip file: %s", err)
			return
		}
		tmpFilename := tmpFile.Name()
		err = tmpFile.Close()
		if err != nil {
			servutil.Fail(w, "failed to close bloblet temporary zip file: %s", err)
			return
		}
		err = blobletZipFile.Close()
		if err != nil {
			servutil.Fail(w, "failed to close bloblet zip file: %s", err)
			return
		}

		// Store bloblet zip file in blob store.
		uploadStart := time.Now()
		blobstore.Add(hash, tmpFilename)
		blobletUploadTime += time.Now().Sub(uploadStart)

		// Unzip bloblet into the application.
		dest := filepath.Join(appDir, path)
		destDir, _ := filepath.Split(dest)
		err = os.MkdirAll(destDir, 0755)
		if err != nil {
			servutil.Fail(w, "creating destination directory for bloblet failed: %s", err)
			return
		}
		err = unzip(tmpFilename, destDir, w)
		if err != nil {
			return
		}

		err = os.RemoveAll(tmpFilename)
		if err != nil {
			servutil.Fail(w, "failed to delete bloblet temporary zip file: %s", err)
			return
		}

	}
	log.Printf("AppHandler unzipped uploaded bloblets and spent %d seconds uploading them to the blobstore\n", blobletUploadTime/time.Second)

	// Add blobs to the application
	log.Println("AppHandler demarshalling resources")
	res := mpForm.Value["resources"]
	presentFiles := []resources.AppFileResource{}
	err = json.Unmarshal([]byte(res[0]), &presentFiles)
	if err != nil {
		servutil.Fail(w, "demarshalling resources failed: %s", err)
		return
	}
	log.Println("AppHandler downloading & unzipping resources from the blobstore")
	var blobletDownloadTime time.Duration = 0
	for _, pf := range presentFiles {
		dest := filepath.Join(appDir, pf.Path)
		destDir, destFile := filepath.Split(dest)
		err = os.MkdirAll(destDir, 0755)
		if err != nil {
			servutil.Fail(w, "creating destination directory for downloaded file failed: %s", err)
			return
		}
		if destFile == bloblet.BlobletFileName {
			fileutils.TempDir("bloblet-download-dir", func(downloadDir string, err error) {
				if err != nil {
					servutil.Fail(w, "bloblet download directory error: %s", err)
					return
				}
				z := filepath.Join(downloadDir, "bloblet.zip")
				downloadStart := time.Now()
				blobstore.Get(pf.Sha1, z)
				blobletDownloadTime += time.Now().Sub(downloadStart)

				err = unzip(z, destDir, w)
				if err != nil {
					return
				}

			})
		} else {
			downloadStart := time.Now()
			blobstore.Get(pf.Sha1, dest)
			blobletDownloadTime += time.Now().Sub(downloadStart)
		}
	}
	log.Printf("AppHandler downloaded & unzipped resources from the blobstore and spent %d seconds downloading them from the blobstore\n", blobletDownloadTime/time.Second)

	log.Println("AppHandler exiting")
}