Esempio n. 1
0
func downloadFile(downloadFileDetails *DownloadFileDetails, logMsgPrefix string, flags *DownloadFlags) error {
	httpClientsDetails := utils.GetArtifactoryHttpClientDetails(flags.ArtDetails)
	bulkDownload := flags.SplitCount == 0 || flags.MinSplitSize < 0 || flags.MinSplitSize*1000 > downloadFileDetails.Size
	if !bulkDownload {
		acceptRange, err := isFileAcceptRange(downloadFileDetails, flags)
		if err != nil {
			return err
		}
		bulkDownload = !acceptRange
	}
	if bulkDownload {
		resp, err := ioutils.DownloadFile(downloadFileDetails.DownloadPath, downloadFileDetails.LocalPath, downloadFileDetails.LocalFileName, httpClientsDetails)
		if err != nil {
			return err
		}
		log.Debug(logMsgPrefix, "Artifactory response:", resp.Status)
	} else {
		concurrentDownloadFlags := ioutils.ConcurrentDownloadFlags{
			DownloadPath: downloadFileDetails.DownloadPath,
			FileName:     downloadFileDetails.LocalFileName,
			LocalPath:    downloadFileDetails.LocalPath,
			FileSize:     downloadFileDetails.Size,
			SplitCount:   flags.SplitCount}

		ioutils.DownloadFileConcurrently(concurrentDownloadFlags, logMsgPrefix, httpClientsDetails)
	}
	return nil
}
Esempio n. 2
0
func DownloadLog(packageDetails *utils.VersionDetails, logName string,
	details *config.BintrayDetails) error {
	if details.User == "" {
		details.User = packageDetails.Subject
	}
	path := details.ApiUrl + "packages/" + packageDetails.Subject + "/" +
		packageDetails.Repo + "/" + packageDetails.Package + "/logs/" + logName
	httpClientsDetails := utils.GetBintrayHttpClientDetails(details)
	log.Info("Downloading logs...")
	resp, err := ioutils.DownloadFile(path, "", logName, httpClientsDetails)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return cliutils.CheckError(errors.New("Bintray response: " + resp.Status))
	}
	log.Debug("Bintray response:", resp.Status)
	log.Info("Downloaded log.")
	return nil
}
Esempio n. 3
0
func saveData(xrsyTmpdir, filesPrefix, zipSuffix, logMsgPrefix string, urlsList []string) (err error) {
	dataDir, err := ioutil.TempDir(xrsyTmpdir, filesPrefix)
	if err != nil {
		return err
	}
	defer func() {
		if cerr := os.RemoveAll(dataDir); cerr != nil && err == nil {
			err = cerr
		}
	}()
	for i, url := range urlsList {
		fileName := filesPrefix + strconv.Itoa(i) + ".json"
		log.Info(logMsgPrefix, "Downloading", url)
		ioutils.DownloadFile(url, dataDir, fileName, ioutils.HttpClientDetails{})
	}
	log.Info("Zipping files.")
	err = zipFolderFiles(dataDir, filesPrefix+zipSuffix+".zip")
	if err != nil {
		return err
	}
	log.Info("Done zipping files.")
	return nil
}
Esempio n. 4
0
func DownloadBintrayFile(bintrayDetails *config.BintrayDetails, pathDetails *PathDetails, targetPath string,
	flags *DownloadFlags, logMsgPrefix string) (err error) {

	cleanPath := strings.Replace(pathDetails.Path, "(", "", -1)
	cleanPath = strings.Replace(cleanPath, ")", "", -1)
	downloadPath := path.Join(pathDetails.Subject, pathDetails.Repo, cleanPath)

	fileName, filePath := ioutils.GetFileAndDirFromPath(cleanPath)

	url := bintrayDetails.DownloadServerUrl + downloadPath
	if flags.IncludeUnpublished {
		url += "?include_unpublished=1"
	}
	log.Info(logMsgPrefix+"Downloading", downloadPath)

	httpClientsDetails := GetBintrayHttpClientDetails(bintrayDetails)
	var details *ioutils.FileDetails
	details, err = ioutils.GetRemoteFileDetails(url, httpClientsDetails)
	if err != nil {
		err = cliutils.CheckError(errors.New("Bintray " + err.Error()))
		if err != nil {
			return
		}
	}

	regexpPattern := cliutils.PathToRegExp(pathDetails.Path)
	placeHolderTarget, e := cliutils.ReformatRegexp(regexpPattern, cleanPath, targetPath)
	if e != nil {
		err = e
		return
	}

	localPath, localFileName := ioutils.GetLocalPathAndFile(fileName, filePath, placeHolderTarget, flags.Flat)
	var shouldDownload bool
	shouldDownload, err = shouldDownloadFile(path.Join(localPath, localFileName), details)
	if err != nil {
		return
	}
	if !shouldDownload {
		log.Info(logMsgPrefix, "File already exists locally.")
		return
	}

	// Check if the file should be downloaded concurrently.
	if flags.SplitCount == 0 || flags.MinSplitSize < 0 || flags.MinSplitSize*1000 > details.Size {
		// File should not be downloaded concurrently. Download it as one block.
		resp, err := ioutils.DownloadFile(url, localPath, localFileName, httpClientsDetails)
		if err != nil {
			return err
		}
		log.Debug(logMsgPrefix, "Bintray response:", resp.Status)
	} else {
		// We should attempt to download the file concurrently, but only if it is provided through the DSN.
		// To check if the file is provided through the DSN, we first attempt to download the file
		// with 'follow redirect' disabled.

		var resp *http.Response
		var redirectUrl string
		resp, redirectUrl, err =
			ioutils.DownloadFileNoRedirect(url, localPath, localFileName, httpClientsDetails)
		// There are two options now. Either the file has just been downloaded as one block, or
		// we got a redirect to DSN download URL. In case of the later, we should download the file
		// concurrently from the DSN URL.
		// 'err' is not nil in case 'redirectUrl' was returned.
		if redirectUrl != "" {
			concurrentDownloadFlags := ioutils.ConcurrentDownloadFlags{
				DownloadPath: redirectUrl,
				FileName:     localFileName,
				LocalPath:    localPath,
				FileSize:     details.Size,
				SplitCount:   flags.SplitCount,
				Flat:         flags.Flat}

			ioutils.DownloadFileConcurrently(concurrentDownloadFlags, "", httpClientsDetails)
		} else {
			err = cliutils.CheckError(err)
			if err != nil {
				return
			}
			log.Info(logMsgPrefix, "Bintray response:", resp.Status)
		}
	}
	return
}