func downloadFile(downloadFileDetails *DownloadFileDetails, logMsgPrefix string, flags *DownloadFlags) error { httpClientsDetails := utils.GetArtifactoryHttpClientDetails(flags.ArtDetails) bulkDownload := flags.SplitCount == 0 || flags.MinSplitSize < 0 || flags.MinSplitSize*1000 > downloadFileDetails.Size if !bulkDownload { acceptRange, err := isFileAcceptRange(downloadFileDetails, flags) if err != nil { return err } bulkDownload = !acceptRange } if bulkDownload { resp, err := ioutils.DownloadFile(downloadFileDetails.DownloadPath, downloadFileDetails.LocalPath, downloadFileDetails.LocalFileName, httpClientsDetails) if err != nil { return err } log.Debug(logMsgPrefix, "Artifactory response:", resp.Status) } else { concurrentDownloadFlags := ioutils.ConcurrentDownloadFlags{ DownloadPath: downloadFileDetails.DownloadPath, FileName: downloadFileDetails.LocalFileName, LocalPath: downloadFileDetails.LocalPath, FileSize: downloadFileDetails.Size, SplitCount: flags.SplitCount} ioutils.DownloadFileConcurrently(concurrentDownloadFlags, logMsgPrefix, httpClientsDetails) } return nil }
func DownloadLog(packageDetails *utils.VersionDetails, logName string, details *config.BintrayDetails) error { if details.User == "" { details.User = packageDetails.Subject } path := details.ApiUrl + "packages/" + packageDetails.Subject + "/" + packageDetails.Repo + "/" + packageDetails.Package + "/logs/" + logName httpClientsDetails := utils.GetBintrayHttpClientDetails(details) log.Info("Downloading logs...") resp, err := ioutils.DownloadFile(path, "", logName, httpClientsDetails) if err != nil { return err } if resp.StatusCode != 200 { return cliutils.CheckError(errors.New("Bintray response: " + resp.Status)) } log.Debug("Bintray response:", resp.Status) log.Info("Downloaded log.") return nil }
func saveData(xrsyTmpdir, filesPrefix, zipSuffix, logMsgPrefix string, urlsList []string) (err error) { dataDir, err := ioutil.TempDir(xrsyTmpdir, filesPrefix) if err != nil { return err } defer func() { if cerr := os.RemoveAll(dataDir); cerr != nil && err == nil { err = cerr } }() for i, url := range urlsList { fileName := filesPrefix + strconv.Itoa(i) + ".json" log.Info(logMsgPrefix, "Downloading", url) ioutils.DownloadFile(url, dataDir, fileName, ioutils.HttpClientDetails{}) } log.Info("Zipping files.") err = zipFolderFiles(dataDir, filesPrefix+zipSuffix+".zip") if err != nil { return err } log.Info("Done zipping files.") return nil }
func DownloadBintrayFile(bintrayDetails *config.BintrayDetails, pathDetails *PathDetails, targetPath string, flags *DownloadFlags, logMsgPrefix string) (err error) { cleanPath := strings.Replace(pathDetails.Path, "(", "", -1) cleanPath = strings.Replace(cleanPath, ")", "", -1) downloadPath := path.Join(pathDetails.Subject, pathDetails.Repo, cleanPath) fileName, filePath := ioutils.GetFileAndDirFromPath(cleanPath) url := bintrayDetails.DownloadServerUrl + downloadPath if flags.IncludeUnpublished { url += "?include_unpublished=1" } log.Info(logMsgPrefix+"Downloading", downloadPath) httpClientsDetails := GetBintrayHttpClientDetails(bintrayDetails) var details *ioutils.FileDetails details, err = ioutils.GetRemoteFileDetails(url, httpClientsDetails) if err != nil { err = cliutils.CheckError(errors.New("Bintray " + err.Error())) if err != nil { return } } regexpPattern := cliutils.PathToRegExp(pathDetails.Path) placeHolderTarget, e := cliutils.ReformatRegexp(regexpPattern, cleanPath, targetPath) if e != nil { err = e return } localPath, localFileName := ioutils.GetLocalPathAndFile(fileName, filePath, placeHolderTarget, flags.Flat) var shouldDownload bool shouldDownload, err = shouldDownloadFile(path.Join(localPath, localFileName), details) if err != nil { return } if !shouldDownload { log.Info(logMsgPrefix, "File already exists locally.") return } // Check if the file should be downloaded concurrently. if flags.SplitCount == 0 || flags.MinSplitSize < 0 || flags.MinSplitSize*1000 > details.Size { // File should not be downloaded concurrently. Download it as one block. resp, err := ioutils.DownloadFile(url, localPath, localFileName, httpClientsDetails) if err != nil { return err } log.Debug(logMsgPrefix, "Bintray response:", resp.Status) } else { // We should attempt to download the file concurrently, but only if it is provided through the DSN. // To check if the file is provided through the DSN, we first attempt to download the file // with 'follow redirect' disabled. var resp *http.Response var redirectUrl string resp, redirectUrl, err = ioutils.DownloadFileNoRedirect(url, localPath, localFileName, httpClientsDetails) // There are two options now. Either the file has just been downloaded as one block, or // we got a redirect to DSN download URL. In case of the later, we should download the file // concurrently from the DSN URL. // 'err' is not nil in case 'redirectUrl' was returned. if redirectUrl != "" { concurrentDownloadFlags := ioutils.ConcurrentDownloadFlags{ DownloadPath: redirectUrl, FileName: localFileName, LocalPath: localPath, FileSize: details.Size, SplitCount: flags.SplitCount, Flat: flags.Flat} ioutils.DownloadFileConcurrently(concurrentDownloadFlags, "", httpClientsDetails) } else { err = cliutils.CheckError(err) if err != nil { return } log.Info(logMsgPrefix, "Bintray response:", resp.Status) } } return }