func downloadAqlResult(dependecies []DownloadData, flags *DownloadFlags) (buildDependencies [][]utils.DependenciesBuildInfo, err error) {
size := len(dependecies)
buildDependencies = make([][]utils.DependenciesBuildInfo, flags.Threads)
var wg sync.WaitGroup
for i := 0; i < flags.Threads; i++ {
wg.Add(1)
go func(threadId int) {
logMsgPrefix := cliutils.GetLogMsgPrefix(threadId, flags.DryRun)
for j := threadId; j < size && err == nil; j += flags.Threads {
downloadPath, e := utils.BuildArtifactoryUrl(flags.ArtDetails.Url, dependecies[j].Dependency.GetFullUrl(), make(map[string]string))
if e != nil {
err = e
break
}
log.Info(logMsgPrefix+"Downloading", dependecies[j].Dependency.GetFullUrl())
if flags.DryRun {
continue
}
regexpPattern := cliutils.PathToRegExp(dependecies[j].DownloadPath)
placeHolderTarget, e := cliutils.ReformatRegexp(regexpPattern, dependecies[j].Dependency.GetFullUrl(), dependecies[j].Target)
if e != nil {
err = e
break
}
localPath, localFileName := ioutils.GetLocalPathAndFile(dependecies[j].Dependency.Name, dependecies[j].Dependency.Path, placeHolderTarget, dependecies[j].Flat)
shouldDownload, e := shouldDownloadFile(path.Join(localPath, dependecies[j].Dependency.Name), dependecies[j].Dependency.Actual_Md5, dependecies[j].Dependency.Actual_Sha1)
if e != nil {
err = e
break
}
dependency := createBuildDependencyItem(dependecies[j].Dependency)
if !shouldDownload {
buildDependencies[threadId] = append(buildDependencies[threadId], dependency)
log.Debug(logMsgPrefix, "File already exists locally.")
continue
}
downloadFileDetails := createDownloadFileDetails(downloadPath, localPath, localFileName, nil, dependecies[j].Dependency.Size)
e = downloadFile(downloadFileDetails, logMsgPrefix, flags)
if e != nil {
err = e
break
}
buildDependencies[threadId] = append(buildDependencies[threadId], dependency)
}
wg.Done()
}(i)
}
wg.Wait()
logDownloadTotals(buildDependencies)
return
}
func moveSimple(fileSpec *Files, flags *MoveFlags, moveType MoveType) (successCount, failedCount int, err error) {
cleanPattern := cliutils.StripChars(fileSpec.Pattern, "()")
patternFileName, _ := ioutils.GetFileAndDirFromPath(fileSpec.Pattern)
regexpPattern := cliutils.PathToRegExp(fileSpec.Pattern)
placeHolderTarget, err := cliutils.ReformatRegexp(regexpPattern, cleanPattern, fileSpec.Target)
if err != nil {
return
}
if strings.HasSuffix(placeHolderTarget, "/") {
placeHolderTarget += patternFileName
}
success, err := moveFile(cleanPattern, placeHolderTarget, flags, moveType)
successCount = cliutils.Bool2Int(success)
failedCount = cliutils.Bool2Int(!success)
return
}
func moveFiles(regexpPath string, resultItems []AqlSearchResultItem, fileSpec *Files, flags *MoveFlags, moveType MoveType) (successCount, failedCount int, err error) {
successCount = 0
failedCount = 0
for _, v := range resultItems {
destPathLocal := fileSpec.Target
isFlat, e := cliutils.StringToBool(fileSpec.Flat, false)
if e != nil {
err = e
return
}
if !isFlat {
if strings.Contains(destPathLocal, "/") {
file, dir := ioutils.GetFileAndDirFromPath(destPathLocal)
destPathLocal = cliutils.TrimPath(dir + "/" + v.Path + "/" + file)
} else {
destPathLocal = cliutils.TrimPath(destPathLocal + "/" + v.Path + "/")
}
}
destFile, e := cliutils.ReformatRegexp(regexpPath, v.GetFullUrl(), destPathLocal)
if e != nil {
err = e
return
}
if strings.HasSuffix(destFile, "/") {
destFile += v.Name
}
success, e := moveFile(v.GetFullUrl(), destFile, flags, moveType)
if e != nil {
err = e
return
}
successCount += cliutils.Bool2Int(success)
failedCount += cliutils.Bool2Int(!success)
}
return
}
func DownloadBintrayFile(bintrayDetails *config.BintrayDetails, pathDetails *PathDetails, targetPath string,
flags *DownloadFlags, logMsgPrefix string) (err error) {
cleanPath := strings.Replace(pathDetails.Path, "(", "", -1)
cleanPath = strings.Replace(cleanPath, ")", "", -1)
downloadPath := path.Join(pathDetails.Subject, pathDetails.Repo, cleanPath)
fileName, filePath := ioutils.GetFileAndDirFromPath(cleanPath)
url := bintrayDetails.DownloadServerUrl + downloadPath
if flags.IncludeUnpublished {
url += "?include_unpublished=1"
}
log.Info(logMsgPrefix+"Downloading", downloadPath)
httpClientsDetails := GetBintrayHttpClientDetails(bintrayDetails)
var details *ioutils.FileDetails
details, err = ioutils.GetRemoteFileDetails(url, httpClientsDetails)
if err != nil {
err = cliutils.CheckError(errors.New("Bintray " + err.Error()))
if err != nil {
return
}
}
regexpPattern := cliutils.PathToRegExp(pathDetails.Path)
placeHolderTarget, e := cliutils.ReformatRegexp(regexpPattern, cleanPath, targetPath)
if e != nil {
err = e
return
}
localPath, localFileName := ioutils.GetLocalPathAndFile(fileName, filePath, placeHolderTarget, flags.Flat)
var shouldDownload bool
shouldDownload, err = shouldDownloadFile(path.Join(localPath, localFileName), details)
if err != nil {
return
}
if !shouldDownload {
log.Info(logMsgPrefix, "File already exists locally.")
return
}
// Check if the file should be downloaded concurrently.
if flags.SplitCount == 0 || flags.MinSplitSize < 0 || flags.MinSplitSize*1000 > details.Size {
// File should not be downloaded concurrently. Download it as one block.
resp, err := ioutils.DownloadFile(url, localPath, localFileName, httpClientsDetails)
if err != nil {
return err
}
log.Debug(logMsgPrefix, "Bintray response:", resp.Status)
} else {
// We should attempt to download the file concurrently, but only if it is provided through the DSN.
// To check if the file is provided through the DSN, we first attempt to download the file
// with 'follow redirect' disabled.
var resp *http.Response
var redirectUrl string
resp, redirectUrl, err =
ioutils.DownloadFileNoRedirect(url, localPath, localFileName, httpClientsDetails)
// There are two options now. Either the file has just been downloaded as one block, or
// we got a redirect to DSN download URL. In case of the later, we should download the file
// concurrently from the DSN URL.
// 'err' is not nil in case 'redirectUrl' was returned.
if redirectUrl != "" {
concurrentDownloadFlags := ioutils.ConcurrentDownloadFlags{
DownloadPath: redirectUrl,
FileName: localFileName,
LocalPath: localPath,
FileSize: details.Size,
SplitCount: flags.SplitCount,
Flat: flags.Flat}
ioutils.DownloadFileConcurrently(concurrentDownloadFlags, "", httpClientsDetails)
} else {
err = cliutils.CheckError(err)
if err != nil {
return
}
log.Info(logMsgPrefix, "Bintray response:", resp.Status)
}
}
return
}