Beispiel #1
0
func downloadAqlResult(dependecies []DownloadData, flags *DownloadFlags) (buildDependencies [][]utils.DependenciesBuildInfo, err error) {
	size := len(dependecies)
	buildDependencies = make([][]utils.DependenciesBuildInfo, flags.Threads)
	var wg sync.WaitGroup
	for i := 0; i < flags.Threads; i++ {
		wg.Add(1)
		go func(threadId int) {
			logMsgPrefix := cliutils.GetLogMsgPrefix(threadId, flags.DryRun)
			for j := threadId; j < size && err == nil; j += flags.Threads {
				downloadPath, e := utils.BuildArtifactoryUrl(flags.ArtDetails.Url, dependecies[j].Dependency.GetFullUrl(), make(map[string]string))
				if e != nil {
					err = e
					break
				}
				log.Info(logMsgPrefix+"Downloading", dependecies[j].Dependency.GetFullUrl())
				if flags.DryRun {
					continue
				}

				regexpPattern := cliutils.PathToRegExp(dependecies[j].DownloadPath)
				placeHolderTarget, e := cliutils.ReformatRegexp(regexpPattern, dependecies[j].Dependency.GetFullUrl(), dependecies[j].Target)
				if e != nil {
					err = e
					break
				}
				localPath, localFileName := ioutils.GetLocalPathAndFile(dependecies[j].Dependency.Name, dependecies[j].Dependency.Path, placeHolderTarget, dependecies[j].Flat)
				shouldDownload, e := shouldDownloadFile(path.Join(localPath, dependecies[j].Dependency.Name), dependecies[j].Dependency.Actual_Md5, dependecies[j].Dependency.Actual_Sha1)
				if e != nil {
					err = e
					break
				}
				dependency := createBuildDependencyItem(dependecies[j].Dependency)
				if !shouldDownload {
					buildDependencies[threadId] = append(buildDependencies[threadId], dependency)
					log.Debug(logMsgPrefix, "File already exists locally.")
					continue
				}

				downloadFileDetails := createDownloadFileDetails(downloadPath, localPath, localFileName, nil, dependecies[j].Dependency.Size)
				e = downloadFile(downloadFileDetails, logMsgPrefix, flags)
				if e != nil {
					err = e
					break
				}
				buildDependencies[threadId] = append(buildDependencies[threadId], dependency)
			}
			wg.Done()
		}(i)
	}
	wg.Wait()
	logDownloadTotals(buildDependencies)
	return
}
Beispiel #2
0
func DownloadBintrayFile(bintrayDetails *config.BintrayDetails, pathDetails *PathDetails, targetPath string,
	flags *DownloadFlags, logMsgPrefix string) (err error) {

	cleanPath := strings.Replace(pathDetails.Path, "(", "", -1)
	cleanPath = strings.Replace(cleanPath, ")", "", -1)
	downloadPath := path.Join(pathDetails.Subject, pathDetails.Repo, cleanPath)

	fileName, filePath := ioutils.GetFileAndDirFromPath(cleanPath)

	url := bintrayDetails.DownloadServerUrl + downloadPath
	if flags.IncludeUnpublished {
		url += "?include_unpublished=1"
	}
	log.Info(logMsgPrefix+"Downloading", downloadPath)

	httpClientsDetails := GetBintrayHttpClientDetails(bintrayDetails)
	var details *ioutils.FileDetails
	details, err = ioutils.GetRemoteFileDetails(url, httpClientsDetails)
	if err != nil {
		err = cliutils.CheckError(errors.New("Bintray " + err.Error()))
		if err != nil {
			return
		}
	}

	regexpPattern := cliutils.PathToRegExp(pathDetails.Path)
	placeHolderTarget, e := cliutils.ReformatRegexp(regexpPattern, cleanPath, targetPath)
	if e != nil {
		err = e
		return
	}

	localPath, localFileName := ioutils.GetLocalPathAndFile(fileName, filePath, placeHolderTarget, flags.Flat)
	var shouldDownload bool
	shouldDownload, err = shouldDownloadFile(path.Join(localPath, localFileName), details)
	if err != nil {
		return
	}
	if !shouldDownload {
		log.Info(logMsgPrefix, "File already exists locally.")
		return
	}

	// Check if the file should be downloaded concurrently.
	if flags.SplitCount == 0 || flags.MinSplitSize < 0 || flags.MinSplitSize*1000 > details.Size {
		// File should not be downloaded concurrently. Download it as one block.
		resp, err := ioutils.DownloadFile(url, localPath, localFileName, httpClientsDetails)
		if err != nil {
			return err
		}
		log.Debug(logMsgPrefix, "Bintray response:", resp.Status)
	} else {
		// We should attempt to download the file concurrently, but only if it is provided through the DSN.
		// To check if the file is provided through the DSN, we first attempt to download the file
		// with 'follow redirect' disabled.

		var resp *http.Response
		var redirectUrl string
		resp, redirectUrl, err =
			ioutils.DownloadFileNoRedirect(url, localPath, localFileName, httpClientsDetails)
		// There are two options now. Either the file has just been downloaded as one block, or
		// we got a redirect to DSN download URL. In case of the later, we should download the file
		// concurrently from the DSN URL.
		// 'err' is not nil in case 'redirectUrl' was returned.
		if redirectUrl != "" {
			concurrentDownloadFlags := ioutils.ConcurrentDownloadFlags{
				DownloadPath: redirectUrl,
				FileName:     localFileName,
				LocalPath:    localPath,
				FileSize:     details.Size,
				SplitCount:   flags.SplitCount,
				Flat:         flags.Flat}

			ioutils.DownloadFileConcurrently(concurrentDownloadFlags, "", httpClientsDetails)
		} else {
			err = cliutils.CheckError(err)
			if err != nil {
				return
			}
			log.Info(logMsgPrefix, "Bintray response:", resp.Status)
		}
	}
	return
}