func downloadFiles(results []VersionFilesResult, versionDetails *utils.VersionDetails, targetPath string,
flags *utils.DownloadFlags) (totalDownloaded int, err error) {
size := len(results)
downloadedForThread := make([]int, flags.Threads)
var wg sync.WaitGroup
for i := 0; i < flags.Threads; i++ {
wg.Add(1)
go func(threadId int) {
logMsgPrefix := cliutils.GetLogMsgPrefix(threadId, false)
for j := threadId; j < size; j += flags.Threads {
pathDetails := &utils.PathDetails{
Subject: versionDetails.Subject,
Repo: versionDetails.Repo,
Path: results[j].Path}
e := utils.DownloadBintrayFile(flags.BintrayDetails, pathDetails, targetPath,
flags, logMsgPrefix)
if e != nil {
err = e
continue
}
downloadedForThread[threadId]++
}
wg.Done()
}(i)
}
wg.Wait()
for i := range downloadedForThread {
totalDownloaded += downloadedForThread[i]
}
return
}
func uploadWildcard(artifacts []UploadData, flags *UploadFlags) (buildInfoArtifacts [][]utils.ArtifactBuildInfo, totalUploaded, totalFailed int, err error) {
minChecksumDeploySize, e := getMinChecksumDeploySize()
if e != nil {
err = e
return
}
size := len(artifacts)
var wg sync.WaitGroup
// Create an array of integers, to store the total file that were uploaded successfully.
// Each array item is used by a single thread.
uploadCount := make([]int, flags.Threads, flags.Threads)
buildInfoArtifacts = make([][]utils.ArtifactBuildInfo, flags.Threads)
for i := 0; i < flags.Threads; i++ {
wg.Add(1)
go func(threadId int) {
logMsgPrefix := cliutils.GetLogMsgPrefix(threadId, flags.DryRun)
for j := threadId; j < size && err == nil; j += flags.Threads {
var e error
var uploaded bool
var target string
var buildInfoArtifact utils.ArtifactBuildInfo
target, e = utils.BuildArtifactoryUrl(flags.ArtDetails.Url, artifacts[j].Artifact.TargetPath, make(map[string]string))
if e != nil {
err = e
break
}
buildInfoArtifact, uploaded, e = uploadFile(artifacts[j].Artifact.LocalPath, target, artifacts[j].Props, flags, minChecksumDeploySize, logMsgPrefix)
if e != nil {
err = e
break
}
if uploaded {
uploadCount[threadId]++
buildInfoArtifacts[threadId] = append(buildInfoArtifacts[threadId], buildInfoArtifact)
}
}
wg.Done()
}(i)
}
wg.Wait()
if err != nil {
return
}
totalUploaded = 0
for _, i := range uploadCount {
totalUploaded += i
}
log.Info("Uploaded", strconv.Itoa(totalUploaded), "artifacts.")
totalFailed = size - totalUploaded
if totalFailed > 0 {
log.Error("Failed uploading", strconv.Itoa(totalFailed), "artifacts.")
}
return
}
func downloadAqlResult(dependecies []DownloadData, flags *DownloadFlags) (buildDependencies [][]utils.DependenciesBuildInfo, err error) {
size := len(dependecies)
buildDependencies = make([][]utils.DependenciesBuildInfo, flags.Threads)
var wg sync.WaitGroup
for i := 0; i < flags.Threads; i++ {
wg.Add(1)
go func(threadId int) {
logMsgPrefix := cliutils.GetLogMsgPrefix(threadId, flags.DryRun)
for j := threadId; j < size && err == nil; j += flags.Threads {
downloadPath, e := utils.BuildArtifactoryUrl(flags.ArtDetails.Url, dependecies[j].Dependency.GetFullUrl(), make(map[string]string))
if e != nil {
err = e
break
}
log.Info(logMsgPrefix+"Downloading", dependecies[j].Dependency.GetFullUrl())
if flags.DryRun {
continue
}
regexpPattern := cliutils.PathToRegExp(dependecies[j].DownloadPath)
placeHolderTarget, e := cliutils.ReformatRegexp(regexpPattern, dependecies[j].Dependency.GetFullUrl(), dependecies[j].Target)
if e != nil {
err = e
break
}
localPath, localFileName := ioutils.GetLocalPathAndFile(dependecies[j].Dependency.Name, dependecies[j].Dependency.Path, placeHolderTarget, dependecies[j].Flat)
shouldDownload, e := shouldDownloadFile(path.Join(localPath, dependecies[j].Dependency.Name), dependecies[j].Dependency.Actual_Md5, dependecies[j].Dependency.Actual_Sha1)
if e != nil {
err = e
break
}
dependency := createBuildDependencyItem(dependecies[j].Dependency)
if !shouldDownload {
buildDependencies[threadId] = append(buildDependencies[threadId], dependency)
log.Debug(logMsgPrefix, "File already exists locally.")
continue
}
downloadFileDetails := createDownloadFileDetails(downloadPath, localPath, localFileName, nil, dependecies[j].Dependency.Size)
e = downloadFile(downloadFileDetails, logMsgPrefix, flags)
if e != nil {
err = e
break
}
buildDependencies[threadId] = append(buildDependencies[threadId], dependency)
}
wg.Done()
}(i)
}
wg.Wait()
logDownloadTotals(buildDependencies)
return
}
func uploadFiles(artifacts []cliutils.Artifact, baseUrl string, flags *UploadFlags) (totalUploaded,
totalFailed int, err error) {
size := len(artifacts)
var wg sync.WaitGroup
// Create an array of integers, to store the total file that were uploaded successfully.
// Each array item is used by a single thread.
uploadCount := make([]int, flags.Threads, flags.Threads)
matrixParams := getMatrixParams(flags)
for i := 0; i < flags.Threads; i++ {
wg.Add(1)
go func(threadId int) {
logMsgPrefix := cliutils.GetLogMsgPrefix(threadId, flags.DryRun)
for j := threadId; j < size; j += flags.Threads {
if err != nil {
break
}
url := baseUrl + artifacts[j].TargetPath + matrixParams
if !flags.DryRun {
uploaded, e := uploadFile(artifacts[j], url, logMsgPrefix, flags.BintrayDetails)
if e != nil {
err = e
break
}
if uploaded {
uploadCount[threadId]++
}
} else {
log.Info("[Dry Run] Uploading artifact:", artifacts[j].LocalPath)
uploadCount[threadId]++
}
}
wg.Done()
}(i)
}
wg.Wait()
totalUploaded = 0
for _, i := range uploadCount {
totalUploaded += i
}
log.Info("Uploaded", strconv.Itoa(totalUploaded), "artifacts.")
totalFailed = size - totalUploaded
if totalFailed > 0 {
log.Error("Failed uploading", strconv.Itoa(totalFailed), "artifacts.")
}
return
}