func downloadFiles(results []VersionFilesResult, versionDetails *utils.VersionDetails, targetPath string,
	flags *utils.DownloadFlags) (totalDownloaded int, err error) {

	size := len(results)
	downloadedForThread := make([]int, flags.Threads)
	var wg sync.WaitGroup
	for i := 0; i < flags.Threads; i++ {
		wg.Add(1)
		go func(threadId int) {
			logMsgPrefix := cliutils.GetLogMsgPrefix(threadId, false)
			for j := threadId; j < size; j += flags.Threads {
				pathDetails := &utils.PathDetails{
					Subject: versionDetails.Subject,
					Repo:    versionDetails.Repo,
					Path:    results[j].Path}

				e := utils.DownloadBintrayFile(flags.BintrayDetails, pathDetails, targetPath,
					flags, logMsgPrefix)
				if e != nil {
					err = e
					continue
				}
				downloadedForThread[threadId]++
			}
			wg.Done()
		}(i)
	}
	wg.Wait()

	for i := range downloadedForThread {
		totalDownloaded += downloadedForThread[i]
	}
	return
}
Exemple #2
0
func uploadWildcard(artifacts []UploadData, flags *UploadFlags) (buildInfoArtifacts [][]utils.ArtifactBuildInfo, totalUploaded, totalFailed int, err error) {
	minChecksumDeploySize, e := getMinChecksumDeploySize()
	if e != nil {
		err = e
		return
	}

	size := len(artifacts)
	var wg sync.WaitGroup

	// Create an array of integers, to store the total file that were uploaded successfully.
	// Each array item is used by a single thread.
	uploadCount := make([]int, flags.Threads, flags.Threads)
	buildInfoArtifacts = make([][]utils.ArtifactBuildInfo, flags.Threads)
	for i := 0; i < flags.Threads; i++ {
		wg.Add(1)
		go func(threadId int) {
			logMsgPrefix := cliutils.GetLogMsgPrefix(threadId, flags.DryRun)
			for j := threadId; j < size && err == nil; j += flags.Threads {
				var e error
				var uploaded bool
				var target string
				var buildInfoArtifact utils.ArtifactBuildInfo
				target, e = utils.BuildArtifactoryUrl(flags.ArtDetails.Url, artifacts[j].Artifact.TargetPath, make(map[string]string))
				if e != nil {
					err = e
					break
				}
				buildInfoArtifact, uploaded, e = uploadFile(artifacts[j].Artifact.LocalPath, target, artifacts[j].Props, flags, minChecksumDeploySize, logMsgPrefix)
				if e != nil {
					err = e
					break
				}
				if uploaded {
					uploadCount[threadId]++
					buildInfoArtifacts[threadId] = append(buildInfoArtifacts[threadId], buildInfoArtifact)
				}
			}
			wg.Done()
		}(i)
	}
	wg.Wait()
	if err != nil {
		return
	}
	totalUploaded = 0
	for _, i := range uploadCount {
		totalUploaded += i
	}

	log.Info("Uploaded", strconv.Itoa(totalUploaded), "artifacts.")
	totalFailed = size - totalUploaded
	if totalFailed > 0 {
		log.Error("Failed uploading", strconv.Itoa(totalFailed), "artifacts.")
	}
	return
}
Exemple #3
0
func downloadAqlResult(dependecies []DownloadData, flags *DownloadFlags) (buildDependencies [][]utils.DependenciesBuildInfo, err error) {
	size := len(dependecies)
	buildDependencies = make([][]utils.DependenciesBuildInfo, flags.Threads)
	var wg sync.WaitGroup
	for i := 0; i < flags.Threads; i++ {
		wg.Add(1)
		go func(threadId int) {
			logMsgPrefix := cliutils.GetLogMsgPrefix(threadId, flags.DryRun)
			for j := threadId; j < size && err == nil; j += flags.Threads {
				downloadPath, e := utils.BuildArtifactoryUrl(flags.ArtDetails.Url, dependecies[j].Dependency.GetFullUrl(), make(map[string]string))
				if e != nil {
					err = e
					break
				}
				log.Info(logMsgPrefix+"Downloading", dependecies[j].Dependency.GetFullUrl())
				if flags.DryRun {
					continue
				}

				regexpPattern := cliutils.PathToRegExp(dependecies[j].DownloadPath)
				placeHolderTarget, e := cliutils.ReformatRegexp(regexpPattern, dependecies[j].Dependency.GetFullUrl(), dependecies[j].Target)
				if e != nil {
					err = e
					break
				}
				localPath, localFileName := ioutils.GetLocalPathAndFile(dependecies[j].Dependency.Name, dependecies[j].Dependency.Path, placeHolderTarget, dependecies[j].Flat)
				shouldDownload, e := shouldDownloadFile(path.Join(localPath, dependecies[j].Dependency.Name), dependecies[j].Dependency.Actual_Md5, dependecies[j].Dependency.Actual_Sha1)
				if e != nil {
					err = e
					break
				}
				dependency := createBuildDependencyItem(dependecies[j].Dependency)
				if !shouldDownload {
					buildDependencies[threadId] = append(buildDependencies[threadId], dependency)
					log.Debug(logMsgPrefix, "File already exists locally.")
					continue
				}

				downloadFileDetails := createDownloadFileDetails(downloadPath, localPath, localFileName, nil, dependecies[j].Dependency.Size)
				e = downloadFile(downloadFileDetails, logMsgPrefix, flags)
				if e != nil {
					err = e
					break
				}
				buildDependencies[threadId] = append(buildDependencies[threadId], dependency)
			}
			wg.Done()
		}(i)
	}
	wg.Wait()
	logDownloadTotals(buildDependencies)
	return
}
Exemple #4
0
func uploadFiles(artifacts []cliutils.Artifact, baseUrl string, flags *UploadFlags) (totalUploaded,
	totalFailed int, err error) {

	size := len(artifacts)
	var wg sync.WaitGroup

	// Create an array of integers, to store the total file that were uploaded successfully.
	// Each array item is used by a single thread.
	uploadCount := make([]int, flags.Threads, flags.Threads)
	matrixParams := getMatrixParams(flags)
	for i := 0; i < flags.Threads; i++ {
		wg.Add(1)
		go func(threadId int) {
			logMsgPrefix := cliutils.GetLogMsgPrefix(threadId, flags.DryRun)
			for j := threadId; j < size; j += flags.Threads {
				if err != nil {
					break
				}
				url := baseUrl + artifacts[j].TargetPath + matrixParams
				if !flags.DryRun {
					uploaded, e := uploadFile(artifacts[j], url, logMsgPrefix, flags.BintrayDetails)
					if e != nil {
						err = e
						break
					}
					if uploaded {
						uploadCount[threadId]++
					}
				} else {
					log.Info("[Dry Run] Uploading artifact:", artifacts[j].LocalPath)
					uploadCount[threadId]++
				}
			}
			wg.Done()
		}(i)
	}
	wg.Wait()

	totalUploaded = 0
	for _, i := range uploadCount {
		totalUploaded += i
	}
	log.Info("Uploaded", strconv.Itoa(totalUploaded), "artifacts.")
	totalFailed = size - totalUploaded
	if totalFailed > 0 {
		log.Error("Failed uploading", strconv.Itoa(totalFailed), "artifacts.")
	}
	return
}