func fetchSource(ac, rc *APIClient, rootPath, taskId string, noPatch bool) error {
task, err := rc.GetTask(taskId)
if err != nil {
return err
}
if task == nil {
return fmt.Errorf("task not found.")
}
config, err := rc.GetConfig(task.Version)
if err != nil {
return err
}
project, err := ac.GetProjectRef(task.Project)
if err != nil {
return err
}
cloneDir := util.CleanForPath(fmt.Sprintf("source-%v", task.Project))
var patch *service.RestPatch
if task.Requester == evergreen.PatchVersionRequester {
cloneDir = util.CleanForPath(fmt.Sprintf("source-patch-%v_%v", task.PatchNumber, task.Project))
patch, err = rc.GetPatch(task.PatchId)
if err != nil {
return err
}
} else {
if len(task.Revision) >= 5 {
cloneDir = util.CleanForPath(fmt.Sprintf("source-%v-%v", task.Project, task.Revision[0:6]))
}
}
cloneDir = filepath.Join(rootPath, cloneDir)
err = cloneSource(task, project, config, cloneDir)
if err != nil {
return err
}
if patch != nil && !noPatch {
err = applyPatch(patch, cloneDir, config, config.FindBuildVariant(task.BuildVariant))
if err != nil {
return err
}
}
return nil
}
// ToModelTestResultAndLog converts an xunit test case into an
// mci task.TestResult and model.TestLog. Logs are only
// generated if the test case did not succeed (this is part of
// the xunit xml file design)
func (tc TestCase) ToModelTestResultAndLog(t *task.Task) (task.TestResult, *model.TestLog) {
res := task.TestResult{}
var log *model.TestLog
if tc.ClassName != "" {
res.TestFile = fmt.Sprintf("%v.%v", tc.ClassName, tc.Name)
} else {
res.TestFile = tc.Name
}
// replace spaces, dashes, etc. with underscores
res.TestFile = util.CleanForPath(res.TestFile)
res.StartTime = float64(time.Now().Unix())
res.EndTime = float64(res.StartTime + tc.Time)
// the presence of the Failure, Error, or Skipped fields
// is used to indicate an unsuccessful test case. Logs
// can only be generated in failure cases, because xunit
// results only include messages if they did *not* succeed.
switch {
case tc.Failure != nil:
res.Status = evergreen.TestFailedStatus
log = tc.Failure.toBasicTestLog("FAILURE")
case tc.Error != nil:
res.Status = evergreen.TestFailedStatus
log = tc.Error.toBasicTestLog("ERROR")
case tc.Skipped != nil:
res.Status = evergreen.TestSkippedStatus
log = tc.Skipped.toBasicTestLog("SKIPPED")
default:
res.Status = evergreen.TestSucceededStatus
}
if log != nil {
log.Name = res.TestFile
log.Task = t.Id
log.TaskExecution = t.Execution
// update the URL of the result to the expected log URL
res.URL = log.URL()
}
return res, log
}
// downloadUrls pulls a set of artifacts from the given channel and downloads them, using up to
// the given number of workers in parallel. The given root directory determines the base location
// where all the artifact files will be downloaded to.
func downloadUrls(root string, urls chan artifactDownload, workers int) error {
if workers <= 0 {
panic("invalid workers count")
}
wg := sync.WaitGroup{}
errs := make(chan error)
wg.Add(workers)
// Keep track of filenames being downloaded, so that if there are collisions, we can detect
// and re-name the file to something else.
fileNamesUsed := struct {
nameCounts map[string]int
sync.Mutex
}{nameCounts: map[string]int{}}
for i := 0; i < workers; i++ {
go func(workerId int) {
defer wg.Done()
counter := 0
for u := range urls {
// Try to determinate the file location for the output.
folder := filepath.Join(root, u.path)
// As a backup plan in case we can't figure out the file name from the URL,
// the file name will just be named after the worker ID and file index.
justFile := fmt.Sprintf("%v_%v", workerId, counter)
parsedUrl, err := url.Parse(u.url)
if err == nil {
// under normal operation, the file name written to disk will match the name
// of the file in the URL. For instance, http://www.website.com/file.tgz
// will assume "file.tgz".
pathParts := strings.Split(parsedUrl.Path, "/")
if len(pathParts) >= 1 {
justFile = util.CleanForPath(pathParts[len(pathParts)-1])
}
}
fileName := filepath.Join(folder, justFile)
fileNamesUsed.Lock()
for {
fileNamesUsed.nameCounts[fileName] += 1
testFileName := fileNameWithIndex(fileName, fileNamesUsed.nameCounts[fileName])
_, err := os.Stat(testFileName)
if err != nil {
if os.IsNotExist(err) {
// we found a file name to safely create without collisions..
fileName = testFileName
break
}
// something else went wrong.
errs <- fmt.Errorf("failed to check if file exists: %v", err)
return
}
}
fileNamesUsed.Unlock()
err = os.MkdirAll(folder, 0777)
if err != nil {
errs <- fmt.Errorf("Couldn't create output directory %v: %v", folder, err)
continue
}
out, err := os.Create(fileName)
if err != nil {
errs <- fmt.Errorf("Couldn't download %v: %v", u.url, err)
continue
}
resp, err := http.Get(u.url)
if err != nil {
errs <- fmt.Errorf("Couldn't download %v: %v", u.url, err)
continue
}
// If we can get the info, determine the file size so that the human can get an
// idea of how long the file might take to download.
// TODO: progress bars.
length, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
sizeLog := ""
if length > 0 {
sizeLog = fmt.Sprintf(" (%s)", humanize.Bytes(uint64(length)))
}
justFile = filepath.Base(fileName)
fmt.Printf("(worker %v) Downloading %v to directory %s%s\n", workerId, justFile, u.path, sizeLog)
//sizeTracker := util.SizeTrackingReader{0, resp.Body}
_, err = io.Copy(out, resp.Body)
if err != nil {
errs <- fmt.Errorf("Couldn't download %v: %v", u.url, err)
continue
}
resp.Body.Close()
out.Close()
counter++
}
}(i)
}
done := make(chan struct{})
var hasErrors error
go func() {
defer close(done)
for e := range errs {
hasErrors = fmt.Errorf("some files could not be downloaded successfully.")
fmt.Println("error: ", e)
}
}()
wg.Wait()
close(errs)
<-done
return hasErrors
}