Example #1
0
func fetchSource(ac, rc *APIClient, rootPath, taskId string, noPatch bool) error {
	task, err := rc.GetTask(taskId)
	if err != nil {
		return err
	}
	if task == nil {
		return fmt.Errorf("task not found.")
	}

	config, err := rc.GetConfig(task.Version)
	if err != nil {
		return err
	}

	project, err := ac.GetProjectRef(task.Project)
	if err != nil {
		return err
	}

	cloneDir := util.CleanForPath(fmt.Sprintf("source-%v", task.Project))
	var patch *service.RestPatch
	if task.Requester == evergreen.PatchVersionRequester {
		cloneDir = util.CleanForPath(fmt.Sprintf("source-patch-%v_%v", task.PatchNumber, task.Project))
		patch, err = rc.GetPatch(task.PatchId)
		if err != nil {
			return err
		}
	} else {
		if len(task.Revision) >= 5 {
			cloneDir = util.CleanForPath(fmt.Sprintf("source-%v-%v", task.Project, task.Revision[0:6]))
		}
	}
	cloneDir = filepath.Join(rootPath, cloneDir)

	err = cloneSource(task, project, config, cloneDir)
	if err != nil {
		return err
	}
	if patch != nil && !noPatch {
		err = applyPatch(patch, cloneDir, config, config.FindBuildVariant(task.BuildVariant))
		if err != nil {
			return err
		}
	}

	return nil
}
Example #2
0
// ToModelTestResultAndLog converts an xunit test case into an
// mci task.TestResult and model.TestLog. Logs are only
// generated if the test case did not succeed (this is part of
// the xunit xml file design)
func (tc TestCase) ToModelTestResultAndLog(t *task.Task) (task.TestResult, *model.TestLog) {

	res := task.TestResult{}
	var log *model.TestLog

	if tc.ClassName != "" {
		res.TestFile = fmt.Sprintf("%v.%v", tc.ClassName, tc.Name)
	} else {
		res.TestFile = tc.Name
	}
	// replace spaces, dashes, etc. with underscores
	res.TestFile = util.CleanForPath(res.TestFile)

	res.StartTime = float64(time.Now().Unix())
	res.EndTime = float64(res.StartTime + tc.Time)

	// the presence of the Failure, Error, or Skipped fields
	// is used to indicate an unsuccessful test case. Logs
	// can only be generated in failure cases, because xunit
	// results only include messages if they did *not* succeed.
	switch {
	case tc.Failure != nil:
		res.Status = evergreen.TestFailedStatus
		log = tc.Failure.toBasicTestLog("FAILURE")
	case tc.Error != nil:
		res.Status = evergreen.TestFailedStatus
		log = tc.Error.toBasicTestLog("ERROR")
	case tc.Skipped != nil:
		res.Status = evergreen.TestSkippedStatus
		log = tc.Skipped.toBasicTestLog("SKIPPED")
	default:
		res.Status = evergreen.TestSucceededStatus
	}

	if log != nil {
		log.Name = res.TestFile
		log.Task = t.Id
		log.TaskExecution = t.Execution

		// update the URL of the result to the expected log URL
		res.URL = log.URL()
	}

	return res, log
}
Example #3
0
// downloadUrls pulls a set of artifacts from the given channel and downloads them, using up to
// the given number of workers in parallel. The given root directory determines the base location
// where all the artifact files will be downloaded to.
func downloadUrls(root string, urls chan artifactDownload, workers int) error {
	if workers <= 0 {
		panic("invalid workers count")
	}
	wg := sync.WaitGroup{}
	errs := make(chan error)
	wg.Add(workers)

	// Keep track of filenames being downloaded, so that if there are collisions, we can detect
	// and re-name the file to something else.
	fileNamesUsed := struct {
		nameCounts map[string]int
		sync.Mutex
	}{nameCounts: map[string]int{}}

	for i := 0; i < workers; i++ {
		go func(workerId int) {
			defer wg.Done()
			counter := 0
			for u := range urls {

				// Try to determinate the file location for the output.
				folder := filepath.Join(root, u.path)
				// As a backup plan in case we can't figure out the file name from the URL,
				// the file name will just be named after the worker ID and file index.
				justFile := fmt.Sprintf("%v_%v", workerId, counter)
				parsedUrl, err := url.Parse(u.url)
				if err == nil {
					// under normal operation, the file name written to disk will match the name
					// of the file in the URL. For instance, http://www.website.com/file.tgz
					// will assume "file.tgz".
					pathParts := strings.Split(parsedUrl.Path, "/")
					if len(pathParts) >= 1 {
						justFile = util.CleanForPath(pathParts[len(pathParts)-1])
					}
				}

				fileName := filepath.Join(folder, justFile)
				fileNamesUsed.Lock()
				for {
					fileNamesUsed.nameCounts[fileName] += 1
					testFileName := fileNameWithIndex(fileName, fileNamesUsed.nameCounts[fileName])
					_, err := os.Stat(testFileName)
					if err != nil {
						if os.IsNotExist(err) {
							// we found a file name to safely create without collisions..
							fileName = testFileName
							break
						}
						// something else went wrong.
						errs <- fmt.Errorf("failed to check if file exists: %v", err)
						return
					}
				}

				fileNamesUsed.Unlock()

				err = os.MkdirAll(folder, 0777)
				if err != nil {
					errs <- fmt.Errorf("Couldn't create output directory %v: %v", folder, err)
					continue
				}

				out, err := os.Create(fileName)
				if err != nil {
					errs <- fmt.Errorf("Couldn't download %v: %v", u.url, err)
					continue
				}
				resp, err := http.Get(u.url)
				if err != nil {
					errs <- fmt.Errorf("Couldn't download %v: %v", u.url, err)
					continue
				}

				// If we can get the info, determine the file size so that the human can get an
				// idea of how long the file might take to download.
				// TODO: progress bars.
				length, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
				sizeLog := ""
				if length > 0 {
					sizeLog = fmt.Sprintf(" (%s)", humanize.Bytes(uint64(length)))
				}

				justFile = filepath.Base(fileName)
				fmt.Printf("(worker %v) Downloading %v to directory %s%s\n", workerId, justFile, u.path, sizeLog)
				//sizeTracker := util.SizeTrackingReader{0, resp.Body}
				_, err = io.Copy(out, resp.Body)
				if err != nil {
					errs <- fmt.Errorf("Couldn't download %v: %v", u.url, err)
					continue
				}
				resp.Body.Close()
				out.Close()
				counter++
			}
		}(i)
	}

	done := make(chan struct{})
	var hasErrors error
	go func() {
		defer close(done)
		for e := range errs {
			hasErrors = fmt.Errorf("some files could not be downloaded successfully.")
			fmt.Println("error: ", e)
		}
	}()
	wg.Wait()
	close(errs)
	<-done

	return hasErrors
}