Пример #1
0
// GCSBasedStable is a version of Stable function that depends on files stored in GCS instead of Jenkis
func (e *RealE2ETester) GCSBasedStable() (allStable, ignorableFlakes bool) {
	allStable = true

	for _, job := range e.JobNames {
		lastBuildNumber, err := e.GoogleGCSBucketUtils.GetLastestBuildNumberFromJenkinsGoogleBucket(job)
		glog.V(4).Infof("Checking status of %v, %v", job, lastBuildNumber)
		if err != nil {
			glog.Errorf("Error while getting data for %v: %v", job, err)
			e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
			continue
		}

		thisResult, err := e.flakeCache.Get(cache.Job(job), cache.Number(lastBuildNumber))
		if err != nil || (!thisResult.Pass && thisResult.UnlistedFlakes) {
			glog.V(4).Infof("Found unstable job: %v, build number: %v: (err: %v) %#v", job, lastBuildNumber, err, thisResult)
			e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
			allStable = false
			continue
		}

		if thisResult.Pass {
			e.setBuildStatus(job, "Stable", strconv.Itoa(lastBuildNumber))
			continue
		}

		lastResult, err := e.flakeCache.Get(cache.Job(job), cache.Number(lastBuildNumber-1))
		if err != nil || lastResult.UnlistedFlakes {
			glog.V(4).Infof("prev job doesn't help: %v, build number: %v (the previous build); (err %v) %#v", job, lastBuildNumber-1, err, lastResult)
			allStable = false
			e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
			continue
		}

		if lastResult.Pass {
			ignorableFlakes = true
			e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(lastBuildNumber))
			continue
		}

		intersection := sets.NewString()
		for testName := range thisResult.Flakes {
			if _, ok := lastResult.Flakes[testName]; ok {
				intersection.Insert(string(testName))
			}
		}
		if len(intersection) == 0 {
			glog.V(2).Infof("Ignoring failure of %v/%v since it didn't happen the previous run this run = %v; prev run = %v.", job, lastBuildNumber, thisResult.Flakes, lastResult.Flakes)
			ignorableFlakes = true
			e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(lastBuildNumber))
			continue
		}
		glog.V(2).Infof("Failure of %v/%v is legit. Tests that failed multiple times in a row: %v", job, lastBuildNumber, intersection)
		allStable = false
		e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
	}

	return allStable, ignorableFlakes
}
Пример #2
0
func getReqKey(req *http.Request) (key resolutionKey, err error) {
	key.job = cache.Job(req.URL.Query().Get("job"))
	var n int
	n, err = strconv.Atoi(req.URL.Query().Get("number"))
	key.number = cache.Number(n)
	return key, err
}
Пример #3
0
// GCSWeakStable is a version of GCSBasedStable with a slightly relaxed condition.
// This function says that e2e's are unstable only if there were real test failures
// (i.e. there was a test that failed, so no timeouts/cluster startup failures counts),
// or test failed for any reason 3 times in a row.
func (e *RealE2ETester) GCSWeakStable() bool {
	allStable := true
	for _, job := range e.WeakStableJobNames {
		lastBuildNumber, err := e.GoogleGCSBucketUtils.GetLastestBuildNumberFromJenkinsGoogleBucket(job)
		glog.V(4).Infof("Checking status of %v, %v", job, lastBuildNumber)
		if err != nil {
			glog.Errorf("Error while getting data for %v: %v", job, err)
			e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
			continue
		}
		if stable, err := e.GoogleGCSBucketUtils.CheckFinishedStatus(job, lastBuildNumber); stable && err == nil {
			e.setBuildStatus(job, "Stable", strconv.Itoa(lastBuildNumber))
			continue
		}

		if e.resolutionTracker.Resolved(cache.Job(job), cache.Number(lastBuildNumber)) {
			e.setBuildStatus(job, "Problem Resolved", strconv.Itoa(lastBuildNumber))
			continue
		}

		failures, err := e.failureReasons(job, lastBuildNumber, false)
		if err != nil {
			glog.Errorf("Error while getting data for %v/%v: %v", job, lastBuildNumber, err)
			e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
			continue
		}

		thisStable := len(failures) == 0

		if thisStable == false {
			allStable = false
			e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
			glog.Infof("WeakStable failed because found a failure in JUnit file for build %v; %v and possibly more failed", lastBuildNumber, failures)
			continue
		}

		// If we're here it means that we weren't able to find a test that failed, which means that the reason of build failure is comming from the infrastructure
		// Check results of previous two builds.
		unstable := make([]int, 0)
		if stable, err := e.GoogleGCSBucketUtils.CheckFinishedStatus(job, lastBuildNumber-1); !stable || err != nil {
			unstable = append(unstable, lastBuildNumber-1)
		}
		if stable, err := e.GoogleGCSBucketUtils.CheckFinishedStatus(job, lastBuildNumber-2); !stable || err != nil {
			unstable = append(unstable, lastBuildNumber-2)
		}
		if len(unstable) > 1 {
			e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
			allStable = false
			glog.Infof("WeakStable failed because found a weak failure in build %v and builds %v failed.", lastBuildNumber, unstable)
			continue
		}
		e.setBuildStatus(job, "Stable", strconv.Itoa(lastBuildNumber))
	}
	return allStable
}
Пример #4
0
func (e *RealE2ETester) checkPassFail(job string, number int) (stable, ignorableFlakes bool) {
	if e.resolutionTracker.Resolved(cache.Job(job), cache.Number(number)) {
		e.setBuildStatus(job, "Problem Resolved", strconv.Itoa(number))
		return true, true
	}

	thisResult, err := e.GetBuildResult(job, number)
	if err != nil || thisResult.Status == cache.ResultFailed {
		glog.V(4).Infof("Found unstable job: %v, build number: %v: (err: %v) %#v", job, number, err, thisResult)
		e.setBuildStatus(job, "Not Stable", strconv.Itoa(number))
		return false, false
	}

	if thisResult.Status == cache.ResultStable {
		e.setBuildStatus(job, "Stable", strconv.Itoa(number))
		return true, false
	}

	lastResult, err := e.GetBuildResult(job, number-1)
	if err != nil || lastResult.Status == cache.ResultFailed {
		glog.V(4).Infof("prev job doesn't help: %v, build number: %v (the previous build); (err %v) %#v", job, number-1, err, lastResult)
		e.setBuildStatus(job, "Not Stable", strconv.Itoa(number))
		return true, false
	}

	if lastResult.Status == cache.ResultStable {
		e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(number))
		return true, true
	}

	intersection := sets.NewString()
	for testName := range thisResult.Flakes {
		if _, ok := lastResult.Flakes[testName]; ok {
			intersection.Insert(string(testName))
		}
	}
	if len(intersection) == 0 {
		glog.V(2).Infof("Ignoring failure of %v/%v since it didn't happen the previous run this run = %v; prev run = %v.", job, number, thisResult.Flakes, lastResult.Flakes)
		e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(number))
		return true, true
	}
	glog.V(2).Infof("Failure of %v/%v is legit. Tests that failed multiple times in a row: %v", job, number, intersection)
	e.setBuildStatus(job, "Not Stable", strconv.Itoa(number))
	return false, false
}
Пример #5
0
// GetBuildResult returns (or gets) the cached result of the job and build. Public.
func (e *RealE2ETester) GetBuildResult(job string, number int) (*cache.Result, error) {
	return e.flakeCache.Get(cache.Job(job), cache.Number(number))
}