// GCSBasedStable is a version of Stable function that depends on files stored in GCS instead of Jenkis
func (e *RealE2ETester) GCSBasedStable() (allStable, ignorableFlakes bool) {
allStable = true
for _, job := range e.JobNames {
lastBuildNumber, err := e.GoogleGCSBucketUtils.GetLastestBuildNumberFromJenkinsGoogleBucket(job)
glog.V(4).Infof("Checking status of %v, %v", job, lastBuildNumber)
if err != nil {
glog.Errorf("Error while getting data for %v: %v", job, err)
e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
continue
}
thisResult, err := e.flakeCache.Get(cache.Job(job), cache.Number(lastBuildNumber))
if err != nil || (!thisResult.Pass && thisResult.UnlistedFlakes) {
glog.V(4).Infof("Found unstable job: %v, build number: %v: (err: %v) %#v", job, lastBuildNumber, err, thisResult)
e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
allStable = false
continue
}
if thisResult.Pass {
e.setBuildStatus(job, "Stable", strconv.Itoa(lastBuildNumber))
continue
}
lastResult, err := e.flakeCache.Get(cache.Job(job), cache.Number(lastBuildNumber-1))
if err != nil || lastResult.UnlistedFlakes {
glog.V(4).Infof("prev job doesn't help: %v, build number: %v (the previous build); (err %v) %#v", job, lastBuildNumber-1, err, lastResult)
allStable = false
e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
continue
}
if lastResult.Pass {
ignorableFlakes = true
e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(lastBuildNumber))
continue
}
intersection := sets.NewString()
for testName := range thisResult.Flakes {
if _, ok := lastResult.Flakes[testName]; ok {
intersection.Insert(string(testName))
}
}
if len(intersection) == 0 {
glog.V(2).Infof("Ignoring failure of %v/%v since it didn't happen the previous run this run = %v; prev run = %v.", job, lastBuildNumber, thisResult.Flakes, lastResult.Flakes)
ignorableFlakes = true
e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(lastBuildNumber))
continue
}
glog.V(2).Infof("Failure of %v/%v is legit. Tests that failed multiple times in a row: %v", job, lastBuildNumber, intersection)
allStable = false
e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
}
return allStable, ignorableFlakes
}
func getReqKey(req *http.Request) (key resolutionKey, err error) {
key.job = cache.Job(req.URL.Query().Get("job"))
var n int
n, err = strconv.Atoi(req.URL.Query().Get("number"))
key.number = cache.Number(n)
return key, err
}
// GCSWeakStable is a version of GCSBasedStable with a slightly relaxed condition.
// This function says that e2e's are unstable only if there were real test failures
// (i.e. there was a test that failed, so no timeouts/cluster startup failures counts),
// or test failed for any reason 3 times in a row.
func (e *RealE2ETester) GCSWeakStable() bool {
allStable := true
for _, job := range e.WeakStableJobNames {
lastBuildNumber, err := e.GoogleGCSBucketUtils.GetLastestBuildNumberFromJenkinsGoogleBucket(job)
glog.V(4).Infof("Checking status of %v, %v", job, lastBuildNumber)
if err != nil {
glog.Errorf("Error while getting data for %v: %v", job, err)
e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
continue
}
if stable, err := e.GoogleGCSBucketUtils.CheckFinishedStatus(job, lastBuildNumber); stable && err == nil {
e.setBuildStatus(job, "Stable", strconv.Itoa(lastBuildNumber))
continue
}
if e.resolutionTracker.Resolved(cache.Job(job), cache.Number(lastBuildNumber)) {
e.setBuildStatus(job, "Problem Resolved", strconv.Itoa(lastBuildNumber))
continue
}
failures, err := e.failureReasons(job, lastBuildNumber, false)
if err != nil {
glog.Errorf("Error while getting data for %v/%v: %v", job, lastBuildNumber, err)
e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
continue
}
thisStable := len(failures) == 0
if thisStable == false {
allStable = false
e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
glog.Infof("WeakStable failed because found a failure in JUnit file for build %v; %v and possibly more failed", lastBuildNumber, failures)
continue
}
// If we're here it means that we weren't able to find a test that failed, which means that the reason of build failure is comming from the infrastructure
// Check results of previous two builds.
unstable := make([]int, 0)
if stable, err := e.GoogleGCSBucketUtils.CheckFinishedStatus(job, lastBuildNumber-1); !stable || err != nil {
unstable = append(unstable, lastBuildNumber-1)
}
if stable, err := e.GoogleGCSBucketUtils.CheckFinishedStatus(job, lastBuildNumber-2); !stable || err != nil {
unstable = append(unstable, lastBuildNumber-2)
}
if len(unstable) > 1 {
e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber))
allStable = false
glog.Infof("WeakStable failed because found a weak failure in build %v and builds %v failed.", lastBuildNumber, unstable)
continue
}
e.setBuildStatus(job, "Stable", strconv.Itoa(lastBuildNumber))
}
return allStable
}
func (e *RealE2ETester) checkPassFail(job string, number int) (stable, ignorableFlakes bool) {
if e.resolutionTracker.Resolved(cache.Job(job), cache.Number(number)) {
e.setBuildStatus(job, "Problem Resolved", strconv.Itoa(number))
return true, true
}
thisResult, err := e.GetBuildResult(job, number)
if err != nil || thisResult.Status == cache.ResultFailed {
glog.V(4).Infof("Found unstable job: %v, build number: %v: (err: %v) %#v", job, number, err, thisResult)
e.setBuildStatus(job, "Not Stable", strconv.Itoa(number))
return false, false
}
if thisResult.Status == cache.ResultStable {
e.setBuildStatus(job, "Stable", strconv.Itoa(number))
return true, false
}
lastResult, err := e.GetBuildResult(job, number-1)
if err != nil || lastResult.Status == cache.ResultFailed {
glog.V(4).Infof("prev job doesn't help: %v, build number: %v (the previous build); (err %v) %#v", job, number-1, err, lastResult)
e.setBuildStatus(job, "Not Stable", strconv.Itoa(number))
return true, false
}
if lastResult.Status == cache.ResultStable {
e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(number))
return true, true
}
intersection := sets.NewString()
for testName := range thisResult.Flakes {
if _, ok := lastResult.Flakes[testName]; ok {
intersection.Insert(string(testName))
}
}
if len(intersection) == 0 {
glog.V(2).Infof("Ignoring failure of %v/%v since it didn't happen the previous run this run = %v; prev run = %v.", job, number, thisResult.Flakes, lastResult.Flakes)
e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(number))
return true, true
}
glog.V(2).Infof("Failure of %v/%v is legit. Tests that failed multiple times in a row: %v", job, number, intersection)
e.setBuildStatus(job, "Not Stable", strconv.Itoa(number))
return false, false
}
// GetBuildResult returns (or gets) the cached result of the job and build. Public.
func (e *RealE2ETester) GetBuildResult(job string, number int) (*cache.Result, error) {
return e.flakeCache.Get(cache.Job(job), cache.Number(number))
}