// GCSBasedStable is a version of Stable function that depends on files stored in GCS instead of Jenkis func (e *RealE2ETester) GCSBasedStable() (allStable, ignorableFlakes bool) { allStable = true for _, job := range e.JobNames { lastBuildNumber, err := e.GoogleGCSBucketUtils.GetLastestBuildNumberFromJenkinsGoogleBucket(job) glog.V(4).Infof("Checking status of %v, %v", job, lastBuildNumber) if err != nil { glog.Errorf("Error while getting data for %v: %v", job, err) e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber)) continue } thisResult, err := e.flakeCache.Get(cache.Job(job), cache.Number(lastBuildNumber)) if err != nil || (!thisResult.Pass && thisResult.UnlistedFlakes) { glog.V(4).Infof("Found unstable job: %v, build number: %v: (err: %v) %#v", job, lastBuildNumber, err, thisResult) e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber)) allStable = false continue } if thisResult.Pass { e.setBuildStatus(job, "Stable", strconv.Itoa(lastBuildNumber)) continue } lastResult, err := e.flakeCache.Get(cache.Job(job), cache.Number(lastBuildNumber-1)) if err != nil || lastResult.UnlistedFlakes { glog.V(4).Infof("prev job doesn't help: %v, build number: %v (the previous build); (err %v) %#v", job, lastBuildNumber-1, err, lastResult) allStable = false e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber)) continue } if lastResult.Pass { ignorableFlakes = true e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(lastBuildNumber)) continue } intersection := sets.NewString() for testName := range thisResult.Flakes { if _, ok := lastResult.Flakes[testName]; ok { intersection.Insert(string(testName)) } } if len(intersection) == 0 { glog.V(2).Infof("Ignoring failure of %v/%v since it didn't happen the previous run this run = %v; prev run = %v.", job, lastBuildNumber, thisResult.Flakes, lastResult.Flakes) ignorableFlakes = true e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(lastBuildNumber)) continue } glog.V(2).Infof("Failure of %v/%v is legit. Tests that failed multiple times in a row: %v", job, lastBuildNumber, intersection) allStable = false e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber)) } return allStable, ignorableFlakes }
func getReqKey(req *http.Request) (key resolutionKey, err error) { key.job = cache.Job(req.URL.Query().Get("job")) var n int n, err = strconv.Atoi(req.URL.Query().Get("number")) key.number = cache.Number(n) return key, err }
// GCSWeakStable is a version of GCSBasedStable with a slightly relaxed condition. // This function says that e2e's are unstable only if there were real test failures // (i.e. there was a test that failed, so no timeouts/cluster startup failures counts), // or test failed for any reason 3 times in a row. func (e *RealE2ETester) GCSWeakStable() bool { allStable := true for _, job := range e.WeakStableJobNames { lastBuildNumber, err := e.GoogleGCSBucketUtils.GetLastestBuildNumberFromJenkinsGoogleBucket(job) glog.V(4).Infof("Checking status of %v, %v", job, lastBuildNumber) if err != nil { glog.Errorf("Error while getting data for %v: %v", job, err) e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber)) continue } if stable, err := e.GoogleGCSBucketUtils.CheckFinishedStatus(job, lastBuildNumber); stable && err == nil { e.setBuildStatus(job, "Stable", strconv.Itoa(lastBuildNumber)) continue } if e.resolutionTracker.Resolved(cache.Job(job), cache.Number(lastBuildNumber)) { e.setBuildStatus(job, "Problem Resolved", strconv.Itoa(lastBuildNumber)) continue } failures, err := e.failureReasons(job, lastBuildNumber, false) if err != nil { glog.Errorf("Error while getting data for %v/%v: %v", job, lastBuildNumber, err) e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber)) continue } thisStable := len(failures) == 0 if thisStable == false { allStable = false e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber)) glog.Infof("WeakStable failed because found a failure in JUnit file for build %v; %v and possibly more failed", lastBuildNumber, failures) continue } // If we're here it means that we weren't able to find a test that failed, which means that the reason of build failure is comming from the infrastructure // Check results of previous two builds. unstable := make([]int, 0) if stable, err := e.GoogleGCSBucketUtils.CheckFinishedStatus(job, lastBuildNumber-1); !stable || err != nil { unstable = append(unstable, lastBuildNumber-1) } if stable, err := e.GoogleGCSBucketUtils.CheckFinishedStatus(job, lastBuildNumber-2); !stable || err != nil { unstable = append(unstable, lastBuildNumber-2) } if len(unstable) > 1 { e.setBuildStatus(job, "Not Stable", strconv.Itoa(lastBuildNumber)) allStable = false glog.Infof("WeakStable failed because found a weak failure in build %v and builds %v failed.", lastBuildNumber, unstable) continue } e.setBuildStatus(job, "Stable", strconv.Itoa(lastBuildNumber)) } return allStable }
func (e *RealE2ETester) checkPassFail(job string, number int) (stable, ignorableFlakes bool) { if e.resolutionTracker.Resolved(cache.Job(job), cache.Number(number)) { e.setBuildStatus(job, "Problem Resolved", strconv.Itoa(number)) return true, true } thisResult, err := e.GetBuildResult(job, number) if err != nil || thisResult.Status == cache.ResultFailed { glog.V(4).Infof("Found unstable job: %v, build number: %v: (err: %v) %#v", job, number, err, thisResult) e.setBuildStatus(job, "Not Stable", strconv.Itoa(number)) return false, false } if thisResult.Status == cache.ResultStable { e.setBuildStatus(job, "Stable", strconv.Itoa(number)) return true, false } lastResult, err := e.GetBuildResult(job, number-1) if err != nil || lastResult.Status == cache.ResultFailed { glog.V(4).Infof("prev job doesn't help: %v, build number: %v (the previous build); (err %v) %#v", job, number-1, err, lastResult) e.setBuildStatus(job, "Not Stable", strconv.Itoa(number)) return true, false } if lastResult.Status == cache.ResultStable { e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(number)) return true, true } intersection := sets.NewString() for testName := range thisResult.Flakes { if _, ok := lastResult.Flakes[testName]; ok { intersection.Insert(string(testName)) } } if len(intersection) == 0 { glog.V(2).Infof("Ignoring failure of %v/%v since it didn't happen the previous run this run = %v; prev run = %v.", job, number, thisResult.Flakes, lastResult.Flakes) e.setBuildStatus(job, "Ignorable flake", strconv.Itoa(number)) return true, true } glog.V(2).Infof("Failure of %v/%v is legit. Tests that failed multiple times in a row: %v", job, number, intersection) e.setBuildStatus(job, "Not Stable", strconv.Itoa(number)) return false, false }
// GetBuildResult returns (or gets) the cached result of the job and build. Public. func (e *RealE2ETester) GetBuildResult(job string, number int) (*cache.Result, error) { return e.flakeCache.Get(cache.Job(job), cache.Number(number)) }