// GCSBasedStable is a version of Stable function that depends on files stored in GCS instead of Jenkis func (e *E2ETester) GCSBasedStable() bool { for _, job := range e.JobNames { lastBuildNumber, err := utils.GetLastestBuildNumberFromJenkinsGoogleBucket(job) glog.V(4).Infof("Checking status of %v, %v", job, lastBuildNumber) if err != nil { glog.Errorf("Error while getting data for %v: %v", job, err) continue } if !checkFinishedStatus(job, lastBuildNumber) { return false } } return true }
func (g *GoogleGCSDownloader) getData() (TestToBuildData, sets.String, sets.String, error) { fmt.Print("Getting Data from GCS...\n") buildLatency := TestToBuildData{} resources := sets.NewString() methods := sets.NewString() buildNumber := g.startFrom lastBuildNo, err := utils.GetLastestBuildNumberFromJenkinsGoogleBucket(jobName) if err != nil { return buildLatency, resources, methods, err } if buildNumber < lastBuildNo-100 { buildNumber = lastBuildNo - 100 } for ; buildNumber <= lastBuildNo; buildNumber++ { fmt.Printf("Fetching build %v...\n", buildNumber) testDataResponse, err := utils.GetFileFromJenkinsGoogleBucket(jobName, buildNumber, logFile) if err != nil { fmt.Fprintf(os.Stderr, "Error while fetching data: %v\n", err) continue } testDataBody := testDataResponse.Body defer testDataBody.Close() testDataScanner := bufio.NewScanner(testDataBody) hist := parseTestOutput(testDataScanner, buildNumber, resources, methods) for k, v := range hist { if _, ok := buildLatency[k]; !ok { buildLatency[k] = make(BuildLatencyData) } buildLatency[k][fmt.Sprintf("%d", buildNumber)] = v } } return buildLatency, resources, methods, nil }
// GCSWeakStable is a version of GCSBasedStable with a slightly relaxed condition. // This function says that e2e's are unstable only if there were real test failures // (i.e. there was a test that failed, so no timeouts/cluster startup failures counts), // or test failed for any reason 3 times in a row. func (e *E2ETester) GCSWeakStable() bool { for _, job := range e.WeakStableJobNames { lastBuildNumber, err := utils.GetLastestBuildNumberFromJenkinsGoogleBucket(job) glog.Infof("Checking status of %v, %v", job, lastBuildNumber) if err != nil { glog.Errorf("Error while getting data for %v: %v", job, err) continue } if checkFinishedStatus(job, lastBuildNumber) { continue } // If we're here it means that build failed, so we need to look for a reason // by iterating over junit_XX.xml files and look for failures i := 0 for { i++ path := fmt.Sprintf("artifacts/junit_%02d.xml", i) response, err := utils.GetFileFromJenkinsGoogleBucket(job, lastBuildNumber, path) if err != nil { glog.Errorf("Error while getting data for %v/%v/%v: %v", job, lastBuildNumber, path, err) continue } if response.StatusCode != 200 { break } defer response.Body.Close() reader := bufio.NewReader(response.Body) body, err := reader.ReadString('\n') if err != nil { glog.Errorf("Failed to read the response for %v/%v/%v: %v", job, lastBuildNumber, path, err) continue } if strings.TrimSpace(body) != expectedXMLHeader { glog.Errorf("Invalid header for %v/%v/%v: %v, expected %v", job, lastBuildNumber, path, body, expectedXMLHeader) continue } body, err = reader.ReadString('\n') if err != nil { glog.Errorf("Failed to read the response for %v/%v/%v: %v", job, lastBuildNumber, path, err) continue } numberOfTests := 0 nubmerOfFailures := 0 timestamp := 0.0 fmt.Sscanf(strings.TrimSpace(body), "<testsuite tests=\"%d\" failures=\"%d\" time=\"%f\">", &numberOfTests, &nubmerOfFailures, ×tamp) glog.V(4).Infof("%v, numberOfTests: %v, numberOfFailures: %v", string(body), numberOfTests, nubmerOfFailures) if nubmerOfFailures > 0 { glog.V(4).Infof("Found failure in %v for job %v build number %v", path, job, lastBuildNumber) return false } } // If we're here it means that we weren't able to find a test that failed, which means that the reason of build failure is comming from the infrastructure // Check results of previous two builds. if !checkFinishedStatus(job, lastBuildNumber-1) && !checkFinishedStatus(job, lastBuildNumber-2) { return false } } return true }