Пример #1
0
func checkFinishedStatus(job string, lastBuildNumber int) bool {
	response, err := utils.GetFileFromJenkinsGoogleBucket(job, lastBuildNumber, "finished.json")
	if err != nil {
		glog.Errorf("Error while getting data for %v/%v/%v: %v", job, lastBuildNumber, "finished.json", err)
		return false
	}

	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		glog.Infof("Got a non-success response %v while reading data for %v/%v/%v", response.StatusCode, job, lastBuildNumber, "finished.json")
		return false
	}
	result := finishedFile{}
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		glog.Errorf("Failed to read the response for %v/%v/%v: %v", job, lastBuildNumber, "finished.json", err)
		return false
	}
	err = json.Unmarshal(body, &result)
	if err != nil {
		glog.Errorf("Failed to unmarshal %v: %v", string(body), err)
		return false
	}
	return result.Result == successString
}
Пример #2
0
func (g *GoogleGCSDownloader) getData() (TestToBuildData, sets.String, sets.String, error) {
	fmt.Print("Getting Data from GCS...\n")
	buildLatency := TestToBuildData{}
	resources := sets.NewString()
	methods := sets.NewString()

	buildNumber := g.startFrom
	lastBuildNo, err := utils.GetLastestBuildNumberFromJenkinsGoogleBucket(jobName)
	if err != nil {
		return buildLatency, resources, methods, err
	}
	if buildNumber < lastBuildNo-100 {
		buildNumber = lastBuildNo - 100
	}

	for ; buildNumber <= lastBuildNo; buildNumber++ {
		fmt.Printf("Fetching build %v...\n", buildNumber)
		testDataResponse, err := utils.GetFileFromJenkinsGoogleBucket(jobName, buildNumber, logFile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error while fetching data: %v\n", err)
			continue
		}

		testDataBody := testDataResponse.Body
		defer testDataBody.Close()
		testDataScanner := bufio.NewScanner(testDataBody)

		hist := parseTestOutput(testDataScanner, buildNumber, resources, methods)

		for k, v := range hist {
			if _, ok := buildLatency[k]; !ok {
				buildLatency[k] = make(BuildLatencyData)
			}
			buildLatency[k][fmt.Sprintf("%d", buildNumber)] = v
		}
	}

	return buildLatency, resources, methods, nil
}
Пример #3
0
// GCSWeakStable is a version of GCSBasedStable with a slightly relaxed condition.
// This function says that e2e's are unstable only if there were real test failures
// (i.e. there was a test that failed, so no timeouts/cluster startup failures counts),
// or test failed for any reason 3 times in a row.
func (e *E2ETester) GCSWeakStable() bool {
	for _, job := range e.WeakStableJobNames {
		lastBuildNumber, err := utils.GetLastestBuildNumberFromJenkinsGoogleBucket(job)
		glog.Infof("Checking status of %v, %v", job, lastBuildNumber)
		if err != nil {
			glog.Errorf("Error while getting data for %v: %v", job, err)
			continue
		}
		if checkFinishedStatus(job, lastBuildNumber) {
			continue
		}

		// If we're here it means that build failed, so we need to look for a reason
		// by iterating over junit_XX.xml files and look for failures
		i := 0
		for {
			i++
			path := fmt.Sprintf("artifacts/junit_%02d.xml", i)
			response, err := utils.GetFileFromJenkinsGoogleBucket(job, lastBuildNumber, path)
			if err != nil {
				glog.Errorf("Error while getting data for %v/%v/%v: %v", job, lastBuildNumber, path, err)
				continue
			}
			if response.StatusCode != 200 {
				break
			}
			defer response.Body.Close()
			reader := bufio.NewReader(response.Body)
			body, err := reader.ReadString('\n')
			if err != nil {
				glog.Errorf("Failed to read the response for %v/%v/%v: %v", job, lastBuildNumber, path, err)
				continue
			}
			if strings.TrimSpace(body) != expectedXMLHeader {
				glog.Errorf("Invalid header for %v/%v/%v: %v, expected %v", job, lastBuildNumber, path, body, expectedXMLHeader)
				continue
			}
			body, err = reader.ReadString('\n')
			if err != nil {
				glog.Errorf("Failed to read the response for %v/%v/%v: %v", job, lastBuildNumber, path, err)
				continue
			}
			numberOfTests := 0
			nubmerOfFailures := 0
			timestamp := 0.0
			fmt.Sscanf(strings.TrimSpace(body), "<testsuite tests=\"%d\" failures=\"%d\" time=\"%f\">", &numberOfTests, &nubmerOfFailures, &timestamp)
			glog.V(4).Infof("%v, numberOfTests: %v, numberOfFailures: %v", string(body), numberOfTests, nubmerOfFailures)
			if nubmerOfFailures > 0 {
				glog.V(4).Infof("Found failure in %v for job %v build number %v", path, job, lastBuildNumber)
				return false
			}
		}

		// If we're here it means that we weren't able to find a test that failed, which means that the reason of build failure is comming from the infrastructure
		// Check results of previous two builds.
		if !checkFinishedStatus(job, lastBuildNumber-1) && !checkFinishedStatus(job, lastBuildNumber-2) {
			return false
		}
	}
	return true
}
Пример #4
0
func main() {
	registerFlags(pflag.CommandLine)
	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
	pflag.Parse()

	if leftBuildNumber == 0 || rightBuildNumber == 0 {
		fmt.Fprintf(os.Stderr, "Need both left and right build numbers")
		return
	}

	leftResp, err := utils.GetFileFromJenkinsGoogleBucket(job, leftBuildNumber, buildFilePath)
	if err != nil {
		panic(err)
	}
	leftBody := leftResp.Body
	defer leftBody.Close()
	leftBodyScanner := bufio.NewScanner(leftBody)
	leftLogs, leftResources, leftMetrics := src.ProcessSingleTest(leftBodyScanner, leftBuildNumber)

	rightResp, err := utils.GetFileFromJenkinsGoogleBucket(job, rightBuildNumber, buildFilePath)
	if err != nil {
		panic(err)
	}
	rightBody := rightResp.Body
	defer rightBody.Close()
	rightBodyScanner := bufio.NewScanner(rightBody)
	rightLogs, rightResources, rightMetrics := src.ProcessSingleTest(rightBodyScanner, rightBuildNumber)

	if len(leftLogs) != 0 && len(rightLogs) != 0 {
		for k := range leftLogs {
			if _, ok := rightLogs[k]; !ok {
				fmt.Printf("Right logs missing for test %v\n", k)
				continue
			}
			violatingLogs := src.CompareLogGenerationSpeed(leftLogs[k], rightLogs[k])
			if len(violatingLogs) == 0 {
				continue
			}
			if enableOutputColoring {
				src.ChangeColor(ct.Cyan, os.Stdout)
			}
			fmt.Printf("Differences for test %v", k)
			if enableOutputColoring {
				src.ResetColor(os.Stdout)
			}
			fmt.Print("\n")
			violatingLogs.PrintToStdout(leftBuildNumber, rightBuildNumber, enableOutputColoring)
		}
	}
	fmt.Println("")

	if len(leftResources) != 0 && len(rightResources) != 0 {
		for k := range leftResources {
			if _, ok := rightResources[k]; !ok {
				fmt.Printf("Right resources missing for test %v\n", k)
				continue
			}
			violatingResources := src.CompareResourceUsages(leftResources[k], rightResources[k])
			if len(violatingResources) == 0 {
				continue
			}
			if enableOutputColoring {
				src.ChangeColor(ct.Cyan, os.Stdout)
			}
			fmt.Printf("Differences for test %v", k)
			if enableOutputColoring {
				src.ResetColor(os.Stdout)
			}
			fmt.Print("\n")
			violatingResources.PrintToStdout(leftBuildNumber, rightBuildNumber, enableOutputColoring)
		}
	}
	fmt.Println("")

	if len(leftMetrics) != 0 && len(rightMetrics) != 0 {
		for k := range rightMetrics {
			if _, ok := rightMetrics[k]; !ok {
				fmt.Printf("Right resources missing for test %v\n", k)
				continue
			}
			violatingMetrics := src.CompareMetrics(leftMetrics[k], rightMetrics[k])
			if len(violatingMetrics) == 0 {
				continue
			}
			if enableOutputColoring {
				src.ChangeColor(ct.Cyan, os.Stdout)
			}
			fmt.Printf("Differences for test %v", k)
			if enableOutputColoring {
				src.ResetColor(os.Stdout)
			}
			fmt.Print("\n")
			violatingMetrics.PrintToStdout(leftBuildNumber, rightBuildNumber, enableOutputColoring)
		}
	}
}