func TestCreateTimestampFile(t *testing.T) { realDir := filepath.Join(os.TempDir(), "util_test") util.Mkdir(realDir, 0755) defer util.RemoveAll(realDir) timestampFilePath := filepath.Join(realDir, TIMESTAMP_FILE_NAME) if err := CreateTimestampFile(realDir); err != nil { t.Errorf("Unexpected error: %s", err) } // Assert timestamp file exists. if _, err := os.Stat(timestampFilePath); err != nil { t.Errorf("Timestamp file %s was not created!", timestampFilePath) } // Assert timestamp file contains int64. fileContent, err := ioutil.ReadFile(timestampFilePath) if err != nil { t.Errorf("Unexpected error: %s", err) } if _, err := strconv.ParseInt(string(fileContent), 10, 64); err != nil { t.Errorf("Unexpected value in %s: %s", timestampFilePath, err) } // Assert error returned when specified dir does not exist. nonexistantDir := filepath.Join(os.TempDir(), "util_test_nonexistant") util.RemoveAll(nonexistantDir) if err := CreateTimestampFile(nonexistantDir); err != nil { // Expected error } else { t.Error("Unexpected lack of error") } }
// Will need a local valid google_storage_token.data file with read write access // to run the below test. func Auth_TestDownloadWorkerArtifacts(t *testing.T) { testPagesetsDirName := filepath.Join("unit-tests", "util", "page_sets") client, _ := GetOAuthClient() gs, err := NewGsUtil(client) if err != nil { t.Errorf("Unexpected error: %s", err) } tmpDir := filepath.Join(os.TempDir(), "util_test") StorageDir = tmpDir defer util.RemoveAll(tmpDir) if err := gs.DownloadWorkerArtifacts(testPagesetsDirName, "10k", 1); err != nil { t.Errorf("Unexpected error: %s", err) } // Examine contents of the local directory. localDir := filepath.Join(tmpDir, testPagesetsDirName, "10k") files, err := ioutil.ReadDir(localDir) if err != nil { t.Errorf("Unexpected error: %s", err) } assert.Equal(t, 3, len(files)) assert.Equal(t, "TIMESTAMP", files[0].Name()) assert.Equal(t, "alexa1-1.py", files[1].Name()) assert.Equal(t, "alexa2-2.py", files[2].Name()) }
func testRasterizer(t *testing.T, rasterizer Rasterizer, expectation string) { assert.True(t, rasterizer.Enabled(), "%s.Enabled() failed.", rasterizer.String()) testDataDir, err := testutils.TestDataDir() assert.Nil(t, err, "TestDataDir missing: %v", err) tempDir, err := ioutil.TempDir("", "pdf_test_") assert.Nil(t, err, "ioutil.TempDir failed") defer util.RemoveAll(tempDir) pdfSrcPath := path.Join(testDataDir, "minimal.pdf") assert.True(t, fileutil.FileExists(pdfSrcPath), "Path '%s' does not exist", pdfSrcPath) pdfInputPath := path.Join(tempDir, "minimal.pdf") err = os.Symlink(pdfSrcPath, pdfInputPath) assert.Nil(t, err, "Symlink failed") assert.True(t, fileutil.FileExists(pdfInputPath), "Path '%s' does not exist", pdfInputPath) outputFileName := path.Join(tempDir, "test.png") badPath := path.Join(tempDir, "this_file_should_really_not_exist.pdf") if err := rasterizer.Rasterize(badPath, outputFileName); err == nil { t.Errorf(": Got '%v' Want '%v'", err, nil) } if err := rasterizer.Rasterize(pdfInputPath, outputFileName); err != nil { t.Errorf(": Got '%v' Want '!nil'", err) } expectedOutput := path.Join(testDataDir, expectation) assert.True(t, filesEqual(outputFileName, expectedOutput), "png output not correct") }
// processResult rasterizes a single PDF result and returns a set of new results. func (xformer *pdfXformer) processResult(res goldingester.Result) []*goldingester.Result { rasterizedResults := []*goldingester.Result{} resultMap, found := xformer.results[res.Digest] if found { // Skip rasterizion steps: big win. for index, rasterizer := range xformer.rasterizers { digest, ok := resultMap[index] if ok { rasterizedResults = append(rasterizedResults, newResult(res.Key, rasterizer.String(), digest)) } else { glog.Errorf("missing rasterizer %s on %s", rasterizer.String(), res.Digest) } } return rasterizedResults } tempdir, err := xformer.makeTmpDir() if err != nil { glog.Errorf("error making temp directory: %s", err) return rasterizedResults } defer util.RemoveAll(tempdir) pdfPath := path.Join(tempdir, fmt.Sprintf("%s.pdf", res.Digest)) objectName := fmt.Sprintf("%s/%s.pdf", *storageImagesDirectory, res.Digest) storageURL := fmt.Sprintf("gs://%s/%s", *storageBucket, objectName) object, err := xformer.client.storageService.Objects.Get(*storageBucket, objectName).Do() if err != nil { glog.Errorf("unable to find %s: %s", storageURL, err) return []*goldingester.Result{} } pdfData, _, err := gsFetch(object, xformer.client) if err != nil { glog.Errorf("unable to retrieve %s: %s", storageURL, err) return []*goldingester.Result{} } err = writeTo(pdfPath, &pdfData) if err != nil { glog.Errorf("unable to write file %s: %s", pdfPath, err) return []*goldingester.Result{} } if !isPDF(pdfPath) { glog.Errorf("%s is not a PDF", objectName) return []*goldingester.Result{} } resultMap = map[int]string{} for index, rasterizer := range xformer.rasterizers { digest, err := xformer.rasterizeOnce(pdfPath, index) if err != nil { glog.Errorf("rasterizer %s failed on %s.pdf: %s", rasterizer, res.Digest, err) continue } rasterizedResults = append(rasterizedResults, newResult(res.Key, rasterizer.String(), digest)) resultMap[index] = digest } xformer.results[res.Digest] = resultMap return rasterizedResults }
func TestIngester(t *testing.T) { defer util.RemoveAll(LOCAL_STATUS_DIR) now := time.Now() beginningOfTime := now.Add(-time.Hour * 24 * 10).Unix() const totalCommits = 100 // Instantiate mock VCS and the source. vcs := getVCS(beginningOfTime, now.Unix(), totalCommits) hashes := vcs.From(time.Unix(0, 0)) assert.Equal(t, totalCommits, len(hashes)) for _, h := range hashes { assert.NotEqual(t, "", h) } sources := []Source{MockSource(t, vcs)} // Instantiate the mock processor. collected := map[string]int{} var mutex sync.Mutex processFn := func(result ResultFileLocation) error { mutex.Lock() defer mutex.Unlock() collected[result.Name()] += 1 return nil } finishFn := func() error { return nil } processor := MockProcessor(processFn, finishFn) // Instantiate ingesterConf conf := &sharedconfig.IngesterConfig{ RunEvery: sharedconfig.TomlDuration{Duration: 1 * time.Second}, NCommits: 50, MinDays: 3, StatusDir: LOCAL_STATUS_DIR, } // Instantiate ingester and start it. ingester, err := NewIngester("test-ingester", conf, vcs, sources, processor) assert.Nil(t, err) ingester.Start() // Give it enough time to run a few ingestion cycles and to shut down. time.Sleep(5 * time.Second) ingester.stop() time.Sleep(5 * time.Second) assert.Equal(t, totalCommits/2, len(collected)) for _, count := range collected { assert.Equal(t, 1, count) } }
func TestAreTimestampsEqual(t *testing.T) { gs, err := NewGsUtil(util.NewTimeoutClient()) if err != nil { t.Errorf("Unexpected error: %s", err) } tmpDir := filepath.Join(os.TempDir(), "util_test") util.Mkdir(tmpDir, 0777) defer util.RemoveAll(tmpDir) f, err := os.Create(filepath.Join(tmpDir, TIMESTAMP_FILE_NAME)) if err != nil { t.Errorf("Unexpected error: %s", err) } defer util.Close(f) // Test with matching timestamps. if _, err := f.WriteString(GS_TEST_TIMESTAMP_VALUE); err != nil { t.Errorf("Unexpected error: %s", err) } result1, err := gs.AreTimeStampsEqual(tmpDir, "unit-tests/util/") if err != nil { t.Errorf("Unexpected error: %s", err) } assert.True(t, result1) // Test with differing timestamps. if _, err := f.WriteString(GS_TEST_TIMESTAMP_VALUE); err != nil { t.Errorf("Unexpected error: %s", err) } result2, err := gs.AreTimeStampsEqual(tmpDir, "unit-tests/util/") if err != nil { t.Errorf("Unexpected error: %s", err) } assert.False(t, result2) // Test with Google Storage timestamp missing. result3, err := gs.AreTimeStampsEqual(tmpDir, "unit-tests/util/dummy_name/") if err == nil { t.Error("Expected an error") } assert.False(t, result3) // Test with local timestamp missing. result4, err := gs.AreTimeStampsEqual(tmpDir+"dummy_name", "unit-tests/util/") if err == nil { t.Error("Expected an error") } assert.False(t, result4) }
// downloadAndExecPython downloads a Python script from the Repo, executes it, and // parses its output in JSON format into the given destination. func downloadAndExecPython(r *gitiles.Repo, dst interface{}, srcPath string, workdir string) error { destDir, err := ioutil.TempDir(workdir, "gitiles") if err != nil { return err } defer util.RemoveAll(destDir) destPath := path.Join(destDir, path.Base(srcPath)) if err := r.DownloadFile(srcPath, destPath); err != nil { return err } output, err := exec.Command("python", destPath).Output() if err != nil { return err } if err := json.Unmarshal(output, dst); err != nil { return err } return nil }
func main() { defer common.LogPanic() worker_common.Init() defer util.TimeTrack(time.Now(), "Fixing archives") defer glog.Flush() // Create the task file so that the master knows this worker is still busy. skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_FIXING_ARCHIVES)) defer util.DeleteTaskFile(util.ACTIVITY_FIXING_ARCHIVES) if *pagesetType == "" { glog.Error("Must specify --pageset_type") return } if *chromiumBuild == "" { glog.Error("Must specify --chromium_build") return } if *runID == "" { glog.Error("Must specify --run_id") return } // Reset the local chromium checkout. if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err) return } // Sync the local chromium checkout. if err := util.SyncDir(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err) return } // Instantiate GsUtil object. gs, err := util.NewGsUtil(nil) if err != nil { glog.Error(err) return } // Download the specified chromium build. if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil { glog.Error(err) return } // Delete the chromium build to save space when we are done. defer skutil.RemoveAll(filepath.Join(util.ChromiumBuildsDir, *chromiumBuild)) chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME) // Download pagesets if they do not exist locally. if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType) // Download archives if they do not exist locally. if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } // Establish output paths. localOutputDir := filepath.Join(util.StorageDir, util.FixArchivesRunsDir, *runID) skutil.RemoveAll(localOutputDir) skutil.MkdirAll(localOutputDir, 0700) defer skutil.RemoveAll(localOutputDir) // Construct path to the ct_run_benchmark python script. _, currentFile, _, _ := runtime.Caller(0) pathToPyFiles := filepath.Join( filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))), "py") timeoutSecs := util.PagesetTypeToInfo[*pagesetType].RunChromiumPerfTimeoutSecs fileInfos, err := ioutil.ReadDir(pathToPagesets) if err != nil { glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err) return } // Location of the WPR logs. wprLogs := filepath.Join(util.ChromiumSrcDir, "webpagereplay_logs", "logs.txt") // Slice that will contain inconsistentArchives := []string{} // Loop through all pagesets. for _, fileInfo := range fileInfos { benchmarkResults := []float64{} resourceMissingCounts := []int{} pagesetBaseName := filepath.Base(fileInfo.Name()) if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" { // Ignore timestamp files and .pyc files. continue } // Convert the filename into a format consumable by the run_benchmarks // binary. pagesetName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName)) pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name()) glog.Infof("===== Processing %s =====", pagesetPath) // Repeat runs the specified number of times. for repeatNum := 1; repeatNum <= *repeatBenchmark; repeatNum++ { // Delete webpagereplay_logs before every run. skutil.RemoveAll(wprLogs) skutil.LogErr(os.Chdir(pathToPyFiles)) args := []string{ util.BINARY_RUN_BENCHMARK, fmt.Sprintf("%s.%s", *benchmarkName, util.BenchmarksToTelemetryName[*benchmarkName]), "--page-set-name=" + pagesetName, "--page-set-base-dir=" + pathToPagesets, "--also-run-disabled-tests", } // Add output dir. outputDirArgValue := filepath.Join(localOutputDir, pagesetName, strconv.Itoa(repeatNum)) args = append(args, "--output-dir="+outputDirArgValue) // Figure out which browser should be used. args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary) // Split benchmark args if not empty and append to args. if *benchmarkArgs != "" { for _, benchmarkArg := range strings.Split(*benchmarkArgs, " ") { args = append(args, benchmarkArg) } } // Add browserArgs if not empty to args. if *browserArgs != "" { args = append(args, "--extra-browser-args="+*browserArgs) } // Set the PYTHONPATH to the pagesets and the telemetry dirs. env := []string{ fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), "DISPLAY=:0", } skutil.LogErr( util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil)) // Examine the results-pivot-table.csv file and store the mean frame time. resultsCSV := filepath.Join(outputDirArgValue, "results-pivot-table.csv") // TODO(rmistry): The format has changed from results.csv to results-pivot-table.csv headers, values, err := getRowsFromCSV(resultsCSV) if err != nil { glog.Errorf("Could not read %s: %s", resultsCSV, err) continue } for i := range headers { if headers[i] == *benchmarkHeaderToCheck { value, _ := strconv.ParseFloat(values[i], 64) benchmarkResults = append(benchmarkResults, value) break } } // Find how many times "Could not replay" showed up in wprLogs. content, err := ioutil.ReadFile(wprLogs) if err != nil { glog.Errorf("Could not read %s: %s", wprLogs, err) continue } resourceMissingCount := strings.Count(string(content), "Could not replay") resourceMissingCounts = append(resourceMissingCounts, resourceMissingCount) } glog.Infof("Benchmark results for %s are: %v", fileInfo.Name(), benchmarkResults) percentageChange := getPercentageChange(benchmarkResults) glog.Infof("Percentage change of results is: %f", percentageChange) glog.Infof("\"Could not replay\" showed up %v times in %s", resourceMissingCounts, wprLogs) maxResourceMissingCount := 0 for _, count := range resourceMissingCounts { if maxResourceMissingCount < count { maxResourceMissingCount = count } } if percentageChange > *percentageChangeThreshold || maxResourceMissingCount > *resourceMissingCountThreshold { glog.Infof("The archive for %s is inconsistent!", fileInfo.Name()) inconsistentArchives = append(inconsistentArchives, fmt.Sprintf("%s percentageChange: %f maxResourceMissingCount: %v", fileInfo.Name(), percentageChange, maxResourceMissingCount)) if *deletePageset { // Delete the pageset. skutil.RemoveAll(pagesetPath) } } } if len(inconsistentArchives) > 0 { glog.Infof("%d archives are inconsistent!", len(inconsistentArchives)) glog.Infof("The list of inconsistentArchives is: %v", inconsistentArchives) if *deletePageset { // Write new timestamp to the pagesets dir. skutil.RemoveAll(filepath.Join(pathToPagesets, util.TIMESTAMP_FILE_NAME)) skutil.LogErr(util.CreateTimestampFile(pathToPagesets)) // Inconsistent pagesets were deleted locally. Upload local pagesets dir // to Google Storage. if err := gs.UploadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } } } }
func main() { defer common.LogPanic() common.Init() defer util.TimeTrack(time.Now(), "Creating Pagesets") defer glog.Flush() // Create the task file so that the master knows this worker is still busy. skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CREATING_PAGESETS)) defer util.DeleteTaskFile(util.ACTIVITY_CREATING_PAGESETS) // Delete and remake the local pagesets directory. pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType) skutil.RemoveAll(pathToPagesets) skutil.MkdirAll(pathToPagesets, 0700) // Get info about the specified pageset type. pagesetTypeInfo := util.PagesetTypeToInfo[*pagesetType] csvSource := pagesetTypeInfo.CSVSource numPages := pagesetTypeInfo.NumPages userAgent := pagesetTypeInfo.UserAgent // Download the CSV file from Google Storage to a tmp location. gs, err := util.NewGsUtil(nil) if err != nil { glog.Error(err) return } respBody, err := gs.GetRemoteFileContents(csvSource) if err != nil { glog.Error(err) return } defer skutil.Close(respBody) csvFile := filepath.Join(os.TempDir(), filepath.Base(csvSource)) out, err := os.Create(csvFile) if err != nil { glog.Errorf("Unable to create file %s: %s", csvFile, err) return } defer skutil.Close(out) defer skutil.Remove(csvFile) if _, err = io.Copy(out, respBody); err != nil { glog.Error(err) return } // Figure out which pagesets this worker should generate. numPagesPerSlave := numPages / util.NUM_WORKERS startNum := (*workerNum-1)*numPagesPerSlave + 1 endNum := *workerNum * numPagesPerSlave // Construct path to the create_page_set.py python script. _, currentFile, _, _ := runtime.Caller(0) createPageSetScript := filepath.Join( filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))), "py", "create_page_set.py") // Execute the create_page_set.py python script. timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CreatePagesetsTimeoutSecs for currNum := startNum; currNum <= endNum; currNum++ { args := []string{ createPageSetScript, "-s", strconv.Itoa(currNum), "-e", strconv.Itoa(currNum), "-c", csvFile, "-p", *pagesetType, "-u", userAgent, "-o", pathToPagesets, } if err := util.ExecuteCmd("python", args, []string{}, time.Duration(timeoutSecs)*time.Second, nil, nil); err != nil { glog.Error(err) return } } // Write timestamp to the pagesets dir. skutil.LogErr(util.CreateTimestampFile(pathToPagesets)) // Upload pagesets dir to Google Storage. if err := gs.UploadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } }
// runID is the unique id of the current run (typically requester + timestamp). // targetPlatform is the platform the benchmark will run on (Android / Linux ). // chromiumHash is the hash the checkout should be synced to. If not specified then // Chromium's Tot hash is used. // skiaHash is the hash the checkout should be synced to. If not specified then // Skia's LKGR hash is used (the hash in Chromium's DEPS file). // applyPatches if true looks for Chromium/Blink/Skia patches in the temp dir and // runs once with the patch applied and once without the patch applied. func CreateChromiumBuild(runID, targetPlatform, chromiumHash, skiaHash string, applyPatches bool) (string, string, error) { // Determine which build dir and fetch target to use. var chromiumBuildDir, fetchTarget string if targetPlatform == "Android" { chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "android_base") fetchTarget = "android" } else if targetPlatform == "Linux" { chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "linux_base") fetchTarget = "chromium" } else { return "", "", fmt.Errorf("Unrecognized target_platform %s", targetPlatform) } util.MkdirAll(chromiumBuildDir, 0700) // Find which Chromium commit hash should be used. var err error if chromiumHash == "" { chromiumHash, err = getChromiumHash() if err != nil { return "", "", fmt.Errorf("Error while finding Chromium's Hash: %s", err) } } // Find which Skia commit hash should be used. if skiaHash == "" { skiaHash, err = getSkiaHash() if err != nil { return "", "", fmt.Errorf("Error while finding Skia's Hash: %s", err) } } // Run chromium sync command using the above commit hashes. // Construct path to the sync_skia_in_chrome python script. _, currentFile, _, _ := runtime.Caller(0) pathToPyFiles := filepath.Join( filepath.Dir((filepath.Dir(filepath.Dir(currentFile)))), "py") syncArgs := []string{ filepath.Join(pathToPyFiles, "sync_skia_in_chrome.py"), "--destination=" + chromiumBuildDir, "--fetch_target=" + fetchTarget, "--chrome_revision=" + chromiumHash, "--skia_revision=" + skiaHash, } if err := ExecuteCmd("python", syncArgs, []string{}, 2*time.Hour, nil, nil); err != nil { glog.Warning("There was an error. Deleting base directory and trying again.") util.RemoveAll(chromiumBuildDir) util.MkdirAll(chromiumBuildDir, 0700) if err := ExecuteCmd("python", syncArgs, []string{}, 2*time.Hour, nil, nil); err != nil { return "", "", fmt.Errorf("There was an error checking out chromium %s + skia %s: %s", chromiumHash, skiaHash, err) } } // Make sure we are starting from a clean slate. if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil { return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err) } googleStorageDirName := fmt.Sprintf("%s-%s-%s", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), runID) if runID == "" { // Do not include the runID in the dir name if it is not specified. googleStorageDirName = fmt.Sprintf("%s-%s", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash)) } if applyPatches { if err := applyRepoPatches(filepath.Join(chromiumBuildDir, "src"), runID); err != nil { return "", "", fmt.Errorf("Could not apply patches in the chromium checkout in %s: %s", chromiumBuildDir, err) } // Add "try" prefix and "withpatch" suffix. googleStorageDirName = fmt.Sprintf("try-%s-withpatch", googleStorageDirName) } // Build chromium. if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil { return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err) } // Upload to Google Storage. gs, err := NewGsUtil(nil) if err != nil { return "", "", fmt.Errorf("Could not create GS object: %s", err) } if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil { return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err) } // Check for the applypatch flag and reset and then build again and copy to // google storage. if applyPatches { // Now build chromium without the patches and upload it to Google Storage. // Make sure we are starting from a clean slate. if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil { return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err) } // Build chromium. if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil { return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err) } // Upload to Google Storage. googleStorageDirName = fmt.Sprintf("try-%s-%s-%s-nopatch", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), runID) if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil { return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err) } } return getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), nil }
func main() { common.Init() webhook.MustInitRequestSaltFromFile(util.WebhookRequestSaltPath) // Send start email. emailsArr := util.ParseEmails(*emails) emailsArr = append(emailsArr, util.CtAdmins...) if len(emailsArr) == 0 { glog.Error("At least one email address must be specified") return } skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Skia correctness")) // Ensure webapp is updated and email is sent even if task fails. defer updateWebappTask() defer sendEmail(emailsArr) // Cleanup tmp files after the run. defer util.CleanTmpDir() // Finish with glog flush and how long the task took. defer util.TimeTrack(time.Now(), "Running skia correctness task on workers") defer glog.Flush() if *pagesetType == "" { glog.Error("Must specify --pageset_type") return } if *chromiumBuild == "" { glog.Error("Must specify --chromium_build") return } if *runID == "" { glog.Error("Must specify --run_id") return } // Instantiate GsUtil object. gs, err := util.NewGsUtil(nil) if err != nil { glog.Error(err) return } remoteOutputDir := filepath.Join(util.SkiaCorrectnessRunsDir, *runID) // Copy the patch to Google Storage. patchName := *runID + ".patch" patchRemoteDir := filepath.Join(remoteOutputDir, "patches") if err := gs.UploadFile(patchName, os.TempDir(), patchRemoteDir); err != nil { glog.Errorf("Could not upload %s to %s: %s", patchName, patchRemoteDir, err) return } skiaPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, patchRemoteDir, patchName) // Run the run_skia_correctness script on all workers. runSkiaCorrCmdTemplate := "DISPLAY=:0 run_skia_correctness --worker_num={{.WorkerNum}} --log_dir={{.LogDir}} " + "--pageset_type={{.PagesetType}} --chromium_build={{.ChromiumBuild}} --run_id={{.RunID}} " + "--render_pictures_args=\"{{.RenderPicturesArgs}}\" --gpu_nopatch_run={{.GpuNoPatchRun}} " + "--gpu_withpatch_run={{.GpuWithPatchRun}};" runSkiaCorrTemplateParsed := template.Must(template.New("run_skia_correctness_cmd").Parse(runSkiaCorrCmdTemplate)) runSkiaCorrCmdBytes := new(bytes.Buffer) if err := runSkiaCorrTemplateParsed.Execute(runSkiaCorrCmdBytes, struct { WorkerNum string LogDir string PagesetType string ChromiumBuild string RunID string RenderPicturesArgs string GpuNoPatchRun string GpuWithPatchRun string }{ WorkerNum: util.WORKER_NUM_KEYWORD, LogDir: util.GLogDir, PagesetType: *pagesetType, ChromiumBuild: *chromiumBuild, RunID: *runID, RenderPicturesArgs: *renderPicturesArgs, GpuNoPatchRun: strconv.FormatBool(*gpuNoPatchRun), GpuWithPatchRun: strconv.FormatBool(*gpuWithPatchRun), }); err != nil { glog.Errorf("Failed to execute template: %s", err) return } cmd := []string{ fmt.Sprintf("cd %s;", util.CtTreeDir), "git pull;", "make all;", // The main command that runs run_skia_correctness on all workers. runSkiaCorrCmdBytes.String(), } if _, err := util.SSH(strings.Join(cmd, " "), util.Slaves, 4*time.Hour); err != nil { glog.Errorf("Error while running cmd %s: %s", cmd, err) return } localOutputDir := filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir, *runID) localSummariesDir := filepath.Join(localOutputDir, "summaries") skutil.MkdirAll(localSummariesDir, 0700) defer skutil.RemoveAll(filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir)) // Copy outputs from all slaves locally. for i := 0; i < util.NUM_WORKERS; i++ { workerNum := i + 1 workerLocalOutputPath := filepath.Join(localSummariesDir, fmt.Sprintf("slave%d", workerNum)+".json") workerRemoteOutputPath := filepath.Join(remoteOutputDir, fmt.Sprintf("slave%d", workerNum), fmt.Sprintf("slave%d", workerNum)+".json") respBody, err := gs.GetRemoteFileContents(workerRemoteOutputPath) if err != nil { glog.Errorf("Could not fetch %s: %s", workerRemoteOutputPath, err) // TODO(rmistry): Should we instead return here? We can only return // here if all 100 slaves reliably run without any failures which they // really should. continue } defer skutil.Close(respBody) out, err := os.Create(workerLocalOutputPath) if err != nil { glog.Errorf("Unable to create file %s: %s", workerLocalOutputPath, err) return } defer skutil.Close(out) defer skutil.Remove(workerLocalOutputPath) if _, err = io.Copy(out, respBody); err != nil { glog.Errorf("Unable to copy to file %s: %s", workerLocalOutputPath, err) return } } // Call json_summary_combiner.py to merge all results into a single results CSV. _, currentFile, _, _ := runtime.Caller(0) pathToPyFiles := filepath.Join( filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))), "py") pathToJsonCombiner := filepath.Join(pathToPyFiles, "json_summary_combiner.py") localHtmlDir := filepath.Join(localOutputDir, "html") remoteHtmlDir := filepath.Join(remoteOutputDir, "html") baseHtmlLink := util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteHtmlDir) + "/" htmlOutputLink = baseHtmlLink + "index.html" skutil.MkdirAll(localHtmlDir, 0700) args := []string{ pathToJsonCombiner, "--json_summaries_dir=" + localSummariesDir, "--output_html_dir=" + localHtmlDir, "--absolute_url=" + baseHtmlLink, "--render_pictures_args=" + *renderPicturesArgs, "--nopatch_gpu=" + strconv.FormatBool(*gpuNoPatchRun), "--withpatch_gpu=" + strconv.FormatBool(*gpuWithPatchRun), } if err := util.ExecuteCmd("python", args, []string{}, 1*time.Hour, nil, nil); err != nil { glog.Errorf("Error running json_summary_combiner.py: %s", err) return } // Copy the HTML files to Google Storage. if err := gs.UploadDir(localHtmlDir, remoteHtmlDir, true); err != nil { glog.Errorf("Could not upload %s to %s: %s", localHtmlDir, remoteHtmlDir, err) return } taskCompletedSuccessfully = true }
func (p *pdfProcessor) rasterizeAndUpload(dmResultName string, dmResults *goldingestion.DMResults, pdfResults []*goldingestion.Result) error { processedResults := make([]*goldingestion.Result, 0, len(pdfResults)*len(p.rasterizers)) // Create a temporary directory to hold the rastered images. tempDir, err := ioutil.TempDir(p.pdfCacheDir, "pdfingestion") if err != nil { return err } defer util.RemoveAll(tempDir) // Go through all results that generated PDF files. for resultIdx, result := range pdfResults { // Fetch the PDF file if it's not in the cache. pdfFileName := fmt.Sprintf("%s.%s", result.Digest, PDF_EXT) pdfPath := filepath.Join(p.pdfCacheDir, pdfFileName) if !fileutil.FileExists(pdfPath) { if err = p.download(p.inImagesBucket, p.inImagesDir, pdfFileName, pdfPath); err != nil { glog.Errorf("Unable to retrieve image: %s. Error: %s", pdfFileName, err) continue } } // Generate an image for each rasterizer. for rasterIdx, rasterizer := range p.rasterizers { tempName := filepath.Join(tempDir, fmt.Sprintf("rastering_%d_%d.%s", resultIdx, rasterIdx, PNG_EXT)) err := rasterizer.Rasterize(pdfPath, tempName) if err != nil { glog.Errorf("Rasterizing %s with %s failed: %s", filepath.Base(pdfPath), rasterizer.String(), err) continue } // Open the generated image and calculate the MD5. file, err := os.Open(tempName) if err != nil { glog.Errorf("Unable to open generated image: %s", err) continue } var buf bytes.Buffer md5, err := util.MD5FromReader(file, &buf) if err != nil { glog.Errorf("Unable to calculate MD5 hash of file %s. Got error: %s", tempName, err) continue } digest := hex.EncodeToString(md5) uploadFileName := fmt.Sprintf("%s.%s", digest, PNG_EXT) if err := p.upload(p.outImagesBucket, p.outImagesDir, uploadFileName, bytes.NewBuffer(buf.Bytes())); err != nil { glog.Errorf("Unable to upload file %s. Error: %s", uploadFileName, err) continue } // Update the result and add it to the successfully processed results. result.Key["rasterizer"] = rasterizer.String() result.Digest = digest result.Options["ext"] = PNG_EXT processedResults = append(processedResults, result) } } // If we have no processed results we consider it an error. if len(processedResults) == 0 { return fmt.Errorf("No input image was processed successfully.") } // Replace the old results in the original result and write it to the cloud. dmResults.Results = processedResults jsonBytes, err := json.MarshalIndent(dmResults, "", " ") if err != nil { return fmt.Errorf("Unable to encode JSON: %s", err) } return p.upload(p.outJsonBucket, p.outJsonDir, dmResultName, bytes.NewBuffer(jsonBytes)) }
func main() { defer common.LogPanic() worker_common.Init() if !*worker_common.Local { defer util.CleanTmpDir() } defer util.TimeTrack(time.Now(), "Capturing SKPs") defer glog.Flush() // Validate required arguments. if *chromiumBuild == "" { glog.Error("Must specify --chromium_build") return } if *runID == "" { glog.Error("Must specify --run_id") return } if *targetPlatform == util.PLATFORM_ANDROID { glog.Error("Android is not yet supported for capturing SKPs.") return } // Reset the local chromium checkout. if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err) return } // Sync the local chromium checkout. if err := util.SyncDir(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err) return } // Create the task file so that the master knows this worker is still busy. skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_SKPS)) defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_SKPS) // Instantiate GsUtil object. gs, err := util.NewGsUtil(nil) if err != nil { glog.Error(err) return } // Download the specified chromium build. if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil { glog.Error(err) return } // Delete the chromium build to save space when we are done. defer skutil.RemoveAll(filepath.Join(util.ChromiumBuildsDir, *chromiumBuild)) chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME) if *targetPlatform == util.PLATFORM_ANDROID { // Install the APK on the Android device. if err := util.InstallChromeAPK(*chromiumBuild); err != nil { glog.Errorf("Could not install the chromium APK: %s", err) return } } // Download pagesets if they do not exist locally. if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType) // Download archives if they do not exist locally. if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } // Create the dir that SKPs will be stored in. pathToSkps := filepath.Join(util.SkpsDir, *pagesetType, *chromiumBuild) // Delete and remake the local SKPs directory. skutil.RemoveAll(pathToSkps) skutil.MkdirAll(pathToSkps, 0700) // Establish output paths. localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID) skutil.RemoveAll(localOutputDir) skutil.MkdirAll(localOutputDir, 0700) defer skutil.RemoveAll(localOutputDir) // Construct path to the ct_run_benchmark python script. _, currentFile, _, _ := runtime.Caller(0) pathToPyFiles := filepath.Join( filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))), "py") timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureSKPsTimeoutSecs fileInfos, err := ioutil.ReadDir(pathToPagesets) if err != nil { glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err) return } // Create channel that contains all pageset file names. This channel will // be consumed by the worker pool. pagesetRequests := util.GetClosedChannelOfPagesets(fileInfos) var wg sync.WaitGroup // Use a RWMutex for the chromeProcessesCleaner goroutine to communicate to // the workers (acting as "readers") when it wants to be the "writer" and // kill all zombie chrome processes. var mutex sync.RWMutex // Loop through workers in the worker pool. for i := 0; i < WORKER_POOL_SIZE; i++ { // Increment the WaitGroup counter. wg.Add(1) // Create and run a goroutine closure that captures SKPs. go func() { // Decrement the WaitGroup counter when the goroutine completes. defer wg.Done() for pagesetName := range pagesetRequests { mutex.RLock() // Read the pageset. pagesetPath := filepath.Join(pathToPagesets, pagesetName) decodedPageset, err := util.ReadPageset(pagesetPath) if err != nil { glog.Errorf("Could not read %s: %s", pagesetPath, err) continue } glog.Infof("===== Processing %s =====", pagesetPath) skutil.LogErr(os.Chdir(pathToPyFiles)) args := []string{ filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK), util.BenchmarksToTelemetryName[util.BENCHMARK_SKPICTURE_PRINTER], "--also-run-disabled-tests", "--page-repeat=1", // Only need one run for SKPs. "--skp-outdir=" + pathToSkps, "--extra-browser-args=" + util.DEFAULT_BROWSER_ARGS, "--user-agent=" + decodedPageset.UserAgent, "--urls-list=" + decodedPageset.UrlsList, "--archive-data-file=" + decodedPageset.ArchiveDataFile, } // Figure out which browser should be used. if *targetPlatform == util.PLATFORM_ANDROID { args = append(args, "--browser=android-chromium") } else { args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary) } // Set the PYTHONPATH to the pagesets and the telemetry dirs. env := []string{ fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), "DISPLAY=:0", } skutil.LogErr( util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil)) mutex.RUnlock() } }() } if !*worker_common.Local { // Start the cleaner. go util.ChromeProcessesCleaner(&mutex, *chromeCleanerTimer) } // Wait for all spawned goroutines to complete. wg.Wait() // Move, validate and upload all SKP files. // List all directories in pathToSkps and copy out the skps. skpFileInfos, err := ioutil.ReadDir(pathToSkps) if err != nil { glog.Errorf("Unable to read %s: %s", pathToSkps, err) return } for _, fileInfo := range skpFileInfos { if !fileInfo.IsDir() { // We are only interested in directories. continue } skpName := fileInfo.Name() // Find the largest layer in this directory. layerInfos, err := ioutil.ReadDir(filepath.Join(pathToSkps, skpName)) if err != nil { glog.Errorf("Unable to read %s: %s", filepath.Join(pathToSkps, skpName), err) } if len(layerInfos) > 0 { largestLayerInfo := layerInfos[0] for _, layerInfo := range layerInfos { fmt.Println(layerInfo.Size()) if layerInfo.Size() > largestLayerInfo.Size() { largestLayerInfo = layerInfo } } // Only save SKPs greater than 6000 bytes. Less than that are probably // malformed. if largestLayerInfo.Size() > 6000 { layerPath := filepath.Join(pathToSkps, skpName, largestLayerInfo.Name()) skutil.Rename(layerPath, filepath.Join(pathToSkps, skpName+".skp")) } else { glog.Warningf("Skipping %s because size was less than 5000 bytes", skpName) } } // We extracted what we needed from the directory, now delete it. skutil.RemoveAll(filepath.Join(pathToSkps, skpName)) } glog.Info("Calling remove_invalid_skps.py") // Sync Skia tree. skutil.LogErr(util.SyncDir(util.SkiaTreeDir)) // Build tools. skutil.LogErr(util.BuildSkiaTools()) // Run remove_invalid_skps.py pathToRemoveSKPs := filepath.Join(pathToPyFiles, "remove_invalid_skps.py") pathToSKPInfo := filepath.Join(util.SkiaTreeDir, "out", "Release", "skpinfo") args := []string{ pathToRemoveSKPs, "--skp_dir=" + pathToSkps, "--path_to_skpinfo=" + pathToSKPInfo, } skutil.LogErr(util.ExecuteCmd("python", args, []string{}, util.REMOVE_INVALID_SKPS_TIMEOUT, nil, nil)) // Write timestamp to the SKPs dir. skutil.LogErr(util.CreateTimestampFile(pathToSkps)) // Upload SKPs dir to Google Storage. if err := gs.UploadWorkerArtifacts(util.SKPS_DIR_NAME, filepath.Join(*pagesetType, *chromiumBuild), *workerNum); err != nil { glog.Error(err) return } }
func main() { common.Init() defer util.TimeTrack(time.Now(), "Running Skia Correctness") defer glog.Flush() if *chromiumBuild == "" { glog.Error("Must specify --chromium_build") return } if *runID == "" { glog.Error("Must specify --run_id") return } // Create the task file so that the master knows this worker is still busy. skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_RUNNING_SKIA_CORRECTNESS)) defer util.DeleteTaskFile(util.ACTIVITY_RUNNING_SKIA_CORRECTNESS) // Establish output paths. localOutputDir := filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir, *runID) skutil.RemoveAll(filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir)) skutil.MkdirAll(localOutputDir, 0700) defer skutil.RemoveAll(localOutputDir) remoteOutputDir := filepath.Join(util.SkiaCorrectnessRunsDir, *runID, fmt.Sprintf("slave%d", *workerNum)) // Instantiate GsUtil object. gs, err := util.NewGsUtil(nil) if err != nil { glog.Error(err) return } // Download SKPs if they do not exist locally. if err := gs.DownloadWorkerArtifacts(util.SKPS_DIR_NAME, filepath.Join(*pagesetType, *chromiumBuild), *workerNum); err != nil { glog.Error(err) return } localSkpsDir := filepath.Join(util.SkpsDir, *pagesetType, *chromiumBuild) // Download the Skia patch for this run from Google storage. patchName := *runID + ".patch" patchLocalPath := filepath.Join(os.TempDir(), patchName) remoteDir := filepath.Join(util.SkiaCorrectnessRunsDir, *runID) patchRemotePath := filepath.Join(remoteDir, "patches", patchName) respBody, err := gs.GetRemoteFileContents(patchRemotePath) if err != nil { glog.Errorf("Could not fetch %s: %s", patchRemotePath, err) return } defer skutil.Close(respBody) out, err := os.Create(patchLocalPath) if err != nil { glog.Errorf("Unable to create file %s: %s", patchLocalPath, err) return } defer skutil.Close(out) defer skutil.Remove(patchLocalPath) if _, err = io.Copy(out, respBody); err != nil { glog.Error(err) return } // Apply the patch to a clean checkout and run render_pictures. // Reset Skia tree. skutil.LogErr(util.ResetCheckout(util.SkiaTreeDir)) // Sync Skia tree. skutil.LogErr(util.SyncDir(util.SkiaTreeDir)) // Apply Skia patch. file, _ := os.Open(patchLocalPath) fileInfo, _ := file.Stat() // It is a valid patch only if it is more than 10 bytes. if fileInfo.Size() > 10 { glog.Info("Attempting to apply %s to %s", patchLocalPath, util.SkiaTreeDir) if err := util.ApplyPatch(patchLocalPath, util.SkiaTreeDir); err != nil { glog.Errorf("Could not apply patch %s to %s: %s", patchLocalPath, util.SkiaTreeDir, err) return } glog.Info("Patch successfully applied") } else { glog.Info("Patch is empty or invalid. Skipping the patch.") } // Build tools. skutil.LogErr(util.BuildSkiaTools()) // Run render_pictures. if err := runRenderPictures(localSkpsDir, filepath.Join(localOutputDir, "withpatch"), filepath.Join(remoteOutputDir, "withpatch"), *gpuWithPatchRun); err != nil { glog.Errorf("Error while running withpatch render_pictures: %s", err) return } // Remove the patch and run render_pictures. // Reset Skia tree. skutil.LogErr(util.ResetCheckout(util.SkiaTreeDir)) // Build tools. skutil.LogErr(util.BuildSkiaTools()) // Run render_pictures. if err := runRenderPictures(localSkpsDir, filepath.Join(localOutputDir, "nopatch"), filepath.Join(remoteOutputDir, "nopatch"), *gpuNoPatchRun); err != nil { glog.Errorf("Error while running nopatch render_pictures: %s", err) return } // Comparing pictures and saving differences in JSON output file. jsonSummaryDir := filepath.Join(localOutputDir, "json_summary") skutil.MkdirAll(jsonSummaryDir, 0700) jsonSummaryPath := filepath.Join(jsonSummaryDir, fmt.Sprintf("slave%d", *workerNum)+".json") // Construct path to the write_json_summary python script. _, currentFile, _, _ := runtime.Caller(0) pathToPyFiles := filepath.Join( filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))), "py") summaryArgs := []string{ filepath.Join(pathToPyFiles, "write_json_summary.py"), "--img_root=" + localOutputDir, "--withpatch_json=" + filepath.Join(localOutputDir, "withpatch", "summary.json"), "--withpatch_images_base_url=file://" + filepath.Join(localOutputDir, "withpatch"), "--nopatch_json=" + filepath.Join(localOutputDir, "nopatch", "summary.json"), "--nopatch_images_base_url=file://" + filepath.Join(localOutputDir, "nopatch"), "--output_file_path=" + jsonSummaryPath, "--gs_output_dir=gs://" + filepath.Join(util.GS_BUCKET_NAME, remoteDir), "--gs_skp_dir=gs://" + filepath.Join(util.GS_BUCKET_NAME, util.SKPS_DIR_NAME, *pagesetType, *chromiumBuild, fmt.Sprintf("slave%d", *workerNum)), "--slave_num=" + strconv.Itoa(*workerNum), } if err := util.ExecuteCmd("python", summaryArgs, []string{}, 15*time.Minute, nil, nil); err != nil { glog.Error(err) return } // Upload artifacts to Google Storage. // Get list of failed file names and upload only those to Google Storage. // Read the JSON file. summaryFile, err := ioutil.ReadFile(jsonSummaryPath) if err != nil { glog.Errorf("Unable to read %s: %s", jsonSummaryPath, err) } var jsontype map[string]interface{} skutil.LogErr(json.Unmarshal(summaryFile, &jsontype)) if jsontype[fmt.Sprintf("slave%d", *workerNum)] != nil { failedFiles := jsontype[fmt.Sprintf("slave%d", *workerNum)].(map[string]interface{})["failedFiles"].([]interface{}) for i := range failedFiles { failedFile := failedFiles[i].(map[string]interface{})["fileName"].(string) // TODO(rmistry): Use goroutines to do the below in parallel. skutil.LogErr( gs.UploadFile(failedFile, filepath.Join(localOutputDir, "withpatch"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "withpatch-images"))) skutil.LogErr( gs.UploadFile(failedFile, filepath.Join(localOutputDir, "nopatch"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "nopatch-images"))) } // Copy the diffs and whitediffs to Google Storage. skutil.LogErr( gs.UploadDir(filepath.Join(localOutputDir, "diffs"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "diffs"), true)) skutil.LogErr( gs.UploadDir(filepath.Join(localOutputDir, "whitediffs"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "whitediffs"), true)) } // Upload the summary file. skutil.LogErr( gs.UploadFile(fmt.Sprintf("slave%d", *workerNum)+".json", jsonSummaryDir, filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum)))) }
// downloadRemoteDir downloads the specified Google Storage dir to the specified // local dir. The local dir will be emptied and recreated. Handles multiple levels // of directories. func (gs *GsUtil) downloadRemoteDir(localDir, gsDir string) error { // Empty the local dir. util.RemoveAll(localDir) // Create the local dir. util.MkdirAll(localDir, 0700) // The channel where the storage objects to be deleted will be sent to. chStorageObjects := make(chan filePathToStorageObject, MAX_CHANNEL_SIZE) req := gs.service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/") for req != nil { resp, err := req.Do() if err != nil { return fmt.Errorf("Error occured while listing %s: %s", gsDir, err) } for _, result := range resp.Items { fileName := filepath.Base(result.Name) // If downloading from subdir then add it to the fileName. fileGsDir := filepath.Dir(result.Name) subDirs := strings.TrimPrefix(fileGsDir, gsDir) if subDirs != "" { dirTokens := strings.Split(subDirs, "/") for i := range dirTokens { fileName = filepath.Join(dirTokens[len(dirTokens)-i-1], fileName) } // Create the local directory. util.MkdirAll(filepath.Join(localDir, filepath.Dir(fileName)), 0700) } chStorageObjects <- filePathToStorageObject{storageObject: result, filePath: fileName} } if len(resp.NextPageToken) > 0 { req.PageToken(resp.NextPageToken) } else { req = nil } } close(chStorageObjects) // Kick off goroutines to download the storage objects. var wg sync.WaitGroup for i := 0; i < GOROUTINE_POOL_SIZE; i++ { wg.Add(1) go func(goroutineNum int) { defer wg.Done() for obj := range chStorageObjects { result := obj.storageObject filePath := obj.filePath respBody, err := getRespBody(result, gs.client) if err != nil { glog.Errorf("Could not fetch %s: %s", result.MediaLink, err) return } defer util.Close(respBody) outputFile := filepath.Join(localDir, filePath) out, err := os.Create(outputFile) if err != nil { glog.Errorf("Unable to create file %s: %s", outputFile, err) return } defer util.Close(out) if _, err = io.Copy(out, respBody); err != nil { glog.Error(err) return } glog.Infof("Downloaded gs://%s/%s to %s with goroutine#%d", GS_BUCKET_NAME, result.Name, outputFile, goroutineNum) // Sleep for a second after downloading file to avoid bombarding Cloud // storage. time.Sleep(time.Second) } }(i + 1) } wg.Wait() return nil }
func main() { common.Init() defer util.TimeTrack(time.Now(), "Capturing Archives") defer glog.Flush() // Create the task file so that the master knows this worker is still busy. skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES)) defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES) if *chromiumBuild == "" { glog.Error("Must specify --chromium_build") return } // Reset the local chromium checkout. if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err) return } // Sync the local chromium checkout. if err := util.SyncDir(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err) return } // Delete and remake the local webpage archives directory. pathToArchives := filepath.Join(util.WebArchivesDir, *pagesetType) skutil.RemoveAll(pathToArchives) skutil.MkdirAll(pathToArchives, 0700) // Instantiate GsUtil object. gs, err := util.NewGsUtil(nil) if err != nil { glog.Error(err) return } // Download the specified chromium build if it does not exist locally. if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil { glog.Error(err) return } // Download pagesets if they do not exist locally. if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType) chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME) recordWprBinary := filepath.Join(util.TelemetryBinariesDir, util.BINARY_RECORD_WPR) timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureArchivesTimeoutSecs // Loop through all pagesets. fileInfos, err := ioutil.ReadDir(pathToPagesets) if err != nil { glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err) return } // TODO(rmistry): Remove this hack once the 1M webpage archives have been captured. glog.Infof("The length of fileInfos is: %s", len(fileInfos)) fileInfos = fileInfos[18500:20000] glog.Infof("The fileInfos are: %s", fileInfos) for _, fileInfo := range fileInfos { pagesetBaseName := filepath.Base(fileInfo.Name()) if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" { // Ignore timestamp files and .pyc files. continue } // Convert the filename into a format consumable by the record_wpr binary. pagesetArchiveName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName)) pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name()) glog.Infof("===== Processing %s =====", pagesetPath) args := []string{ "--extra-browser-args=--disable-setuid-sandbox", "--browser=exact", "--browser-executable=" + chromiumBinary, fmt.Sprintf("%s_page_set", pagesetArchiveName), "--page-set-base-dir=" + pathToPagesets, } env := []string{ fmt.Sprintf("PYTHONPATH=%s:$PYTHONPATH", pathToPagesets), "DISPLAY=:0", } skutil.LogErr(util.ExecuteCmd(recordWprBinary, args, env, time.Duration(timeoutSecs)*time.Second, nil, nil)) } // Write timestamp to the webpage archives dir. skutil.LogErr(util.CreateTimestampFile(pathToArchives)) // Upload webpage archives dir to Google Storage. if err := gs.UploadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } }
func TestInfo(t *testing.T) { levelDBDir := filepath.Join(os.TempDir(), "android-leveldb") defer util.RemoveAll(levelDBDir) db, err := leveldb.OpenFile(levelDBDir, nil) if err != nil { t.Fatalf("Failed to create test leveldb: %s", err) } i := &info{ db: db, commits: mockCommits{}, } if got, want := i.branchtargets(), []string{}; !util.SSliceEqual(got, want) { t.Errorf("Wrong targets: Got %v Want %v", got, want) } i.single_poll() // The first time we Get on an unknown target well get nil, err. commit, err := i.Get("git_master-skia", "razor-userdebug", "100") assert.Nil(t, commit) assert.NotNil(t, err) // After the first time we'll get an error. commit, err = i.Get("git_master-skia", "razor-userdebug", "100") assert.Nil(t, commit) assert.NotNil(t, err) // Now poll. i.single_poll() // Now the commits should be populated so the Get should succeed. commit, err = i.Get("git_master-skia", "razor-userdebug", "100") assert.Nil(t, err) assert.NotNil(t, commit) if got, want := commit.Hash, "1234567890"; got != want { t.Errorf("Wrong commit returned: Got %v Want %v", got, want) } commit, err = i.Get("git_master-skia", "razor-userdebug", "101") assert.Nil(t, err) assert.NotNil(t, commit) if got, want := commit.Hash, "1234567890"; got != want { t.Errorf("Wrong commit returned: Got %v Want %v", got, want) } commit, err = i.Get("git_master-skia", "razor-userdebug", "103") assert.Nil(t, err) assert.NotNil(t, commit) if got, want := commit.Hash, "987654321"; got != want { t.Errorf("Wrong commit returned: Got %v Want %v", got, want) } commit, err = i.Get("git_master-skia", "razor-userdebug", "99") assert.Nil(t, commit) assert.NotNil(t, err) // Now add another target. // The first time we Get on an unknown target well get nil, err. commit, err = i.Get("git_master-skia", "volantis-userdebug", "100") assert.Nil(t, commit) assert.NotNil(t, err) // After the first time we'll get an error. commit, err = i.Get("git_master-skia", "volantis-userdebug", "100") assert.Nil(t, commit) assert.NotNil(t, err) // Now poll. i.single_poll() // Now the commits should be populated so the Get should succeed. commit, err = i.Get("git_master-skia", "volantis-userdebug", "100") assert.Nil(t, err) assert.NotNil(t, commit) if got, want := commit.Hash, "1234567890"; got != want { t.Errorf("Wrong commit returned: Got %v Want %v", got, want) } commit, err = i.Get("git_master-skia", "volantis-userdebug", "101") assert.Nil(t, err) assert.NotNil(t, commit) if got, want := commit.Hash, "1234567890"; got != want { t.Errorf("Wrong commit returned: Got %v Want %v", got, want) } commit, err = i.Get("git_master-skia", "volantis-userdebug", "105") assert.Nil(t, err) assert.NotNil(t, commit) if got, want := commit.Hash, "987654321"; got != want { t.Errorf("Wrong commit returned: Got %v Want %v", got, want) } commit, err = i.Get("git_master-skia", "volantis-userdebug", "99") assert.Nil(t, commit) assert.NotNil(t, err) }
func main() { common.Init() defer util.CleanTmpDir() defer util.TimeTrack(time.Now(), "Running Chromium Perf") defer glog.Flush() // Validate required arguments. if *chromiumBuildNoPatch == "" { glog.Error("Must specify --chromium_build_nopatch") return } if *chromiumBuildWithPatch == "" { glog.Error("Must specify --chromium_build_withpatch") return } if *runIDNoPatch == "" { glog.Error("Must specify --run_id_nopatch") return } if *runIDWithPatch == "" { glog.Error("Must specify --run_id_withpatch") return } if *benchmarkName == "" { glog.Error("Must specify --benchmark_name") return } // Reset the local chromium checkout. if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err) return } // Sync the local chromium checkout. if err := util.SyncDir(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err) return } // Create the task file so that the master knows this worker is still busy. skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_RUNNING_CHROMIUM_PERF)) defer util.DeleteTaskFile(util.ACTIVITY_RUNNING_CHROMIUM_PERF) if *targetPlatform == util.PLATFORM_ANDROID { if err := adb.VerifyLocalDevice(); err != nil { // Android device missing or offline. glog.Errorf("Could not find Android device: %s", err) return } // Make sure adb shell is running as root. skutil.LogErr( util.ExecuteCmd(util.BINARY_ADB, []string{"root"}, []string{}, 5*time.Minute, nil, nil)) } // Instantiate GsUtil object. gs, err := util.NewGsUtil(nil) if err != nil { glog.Error(err) return } // Download the specified chromium builds. for _, chromiumBuild := range []string{*chromiumBuildNoPatch, *chromiumBuildWithPatch} { if err := gs.DownloadChromiumBuild(chromiumBuild); err != nil { glog.Error(err) return } //Delete the chromium build to save space when we are done. defer skutil.RemoveAll(filepath.Join(util.ChromiumBuildsDir, chromiumBuild)) } chromiumBinaryNoPatch := filepath.Join(util.ChromiumBuildsDir, *chromiumBuildNoPatch, util.BINARY_CHROME) chromiumBinaryWithPatch := filepath.Join(util.ChromiumBuildsDir, *chromiumBuildWithPatch, util.BINARY_CHROME) // Download pagesets if they do not exist locally. if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType) // Download archives if they do not exist locally. if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } // Establish nopatch output paths. localOutputDirNoPatch := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runIDNoPatch) skutil.RemoveAll(localOutputDirNoPatch) skutil.MkdirAll(localOutputDirNoPatch, 0700) defer skutil.RemoveAll(localOutputDirNoPatch) remoteDirNoPatch := filepath.Join(util.BenchmarkRunsDir, *runIDNoPatch) // Establish withpatch output paths. localOutputDirWithPatch := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runIDWithPatch) skutil.RemoveAll(localOutputDirWithPatch) skutil.MkdirAll(localOutputDirWithPatch, 0700) defer skutil.RemoveAll(localOutputDirWithPatch) remoteDirWithPatch := filepath.Join(util.BenchmarkRunsDir, *runIDWithPatch) // Construct path to the ct_run_benchmark python script. _, currentFile, _, _ := runtime.Caller(0) pathToPyFiles := filepath.Join( filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))), "py") fileInfos, err := ioutil.ReadDir(pathToPagesets) if err != nil { glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err) return } // Loop through all pagesets and run once without the patch and once with the // patch. for _, fileInfo := range fileInfos { if err := runBenchmark(fileInfo.Name(), pathToPagesets, pathToPyFiles, localOutputDirNoPatch, *chromiumBuildNoPatch, chromiumBinaryNoPatch, *runIDNoPatch, *browserExtraArgsNoPatch); err != nil { glog.Errorf("Error while running nopatch benchmark: %s", err) return } if err := runBenchmark(fileInfo.Name(), pathToPagesets, pathToPyFiles, localOutputDirWithPatch, *chromiumBuildWithPatch, chromiumBinaryWithPatch, *runIDWithPatch, *browserExtraArgsWithPatch); err != nil { glog.Errorf("Error while running withpatch benchmark: %s", err) return } } // If "--output-format=csv-pivot-table" was specified then merge all CSV files and upload. if strings.Contains(*benchmarkExtraArgs, "--output-format=csv-pivot-table") { if err := mergeUploadCSVFiles(localOutputDirNoPatch, pathToPyFiles, *runIDNoPatch, remoteDirNoPatch, gs); err != nil { glog.Errorf("Error while processing nopatch CSV files: %s", err) return } if err := mergeUploadCSVFiles(localOutputDirWithPatch, pathToPyFiles, *runIDWithPatch, remoteDirWithPatch, gs); err != nil { glog.Errorf("Error while processing withpatch CSV files: %s", err) return } } }
// CleanTmpDir deletes all tmp files from the caller because telemetry tends to // generate a lot of temporary artifacts there and they take up root disk space. func CleanTmpDir() { files, _ := ioutil.ReadDir(os.TempDir()) for _, f := range files { util.RemoveAll(filepath.Join(os.TempDir(), f.Name())) } }
func main() { defer common.LogPanic() worker_common.Init() if !*worker_common.Local { defer util.CleanTmpDir() } defer util.TimeTrack(time.Now(), "Running Chromium Perf") defer glog.Flush() // Validate required arguments. if *chromiumBuildNoPatch == "" { glog.Error("Must specify --chromium_build_nopatch") return } if *chromiumBuildWithPatch == "" { glog.Error("Must specify --chromium_build_withpatch") return } if *runID == "" { glog.Error("Must specify --run_id") return } if *runIDNoPatch == "" { glog.Error("Must specify --run_id_nopatch") return } if *runIDWithPatch == "" { glog.Error("Must specify --run_id_withpatch") return } if *benchmarkName == "" { glog.Error("Must specify --benchmark_name") return } // Reset the local chromium checkout. if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err) return } // Sync the local chromium checkout. if err := util.SyncDir(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err) return } // Create the task file so that the master knows this worker is still busy. skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_RUNNING_CHROMIUM_PERF)) defer util.DeleteTaskFile(util.ACTIVITY_RUNNING_CHROMIUM_PERF) if *targetPlatform == util.PLATFORM_ANDROID { if err := adb.VerifyLocalDevice(); err != nil { // Android device missing or offline. glog.Errorf("Could not find Android device: %s", err) return } // Kill adb server to make sure we start from a clean slate. skutil.LogErr(util.ExecuteCmd(util.BINARY_ADB, []string{"kill-server"}, []string{}, util.ADB_ROOT_TIMEOUT, nil, nil)) // Make sure adb shell is running as root. skutil.LogErr(util.ExecuteCmd(util.BINARY_ADB, []string{"root"}, []string{}, util.ADB_ROOT_TIMEOUT, nil, nil)) } // Instantiate GsUtil object. gs, err := util.NewGsUtil(nil) if err != nil { glog.Error(err) return } // Download the benchmark patch for this run from Google storage. benchmarkPatchName := *runID + ".benchmark.patch" benchmarkPatchLocalPath := filepath.Join(os.TempDir(), benchmarkPatchName) remoteDir := filepath.Join(util.ChromiumPerfRunsDir, *runID) benchmarkPatchRemotePath := filepath.Join(remoteDir, benchmarkPatchName) respBody, err := gs.GetRemoteFileContents(benchmarkPatchRemotePath) if err != nil { glog.Errorf("Could not fetch %s: %s", benchmarkPatchRemotePath, err) return } defer skutil.Close(respBody) buf := new(bytes.Buffer) if _, err := buf.ReadFrom(respBody); err != nil { glog.Errorf("Could not read from %s: %s", benchmarkPatchRemotePath, err) return } if err := ioutil.WriteFile(benchmarkPatchLocalPath, buf.Bytes(), 0666); err != nil { glog.Errorf("Unable to create file %s: %s", benchmarkPatchLocalPath, err) return } defer skutil.Remove(benchmarkPatchLocalPath) // Apply benchmark patch to the local chromium checkout. if buf.Len() > 10 { if err := util.ApplyPatch(benchmarkPatchLocalPath, util.ChromiumSrcDir); err != nil { glog.Errorf("Could not apply Telemetry's patch in %s: %s", util.ChromiumSrcDir, err) return } } // Download the specified chromium builds. for _, chromiumBuild := range []string{*chromiumBuildNoPatch, *chromiumBuildWithPatch} { if err := gs.DownloadChromiumBuild(chromiumBuild); err != nil { glog.Error(err) return } //Delete the chromium build to save space when we are done. defer skutil.RemoveAll(filepath.Join(util.ChromiumBuildsDir, chromiumBuild)) } chromiumBinaryNoPatch := filepath.Join(util.ChromiumBuildsDir, *chromiumBuildNoPatch, util.BINARY_CHROME) chromiumBinaryWithPatch := filepath.Join(util.ChromiumBuildsDir, *chromiumBuildWithPatch, util.BINARY_CHROME) // Download pagesets if they do not exist locally. if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType) // Download archives if they do not exist locally. if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } // Establish nopatch output paths. localOutputDirNoPatch := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runIDNoPatch) skutil.RemoveAll(localOutputDirNoPatch) skutil.MkdirAll(localOutputDirNoPatch, 0700) defer skutil.RemoveAll(localOutputDirNoPatch) remoteDirNoPatch := filepath.Join(util.BenchmarkRunsDir, *runIDNoPatch) // Establish withpatch output paths. localOutputDirWithPatch := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runIDWithPatch) skutil.RemoveAll(localOutputDirWithPatch) skutil.MkdirAll(localOutputDirWithPatch, 0700) defer skutil.RemoveAll(localOutputDirWithPatch) remoteDirWithPatch := filepath.Join(util.BenchmarkRunsDir, *runIDWithPatch) // Construct path to the ct_run_benchmark python script. _, currentFile, _, _ := runtime.Caller(0) pathToPyFiles := filepath.Join( filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))), "py") fileInfos, err := ioutil.ReadDir(pathToPagesets) if err != nil { glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err) return } numWorkers := WORKER_POOL_SIZE if *targetPlatform == util.PLATFORM_ANDROID || !*runInParallel { // Do not run page sets in parallel if the target platform is Android. // This is because the nopatch/withpatch APK needs to be installed prior to // each run and this will interfere with the parallel runs. Instead of trying // to find a complicated solution to this, it makes sense for Android to // continue to be serial because it will help guard against // crashes/flakiness/inconsistencies which are more prevalent in mobile runs. numWorkers = 1 glog.Infoln("===== Going to run the task serially =====") } else { glog.Infoln("===== Going to run the task with parallel chrome processes =====") } // Create channel that contains all pageset file names. This channel will // be consumed by the worker pool. pagesetRequests := util.GetClosedChannelOfPagesets(fileInfos) var wg sync.WaitGroup // Use a RWMutex for the chromeProcessesCleaner goroutine to communicate to // the workers (acting as "readers") when it wants to be the "writer" and // kill all zombie chrome processes. var mutex sync.RWMutex // Loop through workers in the worker pool. for i := 0; i < numWorkers; i++ { // Increment the WaitGroup counter. wg.Add(1) // Create and run a goroutine closure that captures SKPs. go func() { // Decrement the WaitGroup counter when the goroutine completes. defer wg.Done() for pagesetName := range pagesetRequests { mutex.RLock() if err := runBenchmark(pagesetName, pathToPagesets, pathToPyFiles, localOutputDirNoPatch, *chromiumBuildNoPatch, chromiumBinaryNoPatch, *runIDNoPatch, *browserExtraArgsNoPatch); err != nil { glog.Errorf("Error while running nopatch benchmark: %s", err) return } if err := runBenchmark(pagesetName, pathToPagesets, pathToPyFiles, localOutputDirWithPatch, *chromiumBuildWithPatch, chromiumBinaryWithPatch, *runIDWithPatch, *browserExtraArgsWithPatch); err != nil { glog.Errorf("Error while running withpatch benchmark: %s", err) return } mutex.RUnlock() } }() } if !*worker_common.Local { // Start the cleaner. go util.ChromeProcessesCleaner(&mutex, *chromeCleanerTimer) } // Wait for all spawned goroutines to complete. wg.Wait() // If "--output-format=csv-pivot-table" was specified then merge all CSV files and upload. if strings.Contains(*benchmarkExtraArgs, "--output-format=csv-pivot-table") { if err := mergeUploadCSVFiles(localOutputDirNoPatch, pathToPyFiles, *runIDNoPatch, remoteDirNoPatch, gs); err != nil { glog.Errorf("Error while processing nopatch CSV files: %s", err) return } if err := mergeUploadCSVFiles(localOutputDirWithPatch, pathToPyFiles, *runIDWithPatch, remoteDirWithPatch, gs); err != nil { glog.Errorf("Error while processing withpatch CSV files: %s", err) return } } }
func main() { defer common.LogPanic() worker_common.Init() defer util.TimeTrack(time.Now(), "Capturing Archives") defer glog.Flush() // Create the task file so that the master knows this worker is still busy. skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES)) defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES) if *chromiumBuild == "" { glog.Error("Must specify --chromium_build") return } // Reset the local chromium checkout. if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err) return } // Sync the local chromium checkout. if err := util.SyncDir(util.ChromiumSrcDir); err != nil { glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err) return } // Delete and remake the local webpage archives directory. pathToArchives := filepath.Join(util.WebArchivesDir, *pagesetType) skutil.RemoveAll(pathToArchives) skutil.MkdirAll(pathToArchives, 0700) // Instantiate GsUtil object. gs, err := util.NewGsUtil(nil) if err != nil { glog.Error(err) return } // Download the specified chromium build if it does not exist locally. if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil { glog.Error(err) return } // Download pagesets if they do not exist locally. if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType) chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME) recordWprBinary := filepath.Join(util.TelemetryBinariesDir, util.BINARY_RECORD_WPR) timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureArchivesTimeoutSecs // Loop through all pagesets. fileInfos, err := ioutil.ReadDir(pathToPagesets) if err != nil { glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err) return } glog.Infof("The %s fileInfos are: %s", len(fileInfos), fileInfos) for _, fileInfo := range fileInfos { pagesetBaseName := filepath.Base(fileInfo.Name()) if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" { // Ignore timestamp files and .pyc files. continue } // Read the pageset. pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name()) decodedPageset, err := util.ReadPageset(pagesetPath) if err != nil { glog.Errorf("Could not read %s: %s", pagesetPath, err) return } glog.Infof("===== Processing %s =====", pagesetPath) args := []string{ util.CAPTURE_ARCHIVES_DEFAULT_CT_BENCHMARK, "--extra-browser-args=--disable-setuid-sandbox", "--browser=exact", "--browser-executable=" + chromiumBinary, "--user-agent=" + decodedPageset.UserAgent, "--urls-list=" + decodedPageset.UrlsList, "--archive-data-file=" + decodedPageset.ArchiveDataFile, } env := []string{ fmt.Sprintf("PYTHONPATH=%s:$PYTHONPATH", pathToPagesets), "DISPLAY=:0", } skutil.LogErr(util.ExecuteCmd(recordWprBinary, args, env, time.Duration(timeoutSecs)*time.Second, nil, nil)) } // Write timestamp to the webpage archives dir. skutil.LogErr(util.CreateTimestampFile(pathToArchives)) // Upload webpage archives dir to Google Storage. if err := gs.UploadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil { glog.Error(err) return } }
func TestPDFProcessor(t *testing.T) { testutils.SkipIfShort(t) // Get the service account client from meta data or a local config file. client, err := auth.NewJWTServiceAccountClient("", auth.DEFAULT_JWT_FILENAME, nil, storage.ScopeFullControl) assert.Nil(t, err) cacheDir, err := fileutil.EnsureDirExists(CACHE_DIR) assert.Nil(t, err) // Clean up after the test. defer func() { defer util.RemoveAll(cacheDir) deleteFolderContent(t, TEST_BUCKET, IMAGES_OUT_DIR, client) deleteFolderContent(t, TEST_BUCKET, JSON_OUT_DIR, client) }() // Configure the processor. ingesterConf := &sharedconfig.IngesterConfig{ ExtraParams: map[string]string{ CONFIG_INPUT_IMAGES_BUCKET: TEST_BUCKET, CONFIG_INPUT_IMAGES_DIR: IMAGES_IN_DIR, CONFIG_OUTPUT_JSON_BUCKET: TEST_BUCKET, CONFIG_OUTPUT_JSON_DIR: JSON_OUT_DIR, CONFIG_OUTPUT_IMAGES_BUCKET: TEST_BUCKET, CONFIG_OUTPUT_IMAGES_DIR: IMAGES_OUT_DIR, CONFIG_PDF_CACHEDIR: cacheDir, }, } processor, err := newPDFProcessor(nil, ingesterConf, client) assert.Nil(t, err) // Load the example file and process it. fsResult, err := ingestion.FileSystemResult(TEST_INGESTION_FILE, "./") assert.Nil(t, err) err = processor.Process(fsResult) assert.Nil(t, err) // Fetch the json output and parse it. pProcessor := processor.(*pdfProcessor) // download the result. resultFileName := filepath.Join(CACHE_DIR, "result-file.json") assert.Nil(t, pProcessor.download(TEST_BUCKET, JSON_OUT_DIR, fsResult.Name(), resultFileName)) // Make sure we get the expected result. fsResult, err = ingestion.FileSystemResult(TEST_INGESTION_FILE, "./") assert.Nil(t, err) r, err := fsResult.Open() assert.Nil(t, err) fsDMResults, err := goldingestion.ParseDMResultsFromReader(r) assert.Nil(t, err) foundResult, err := ingestion.FileSystemResult(resultFileName, "./") assert.Nil(t, err) r, err = foundResult.Open() assert.Nil(t, err) foundDMResults, err := goldingestion.ParseDMResultsFromReader(r) assert.Nil(t, err) dmResult1 := *fsDMResults dmResult2 := *foundDMResults dmResult1.Results = nil dmResult2.Results = nil assert.Equal(t, dmResult1, dmResult2) foundIdx := 0 srcResults := fsDMResults.Results tgtResults := foundDMResults.Results for _, result := range srcResults { assert.True(t, foundIdx < len(tgtResults)) if result.Options["ext"] == "pdf" { for ; (foundIdx < len(tgtResults)) && (result.Key["name"] == tgtResults[foundIdx].Key["name"]); foundIdx++ { assert.True(t, tgtResults[foundIdx].Key["rasterizer"] != "") delete(tgtResults[foundIdx].Key, "rasterizer") assert.Equal(t, result.Key, tgtResults[foundIdx].Key) assert.Equal(t, "png", tgtResults[foundIdx].Options["ext"]) } } } assert.Equal(t, len(foundDMResults.Results), foundIdx) }
func main() { common.Init() webhook.MustInitRequestSaltFromFile(util.WebhookRequestSaltPath) // Send start email. emailsArr := util.ParseEmails(*emails) emailsArr = append(emailsArr, util.CtAdmins...) if len(emailsArr) == 0 { glog.Error("At least one email address must be specified") return } skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&frontend.ChromiumPerfUpdateVars{}, *gaeTaskID)) skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Chromium perf")) // Ensure webapp is updated and email is sent even if task fails. defer updateWebappTask() defer sendEmail(emailsArr) // Cleanup dirs after run completes. defer skutil.RemoveAll(filepath.Join(util.StorageDir, util.ChromiumPerfRunsDir)) defer skutil.RemoveAll(filepath.Join(util.StorageDir, util.BenchmarkRunsDir)) // Cleanup tmp files after the run. defer util.CleanTmpDir() // Finish with glog flush and how long the task took. defer util.TimeTrack(time.Now(), "Running chromium perf task on workers") defer glog.Flush() if *pagesetType == "" { glog.Error("Must specify --pageset_type") return } if *benchmarkName == "" { glog.Error("Must specify --benchmark_name") return } if *runID == "" { glog.Error("Must specify --run_id") return } // Instantiate GsUtil object. gs, err := util.NewGsUtil(nil) if err != nil { glog.Errorf("Could not instantiate gsutil object: %s", err) return } remoteOutputDir := filepath.Join(util.ChromiumPerfRunsDir, *runID) // Copy the patches to Google Storage. skiaPatchName := *runID + ".skia.patch" blinkPatchName := *runID + ".blink.patch" chromiumPatchName := *runID + ".chromium.patch" for _, patchName := range []string{skiaPatchName, blinkPatchName, chromiumPatchName} { if err := gs.UploadFile(patchName, os.TempDir(), remoteOutputDir); err != nil { glog.Errorf("Could not upload %s to %s: %s", patchName, remoteOutputDir, err) return } } skiaPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteOutputDir, skiaPatchName) blinkPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteOutputDir, blinkPatchName) chromiumPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteOutputDir, chromiumPatchName) // Create the two required chromium builds (with patch and without the patch). chromiumHash, skiaHash, err := util.CreateChromiumBuild(*runID, *targetPlatform, "", "", true) if err != nil { glog.Errorf("Could not create chromium build: %s", err) return } // Reboot all workers to start from a clean slate. util.RebootWorkers() // Run the run_chromium_perf script on all workers. runIDNoPatch := *runID + "-nopatch" runIDWithPatch := *runID + "-withpatch" chromiumBuildNoPatch := fmt.Sprintf("try-%s-%s-%s", chromiumHash, skiaHash, runIDNoPatch) chromiumBuildWithPatch := fmt.Sprintf("try-%s-%s-%s", chromiumHash, skiaHash, runIDWithPatch) runChromiumPerfCmdTemplate := "DISPLAY=:0 run_chromium_perf " + "--worker_num={{.WorkerNum}} --log_dir={{.LogDir}} --pageset_type={{.PagesetType}} " + "--chromium_build_nopatch={{.ChromiumBuildNoPatch}} --chromium_build_withpatch={{.ChromiumBuildWithPatch}} " + "--run_id_nopatch={{.RunIDNoPatch}} --run_id_withpatch={{.RunIDWithPatch}} " + "--benchmark_name={{.BenchmarkName}} --benchmark_extra_args=\"{{.BenchmarkExtraArgs}}\" " + "--browser_extra_args_nopatch=\"{{.BrowserExtraArgsNoPatch}}\" --browser_extra_args_withpatch=\"{{.BrowserExtraArgsWithPatch}}\" " + "--repeat_benchmark={{.RepeatBenchmark}} --target_platform={{.TargetPlatform}};" runChromiumPerfTemplateParsed := template.Must(template.New("run_chromium_perf_cmd").Parse(runChromiumPerfCmdTemplate)) runChromiumPerfCmdBytes := new(bytes.Buffer) if err := runChromiumPerfTemplateParsed.Execute(runChromiumPerfCmdBytes, struct { WorkerNum string LogDir string PagesetType string ChromiumBuildNoPatch string ChromiumBuildWithPatch string RunIDNoPatch string RunIDWithPatch string BenchmarkName string BenchmarkExtraArgs string BrowserExtraArgsNoPatch string BrowserExtraArgsWithPatch string RepeatBenchmark int TargetPlatform string }{ WorkerNum: util.WORKER_NUM_KEYWORD, LogDir: util.GLogDir, PagesetType: *pagesetType, ChromiumBuildNoPatch: chromiumBuildNoPatch, ChromiumBuildWithPatch: chromiumBuildWithPatch, RunIDNoPatch: runIDNoPatch, RunIDWithPatch: runIDWithPatch, BenchmarkName: *benchmarkName, BenchmarkExtraArgs: *benchmarkExtraArgs, BrowserExtraArgsNoPatch: *browserExtraArgsNoPatch, BrowserExtraArgsWithPatch: *browserExtraArgsWithPatch, RepeatBenchmark: *repeatBenchmark, TargetPlatform: *targetPlatform, }); err != nil { glog.Errorf("Failed to execute template: %s", err) return } cmd := []string{ fmt.Sprintf("cd %s;", util.CtTreeDir), "git pull;", "make all;", // The main command that runs run_chromium_perf on all workers. runChromiumPerfCmdBytes.String(), } // Setting a 1 day timeout since it may take a while run benchmarks with many // repeats. if _, err := util.SSH(strings.Join(cmd, " "), util.Slaves, 1*24*time.Hour); err != nil { glog.Errorf("Error while running cmd %s: %s", cmd, err) return } // If "--output-format=csv-pivot-table" was specified then merge all CSV files and upload. if strings.Contains(*benchmarkExtraArgs, "--output-format=csv-pivot-table") { for _, runID := range []string{runIDNoPatch, runIDWithPatch} { if err := mergeUploadCSVFiles(runID, gs); err != nil { glog.Errorf("Unable to merge and upload CSV files for %s: %s", runID, err) } } } // Compare the resultant CSV files using csv_comparer.py noPatchCSVPath := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, runIDNoPatch, runIDNoPatch+".output") withPatchCSVPath := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, runIDWithPatch, runIDWithPatch+".output") htmlOutputDir := filepath.Join(util.StorageDir, util.ChromiumPerfRunsDir, *runID, "html") skutil.MkdirAll(htmlOutputDir, 0700) htmlRemoteDir := filepath.Join(remoteOutputDir, "html") htmlOutputLinkBase := util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, htmlRemoteDir) + "/" htmlOutputLink = htmlOutputLinkBase + "index.html" noPatchOutputLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, util.BenchmarkRunsDir, runIDNoPatch, "consolidated_outputs", runIDNoPatch+".output") withPatchOutputLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, util.BenchmarkRunsDir, runIDWithPatch, "consolidated_outputs", runIDWithPatch+".output") // Construct path to the csv_comparer python script. _, currentFile, _, _ := runtime.Caller(0) pathToPyFiles := filepath.Join( filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))), "py") pathToCsvComparer := filepath.Join(pathToPyFiles, "csv_comparer.py") args := []string{ pathToCsvComparer, "--csv_file1=" + noPatchCSVPath, "--csv_file2=" + withPatchCSVPath, "--output_html=" + htmlOutputDir, "--variance_threshold=" + strconv.FormatFloat(*varianceThreshold, 'f', 2, 64), "--discard_outliers=" + strconv.FormatFloat(*discardOutliers, 'f', 2, 64), "--absolute_url=" + htmlOutputLinkBase, "--requester_email=" + *emails, "--skia_patch_link=" + skiaPatchLink, "--blink_patch_link=" + blinkPatchLink, "--chromium_patch_link=" + chromiumPatchLink, "--raw_csv_nopatch=" + noPatchOutputLink, "--raw_csv_withpatch=" + withPatchOutputLink, "--num_repeated=" + strconv.Itoa(*repeatBenchmark), "--target_platform=" + *targetPlatform, "--browser_args_nopatch=" + *browserExtraArgsNoPatch, "--browser_args_withpatch=" + *browserExtraArgsWithPatch, "--pageset_type=" + *pagesetType, "--chromium_hash=" + chromiumHash, "--skia_hash=" + skiaHash, } if err := util.ExecuteCmd("python", args, []string{}, 2*time.Hour, nil, nil); err != nil { glog.Errorf("Error running csv_comparer.py: %s", err) return } // Copy the HTML files to Google Storage. if err := gs.UploadDir(htmlOutputDir, htmlRemoteDir, true); err != nil { glog.Errorf("Could not upload %s to %s: %s", htmlOutputDir, htmlRemoteDir, err) return } taskCompletedSuccessfully = true }
func TestShareDB(t *testing.T) { // Create the server and start it. serverImpl := NewServer(DATA_DIR) defer util.RemoveAll(DATA_DIR) grpcServer, client, err := startServer(t, serverImpl) assert.Nil(t, err) defer grpcServer.Stop() dbName := "database001" bucketName := "bucket_01" ctx := context.Background() allKeys := []string{} for k := 0; k < MAX_KEYS; k++ { key := fmt.Sprintf("key_%04d", k) value := fmt.Sprintf("val_%04d", k) ack, err := client.Put(ctx, &PutRequest{dbName, bucketName, key, []byte(value)}) assert.Nil(t, err) assert.True(t, ack.Ok) foundResp, err := client.Get(ctx, &GetRequest{dbName, bucketName, key}) assert.Nil(t, err) assert.Equal(t, value, string(foundResp.Value)) allKeys = append(allKeys, key) } foundDBs, err := client.Databases(ctx, &DatabasesRequest{}) assert.Nil(t, err) assert.Equal(t, []string{dbName}, foundDBs.Values) foundBuckets, err := client.Buckets(ctx, &BucketsRequest{dbName}) assert.Nil(t, err) assert.Equal(t, []string{bucketName}, foundBuckets.Values) foundKeys, err := client.Keys(ctx, &KeysRequest{dbName, bucketName, "", "", ""}) assert.Nil(t, err) sort.Strings(foundKeys.Values) sort.Strings(allKeys) assert.Equal(t, allKeys, foundKeys.Values) // Test a min-max range scan. foundKeys, err = client.Keys(ctx, &KeysRequest{dbName, bucketName, "", "key_0010", "key_0015"}) assert.Nil(t, err) assert.Equal(t, []string{"key_0010", "key_0011", "key_0012", "key_0013", "key_0014", "key_0015"}, foundKeys.Values) // Test a min to end range scan. foundKeys, err = client.Keys(ctx, &KeysRequest{dbName, bucketName, "", "key_0015", ""}) assert.Nil(t, err) assert.Equal(t, []string{"key_0015", "key_0016", "key_0017", "key_0018", "key_0019"}, foundKeys.Values[0:5]) // Test a start to max range scan. foundKeys, err = client.Keys(ctx, &KeysRequest{dbName, bucketName, "", "", "key_0004"}) assert.Nil(t, err) assert.Equal(t, []string{"key_0000", "key_0001", "key_0002", "key_0003", "key_0004"}, foundKeys.Values) // Test a prefix scan. foundKeys, err = client.Keys(ctx, &KeysRequest{dbName, bucketName, "key_000", "", ""}) assert.Nil(t, err) exp := []string{"key_0000", "key_0001", "key_0002", "key_0003", "key_0004", "key_0005", "key_0006", "key_0007", "key_0008", "key_0009"} assert.Equal(t, exp, foundKeys.Values) for _, k := range allKeys { ack, err := client.Delete(ctx, &DeleteRequest{dbName, bucketName, k}) assert.Nil(t, err) assert.True(t, ack.Ok) foundVal, err := client.Get(ctx, &GetRequest{dbName, bucketName, k}) assert.Nil(t, err) assert.Nil(t, foundVal.Value) } foundKeys, err = client.Keys(ctx, &KeysRequest{dbName, bucketName, "", "", ""}) assert.Nil(t, err) assert.Equal(t, 0, len(foundKeys.Values)) foundResp, err := client.Get(ctx, &GetRequest{"Does-not-exist-either", bucketName, "does-not-exist"}) assert.NotNil(t, err) assert.True(t, strings.Contains(err.Error(), "Does-not-exist-either")) foundResp, err = client.Get(ctx, &GetRequest{dbName, bucketName, "does-not-exist"}) assert.Nil(t, err) assert.Equal(t, "", string(foundResp.Value)) }