func TestCreateTimestampFile(t *testing.T) {
realDir := filepath.Join(os.TempDir(), "util_test")
util.Mkdir(realDir, 0755)
defer util.RemoveAll(realDir)
timestampFilePath := filepath.Join(realDir, TIMESTAMP_FILE_NAME)
if err := CreateTimestampFile(realDir); err != nil {
t.Errorf("Unexpected error: %s", err)
}
// Assert timestamp file exists.
if _, err := os.Stat(timestampFilePath); err != nil {
t.Errorf("Timestamp file %s was not created!", timestampFilePath)
}
// Assert timestamp file contains int64.
fileContent, err := ioutil.ReadFile(timestampFilePath)
if err != nil {
t.Errorf("Unexpected error: %s", err)
}
if _, err := strconv.ParseInt(string(fileContent), 10, 64); err != nil {
t.Errorf("Unexpected value in %s: %s", timestampFilePath, err)
}
// Assert error returned when specified dir does not exist.
nonexistantDir := filepath.Join(os.TempDir(), "util_test_nonexistant")
util.RemoveAll(nonexistantDir)
if err := CreateTimestampFile(nonexistantDir); err != nil {
// Expected error
} else {
t.Error("Unexpected lack of error")
}
}
// Will need a local valid google_storage_token.data file with read write access
// to run the below test.
func Auth_TestDownloadWorkerArtifacts(t *testing.T) {
testPagesetsDirName := filepath.Join("unit-tests", "util", "page_sets")
client, _ := GetOAuthClient()
gs, err := NewGsUtil(client)
if err != nil {
t.Errorf("Unexpected error: %s", err)
}
tmpDir := filepath.Join(os.TempDir(), "util_test")
StorageDir = tmpDir
defer util.RemoveAll(tmpDir)
if err := gs.DownloadWorkerArtifacts(testPagesetsDirName, "10k", 1); err != nil {
t.Errorf("Unexpected error: %s", err)
}
// Examine contents of the local directory.
localDir := filepath.Join(tmpDir, testPagesetsDirName, "10k")
files, err := ioutil.ReadDir(localDir)
if err != nil {
t.Errorf("Unexpected error: %s", err)
}
assert.Equal(t, 3, len(files))
assert.Equal(t, "TIMESTAMP", files[0].Name())
assert.Equal(t, "alexa1-1.py", files[1].Name())
assert.Equal(t, "alexa2-2.py", files[2].Name())
}
func testRasterizer(t *testing.T, rasterizer Rasterizer, expectation string) {
assert.True(t, rasterizer.Enabled(), "%s.Enabled() failed.", rasterizer.String())
testDataDir, err := testutils.TestDataDir()
assert.Nil(t, err, "TestDataDir missing: %v", err)
tempDir, err := ioutil.TempDir("", "pdf_test_")
assert.Nil(t, err, "ioutil.TempDir failed")
defer util.RemoveAll(tempDir)
pdfSrcPath := path.Join(testDataDir, "minimal.pdf")
assert.True(t, fileutil.FileExists(pdfSrcPath), "Path '%s' does not exist", pdfSrcPath)
pdfInputPath := path.Join(tempDir, "minimal.pdf")
err = os.Symlink(pdfSrcPath, pdfInputPath)
assert.Nil(t, err, "Symlink failed")
assert.True(t, fileutil.FileExists(pdfInputPath), "Path '%s' does not exist", pdfInputPath)
outputFileName := path.Join(tempDir, "test.png")
badPath := path.Join(tempDir, "this_file_should_really_not_exist.pdf")
if err := rasterizer.Rasterize(badPath, outputFileName); err == nil {
t.Errorf(": Got '%v' Want '%v'", err, nil)
}
if err := rasterizer.Rasterize(pdfInputPath, outputFileName); err != nil {
t.Errorf(": Got '%v' Want '!nil'", err)
}
expectedOutput := path.Join(testDataDir, expectation)
assert.True(t, filesEqual(outputFileName, expectedOutput), "png output not correct")
}
// processResult rasterizes a single PDF result and returns a set of new results.
func (xformer *pdfXformer) processResult(res goldingester.Result) []*goldingester.Result {
rasterizedResults := []*goldingester.Result{}
resultMap, found := xformer.results[res.Digest]
if found {
// Skip rasterizion steps: big win.
for index, rasterizer := range xformer.rasterizers {
digest, ok := resultMap[index]
if ok {
rasterizedResults = append(rasterizedResults,
newResult(res.Key, rasterizer.String(), digest))
} else {
glog.Errorf("missing rasterizer %s on %s", rasterizer.String(), res.Digest)
}
}
return rasterizedResults
}
tempdir, err := xformer.makeTmpDir()
if err != nil {
glog.Errorf("error making temp directory: %s", err)
return rasterizedResults
}
defer util.RemoveAll(tempdir)
pdfPath := path.Join(tempdir, fmt.Sprintf("%s.pdf", res.Digest))
objectName := fmt.Sprintf("%s/%s.pdf", *storageImagesDirectory, res.Digest)
storageURL := fmt.Sprintf("gs://%s/%s", *storageBucket, objectName)
object, err := xformer.client.storageService.Objects.Get(*storageBucket, objectName).Do()
if err != nil {
glog.Errorf("unable to find %s: %s", storageURL, err)
return []*goldingester.Result{}
}
pdfData, _, err := gsFetch(object, xformer.client)
if err != nil {
glog.Errorf("unable to retrieve %s: %s", storageURL, err)
return []*goldingester.Result{}
}
err = writeTo(pdfPath, &pdfData)
if err != nil {
glog.Errorf("unable to write file %s: %s", pdfPath, err)
return []*goldingester.Result{}
}
if !isPDF(pdfPath) {
glog.Errorf("%s is not a PDF", objectName)
return []*goldingester.Result{}
}
resultMap = map[int]string{}
for index, rasterizer := range xformer.rasterizers {
digest, err := xformer.rasterizeOnce(pdfPath, index)
if err != nil {
glog.Errorf("rasterizer %s failed on %s.pdf: %s", rasterizer, res.Digest, err)
continue
}
rasterizedResults = append(rasterizedResults,
newResult(res.Key, rasterizer.String(), digest))
resultMap[index] = digest
}
xformer.results[res.Digest] = resultMap
return rasterizedResults
}
func TestIngester(t *testing.T) {
defer util.RemoveAll(LOCAL_STATUS_DIR)
now := time.Now()
beginningOfTime := now.Add(-time.Hour * 24 * 10).Unix()
const totalCommits = 100
// Instantiate mock VCS and the source.
vcs := getVCS(beginningOfTime, now.Unix(), totalCommits)
hashes := vcs.From(time.Unix(0, 0))
assert.Equal(t, totalCommits, len(hashes))
for _, h := range hashes {
assert.NotEqual(t, "", h)
}
sources := []Source{MockSource(t, vcs)}
// Instantiate the mock processor.
collected := map[string]int{}
var mutex sync.Mutex
processFn := func(result ResultFileLocation) error {
mutex.Lock()
defer mutex.Unlock()
collected[result.Name()] += 1
return nil
}
finishFn := func() error { return nil }
processor := MockProcessor(processFn, finishFn)
// Instantiate ingesterConf
conf := &sharedconfig.IngesterConfig{
RunEvery: sharedconfig.TomlDuration{Duration: 1 * time.Second},
NCommits: 50,
MinDays: 3,
StatusDir: LOCAL_STATUS_DIR,
}
// Instantiate ingester and start it.
ingester, err := NewIngester("test-ingester", conf, vcs, sources, processor)
assert.Nil(t, err)
ingester.Start()
// Give it enough time to run a few ingestion cycles and to shut down.
time.Sleep(5 * time.Second)
ingester.stop()
time.Sleep(5 * time.Second)
assert.Equal(t, totalCommits/2, len(collected))
for _, count := range collected {
assert.Equal(t, 1, count)
}
}
func TestAreTimestampsEqual(t *testing.T) {
gs, err := NewGsUtil(util.NewTimeoutClient())
if err != nil {
t.Errorf("Unexpected error: %s", err)
}
tmpDir := filepath.Join(os.TempDir(), "util_test")
util.Mkdir(tmpDir, 0777)
defer util.RemoveAll(tmpDir)
f, err := os.Create(filepath.Join(tmpDir, TIMESTAMP_FILE_NAME))
if err != nil {
t.Errorf("Unexpected error: %s", err)
}
defer util.Close(f)
// Test with matching timestamps.
if _, err := f.WriteString(GS_TEST_TIMESTAMP_VALUE); err != nil {
t.Errorf("Unexpected error: %s", err)
}
result1, err := gs.AreTimeStampsEqual(tmpDir, "unit-tests/util/")
if err != nil {
t.Errorf("Unexpected error: %s", err)
}
assert.True(t, result1)
// Test with differing timestamps.
if _, err := f.WriteString(GS_TEST_TIMESTAMP_VALUE); err != nil {
t.Errorf("Unexpected error: %s", err)
}
result2, err := gs.AreTimeStampsEqual(tmpDir, "unit-tests/util/")
if err != nil {
t.Errorf("Unexpected error: %s", err)
}
assert.False(t, result2)
// Test with Google Storage timestamp missing.
result3, err := gs.AreTimeStampsEqual(tmpDir, "unit-tests/util/dummy_name/")
if err == nil {
t.Error("Expected an error")
}
assert.False(t, result3)
// Test with local timestamp missing.
result4, err := gs.AreTimeStampsEqual(tmpDir+"dummy_name", "unit-tests/util/")
if err == nil {
t.Error("Expected an error")
}
assert.False(t, result4)
}
// downloadAndExecPython downloads a Python script from the Repo, executes it, and
// parses its output in JSON format into the given destination.
func downloadAndExecPython(r *gitiles.Repo, dst interface{}, srcPath string, workdir string) error {
destDir, err := ioutil.TempDir(workdir, "gitiles")
if err != nil {
return err
}
defer util.RemoveAll(destDir)
destPath := path.Join(destDir, path.Base(srcPath))
if err := r.DownloadFile(srcPath, destPath); err != nil {
return err
}
output, err := exec.Command("python", destPath).Output()
if err != nil {
return err
}
if err := json.Unmarshal(output, dst); err != nil {
return err
}
return nil
}
func main() {
defer common.LogPanic()
worker_common.Init()
defer util.TimeTrack(time.Now(), "Fixing archives")
defer glog.Flush()
// Create the task file so that the master knows this worker is still busy.
skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_FIXING_ARCHIVES))
defer util.DeleteTaskFile(util.ACTIVITY_FIXING_ARCHIVES)
if *pagesetType == "" {
glog.Error("Must specify --pageset_type")
return
}
if *chromiumBuild == "" {
glog.Error("Must specify --chromium_build")
return
}
if *runID == "" {
glog.Error("Must specify --run_id")
return
}
// Reset the local chromium checkout.
if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
return
}
// Sync the local chromium checkout.
if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
return
}
// Instantiate GsUtil object.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Error(err)
return
}
// Download the specified chromium build.
if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
glog.Error(err)
return
}
// Delete the chromium build to save space when we are done.
defer skutil.RemoveAll(filepath.Join(util.ChromiumBuildsDir, *chromiumBuild))
chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)
// Download pagesets if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
// Download archives if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
// Establish output paths.
localOutputDir := filepath.Join(util.StorageDir, util.FixArchivesRunsDir, *runID)
skutil.RemoveAll(localOutputDir)
skutil.MkdirAll(localOutputDir, 0700)
defer skutil.RemoveAll(localOutputDir)
// Construct path to the ct_run_benchmark python script.
_, currentFile, _, _ := runtime.Caller(0)
pathToPyFiles := filepath.Join(
filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
"py")
timeoutSecs := util.PagesetTypeToInfo[*pagesetType].RunChromiumPerfTimeoutSecs
fileInfos, err := ioutil.ReadDir(pathToPagesets)
if err != nil {
glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
return
}
// Location of the WPR logs.
wprLogs := filepath.Join(util.ChromiumSrcDir, "webpagereplay_logs", "logs.txt")
// Slice that will contain
inconsistentArchives := []string{}
// Loop through all pagesets.
for _, fileInfo := range fileInfos {
benchmarkResults := []float64{}
resourceMissingCounts := []int{}
pagesetBaseName := filepath.Base(fileInfo.Name())
if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
// Ignore timestamp files and .pyc files.
continue
}
// Convert the filename into a format consumable by the run_benchmarks
// binary.
pagesetName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name())
glog.Infof("===== Processing %s =====", pagesetPath)
// Repeat runs the specified number of times.
for repeatNum := 1; repeatNum <= *repeatBenchmark; repeatNum++ {
// Delete webpagereplay_logs before every run.
skutil.RemoveAll(wprLogs)
skutil.LogErr(os.Chdir(pathToPyFiles))
args := []string{
util.BINARY_RUN_BENCHMARK,
fmt.Sprintf("%s.%s", *benchmarkName, util.BenchmarksToTelemetryName[*benchmarkName]),
"--page-set-name=" + pagesetName,
"--page-set-base-dir=" + pathToPagesets,
"--also-run-disabled-tests",
}
// Add output dir.
outputDirArgValue := filepath.Join(localOutputDir, pagesetName, strconv.Itoa(repeatNum))
args = append(args, "--output-dir="+outputDirArgValue)
// Figure out which browser should be used.
args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary)
// Split benchmark args if not empty and append to args.
if *benchmarkArgs != "" {
for _, benchmarkArg := range strings.Split(*benchmarkArgs, " ") {
args = append(args, benchmarkArg)
}
}
// Add browserArgs if not empty to args.
if *browserArgs != "" {
args = append(args, "--extra-browser-args="+*browserArgs)
}
// Set the PYTHONPATH to the pagesets and the telemetry dirs.
env := []string{
fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir),
"DISPLAY=:0",
}
skutil.LogErr(
util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil))
// Examine the results-pivot-table.csv file and store the mean frame time.
resultsCSV := filepath.Join(outputDirArgValue, "results-pivot-table.csv")
// TODO(rmistry): The format has changed from results.csv to results-pivot-table.csv
headers, values, err := getRowsFromCSV(resultsCSV)
if err != nil {
glog.Errorf("Could not read %s: %s", resultsCSV, err)
continue
}
for i := range headers {
if headers[i] == *benchmarkHeaderToCheck {
value, _ := strconv.ParseFloat(values[i], 64)
benchmarkResults = append(benchmarkResults, value)
break
}
}
// Find how many times "Could not replay" showed up in wprLogs.
content, err := ioutil.ReadFile(wprLogs)
if err != nil {
glog.Errorf("Could not read %s: %s", wprLogs, err)
continue
}
resourceMissingCount := strings.Count(string(content), "Could not replay")
resourceMissingCounts = append(resourceMissingCounts, resourceMissingCount)
}
glog.Infof("Benchmark results for %s are: %v", fileInfo.Name(), benchmarkResults)
percentageChange := getPercentageChange(benchmarkResults)
glog.Infof("Percentage change of results is: %f", percentageChange)
glog.Infof("\"Could not replay\" showed up %v times in %s", resourceMissingCounts, wprLogs)
maxResourceMissingCount := 0
for _, count := range resourceMissingCounts {
if maxResourceMissingCount < count {
maxResourceMissingCount = count
}
}
if percentageChange > *percentageChangeThreshold || maxResourceMissingCount > *resourceMissingCountThreshold {
glog.Infof("The archive for %s is inconsistent!", fileInfo.Name())
inconsistentArchives = append(inconsistentArchives, fmt.Sprintf("%s percentageChange: %f maxResourceMissingCount: %v", fileInfo.Name(), percentageChange, maxResourceMissingCount))
if *deletePageset {
// Delete the pageset.
skutil.RemoveAll(pagesetPath)
}
}
}
if len(inconsistentArchives) > 0 {
glog.Infof("%d archives are inconsistent!", len(inconsistentArchives))
glog.Infof("The list of inconsistentArchives is: %v", inconsistentArchives)
if *deletePageset {
// Write new timestamp to the pagesets dir.
skutil.RemoveAll(filepath.Join(pathToPagesets, util.TIMESTAMP_FILE_NAME))
skutil.LogErr(util.CreateTimestampFile(pathToPagesets))
// Inconsistent pagesets were deleted locally. Upload local pagesets dir
// to Google Storage.
if err := gs.UploadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
}
}
}
func main() {
defer common.LogPanic()
common.Init()
defer util.TimeTrack(time.Now(), "Creating Pagesets")
defer glog.Flush()
// Create the task file so that the master knows this worker is still busy.
skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CREATING_PAGESETS))
defer util.DeleteTaskFile(util.ACTIVITY_CREATING_PAGESETS)
// Delete and remake the local pagesets directory.
pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
skutil.RemoveAll(pathToPagesets)
skutil.MkdirAll(pathToPagesets, 0700)
// Get info about the specified pageset type.
pagesetTypeInfo := util.PagesetTypeToInfo[*pagesetType]
csvSource := pagesetTypeInfo.CSVSource
numPages := pagesetTypeInfo.NumPages
userAgent := pagesetTypeInfo.UserAgent
// Download the CSV file from Google Storage to a tmp location.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Error(err)
return
}
respBody, err := gs.GetRemoteFileContents(csvSource)
if err != nil {
glog.Error(err)
return
}
defer skutil.Close(respBody)
csvFile := filepath.Join(os.TempDir(), filepath.Base(csvSource))
out, err := os.Create(csvFile)
if err != nil {
glog.Errorf("Unable to create file %s: %s", csvFile, err)
return
}
defer skutil.Close(out)
defer skutil.Remove(csvFile)
if _, err = io.Copy(out, respBody); err != nil {
glog.Error(err)
return
}
// Figure out which pagesets this worker should generate.
numPagesPerSlave := numPages / util.NUM_WORKERS
startNum := (*workerNum-1)*numPagesPerSlave + 1
endNum := *workerNum * numPagesPerSlave
// Construct path to the create_page_set.py python script.
_, currentFile, _, _ := runtime.Caller(0)
createPageSetScript := filepath.Join(
filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
"py", "create_page_set.py")
// Execute the create_page_set.py python script.
timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CreatePagesetsTimeoutSecs
for currNum := startNum; currNum <= endNum; currNum++ {
args := []string{
createPageSetScript,
"-s", strconv.Itoa(currNum),
"-e", strconv.Itoa(currNum),
"-c", csvFile,
"-p", *pagesetType,
"-u", userAgent,
"-o", pathToPagesets,
}
if err := util.ExecuteCmd("python", args, []string{}, time.Duration(timeoutSecs)*time.Second, nil, nil); err != nil {
glog.Error(err)
return
}
}
// Write timestamp to the pagesets dir.
skutil.LogErr(util.CreateTimestampFile(pathToPagesets))
// Upload pagesets dir to Google Storage.
if err := gs.UploadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
}
// runID is the unique id of the current run (typically requester + timestamp).
// targetPlatform is the platform the benchmark will run on (Android / Linux ).
// chromiumHash is the hash the checkout should be synced to. If not specified then
// Chromium's Tot hash is used.
// skiaHash is the hash the checkout should be synced to. If not specified then
// Skia's LKGR hash is used (the hash in Chromium's DEPS file).
// applyPatches if true looks for Chromium/Blink/Skia patches in the temp dir and
// runs once with the patch applied and once without the patch applied.
func CreateChromiumBuild(runID, targetPlatform, chromiumHash, skiaHash string, applyPatches bool) (string, string, error) {
// Determine which build dir and fetch target to use.
var chromiumBuildDir, fetchTarget string
if targetPlatform == "Android" {
chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "android_base")
fetchTarget = "android"
} else if targetPlatform == "Linux" {
chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "linux_base")
fetchTarget = "chromium"
} else {
return "", "", fmt.Errorf("Unrecognized target_platform %s", targetPlatform)
}
util.MkdirAll(chromiumBuildDir, 0700)
// Find which Chromium commit hash should be used.
var err error
if chromiumHash == "" {
chromiumHash, err = getChromiumHash()
if err != nil {
return "", "", fmt.Errorf("Error while finding Chromium's Hash: %s", err)
}
}
// Find which Skia commit hash should be used.
if skiaHash == "" {
skiaHash, err = getSkiaHash()
if err != nil {
return "", "", fmt.Errorf("Error while finding Skia's Hash: %s", err)
}
}
// Run chromium sync command using the above commit hashes.
// Construct path to the sync_skia_in_chrome python script.
_, currentFile, _, _ := runtime.Caller(0)
pathToPyFiles := filepath.Join(
filepath.Dir((filepath.Dir(filepath.Dir(currentFile)))),
"py")
syncArgs := []string{
filepath.Join(pathToPyFiles, "sync_skia_in_chrome.py"),
"--destination=" + chromiumBuildDir,
"--fetch_target=" + fetchTarget,
"--chrome_revision=" + chromiumHash,
"--skia_revision=" + skiaHash,
}
if err := ExecuteCmd("python", syncArgs, []string{}, 2*time.Hour, nil, nil); err != nil {
glog.Warning("There was an error. Deleting base directory and trying again.")
util.RemoveAll(chromiumBuildDir)
util.MkdirAll(chromiumBuildDir, 0700)
if err := ExecuteCmd("python", syncArgs, []string{}, 2*time.Hour, nil, nil); err != nil {
return "", "", fmt.Errorf("There was an error checking out chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
}
}
// Make sure we are starting from a clean slate.
if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
}
googleStorageDirName := fmt.Sprintf("%s-%s-%s", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), runID)
if runID == "" {
// Do not include the runID in the dir name if it is not specified.
googleStorageDirName = fmt.Sprintf("%s-%s", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash))
}
if applyPatches {
if err := applyRepoPatches(filepath.Join(chromiumBuildDir, "src"), runID); err != nil {
return "", "", fmt.Errorf("Could not apply patches in the chromium checkout in %s: %s", chromiumBuildDir, err)
}
// Add "try" prefix and "withpatch" suffix.
googleStorageDirName = fmt.Sprintf("try-%s-withpatch", googleStorageDirName)
}
// Build chromium.
if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil {
return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
}
// Upload to Google Storage.
gs, err := NewGsUtil(nil)
if err != nil {
return "", "", fmt.Errorf("Could not create GS object: %s", err)
}
if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
}
// Check for the applypatch flag and reset and then build again and copy to
// google storage.
if applyPatches {
// Now build chromium without the patches and upload it to Google Storage.
// Make sure we are starting from a clean slate.
if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
}
// Build chromium.
if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil {
return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
}
// Upload to Google Storage.
googleStorageDirName = fmt.Sprintf("try-%s-%s-%s-nopatch", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), runID)
if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
}
}
return getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), nil
}
func main() {
common.Init()
webhook.MustInitRequestSaltFromFile(util.WebhookRequestSaltPath)
// Send start email.
emailsArr := util.ParseEmails(*emails)
emailsArr = append(emailsArr, util.CtAdmins...)
if len(emailsArr) == 0 {
glog.Error("At least one email address must be specified")
return
}
skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Skia correctness"))
// Ensure webapp is updated and email is sent even if task fails.
defer updateWebappTask()
defer sendEmail(emailsArr)
// Cleanup tmp files after the run.
defer util.CleanTmpDir()
// Finish with glog flush and how long the task took.
defer util.TimeTrack(time.Now(), "Running skia correctness task on workers")
defer glog.Flush()
if *pagesetType == "" {
glog.Error("Must specify --pageset_type")
return
}
if *chromiumBuild == "" {
glog.Error("Must specify --chromium_build")
return
}
if *runID == "" {
glog.Error("Must specify --run_id")
return
}
// Instantiate GsUtil object.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Error(err)
return
}
remoteOutputDir := filepath.Join(util.SkiaCorrectnessRunsDir, *runID)
// Copy the patch to Google Storage.
patchName := *runID + ".patch"
patchRemoteDir := filepath.Join(remoteOutputDir, "patches")
if err := gs.UploadFile(patchName, os.TempDir(), patchRemoteDir); err != nil {
glog.Errorf("Could not upload %s to %s: %s", patchName, patchRemoteDir, err)
return
}
skiaPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, patchRemoteDir, patchName)
// Run the run_skia_correctness script on all workers.
runSkiaCorrCmdTemplate := "DISPLAY=:0 run_skia_correctness --worker_num={{.WorkerNum}} --log_dir={{.LogDir}} " +
"--pageset_type={{.PagesetType}} --chromium_build={{.ChromiumBuild}} --run_id={{.RunID}} " +
"--render_pictures_args=\"{{.RenderPicturesArgs}}\" --gpu_nopatch_run={{.GpuNoPatchRun}} " +
"--gpu_withpatch_run={{.GpuWithPatchRun}};"
runSkiaCorrTemplateParsed := template.Must(template.New("run_skia_correctness_cmd").Parse(runSkiaCorrCmdTemplate))
runSkiaCorrCmdBytes := new(bytes.Buffer)
if err := runSkiaCorrTemplateParsed.Execute(runSkiaCorrCmdBytes, struct {
WorkerNum string
LogDir string
PagesetType string
ChromiumBuild string
RunID string
RenderPicturesArgs string
GpuNoPatchRun string
GpuWithPatchRun string
}{
WorkerNum: util.WORKER_NUM_KEYWORD,
LogDir: util.GLogDir,
PagesetType: *pagesetType,
ChromiumBuild: *chromiumBuild,
RunID: *runID,
RenderPicturesArgs: *renderPicturesArgs,
GpuNoPatchRun: strconv.FormatBool(*gpuNoPatchRun),
GpuWithPatchRun: strconv.FormatBool(*gpuWithPatchRun),
}); err != nil {
glog.Errorf("Failed to execute template: %s", err)
return
}
cmd := []string{
fmt.Sprintf("cd %s;", util.CtTreeDir),
"git pull;",
"make all;",
// The main command that runs run_skia_correctness on all workers.
runSkiaCorrCmdBytes.String(),
}
if _, err := util.SSH(strings.Join(cmd, " "), util.Slaves, 4*time.Hour); err != nil {
glog.Errorf("Error while running cmd %s: %s", cmd, err)
return
}
localOutputDir := filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir, *runID)
localSummariesDir := filepath.Join(localOutputDir, "summaries")
skutil.MkdirAll(localSummariesDir, 0700)
defer skutil.RemoveAll(filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir))
// Copy outputs from all slaves locally.
for i := 0; i < util.NUM_WORKERS; i++ {
workerNum := i + 1
workerLocalOutputPath := filepath.Join(localSummariesDir, fmt.Sprintf("slave%d", workerNum)+".json")
workerRemoteOutputPath := filepath.Join(remoteOutputDir, fmt.Sprintf("slave%d", workerNum), fmt.Sprintf("slave%d", workerNum)+".json")
respBody, err := gs.GetRemoteFileContents(workerRemoteOutputPath)
if err != nil {
glog.Errorf("Could not fetch %s: %s", workerRemoteOutputPath, err)
// TODO(rmistry): Should we instead return here? We can only return
// here if all 100 slaves reliably run without any failures which they
// really should.
continue
}
defer skutil.Close(respBody)
out, err := os.Create(workerLocalOutputPath)
if err != nil {
glog.Errorf("Unable to create file %s: %s", workerLocalOutputPath, err)
return
}
defer skutil.Close(out)
defer skutil.Remove(workerLocalOutputPath)
if _, err = io.Copy(out, respBody); err != nil {
glog.Errorf("Unable to copy to file %s: %s", workerLocalOutputPath, err)
return
}
}
// Call json_summary_combiner.py to merge all results into a single results CSV.
_, currentFile, _, _ := runtime.Caller(0)
pathToPyFiles := filepath.Join(
filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
"py")
pathToJsonCombiner := filepath.Join(pathToPyFiles, "json_summary_combiner.py")
localHtmlDir := filepath.Join(localOutputDir, "html")
remoteHtmlDir := filepath.Join(remoteOutputDir, "html")
baseHtmlLink := util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteHtmlDir) + "/"
htmlOutputLink = baseHtmlLink + "index.html"
skutil.MkdirAll(localHtmlDir, 0700)
args := []string{
pathToJsonCombiner,
"--json_summaries_dir=" + localSummariesDir,
"--output_html_dir=" + localHtmlDir,
"--absolute_url=" + baseHtmlLink,
"--render_pictures_args=" + *renderPicturesArgs,
"--nopatch_gpu=" + strconv.FormatBool(*gpuNoPatchRun),
"--withpatch_gpu=" + strconv.FormatBool(*gpuWithPatchRun),
}
if err := util.ExecuteCmd("python", args, []string{}, 1*time.Hour, nil, nil); err != nil {
glog.Errorf("Error running json_summary_combiner.py: %s", err)
return
}
// Copy the HTML files to Google Storage.
if err := gs.UploadDir(localHtmlDir, remoteHtmlDir, true); err != nil {
glog.Errorf("Could not upload %s to %s: %s", localHtmlDir, remoteHtmlDir, err)
return
}
taskCompletedSuccessfully = true
}
func (p *pdfProcessor) rasterizeAndUpload(dmResultName string, dmResults *goldingestion.DMResults, pdfResults []*goldingestion.Result) error {
processedResults := make([]*goldingestion.Result, 0, len(pdfResults)*len(p.rasterizers))
// Create a temporary directory to hold the rastered images.
tempDir, err := ioutil.TempDir(p.pdfCacheDir, "pdfingestion")
if err != nil {
return err
}
defer util.RemoveAll(tempDir)
// Go through all results that generated PDF files.
for resultIdx, result := range pdfResults {
// Fetch the PDF file if it's not in the cache.
pdfFileName := fmt.Sprintf("%s.%s", result.Digest, PDF_EXT)
pdfPath := filepath.Join(p.pdfCacheDir, pdfFileName)
if !fileutil.FileExists(pdfPath) {
if err = p.download(p.inImagesBucket, p.inImagesDir, pdfFileName, pdfPath); err != nil {
glog.Errorf("Unable to retrieve image: %s. Error: %s", pdfFileName, err)
continue
}
}
// Generate an image for each rasterizer.
for rasterIdx, rasterizer := range p.rasterizers {
tempName := filepath.Join(tempDir, fmt.Sprintf("rastering_%d_%d.%s", resultIdx, rasterIdx, PNG_EXT))
err := rasterizer.Rasterize(pdfPath, tempName)
if err != nil {
glog.Errorf("Rasterizing %s with %s failed: %s", filepath.Base(pdfPath), rasterizer.String(), err)
continue
}
// Open the generated image and calculate the MD5.
file, err := os.Open(tempName)
if err != nil {
glog.Errorf("Unable to open generated image: %s", err)
continue
}
var buf bytes.Buffer
md5, err := util.MD5FromReader(file, &buf)
if err != nil {
glog.Errorf("Unable to calculate MD5 hash of file %s. Got error: %s", tempName, err)
continue
}
digest := hex.EncodeToString(md5)
uploadFileName := fmt.Sprintf("%s.%s", digest, PNG_EXT)
if err := p.upload(p.outImagesBucket, p.outImagesDir, uploadFileName, bytes.NewBuffer(buf.Bytes())); err != nil {
glog.Errorf("Unable to upload file %s. Error: %s", uploadFileName, err)
continue
}
// Update the result and add it to the successfully processed results.
result.Key["rasterizer"] = rasterizer.String()
result.Digest = digest
result.Options["ext"] = PNG_EXT
processedResults = append(processedResults, result)
}
}
// If we have no processed results we consider it an error.
if len(processedResults) == 0 {
return fmt.Errorf("No input image was processed successfully.")
}
// Replace the old results in the original result and write it to the cloud.
dmResults.Results = processedResults
jsonBytes, err := json.MarshalIndent(dmResults, "", " ")
if err != nil {
return fmt.Errorf("Unable to encode JSON: %s", err)
}
return p.upload(p.outJsonBucket, p.outJsonDir, dmResultName, bytes.NewBuffer(jsonBytes))
}
func main() {
defer common.LogPanic()
worker_common.Init()
if !*worker_common.Local {
defer util.CleanTmpDir()
}
defer util.TimeTrack(time.Now(), "Capturing SKPs")
defer glog.Flush()
// Validate required arguments.
if *chromiumBuild == "" {
glog.Error("Must specify --chromium_build")
return
}
if *runID == "" {
glog.Error("Must specify --run_id")
return
}
if *targetPlatform == util.PLATFORM_ANDROID {
glog.Error("Android is not yet supported for capturing SKPs.")
return
}
// Reset the local chromium checkout.
if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
return
}
// Sync the local chromium checkout.
if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
return
}
// Create the task file so that the master knows this worker is still busy.
skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_SKPS))
defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_SKPS)
// Instantiate GsUtil object.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Error(err)
return
}
// Download the specified chromium build.
if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
glog.Error(err)
return
}
// Delete the chromium build to save space when we are done.
defer skutil.RemoveAll(filepath.Join(util.ChromiumBuildsDir, *chromiumBuild))
chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)
if *targetPlatform == util.PLATFORM_ANDROID {
// Install the APK on the Android device.
if err := util.InstallChromeAPK(*chromiumBuild); err != nil {
glog.Errorf("Could not install the chromium APK: %s", err)
return
}
}
// Download pagesets if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
// Download archives if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
// Create the dir that SKPs will be stored in.
pathToSkps := filepath.Join(util.SkpsDir, *pagesetType, *chromiumBuild)
// Delete and remake the local SKPs directory.
skutil.RemoveAll(pathToSkps)
skutil.MkdirAll(pathToSkps, 0700)
// Establish output paths.
localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID)
skutil.RemoveAll(localOutputDir)
skutil.MkdirAll(localOutputDir, 0700)
defer skutil.RemoveAll(localOutputDir)
// Construct path to the ct_run_benchmark python script.
_, currentFile, _, _ := runtime.Caller(0)
pathToPyFiles := filepath.Join(
filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
"py")
timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureSKPsTimeoutSecs
fileInfos, err := ioutil.ReadDir(pathToPagesets)
if err != nil {
glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
return
}
// Create channel that contains all pageset file names. This channel will
// be consumed by the worker pool.
pagesetRequests := util.GetClosedChannelOfPagesets(fileInfos)
var wg sync.WaitGroup
// Use a RWMutex for the chromeProcessesCleaner goroutine to communicate to
// the workers (acting as "readers") when it wants to be the "writer" and
// kill all zombie chrome processes.
var mutex sync.RWMutex
// Loop through workers in the worker pool.
for i := 0; i < WORKER_POOL_SIZE; i++ {
// Increment the WaitGroup counter.
wg.Add(1)
// Create and run a goroutine closure that captures SKPs.
go func() {
// Decrement the WaitGroup counter when the goroutine completes.
defer wg.Done()
for pagesetName := range pagesetRequests {
mutex.RLock()
// Read the pageset.
pagesetPath := filepath.Join(pathToPagesets, pagesetName)
decodedPageset, err := util.ReadPageset(pagesetPath)
if err != nil {
glog.Errorf("Could not read %s: %s", pagesetPath, err)
continue
}
glog.Infof("===== Processing %s =====", pagesetPath)
skutil.LogErr(os.Chdir(pathToPyFiles))
args := []string{
filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
util.BenchmarksToTelemetryName[util.BENCHMARK_SKPICTURE_PRINTER],
"--also-run-disabled-tests",
"--page-repeat=1", // Only need one run for SKPs.
"--skp-outdir=" + pathToSkps,
"--extra-browser-args=" + util.DEFAULT_BROWSER_ARGS,
"--user-agent=" + decodedPageset.UserAgent,
"--urls-list=" + decodedPageset.UrlsList,
"--archive-data-file=" + decodedPageset.ArchiveDataFile,
}
// Figure out which browser should be used.
if *targetPlatform == util.PLATFORM_ANDROID {
args = append(args, "--browser=android-chromium")
} else {
args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary)
}
// Set the PYTHONPATH to the pagesets and the telemetry dirs.
env := []string{
fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir),
"DISPLAY=:0",
}
skutil.LogErr(
util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil))
mutex.RUnlock()
}
}()
}
if !*worker_common.Local {
// Start the cleaner.
go util.ChromeProcessesCleaner(&mutex, *chromeCleanerTimer)
}
// Wait for all spawned goroutines to complete.
wg.Wait()
// Move, validate and upload all SKP files.
// List all directories in pathToSkps and copy out the skps.
skpFileInfos, err := ioutil.ReadDir(pathToSkps)
if err != nil {
glog.Errorf("Unable to read %s: %s", pathToSkps, err)
return
}
for _, fileInfo := range skpFileInfos {
if !fileInfo.IsDir() {
// We are only interested in directories.
continue
}
skpName := fileInfo.Name()
// Find the largest layer in this directory.
layerInfos, err := ioutil.ReadDir(filepath.Join(pathToSkps, skpName))
if err != nil {
glog.Errorf("Unable to read %s: %s", filepath.Join(pathToSkps, skpName), err)
}
if len(layerInfos) > 0 {
largestLayerInfo := layerInfos[0]
for _, layerInfo := range layerInfos {
fmt.Println(layerInfo.Size())
if layerInfo.Size() > largestLayerInfo.Size() {
largestLayerInfo = layerInfo
}
}
// Only save SKPs greater than 6000 bytes. Less than that are probably
// malformed.
if largestLayerInfo.Size() > 6000 {
layerPath := filepath.Join(pathToSkps, skpName, largestLayerInfo.Name())
skutil.Rename(layerPath, filepath.Join(pathToSkps, skpName+".skp"))
} else {
glog.Warningf("Skipping %s because size was less than 5000 bytes", skpName)
}
}
// We extracted what we needed from the directory, now delete it.
skutil.RemoveAll(filepath.Join(pathToSkps, skpName))
}
glog.Info("Calling remove_invalid_skps.py")
// Sync Skia tree.
skutil.LogErr(util.SyncDir(util.SkiaTreeDir))
// Build tools.
skutil.LogErr(util.BuildSkiaTools())
// Run remove_invalid_skps.py
pathToRemoveSKPs := filepath.Join(pathToPyFiles, "remove_invalid_skps.py")
pathToSKPInfo := filepath.Join(util.SkiaTreeDir, "out", "Release", "skpinfo")
args := []string{
pathToRemoveSKPs,
"--skp_dir=" + pathToSkps,
"--path_to_skpinfo=" + pathToSKPInfo,
}
skutil.LogErr(util.ExecuteCmd("python", args, []string{}, util.REMOVE_INVALID_SKPS_TIMEOUT,
nil, nil))
// Write timestamp to the SKPs dir.
skutil.LogErr(util.CreateTimestampFile(pathToSkps))
// Upload SKPs dir to Google Storage.
if err := gs.UploadWorkerArtifacts(util.SKPS_DIR_NAME, filepath.Join(*pagesetType, *chromiumBuild), *workerNum); err != nil {
glog.Error(err)
return
}
}
func main() {
common.Init()
defer util.TimeTrack(time.Now(), "Running Skia Correctness")
defer glog.Flush()
if *chromiumBuild == "" {
glog.Error("Must specify --chromium_build")
return
}
if *runID == "" {
glog.Error("Must specify --run_id")
return
}
// Create the task file so that the master knows this worker is still busy.
skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_RUNNING_SKIA_CORRECTNESS))
defer util.DeleteTaskFile(util.ACTIVITY_RUNNING_SKIA_CORRECTNESS)
// Establish output paths.
localOutputDir := filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir, *runID)
skutil.RemoveAll(filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir))
skutil.MkdirAll(localOutputDir, 0700)
defer skutil.RemoveAll(localOutputDir)
remoteOutputDir := filepath.Join(util.SkiaCorrectnessRunsDir, *runID, fmt.Sprintf("slave%d", *workerNum))
// Instantiate GsUtil object.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Error(err)
return
}
// Download SKPs if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.SKPS_DIR_NAME, filepath.Join(*pagesetType, *chromiumBuild), *workerNum); err != nil {
glog.Error(err)
return
}
localSkpsDir := filepath.Join(util.SkpsDir, *pagesetType, *chromiumBuild)
// Download the Skia patch for this run from Google storage.
patchName := *runID + ".patch"
patchLocalPath := filepath.Join(os.TempDir(), patchName)
remoteDir := filepath.Join(util.SkiaCorrectnessRunsDir, *runID)
patchRemotePath := filepath.Join(remoteDir, "patches", patchName)
respBody, err := gs.GetRemoteFileContents(patchRemotePath)
if err != nil {
glog.Errorf("Could not fetch %s: %s", patchRemotePath, err)
return
}
defer skutil.Close(respBody)
out, err := os.Create(patchLocalPath)
if err != nil {
glog.Errorf("Unable to create file %s: %s", patchLocalPath, err)
return
}
defer skutil.Close(out)
defer skutil.Remove(patchLocalPath)
if _, err = io.Copy(out, respBody); err != nil {
glog.Error(err)
return
}
// Apply the patch to a clean checkout and run render_pictures.
// Reset Skia tree.
skutil.LogErr(util.ResetCheckout(util.SkiaTreeDir))
// Sync Skia tree.
skutil.LogErr(util.SyncDir(util.SkiaTreeDir))
// Apply Skia patch.
file, _ := os.Open(patchLocalPath)
fileInfo, _ := file.Stat()
// It is a valid patch only if it is more than 10 bytes.
if fileInfo.Size() > 10 {
glog.Info("Attempting to apply %s to %s", patchLocalPath, util.SkiaTreeDir)
if err := util.ApplyPatch(patchLocalPath, util.SkiaTreeDir); err != nil {
glog.Errorf("Could not apply patch %s to %s: %s", patchLocalPath, util.SkiaTreeDir, err)
return
}
glog.Info("Patch successfully applied")
} else {
glog.Info("Patch is empty or invalid. Skipping the patch.")
}
// Build tools.
skutil.LogErr(util.BuildSkiaTools())
// Run render_pictures.
if err := runRenderPictures(localSkpsDir, filepath.Join(localOutputDir, "withpatch"), filepath.Join(remoteOutputDir, "withpatch"), *gpuWithPatchRun); err != nil {
glog.Errorf("Error while running withpatch render_pictures: %s", err)
return
}
// Remove the patch and run render_pictures.
// Reset Skia tree.
skutil.LogErr(util.ResetCheckout(util.SkiaTreeDir))
// Build tools.
skutil.LogErr(util.BuildSkiaTools())
// Run render_pictures.
if err := runRenderPictures(localSkpsDir, filepath.Join(localOutputDir, "nopatch"), filepath.Join(remoteOutputDir, "nopatch"), *gpuNoPatchRun); err != nil {
glog.Errorf("Error while running nopatch render_pictures: %s", err)
return
}
// Comparing pictures and saving differences in JSON output file.
jsonSummaryDir := filepath.Join(localOutputDir, "json_summary")
skutil.MkdirAll(jsonSummaryDir, 0700)
jsonSummaryPath := filepath.Join(jsonSummaryDir, fmt.Sprintf("slave%d", *workerNum)+".json")
// Construct path to the write_json_summary python script.
_, currentFile, _, _ := runtime.Caller(0)
pathToPyFiles := filepath.Join(
filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
"py")
summaryArgs := []string{
filepath.Join(pathToPyFiles, "write_json_summary.py"),
"--img_root=" + localOutputDir,
"--withpatch_json=" + filepath.Join(localOutputDir, "withpatch", "summary.json"),
"--withpatch_images_base_url=file://" + filepath.Join(localOutputDir, "withpatch"),
"--nopatch_json=" + filepath.Join(localOutputDir, "nopatch", "summary.json"),
"--nopatch_images_base_url=file://" + filepath.Join(localOutputDir, "nopatch"),
"--output_file_path=" + jsonSummaryPath,
"--gs_output_dir=gs://" + filepath.Join(util.GS_BUCKET_NAME, remoteDir),
"--gs_skp_dir=gs://" + filepath.Join(util.GS_BUCKET_NAME, util.SKPS_DIR_NAME, *pagesetType, *chromiumBuild, fmt.Sprintf("slave%d", *workerNum)),
"--slave_num=" + strconv.Itoa(*workerNum),
}
if err := util.ExecuteCmd("python", summaryArgs, []string{}, 15*time.Minute, nil, nil); err != nil {
glog.Error(err)
return
}
// Upload artifacts to Google Storage.
// Get list of failed file names and upload only those to Google Storage.
// Read the JSON file.
summaryFile, err := ioutil.ReadFile(jsonSummaryPath)
if err != nil {
glog.Errorf("Unable to read %s: %s", jsonSummaryPath, err)
}
var jsontype map[string]interface{}
skutil.LogErr(json.Unmarshal(summaryFile, &jsontype))
if jsontype[fmt.Sprintf("slave%d", *workerNum)] != nil {
failedFiles := jsontype[fmt.Sprintf("slave%d", *workerNum)].(map[string]interface{})["failedFiles"].([]interface{})
for i := range failedFiles {
failedFile := failedFiles[i].(map[string]interface{})["fileName"].(string)
// TODO(rmistry): Use goroutines to do the below in parallel.
skutil.LogErr(
gs.UploadFile(failedFile, filepath.Join(localOutputDir, "withpatch"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "withpatch-images")))
skutil.LogErr(
gs.UploadFile(failedFile, filepath.Join(localOutputDir, "nopatch"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "nopatch-images")))
}
// Copy the diffs and whitediffs to Google Storage.
skutil.LogErr(
gs.UploadDir(filepath.Join(localOutputDir, "diffs"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "diffs"), true))
skutil.LogErr(
gs.UploadDir(filepath.Join(localOutputDir, "whitediffs"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "whitediffs"), true))
}
// Upload the summary file.
skutil.LogErr(
gs.UploadFile(fmt.Sprintf("slave%d", *workerNum)+".json", jsonSummaryDir, filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum))))
}
// downloadRemoteDir downloads the specified Google Storage dir to the specified
// local dir. The local dir will be emptied and recreated. Handles multiple levels
// of directories.
func (gs *GsUtil) downloadRemoteDir(localDir, gsDir string) error {
// Empty the local dir.
util.RemoveAll(localDir)
// Create the local dir.
util.MkdirAll(localDir, 0700)
// The channel where the storage objects to be deleted will be sent to.
chStorageObjects := make(chan filePathToStorageObject, MAX_CHANNEL_SIZE)
req := gs.service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/")
for req != nil {
resp, err := req.Do()
if err != nil {
return fmt.Errorf("Error occured while listing %s: %s", gsDir, err)
}
for _, result := range resp.Items {
fileName := filepath.Base(result.Name)
// If downloading from subdir then add it to the fileName.
fileGsDir := filepath.Dir(result.Name)
subDirs := strings.TrimPrefix(fileGsDir, gsDir)
if subDirs != "" {
dirTokens := strings.Split(subDirs, "/")
for i := range dirTokens {
fileName = filepath.Join(dirTokens[len(dirTokens)-i-1], fileName)
}
// Create the local directory.
util.MkdirAll(filepath.Join(localDir, filepath.Dir(fileName)), 0700)
}
chStorageObjects <- filePathToStorageObject{storageObject: result, filePath: fileName}
}
if len(resp.NextPageToken) > 0 {
req.PageToken(resp.NextPageToken)
} else {
req = nil
}
}
close(chStorageObjects)
// Kick off goroutines to download the storage objects.
var wg sync.WaitGroup
for i := 0; i < GOROUTINE_POOL_SIZE; i++ {
wg.Add(1)
go func(goroutineNum int) {
defer wg.Done()
for obj := range chStorageObjects {
result := obj.storageObject
filePath := obj.filePath
respBody, err := getRespBody(result, gs.client)
if err != nil {
glog.Errorf("Could not fetch %s: %s", result.MediaLink, err)
return
}
defer util.Close(respBody)
outputFile := filepath.Join(localDir, filePath)
out, err := os.Create(outputFile)
if err != nil {
glog.Errorf("Unable to create file %s: %s", outputFile, err)
return
}
defer util.Close(out)
if _, err = io.Copy(out, respBody); err != nil {
glog.Error(err)
return
}
glog.Infof("Downloaded gs://%s/%s to %s with goroutine#%d", GS_BUCKET_NAME, result.Name, outputFile, goroutineNum)
// Sleep for a second after downloading file to avoid bombarding Cloud
// storage.
time.Sleep(time.Second)
}
}(i + 1)
}
wg.Wait()
return nil
}
func main() {
common.Init()
defer util.TimeTrack(time.Now(), "Capturing Archives")
defer glog.Flush()
// Create the task file so that the master knows this worker is still busy.
skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES))
defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES)
if *chromiumBuild == "" {
glog.Error("Must specify --chromium_build")
return
}
// Reset the local chromium checkout.
if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
return
}
// Sync the local chromium checkout.
if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
return
}
// Delete and remake the local webpage archives directory.
pathToArchives := filepath.Join(util.WebArchivesDir, *pagesetType)
skutil.RemoveAll(pathToArchives)
skutil.MkdirAll(pathToArchives, 0700)
// Instantiate GsUtil object.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Error(err)
return
}
// Download the specified chromium build if it does not exist locally.
if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
glog.Error(err)
return
}
// Download pagesets if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)
recordWprBinary := filepath.Join(util.TelemetryBinariesDir, util.BINARY_RECORD_WPR)
timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureArchivesTimeoutSecs
// Loop through all pagesets.
fileInfos, err := ioutil.ReadDir(pathToPagesets)
if err != nil {
glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
return
}
// TODO(rmistry): Remove this hack once the 1M webpage archives have been captured.
glog.Infof("The length of fileInfos is: %s", len(fileInfos))
fileInfos = fileInfos[18500:20000]
glog.Infof("The fileInfos are: %s", fileInfos)
for _, fileInfo := range fileInfos {
pagesetBaseName := filepath.Base(fileInfo.Name())
if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
// Ignore timestamp files and .pyc files.
continue
}
// Convert the filename into a format consumable by the record_wpr binary.
pagesetArchiveName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name())
glog.Infof("===== Processing %s =====", pagesetPath)
args := []string{
"--extra-browser-args=--disable-setuid-sandbox",
"--browser=exact",
"--browser-executable=" + chromiumBinary,
fmt.Sprintf("%s_page_set", pagesetArchiveName),
"--page-set-base-dir=" + pathToPagesets,
}
env := []string{
fmt.Sprintf("PYTHONPATH=%s:$PYTHONPATH", pathToPagesets),
"DISPLAY=:0",
}
skutil.LogErr(util.ExecuteCmd(recordWprBinary, args, env, time.Duration(timeoutSecs)*time.Second, nil, nil))
}
// Write timestamp to the webpage archives dir.
skutil.LogErr(util.CreateTimestampFile(pathToArchives))
// Upload webpage archives dir to Google Storage.
if err := gs.UploadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
}
func TestInfo(t *testing.T) {
levelDBDir := filepath.Join(os.TempDir(), "android-leveldb")
defer util.RemoveAll(levelDBDir)
db, err := leveldb.OpenFile(levelDBDir, nil)
if err != nil {
t.Fatalf("Failed to create test leveldb: %s", err)
}
i := &info{
db: db,
commits: mockCommits{},
}
if got, want := i.branchtargets(), []string{}; !util.SSliceEqual(got, want) {
t.Errorf("Wrong targets: Got %v Want %v", got, want)
}
i.single_poll()
// The first time we Get on an unknown target well get nil, err.
commit, err := i.Get("git_master-skia", "razor-userdebug", "100")
assert.Nil(t, commit)
assert.NotNil(t, err)
// After the first time we'll get an error.
commit, err = i.Get("git_master-skia", "razor-userdebug", "100")
assert.Nil(t, commit)
assert.NotNil(t, err)
// Now poll.
i.single_poll()
// Now the commits should be populated so the Get should succeed.
commit, err = i.Get("git_master-skia", "razor-userdebug", "100")
assert.Nil(t, err)
assert.NotNil(t, commit)
if got, want := commit.Hash, "1234567890"; got != want {
t.Errorf("Wrong commit returned: Got %v Want %v", got, want)
}
commit, err = i.Get("git_master-skia", "razor-userdebug", "101")
assert.Nil(t, err)
assert.NotNil(t, commit)
if got, want := commit.Hash, "1234567890"; got != want {
t.Errorf("Wrong commit returned: Got %v Want %v", got, want)
}
commit, err = i.Get("git_master-skia", "razor-userdebug", "103")
assert.Nil(t, err)
assert.NotNil(t, commit)
if got, want := commit.Hash, "987654321"; got != want {
t.Errorf("Wrong commit returned: Got %v Want %v", got, want)
}
commit, err = i.Get("git_master-skia", "razor-userdebug", "99")
assert.Nil(t, commit)
assert.NotNil(t, err)
// Now add another target.
// The first time we Get on an unknown target well get nil, err.
commit, err = i.Get("git_master-skia", "volantis-userdebug", "100")
assert.Nil(t, commit)
assert.NotNil(t, err)
// After the first time we'll get an error.
commit, err = i.Get("git_master-skia", "volantis-userdebug", "100")
assert.Nil(t, commit)
assert.NotNil(t, err)
// Now poll.
i.single_poll()
// Now the commits should be populated so the Get should succeed.
commit, err = i.Get("git_master-skia", "volantis-userdebug", "100")
assert.Nil(t, err)
assert.NotNil(t, commit)
if got, want := commit.Hash, "1234567890"; got != want {
t.Errorf("Wrong commit returned: Got %v Want %v", got, want)
}
commit, err = i.Get("git_master-skia", "volantis-userdebug", "101")
assert.Nil(t, err)
assert.NotNil(t, commit)
if got, want := commit.Hash, "1234567890"; got != want {
t.Errorf("Wrong commit returned: Got %v Want %v", got, want)
}
commit, err = i.Get("git_master-skia", "volantis-userdebug", "105")
assert.Nil(t, err)
assert.NotNil(t, commit)
if got, want := commit.Hash, "987654321"; got != want {
t.Errorf("Wrong commit returned: Got %v Want %v", got, want)
}
commit, err = i.Get("git_master-skia", "volantis-userdebug", "99")
assert.Nil(t, commit)
assert.NotNil(t, err)
}
func main() {
common.Init()
defer util.CleanTmpDir()
defer util.TimeTrack(time.Now(), "Running Chromium Perf")
defer glog.Flush()
// Validate required arguments.
if *chromiumBuildNoPatch == "" {
glog.Error("Must specify --chromium_build_nopatch")
return
}
if *chromiumBuildWithPatch == "" {
glog.Error("Must specify --chromium_build_withpatch")
return
}
if *runIDNoPatch == "" {
glog.Error("Must specify --run_id_nopatch")
return
}
if *runIDWithPatch == "" {
glog.Error("Must specify --run_id_withpatch")
return
}
if *benchmarkName == "" {
glog.Error("Must specify --benchmark_name")
return
}
// Reset the local chromium checkout.
if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
return
}
// Sync the local chromium checkout.
if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
return
}
// Create the task file so that the master knows this worker is still busy.
skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_RUNNING_CHROMIUM_PERF))
defer util.DeleteTaskFile(util.ACTIVITY_RUNNING_CHROMIUM_PERF)
if *targetPlatform == util.PLATFORM_ANDROID {
if err := adb.VerifyLocalDevice(); err != nil {
// Android device missing or offline.
glog.Errorf("Could not find Android device: %s", err)
return
}
// Make sure adb shell is running as root.
skutil.LogErr(
util.ExecuteCmd(util.BINARY_ADB, []string{"root"}, []string{}, 5*time.Minute, nil, nil))
}
// Instantiate GsUtil object.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Error(err)
return
}
// Download the specified chromium builds.
for _, chromiumBuild := range []string{*chromiumBuildNoPatch, *chromiumBuildWithPatch} {
if err := gs.DownloadChromiumBuild(chromiumBuild); err != nil {
glog.Error(err)
return
}
//Delete the chromium build to save space when we are done.
defer skutil.RemoveAll(filepath.Join(util.ChromiumBuildsDir, chromiumBuild))
}
chromiumBinaryNoPatch := filepath.Join(util.ChromiumBuildsDir, *chromiumBuildNoPatch, util.BINARY_CHROME)
chromiumBinaryWithPatch := filepath.Join(util.ChromiumBuildsDir, *chromiumBuildWithPatch, util.BINARY_CHROME)
// Download pagesets if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
// Download archives if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
// Establish nopatch output paths.
localOutputDirNoPatch := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runIDNoPatch)
skutil.RemoveAll(localOutputDirNoPatch)
skutil.MkdirAll(localOutputDirNoPatch, 0700)
defer skutil.RemoveAll(localOutputDirNoPatch)
remoteDirNoPatch := filepath.Join(util.BenchmarkRunsDir, *runIDNoPatch)
// Establish withpatch output paths.
localOutputDirWithPatch := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runIDWithPatch)
skutil.RemoveAll(localOutputDirWithPatch)
skutil.MkdirAll(localOutputDirWithPatch, 0700)
defer skutil.RemoveAll(localOutputDirWithPatch)
remoteDirWithPatch := filepath.Join(util.BenchmarkRunsDir, *runIDWithPatch)
// Construct path to the ct_run_benchmark python script.
_, currentFile, _, _ := runtime.Caller(0)
pathToPyFiles := filepath.Join(
filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
"py")
fileInfos, err := ioutil.ReadDir(pathToPagesets)
if err != nil {
glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
return
}
// Loop through all pagesets and run once without the patch and once with the
// patch.
for _, fileInfo := range fileInfos {
if err := runBenchmark(fileInfo.Name(), pathToPagesets, pathToPyFiles, localOutputDirNoPatch, *chromiumBuildNoPatch, chromiumBinaryNoPatch, *runIDNoPatch, *browserExtraArgsNoPatch); err != nil {
glog.Errorf("Error while running nopatch benchmark: %s", err)
return
}
if err := runBenchmark(fileInfo.Name(), pathToPagesets, pathToPyFiles, localOutputDirWithPatch, *chromiumBuildWithPatch, chromiumBinaryWithPatch, *runIDWithPatch, *browserExtraArgsWithPatch); err != nil {
glog.Errorf("Error while running withpatch benchmark: %s", err)
return
}
}
// If "--output-format=csv-pivot-table" was specified then merge all CSV files and upload.
if strings.Contains(*benchmarkExtraArgs, "--output-format=csv-pivot-table") {
if err := mergeUploadCSVFiles(localOutputDirNoPatch, pathToPyFiles, *runIDNoPatch, remoteDirNoPatch, gs); err != nil {
glog.Errorf("Error while processing nopatch CSV files: %s", err)
return
}
if err := mergeUploadCSVFiles(localOutputDirWithPatch, pathToPyFiles, *runIDWithPatch, remoteDirWithPatch, gs); err != nil {
glog.Errorf("Error while processing withpatch CSV files: %s", err)
return
}
}
}
// CleanTmpDir deletes all tmp files from the caller because telemetry tends to
// generate a lot of temporary artifacts there and they take up root disk space.
func CleanTmpDir() {
files, _ := ioutil.ReadDir(os.TempDir())
for _, f := range files {
util.RemoveAll(filepath.Join(os.TempDir(), f.Name()))
}
}
func main() {
defer common.LogPanic()
worker_common.Init()
if !*worker_common.Local {
defer util.CleanTmpDir()
}
defer util.TimeTrack(time.Now(), "Running Chromium Perf")
defer glog.Flush()
// Validate required arguments.
if *chromiumBuildNoPatch == "" {
glog.Error("Must specify --chromium_build_nopatch")
return
}
if *chromiumBuildWithPatch == "" {
glog.Error("Must specify --chromium_build_withpatch")
return
}
if *runID == "" {
glog.Error("Must specify --run_id")
return
}
if *runIDNoPatch == "" {
glog.Error("Must specify --run_id_nopatch")
return
}
if *runIDWithPatch == "" {
glog.Error("Must specify --run_id_withpatch")
return
}
if *benchmarkName == "" {
glog.Error("Must specify --benchmark_name")
return
}
// Reset the local chromium checkout.
if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
return
}
// Sync the local chromium checkout.
if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
return
}
// Create the task file so that the master knows this worker is still busy.
skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_RUNNING_CHROMIUM_PERF))
defer util.DeleteTaskFile(util.ACTIVITY_RUNNING_CHROMIUM_PERF)
if *targetPlatform == util.PLATFORM_ANDROID {
if err := adb.VerifyLocalDevice(); err != nil {
// Android device missing or offline.
glog.Errorf("Could not find Android device: %s", err)
return
}
// Kill adb server to make sure we start from a clean slate.
skutil.LogErr(util.ExecuteCmd(util.BINARY_ADB, []string{"kill-server"}, []string{},
util.ADB_ROOT_TIMEOUT, nil, nil))
// Make sure adb shell is running as root.
skutil.LogErr(util.ExecuteCmd(util.BINARY_ADB, []string{"root"}, []string{},
util.ADB_ROOT_TIMEOUT, nil, nil))
}
// Instantiate GsUtil object.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Error(err)
return
}
// Download the benchmark patch for this run from Google storage.
benchmarkPatchName := *runID + ".benchmark.patch"
benchmarkPatchLocalPath := filepath.Join(os.TempDir(), benchmarkPatchName)
remoteDir := filepath.Join(util.ChromiumPerfRunsDir, *runID)
benchmarkPatchRemotePath := filepath.Join(remoteDir, benchmarkPatchName)
respBody, err := gs.GetRemoteFileContents(benchmarkPatchRemotePath)
if err != nil {
glog.Errorf("Could not fetch %s: %s", benchmarkPatchRemotePath, err)
return
}
defer skutil.Close(respBody)
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(respBody); err != nil {
glog.Errorf("Could not read from %s: %s", benchmarkPatchRemotePath, err)
return
}
if err := ioutil.WriteFile(benchmarkPatchLocalPath, buf.Bytes(), 0666); err != nil {
glog.Errorf("Unable to create file %s: %s", benchmarkPatchLocalPath, err)
return
}
defer skutil.Remove(benchmarkPatchLocalPath)
// Apply benchmark patch to the local chromium checkout.
if buf.Len() > 10 {
if err := util.ApplyPatch(benchmarkPatchLocalPath, util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not apply Telemetry's patch in %s: %s", util.ChromiumSrcDir, err)
return
}
}
// Download the specified chromium builds.
for _, chromiumBuild := range []string{*chromiumBuildNoPatch, *chromiumBuildWithPatch} {
if err := gs.DownloadChromiumBuild(chromiumBuild); err != nil {
glog.Error(err)
return
}
//Delete the chromium build to save space when we are done.
defer skutil.RemoveAll(filepath.Join(util.ChromiumBuildsDir, chromiumBuild))
}
chromiumBinaryNoPatch := filepath.Join(util.ChromiumBuildsDir, *chromiumBuildNoPatch, util.BINARY_CHROME)
chromiumBinaryWithPatch := filepath.Join(util.ChromiumBuildsDir, *chromiumBuildWithPatch, util.BINARY_CHROME)
// Download pagesets if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
// Download archives if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
// Establish nopatch output paths.
localOutputDirNoPatch := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runIDNoPatch)
skutil.RemoveAll(localOutputDirNoPatch)
skutil.MkdirAll(localOutputDirNoPatch, 0700)
defer skutil.RemoveAll(localOutputDirNoPatch)
remoteDirNoPatch := filepath.Join(util.BenchmarkRunsDir, *runIDNoPatch)
// Establish withpatch output paths.
localOutputDirWithPatch := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runIDWithPatch)
skutil.RemoveAll(localOutputDirWithPatch)
skutil.MkdirAll(localOutputDirWithPatch, 0700)
defer skutil.RemoveAll(localOutputDirWithPatch)
remoteDirWithPatch := filepath.Join(util.BenchmarkRunsDir, *runIDWithPatch)
// Construct path to the ct_run_benchmark python script.
_, currentFile, _, _ := runtime.Caller(0)
pathToPyFiles := filepath.Join(
filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
"py")
fileInfos, err := ioutil.ReadDir(pathToPagesets)
if err != nil {
glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
return
}
numWorkers := WORKER_POOL_SIZE
if *targetPlatform == util.PLATFORM_ANDROID || !*runInParallel {
// Do not run page sets in parallel if the target platform is Android.
// This is because the nopatch/withpatch APK needs to be installed prior to
// each run and this will interfere with the parallel runs. Instead of trying
// to find a complicated solution to this, it makes sense for Android to
// continue to be serial because it will help guard against
// crashes/flakiness/inconsistencies which are more prevalent in mobile runs.
numWorkers = 1
glog.Infoln("===== Going to run the task serially =====")
} else {
glog.Infoln("===== Going to run the task with parallel chrome processes =====")
}
// Create channel that contains all pageset file names. This channel will
// be consumed by the worker pool.
pagesetRequests := util.GetClosedChannelOfPagesets(fileInfos)
var wg sync.WaitGroup
// Use a RWMutex for the chromeProcessesCleaner goroutine to communicate to
// the workers (acting as "readers") when it wants to be the "writer" and
// kill all zombie chrome processes.
var mutex sync.RWMutex
// Loop through workers in the worker pool.
for i := 0; i < numWorkers; i++ {
// Increment the WaitGroup counter.
wg.Add(1)
// Create and run a goroutine closure that captures SKPs.
go func() {
// Decrement the WaitGroup counter when the goroutine completes.
defer wg.Done()
for pagesetName := range pagesetRequests {
mutex.RLock()
if err := runBenchmark(pagesetName, pathToPagesets, pathToPyFiles, localOutputDirNoPatch, *chromiumBuildNoPatch, chromiumBinaryNoPatch, *runIDNoPatch, *browserExtraArgsNoPatch); err != nil {
glog.Errorf("Error while running nopatch benchmark: %s", err)
return
}
if err := runBenchmark(pagesetName, pathToPagesets, pathToPyFiles, localOutputDirWithPatch, *chromiumBuildWithPatch, chromiumBinaryWithPatch, *runIDWithPatch, *browserExtraArgsWithPatch); err != nil {
glog.Errorf("Error while running withpatch benchmark: %s", err)
return
}
mutex.RUnlock()
}
}()
}
if !*worker_common.Local {
// Start the cleaner.
go util.ChromeProcessesCleaner(&mutex, *chromeCleanerTimer)
}
// Wait for all spawned goroutines to complete.
wg.Wait()
// If "--output-format=csv-pivot-table" was specified then merge all CSV files and upload.
if strings.Contains(*benchmarkExtraArgs, "--output-format=csv-pivot-table") {
if err := mergeUploadCSVFiles(localOutputDirNoPatch, pathToPyFiles, *runIDNoPatch, remoteDirNoPatch, gs); err != nil {
glog.Errorf("Error while processing nopatch CSV files: %s", err)
return
}
if err := mergeUploadCSVFiles(localOutputDirWithPatch, pathToPyFiles, *runIDWithPatch, remoteDirWithPatch, gs); err != nil {
glog.Errorf("Error while processing withpatch CSV files: %s", err)
return
}
}
}
func main() {
defer common.LogPanic()
worker_common.Init()
defer util.TimeTrack(time.Now(), "Capturing Archives")
defer glog.Flush()
// Create the task file so that the master knows this worker is still busy.
skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES))
defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES)
if *chromiumBuild == "" {
glog.Error("Must specify --chromium_build")
return
}
// Reset the local chromium checkout.
if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
return
}
// Sync the local chromium checkout.
if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
return
}
// Delete and remake the local webpage archives directory.
pathToArchives := filepath.Join(util.WebArchivesDir, *pagesetType)
skutil.RemoveAll(pathToArchives)
skutil.MkdirAll(pathToArchives, 0700)
// Instantiate GsUtil object.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Error(err)
return
}
// Download the specified chromium build if it does not exist locally.
if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
glog.Error(err)
return
}
// Download pagesets if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)
recordWprBinary := filepath.Join(util.TelemetryBinariesDir, util.BINARY_RECORD_WPR)
timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureArchivesTimeoutSecs
// Loop through all pagesets.
fileInfos, err := ioutil.ReadDir(pathToPagesets)
if err != nil {
glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
return
}
glog.Infof("The %s fileInfos are: %s", len(fileInfos), fileInfos)
for _, fileInfo := range fileInfos {
pagesetBaseName := filepath.Base(fileInfo.Name())
if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
// Ignore timestamp files and .pyc files.
continue
}
// Read the pageset.
pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name())
decodedPageset, err := util.ReadPageset(pagesetPath)
if err != nil {
glog.Errorf("Could not read %s: %s", pagesetPath, err)
return
}
glog.Infof("===== Processing %s =====", pagesetPath)
args := []string{
util.CAPTURE_ARCHIVES_DEFAULT_CT_BENCHMARK,
"--extra-browser-args=--disable-setuid-sandbox",
"--browser=exact",
"--browser-executable=" + chromiumBinary,
"--user-agent=" + decodedPageset.UserAgent,
"--urls-list=" + decodedPageset.UrlsList,
"--archive-data-file=" + decodedPageset.ArchiveDataFile,
}
env := []string{
fmt.Sprintf("PYTHONPATH=%s:$PYTHONPATH", pathToPagesets),
"DISPLAY=:0",
}
skutil.LogErr(util.ExecuteCmd(recordWprBinary, args, env, time.Duration(timeoutSecs)*time.Second, nil, nil))
}
// Write timestamp to the webpage archives dir.
skutil.LogErr(util.CreateTimestampFile(pathToArchives))
// Upload webpage archives dir to Google Storage.
if err := gs.UploadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
}
func TestPDFProcessor(t *testing.T) {
testutils.SkipIfShort(t)
// Get the service account client from meta data or a local config file.
client, err := auth.NewJWTServiceAccountClient("", auth.DEFAULT_JWT_FILENAME, nil, storage.ScopeFullControl)
assert.Nil(t, err)
cacheDir, err := fileutil.EnsureDirExists(CACHE_DIR)
assert.Nil(t, err)
// Clean up after the test.
defer func() {
defer util.RemoveAll(cacheDir)
deleteFolderContent(t, TEST_BUCKET, IMAGES_OUT_DIR, client)
deleteFolderContent(t, TEST_BUCKET, JSON_OUT_DIR, client)
}()
// Configure the processor.
ingesterConf := &sharedconfig.IngesterConfig{
ExtraParams: map[string]string{
CONFIG_INPUT_IMAGES_BUCKET: TEST_BUCKET,
CONFIG_INPUT_IMAGES_DIR: IMAGES_IN_DIR,
CONFIG_OUTPUT_JSON_BUCKET: TEST_BUCKET,
CONFIG_OUTPUT_JSON_DIR: JSON_OUT_DIR,
CONFIG_OUTPUT_IMAGES_BUCKET: TEST_BUCKET,
CONFIG_OUTPUT_IMAGES_DIR: IMAGES_OUT_DIR,
CONFIG_PDF_CACHEDIR: cacheDir,
},
}
processor, err := newPDFProcessor(nil, ingesterConf, client)
assert.Nil(t, err)
// Load the example file and process it.
fsResult, err := ingestion.FileSystemResult(TEST_INGESTION_FILE, "./")
assert.Nil(t, err)
err = processor.Process(fsResult)
assert.Nil(t, err)
// Fetch the json output and parse it.
pProcessor := processor.(*pdfProcessor)
// download the result.
resultFileName := filepath.Join(CACHE_DIR, "result-file.json")
assert.Nil(t, pProcessor.download(TEST_BUCKET, JSON_OUT_DIR, fsResult.Name(), resultFileName))
// Make sure we get the expected result.
fsResult, err = ingestion.FileSystemResult(TEST_INGESTION_FILE, "./")
assert.Nil(t, err)
r, err := fsResult.Open()
assert.Nil(t, err)
fsDMResults, err := goldingestion.ParseDMResultsFromReader(r)
assert.Nil(t, err)
foundResult, err := ingestion.FileSystemResult(resultFileName, "./")
assert.Nil(t, err)
r, err = foundResult.Open()
assert.Nil(t, err)
foundDMResults, err := goldingestion.ParseDMResultsFromReader(r)
assert.Nil(t, err)
dmResult1 := *fsDMResults
dmResult2 := *foundDMResults
dmResult1.Results = nil
dmResult2.Results = nil
assert.Equal(t, dmResult1, dmResult2)
foundIdx := 0
srcResults := fsDMResults.Results
tgtResults := foundDMResults.Results
for _, result := range srcResults {
assert.True(t, foundIdx < len(tgtResults))
if result.Options["ext"] == "pdf" {
for ; (foundIdx < len(tgtResults)) && (result.Key["name"] == tgtResults[foundIdx].Key["name"]); foundIdx++ {
assert.True(t, tgtResults[foundIdx].Key["rasterizer"] != "")
delete(tgtResults[foundIdx].Key, "rasterizer")
assert.Equal(t, result.Key, tgtResults[foundIdx].Key)
assert.Equal(t, "png", tgtResults[foundIdx].Options["ext"])
}
}
}
assert.Equal(t, len(foundDMResults.Results), foundIdx)
}
func main() {
common.Init()
webhook.MustInitRequestSaltFromFile(util.WebhookRequestSaltPath)
// Send start email.
emailsArr := util.ParseEmails(*emails)
emailsArr = append(emailsArr, util.CtAdmins...)
if len(emailsArr) == 0 {
glog.Error("At least one email address must be specified")
return
}
skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&frontend.ChromiumPerfUpdateVars{}, *gaeTaskID))
skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Chromium perf"))
// Ensure webapp is updated and email is sent even if task fails.
defer updateWebappTask()
defer sendEmail(emailsArr)
// Cleanup dirs after run completes.
defer skutil.RemoveAll(filepath.Join(util.StorageDir, util.ChromiumPerfRunsDir))
defer skutil.RemoveAll(filepath.Join(util.StorageDir, util.BenchmarkRunsDir))
// Cleanup tmp files after the run.
defer util.CleanTmpDir()
// Finish with glog flush and how long the task took.
defer util.TimeTrack(time.Now(), "Running chromium perf task on workers")
defer glog.Flush()
if *pagesetType == "" {
glog.Error("Must specify --pageset_type")
return
}
if *benchmarkName == "" {
glog.Error("Must specify --benchmark_name")
return
}
if *runID == "" {
glog.Error("Must specify --run_id")
return
}
// Instantiate GsUtil object.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Errorf("Could not instantiate gsutil object: %s", err)
return
}
remoteOutputDir := filepath.Join(util.ChromiumPerfRunsDir, *runID)
// Copy the patches to Google Storage.
skiaPatchName := *runID + ".skia.patch"
blinkPatchName := *runID + ".blink.patch"
chromiumPatchName := *runID + ".chromium.patch"
for _, patchName := range []string{skiaPatchName, blinkPatchName, chromiumPatchName} {
if err := gs.UploadFile(patchName, os.TempDir(), remoteOutputDir); err != nil {
glog.Errorf("Could not upload %s to %s: %s", patchName, remoteOutputDir, err)
return
}
}
skiaPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteOutputDir, skiaPatchName)
blinkPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteOutputDir, blinkPatchName)
chromiumPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteOutputDir, chromiumPatchName)
// Create the two required chromium builds (with patch and without the patch).
chromiumHash, skiaHash, err := util.CreateChromiumBuild(*runID, *targetPlatform, "", "", true)
if err != nil {
glog.Errorf("Could not create chromium build: %s", err)
return
}
// Reboot all workers to start from a clean slate.
util.RebootWorkers()
// Run the run_chromium_perf script on all workers.
runIDNoPatch := *runID + "-nopatch"
runIDWithPatch := *runID + "-withpatch"
chromiumBuildNoPatch := fmt.Sprintf("try-%s-%s-%s", chromiumHash, skiaHash, runIDNoPatch)
chromiumBuildWithPatch := fmt.Sprintf("try-%s-%s-%s", chromiumHash, skiaHash, runIDWithPatch)
runChromiumPerfCmdTemplate := "DISPLAY=:0 run_chromium_perf " +
"--worker_num={{.WorkerNum}} --log_dir={{.LogDir}} --pageset_type={{.PagesetType}} " +
"--chromium_build_nopatch={{.ChromiumBuildNoPatch}} --chromium_build_withpatch={{.ChromiumBuildWithPatch}} " +
"--run_id_nopatch={{.RunIDNoPatch}} --run_id_withpatch={{.RunIDWithPatch}} " +
"--benchmark_name={{.BenchmarkName}} --benchmark_extra_args=\"{{.BenchmarkExtraArgs}}\" " +
"--browser_extra_args_nopatch=\"{{.BrowserExtraArgsNoPatch}}\" --browser_extra_args_withpatch=\"{{.BrowserExtraArgsWithPatch}}\" " +
"--repeat_benchmark={{.RepeatBenchmark}} --target_platform={{.TargetPlatform}};"
runChromiumPerfTemplateParsed := template.Must(template.New("run_chromium_perf_cmd").Parse(runChromiumPerfCmdTemplate))
runChromiumPerfCmdBytes := new(bytes.Buffer)
if err := runChromiumPerfTemplateParsed.Execute(runChromiumPerfCmdBytes, struct {
WorkerNum string
LogDir string
PagesetType string
ChromiumBuildNoPatch string
ChromiumBuildWithPatch string
RunIDNoPatch string
RunIDWithPatch string
BenchmarkName string
BenchmarkExtraArgs string
BrowserExtraArgsNoPatch string
BrowserExtraArgsWithPatch string
RepeatBenchmark int
TargetPlatform string
}{
WorkerNum: util.WORKER_NUM_KEYWORD,
LogDir: util.GLogDir,
PagesetType: *pagesetType,
ChromiumBuildNoPatch: chromiumBuildNoPatch,
ChromiumBuildWithPatch: chromiumBuildWithPatch,
RunIDNoPatch: runIDNoPatch,
RunIDWithPatch: runIDWithPatch,
BenchmarkName: *benchmarkName,
BenchmarkExtraArgs: *benchmarkExtraArgs,
BrowserExtraArgsNoPatch: *browserExtraArgsNoPatch,
BrowserExtraArgsWithPatch: *browserExtraArgsWithPatch,
RepeatBenchmark: *repeatBenchmark,
TargetPlatform: *targetPlatform,
}); err != nil {
glog.Errorf("Failed to execute template: %s", err)
return
}
cmd := []string{
fmt.Sprintf("cd %s;", util.CtTreeDir),
"git pull;",
"make all;",
// The main command that runs run_chromium_perf on all workers.
runChromiumPerfCmdBytes.String(),
}
// Setting a 1 day timeout since it may take a while run benchmarks with many
// repeats.
if _, err := util.SSH(strings.Join(cmd, " "), util.Slaves, 1*24*time.Hour); err != nil {
glog.Errorf("Error while running cmd %s: %s", cmd, err)
return
}
// If "--output-format=csv-pivot-table" was specified then merge all CSV files and upload.
if strings.Contains(*benchmarkExtraArgs, "--output-format=csv-pivot-table") {
for _, runID := range []string{runIDNoPatch, runIDWithPatch} {
if err := mergeUploadCSVFiles(runID, gs); err != nil {
glog.Errorf("Unable to merge and upload CSV files for %s: %s", runID, err)
}
}
}
// Compare the resultant CSV files using csv_comparer.py
noPatchCSVPath := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, runIDNoPatch, runIDNoPatch+".output")
withPatchCSVPath := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, runIDWithPatch, runIDWithPatch+".output")
htmlOutputDir := filepath.Join(util.StorageDir, util.ChromiumPerfRunsDir, *runID, "html")
skutil.MkdirAll(htmlOutputDir, 0700)
htmlRemoteDir := filepath.Join(remoteOutputDir, "html")
htmlOutputLinkBase := util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, htmlRemoteDir) + "/"
htmlOutputLink = htmlOutputLinkBase + "index.html"
noPatchOutputLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, util.BenchmarkRunsDir, runIDNoPatch, "consolidated_outputs", runIDNoPatch+".output")
withPatchOutputLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, util.BenchmarkRunsDir, runIDWithPatch, "consolidated_outputs", runIDWithPatch+".output")
// Construct path to the csv_comparer python script.
_, currentFile, _, _ := runtime.Caller(0)
pathToPyFiles := filepath.Join(
filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
"py")
pathToCsvComparer := filepath.Join(pathToPyFiles, "csv_comparer.py")
args := []string{
pathToCsvComparer,
"--csv_file1=" + noPatchCSVPath,
"--csv_file2=" + withPatchCSVPath,
"--output_html=" + htmlOutputDir,
"--variance_threshold=" + strconv.FormatFloat(*varianceThreshold, 'f', 2, 64),
"--discard_outliers=" + strconv.FormatFloat(*discardOutliers, 'f', 2, 64),
"--absolute_url=" + htmlOutputLinkBase,
"--requester_email=" + *emails,
"--skia_patch_link=" + skiaPatchLink,
"--blink_patch_link=" + blinkPatchLink,
"--chromium_patch_link=" + chromiumPatchLink,
"--raw_csv_nopatch=" + noPatchOutputLink,
"--raw_csv_withpatch=" + withPatchOutputLink,
"--num_repeated=" + strconv.Itoa(*repeatBenchmark),
"--target_platform=" + *targetPlatform,
"--browser_args_nopatch=" + *browserExtraArgsNoPatch,
"--browser_args_withpatch=" + *browserExtraArgsWithPatch,
"--pageset_type=" + *pagesetType,
"--chromium_hash=" + chromiumHash,
"--skia_hash=" + skiaHash,
}
if err := util.ExecuteCmd("python", args, []string{}, 2*time.Hour, nil, nil); err != nil {
glog.Errorf("Error running csv_comparer.py: %s", err)
return
}
// Copy the HTML files to Google Storage.
if err := gs.UploadDir(htmlOutputDir, htmlRemoteDir, true); err != nil {
glog.Errorf("Could not upload %s to %s: %s", htmlOutputDir, htmlRemoteDir, err)
return
}
taskCompletedSuccessfully = true
}
func TestShareDB(t *testing.T) {
// Create the server and start it.
serverImpl := NewServer(DATA_DIR)
defer util.RemoveAll(DATA_DIR)
grpcServer, client, err := startServer(t, serverImpl)
assert.Nil(t, err)
defer grpcServer.Stop()
dbName := "database001"
bucketName := "bucket_01"
ctx := context.Background()
allKeys := []string{}
for k := 0; k < MAX_KEYS; k++ {
key := fmt.Sprintf("key_%04d", k)
value := fmt.Sprintf("val_%04d", k)
ack, err := client.Put(ctx, &PutRequest{dbName, bucketName, key, []byte(value)})
assert.Nil(t, err)
assert.True(t, ack.Ok)
foundResp, err := client.Get(ctx, &GetRequest{dbName, bucketName, key})
assert.Nil(t, err)
assert.Equal(t, value, string(foundResp.Value))
allKeys = append(allKeys, key)
}
foundDBs, err := client.Databases(ctx, &DatabasesRequest{})
assert.Nil(t, err)
assert.Equal(t, []string{dbName}, foundDBs.Values)
foundBuckets, err := client.Buckets(ctx, &BucketsRequest{dbName})
assert.Nil(t, err)
assert.Equal(t, []string{bucketName}, foundBuckets.Values)
foundKeys, err := client.Keys(ctx, &KeysRequest{dbName, bucketName, "", "", ""})
assert.Nil(t, err)
sort.Strings(foundKeys.Values)
sort.Strings(allKeys)
assert.Equal(t, allKeys, foundKeys.Values)
// Test a min-max range scan.
foundKeys, err = client.Keys(ctx, &KeysRequest{dbName, bucketName, "", "key_0010", "key_0015"})
assert.Nil(t, err)
assert.Equal(t, []string{"key_0010", "key_0011", "key_0012", "key_0013", "key_0014", "key_0015"}, foundKeys.Values)
// Test a min to end range scan.
foundKeys, err = client.Keys(ctx, &KeysRequest{dbName, bucketName, "", "key_0015", ""})
assert.Nil(t, err)
assert.Equal(t, []string{"key_0015", "key_0016", "key_0017", "key_0018", "key_0019"}, foundKeys.Values[0:5])
// Test a start to max range scan.
foundKeys, err = client.Keys(ctx, &KeysRequest{dbName, bucketName, "", "", "key_0004"})
assert.Nil(t, err)
assert.Equal(t, []string{"key_0000", "key_0001", "key_0002", "key_0003", "key_0004"}, foundKeys.Values)
// Test a prefix scan.
foundKeys, err = client.Keys(ctx, &KeysRequest{dbName, bucketName, "key_000", "", ""})
assert.Nil(t, err)
exp := []string{"key_0000", "key_0001", "key_0002", "key_0003", "key_0004",
"key_0005", "key_0006", "key_0007", "key_0008", "key_0009"}
assert.Equal(t, exp, foundKeys.Values)
for _, k := range allKeys {
ack, err := client.Delete(ctx, &DeleteRequest{dbName, bucketName, k})
assert.Nil(t, err)
assert.True(t, ack.Ok)
foundVal, err := client.Get(ctx, &GetRequest{dbName, bucketName, k})
assert.Nil(t, err)
assert.Nil(t, foundVal.Value)
}
foundKeys, err = client.Keys(ctx, &KeysRequest{dbName, bucketName, "", "", ""})
assert.Nil(t, err)
assert.Equal(t, 0, len(foundKeys.Values))
foundResp, err := client.Get(ctx, &GetRequest{"Does-not-exist-either", bucketName, "does-not-exist"})
assert.NotNil(t, err)
assert.True(t, strings.Contains(err.Error(), "Does-not-exist-either"))
foundResp, err = client.Get(ctx, &GetRequest{dbName, bucketName, "does-not-exist"})
assert.Nil(t, err)
assert.Equal(t, "", string(foundResp.Value))
}