Exemple #1
0
func (task *LuaScriptTask) Execute() error {
	runId := runId(task)
	chromiumBuildDir := ctutil.ChromiumBuildDir(task.ChromiumRev, task.SkiaRev, "")
	// TODO(benjaminwagner): Since run_lua_on_workers only reads the lua script in order to
	// upload to Google Storage, eventually we should move the upload step here to avoid writing
	// to disk. Not sure if we can/should do the same for the aggregator script.
	luaScriptName := runId + ".lua"
	luaScriptPath := filepath.Join(os.TempDir(), luaScriptName)
	if err := ioutil.WriteFile(luaScriptPath, []byte(task.LuaScript), 0666); err != nil {
		return err
	}
	defer skutil.Remove(luaScriptPath)
	if task.LuaAggregatorScript != "" {
		luaAggregatorName := runId + ".aggregator"
		luaAggregatorPath := filepath.Join(os.TempDir(), luaAggregatorName)
		if err := ioutil.WriteFile(luaAggregatorPath, []byte(task.LuaAggregatorScript), 0666); err != nil {
			return err
		}
		defer skutil.Remove(luaAggregatorPath)
	}
	return exec.Run(&exec.Command{
		Name: "run_lua_on_workers",
		Args: []string{
			"--emails=" + task.Username,
			"--description=" + task.Description,
			"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
			"--pageset_type=" + task.PageSets,
			"--chromium_build=" + chromiumBuildDir,
			"--run_id=" + runId,
			"--log_dir=" + logDir,
			"--log_id=" + runId,
		},
		Timeout: ctutil.MASTER_SCRIPT_RUN_LUA_TIMEOUT,
	})
}
// download fetches the content of the given location in GS and stores it at the given
// output path.
func (p *pdfProcessor) download(bucket, dir, fileName, outputPath string) error {
	objectPath := dir + "/" + fileName
	r, err := p.storageClient.Bucket(bucket).Object(objectPath).NewReader(context.Background())
	if err != nil {
		return err
	}
	defer util.Close(r)

	tempFile, err := ioutil.TempFile(p.pdfCacheDir, "pdfingestion-download")
	if err != nil {
		return err
	}

	_, err = io.Copy(tempFile, r)
	util.Close(tempFile)
	if err != nil {
		util.Remove(tempFile.Name())
		return err
	}

	if err := os.Rename(tempFile.Name(), outputPath); err != nil {
		return err
	}

	glog.Infof("Downloaded: %s/%s", bucket, objectPath)
	return nil
}
func TestMustInitRequestSaltFromFileSuccess(t *testing.T) {
	f, err := ioutil.TempFile("", "webhook_test_salt")
	assert.NoError(t, err)
	defer util.Remove(f.Name())
	_, err = f.WriteString(TEST_SALT_BASE64)
	assert.NoError(t, err)
	MustInitRequestSaltFromFile(f.Name())
	expect.Equal(t, []byte(TEST_SALT), requestSalt)
}
Exemple #4
0
func mergeUploadCSVFiles(runID string, gs *util.GsUtil) error {
	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, runID)
	skutil.MkdirAll(localOutputDir, 0700)
	// Copy outputs from all slaves locally.
	for i := 0; i < util.NUM_WORKERS; i++ {
		workerNum := i + 1
		workerLocalOutputPath := filepath.Join(localOutputDir, fmt.Sprintf("slave%d", workerNum)+".csv")
		workerRemoteOutputPath := filepath.Join(util.BenchmarkRunsDir, runID, fmt.Sprintf("slave%d", workerNum), "outputs", runID+".output")
		respBody, err := gs.GetRemoteFileContents(workerRemoteOutputPath)
		if err != nil {
			glog.Errorf("Could not fetch %s: %s", workerRemoteOutputPath, err)
			// TODO(rmistry): Should we instead return here? We can only return
			// here if all 100 slaves reliably run without any failures which they
			// really should.
			continue
		}
		defer skutil.Close(respBody)
		out, err := os.Create(workerLocalOutputPath)
		if err != nil {
			return fmt.Errorf("Unable to create file %s: %s", workerLocalOutputPath, err)
		}
		defer skutil.Close(out)
		defer skutil.Remove(workerLocalOutputPath)
		if _, err = io.Copy(out, respBody); err != nil {
			return fmt.Errorf("Unable to copy to file %s: %s", workerLocalOutputPath, err)
		}
	}
	// Call csv_merger.py to merge all results into a single results CSV.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
		"py")
	pathToCsvMerger := filepath.Join(pathToPyFiles, "csv_merger.py")
	outputFileName := runID + ".output"
	args := []string{
		pathToCsvMerger,
		"--csv_dir=" + localOutputDir,
		"--output_csv_name=" + filepath.Join(localOutputDir, outputFileName),
	}
	if err := util.ExecuteCmd("python", args, []string{}, 1*time.Hour, nil, nil); err != nil {
		return fmt.Errorf("Error running csv_merger.py: %s", err)
	}
	// Copy the output file to Google Storage.
	remoteOutputDir := filepath.Join(util.BenchmarkRunsDir, runID, "consolidated_outputs")
	if err := gs.UploadFile(outputFileName, localOutputDir, remoteOutputDir); err != nil {
		return fmt.Errorf("Unable to upload %s to %s: %s", outputFileName, remoteOutputDir, err)
	}
	return nil
}
Exemple #5
0
func mergeUploadCSVFiles(runID string, gs *util.GsUtil) ([]string, error) {
	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, runID)
	skutil.MkdirAll(localOutputDir, 0700)
	noOutputSlaves := []string{}
	// Copy outputs from all slaves locally.
	for i := 0; i < util.NumWorkers(); i++ {
		workerNum := i + 1
		workerLocalOutputPath := filepath.Join(localOutputDir, fmt.Sprintf("slave%d", workerNum)+".csv")
		workerRemoteOutputPath := filepath.Join(util.BenchmarkRunsDir, runID, fmt.Sprintf("slave%d", workerNum), "outputs", runID+".output")
		respBody, err := gs.GetRemoteFileContents(workerRemoteOutputPath)
		if err != nil {
			glog.Errorf("Could not fetch %s: %s", workerRemoteOutputPath, err)
			noOutputSlaves = append(noOutputSlaves, fmt.Sprintf(util.WORKER_NAME_TEMPLATE, workerNum))
			continue
		}
		defer skutil.Close(respBody)
		out, err := os.Create(workerLocalOutputPath)
		if err != nil {
			return noOutputSlaves, fmt.Errorf("Unable to create file %s: %s", workerLocalOutputPath, err)
		}
		defer skutil.Close(out)
		defer skutil.Remove(workerLocalOutputPath)
		if _, err = io.Copy(out, respBody); err != nil {
			return noOutputSlaves, fmt.Errorf("Unable to copy to file %s: %s", workerLocalOutputPath, err)
		}
	}
	// Call csv_merger.py to merge all results into a single results CSV.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
		"py")
	pathToCsvMerger := filepath.Join(pathToPyFiles, "csv_merger.py")
	outputFileName := runID + ".output"
	args := []string{
		pathToCsvMerger,
		"--csv_dir=" + localOutputDir,
		"--output_csv_name=" + filepath.Join(localOutputDir, outputFileName),
	}
	err := util.ExecuteCmd("python", args, []string{}, util.CSV_MERGER_TIMEOUT, nil, nil)
	if err != nil {
		return noOutputSlaves, fmt.Errorf("Error running csv_merger.py: %s", err)
	}
	// Copy the output file to Google Storage.
	remoteOutputDir := filepath.Join(util.BenchmarkRunsDir, runID, "consolidated_outputs")
	if err := gs.UploadFile(outputFileName, localOutputDir, remoteOutputDir); err != nil {
		return noOutputSlaves, fmt.Errorf("Unable to upload %s to %s: %s", outputFileName, remoteOutputDir, err)
	}
	return noOutputSlaves, nil
}
Exemple #6
0
func (task *ChromiumPerfTask) Execute() error {
	token := statusTracker.StartTask(CHROMIUM_PERF)
	runId := runId(task)
	// TODO(benjaminwagner): Since run_chromium_perf_on_workers only reads these in order to
	// upload to Google Storage, eventually we should move the upload step here to avoid writing
	// to disk.
	for fileSuffix, patch := range map[string]string{
		".chromium.patch":  task.ChromiumPatch,
		".skia.patch":      task.SkiaPatch,
		".benchmark.patch": task.BenchmarkPatch,
	} {
		// Add an extra newline at the end because git sometimes rejects patches due to
		// missing newlines.
		patch = patch + "\n"
		patchPath := filepath.Join(os.TempDir(), runId+fileSuffix)
		if err := ioutil.WriteFile(patchPath, []byte(patch), 0666); err != nil {
			return err
		}
		defer skutil.Remove(patchPath)
	}
	err := exec.Run(&exec.Command{
		Name: "run_chromium_perf_on_workers",
		Args: []string{
			"--emails=" + task.Username,
			"--description=" + task.Description,
			"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
			"--pageset_type=" + task.PageSets,
			"--benchmark_name=" + task.Benchmark,
			"--benchmark_extra_args=" + task.BenchmarkArgs,
			"--browser_extra_args_nopatch=" + task.BrowserArgsNoPatch,
			"--browser_extra_args_withpatch=" + task.BrowserArgsWithPatch,
			"--repeat_benchmark=" + strconv.FormatInt(task.RepeatRuns, 10),
			"--run_in_parallel=" + strconv.FormatBool(task.RunInParallel),
			"--target_platform=" + task.Platform,
			"--run_id=" + runId,
			"--log_dir=" + logDir,
			"--log_id=" + runId,
			fmt.Sprintf("--local=%t", *master_common.Local),
		},
		Timeout: ctutil.MASTER_SCRIPT_RUN_CHROMIUM_PERF_TIMEOUT,
	})
	statusTracker.FinishTask(token, err)
	return err
}
func getChromiumHash() (string, error) {
	// Find Chromium's Tot commit hash.
	stdoutFilePath := filepath.Join(os.TempDir(), "chromium-tot")
	stdoutFile, err := os.Create(stdoutFilePath)
	defer util.Close(stdoutFile)
	defer util.Remove(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Could not create %s: %s", stdoutFilePath, err)
	}
	totArgs := []string{"ls-remote", "https://chromium.googlesource.com/chromium/src.git", "--verify", "refs/heads/master"}
	if err := ExecuteCmd(BINARY_GIT, totArgs, []string{}, time.Minute*5, stdoutFile, nil); err != nil {
		return "", fmt.Errorf("Error while finding Chromium's ToT: %s", err)
	}
	output, err := ioutil.ReadFile(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Cannot read %s: %s", stdoutFilePath, err)
	}
	tokens := strings.Split(string(output), "\t")
	return tokens[0], nil
}
func TestTaskFileUtils(t *testing.T) {
	TaskFileDir = os.TempDir()
	taskFilePath := filepath.Join(TaskFileDir, TEST_FILE_NAME)
	defer util.Remove(taskFilePath)

	// Assert that the task file is created.
	if err := CreateTaskFile(TEST_FILE_NAME); err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if _, err := os.Stat(taskFilePath); err != nil {
		t.Errorf("Task file %s was not created!", taskFilePath)
	}

	// Assert that DeleteTaskFile deletes the task file.
	DeleteTaskFile(TEST_FILE_NAME)
	if _, err := os.Stat(taskFilePath); err != nil {
		// Expected error
	} else {
		t.Error("Unexpected lack of error")
	}
}
Exemple #9
0
// Rasterize assumes that filepath.Dir(pdfInputPath) is writable
func (Pdfium) Rasterize(pdfInputPath, pngOutputPath string) error {
	if !(Pdfium{}).Enabled() {
		return fmt.Errorf("pdfium_test is missing")
	}

	// Check input
	if !fileutil.FileExists(pdfInputPath) {
		return fmt.Errorf("Path '%s' does not exist", pdfInputPath)
	}

	// Remove any files created by pdfiumExecutable
	defer func() {
		// Assume pdfInputPath has glob characters.
		matches, _ := filepath.Glob(fmt.Sprintf("%s.*.png", pdfInputPath))
		for _, match := range matches {
			util.Remove(match)
		}
	}()

	command := exec.Command(pdfiumExecutable, "--png", pdfInputPath)
	if err := command.Start(); err != nil {
		return err
	}
	go func() {
		time.Sleep(5 * time.Second)
		_ = command.Process.Kill()
	}()
	if err := command.Wait(); err != nil {
		return err
	}

	firstPagePath := fmt.Sprintf("%s.0.png", pdfInputPath)
	if !fileutil.FileExists(firstPagePath) {
		return fmt.Errorf("First rasterized page (%s) not found.", firstPagePath)
	}
	if err := os.Rename(firstPagePath, pngOutputPath); err != nil {
		return err
	}
	return nil
}
Exemple #10
0
func main() {
	defer common.LogPanic()
	common.Init()
	defer util.TimeTrack(time.Now(), "Creating Pagesets")
	defer glog.Flush()
	// Create the task file so that the master knows this worker is still busy.
	skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CREATING_PAGESETS))
	defer util.DeleteTaskFile(util.ACTIVITY_CREATING_PAGESETS)

	// Delete and remake the local pagesets directory.
	pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
	skutil.RemoveAll(pathToPagesets)
	skutil.MkdirAll(pathToPagesets, 0700)

	// Get info about the specified pageset type.
	pagesetTypeInfo := util.PagesetTypeToInfo[*pagesetType]
	csvSource := pagesetTypeInfo.CSVSource
	numPages := pagesetTypeInfo.NumPages
	userAgent := pagesetTypeInfo.UserAgent

	// Download the CSV file from Google Storage to a tmp location.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}
	respBody, err := gs.GetRemoteFileContents(csvSource)
	if err != nil {
		glog.Error(err)
		return
	}
	defer skutil.Close(respBody)
	csvFile := filepath.Join(os.TempDir(), filepath.Base(csvSource))
	out, err := os.Create(csvFile)
	if err != nil {
		glog.Errorf("Unable to create file %s: %s", csvFile, err)
		return
	}
	defer skutil.Close(out)
	defer skutil.Remove(csvFile)
	if _, err = io.Copy(out, respBody); err != nil {
		glog.Error(err)
		return
	}

	// Figure out which pagesets this worker should generate.
	numPagesPerSlave := numPages / util.NUM_WORKERS
	startNum := (*workerNum-1)*numPagesPerSlave + 1
	endNum := *workerNum * numPagesPerSlave

	// Construct path to the create_page_set.py python script.
	_, currentFile, _, _ := runtime.Caller(0)
	createPageSetScript := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
		"py", "create_page_set.py")

	// Execute the create_page_set.py python script.
	timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CreatePagesetsTimeoutSecs
	for currNum := startNum; currNum <= endNum; currNum++ {
		args := []string{
			createPageSetScript,
			"-s", strconv.Itoa(currNum),
			"-e", strconv.Itoa(currNum),
			"-c", csvFile,
			"-p", *pagesetType,
			"-u", userAgent,
			"-o", pathToPagesets,
		}
		if err := util.ExecuteCmd("python", args, []string{}, time.Duration(timeoutSecs)*time.Second, nil, nil); err != nil {
			glog.Error(err)
			return
		}
	}
	// Write timestamp to the pagesets dir.
	skutil.LogErr(util.CreateTimestampFile(pathToPagesets))

	// Upload pagesets dir to Google Storage.
	if err := gs.UploadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}
}
Exemple #11
0
func main() {
	defer common.LogPanic()
	common.Init()
	defer util.TimeTrack(time.Now(), "Running Lua Scripts")
	defer glog.Flush()

	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *runID == "" {
		glog.Error("Must specify --run_id")
		return
	}

	// Create the task file so that the master knows this worker is still busy.
	skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_RUNNING_LUA_SCRIPTS))
	defer util.DeleteTaskFile(util.ACTIVITY_RUNNING_LUA_SCRIPTS)

	// Sync Skia tree.
	skutil.LogErr(util.SyncDir(util.SkiaTreeDir))

	// Build tools.
	skutil.LogErr(util.BuildSkiaTools())

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Download SKPs if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.SKPS_DIR_NAME, filepath.Join(*pagesetType, *chromiumBuild), *workerNum); err != nil {
		glog.Error(err)
		return
	}
	localSkpsDir := filepath.Join(util.SkpsDir, *pagesetType, *chromiumBuild)

	// Download the lua script for this run from Google storage.
	luaScriptName := *runID + ".lua"
	luaScriptLocalPath := filepath.Join(os.TempDir(), luaScriptName)
	remoteDir := filepath.Join(util.LuaRunsDir, *runID)
	luaScriptRemotePath := filepath.Join(remoteDir, "scripts", luaScriptName)
	respBody, err := gs.GetRemoteFileContents(luaScriptRemotePath)
	if err != nil {
		glog.Errorf("Could not fetch %s: %s", luaScriptRemotePath, err)
		return
	}
	defer skutil.Close(respBody)
	out, err := os.Create(luaScriptLocalPath)
	if err != nil {
		glog.Errorf("Unable to create file %s: %s", luaScriptLocalPath, err)
		return
	}
	defer skutil.Close(out)
	defer skutil.Remove(luaScriptLocalPath)
	if _, err = io.Copy(out, respBody); err != nil {
		glog.Error(err)
		return
	}

	// Run lua_pictures and save stdout and stderr in files.
	stdoutFileName := *runID + ".output"
	stdoutFilePath := filepath.Join(os.TempDir(), stdoutFileName)
	stdoutFile, err := os.Create(stdoutFilePath)
	defer skutil.Close(stdoutFile)
	defer skutil.Remove(stdoutFilePath)
	if err != nil {
		glog.Errorf("Could not create %s: %s", stdoutFilePath, err)
		return
	}
	stderrFileName := *runID + ".err"
	stderrFilePath := filepath.Join(os.TempDir(), stderrFileName)
	stderrFile, err := os.Create(stderrFilePath)
	defer skutil.Close(stderrFile)
	defer skutil.Remove(stderrFilePath)
	if err != nil {
		glog.Errorf("Could not create %s: %s", stderrFilePath, err)
		return
	}
	args := []string{
		"--skpPath", localSkpsDir,
		"--luaFile", luaScriptLocalPath,
	}
	err = util.ExecuteCmd(
		filepath.Join(util.SkiaTreeDir, "out", "Release", util.BINARY_LUA_PICTURES), args,
		[]string{}, util.LUA_PICTURES_TIMEOUT, stdoutFile, stderrFile)
	if err != nil {
		glog.Error(err)
		return
	}

	// Copy stdout and stderr files to Google Storage.
	skutil.LogErr(
		gs.UploadFile(stdoutFileName, os.TempDir(), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "outputs")))
	skutil.LogErr(
		gs.UploadFile(stderrFileName, os.TempDir(), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "errors")))
}
Exemple #12
0
func main() {
	common.Init()
	webhook.MustInitRequestSaltFromFile(util.WebhookRequestSaltPath)

	// Send start email.
	emailsArr := util.ParseEmails(*emails)
	emailsArr = append(emailsArr, util.CtAdmins...)
	if len(emailsArr) == 0 {
		glog.Error("At least one email address must be specified")
		return
	}
	skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&frontend.LuaScriptUpdateVars{}, *gaeTaskID))
	skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Lua script"))
	// Ensure webapp is updated and email is sent even if task fails.
	defer updateWebappTask()
	defer sendEmail(emailsArr)
	// Cleanup tmp files after the run.
	defer util.CleanTmpDir()
	// Finish with glog flush and how long the task took.
	defer util.TimeTrack(time.Now(), "Running Lua script on workers")
	defer glog.Flush()

	if *pagesetType == "" {
		glog.Error("Must specify --pageset_type")
		return
	}
	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *runID == "" {
		glog.Error("Must specify --run_id")
		return
	}

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Upload the lua script for this run to Google storage.
	luaScriptName := *runID + ".lua"
	defer skutil.Remove(filepath.Join(os.TempDir(), luaScriptName))
	luaScriptRemoteDir := filepath.Join(util.LuaRunsDir, *runID, "scripts")
	luaScriptRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, luaScriptRemoteDir, luaScriptName)
	if err := gs.UploadFile(luaScriptName, os.TempDir(), luaScriptRemoteDir); err != nil {
		glog.Errorf("Could not upload %s to %s: %s", luaScriptName, luaScriptRemoteDir, err)
		return
	}

	// Run the run_lua script on all workers.
	runLuaCmdTemplate := "DISPLAY=:0 run_lua --worker_num={{.WorkerNum}} --log_dir={{.LogDir}} --pageset_type={{.PagesetType}} --chromium_build={{.ChromiumBuild}} --run_id={{.RunID}};"
	runLuaTemplateParsed := template.Must(template.New("run_lua_cmd").Parse(runLuaCmdTemplate))
	luaCmdBytes := new(bytes.Buffer)
	if err := runLuaTemplateParsed.Execute(luaCmdBytes, struct {
		WorkerNum     string
		LogDir        string
		PagesetType   string
		ChromiumBuild string
		RunID         string
	}{
		WorkerNum:     util.WORKER_NUM_KEYWORD,
		LogDir:        util.GLogDir,
		PagesetType:   *pagesetType,
		ChromiumBuild: *chromiumBuild,
		RunID:         *runID,
	}); err != nil {
		glog.Errorf("Failed to execute template: %s", err)
		return
	}
	cmd := []string{
		fmt.Sprintf("cd %s;", util.CtTreeDir),
		"git pull;",
		"make all;",
		// The main command that runs run_lua on all workers.
		luaCmdBytes.String(),
	}
	if _, err := util.SSH(strings.Join(cmd, " "), util.Slaves, 2*time.Hour); err != nil {
		glog.Errorf("Error while running cmd %s: %s", cmd, err)
		return
	}

	// Copy outputs from all slaves locally and combine it into one file.
	consolidatedFileName := "lua-output"
	consolidatedLuaOutput := filepath.Join(os.TempDir(), consolidatedFileName)
	if err := ioutil.WriteFile(consolidatedLuaOutput, []byte{}, 0660); err != nil {
		glog.Errorf("Could not create %s: %s", consolidatedLuaOutput, err)
		return
	}
	for i := 0; i < util.NUM_WORKERS; i++ {
		workerNum := i + 1
		workerRemoteOutputPath := filepath.Join(util.LuaRunsDir, *runID, fmt.Sprintf("slave%d", workerNum), "outputs", *runID+".output")
		respBody, err := gs.GetRemoteFileContents(workerRemoteOutputPath)
		if err != nil {
			glog.Errorf("Could not fetch %s: %s", workerRemoteOutputPath, err)
			// TODO(rmistry): Should we instead return here? We can only return
			// here if all 100 slaves reliably run without any failures which they
			// really should.
			continue
		}
		defer skutil.Close(respBody)
		out, err := os.OpenFile(consolidatedLuaOutput, os.O_RDWR|os.O_APPEND, 0660)
		if err != nil {
			glog.Errorf("Unable to open file %s: %s", consolidatedLuaOutput, err)
			return
		}
		defer skutil.Close(out)
		defer skutil.Remove(consolidatedLuaOutput)
		if _, err = io.Copy(out, respBody); err != nil {
			glog.Errorf("Unable to write out %s to %s: %s", workerRemoteOutputPath, consolidatedLuaOutput, err)
			return
		}
	}
	// Copy the consolidated file into Google Storage.
	consolidatedOutputRemoteDir := filepath.Join(util.LuaRunsDir, *runID, "consolidated_outputs")
	luaOutputRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, consolidatedOutputRemoteDir, consolidatedFileName)
	if err := gs.UploadFile(consolidatedFileName, os.TempDir(), consolidatedOutputRemoteDir); err != nil {
		glog.Errorf("Unable to upload %s to %s: %s", consolidatedLuaOutput, consolidatedOutputRemoteDir, err)
		return
	}

	// Upload the lua aggregator (if specified) for this run to Google storage.
	luaAggregatorName := *runID + ".aggregator"
	luaAggregatorPath := filepath.Join(os.TempDir(), luaAggregatorName)
	defer skutil.Remove(luaAggregatorPath)
	luaAggregatorRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, luaScriptRemoteDir, luaAggregatorName)
	luaAggregatorFileInfo, err := os.Stat(luaAggregatorPath)
	if !os.IsNotExist(err) && luaAggregatorFileInfo.Size() > 10 {
		if err := gs.UploadFile(luaAggregatorName, os.TempDir(), luaScriptRemoteDir); err != nil {
			glog.Errorf("Could not upload %s to %s: %s", luaAggregatorName, luaScriptRemoteDir, err)
			return
		}
		// Run the aggregator and save stdout.
		luaAggregatorOutputFileName := *runID + ".agg.output"
		luaAggregatorOutputFilePath := filepath.Join(os.TempDir(), luaAggregatorOutputFileName)
		luaAggregatorOutputFile, err := os.Create(luaAggregatorOutputFilePath)
		defer skutil.Close(luaAggregatorOutputFile)
		defer skutil.Remove(luaAggregatorOutputFilePath)
		if err != nil {
			glog.Errorf("Could not create %s: %s", luaAggregatorOutputFilePath, err)
			return
		}
		if err := util.ExecuteCmd(util.BINARY_LUA, []string{luaAggregatorPath}, []string{}, time.Hour, luaAggregatorOutputFile, nil); err != nil {
			glog.Errorf("Could not execute the lua aggregator %s: %s", luaAggregatorPath, err)
			return
		}
		// Copy the aggregator output into Google Storage.
		luaAggregatorOutputRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, consolidatedOutputRemoteDir, luaAggregatorOutputFileName)
		if err := gs.UploadFile(luaAggregatorOutputFileName, os.TempDir(), consolidatedOutputRemoteDir); err != nil {
			glog.Errorf("Unable to upload %s to %s: %s", luaAggregatorOutputFileName, consolidatedOutputRemoteDir, err)
			return
		}
	} else {
		glog.Info("A lua aggregator has not been specified.")
	}

	taskCompletedSuccessfully = true
}
Exemple #13
0
func main() {
	defer common.LogPanic()
	worker_common.Init()
	if !*worker_common.Local {
		defer util.CleanTmpDir()
	}
	defer util.TimeTrack(time.Now(), "Running Chromium Perf")
	defer glog.Flush()

	// Validate required arguments.
	if *chromiumBuildNoPatch == "" {
		glog.Error("Must specify --chromium_build_nopatch")
		return
	}
	if *chromiumBuildWithPatch == "" {
		glog.Error("Must specify --chromium_build_withpatch")
		return
	}
	if *runID == "" {
		glog.Error("Must specify --run_id")
		return
	}
	if *runIDNoPatch == "" {
		glog.Error("Must specify --run_id_nopatch")
		return
	}
	if *runIDWithPatch == "" {
		glog.Error("Must specify --run_id_withpatch")
		return
	}
	if *benchmarkName == "" {
		glog.Error("Must specify --benchmark_name")
		return
	}

	// Reset the local chromium checkout.
	if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
		return
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
		return
	}

	// Create the task file so that the master knows this worker is still busy.
	skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_RUNNING_CHROMIUM_PERF))
	defer util.DeleteTaskFile(util.ACTIVITY_RUNNING_CHROMIUM_PERF)

	if *targetPlatform == util.PLATFORM_ANDROID {
		if err := adb.VerifyLocalDevice(); err != nil {
			// Android device missing or offline.
			glog.Errorf("Could not find Android device: %s", err)
			return
		}
		// Kill adb server to make sure we start from a clean slate.
		skutil.LogErr(util.ExecuteCmd(util.BINARY_ADB, []string{"kill-server"}, []string{},
			util.ADB_ROOT_TIMEOUT, nil, nil))
		// Make sure adb shell is running as root.
		skutil.LogErr(util.ExecuteCmd(util.BINARY_ADB, []string{"root"}, []string{},
			util.ADB_ROOT_TIMEOUT, nil, nil))
	}

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Download the benchmark patch for this run from Google storage.
	benchmarkPatchName := *runID + ".benchmark.patch"
	benchmarkPatchLocalPath := filepath.Join(os.TempDir(), benchmarkPatchName)
	remoteDir := filepath.Join(util.ChromiumPerfRunsDir, *runID)
	benchmarkPatchRemotePath := filepath.Join(remoteDir, benchmarkPatchName)
	respBody, err := gs.GetRemoteFileContents(benchmarkPatchRemotePath)
	if err != nil {
		glog.Errorf("Could not fetch %s: %s", benchmarkPatchRemotePath, err)
		return
	}
	defer skutil.Close(respBody)
	buf := new(bytes.Buffer)
	if _, err := buf.ReadFrom(respBody); err != nil {
		glog.Errorf("Could not read from %s: %s", benchmarkPatchRemotePath, err)
		return
	}
	if err := ioutil.WriteFile(benchmarkPatchLocalPath, buf.Bytes(), 0666); err != nil {
		glog.Errorf("Unable to create file %s: %s", benchmarkPatchLocalPath, err)
		return
	}
	defer skutil.Remove(benchmarkPatchLocalPath)
	// Apply benchmark patch to the local chromium checkout.
	if buf.Len() > 10 {
		if err := util.ApplyPatch(benchmarkPatchLocalPath, util.ChromiumSrcDir); err != nil {
			glog.Errorf("Could not apply Telemetry's patch in %s: %s", util.ChromiumSrcDir, err)
			return
		}
	}

	// Download the specified chromium builds.
	for _, chromiumBuild := range []string{*chromiumBuildNoPatch, *chromiumBuildWithPatch} {
		if err := gs.DownloadChromiumBuild(chromiumBuild); err != nil {
			glog.Error(err)
			return
		}
		//Delete the chromium build to save space when we are done.
		defer skutil.RemoveAll(filepath.Join(util.ChromiumBuildsDir, chromiumBuild))
	}

	chromiumBinaryNoPatch := filepath.Join(util.ChromiumBuildsDir, *chromiumBuildNoPatch, util.BINARY_CHROME)
	chromiumBinaryWithPatch := filepath.Join(util.ChromiumBuildsDir, *chromiumBuildWithPatch, util.BINARY_CHROME)

	// Download pagesets if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}
	pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)

	// Download archives if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}

	// Establish nopatch output paths.
	localOutputDirNoPatch := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runIDNoPatch)
	skutil.RemoveAll(localOutputDirNoPatch)
	skutil.MkdirAll(localOutputDirNoPatch, 0700)
	defer skutil.RemoveAll(localOutputDirNoPatch)
	remoteDirNoPatch := filepath.Join(util.BenchmarkRunsDir, *runIDNoPatch)

	// Establish withpatch output paths.
	localOutputDirWithPatch := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runIDWithPatch)
	skutil.RemoveAll(localOutputDirWithPatch)
	skutil.MkdirAll(localOutputDirWithPatch, 0700)
	defer skutil.RemoveAll(localOutputDirWithPatch)
	remoteDirWithPatch := filepath.Join(util.BenchmarkRunsDir, *runIDWithPatch)

	// Construct path to the ct_run_benchmark python script.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
		"py")

	fileInfos, err := ioutil.ReadDir(pathToPagesets)
	if err != nil {
		glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
		return
	}

	numWorkers := WORKER_POOL_SIZE
	if *targetPlatform == util.PLATFORM_ANDROID || !*runInParallel {
		// Do not run page sets in parallel if the target platform is Android.
		// This is because the nopatch/withpatch APK needs to be installed prior to
		// each run and this will interfere with the parallel runs. Instead of trying
		// to find a complicated solution to this, it makes sense for Android to
		// continue to be serial because it will help guard against
		// crashes/flakiness/inconsistencies which are more prevalent in mobile runs.
		numWorkers = 1
		glog.Infoln("===== Going to run the task serially =====")
	} else {
		glog.Infoln("===== Going to run the task with parallel chrome processes =====")
	}

	// Create channel that contains all pageset file names. This channel will
	// be consumed by the worker pool.
	pagesetRequests := util.GetClosedChannelOfPagesets(fileInfos)

	var wg sync.WaitGroup
	// Use a RWMutex for the chromeProcessesCleaner goroutine to communicate to
	// the workers (acting as "readers") when it wants to be the "writer" and
	// kill all zombie chrome processes.
	var mutex sync.RWMutex

	// Loop through workers in the worker pool.
	for i := 0; i < numWorkers; i++ {
		// Increment the WaitGroup counter.
		wg.Add(1)

		// Create and run a goroutine closure that captures SKPs.
		go func() {
			// Decrement the WaitGroup counter when the goroutine completes.
			defer wg.Done()

			for pagesetName := range pagesetRequests {

				mutex.RLock()

				if err := runBenchmark(pagesetName, pathToPagesets, pathToPyFiles, localOutputDirNoPatch, *chromiumBuildNoPatch, chromiumBinaryNoPatch, *runIDNoPatch, *browserExtraArgsNoPatch); err != nil {
					glog.Errorf("Error while running nopatch benchmark: %s", err)
					return
				}
				if err := runBenchmark(pagesetName, pathToPagesets, pathToPyFiles, localOutputDirWithPatch, *chromiumBuildWithPatch, chromiumBinaryWithPatch, *runIDWithPatch, *browserExtraArgsWithPatch); err != nil {
					glog.Errorf("Error while running withpatch benchmark: %s", err)
					return
				}
				mutex.RUnlock()
			}
		}()
	}

	if !*worker_common.Local {
		// Start the cleaner.
		go util.ChromeProcessesCleaner(&mutex, *chromeCleanerTimer)
	}

	// Wait for all spawned goroutines to complete.
	wg.Wait()

	// If "--output-format=csv-pivot-table" was specified then merge all CSV files and upload.
	if strings.Contains(*benchmarkExtraArgs, "--output-format=csv-pivot-table") {
		if err := mergeUploadCSVFiles(localOutputDirNoPatch, pathToPyFiles, *runIDNoPatch, remoteDirNoPatch, gs); err != nil {
			glog.Errorf("Error while processing nopatch CSV files: %s", err)
			return
		}
		if err := mergeUploadCSVFiles(localOutputDirWithPatch, pathToPyFiles, *runIDWithPatch, remoteDirWithPatch, gs); err != nil {
			glog.Errorf("Error while processing withpatch CSV files: %s", err)
			return
		}
	}
}
Exemple #14
0
func main() {
	common.Init()
	webhook.MustInitRequestSaltFromFile(util.WebhookRequestSaltPath)

	// Send start email.
	emailsArr := util.ParseEmails(*emails)
	emailsArr = append(emailsArr, util.CtAdmins...)
	if len(emailsArr) == 0 {
		glog.Error("At least one email address must be specified")
		return
	}
	skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&frontend.ChromiumBuildUpdateVars{}, *gaeTaskID))
	skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Build chromium"))
	// Ensure webapp is updated and completion email is sent even if task fails.
	defer updateWebappTask()
	defer sendEmail(emailsArr)
	// Cleanup tmp files after the run.
	defer util.CleanTmpDir()
	// Finish with glog flush and how long the task took.
	defer util.TimeTrack(time.Now(), "Running build chromium")
	defer glog.Flush()

	if *chromiumHash == "" {
		glog.Error("Must specify --chromium_hash")
		return
	}
	if *skiaHash == "" {
		glog.Error("Must specify --skia_hash")
		return
	}

	if _, _, err := util.CreateChromiumBuild("", *targetPlatform, *chromiumHash, *skiaHash, *applyPatches); err != nil {
		glog.Errorf("Error while creating the Chromium build: %s", err)
		return
	}

	// Find when the requested Chromium revision was submitted.
	stdoutFileName := *runID + ".out"
	stdoutFilePath := filepath.Join(os.TempDir(), stdoutFileName)
	stdoutFile, err := os.Create(stdoutFilePath)
	defer skutil.Close(stdoutFile)
	defer skutil.Remove(stdoutFilePath)
	if err != nil {
		glog.Errorf("Could not create %s: %s", stdoutFilePath, err)
		return
	}
	var chromiumBuildDir string
	if *targetPlatform == "Android" {
		chromiumBuildDir = filepath.Join(util.ChromiumBuildsDir, "android_base")
	} else if *targetPlatform == "Linux" {
		chromiumBuildDir = filepath.Join(util.ChromiumBuildsDir, "linux_base")
	}
	if err := os.Chdir(filepath.Join(chromiumBuildDir, "src")); err != nil {
		glog.Errorf("Could not chdir to %s: %s", chromiumBuildDir, err)
		return
	}
	// Run git log --pretty=format="%at" -1
	if err := util.ExecuteCmd(util.BINARY_GIT, []string{"log", "--pretty=format:%at", "-1"}, []string{}, 5*time.Minute, stdoutFile, nil); err != nil {
		glog.Errorf("Could not run git log cmd: %s", err)
		return
	}
	content, err := ioutil.ReadFile(stdoutFilePath)
	if err != nil {
		glog.Errorf("Could not read %s: %s", stdoutFilePath, err)
		return
	}
	chromiumBuildTimestamp = string(content)

	taskCompletedSuccessfully = true
}
Exemple #15
0
func main() {
	common.Init()
	defer util.TimeTrack(time.Now(), "Running Skia Correctness")
	defer glog.Flush()

	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *runID == "" {
		glog.Error("Must specify --run_id")
		return
	}

	// Create the task file so that the master knows this worker is still busy.
	skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_RUNNING_SKIA_CORRECTNESS))
	defer util.DeleteTaskFile(util.ACTIVITY_RUNNING_SKIA_CORRECTNESS)

	// Establish output paths.
	localOutputDir := filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir, *runID)
	skutil.RemoveAll(filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir))
	skutil.MkdirAll(localOutputDir, 0700)
	defer skutil.RemoveAll(localOutputDir)
	remoteOutputDir := filepath.Join(util.SkiaCorrectnessRunsDir, *runID, fmt.Sprintf("slave%d", *workerNum))

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Download SKPs if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.SKPS_DIR_NAME, filepath.Join(*pagesetType, *chromiumBuild), *workerNum); err != nil {
		glog.Error(err)
		return
	}
	localSkpsDir := filepath.Join(util.SkpsDir, *pagesetType, *chromiumBuild)

	// Download the Skia patch for this run from Google storage.
	patchName := *runID + ".patch"
	patchLocalPath := filepath.Join(os.TempDir(), patchName)
	remoteDir := filepath.Join(util.SkiaCorrectnessRunsDir, *runID)
	patchRemotePath := filepath.Join(remoteDir, "patches", patchName)
	respBody, err := gs.GetRemoteFileContents(patchRemotePath)
	if err != nil {
		glog.Errorf("Could not fetch %s: %s", patchRemotePath, err)
		return
	}
	defer skutil.Close(respBody)
	out, err := os.Create(patchLocalPath)
	if err != nil {
		glog.Errorf("Unable to create file %s: %s", patchLocalPath, err)
		return
	}
	defer skutil.Close(out)
	defer skutil.Remove(patchLocalPath)
	if _, err = io.Copy(out, respBody); err != nil {
		glog.Error(err)
		return
	}

	// Apply the patch to a clean checkout and run render_pictures.
	// Reset Skia tree.
	skutil.LogErr(util.ResetCheckout(util.SkiaTreeDir))
	// Sync Skia tree.
	skutil.LogErr(util.SyncDir(util.SkiaTreeDir))
	// Apply Skia patch.
	file, _ := os.Open(patchLocalPath)
	fileInfo, _ := file.Stat()
	// It is a valid patch only if it is more than 10 bytes.
	if fileInfo.Size() > 10 {
		glog.Info("Attempting to apply %s to %s", patchLocalPath, util.SkiaTreeDir)
		if err := util.ApplyPatch(patchLocalPath, util.SkiaTreeDir); err != nil {
			glog.Errorf("Could not apply patch %s to %s: %s", patchLocalPath, util.SkiaTreeDir, err)
			return
		}
		glog.Info("Patch successfully applied")
	} else {
		glog.Info("Patch is empty or invalid. Skipping the patch.")
	}
	// Build tools.
	skutil.LogErr(util.BuildSkiaTools())
	// Run render_pictures.
	if err := runRenderPictures(localSkpsDir, filepath.Join(localOutputDir, "withpatch"), filepath.Join(remoteOutputDir, "withpatch"), *gpuWithPatchRun); err != nil {
		glog.Errorf("Error while running withpatch render_pictures: %s", err)
		return
	}

	// Remove the patch and run render_pictures.
	// Reset Skia tree.
	skutil.LogErr(util.ResetCheckout(util.SkiaTreeDir))
	// Build tools.
	skutil.LogErr(util.BuildSkiaTools())
	// Run render_pictures.
	if err := runRenderPictures(localSkpsDir, filepath.Join(localOutputDir, "nopatch"), filepath.Join(remoteOutputDir, "nopatch"), *gpuNoPatchRun); err != nil {
		glog.Errorf("Error while running nopatch render_pictures: %s", err)
		return
	}

	// Comparing pictures and saving differences in JSON output file.
	jsonSummaryDir := filepath.Join(localOutputDir, "json_summary")
	skutil.MkdirAll(jsonSummaryDir, 0700)
	jsonSummaryPath := filepath.Join(jsonSummaryDir, fmt.Sprintf("slave%d", *workerNum)+".json")
	// Construct path to the write_json_summary python script.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
		"py")
	summaryArgs := []string{
		filepath.Join(pathToPyFiles, "write_json_summary.py"),
		"--img_root=" + localOutputDir,
		"--withpatch_json=" + filepath.Join(localOutputDir, "withpatch", "summary.json"),
		"--withpatch_images_base_url=file://" + filepath.Join(localOutputDir, "withpatch"),
		"--nopatch_json=" + filepath.Join(localOutputDir, "nopatch", "summary.json"),
		"--nopatch_images_base_url=file://" + filepath.Join(localOutputDir, "nopatch"),
		"--output_file_path=" + jsonSummaryPath,
		"--gs_output_dir=gs://" + filepath.Join(util.GS_BUCKET_NAME, remoteDir),
		"--gs_skp_dir=gs://" + filepath.Join(util.GS_BUCKET_NAME, util.SKPS_DIR_NAME, *pagesetType, *chromiumBuild, fmt.Sprintf("slave%d", *workerNum)),
		"--slave_num=" + strconv.Itoa(*workerNum),
	}
	if err := util.ExecuteCmd("python", summaryArgs, []string{}, 15*time.Minute, nil, nil); err != nil {
		glog.Error(err)
		return
	}

	// Upload artifacts to Google Storage.
	// Get list of failed file names and upload only those to Google Storage.
	// Read the JSON file.
	summaryFile, err := ioutil.ReadFile(jsonSummaryPath)
	if err != nil {
		glog.Errorf("Unable to read %s: %s", jsonSummaryPath, err)
	}
	var jsontype map[string]interface{}
	skutil.LogErr(json.Unmarshal(summaryFile, &jsontype))
	if jsontype[fmt.Sprintf("slave%d", *workerNum)] != nil {
		failedFiles := jsontype[fmt.Sprintf("slave%d", *workerNum)].(map[string]interface{})["failedFiles"].([]interface{})
		for i := range failedFiles {
			failedFile := failedFiles[i].(map[string]interface{})["fileName"].(string)
			// TODO(rmistry): Use goroutines to do the below in parallel.
			skutil.LogErr(
				gs.UploadFile(failedFile, filepath.Join(localOutputDir, "withpatch"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "withpatch-images")))
			skutil.LogErr(
				gs.UploadFile(failedFile, filepath.Join(localOutputDir, "nopatch"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "nopatch-images")))
		}
		// Copy the diffs and whitediffs to Google Storage.
		skutil.LogErr(
			gs.UploadDir(filepath.Join(localOutputDir, "diffs"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "diffs"), true))
		skutil.LogErr(
			gs.UploadDir(filepath.Join(localOutputDir, "whitediffs"), filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum), "whitediffs"), true))
	}
	// Upload the summary file.
	skutil.LogErr(
		gs.UploadFile(fmt.Sprintf("slave%d", *workerNum)+".json", jsonSummaryDir, filepath.Join(remoteDir, fmt.Sprintf("slave%d", *workerNum))))
}
Exemple #16
0
func main() {
	common.Init()
	webhook.MustInitRequestSaltFromFile(util.WebhookRequestSaltPath)

	// Send start email.
	emailsArr := util.ParseEmails(*emails)
	emailsArr = append(emailsArr, util.CtAdmins...)
	if len(emailsArr) == 0 {
		glog.Error("At least one email address must be specified")
		return
	}
	skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Skia correctness"))
	// Ensure webapp is updated and email is sent even if task fails.
	defer updateWebappTask()
	defer sendEmail(emailsArr)
	// Cleanup tmp files after the run.
	defer util.CleanTmpDir()
	// Finish with glog flush and how long the task took.
	defer util.TimeTrack(time.Now(), "Running skia correctness task on workers")
	defer glog.Flush()

	if *pagesetType == "" {
		glog.Error("Must specify --pageset_type")
		return
	}
	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *runID == "" {
		glog.Error("Must specify --run_id")
		return
	}

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}
	remoteOutputDir := filepath.Join(util.SkiaCorrectnessRunsDir, *runID)

	// Copy the patch to Google Storage.
	patchName := *runID + ".patch"
	patchRemoteDir := filepath.Join(remoteOutputDir, "patches")
	if err := gs.UploadFile(patchName, os.TempDir(), patchRemoteDir); err != nil {
		glog.Errorf("Could not upload %s to %s: %s", patchName, patchRemoteDir, err)
		return
	}
	skiaPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, patchRemoteDir, patchName)

	// Run the run_skia_correctness script on all workers.
	runSkiaCorrCmdTemplate := "DISPLAY=:0 run_skia_correctness --worker_num={{.WorkerNum}} --log_dir={{.LogDir}} " +
		"--pageset_type={{.PagesetType}} --chromium_build={{.ChromiumBuild}} --run_id={{.RunID}} " +
		"--render_pictures_args=\"{{.RenderPicturesArgs}}\" --gpu_nopatch_run={{.GpuNoPatchRun}} " +
		"--gpu_withpatch_run={{.GpuWithPatchRun}};"
	runSkiaCorrTemplateParsed := template.Must(template.New("run_skia_correctness_cmd").Parse(runSkiaCorrCmdTemplate))
	runSkiaCorrCmdBytes := new(bytes.Buffer)
	if err := runSkiaCorrTemplateParsed.Execute(runSkiaCorrCmdBytes, struct {
		WorkerNum          string
		LogDir             string
		PagesetType        string
		ChromiumBuild      string
		RunID              string
		RenderPicturesArgs string
		GpuNoPatchRun      string
		GpuWithPatchRun    string
	}{
		WorkerNum:          util.WORKER_NUM_KEYWORD,
		LogDir:             util.GLogDir,
		PagesetType:        *pagesetType,
		ChromiumBuild:      *chromiumBuild,
		RunID:              *runID,
		RenderPicturesArgs: *renderPicturesArgs,
		GpuNoPatchRun:      strconv.FormatBool(*gpuNoPatchRun),
		GpuWithPatchRun:    strconv.FormatBool(*gpuWithPatchRun),
	}); err != nil {
		glog.Errorf("Failed to execute template: %s", err)
		return
	}
	cmd := []string{
		fmt.Sprintf("cd %s;", util.CtTreeDir),
		"git pull;",
		"make all;",
		// The main command that runs run_skia_correctness on all workers.
		runSkiaCorrCmdBytes.String(),
	}
	if _, err := util.SSH(strings.Join(cmd, " "), util.Slaves, 4*time.Hour); err != nil {
		glog.Errorf("Error while running cmd %s: %s", cmd, err)
		return
	}

	localOutputDir := filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir, *runID)
	localSummariesDir := filepath.Join(localOutputDir, "summaries")
	skutil.MkdirAll(localSummariesDir, 0700)
	defer skutil.RemoveAll(filepath.Join(util.StorageDir, util.SkiaCorrectnessRunsDir))
	// Copy outputs from all slaves locally.
	for i := 0; i < util.NUM_WORKERS; i++ {
		workerNum := i + 1
		workerLocalOutputPath := filepath.Join(localSummariesDir, fmt.Sprintf("slave%d", workerNum)+".json")
		workerRemoteOutputPath := filepath.Join(remoteOutputDir, fmt.Sprintf("slave%d", workerNum), fmt.Sprintf("slave%d", workerNum)+".json")
		respBody, err := gs.GetRemoteFileContents(workerRemoteOutputPath)
		if err != nil {
			glog.Errorf("Could not fetch %s: %s", workerRemoteOutputPath, err)
			// TODO(rmistry): Should we instead return here? We can only return
			// here if all 100 slaves reliably run without any failures which they
			// really should.
			continue
		}
		defer skutil.Close(respBody)
		out, err := os.Create(workerLocalOutputPath)
		if err != nil {
			glog.Errorf("Unable to create file %s: %s", workerLocalOutputPath, err)
			return
		}
		defer skutil.Close(out)
		defer skutil.Remove(workerLocalOutputPath)
		if _, err = io.Copy(out, respBody); err != nil {
			glog.Errorf("Unable to copy to file %s: %s", workerLocalOutputPath, err)
			return
		}
	}

	// Call json_summary_combiner.py to merge all results into a single results CSV.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
		"py")
	pathToJsonCombiner := filepath.Join(pathToPyFiles, "json_summary_combiner.py")
	localHtmlDir := filepath.Join(localOutputDir, "html")
	remoteHtmlDir := filepath.Join(remoteOutputDir, "html")
	baseHtmlLink := util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteHtmlDir) + "/"
	htmlOutputLink = baseHtmlLink + "index.html"
	skutil.MkdirAll(localHtmlDir, 0700)
	args := []string{
		pathToJsonCombiner,
		"--json_summaries_dir=" + localSummariesDir,
		"--output_html_dir=" + localHtmlDir,
		"--absolute_url=" + baseHtmlLink,
		"--render_pictures_args=" + *renderPicturesArgs,
		"--nopatch_gpu=" + strconv.FormatBool(*gpuNoPatchRun),
		"--withpatch_gpu=" + strconv.FormatBool(*gpuWithPatchRun),
	}
	if err := util.ExecuteCmd("python", args, []string{}, 1*time.Hour, nil, nil); err != nil {
		glog.Errorf("Error running json_summary_combiner.py: %s", err)
		return
	}

	// Copy the HTML files to Google Storage.
	if err := gs.UploadDir(localHtmlDir, remoteHtmlDir, true); err != nil {
		glog.Errorf("Could not upload %s to %s: %s", localHtmlDir, remoteHtmlDir, err)
		return
	}

	taskCompletedSuccessfully = true
}