Beispiel #1
0
func runBenchmark(fileInfoName, pathToPagesets, pathToPyFiles, localOutputDir, chromiumBuildName, chromiumBinary, runID, browserExtraArgs string) error {
	pagesetBaseName := filepath.Base(fileInfoName)
	if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
		// Ignore timestamp files and .pyc files.
		return nil
	}

	// Convert the filename into a format consumable by the run_benchmarks
	// binary.
	pagesetName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
	pagesetPath := filepath.Join(pathToPagesets, fileInfoName)

	glog.Infof("===== Processing %s for %s =====", pagesetPath, runID)

	skutil.LogErr(os.Chdir(pathToPyFiles))
	args := []string{
		util.BINARY_RUN_BENCHMARK,
		fmt.Sprintf("%s.%s", *benchmarkName, util.BenchmarksToPagesetName[*benchmarkName]),
		"--page-set-name=" + pagesetName,
		"--page-set-base-dir=" + pathToPagesets,
		"--also-run-disabled-tests",
	}

	// Need to capture output for all benchmarks.
	outputDirArgValue := filepath.Join(localOutputDir, pagesetName)
	args = append(args, "--output-dir="+outputDirArgValue)
	// Figure out which browser should be used.
	if *targetPlatform == util.PLATFORM_ANDROID {
		if err := installChromeAPK(chromiumBuildName); err != nil {
			return fmt.Errorf("Error while installing APK: %s", err)
		}
		args = append(args, "--browser=android-chrome-shell")
	} else {
		args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary)
	}
	// Split benchmark args if not empty and append to args.
	if *benchmarkExtraArgs != "" {
		for _, benchmarkArg := range strings.Split(*benchmarkExtraArgs, " ") {
			args = append(args, benchmarkArg)
		}
	}
	// Add the number of times to repeat.
	args = append(args, fmt.Sprintf("--page-repeat=%d", *repeatBenchmark))
	// Add browserArgs if not empty to args.
	if browserExtraArgs != "" {
		args = append(args, "--extra-browser-args="+browserExtraArgs)
	}
	// Set the PYTHONPATH to the pagesets and the telemetry dirs.
	env := []string{
		fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir),
		"DISPLAY=:0",
	}
	timeoutSecs := util.PagesetTypeToInfo[*pagesetType].RunChromiumPerfTimeoutSecs
	if err := util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil); err != nil {
		glog.Errorf("Run benchmark command failed with: %s", err)
		glog.Errorf("Killing all running chrome processes in case there is a non-recoverable error.")
		skutil.LogErr(util.ExecuteCmd("pkill", []string{"-9", "chrome"}, []string{}, 5*time.Minute, nil, nil))
	}
	return nil
}
Beispiel #2
0
// writeWorkerResult runs the worker and writes the result to Redis.
func (r *RedisRTC) writeWorkerResult(priority int64, id string) {
	result, err := r.worker(priority, id)
	var writeKey string
	var writeData []byte
	if err != nil {
		writeKey = r.errorKey(id)
		writeData = []byte(err.Error())
	} else {
		if writeData, err = r.codec.Encode(result); err != nil {
			writeKey = r.errorKey(id)
			writeData = []byte(fmt.Sprintf("Error encoding worker result: %s", err))
		} else {
			writeKey = r.key(id)
		}
	}

	c := r.redisPool.Get()
	defer util.Close(c)

	util.LogErr(c.Send("MULTI"))
	util.LogErr(c.Send("SET", writeKey, writeData))
	util.LogErr(c.Send("SREM", r.inProgressKey, id))

	// Expire the error after 10 seconds to let the client decide
	// whether we need to retry.
	if err != nil {
		util.LogErr(c.Send("EXPIRE", writeKey, 10))
	}
	util.LogErr(c.Send("PUBLISH", r.finishedChannel, id))
	if _, err = c.Do("EXEC"); err != nil {
		glog.Errorf("Error writing result to redis: %s", err)
	}
}
Beispiel #3
0
// Add, see the diff.LRUCache interface for details.
func (c *RedisLRUCache) Add(key, value interface{}) bool {
	prefixedKey, rawKey, err := c.encodeKey(key)
	if err != nil {
		glog.Errorf("Unable to create redis key: %s", err)
		return false
	}

	byteVal, err := c.encodeVal(value)
	if err != nil {
		glog.Errorf("Unable to create redis value: %s", err)
		return false
	}

	conn := c.pool.Get()
	defer util.Close(conn)

	util.LogErr(conn.Send("MULTI"))
	util.LogErr(conn.Send("SET", prefixedKey, byteVal))
	util.LogErr(conn.Send("SADD", c.indexSetKey, rawKey))
	_, err = conn.Do("EXEC")
	if err != nil {
		glog.Errorf("Unable to add key: %s", err)
		return false
	}
	return true
}
Beispiel #4
0
// getResultErr returns the cached version of the item identified by 'id' or
// nil if it's not available.
func (r *RedisRTC) getResultErr(id string, returnBytes bool) (interface{}, error) {
	c := r.redisPool.Get()
	defer util.Close(c)

	util.LogErr(c.Send("MULTI"))
	util.LogErr(c.Send("GET", r.key(id)))
	util.LogErr(c.Send("GET", r.errorKey(id)))
	resArr, err := redis.Values(c.Do("EXEC"))
	if err != nil {
		return nil, err
	}
	retBytes, errBytes := resArr[0], resArr[1]
	if errBytes != nil {
		return nil, fmt.Errorf("For %s we received error: %s", id, string(errBytes.([]byte)))
	}

	if retBytes != nil {
		if returnBytes {
			return retBytes, nil
		}
		return r.codec.Decode(retBytes.([]byte))
	}

	// We have neither an error nor any data.
	return nil, nil
}
Beispiel #5
0
// ResetCheckout resets the specified Git checkout.
func ResetCheckout(dir string) error {
	if err := os.Chdir(dir); err != nil {
		return fmt.Errorf("Could not chdir to %s: %s", dir, err)
	}
	// Run "git reset --hard HEAD"
	resetArgs := []string{"reset", "--hard", "HEAD"}
	util.LogErr(ExecuteCmd(BINARY_GIT, resetArgs, []string{}, GIT_RESET_TIMEOUT, nil, nil))
	// Run "git clean -f -d"
	cleanArgs := []string{"clean", "-f", "-d"}
	util.LogErr(ExecuteCmd(BINARY_GIT, cleanArgs, []string{}, GIT_CLEAN_TIMEOUT, nil, nil))

	return nil
}
Beispiel #6
0
// isFinished returns true if the task has finished.
func (r *RedisRTC) isFinished(id string) bool {
	c := r.redisPool.Get()
	defer util.Close(c)

	util.LogErr(c.Send("MULTI"))
	util.LogErr(c.Send("EXISTS", r.key(id)))
	util.LogErr(c.Send("EXISTS", r.errorKey(id)))
	resArr, err := redis.Ints(c.Do("EXEC"))
	if err != nil {
		glog.Errorf("Unable to check if key exits: %s", err)
		return false
	}

	return (resArr[0] + resArr[1]) > 0
}
Beispiel #7
0
func updateWebappTask() {
	if frontend.CTFE_V2 {
		vars := frontend.LuaScriptUpdateVars{}
		vars.Id = *gaeTaskID
		vars.SetCompleted(taskCompletedSuccessfully)
		vars.ScriptOutput = sql.NullString{String: luaOutputRemoteLink, Valid: true}
		if luaAggregatorOutputRemoteLink != "" {
			vars.AggregatedOutput = sql.NullString{String: luaAggregatorOutputRemoteLink, Valid: true}
		}
		skutil.LogErr(frontend.UpdateWebappTaskV2(&vars))
		return
	}
	outputLink := luaOutputRemoteLink
	if luaAggregatorOutputRemoteLink != "" {
		// Use the aggregated output if it exists.
		outputLink = luaAggregatorOutputRemoteLink
	}
	extraData := map[string]string{
		"lua_script_link":     luaScriptRemoteLink,
		"lua_aggregator_link": luaAggregatorRemoteLink,
		"lua_output_link":     outputLink,
	}
	if err := frontend.UpdateWebappTask(*gaeTaskID, frontend.UpdateLuaTasksWebapp, extraData); err != nil {
		glog.Errorf("Error while updating webapp task: %s", err)
		return
	}
}
Beispiel #8
0
// ExecuteCmd executes the specified binary with the specified args and env.
// Stdout and Stderr are written to stdoutFile and stderrFile respectively if
// specified. If not specified then stdout and stderr will be outputted only to
// glog. Note: It is the responsibility of the caller to close stdoutFile and
// stderrFile.
func ExecuteCmd(binary string, args, env []string, timeout time.Duration, stdoutFile, stderrFile *os.File) error {
	// Add the current PATH to the env.
	env = append(env, "PATH="+os.Getenv("PATH"))

	// Create the cmd obj.
	cmd := exec.Command(binary, args...)
	cmd.Env = env

	// Attach WriteLog to command.
	cmd.Stdout = WriteLog{LogFunc: glog.Infof, OutputFile: stdoutFile}
	cmd.Stderr = WriteLog{LogFunc: glog.Errorf, OutputFile: stderrFile}

	// Execute cmd.
	glog.Infof("Executing %s %s", strings.Join(cmd.Env, " "), strings.Join(cmd.Args, " "))
	util.LogErr(cmd.Start())
	done := make(chan error)
	go func() {
		done <- cmd.Wait()
	}()
	select {
	case <-time.After(timeout):
		if err := cmd.Process.Kill(); err != nil {
			return fmt.Errorf("Failed to kill timed out process: %s", err)
		}
		<-done // allow goroutine to exit
		glog.Errorf("Command killed since it took longer than %f secs", timeout.Seconds())
		return fmt.Errorf("Command killed since it took longer than %f secs", timeout.Seconds())
	case err := <-done:
		if err != nil {
			return fmt.Errorf("Process done with error: %s", err)
		}
	}
	return nil
}
Beispiel #9
0
func executeCmd(cmd, hostname string, config *ssh.ClientConfig, timeout time.Duration) (string, error) {
	// Dial up TCP connection to remote machine.
	conn, err := net.Dial("tcp", hostname+":22")
	if err != nil {
		return "", fmt.Errorf("Failed to ssh connect to %s. Make sure \"PubkeyAuthentication yes\" is in your sshd_config: %s", hostname, err)
	}
	defer util.Close(conn)
	util.LogErr(conn.SetDeadline(time.Now().Add(timeout)))

	// Create new SSH client connection.
	sshConn, sshChan, req, err := ssh.NewClientConn(conn, hostname+":22", config)
	if err != nil {
		return "", fmt.Errorf("Failed to ssh connect to %s: %s", hostname, err)
	}
	// Use client connection to create new client.
	client := ssh.NewClient(sshConn, sshChan, req)

	// Client connections can support multiple interactive sessions.
	session, err := client.NewSession()
	if err != nil {
		return "", fmt.Errorf("Failed to ssh connect to %s: %s", hostname, err)
	}

	var stdoutBuf bytes.Buffer
	session.Stdout = &stdoutBuf
	if err := session.Run(cmd); err != nil {
		return "", fmt.Errorf("Errored or Timeout out while running \"%s\" on %s: %s", cmd, hostname, err)
	}
	return stdoutBuf.String(), nil
}
Beispiel #10
0
// Will need a local valid google_storage_token.data file with read write access
// to run the below test.
func Auth_TestUploadWorkerArtifacts(t *testing.T) {
	client, _ := GetOAuthClient()
	gs, err := NewGsUtil(client)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	testDir := "testupload"
	testPagesetType := "10ktest"
	StorageDir = "testdata"
	if err := gs.UploadWorkerArtifacts(testDir, testPagesetType, 1); err != nil {
		t.Errorf("Unexpected error: %s", err)
	}

	// Examine contents of the remote directory and then clean it up.
	service, err := storage.New(gs.client)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	gsDir := filepath.Join(testDir, testPagesetType, "slave1")
	resp, err := service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/").Do()
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	assert.Equal(t, 3, len(resp.Items))
	for index, fileName := range []string{"TIMESTAMP", "alexa1-1.py", "alexa2-2.py"} {
		filePath := fmt.Sprintf("%s/%s", gsDir, fileName)
		defer util.LogErr(service.Objects.Delete(GS_BUCKET_NAME, filePath).Do())
		assert.Equal(t, filePath, resp.Items[index].Name)
	}
}
Beispiel #11
0
// NewDefaultRepoManager returns a RepoManager instance which operates in the given
// working directory and updates at the given frequency.
func NewDefaultRepoManager(workdir string, frequency time.Duration, depot_tools string) (RepoManager, error) {
	chromiumParentDir := path.Join(workdir, "chromium")
	skiaDir := path.Join(workdir, "skia")
	skiaRepo, err := gitinfo.CloneOrUpdate(REPO_SKIA, skiaDir, true)
	if err != nil {
		return nil, err
	}
	gclient := GCLIENT
	rollDep := ROLL_DEP
	if depot_tools != "" {
		gclient = path.Join(depot_tools, gclient)
		rollDep = path.Join(depot_tools, rollDep)
	}

	r := &repoManager{
		chromiumDir:       path.Join(chromiumParentDir, "src"),
		chromiumParentDir: chromiumParentDir,
		depot_tools:       depot_tools,
		gclient:           gclient,
		rollDep:           rollDep,
		skiaDir:           skiaDir,
		skiaRepo:          skiaRepo,
	}
	if err := r.update(); err != nil {
		return nil, err
	}
	go func() {
		for _ = range time.Tick(frequency) {
			util.LogErr(r.update())
		}
	}()
	return r, nil
}
func buildChromium(chromiumDir, targetPlatform string) error {
	if err := os.Chdir(filepath.Join(chromiumDir, "src")); err != nil {
		return fmt.Errorf("Could not chdir to %s/src: %s", chromiumDir, err)
	}

	// Find the build target to use while building chromium.
	buildTarget := "chrome"
	if targetPlatform == "Android" {
		util.LogErr(
			ioutil.WriteFile(filepath.Join(chromiumDir, "chromium.gyp_env"), []byte("{ 'GYP_DEFINES': 'OS=android', }\n"), 0777))
		buildTarget = "chrome_shell_apk"
	}

	// Restart Goma's compiler proxy right before building the checkout.
	if err := ExecuteCmd("python", []string{filepath.Join(GomaDir, "goma_ctl.py"), "restart"}, os.Environ(), 10*time.Minute, nil, nil); err != nil {
		return fmt.Errorf("Error while restarting goma compiler proxy: %s", err)
	}

	// Run "GYP_DEFINES='gomadir=/b/build/goma' GYP_GENERATORS='ninja' build/gyp_chromium -Duse_goma=1".
	env := []string{fmt.Sprintf("GYP_DEFINES=gomadir=%s", GomaDir), "GYP_GENERATORS=ninja"}
	if err := ExecuteCmd(filepath.Join("build", "gyp_chromium"), []string{"-Duse_goma=1"}, env, 30*time.Minute, nil, nil); err != nil {
		return fmt.Errorf("Error while running gyp_chromium: %s", err)
	}
	// Run "ninja -C out/Release -j100 ${build_target}".
	// Use the full system env while building chromium.
	args := []string{"-C", "out/Release", "-j100", buildTarget}
	return ExecuteCmd("ninja", args, os.Environ(), 2*time.Hour, nil, nil)
}
Beispiel #13
0
func updateWebappTask() {
	if frontend.CTFE_V2 {
		vars := frontend.ChromiumPerfUpdateVars{}
		vars.Id = *gaeTaskID
		vars.SetCompleted(taskCompletedSuccessfully)
		vars.Results = sql.NullString{String: htmlOutputLink, Valid: true}
		vars.NoPatchRawOutput = sql.NullString{String: noPatchOutputLink, Valid: true}
		vars.WithPatchRawOutput = sql.NullString{String: withPatchOutputLink, Valid: true}
		skutil.LogErr(frontend.UpdateWebappTaskV2(&vars))
		return
	}
	// TODO(rmistry): Add ability to update the webapp without providing links.
	extraData := map[string]string{
		"skia_patch_link":              skiaPatchLink,
		"blink_patch_link":             blinkPatchLink,
		"chromium_patch_link":          chromiumPatchLink,
		"telemetry_nopatch_log_link":   noPatchOutputLink,
		"telemetry_withpatch_log_link": withPatchOutputLink,
		"html_output_link":             htmlOutputLink,
		"build_log_link":               util.MASTER_LOGSERVER_LINK,
	}
	if err := frontend.UpdateWebappTask(*gaeTaskID, frontend.UpdateChromiumPerfTasksWebapp, extraData); err != nil {
		glog.Errorf("Error while updating webapp task: %s", err)
		return
	}
}
Beispiel #14
0
func main() {
	defer common.LogPanic()
	common.Init()
	frontend.MustInit()

	// Send start email.
	emailsArr := util.ParseEmails(*emails)
	emailsArr = append(emailsArr, util.CtAdmins...)
	if len(emailsArr) == 0 {
		glog.Error("At least one email address must be specified")
		return
	}
	skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&admin_tasks.RecreateWebpageArchivesUpdateVars{}, *gaeTaskID))
	skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Capture archives", util.GetMasterLogLink(*runID), ""))
	// Ensure webapp is updated and completion email is sent even if task fails.
	defer updateWebappTask()
	defer sendEmail(emailsArr)
	// Cleanup tmp files after the run.
	defer util.CleanTmpDir()
	// Finish with glog flush and how long the task took.
	defer util.TimeTrack(time.Now(), "Capture archives on Workers")
	defer glog.Flush()

	if *pagesetType == "" {
		glog.Error("Must specify --pageset_type")
		return
	}
	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}

	cmd := []string{
		fmt.Sprintf("cd %s;", util.CtTreeDir),
		"git pull;",
		"make all;",
		// The main command that runs capture_archives on all workers.
		fmt.Sprintf("DISPLAY=:0 capture_archives --worker_num=%s --log_dir=%s --log_id=%s --pageset_type=%s --chromium_build=%s;", util.WORKER_NUM_KEYWORD, util.GLogDir, *runID, *pagesetType, *chromiumBuild),
	}

	_, err := util.SSH(strings.Join(cmd, " "), util.Slaves, util.CAPTURE_ARCHIVES_TIMEOUT)
	if err != nil {
		glog.Errorf("Error while running cmd %s: %s", cmd, err)
		return
	}
	*taskCompletedSuccessfully = true
}
Beispiel #15
0
// Remove, see the diff.LRUCache interface for details.
func (c *RedisLRUCache) Remove(key interface{}) {
	conn := c.pool.Get()
	defer util.Close(conn)

	prefixedKey, rawKey, err := c.encodeKey(key)
	if err != nil {
		glog.Errorf("Unable to create redis key %s", err)
	}

	util.LogErr(conn.Send("MULTI"))
	util.LogErr(conn.Send("DEL", prefixedKey))
	util.LogErr(conn.Send("SREM", c.indexSetKey, rawKey))
	_, err = conn.Do("EXEC")
	if err != nil {
		glog.Errorf("Error deleting key:%s", err)
	}
}
Beispiel #16
0
// BuildSkiaTools builds "tools" in the Skia trunk directory.
func BuildSkiaTools() error {
	if err := os.Chdir(SkiaTreeDir); err != nil {
		return fmt.Errorf("Could not chdir to %s: %s", SkiaTreeDir, err)
	}
	// Run "make clean".
	util.LogErr(ExecuteCmd(BINARY_MAKE, []string{"clean"}, []string{}, 5*time.Minute, nil, nil))
	// Build tools.
	return ExecuteCmd(BINARY_MAKE, []string{"tools", "BUILDTYPE=Release"}, []string{"GYP_DEFINES=\"skia_warnings_as_errors=0\""}, 5*time.Minute, nil, nil)
}
Beispiel #17
0
func updateWebappTask() {
	vars := chromium_perf.UpdateVars{}
	vars.Id = *gaeTaskID
	vars.SetCompleted(taskCompletedSuccessfully)
	vars.Results = sql.NullString{String: htmlOutputLink, Valid: true}
	vars.NoPatchRawOutput = sql.NullString{String: noPatchOutputLink, Valid: true}
	vars.WithPatchRawOutput = sql.NullString{String: withPatchOutputLink, Valid: true}
	skutil.LogErr(frontend.UpdateWebappTaskV2(&vars))
}
Beispiel #18
0
// Running benchmarks in parallel leads to multiple chrome instances coming up
// at the same time, when there are crashes chrome processes stick around which
// can severely impact the machine's performance. To stop this from
// happening chrome zombie processes are periodically killed.
func ChromeProcessesCleaner(locker sync.Locker, chromeCleanerTimer time.Duration) {
	for _ = range time.Tick(chromeCleanerTimer) {
		glog.Info("The chromeProcessesCleaner goroutine has started")
		glog.Info("Waiting for all existing tasks to complete before killing zombie chrome processes")
		locker.Lock()
		util.LogErr(ExecuteCmd("pkill", []string{"-9", "chrome"}, []string{}, PKILL_TIMEOUT, nil, nil))
		locker.Unlock()
	}
}
Beispiel #19
0
// UploadDir uploads the specified local dir into the specified Google Storage dir.
func (gs *GsUtil) UploadDir(localDir, gsDir string, cleanDir bool) error {
	if cleanDir {
		// Empty the remote dir before uploading to it.
		util.LogErr(gs.deleteRemoteDir(gsDir))
	}

	// Construct a dictionary of file paths to their file infos.
	pathsToFileInfos := map[string]os.FileInfo{}
	visit := func(path string, f os.FileInfo, err error) error {
		if f.IsDir() {
			return nil
		}
		pathsToFileInfos[path] = f
		return nil
	}
	if err := filepath.Walk(localDir, visit); err != nil {
		return fmt.Errorf("Unable to read the local dir %s: %s", localDir, err)
	}

	// The channel where the filepaths to be uploaded will be sent to.
	chFilePaths := make(chan string, MAX_CHANNEL_SIZE)
	// File filepaths and send it to the above channel.
	for path, fileInfo := range pathsToFileInfos {
		fileName := fileInfo.Name()
		containingDir := strings.TrimSuffix(path, fileName)
		subDirs := strings.TrimPrefix(containingDir, localDir)
		if subDirs != "" {
			dirTokens := strings.Split(subDirs, "/")
			for i := range dirTokens {
				fileName = filepath.Join(dirTokens[len(dirTokens)-i-1], fileName)
			}
		}
		chFilePaths <- fileName
	}
	close(chFilePaths)

	// Kick off goroutines to upload the file paths.
	var wg sync.WaitGroup
	for i := 0; i < GOROUTINE_POOL_SIZE; i++ {
		wg.Add(1)
		go func(goroutineNum int) {
			defer wg.Done()
			for filePath := range chFilePaths {
				glog.Infof("Uploading %s to %s with goroutine#%d", filePath, gsDir, goroutineNum)
				if err := gs.UploadFile(filePath, localDir, gsDir); err != nil {
					glog.Errorf("Goroutine#%d could not upload %s to %s: %s", goroutineNum, filePath, localDir, err)
				}
				// Sleep for a second after uploading file to avoid bombarding Cloud
				// storage.
				time.Sleep(time.Second)
			}
		}(i + 1)
	}
	wg.Wait()
	return nil
}
Beispiel #20
0
func main() {
	common.Init()
	webhook.MustInitRequestSaltFromFile(util.WebhookRequestSaltPath)

	// Send start email.
	emailsArr := util.ParseEmails(*emails)
	emailsArr = append(emailsArr, util.CtAdmins...)
	if len(emailsArr) == 0 {
		glog.Error("At least one email address must be specified")
		return
	}
	skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&frontend.RecreatePageSetsUpdateVars{}, *gaeTaskID))
	skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Creating pagesets"))
	// Ensure webapp is updated and completion email is sent even if task fails.
	defer updateWebappTask()
	defer sendEmail(emailsArr)
	// Cleanup tmp files after the run.
	defer util.CleanTmpDir()
	// Finish with glog flush and how long the task took.
	defer util.TimeTrack(time.Now(), "Creating Pagesets on Workers")
	defer glog.Flush()

	if *pagesetType == "" {
		glog.Error("Must specify --pageset_type")
		return
	}

	cmd := []string{
		fmt.Sprintf("cd %s;", util.CtTreeDir),
		"git pull;",
		"make all;",
		// The main command that runs create_pagesets on all workers.
		fmt.Sprintf("create_pagesets --worker_num=%s --log_dir=%s --pageset_type=%s;", util.WORKER_NUM_KEYWORD, util.GLogDir, *pagesetType),
	}

	// Setting a 4 hour timeout since it may take a while to upload page sets to
	// Google Storage when doing 10k page sets per worker.
	if _, err := util.SSH(strings.Join(cmd, " "), util.Slaves, 4*time.Hour); err != nil {
		glog.Errorf("Error while running cmd %s: %s", cmd, err)
		return
	}
	*taskCompletedSuccessfully = true
}
Beispiel #21
0
// SKPs are captured in parallel leading to multiple chrome instances coming up
// at the same time, when there are crashes chrome processes stick around which
// can severely impact the machine's performance. To stop this from
// happening chrome zombie processes are periodically killed.
func chromeProcessesCleaner(mutex *sync.RWMutex) {
	for _ = range time.Tick(*chromeCleanerTimer) {
		glog.Info("The chromeProcessesCleaner goroutine has started")
		glog.Info("Waiting for all existing tasks to complete before killing zombie chrome processes")
		mutex.Lock()
		skutil.LogErr(util.ExecuteCmd("pkill", []string{"-9", "chrome"}, []string{},
			util.PKILL_TIMEOUT, nil, nil))
		mutex.Unlock()
	}
}
Beispiel #22
0
func main() {
	defer common.LogPanic()
	master_common.Init()

	// Send start email.
	emailsArr := util.ParseEmails(*emails)
	emailsArr = append(emailsArr, util.CtAdmins...)
	if len(emailsArr) == 0 {
		glog.Error("At least one email address must be specified")
		return
	}
	skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&admin_tasks.RecreatePageSetsUpdateVars{}, *gaeTaskID))
	skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Creating pagesets", util.GetMasterLogLink(*runID), ""))
	// Ensure webapp is updated and completion email is sent even if task fails.
	defer updateWebappTask()
	defer sendEmail(emailsArr)
	if !*master_common.Local {
		// Cleanup tmp files after the run.
		defer util.CleanTmpDir()
	}
	// Finish with glog flush and how long the task took.
	defer util.TimeTrack(time.Now(), "Creating Pagesets on Workers")
	defer glog.Flush()

	if *pagesetType == "" {
		glog.Error("Must specify --pageset_type")
		return
	}

	cmd := append(master_common.WorkerSetupCmds(),
		// The main command that runs create_pagesets on all workers.
		fmt.Sprintf(
			"create_pagesets --worker_num=%s --log_dir=%s --log_id=%s --pageset_type=%s --local=%t;",
			util.WORKER_NUM_KEYWORD, util.GLogDir, *runID, *pagesetType, *master_common.Local))

	_, err := util.SSH(strings.Join(cmd, " "), util.Slaves, util.CREATE_PAGESETS_TIMEOUT)
	if err != nil {
		glog.Errorf("Error while running cmd %s: %s", cmd, err)
		return
	}
	*taskCompletedSuccessfully = true
}
Beispiel #23
0
func updateWebappTask() {
	vars := lua_scripts.UpdateVars{}
	vars.Id = *gaeTaskID
	vars.SetCompleted(taskCompletedSuccessfully)
	if luaOutputRemoteLink != "" {
		vars.ScriptOutput = sql.NullString{String: luaOutputRemoteLink, Valid: true}
	}
	if luaAggregatorOutputRemoteLink != "" {
		vars.AggregatedOutput = sql.NullString{String: luaAggregatorOutputRemoteLink, Valid: true}
	}
	skutil.LogErr(frontend.UpdateWebappTaskV2(&vars))
}
Beispiel #24
0
func main() {
	defer common.LogPanic()
	master_common.Init()

	// Send start email.
	emailsArr := util.ParseEmails(*emails)
	emailsArr = append(emailsArr, util.CtAdmins...)
	if len(emailsArr) == 0 {
		glog.Error("At least one email address must be specified")
		return
	}
	skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&chromium_builds.UpdateVars{}, *gaeTaskID))
	skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Build chromium", util.GetMasterLogLink(*runID), ""))
	// Ensure webapp is updated and completion email is sent even if task fails.
	defer updateWebappTask()
	defer sendEmail(emailsArr)
	if !*master_common.Local {
		// Cleanup tmp files after the run.
		defer util.CleanTmpDir()
	}
	// Finish with glog flush and how long the task took.
	defer util.TimeTrack(time.Now(), "Running build chromium")
	defer glog.Flush()

	if *chromiumHash == "" {
		glog.Error("Must specify --chromium_hash")
		return
	}
	if *skiaHash == "" {
		glog.Error("Must specify --skia_hash")
		return
	}

	if _, _, err := util.CreateChromiumBuild("", *targetPlatform, *chromiumHash, *skiaHash, *applyPatches); err != nil {
		glog.Errorf("Error while creating the Chromium build: %s", err)
		return
	}

	taskCompletedSuccessfully = true
}
Beispiel #25
0
// dequeue returns a task to be performed by the worker.
// It returns a triple: workerTask, itemsLeft_in_the_task_queue, error.
func (r *RedisRTC) dequeue() (*workerTask, int, error) {
	c := r.redisPool.Get()
	defer util.Close(c)

	// TODO: this needs to be a transaction.
	util.LogErr(c.Send("MULTI"))
	util.LogErr(c.Send("ZRANGE", r.queueKey, 0, 0, "WITHSCORES"))
	util.LogErr(c.Send("ZREMRANGEBYRANK", r.queueKey, 0, 0))
	util.LogErr(c.Send("ZCARD", r.queueKey))
	combinedVals, err := redis.Values(c.Do("EXEC"))

	if err != nil {
		return nil, 0, err
	}

	// Get the number of found elements.
	count := combinedVals[1].(int64)

	// If there are no values, we are done.
	if count == 0 {
		return nil, 0, nil
	}

	result := combinedVals[0].([]interface{})
	id := string(result[0].([]byte))
	priority, err := redis.Int64(result[1], nil)
	if err != nil {
		return nil, 0, err
	}
	itemsLeft := int(combinedVals[2].(int64))
	ret := &workerTask{id, priority}

	args := append([]interface{}{r.inProgressKey}, id)
	if _, err := c.Do("SADD", args...); err != nil {
		return nil, 0, err
	}

	return ret, itemsLeft, nil
}
Beispiel #26
0
func updateWebappTask() {
	if frontend.CTFE_V2 {
		vars := frontend.RecreatePageSetsUpdateVars{}
		vars.Id = *gaeTaskID
		vars.SetCompleted(*taskCompletedSuccessfully)
		skutil.LogErr(frontend.UpdateWebappTaskV2(&vars))
		return
	}
	if err := frontend.UpdateWebappTask(*gaeTaskID, frontend.UpdateAdminTasksWebapp, map[string]string{}); err != nil {
		glog.Errorf("Error while updating webapp task: %s", err)
		return
	}
}
Beispiel #27
0
// enqueue adds the given task and priority to the task queue. Updating the
// priority if necessary.
func (r *RedisRTC) enqueue(id string, priority int64) (bool, error) {
	c := r.redisPool.Get()
	defer util.Close(c)

	util.LogErr(c.Send("MULTI"))
	util.LogErr(c.Send("ZSCORE", r.queueKey, id))
	util.LogErr(c.Send("SISMEMBER", r.inProgressKey, id))
	util.LogErr(c.Send("EXISTS", r.key(id)))
	util.LogErr(c.Send("EXISTS", r.errorKey(id)))
	retVals, err := redis.Values(c.Do("EXEC"))
	if err != nil {
		return false, err
	}

	// See if the id is in the queue or in progress.
	inQueueScore, isInProgress := retVals[0], retVals[1].(int64) == 1
	foundResult, foundError := retVals[2].(int64), retVals[3].(int64)
	found := (foundResult + foundError) > 0

	// If the calculation is in process we don't have todo anything.
	if !isInProgress && !found {
		saveId := true

		// Only update the queue if this has a lower score.
		if inQueueScore != nil {
			oldPriority, _ := redis.Int64(inQueueScore, nil)
			saveId = priority < oldPriority
		}

		if saveId {
			if _, err = c.Do("ZADD", r.queueKey, priority, id); err != nil {
				return false, err
			}
		}
	}

	return found, nil
}
Beispiel #28
0
// VerifyLocalDevice does not throw an error if an Android device is connected and
// online. An error is returned if either "adb" is not installed or if the Android
// device is offline or missing.
func VerifyLocalDevice() error {
	// Run "adb version".
	// Command should return without an error.
	err := util.ExecuteCmd(util.BINARY_ADB, []string{"version"}, []string{},
		util.ADB_VERSION_TIMEOUT, nil, nil)
	if err != nil {
		return fmt.Errorf("adb not installed or not found: %s", err)
	}

	// Run "adb devices | grep offline".
	// Command should return with an error.
	devicesCmd := exec.Command(util.BINARY_ADB, "devices")
	offlineCmd := exec.Command("grep", "offline")
	offlineCmd.Stdin, _ = devicesCmd.StdoutPipe()
	offlineCmd.Stdout = skexec.WriteInfoLog
	skutil.LogErr(offlineCmd.Start())
	skutil.LogErr(devicesCmd.Run())
	if err := offlineCmd.Wait(); err == nil {
		// A nil error here means that an offline device was found.
		return fmt.Errorf("Android device is offline: %s", err)
	}

	// Running "adb devices | grep device$
	// Command should return without an error.
	devicesCmd = exec.Command(util.BINARY_ADB, "devices")
	missingCmd := exec.Command("grep", "device$")
	missingCmd.Stdin, _ = devicesCmd.StdoutPipe()
	missingCmd.Stdout = skexec.WriteInfoLog
	skutil.LogErr(missingCmd.Start())
	skutil.LogErr(devicesCmd.Run())
	if err := missingCmd.Wait(); err != nil {
		// An error here means that the device is missing.
		return fmt.Errorf("Android device is missing: %s", err)
	}

	return nil
}
Beispiel #29
0
// DownloadChromiumBuild downloads the specified Chromium build from Google
// Storage to a local dir.
func (gs *GsUtil) DownloadChromiumBuild(chromiumBuild string) error {
	localDir := filepath.Join(ChromiumBuildsDir, chromiumBuild)
	gsDir := filepath.Join(CHROMIUM_BUILDS_DIR_NAME, chromiumBuild)
	if equal, _ := gs.AreTimeStampsEqual(localDir, gsDir); equal {
		glog.Infof("Not downloading %s because TIMESTAMPS match", gsDir)
		return nil
	}
	glog.Infof("Timestamps between %s and %s are different. Downloading from Google Storage", localDir, gsDir)
	if err := gs.downloadRemoteDir(localDir, gsDir); err != nil {
		return fmt.Errorf("Error downloading %s into %s: %s", gsDir, localDir, err)
	}
	// Downloaded chrome binary needs to be set as an executable.
	util.LogErr(os.Chmod(filepath.Join(localDir, "chrome"), 0777))

	return nil
}
Beispiel #30
0
func updateWebappTask() {
	if frontend.CTFE_V2 {
		vars := frontend.CaptureSkpsUpdateVars{}
		vars.Id = *gaeTaskID
		vars.SetCompleted(taskCompletedSuccessfully)
		skutil.LogErr(frontend.UpdateWebappTaskV2(&vars))
		return
	}
	extraData := map[string]string{
		"output_link": outputRemoteLink,
	}
	if err := frontend.UpdateWebappTask(*gaeTaskID, frontend.UpdateCaptureSKPsTasksWebapp, extraData); err != nil {
		glog.Errorf("Error while updating webapp task: %s", err)
		return
	}
}