// Runs "git pull; make all". func updateAndBuild() error { token := statusTracker.StartTask(UPDATE_AND_BUILD) makefilePath := ctutil.CtTreeDir // TODO(benjaminwagner): Should this also do 'go get -u ...' and/or 'gclient sync'? err := exec.Run(&exec.Command{ Name: "git", Args: []string{"pull"}, Dir: makefilePath, Timeout: ctutil.GIT_PULL_TIMEOUT, LogStdout: true, LogStderr: true, }) if err != nil { statusTracker.FinishTask(token, err) return err } err = exec.Run(&exec.Command{ Name: "make", Args: []string{"all"}, Dir: makefilePath, Timeout: ctutil.MAKE_ALL_TIMEOUT, LogStdout: true, LogStderr: true, }) statusTracker.FinishTask(token, err) return err }
func (task *LuaScriptTask) Execute() error { runId := runId(task) chromiumBuildDir := ctutil.ChromiumBuildDir(task.ChromiumRev, task.SkiaRev, "") // TODO(benjaminwagner): Since run_lua_on_workers only reads the lua script in order to // upload to Google Storage, eventually we should move the upload step here to avoid writing // to disk. Not sure if we can/should do the same for the aggregator script. luaScriptName := runId + ".lua" luaScriptPath := filepath.Join(os.TempDir(), luaScriptName) if err := ioutil.WriteFile(luaScriptPath, []byte(task.LuaScript), 0666); err != nil { return err } defer skutil.Remove(luaScriptPath) if task.LuaAggregatorScript != "" { luaAggregatorName := runId + ".aggregator" luaAggregatorPath := filepath.Join(os.TempDir(), luaAggregatorName) if err := ioutil.WriteFile(luaAggregatorPath, []byte(task.LuaAggregatorScript), 0666); err != nil { return err } defer skutil.Remove(luaAggregatorPath) } return exec.Run(&exec.Command{ Name: "run_lua_on_workers", Args: []string{ "--emails=" + task.Username, "--description=" + task.Description, "--gae_task_id=" + strconv.FormatInt(task.Id, 10), "--pageset_type=" + task.PageSets, "--chromium_build=" + chromiumBuildDir, "--run_id=" + runId, "--log_dir=" + logDir, "--log_id=" + runId, }, Timeout: ctutil.MASTER_SCRIPT_RUN_LUA_TIMEOUT, }) }
func checkWorkerHealth() error { return exec.Run(&exec.Command{ Name: "check_workers_health", Args: []string{"--log_dir=" + logDir}, Timeout: ctutil.CHECK_WORKERS_HEALTH_TIMEOUT, }) }
// performBinaryAnalysis executes a command like: // timeout AnalysisTimeout catchsegv ./parse_foo_debug --input badbeef // from the working dir specified. // GNU timeout is used instead of the option on exec.Command because experimentation with the latter // showed evidence of that way leaking processes, which lead to OOM errors. // GNU catchsegv generates human readable dumps of crashes, which can then be scanned for stacktrace // information. The dumps (which come via standard out) and standard errors are recorded as strings. func performBinaryAnalysis(workingDirPath, baseExecutableName, fileName string, isDebug bool) (string, string, error) { suffix := "_release" if isDebug { suffix = "_debug" } pathToFile := filepath.Join(config.Aggregator.BinaryFuzzPath, fileName) pathToExecutable := fmt.Sprintf("./%s%s", baseExecutableName, suffix) timeoutInSeconds := fmt.Sprintf("%ds", config.Aggregator.AnalysisTimeout/time.Second) var dump bytes.Buffer var stdErr bytes.Buffer cmd := &exec.Command{ Name: "timeout", Args: []string{timeoutInSeconds, "catchsegv", pathToExecutable, "--src", "skp", "--skps", pathToFile, "--config", "8888"}, LogStdout: false, LogStderr: false, Stdout: &dump, Stderr: &stdErr, Dir: workingDirPath, } //errors are fine/expected from this, as we are dealing with bad fuzzes if err := exec.Run(cmd); err != nil { return dump.String(), stdErr.String(), nil } return dump.String(), stdErr.String(), nil }
// downloadSkia uses git to clone Skia from googlesource.com and check it out to the specified version. // Upon sucess, the SkiaVersion in config is set to be the current version and any dependencies // needed to compile Skia have been installed (e.g. the latest version of gyp). // It returns an error on failure. func DownloadSkia(version, path string, v config.VersionSetter) error { glog.Infof("Cloning Skia version %s to %s", version, path) repo, err := gitinfo.CloneOrUpdate("https://skia.googlesource.com/skia", path, false) if err != nil { return fmt.Errorf("Failed cloning Skia: %s", err) } if err = repo.SetToCommit(version); err != nil { return fmt.Errorf("Problem setting Skia to version %s: %s", version, err) } syncCmd := &exec.Command{ Name: "bin/sync-and-gyp", Dir: path, } if err := exec.Run(syncCmd); err != nil { return fmt.Errorf("Failed syncing and setting up gyp: %s", err) } if v != nil { if lc, err := repo.Details(version, false); err != nil { glog.Errorf("Could not get git details for skia version %s: %s", version, err) } else { v.SetSkiaVersion(lc) } } return nil }
func checkWorkerHealth() error { token := statusTracker.StartTask(CHECK_WORKER_HEALTH) err := exec.Run(&exec.Command{ Name: "check_workers_health", Args: []string{ "--log_dir=" + logDir, fmt.Sprintf("--local=%t", *master_common.Local), }, Timeout: ctutil.CHECK_WORKERS_HEALTH_TIMEOUT, }) statusTracker.FinishTask(token, err) return err }
// ExecuteCmd executes the specified binary with the specified args and env. Stdout and Stderr are // written to stdout and stderr respectively if specified. If not specified then Stdout and Stderr // will be outputted only to glog. func ExecuteCmd(binary string, args, env []string, timeout time.Duration, stdout, stderr io.Writer) error { return exec.Run(&exec.Command{ Name: binary, Args: args, Env: env, InheritPath: true, Timeout: timeout, LogStdout: true, Stdout: stdout, LogStderr: true, Stderr: stderr, }) }
// buildDM builds the test harness for parsing skp (and other) files. // First it creates a hard link for the gyp and cpp files. The gyp file is linked into Skia's gyp folder and the cpp file is linked into SKIA_ROOT/../fuzzer_cache/src, which is where the gyp file is configured to point. // Then it activates Skia's gyp command, which creates the build (ninja) files. // Finally, it runs those build files. // If any step fails in unexpected ways, it returns an error. func buildDM(buildType string, isClean bool, buildVars []string) error { glog.Infof("Building %s dm", buildType) // clean previous build if specified buildLocation := filepath.Join("out", buildType) if isClean { if err := os.RemoveAll(filepath.Join(config.Generator.SkiaRoot, buildLocation)); err != nil { return fmt.Errorf("Could not clear out %s before building: %s", filepath.Join(config.Generator.SkiaRoot, buildLocation), err) } } gypCmd := &exec.Command{ Name: "./gyp_skia", Dir: config.Generator.SkiaRoot, LogStdout: false, LogStderr: false, Env: append(buildVars, "GYP_DEFINES=skia_clang_build=1"), } // run gyp if err := exec.Run(gypCmd); err != nil { return fmt.Errorf("Failed gyp: %s", err) } ninjaPath := filepath.Join(config.Common.DepotToolsPath, "ninja") ninjaCmd := &exec.Command{ Name: ninjaPath, Args: []string{"-C", buildLocation, "dm"}, LogStdout: true, LogStderr: true, InheritPath: true, Dir: config.Generator.SkiaRoot, Env: buildVars, } // run ninja return exec.Run(ninjaCmd) }
// Runs "git pull; make all". func updateAndBuild() error { makefilePath := ctutil.CtTreeDir // TODO(benjaminwagner): Should this also do 'go get -u ...' and/or 'gclient sync'? err := exec.Run(&exec.Command{ Name: "git", Args: []string{"pull"}, Dir: makefilePath, Timeout: ctutil.GIT_PULL_TIMEOUT, LogStdout: true, LogStderr: true, }) if err != nil { return err } return exec.Run(&exec.Command{ Name: "make", Args: []string{"all"}, Dir: makefilePath, Timeout: ctutil.MAKE_ALL_TIMEOUT, LogStdout: true, LogStderr: true, }) }
func (task *RecreatePageSetsTask) Execute() error { runId := runId(task) return exec.Run(&exec.Command{ Name: "create_pagesets_on_workers", Args: []string{ "--emails=" + task.Username, "--gae_task_id=" + strconv.FormatInt(task.Id, 10), "--run_id=" + runId, "--pageset_type=" + task.PageSets, "--log_dir=" + logDir, "--log_id=" + runId, }, Timeout: ctutil.MASTER_SCRIPT_CREATE_PAGESETS_TIMEOUT, }) }
func (task *RecreateWebpageArchivesTask) Execute() error { runId := runId(task) chromiumBuildDir := ctutil.ChromiumBuildDir(task.ChromiumRev, task.SkiaRev, "") return exec.Run(&exec.Command{ Name: "capture_archives_on_workers", Args: []string{ "--emails=" + task.Username, "--gae_task_id=" + strconv.FormatInt(task.Id, 10), "--run_id=" + runId, "--pageset_type=" + task.PageSets, "--chromium_build=" + chromiumBuildDir, "--log_dir=" + logDir, "--log_id=" + runId, }, Timeout: ctutil.MASTER_SCRIPT_CAPTURE_ARCHIVES_TIMEOUT, }) }
func (task *ChromiumBuildTask) Execute() error { runId := runId(task) return exec.Run(&exec.Command{ Name: "build_chromium", Args: []string{ "--emails=" + task.Username, "--gae_task_id=" + strconv.FormatInt(task.Id, 10), "--run_id=" + runId, "--target_platform=Linux", "--apply_patches=false", "--chromium_hash=" + task.ChromiumRev, "--skia_hash=" + task.SkiaRev, "--log_dir=" + logDir, "--log_id=" + runId, }, Timeout: ctutil.MASTER_SCRIPT_BUILD_CHROMIUM_TIMEOUT, }) }
func (task *ChromiumPerfTask) Execute() error { token := statusTracker.StartTask(CHROMIUM_PERF) runId := runId(task) // TODO(benjaminwagner): Since run_chromium_perf_on_workers only reads these in order to // upload to Google Storage, eventually we should move the upload step here to avoid writing // to disk. for fileSuffix, patch := range map[string]string{ ".chromium.patch": task.ChromiumPatch, ".skia.patch": task.SkiaPatch, ".benchmark.patch": task.BenchmarkPatch, } { // Add an extra newline at the end because git sometimes rejects patches due to // missing newlines. patch = patch + "\n" patchPath := filepath.Join(os.TempDir(), runId+fileSuffix) if err := ioutil.WriteFile(patchPath, []byte(patch), 0666); err != nil { return err } defer skutil.Remove(patchPath) } err := exec.Run(&exec.Command{ Name: "run_chromium_perf_on_workers", Args: []string{ "--emails=" + task.Username, "--description=" + task.Description, "--gae_task_id=" + strconv.FormatInt(task.Id, 10), "--pageset_type=" + task.PageSets, "--benchmark_name=" + task.Benchmark, "--benchmark_extra_args=" + task.BenchmarkArgs, "--browser_extra_args_nopatch=" + task.BrowserArgsNoPatch, "--browser_extra_args_withpatch=" + task.BrowserArgsWithPatch, "--repeat_benchmark=" + strconv.FormatInt(task.RepeatRuns, 10), "--run_in_parallel=" + strconv.FormatBool(task.RunInParallel), "--target_platform=" + task.Platform, "--run_id=" + runId, "--log_dir=" + logDir, "--log_id=" + runId, fmt.Sprintf("--local=%t", *master_common.Local), }, Timeout: ctutil.MASTER_SCRIPT_RUN_CHROMIUM_PERF_TIMEOUT, }) statusTracker.FinishTask(token, err) return err }
func (task *CaptureSkpsTask) Execute() error { runId := runId(task) chromiumBuildDir := ctutil.ChromiumBuildDir(task.ChromiumRev, task.SkiaRev, "") return exec.Run(&exec.Command{ Name: "capture_skps_on_workers", Args: []string{ "--emails=" + task.Username, "--description=" + task.Description, "--gae_task_id=" + strconv.FormatInt(task.Id, 10), "--pageset_type=" + task.PageSets, "--chromium_build=" + chromiumBuildDir, "--target_platform=Linux", "--run_id=" + runId, "--log_dir=" + logDir, "--log_id=" + runId, }, Timeout: ctutil.MASTER_SCRIPT_CAPTURE_SKPS_TIMEOUT, }) }
func (task *RecreatePageSetsTask) Execute() error { token := statusTracker.StartTask(RECREATE_PAGE_SETS) runId := runId(task) err := exec.Run(&exec.Command{ Name: "create_pagesets_on_workers", Args: []string{ "--emails=" + task.Username, "--gae_task_id=" + strconv.FormatInt(task.Id, 10), "--run_id=" + runId, "--pageset_type=" + task.PageSets, "--log_dir=" + logDir, "--log_id=" + runId, fmt.Sprintf("--local=%t", *master_common.Local), }, Timeout: ctutil.MASTER_SCRIPT_CREATE_PAGESETS_TIMEOUT, }) statusTracker.FinishTask(token, err) return err }
func (task *RecreateWebpageArchivesTask) Execute() error { token := statusTracker.StartTask(RECREATE_WEBPAGE_ARCHIVES) runId := runId(task) chromiumBuildDir := ctutil.ChromiumBuildDir(task.ChromiumRev, task.SkiaRev, "") err := exec.Run(&exec.Command{ Name: "capture_archives_on_workers", Args: []string{ "--emails=" + task.Username, "--gae_task_id=" + strconv.FormatInt(task.Id, 10), "--run_id=" + runId, "--pageset_type=" + task.PageSets, "--chromium_build=" + chromiumBuildDir, "--log_dir=" + logDir, "--log_id=" + runId, fmt.Sprintf("--local=%t", *master_common.Local), }, Timeout: ctutil.MASTER_SCRIPT_CAPTURE_ARCHIVES_TIMEOUT, }) statusTracker.FinishTask(token, err) return err }
func (task *ChromiumBuildTask) Execute() error { token := statusTracker.StartTask(CHROMIUM_BUILD) runId := runId(task) err := exec.Run(&exec.Command{ Name: "build_chromium", Args: []string{ "--emails=" + task.Username, "--gae_task_id=" + strconv.FormatInt(task.Id, 10), "--run_id=" + runId, "--target_platform=Linux", "--apply_patches=false", "--chromium_hash=" + task.ChromiumRev, "--skia_hash=" + task.SkiaRev, "--log_dir=" + logDir, "--log_id=" + runId, fmt.Sprintf("--local=%t", *master_common.Local), }, Timeout: ctutil.MASTER_SCRIPT_BUILD_CHROMIUM_TIMEOUT, }) statusTracker.FinishTask(token, err) return err }
func (task *CaptureSkpsTask) Execute() error { token := statusTracker.StartTask(CAPTURE_SKPS) runId := runId(task) chromiumBuildDir := ctutil.ChromiumBuildDir(task.ChromiumRev, task.SkiaRev, "") err := exec.Run(&exec.Command{ Name: "capture_skps_on_workers", Args: []string{ "--emails=" + task.Username, "--description=" + task.Description, "--gae_task_id=" + strconv.FormatInt(task.Id, 10), "--pageset_type=" + task.PageSets, "--chromium_build=" + chromiumBuildDir, "--target_platform=Linux", "--run_id=" + runId, "--log_dir=" + logDir, "--log_id=" + runId, fmt.Sprintf("--local=%t", *master_common.Local), }, Timeout: ctutil.MASTER_SCRIPT_CAPTURE_SKPS_TIMEOUT, }) statusTracker.FinishTask(token, err) return err }
// LaTex finds <latex-pic> nodes in the html and // replaces them with PNG images of the rendered LaTex. func LaTex(node *html.Node, root string) error { latexNodes := []*html.Node{} var f func(*html.Node) error f = func(n *html.Node) error { if n.Type == html.ElementNode && n.Data == "latex-pic" { // Create a tmp file to write the Latex code into. file, err := ioutil.TempFile("/tmp", "piccolo-latex-") if err != nil { return fmt.Errorf("Couldn't create temp file: %s", err) } _, err = file.Write([]byte(n.FirstChild.Data)) if err != nil { return fmt.Errorf("Failed to write file: %s", err) } file.Close() defer os.Remove(file.Name()) // And create a tmp file to receive the PNG. dest, err := ioutil.TempFile("/tmp", "piccolo-latex-") if err != nil { return fmt.Errorf("Couldn't create temp file: %s", err) } dest.Close() defer os.Remove(dest.Name()) // Convert the latex to a PNG with: // // tex2im -z -a -o ./dst/test.png test.tex args := fmt.Sprintf("-z -a -r 100x100 -x %s/tex2im_header -o %s %s", root, dest.Name(), file.Name()) output := bytes.Buffer{} err = exec.Run(&exec.Command{ Name: "tex2im", Args: strings.Split(args, " "), Env: []string{}, CombinedOutput: &output, Timeout: 10 * time.Minute, InheritPath: true, }) if err != nil { return fmt.Errorf("Failed to run tex2im: %q %s", output, err) } b, err := ioutil.ReadFile(dest.Name()) if err != nil { return fmt.Errorf("Failed to read PNG: %s", err) } uri := fmt.Sprintf("data:image/png;base64,%s", base64.StdEncoding.EncodeToString(b)) // Create an img node. imgNode := &html.Node{ Type: html.ElementNode, Data: "img", Attr: []html.Attribute{ html.Attribute{ Key: "src", Val: uri, }, html.Attribute{ Key: "alt", Val: n.FirstChild.Data, }, html.Attribute{ Key: "title", Val: n.FirstChild.Data, }, }, } // Insert it just before the latex-pic element. n.Parent.InsertBefore(imgNode, n) // Remove the original latex-pic element later. latexNodes = append(latexNodes, n) } for c := n.FirstChild; c != nil; c = c.NextSibling { f(c) } return nil } err := f(node) for _, n := range latexNodes { n.Parent.RemoveChild(n) } return err }
// buildSkiaAST returns a ~13GB dump of all ASTs created when building Skia. // It builds a release version of Skia using Clang, once without the -ast-dump flag // and once with. The output of the latter is returned. func buildSkiaAST() ([]byte, error) { // TODO(kjlubick): Refactor this to share functionality with common.BuildClangDM? // clean previous build buildLocation := filepath.Join("out", "Release") if err := os.RemoveAll(filepath.Join(config.FrontEnd.SkiaRoot, buildLocation)); err != nil { return nil, err } gypCmd := &exec.Command{ Name: "./gyp_skia", Dir: config.FrontEnd.SkiaRoot, LogStdout: false, LogStderr: false, Env: []string{ `GYP_DEFINES=skia_clang_build=1`, fmt.Sprintf("CC=%s", config.Common.ClangPath), fmt.Sprintf("CXX=%s", config.Common.ClangPlusPlusPath), }, } // run gyp if err := exec.Run(gypCmd); err != nil { glog.Errorf("Failed gyp: %s", err) return nil, err } ninjaPath := filepath.Join(config.Common.DepotToolsPath, "ninja") ninjaCmd := &exec.Command{ Name: ninjaPath, Args: []string{"-C", buildLocation}, LogStdout: true, LogStderr: true, Dir: config.FrontEnd.SkiaRoot, Env: []string{ fmt.Sprintf("CC=%s", config.Common.ClangPath), fmt.Sprintf("CXX=%s", config.Common.ClangPlusPlusPath), }, InheritPath: true, } // first build // Skia needs to be built once without the -ast-dump flag before it is built // with -ast-dump, otherwise, the build fails with an error about deleting a file // that doesn't exist. if err := exec.Run(ninjaCmd); err != nil { glog.Errorf("Failed ninja: %s", err) return nil, err } // Quotes are NOT needed around the params. Doing so actually causes // a failure that "-Xclang -ast-dump -fsyntax-only" is an unused argument. gypCmd.Env = append(gypCmd.Env, `CXXFLAGS=-Xclang -ast-dump -fsyntax-only`) // run gyp again to remake build files with ast-dump flags if err := exec.Run(gypCmd); err != nil { glog.Errorf("Failed gyp message: %s", err) return nil, err } // Run ninja again, which will dump the ast to std out var ast bytes.Buffer var stdErr bytes.Buffer ninjaCmd = &exec.Command{ Name: ninjaPath, Args: []string{"-C", buildLocation}, LogStdout: false, LogStderr: false, Stdout: &ast, Stderr: &stdErr, Dir: config.FrontEnd.SkiaRoot, Env: []string{ fmt.Sprintf("CC=%s", config.Common.ClangPath), fmt.Sprintf("CXX=%s", config.Common.ClangPlusPlusPath), }, InheritPath: true, } glog.Info("Generating AST") if err := exec.Run(ninjaCmd); err != nil { return nil, fmt.Errorf("Error generating AST %s:\nstderr: %s", err, stdErr.String()) } glog.Info("Done generating AST") return ast.Bytes(), nil }
// buildSkiaAST returns a ~13GB dump of all ASTs created when building Skia. // It builds a release version of Skia using Clang, once without the -ast-dump flag // and once with. The output of the latter is returned. func buildSkiaAST() ([]byte, error) { // both gyp_skia and ninja need the environment variables set, so it is easier // to just set them with os.Setenv rather than using the Command's Env: variables if err := os.Setenv("CC", config.Generator.ClangPath); err != nil { return nil, err } if err := os.Setenv("CXX", config.Generator.ClangPlusPlusPath); err != nil { return nil, err } // clean previous build buildLocation := filepath.Join("out", "Release") if err := os.RemoveAll(filepath.Join(config.Generator.SkiaRoot, buildLocation)); err != nil { return nil, err } gypCmd := &exec.Command{ Name: "./gyp_skia", Dir: config.Generator.SkiaRoot, } // run gyp if err := exec.Run(gypCmd); err != nil { glog.Errorf("Failed gyp: %s", err) return nil, err } ninjaCmd := &exec.Command{ Name: "ninja", Args: []string{"-C", buildLocation}, LogStdout: false, LogStderr: false, Dir: config.Generator.SkiaRoot, } // first build // Skia needs to be built once without the -ast-dump flag before it is built // with -ast-dump, otherwise, the build fails with an error about deleting a file // that doesn't exist. if err := exec.Run(ninjaCmd); err != nil { glog.Errorf("Failed ninja: %s", err) return nil, err } if err := os.Setenv("CXXFLAGS", "-Xclang -ast-dump -fsyntax-only"); err != nil { return nil, err } // run gyp again to remake build files with ast-dump flags if err := exec.Run(gypCmd); err != nil { glog.Errorf("Failed gyp message: %s", err) return nil, err } // Run ninja again, which will dump the ast to std out var ast bytes.Buffer var stdErr bytes.Buffer ninjaCmd = &exec.Command{ Name: "ninja", Args: []string{"-C", buildLocation}, LogStdout: false, LogStderr: false, Stdout: &ast, Stderr: &stdErr, Dir: config.Generator.SkiaRoot, } if err := exec.Run(ninjaCmd); err != nil { return nil, fmt.Errorf("Error generating AST %s:\nstderr: %s", err, stdErr.String()) } glog.Info("Done parsing ast") return ast.Bytes(), nil }