// Runs "git pull; make all".
func updateAndBuild() error {
token := statusTracker.StartTask(UPDATE_AND_BUILD)
makefilePath := ctutil.CtTreeDir
// TODO(benjaminwagner): Should this also do 'go get -u ...' and/or 'gclient sync'?
err := exec.Run(&exec.Command{
Name: "git",
Args: []string{"pull"},
Dir: makefilePath,
Timeout: ctutil.GIT_PULL_TIMEOUT,
LogStdout: true,
LogStderr: true,
})
if err != nil {
statusTracker.FinishTask(token, err)
return err
}
err = exec.Run(&exec.Command{
Name: "make",
Args: []string{"all"},
Dir: makefilePath,
Timeout: ctutil.MAKE_ALL_TIMEOUT,
LogStdout: true,
LogStderr: true,
})
statusTracker.FinishTask(token, err)
return err
}
func (task *LuaScriptTask) Execute() error {
runId := runId(task)
chromiumBuildDir := ctutil.ChromiumBuildDir(task.ChromiumRev, task.SkiaRev, "")
// TODO(benjaminwagner): Since run_lua_on_workers only reads the lua script in order to
// upload to Google Storage, eventually we should move the upload step here to avoid writing
// to disk. Not sure if we can/should do the same for the aggregator script.
luaScriptName := runId + ".lua"
luaScriptPath := filepath.Join(os.TempDir(), luaScriptName)
if err := ioutil.WriteFile(luaScriptPath, []byte(task.LuaScript), 0666); err != nil {
return err
}
defer skutil.Remove(luaScriptPath)
if task.LuaAggregatorScript != "" {
luaAggregatorName := runId + ".aggregator"
luaAggregatorPath := filepath.Join(os.TempDir(), luaAggregatorName)
if err := ioutil.WriteFile(luaAggregatorPath, []byte(task.LuaAggregatorScript), 0666); err != nil {
return err
}
defer skutil.Remove(luaAggregatorPath)
}
return exec.Run(&exec.Command{
Name: "run_lua_on_workers",
Args: []string{
"--emails=" + task.Username,
"--description=" + task.Description,
"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
"--pageset_type=" + task.PageSets,
"--chromium_build=" + chromiumBuildDir,
"--run_id=" + runId,
"--log_dir=" + logDir,
"--log_id=" + runId,
},
Timeout: ctutil.MASTER_SCRIPT_RUN_LUA_TIMEOUT,
})
}
func checkWorkerHealth() error {
return exec.Run(&exec.Command{
Name: "check_workers_health",
Args: []string{"--log_dir=" + logDir},
Timeout: ctutil.CHECK_WORKERS_HEALTH_TIMEOUT,
})
}
// performBinaryAnalysis executes a command like:
// timeout AnalysisTimeout catchsegv ./parse_foo_debug --input badbeef
// from the working dir specified.
// GNU timeout is used instead of the option on exec.Command because experimentation with the latter
// showed evidence of that way leaking processes, which lead to OOM errors.
// GNU catchsegv generates human readable dumps of crashes, which can then be scanned for stacktrace
// information. The dumps (which come via standard out) and standard errors are recorded as strings.
func performBinaryAnalysis(workingDirPath, baseExecutableName, fileName string, isDebug bool) (string, string, error) {
suffix := "_release"
if isDebug {
suffix = "_debug"
}
pathToFile := filepath.Join(config.Aggregator.BinaryFuzzPath, fileName)
pathToExecutable := fmt.Sprintf("./%s%s", baseExecutableName, suffix)
timeoutInSeconds := fmt.Sprintf("%ds", config.Aggregator.AnalysisTimeout/time.Second)
var dump bytes.Buffer
var stdErr bytes.Buffer
cmd := &exec.Command{
Name: "timeout",
Args: []string{timeoutInSeconds, "catchsegv", pathToExecutable, "--src", "skp", "--skps", pathToFile, "--config", "8888"},
LogStdout: false,
LogStderr: false,
Stdout: &dump,
Stderr: &stdErr,
Dir: workingDirPath,
}
//errors are fine/expected from this, as we are dealing with bad fuzzes
if err := exec.Run(cmd); err != nil {
return dump.String(), stdErr.String(), nil
}
return dump.String(), stdErr.String(), nil
}
// downloadSkia uses git to clone Skia from googlesource.com and check it out to the specified version.
// Upon sucess, the SkiaVersion in config is set to be the current version and any dependencies
// needed to compile Skia have been installed (e.g. the latest version of gyp).
// It returns an error on failure.
func DownloadSkia(version, path string, v config.VersionSetter) error {
glog.Infof("Cloning Skia version %s to %s", version, path)
repo, err := gitinfo.CloneOrUpdate("https://skia.googlesource.com/skia", path, false)
if err != nil {
return fmt.Errorf("Failed cloning Skia: %s", err)
}
if err = repo.SetToCommit(version); err != nil {
return fmt.Errorf("Problem setting Skia to version %s: %s", version, err)
}
syncCmd := &exec.Command{
Name: "bin/sync-and-gyp",
Dir: path,
}
if err := exec.Run(syncCmd); err != nil {
return fmt.Errorf("Failed syncing and setting up gyp: %s", err)
}
if v != nil {
if lc, err := repo.Details(version, false); err != nil {
glog.Errorf("Could not get git details for skia version %s: %s", version, err)
} else {
v.SetSkiaVersion(lc)
}
}
return nil
}
func checkWorkerHealth() error {
token := statusTracker.StartTask(CHECK_WORKER_HEALTH)
err := exec.Run(&exec.Command{
Name: "check_workers_health",
Args: []string{
"--log_dir=" + logDir,
fmt.Sprintf("--local=%t", *master_common.Local),
},
Timeout: ctutil.CHECK_WORKERS_HEALTH_TIMEOUT,
})
statusTracker.FinishTask(token, err)
return err
}
// ExecuteCmd executes the specified binary with the specified args and env. Stdout and Stderr are
// written to stdout and stderr respectively if specified. If not specified then Stdout and Stderr
// will be outputted only to glog.
func ExecuteCmd(binary string, args, env []string, timeout time.Duration, stdout, stderr io.Writer) error {
return exec.Run(&exec.Command{
Name: binary,
Args: args,
Env: env,
InheritPath: true,
Timeout: timeout,
LogStdout: true,
Stdout: stdout,
LogStderr: true,
Stderr: stderr,
})
}
// buildDM builds the test harness for parsing skp (and other) files.
// First it creates a hard link for the gyp and cpp files. The gyp file is linked into Skia's gyp folder and the cpp file is linked into SKIA_ROOT/../fuzzer_cache/src, which is where the gyp file is configured to point.
// Then it activates Skia's gyp command, which creates the build (ninja) files.
// Finally, it runs those build files.
// If any step fails in unexpected ways, it returns an error.
func buildDM(buildType string, isClean bool, buildVars []string) error {
glog.Infof("Building %s dm", buildType)
// clean previous build if specified
buildLocation := filepath.Join("out", buildType)
if isClean {
if err := os.RemoveAll(filepath.Join(config.Generator.SkiaRoot, buildLocation)); err != nil {
return fmt.Errorf("Could not clear out %s before building: %s", filepath.Join(config.Generator.SkiaRoot, buildLocation), err)
}
}
gypCmd := &exec.Command{
Name: "./gyp_skia",
Dir: config.Generator.SkiaRoot,
LogStdout: false,
LogStderr: false,
Env: append(buildVars, "GYP_DEFINES=skia_clang_build=1"),
}
// run gyp
if err := exec.Run(gypCmd); err != nil {
return fmt.Errorf("Failed gyp: %s", err)
}
ninjaPath := filepath.Join(config.Common.DepotToolsPath, "ninja")
ninjaCmd := &exec.Command{
Name: ninjaPath,
Args: []string{"-C", buildLocation, "dm"},
LogStdout: true,
LogStderr: true,
InheritPath: true,
Dir: config.Generator.SkiaRoot,
Env: buildVars,
}
// run ninja
return exec.Run(ninjaCmd)
}
// Runs "git pull; make all".
func updateAndBuild() error {
makefilePath := ctutil.CtTreeDir
// TODO(benjaminwagner): Should this also do 'go get -u ...' and/or 'gclient sync'?
err := exec.Run(&exec.Command{
Name: "git",
Args: []string{"pull"},
Dir: makefilePath,
Timeout: ctutil.GIT_PULL_TIMEOUT,
LogStdout: true,
LogStderr: true,
})
if err != nil {
return err
}
return exec.Run(&exec.Command{
Name: "make",
Args: []string{"all"},
Dir: makefilePath,
Timeout: ctutil.MAKE_ALL_TIMEOUT,
LogStdout: true,
LogStderr: true,
})
}
func (task *RecreatePageSetsTask) Execute() error {
runId := runId(task)
return exec.Run(&exec.Command{
Name: "create_pagesets_on_workers",
Args: []string{
"--emails=" + task.Username,
"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
"--run_id=" + runId,
"--pageset_type=" + task.PageSets,
"--log_dir=" + logDir,
"--log_id=" + runId,
},
Timeout: ctutil.MASTER_SCRIPT_CREATE_PAGESETS_TIMEOUT,
})
}
func (task *RecreateWebpageArchivesTask) Execute() error {
runId := runId(task)
chromiumBuildDir := ctutil.ChromiumBuildDir(task.ChromiumRev, task.SkiaRev, "")
return exec.Run(&exec.Command{
Name: "capture_archives_on_workers",
Args: []string{
"--emails=" + task.Username,
"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
"--run_id=" + runId,
"--pageset_type=" + task.PageSets,
"--chromium_build=" + chromiumBuildDir,
"--log_dir=" + logDir,
"--log_id=" + runId,
},
Timeout: ctutil.MASTER_SCRIPT_CAPTURE_ARCHIVES_TIMEOUT,
})
}
func (task *ChromiumBuildTask) Execute() error {
runId := runId(task)
return exec.Run(&exec.Command{
Name: "build_chromium",
Args: []string{
"--emails=" + task.Username,
"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
"--run_id=" + runId,
"--target_platform=Linux",
"--apply_patches=false",
"--chromium_hash=" + task.ChromiumRev,
"--skia_hash=" + task.SkiaRev,
"--log_dir=" + logDir,
"--log_id=" + runId,
},
Timeout: ctutil.MASTER_SCRIPT_BUILD_CHROMIUM_TIMEOUT,
})
}
func (task *ChromiumPerfTask) Execute() error {
token := statusTracker.StartTask(CHROMIUM_PERF)
runId := runId(task)
// TODO(benjaminwagner): Since run_chromium_perf_on_workers only reads these in order to
// upload to Google Storage, eventually we should move the upload step here to avoid writing
// to disk.
for fileSuffix, patch := range map[string]string{
".chromium.patch": task.ChromiumPatch,
".skia.patch": task.SkiaPatch,
".benchmark.patch": task.BenchmarkPatch,
} {
// Add an extra newline at the end because git sometimes rejects patches due to
// missing newlines.
patch = patch + "\n"
patchPath := filepath.Join(os.TempDir(), runId+fileSuffix)
if err := ioutil.WriteFile(patchPath, []byte(patch), 0666); err != nil {
return err
}
defer skutil.Remove(patchPath)
}
err := exec.Run(&exec.Command{
Name: "run_chromium_perf_on_workers",
Args: []string{
"--emails=" + task.Username,
"--description=" + task.Description,
"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
"--pageset_type=" + task.PageSets,
"--benchmark_name=" + task.Benchmark,
"--benchmark_extra_args=" + task.BenchmarkArgs,
"--browser_extra_args_nopatch=" + task.BrowserArgsNoPatch,
"--browser_extra_args_withpatch=" + task.BrowserArgsWithPatch,
"--repeat_benchmark=" + strconv.FormatInt(task.RepeatRuns, 10),
"--run_in_parallel=" + strconv.FormatBool(task.RunInParallel),
"--target_platform=" + task.Platform,
"--run_id=" + runId,
"--log_dir=" + logDir,
"--log_id=" + runId,
fmt.Sprintf("--local=%t", *master_common.Local),
},
Timeout: ctutil.MASTER_SCRIPT_RUN_CHROMIUM_PERF_TIMEOUT,
})
statusTracker.FinishTask(token, err)
return err
}
func (task *CaptureSkpsTask) Execute() error {
runId := runId(task)
chromiumBuildDir := ctutil.ChromiumBuildDir(task.ChromiumRev, task.SkiaRev, "")
return exec.Run(&exec.Command{
Name: "capture_skps_on_workers",
Args: []string{
"--emails=" + task.Username,
"--description=" + task.Description,
"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
"--pageset_type=" + task.PageSets,
"--chromium_build=" + chromiumBuildDir,
"--target_platform=Linux",
"--run_id=" + runId,
"--log_dir=" + logDir,
"--log_id=" + runId,
},
Timeout: ctutil.MASTER_SCRIPT_CAPTURE_SKPS_TIMEOUT,
})
}
func (task *RecreatePageSetsTask) Execute() error {
token := statusTracker.StartTask(RECREATE_PAGE_SETS)
runId := runId(task)
err := exec.Run(&exec.Command{
Name: "create_pagesets_on_workers",
Args: []string{
"--emails=" + task.Username,
"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
"--run_id=" + runId,
"--pageset_type=" + task.PageSets,
"--log_dir=" + logDir,
"--log_id=" + runId,
fmt.Sprintf("--local=%t", *master_common.Local),
},
Timeout: ctutil.MASTER_SCRIPT_CREATE_PAGESETS_TIMEOUT,
})
statusTracker.FinishTask(token, err)
return err
}
func (task *RecreateWebpageArchivesTask) Execute() error {
token := statusTracker.StartTask(RECREATE_WEBPAGE_ARCHIVES)
runId := runId(task)
chromiumBuildDir := ctutil.ChromiumBuildDir(task.ChromiumRev, task.SkiaRev, "")
err := exec.Run(&exec.Command{
Name: "capture_archives_on_workers",
Args: []string{
"--emails=" + task.Username,
"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
"--run_id=" + runId,
"--pageset_type=" + task.PageSets,
"--chromium_build=" + chromiumBuildDir,
"--log_dir=" + logDir,
"--log_id=" + runId,
fmt.Sprintf("--local=%t", *master_common.Local),
},
Timeout: ctutil.MASTER_SCRIPT_CAPTURE_ARCHIVES_TIMEOUT,
})
statusTracker.FinishTask(token, err)
return err
}
func (task *ChromiumBuildTask) Execute() error {
token := statusTracker.StartTask(CHROMIUM_BUILD)
runId := runId(task)
err := exec.Run(&exec.Command{
Name: "build_chromium",
Args: []string{
"--emails=" + task.Username,
"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
"--run_id=" + runId,
"--target_platform=Linux",
"--apply_patches=false",
"--chromium_hash=" + task.ChromiumRev,
"--skia_hash=" + task.SkiaRev,
"--log_dir=" + logDir,
"--log_id=" + runId,
fmt.Sprintf("--local=%t", *master_common.Local),
},
Timeout: ctutil.MASTER_SCRIPT_BUILD_CHROMIUM_TIMEOUT,
})
statusTracker.FinishTask(token, err)
return err
}
func (task *CaptureSkpsTask) Execute() error {
token := statusTracker.StartTask(CAPTURE_SKPS)
runId := runId(task)
chromiumBuildDir := ctutil.ChromiumBuildDir(task.ChromiumRev, task.SkiaRev, "")
err := exec.Run(&exec.Command{
Name: "capture_skps_on_workers",
Args: []string{
"--emails=" + task.Username,
"--description=" + task.Description,
"--gae_task_id=" + strconv.FormatInt(task.Id, 10),
"--pageset_type=" + task.PageSets,
"--chromium_build=" + chromiumBuildDir,
"--target_platform=Linux",
"--run_id=" + runId,
"--log_dir=" + logDir,
"--log_id=" + runId,
fmt.Sprintf("--local=%t", *master_common.Local),
},
Timeout: ctutil.MASTER_SCRIPT_CAPTURE_SKPS_TIMEOUT,
})
statusTracker.FinishTask(token, err)
return err
}
// LaTex finds <latex-pic> nodes in the html and
// replaces them with PNG images of the rendered LaTex.
func LaTex(node *html.Node, root string) error {
latexNodes := []*html.Node{}
var f func(*html.Node) error
f = func(n *html.Node) error {
if n.Type == html.ElementNode && n.Data == "latex-pic" {
// Create a tmp file to write the Latex code into.
file, err := ioutil.TempFile("/tmp", "piccolo-latex-")
if err != nil {
return fmt.Errorf("Couldn't create temp file: %s", err)
}
_, err = file.Write([]byte(n.FirstChild.Data))
if err != nil {
return fmt.Errorf("Failed to write file: %s", err)
}
file.Close()
defer os.Remove(file.Name())
// And create a tmp file to receive the PNG.
dest, err := ioutil.TempFile("/tmp", "piccolo-latex-")
if err != nil {
return fmt.Errorf("Couldn't create temp file: %s", err)
}
dest.Close()
defer os.Remove(dest.Name())
// Convert the latex to a PNG with:
//
// tex2im -z -a -o ./dst/test.png test.tex
args := fmt.Sprintf("-z -a -r 100x100 -x %s/tex2im_header -o %s %s", root, dest.Name(), file.Name())
output := bytes.Buffer{}
err = exec.Run(&exec.Command{
Name: "tex2im",
Args: strings.Split(args, " "),
Env: []string{},
CombinedOutput: &output,
Timeout: 10 * time.Minute,
InheritPath: true,
})
if err != nil {
return fmt.Errorf("Failed to run tex2im: %q %s", output, err)
}
b, err := ioutil.ReadFile(dest.Name())
if err != nil {
return fmt.Errorf("Failed to read PNG: %s", err)
}
uri := fmt.Sprintf("data:image/png;base64,%s", base64.StdEncoding.EncodeToString(b))
// Create an img node.
imgNode := &html.Node{
Type: html.ElementNode,
Data: "img",
Attr: []html.Attribute{
html.Attribute{
Key: "src",
Val: uri,
},
html.Attribute{
Key: "alt",
Val: n.FirstChild.Data,
},
html.Attribute{
Key: "title",
Val: n.FirstChild.Data,
},
},
}
// Insert it just before the latex-pic element.
n.Parent.InsertBefore(imgNode, n)
// Remove the original latex-pic element later.
latexNodes = append(latexNodes, n)
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
f(c)
}
return nil
}
err := f(node)
for _, n := range latexNodes {
n.Parent.RemoveChild(n)
}
return err
}
// buildSkiaAST returns a ~13GB dump of all ASTs created when building Skia.
// It builds a release version of Skia using Clang, once without the -ast-dump flag
// and once with. The output of the latter is returned.
func buildSkiaAST() ([]byte, error) {
// TODO(kjlubick): Refactor this to share functionality with common.BuildClangDM?
// clean previous build
buildLocation := filepath.Join("out", "Release")
if err := os.RemoveAll(filepath.Join(config.FrontEnd.SkiaRoot, buildLocation)); err != nil {
return nil, err
}
gypCmd := &exec.Command{
Name: "./gyp_skia",
Dir: config.FrontEnd.SkiaRoot,
LogStdout: false,
LogStderr: false,
Env: []string{
`GYP_DEFINES=skia_clang_build=1`,
fmt.Sprintf("CC=%s", config.Common.ClangPath),
fmt.Sprintf("CXX=%s", config.Common.ClangPlusPlusPath),
},
}
// run gyp
if err := exec.Run(gypCmd); err != nil {
glog.Errorf("Failed gyp: %s", err)
return nil, err
}
ninjaPath := filepath.Join(config.Common.DepotToolsPath, "ninja")
ninjaCmd := &exec.Command{
Name: ninjaPath,
Args: []string{"-C", buildLocation},
LogStdout: true,
LogStderr: true,
Dir: config.FrontEnd.SkiaRoot,
Env: []string{
fmt.Sprintf("CC=%s", config.Common.ClangPath),
fmt.Sprintf("CXX=%s", config.Common.ClangPlusPlusPath),
},
InheritPath: true,
}
// first build
// Skia needs to be built once without the -ast-dump flag before it is built
// with -ast-dump, otherwise, the build fails with an error about deleting a file
// that doesn't exist.
if err := exec.Run(ninjaCmd); err != nil {
glog.Errorf("Failed ninja: %s", err)
return nil, err
}
// Quotes are NOT needed around the params. Doing so actually causes
// a failure that "-Xclang -ast-dump -fsyntax-only" is an unused argument.
gypCmd.Env = append(gypCmd.Env, `CXXFLAGS=-Xclang -ast-dump -fsyntax-only`)
// run gyp again to remake build files with ast-dump flags
if err := exec.Run(gypCmd); err != nil {
glog.Errorf("Failed gyp message: %s", err)
return nil, err
}
// Run ninja again, which will dump the ast to std out
var ast bytes.Buffer
var stdErr bytes.Buffer
ninjaCmd = &exec.Command{
Name: ninjaPath,
Args: []string{"-C", buildLocation},
LogStdout: false,
LogStderr: false,
Stdout: &ast,
Stderr: &stdErr,
Dir: config.FrontEnd.SkiaRoot,
Env: []string{
fmt.Sprintf("CC=%s", config.Common.ClangPath),
fmt.Sprintf("CXX=%s", config.Common.ClangPlusPlusPath),
},
InheritPath: true,
}
glog.Info("Generating AST")
if err := exec.Run(ninjaCmd); err != nil {
return nil, fmt.Errorf("Error generating AST %s:\nstderr: %s", err, stdErr.String())
}
glog.Info("Done generating AST")
return ast.Bytes(), nil
}
// buildSkiaAST returns a ~13GB dump of all ASTs created when building Skia.
// It builds a release version of Skia using Clang, once without the -ast-dump flag
// and once with. The output of the latter is returned.
func buildSkiaAST() ([]byte, error) {
// both gyp_skia and ninja need the environment variables set, so it is easier
// to just set them with os.Setenv rather than using the Command's Env: variables
if err := os.Setenv("CC", config.Generator.ClangPath); err != nil {
return nil, err
}
if err := os.Setenv("CXX", config.Generator.ClangPlusPlusPath); err != nil {
return nil, err
}
// clean previous build
buildLocation := filepath.Join("out", "Release")
if err := os.RemoveAll(filepath.Join(config.Generator.SkiaRoot, buildLocation)); err != nil {
return nil, err
}
gypCmd := &exec.Command{
Name: "./gyp_skia",
Dir: config.Generator.SkiaRoot,
}
// run gyp
if err := exec.Run(gypCmd); err != nil {
glog.Errorf("Failed gyp: %s", err)
return nil, err
}
ninjaCmd := &exec.Command{
Name: "ninja",
Args: []string{"-C", buildLocation},
LogStdout: false,
LogStderr: false,
Dir: config.Generator.SkiaRoot,
}
// first build
// Skia needs to be built once without the -ast-dump flag before it is built
// with -ast-dump, otherwise, the build fails with an error about deleting a file
// that doesn't exist.
if err := exec.Run(ninjaCmd); err != nil {
glog.Errorf("Failed ninja: %s", err)
return nil, err
}
if err := os.Setenv("CXXFLAGS", "-Xclang -ast-dump -fsyntax-only"); err != nil {
return nil, err
}
// run gyp again to remake build files with ast-dump flags
if err := exec.Run(gypCmd); err != nil {
glog.Errorf("Failed gyp message: %s", err)
return nil, err
}
// Run ninja again, which will dump the ast to std out
var ast bytes.Buffer
var stdErr bytes.Buffer
ninjaCmd = &exec.Command{
Name: "ninja",
Args: []string{"-C", buildLocation},
LogStdout: false,
LogStderr: false,
Stdout: &ast,
Stderr: &stdErr,
Dir: config.Generator.SkiaRoot,
}
if err := exec.Run(ninjaCmd); err != nil {
return nil, fmt.Errorf("Error generating AST %s:\nstderr: %s", err, stdErr.String())
}
glog.Info("Done parsing ast")
return ast.Bytes(), nil
}