// buildDM builds the test harness for parsing skp (and other) files. // First it creates a hard link for the gyp and cpp files. The gyp file is linked into Skia's gyp folder and the cpp file is linked into SKIA_ROOT/../fuzzer_cache/src, which is where the gyp file is configured to point. // Then it activates Skia's gyp command, which creates the build (ninja) files. // Finally, it runs those build files. // If any step fails in unexpected ways, it returns an error. func buildDM(buildType string, isClean bool) error { glog.Infof("Building %s dm", buildType) // Change directory to skia folder if err := os.Chdir(config.Generator.SkiaRoot); err != nil { return err } // clean previous build if specified buildLocation := filepath.Join("out", buildType) if isClean { if err := os.RemoveAll(buildLocation); err != nil { return err } } // run gyp if message, err := exec.RunSimple("./gyp_skia"); err != nil { glog.Errorf("Failed gyp message: %s", message) return err } // run ninja cmd := fmt.Sprintf("ninja -C %s dm", buildLocation) if message, err := exec.RunSimple(cmd); err != nil { glog.Errorf("Failed ninja message: %s", message) return err } return nil }
func main() { output, err := exec.RunSimple("gcloud compute project-info describe --format=json") if err != nil { glog.Fatalf("Failed to execute gcloud: %s", err) } pi := &ProjectInfo{ CommonInstanceMetadata: Metadata{ Items: []*Item{}, }, } if err := json.Unmarshal([]byte(output), pi); err != nil { glog.Fatalf("Failed to parse gcloud output: %s", err) } items := pi.CommonInstanceMetadata.Items body := "" for _, item := range items { if item.Key == metadata.JWT_SERVICE_ACCOUNT { body = item.Value break } } if body == "" { glog.Fatalf("Failed to find the JST service account in the metadata.") } if err := ioutil.WriteFile(OUTPUT_FILENAME, []byte(body), 0600); err != nil { glog.Fatalf("Failed to write %q: %s", OUTPUT_FILENAME, err) } }
// installDependencies installs all dependencies that are named in the // 'Depends' field of the control file via apt-get. func installDependencies(packageFileName string) error { dependencies, err := getDependencies(packageFileName) if err != nil { return fmt.Errorf("Error getting dependencies for %s: %s", packageFileName, err) } if dependencies != "" { if output, err := iexec.RunSimple("sudo apt-get update"); err != nil { return fmt.Errorf("Unable to update package cache Got error %s\n\n and output: %s\n\n", err, output) } glog.Infof("Installing via apt-get: %s", dependencies) if output, err := iexec.RunSimple(fmt.Sprintf("sudo apt-get -y install %s", dependencies)); err != nil { return fmt.Errorf("Unable to install dependencies for %s. Got error: \n %s \n\n and output:\n\n%s", packageFileName, err, output) } } else { glog.Infof("No deps found.") } return nil }
func runFuzz(hash string) error { cacheDir := config.Config.Fuzzer.CachePath skiaDir := config.Config.Fuzzer.SkiaSourceDir err := os.Chdir(skiaDir) if err != nil { glog.Fatalf("Couldn't change to the skia dir %s: %s", skiaDir, err) } gypFilename := fmt.Sprintf("%s.gyp", hash) glog.Infof("Moving %s to %s", filepath.Join(cacheDir, gypFilename), filepath.Join(skiaDir, "gyp", gypFilename)) outPath := filepath.Join(skiaDir, "gyp", gypFilename) err = os.Rename(filepath.Join(cacheDir, gypFilename), outPath) if err != nil { glog.Fatalf("Couldn't copy the generated gyp file to %s: %s", outPath, err) } glog.Infof("Running gyp for %s...", hash) cmd := fmt.Sprintf("./gyp_skia gyp/%s.gyp gyp/most.gyp -Dskia_mesa=1", hash) message, err := exec.RunSimple(cmd) if err != nil { glog.Fatalf("Couldn't run gyp: %s", err) } glog.Infof("Running ninja for %s...", hash) cmd = fmt.Sprintf("ninja -C %s/out/Release_Developer %s", skiaDir, hash) message, err = exec.RunSimple(cmd) if err != nil { glog.Fatalf("Couldn't run ninja: %s", err) } cmd = fmt.Sprintf("%s/out/Release_Developer/%s --out %s/%s", skiaDir, hash, cacheDir, hash) message, err = exec.RunSimple(cmd) glog.Infof(message) return err }
// generateStats runs the statistics generation script, parses its output, and // reports the data into InfluxDB. // // The script produces data in this format: // { // "Nexus_5X": { // "002e3da61560d3d4": { // "battery": { // "ac": 0, // "health": 2, // "level": 100, // "max": 500000, // "present": 1, // "status": 5, // "temp": 282, // "usb": 1, // "voltage": 4311, // "wireless": 0 // }, // "temperature": 28.0 // } // } // } // func generateStats() error { output, err := exec.RunSimple(*statsScript) if err != nil { return err } res := map[string]interface{}{} if err := json.Unmarshal([]byte(output), &res); err != nil { return err } parseAndReportStats("androidstats", res) return nil }
// getDependencies returns the value of the 'Depends' field in the control // file of the given package. func getDependencies(packageName string) (string, error) { const DEPENDS_PREFIX = "Depends:" output, err := iexec.RunSimple(fmt.Sprintf("dpkg -I %s", packageName)) if err != nil { return "", err } glog.Infof("Got output for %s :\n\n%s", packageName, output) buf := bytes.NewBuffer([]byte(output)) scanner := bufio.NewScanner(buf) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if strings.HasPrefix(line, DEPENDS_PREFIX) { return strings.TrimSpace(strings.TrimPrefix(line, DEPENDS_PREFIX)), nil } } return "", nil }
// Clone creates a new GitInfo by running "git clone" in the given directory. func Clone(repoUrl, dir string, allBranches bool) (*GitInfo, error) { if _, err := exec.RunSimple(fmt.Sprintf("git clone %s %s", repoUrl, dir)); err != nil { return nil, fmt.Errorf("Failed to clone %s into %s: %s", repoUrl, dir, err) } return NewGitInfo(dir, false, allBranches) }
// mainHandler handles the GET and POST of the main page. func mainHandler(w http.ResponseWriter, r *http.Request) { glog.Infof("Main Handler: %q\n", r.URL.Path) requestsCounter.Inc(1) if r.Method == "GET" { rasterURL := "" pdfURL := "" gpuURL := "" rasterMod := time.Now() gpuMod := time.Now() pdfMod := time.Now() code := DEFAULT_SAMPLE source := 0 width := 256 height := 256 match := directLink.FindStringSubmatch(r.URL.Path) var hash string if len(match) == 2 && r.URL.Path != "/" { hash = match[1] if db == nil { http.NotFound(w, r) return } // Update 'code' with the code found in the database. if err := db.QueryRow("SELECT code, width, height, source_image_id FROM webtry WHERE hash=?", hash).Scan(&code, &width, &height, &source); err != nil { http.NotFound(w, r) return } rasterURL, gpuURL, pdfURL, rasterMod, gpuMod, pdfMod = getOutputURLS(hash) } // Expand the template. w.Header().Set("Content-Type", "text/html") context := userCode{ Code: code, PDFURL: pdfURL, RasterURL: rasterURL, GPUURL: gpuURL, PDFMod: pdfMod.Format(TIME_LAYOUT), RasterMod: rasterMod.Format(TIME_LAYOUT), GPUMod: gpuMod.Format(TIME_LAYOUT), BugURL: makeBugURL(hash), Hash: hash, Source: source, Embedded: false, Width: width, Height: height, Titlebar: Titlebar{GitHash: gitHash, GitInfo: gitInfo}, } if err := indexTemplate.Execute(w, context); err != nil { glog.Errorf("Failed to expand template: %q\n", err) } } else if r.Method == "POST" { w.Header().Set("Content-Type", "application/json") buf := bytes.NewBuffer(make([]byte, 0, MAX_TRY_SIZE)) n, err := buf.ReadFrom(r.Body) if err != nil { reportTryError(w, r, err, "Failed to read a request body.", "") return } if n == MAX_TRY_SIZE { err := fmt.Errorf("Code length equal to, or exceeded, %d", MAX_TRY_SIZE) reportTryError(w, r, err, "Code too large.", "") return } request := TryRequest{} if err := json.Unmarshal(buf.Bytes(), &request); err != nil { reportTryError(w, r, err, "Coulnd't decode JSON.", "") return } if !(request.GPU || request.Raster || request.PDF) { reportTryError(w, r, nil, "No run configuration supplied...", "") return } if hasPreProcessor(request.Code) { err := fmt.Errorf("Found preprocessor macro in code.") reportTryError(w, r, err, "Preprocessor macros aren't allowed.", "") return } hash, err := expandCode(LineNumbers(request.Code), request.Source, request.Width, request.Height) if err != nil { reportTryError(w, r, err, "Failed to write the code to compile.", hash) return } writeToDatabase(hash, request.Code, request.Name, request.Source, request.Width, request.Height) err = expandGyp(hash) if err != nil { reportTryError(w, r, err, "Failed to write the gyp file.", hash) return } cmd := fmt.Sprintf("scripts/fiddle_wrapper %s --width %d --height %d", hash, request.Width, request.Height) if request.Raster { cmd += " --raster" } if request.GPU { cmd += " --gpu" } if request.PDF { cmd += " --pdf" } if config.Fiddle.UseChroot { cmd = "schroot -c webtry --directory=/ -- /skia_build/" + cmd } if request.Source > 0 { cmd += fmt.Sprintf(" --source image-%d.png", request.Source) } message, err := exec.RunSimple(cmd) buildAndRunOutput := strings.SplitN(message, "-=-=-=-=-=-=-", 2) outputLines := strings.Split(buildAndRunOutput[0], "\n") errorLines := []compileError{} for _, line := range outputLines { match := errorRE.FindStringSubmatch(line) if len(match) > 0 { lineNumber, parseError := strconv.Atoi(match[1]) if parseError != nil { glog.Errorf("ERROR: Couldn't parse line number from %s\n", match[1]) continue } columnNumber, parseError := strconv.Atoi(match[2]) if parseError != nil { glog.Errorf("ERROR: Couldn't parse column number from %s\n", match[2]) continue } errorLines = append(errorLines, compileError{ Line: lineNumber, Column: columnNumber, Error: match[3], }) } } if err != nil { if len(errorLines) > 0 { reportCompileError(w, r, errorLines, hash) } else { reportTryError(w, r, err, "Failed to run the code:\n"+message, hash) } return } var outputImages OutputImages if err = json.NewDecoder(strings.NewReader(buildAndRunOutput[1])).Decode(&outputImages); err != nil { reportTryError(w, r, err, "Failed to decode the fiddle output.", hash) return } err = writeOutputImages(hash, outputImages) if err != nil { reportTryError(w, r, err, "Couldn't write the generated images to disk.", hash) return } m := response{ Hash: hash, BugURL: makeBugURL(hash), } rasterImg, gpuImg, pdfURL, rasterMod, gpuMod, pdfMod, err, errMsg := getOutputData(hash) if err != nil { reportTryError(w, r, err, errMsg, hash) return } m.RasterImg = rasterImg m.GPUImg = gpuImg m.PDFURL = pdfURL m.RasterMod = rasterMod.Format(TIME_LAYOUT) m.GPUMod = gpuMod.Format(TIME_LAYOUT) m.PDFMod = pdfMod.Format(TIME_LAYOUT) w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(m); err != nil { reportTryError(w, r, err, "Failed to serialize a response.", hash) return } } }
func Init() { defer common.LogPanic() rand.Seed(time.Now().UnixNano()) config.Fiddle.UseChroot = false config.Fiddle.Port = ":8000" config.Fiddle.ResourcePath = "" config.Fiddle.CachePath = "../../cache" config.Fiddle.InoutPath = "../../inout" config.Fiddle.UseMetadata = true common.DecodeTomlFile(*configFilename, &config.Fiddle) path, err := filepath.Abs(config.Fiddle.ResourcePath) if err != nil { glog.Fatal(err) } if err := os.Chdir(path); err != nil { glog.Fatal(err) } codeTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.cpp"))) gypTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.gyp"))) indexTemplate = htemplate.Must(htemplate.ParseFiles( filepath.Join(path, "templates/index.html"), filepath.Join(path, "templates/titlebar.html"), filepath.Join(path, "templates/content.html"), filepath.Join(path, "templates/header.html"), filepath.Join(path, "templates/footer.html"), )) iframeTemplate = htemplate.Must(htemplate.ParseFiles( filepath.Join(path, "templates/iframe.html"), filepath.Join(path, "templates/content.html"), filepath.Join(path, "templates/header.html"), filepath.Join(path, "templates/footer.html"), )) recentTemplate = htemplate.Must(htemplate.ParseFiles( filepath.Join(path, "templates/recent.html"), filepath.Join(path, "templates/titlebar.html"), filepath.Join(path, "templates/header.html"), filepath.Join(path, "templates/footer.html"), )) workspaceTemplate = htemplate.Must(htemplate.ParseFiles( filepath.Join(path, "templates/workspace.html"), filepath.Join(path, "templates/titlebar.html"), filepath.Join(path, "templates/content.html"), filepath.Join(path, "templates/header.html"), filepath.Join(path, "templates/footer.html"), )) // The git command returns output of the format: // // f672cead70404080a991ebfb86c38316a4589b23 2014-04-27 19:21:51 +0000 // logOutput, err := exec.RunSimple(`git log --format=%H%x20%ai HEAD^..HEAD`) if err != nil { panic(err) } logInfo := strings.Split(logOutput, " ") gitHash = logInfo[0] gitInfo = logInfo[1] + " " + logInfo[2] + " " + logInfo[0][0:6] if config.Fiddle.UseMetadata { password := metadata.MustGet(PASSWORD_METADATA_KEY) // The IP address of the database is found here: // https://console.developers.google.com/project/31977622648/sql/instances/webtry/overview // And 3306 is the default port for MySQL. db, err = sql.Open("mysql", fmt.Sprintf("webtry:%s@tcp(173.194.83.52:3306)/webtry?parseTime=true", password)) if err != nil { glog.Fatalf("ERROR: Failed to open connection to SQL server: %q\n", err) } } else { // Fallback to sqlite for local use. db, err = sql.Open("sqlite3", "./webtry.db") if err != nil { glog.Errorf("Failed to open: %q\n", err) panic(err) } sql := `CREATE TABLE IF NOT EXISTS source_images ( id INTEGER PRIMARY KEY NOT NULL, image MEDIUMBLOB DEFAULT '' NOT NULL, -- formatted as a PNG. width INTEGER DEFAULT 0 NOT NULL, height INTEGER DEFAULT 0 NOT NULL, create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, hidden INTEGER DEFAULT 0 NOT NULL )` _, err = db.Exec(sql) if err != nil { glog.Errorf("Creating source_images table failed: %s", err) } sql = `CREATE TABLE IF NOT EXISTS webtry ( code TEXT DEFAULT '' NOT NULL, create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, hash CHAR(64) DEFAULT '' NOT NULL, width INTEGER DEFAULT 256 NOT NULL, height INTEGER DEFAULT 256 NOT NULL, source_image_id INTEGER DEFAULT 0 NOT NULL, PRIMARY KEY(hash) )` _, err = db.Exec(sql) if err != nil { glog.Errorf("Creating webtry table failed: %s", err) } sql = `CREATE TABLE IF NOT EXISTS workspace ( name CHAR(64) DEFAULT '' NOT NULL, create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, PRIMARY KEY(name) )` _, err = db.Exec(sql) if err != nil { glog.Errorf("Creating workspace table failed: %s", err) } sql = `CREATE TABLE IF NOT EXISTS workspacetry ( name CHAR(64) DEFAULT '' NOT NULL, create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, hash CHAR(64) DEFAULT '' NOT NULL, width INTEGER DEFAULT 256 NOT NULL, height INTEGER DEFAULT 256 NOT NULL, hidden INTEGER DEFAULT 0 NOT NULL, source_image_id INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY (name) REFERENCES workspace(name) )` _, err = db.Exec(sql) if err != nil { glog.Errorf("Creating workspacetry table failed: %s", err) } } // Ping the database to keep the connection fresh. go func() { c := time.Tick(1 * time.Minute) for _ = range c { if err := db.Ping(); err != nil { glog.Errorf("Database failed to respond: %q\n", err) } } }() common.InitWithMetrics("webtry", graphiteServer) writeOutAllSourceImages() }