// buildDM builds the test harness for parsing skp (and other) files.
// First it creates a hard link for the gyp and cpp files. The gyp file is linked into Skia's gyp folder and the cpp file is linked into SKIA_ROOT/../fuzzer_cache/src, which is where the gyp file is configured to point.
// Then it activates Skia's gyp command, which creates the build (ninja) files.
// Finally, it runs those build files.
// If any step fails in unexpected ways, it returns an error.
func buildDM(buildType string, isClean bool) error {
glog.Infof("Building %s dm", buildType)
// Change directory to skia folder
if err := os.Chdir(config.Generator.SkiaRoot); err != nil {
return err
}
// clean previous build if specified
buildLocation := filepath.Join("out", buildType)
if isClean {
if err := os.RemoveAll(buildLocation); err != nil {
return err
}
}
// run gyp
if message, err := exec.RunSimple("./gyp_skia"); err != nil {
glog.Errorf("Failed gyp message: %s", message)
return err
}
// run ninja
cmd := fmt.Sprintf("ninja -C %s dm", buildLocation)
if message, err := exec.RunSimple(cmd); err != nil {
glog.Errorf("Failed ninja message: %s", message)
return err
}
return nil
}
func main() {
output, err := exec.RunSimple("gcloud compute project-info describe --format=json")
if err != nil {
glog.Fatalf("Failed to execute gcloud: %s", err)
}
pi := &ProjectInfo{
CommonInstanceMetadata: Metadata{
Items: []*Item{},
},
}
if err := json.Unmarshal([]byte(output), pi); err != nil {
glog.Fatalf("Failed to parse gcloud output: %s", err)
}
items := pi.CommonInstanceMetadata.Items
body := ""
for _, item := range items {
if item.Key == metadata.JWT_SERVICE_ACCOUNT {
body = item.Value
break
}
}
if body == "" {
glog.Fatalf("Failed to find the JST service account in the metadata.")
}
if err := ioutil.WriteFile(OUTPUT_FILENAME, []byte(body), 0600); err != nil {
glog.Fatalf("Failed to write %q: %s", OUTPUT_FILENAME, err)
}
}
// installDependencies installs all dependencies that are named in the
// 'Depends' field of the control file via apt-get.
func installDependencies(packageFileName string) error {
dependencies, err := getDependencies(packageFileName)
if err != nil {
return fmt.Errorf("Error getting dependencies for %s: %s", packageFileName, err)
}
if dependencies != "" {
if output, err := iexec.RunSimple("sudo apt-get update"); err != nil {
return fmt.Errorf("Unable to update package cache Got error %s\n\n and output: %s\n\n", err, output)
}
glog.Infof("Installing via apt-get: %s", dependencies)
if output, err := iexec.RunSimple(fmt.Sprintf("sudo apt-get -y install %s", dependencies)); err != nil {
return fmt.Errorf("Unable to install dependencies for %s. Got error: \n %s \n\n and output:\n\n%s", packageFileName, err, output)
}
} else {
glog.Infof("No deps found.")
}
return nil
}
func runFuzz(hash string) error {
cacheDir := config.Config.Fuzzer.CachePath
skiaDir := config.Config.Fuzzer.SkiaSourceDir
err := os.Chdir(skiaDir)
if err != nil {
glog.Fatalf("Couldn't change to the skia dir %s: %s", skiaDir, err)
}
gypFilename := fmt.Sprintf("%s.gyp", hash)
glog.Infof("Moving %s to %s", filepath.Join(cacheDir, gypFilename), filepath.Join(skiaDir, "gyp", gypFilename))
outPath := filepath.Join(skiaDir, "gyp", gypFilename)
err = os.Rename(filepath.Join(cacheDir, gypFilename), outPath)
if err != nil {
glog.Fatalf("Couldn't copy the generated gyp file to %s: %s", outPath, err)
}
glog.Infof("Running gyp for %s...", hash)
cmd := fmt.Sprintf("./gyp_skia gyp/%s.gyp gyp/most.gyp -Dskia_mesa=1", hash)
message, err := exec.RunSimple(cmd)
if err != nil {
glog.Fatalf("Couldn't run gyp: %s", err)
}
glog.Infof("Running ninja for %s...", hash)
cmd = fmt.Sprintf("ninja -C %s/out/Release_Developer %s", skiaDir, hash)
message, err = exec.RunSimple(cmd)
if err != nil {
glog.Fatalf("Couldn't run ninja: %s", err)
}
cmd = fmt.Sprintf("%s/out/Release_Developer/%s --out %s/%s", skiaDir, hash, cacheDir, hash)
message, err = exec.RunSimple(cmd)
glog.Infof(message)
return err
}
// generateStats runs the statistics generation script, parses its output, and
// reports the data into InfluxDB.
//
// The script produces data in this format:
// {
// "Nexus_5X": {
// "002e3da61560d3d4": {
// "battery": {
// "ac": 0,
// "health": 2,
// "level": 100,
// "max": 500000,
// "present": 1,
// "status": 5,
// "temp": 282,
// "usb": 1,
// "voltage": 4311,
// "wireless": 0
// },
// "temperature": 28.0
// }
// }
// }
//
func generateStats() error {
output, err := exec.RunSimple(*statsScript)
if err != nil {
return err
}
res := map[string]interface{}{}
if err := json.Unmarshal([]byte(output), &res); err != nil {
return err
}
parseAndReportStats("androidstats", res)
return nil
}
// getDependencies returns the value of the 'Depends' field in the control
// file of the given package.
func getDependencies(packageName string) (string, error) {
const DEPENDS_PREFIX = "Depends:"
output, err := iexec.RunSimple(fmt.Sprintf("dpkg -I %s", packageName))
if err != nil {
return "", err
}
glog.Infof("Got output for %s :\n\n%s", packageName, output)
buf := bytes.NewBuffer([]byte(output))
scanner := bufio.NewScanner(buf)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if strings.HasPrefix(line, DEPENDS_PREFIX) {
return strings.TrimSpace(strings.TrimPrefix(line, DEPENDS_PREFIX)), nil
}
}
return "", nil
}
// Clone creates a new GitInfo by running "git clone" in the given directory.
func Clone(repoUrl, dir string, allBranches bool) (*GitInfo, error) {
if _, err := exec.RunSimple(fmt.Sprintf("git clone %s %s", repoUrl, dir)); err != nil {
return nil, fmt.Errorf("Failed to clone %s into %s: %s", repoUrl, dir, err)
}
return NewGitInfo(dir, false, allBranches)
}
// mainHandler handles the GET and POST of the main page.
func mainHandler(w http.ResponseWriter, r *http.Request) {
glog.Infof("Main Handler: %q\n", r.URL.Path)
requestsCounter.Inc(1)
if r.Method == "GET" {
rasterURL := ""
pdfURL := ""
gpuURL := ""
rasterMod := time.Now()
gpuMod := time.Now()
pdfMod := time.Now()
code := DEFAULT_SAMPLE
source := 0
width := 256
height := 256
match := directLink.FindStringSubmatch(r.URL.Path)
var hash string
if len(match) == 2 && r.URL.Path != "/" {
hash = match[1]
if db == nil {
http.NotFound(w, r)
return
}
// Update 'code' with the code found in the database.
if err := db.QueryRow("SELECT code, width, height, source_image_id FROM webtry WHERE hash=?", hash).Scan(&code, &width, &height, &source); err != nil {
http.NotFound(w, r)
return
}
rasterURL, gpuURL, pdfURL, rasterMod, gpuMod, pdfMod = getOutputURLS(hash)
}
// Expand the template.
w.Header().Set("Content-Type", "text/html")
context := userCode{
Code: code,
PDFURL: pdfURL,
RasterURL: rasterURL,
GPUURL: gpuURL,
PDFMod: pdfMod.Format(TIME_LAYOUT),
RasterMod: rasterMod.Format(TIME_LAYOUT),
GPUMod: gpuMod.Format(TIME_LAYOUT),
BugURL: makeBugURL(hash),
Hash: hash,
Source: source,
Embedded: false,
Width: width,
Height: height,
Titlebar: Titlebar{GitHash: gitHash, GitInfo: gitInfo},
}
if err := indexTemplate.Execute(w, context); err != nil {
glog.Errorf("Failed to expand template: %q\n", err)
}
} else if r.Method == "POST" {
w.Header().Set("Content-Type", "application/json")
buf := bytes.NewBuffer(make([]byte, 0, MAX_TRY_SIZE))
n, err := buf.ReadFrom(r.Body)
if err != nil {
reportTryError(w, r, err, "Failed to read a request body.", "")
return
}
if n == MAX_TRY_SIZE {
err := fmt.Errorf("Code length equal to, or exceeded, %d", MAX_TRY_SIZE)
reportTryError(w, r, err, "Code too large.", "")
return
}
request := TryRequest{}
if err := json.Unmarshal(buf.Bytes(), &request); err != nil {
reportTryError(w, r, err, "Coulnd't decode JSON.", "")
return
}
if !(request.GPU || request.Raster || request.PDF) {
reportTryError(w, r, nil, "No run configuration supplied...", "")
return
}
if hasPreProcessor(request.Code) {
err := fmt.Errorf("Found preprocessor macro in code.")
reportTryError(w, r, err, "Preprocessor macros aren't allowed.", "")
return
}
hash, err := expandCode(LineNumbers(request.Code), request.Source, request.Width, request.Height)
if err != nil {
reportTryError(w, r, err, "Failed to write the code to compile.", hash)
return
}
writeToDatabase(hash, request.Code, request.Name, request.Source, request.Width, request.Height)
err = expandGyp(hash)
if err != nil {
reportTryError(w, r, err, "Failed to write the gyp file.", hash)
return
}
cmd := fmt.Sprintf("scripts/fiddle_wrapper %s --width %d --height %d", hash, request.Width, request.Height)
if request.Raster {
cmd += " --raster"
}
if request.GPU {
cmd += " --gpu"
}
if request.PDF {
cmd += " --pdf"
}
if config.Fiddle.UseChroot {
cmd = "schroot -c webtry --directory=/ -- /skia_build/" + cmd
}
if request.Source > 0 {
cmd += fmt.Sprintf(" --source image-%d.png", request.Source)
}
message, err := exec.RunSimple(cmd)
buildAndRunOutput := strings.SplitN(message, "-=-=-=-=-=-=-", 2)
outputLines := strings.Split(buildAndRunOutput[0], "\n")
errorLines := []compileError{}
for _, line := range outputLines {
match := errorRE.FindStringSubmatch(line)
if len(match) > 0 {
lineNumber, parseError := strconv.Atoi(match[1])
if parseError != nil {
glog.Errorf("ERROR: Couldn't parse line number from %s\n", match[1])
continue
}
columnNumber, parseError := strconv.Atoi(match[2])
if parseError != nil {
glog.Errorf("ERROR: Couldn't parse column number from %s\n", match[2])
continue
}
errorLines = append(errorLines,
compileError{
Line: lineNumber,
Column: columnNumber,
Error: match[3],
})
}
}
if err != nil {
if len(errorLines) > 0 {
reportCompileError(w, r, errorLines, hash)
} else {
reportTryError(w, r, err, "Failed to run the code:\n"+message, hash)
}
return
}
var outputImages OutputImages
if err = json.NewDecoder(strings.NewReader(buildAndRunOutput[1])).Decode(&outputImages); err != nil {
reportTryError(w, r, err, "Failed to decode the fiddle output.", hash)
return
}
err = writeOutputImages(hash, outputImages)
if err != nil {
reportTryError(w, r, err, "Couldn't write the generated images to disk.", hash)
return
}
m := response{
Hash: hash,
BugURL: makeBugURL(hash),
}
rasterImg, gpuImg, pdfURL, rasterMod, gpuMod, pdfMod, err, errMsg := getOutputData(hash)
if err != nil {
reportTryError(w, r, err, errMsg, hash)
return
}
m.RasterImg = rasterImg
m.GPUImg = gpuImg
m.PDFURL = pdfURL
m.RasterMod = rasterMod.Format(TIME_LAYOUT)
m.GPUMod = gpuMod.Format(TIME_LAYOUT)
m.PDFMod = pdfMod.Format(TIME_LAYOUT)
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(m); err != nil {
reportTryError(w, r, err, "Failed to serialize a response.", hash)
return
}
}
}
func Init() {
defer common.LogPanic()
rand.Seed(time.Now().UnixNano())
config.Fiddle.UseChroot = false
config.Fiddle.Port = ":8000"
config.Fiddle.ResourcePath = ""
config.Fiddle.CachePath = "../../cache"
config.Fiddle.InoutPath = "../../inout"
config.Fiddle.UseMetadata = true
common.DecodeTomlFile(*configFilename, &config.Fiddle)
path, err := filepath.Abs(config.Fiddle.ResourcePath)
if err != nil {
glog.Fatal(err)
}
if err := os.Chdir(path); err != nil {
glog.Fatal(err)
}
codeTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.cpp")))
gypTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.gyp")))
indexTemplate = htemplate.Must(htemplate.ParseFiles(
filepath.Join(path, "templates/index.html"),
filepath.Join(path, "templates/titlebar.html"),
filepath.Join(path, "templates/content.html"),
filepath.Join(path, "templates/header.html"),
filepath.Join(path, "templates/footer.html"),
))
iframeTemplate = htemplate.Must(htemplate.ParseFiles(
filepath.Join(path, "templates/iframe.html"),
filepath.Join(path, "templates/content.html"),
filepath.Join(path, "templates/header.html"),
filepath.Join(path, "templates/footer.html"),
))
recentTemplate = htemplate.Must(htemplate.ParseFiles(
filepath.Join(path, "templates/recent.html"),
filepath.Join(path, "templates/titlebar.html"),
filepath.Join(path, "templates/header.html"),
filepath.Join(path, "templates/footer.html"),
))
workspaceTemplate = htemplate.Must(htemplate.ParseFiles(
filepath.Join(path, "templates/workspace.html"),
filepath.Join(path, "templates/titlebar.html"),
filepath.Join(path, "templates/content.html"),
filepath.Join(path, "templates/header.html"),
filepath.Join(path, "templates/footer.html"),
))
// The git command returns output of the format:
//
// f672cead70404080a991ebfb86c38316a4589b23 2014-04-27 19:21:51 +0000
//
logOutput, err := exec.RunSimple(`git log --format=%H%x20%ai HEAD^..HEAD`)
if err != nil {
panic(err)
}
logInfo := strings.Split(logOutput, " ")
gitHash = logInfo[0]
gitInfo = logInfo[1] + " " + logInfo[2] + " " + logInfo[0][0:6]
if config.Fiddle.UseMetadata {
password := metadata.MustGet(PASSWORD_METADATA_KEY)
// The IP address of the database is found here:
// https://console.developers.google.com/project/31977622648/sql/instances/webtry/overview
// And 3306 is the default port for MySQL.
db, err = sql.Open("mysql", fmt.Sprintf("webtry:%s@tcp(173.194.83.52:3306)/webtry?parseTime=true", password))
if err != nil {
glog.Fatalf("ERROR: Failed to open connection to SQL server: %q\n", err)
}
} else {
// Fallback to sqlite for local use.
db, err = sql.Open("sqlite3", "./webtry.db")
if err != nil {
glog.Errorf("Failed to open: %q\n", err)
panic(err)
}
sql := `CREATE TABLE IF NOT EXISTS source_images (
id INTEGER PRIMARY KEY NOT NULL,
image MEDIUMBLOB DEFAULT '' NOT NULL, -- formatted as a PNG.
width INTEGER DEFAULT 0 NOT NULL,
height INTEGER DEFAULT 0 NOT NULL,
create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
hidden INTEGER DEFAULT 0 NOT NULL
)`
_, err = db.Exec(sql)
if err != nil {
glog.Errorf("Creating source_images table failed: %s", err)
}
sql = `CREATE TABLE IF NOT EXISTS webtry (
code TEXT DEFAULT '' NOT NULL,
create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
hash CHAR(64) DEFAULT '' NOT NULL,
width INTEGER DEFAULT 256 NOT NULL,
height INTEGER DEFAULT 256 NOT NULL,
source_image_id INTEGER DEFAULT 0 NOT NULL,
PRIMARY KEY(hash)
)`
_, err = db.Exec(sql)
if err != nil {
glog.Errorf("Creating webtry table failed: %s", err)
}
sql = `CREATE TABLE IF NOT EXISTS workspace (
name CHAR(64) DEFAULT '' NOT NULL,
create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
PRIMARY KEY(name)
)`
_, err = db.Exec(sql)
if err != nil {
glog.Errorf("Creating workspace table failed: %s", err)
}
sql = `CREATE TABLE IF NOT EXISTS workspacetry (
name CHAR(64) DEFAULT '' NOT NULL,
create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
hash CHAR(64) DEFAULT '' NOT NULL,
width INTEGER DEFAULT 256 NOT NULL,
height INTEGER DEFAULT 256 NOT NULL,
hidden INTEGER DEFAULT 0 NOT NULL,
source_image_id INTEGER DEFAULT 0 NOT NULL,
FOREIGN KEY (name) REFERENCES workspace(name)
)`
_, err = db.Exec(sql)
if err != nil {
glog.Errorf("Creating workspacetry table failed: %s", err)
}
}
// Ping the database to keep the connection fresh.
go func() {
c := time.Tick(1 * time.Minute)
for _ = range c {
if err := db.Ping(); err != nil {
glog.Errorf("Database failed to respond: %q\n", err)
}
}
}()
common.InitWithMetrics("webtry", graphiteServer)
writeOutAllSourceImages()
}
