コード例 #1
0
ファイル: build_skia.go プロジェクト: 1394/skia-buildbot
// buildDM builds the test harness for parsing skp (and other) files.
// First it creates a hard link for the gyp and cpp files. The gyp file is linked into Skia's gyp folder and the cpp file is linked into SKIA_ROOT/../fuzzer_cache/src, which is where the gyp file is configured to point.
// Then it activates Skia's gyp command, which creates the build (ninja) files.
// Finally, it runs those build files.
// If any step fails in unexpected ways, it returns an error.
func buildDM(buildType string, isClean bool) error {
	glog.Infof("Building %s dm", buildType)

	// Change directory to skia folder
	if err := os.Chdir(config.Generator.SkiaRoot); err != nil {
		return err
	}

	// clean previous build if specified
	buildLocation := filepath.Join("out", buildType)
	if isClean {
		if err := os.RemoveAll(buildLocation); err != nil {
			return err
		}
	}

	// run gyp
	if message, err := exec.RunSimple("./gyp_skia"); err != nil {
		glog.Errorf("Failed gyp message: %s", message)
		return err
	}

	// run ninja
	cmd := fmt.Sprintf("ninja -C %s dm", buildLocation)
	if message, err := exec.RunSimple(cmd); err != nil {
		glog.Errorf("Failed ninja message: %s", message)
		return err
	}
	return nil
}
コード例 #2
0
ファイル: main.go プロジェクト: saltmueller/skia-buildbot
func main() {
	output, err := exec.RunSimple("gcloud compute project-info describe --format=json")
	if err != nil {
		glog.Fatalf("Failed to execute gcloud: %s", err)
	}
	pi := &ProjectInfo{
		CommonInstanceMetadata: Metadata{
			Items: []*Item{},
		},
	}
	if err := json.Unmarshal([]byte(output), pi); err != nil {
		glog.Fatalf("Failed to parse gcloud output: %s", err)
	}
	items := pi.CommonInstanceMetadata.Items
	body := ""
	for _, item := range items {
		if item.Key == metadata.JWT_SERVICE_ACCOUNT {
			body = item.Value
			break
		}
	}
	if body == "" {
		glog.Fatalf("Failed to find the JST service account in the metadata.")
	}
	if err := ioutil.WriteFile(OUTPUT_FILENAME, []byte(body), 0600); err != nil {
		glog.Fatalf("Failed to write %q: %s", OUTPUT_FILENAME, err)
	}
}
コード例 #3
0
ファイル: packages.go プロジェクト: Tiger66639/skia-buildbot
// installDependencies installs all dependencies that are named in the
// 'Depends' field of the control file via apt-get.
func installDependencies(packageFileName string) error {
	dependencies, err := getDependencies(packageFileName)
	if err != nil {
		return fmt.Errorf("Error getting dependencies for %s: %s", packageFileName, err)
	}

	if dependencies != "" {
		if output, err := iexec.RunSimple("sudo apt-get update"); err != nil {
			return fmt.Errorf("Unable to update package cache Got error  %s\n\n and output: %s\n\n", err, output)
		}

		glog.Infof("Installing via apt-get: %s", dependencies)
		if output, err := iexec.RunSimple(fmt.Sprintf("sudo apt-get -y install %s", dependencies)); err != nil {
			return fmt.Errorf("Unable to install dependencies for %s. Got error: \n %s \n\n and output:\n\n%s", packageFileName, err, output)
		}
	} else {
		glog.Infof("No deps found.")
	}
	return nil
}
コード例 #4
0
ファイル: main.go プロジェクト: Tiger66639/skia-buildbot
func runFuzz(hash string) error {
	cacheDir := config.Config.Fuzzer.CachePath
	skiaDir := config.Config.Fuzzer.SkiaSourceDir

	err := os.Chdir(skiaDir)
	if err != nil {
		glog.Fatalf("Couldn't change to the skia dir %s: %s", skiaDir, err)
	}

	gypFilename := fmt.Sprintf("%s.gyp", hash)

	glog.Infof("Moving %s to %s", filepath.Join(cacheDir, gypFilename), filepath.Join(skiaDir, "gyp", gypFilename))
	outPath := filepath.Join(skiaDir, "gyp", gypFilename)
	err = os.Rename(filepath.Join(cacheDir, gypFilename), outPath)
	if err != nil {
		glog.Fatalf("Couldn't copy the generated gyp file to %s: %s", outPath, err)
	}
	glog.Infof("Running gyp for %s...", hash)

	cmd := fmt.Sprintf("./gyp_skia gyp/%s.gyp gyp/most.gyp -Dskia_mesa=1", hash)
	message, err := exec.RunSimple(cmd)
	if err != nil {
		glog.Fatalf("Couldn't run gyp: %s", err)
	}

	glog.Infof("Running ninja for %s...", hash)

	cmd = fmt.Sprintf("ninja -C %s/out/Release_Developer %s", skiaDir, hash)
	message, err = exec.RunSimple(cmd)
	if err != nil {
		glog.Fatalf("Couldn't run ninja: %s", err)
	}

	cmd = fmt.Sprintf("%s/out/Release_Developer/%s --out %s/%s", skiaDir, hash, cacheDir, hash)
	message, err = exec.RunSimple(cmd)

	glog.Infof(message)

	return err
}
コード例 #5
0
ファイル: main.go プロジェクト: saltmueller/skia-buildbot
// generateStats runs the statistics generation script, parses its output, and
// reports the data into InfluxDB.
//
// The script produces data in this format:
// {
//   "Nexus_5X": {
//     "002e3da61560d3d4": {
//       "battery": {
//         "ac": 0,
//         "health": 2,
//         "level": 100,
//         "max": 500000,
//         "present": 1,
//         "status": 5,
//         "temp": 282,
//         "usb": 1,
//         "voltage": 4311,
//         "wireless": 0
//       },
//       "temperature": 28.0
//     }
//   }
// }
//
func generateStats() error {
	output, err := exec.RunSimple(*statsScript)
	if err != nil {
		return err
	}

	res := map[string]interface{}{}
	if err := json.Unmarshal([]byte(output), &res); err != nil {
		return err
	}

	parseAndReportStats("androidstats", res)

	return nil
}
コード例 #6
0
ファイル: packages.go プロジェクト: Tiger66639/skia-buildbot
// getDependencies returns the value of the 'Depends' field in the control
// file of the given package.
func getDependencies(packageName string) (string, error) {
	const DEPENDS_PREFIX = "Depends:"

	output, err := iexec.RunSimple(fmt.Sprintf("dpkg -I %s", packageName))
	if err != nil {
		return "", err
	}
	glog.Infof("Got output for %s :\n\n%s", packageName, output)

	buf := bytes.NewBuffer([]byte(output))
	scanner := bufio.NewScanner(buf)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if strings.HasPrefix(line, DEPENDS_PREFIX) {
			return strings.TrimSpace(strings.TrimPrefix(line, DEPENDS_PREFIX)), nil
		}
	}
	return "", nil
}
コード例 #7
0
ファイル: gitinfo.go プロジェクト: saltmueller/skia-buildbot
// Clone creates a new GitInfo by running "git clone" in the given directory.
func Clone(repoUrl, dir string, allBranches bool) (*GitInfo, error) {
	if _, err := exec.RunSimple(fmt.Sprintf("git clone %s %s", repoUrl, dir)); err != nil {
		return nil, fmt.Errorf("Failed to clone %s into %s: %s", repoUrl, dir, err)
	}
	return NewGitInfo(dir, false, allBranches)
}
コード例 #8
0
ファイル: main.go プロジェクト: saltmueller/skia-buildbot
// mainHandler handles the GET and POST of the main page.
func mainHandler(w http.ResponseWriter, r *http.Request) {
	glog.Infof("Main Handler: %q\n", r.URL.Path)
	requestsCounter.Inc(1)
	if r.Method == "GET" {
		rasterURL := ""
		pdfURL := ""
		gpuURL := ""
		rasterMod := time.Now()
		gpuMod := time.Now()
		pdfMod := time.Now()

		code := DEFAULT_SAMPLE
		source := 0
		width := 256
		height := 256
		match := directLink.FindStringSubmatch(r.URL.Path)

		var hash string
		if len(match) == 2 && r.URL.Path != "/" {
			hash = match[1]
			if db == nil {
				http.NotFound(w, r)
				return
			}
			// Update 'code' with the code found in the database.
			if err := db.QueryRow("SELECT code, width, height, source_image_id FROM webtry WHERE hash=?", hash).Scan(&code, &width, &height, &source); err != nil {
				http.NotFound(w, r)
				return
			}

			rasterURL, gpuURL, pdfURL, rasterMod, gpuMod, pdfMod = getOutputURLS(hash)
		}

		// Expand the template.
		w.Header().Set("Content-Type", "text/html")
		context := userCode{
			Code:      code,
			PDFURL:    pdfURL,
			RasterURL: rasterURL,
			GPUURL:    gpuURL,
			PDFMod:    pdfMod.Format(TIME_LAYOUT),
			RasterMod: rasterMod.Format(TIME_LAYOUT),
			GPUMod:    gpuMod.Format(TIME_LAYOUT),
			BugURL:    makeBugURL(hash),
			Hash:      hash,
			Source:    source,
			Embedded:  false,
			Width:     width,
			Height:    height,
			Titlebar:  Titlebar{GitHash: gitHash, GitInfo: gitInfo},
		}
		if err := indexTemplate.Execute(w, context); err != nil {
			glog.Errorf("Failed to expand template: %q\n", err)
		}
	} else if r.Method == "POST" {
		w.Header().Set("Content-Type", "application/json")
		buf := bytes.NewBuffer(make([]byte, 0, MAX_TRY_SIZE))
		n, err := buf.ReadFrom(r.Body)
		if err != nil {
			reportTryError(w, r, err, "Failed to read a request body.", "")
			return
		}
		if n == MAX_TRY_SIZE {
			err := fmt.Errorf("Code length equal to, or exceeded, %d", MAX_TRY_SIZE)
			reportTryError(w, r, err, "Code too large.", "")
			return
		}
		request := TryRequest{}
		if err := json.Unmarshal(buf.Bytes(), &request); err != nil {
			reportTryError(w, r, err, "Coulnd't decode JSON.", "")
			return
		}
		if !(request.GPU || request.Raster || request.PDF) {
			reportTryError(w, r, nil, "No run configuration supplied...", "")
			return
		}
		if hasPreProcessor(request.Code) {
			err := fmt.Errorf("Found preprocessor macro in code.")
			reportTryError(w, r, err, "Preprocessor macros aren't allowed.", "")
			return
		}
		hash, err := expandCode(LineNumbers(request.Code), request.Source, request.Width, request.Height)
		if err != nil {
			reportTryError(w, r, err, "Failed to write the code to compile.", hash)
			return
		}
		writeToDatabase(hash, request.Code, request.Name, request.Source, request.Width, request.Height)
		err = expandGyp(hash)
		if err != nil {
			reportTryError(w, r, err, "Failed to write the gyp file.", hash)
			return
		}
		cmd := fmt.Sprintf("scripts/fiddle_wrapper %s --width %d --height %d", hash, request.Width, request.Height)
		if request.Raster {
			cmd += " --raster"
		}
		if request.GPU {
			cmd += " --gpu"
		}
		if request.PDF {
			cmd += " --pdf"
		}
		if config.Fiddle.UseChroot {
			cmd = "schroot -c webtry --directory=/ -- /skia_build/" + cmd
		}
		if request.Source > 0 {
			cmd += fmt.Sprintf(" --source image-%d.png", request.Source)
		}

		message, err := exec.RunSimple(cmd)

		buildAndRunOutput := strings.SplitN(message, "-=-=-=-=-=-=-", 2)

		outputLines := strings.Split(buildAndRunOutput[0], "\n")
		errorLines := []compileError{}

		for _, line := range outputLines {
			match := errorRE.FindStringSubmatch(line)
			if len(match) > 0 {
				lineNumber, parseError := strconv.Atoi(match[1])
				if parseError != nil {
					glog.Errorf("ERROR: Couldn't parse line number from %s\n", match[1])
					continue
				}
				columnNumber, parseError := strconv.Atoi(match[2])
				if parseError != nil {
					glog.Errorf("ERROR: Couldn't parse column number from %s\n", match[2])
					continue
				}
				errorLines = append(errorLines,
					compileError{
						Line:   lineNumber,
						Column: columnNumber,
						Error:  match[3],
					})
			}
		}

		if err != nil {
			if len(errorLines) > 0 {
				reportCompileError(w, r, errorLines, hash)
			} else {
				reportTryError(w, r, err, "Failed to run the code:\n"+message, hash)
			}
			return
		}

		var outputImages OutputImages

		if err = json.NewDecoder(strings.NewReader(buildAndRunOutput[1])).Decode(&outputImages); err != nil {
			reportTryError(w, r, err, "Failed to decode the fiddle output.", hash)
			return
		}

		err = writeOutputImages(hash, outputImages)
		if err != nil {
			reportTryError(w, r, err, "Couldn't write the generated images to disk.", hash)
			return
		}

		m := response{
			Hash:   hash,
			BugURL: makeBugURL(hash),
		}

		rasterImg, gpuImg, pdfURL, rasterMod, gpuMod, pdfMod, err, errMsg := getOutputData(hash)
		if err != nil {
			reportTryError(w, r, err, errMsg, hash)
			return
		}

		m.RasterImg = rasterImg
		m.GPUImg = gpuImg
		m.PDFURL = pdfURL
		m.RasterMod = rasterMod.Format(TIME_LAYOUT)
		m.GPUMod = gpuMod.Format(TIME_LAYOUT)
		m.PDFMod = pdfMod.Format(TIME_LAYOUT)

		w.Header().Set("Content-Type", "application/json")
		if err := json.NewEncoder(w).Encode(m); err != nil {
			reportTryError(w, r, err, "Failed to serialize a response.", hash)
			return
		}
	}
}
コード例 #9
0
ファイル: main.go プロジェクト: saltmueller/skia-buildbot
func Init() {
	defer common.LogPanic()
	rand.Seed(time.Now().UnixNano())

	config.Fiddle.UseChroot = false
	config.Fiddle.Port = ":8000"
	config.Fiddle.ResourcePath = ""
	config.Fiddle.CachePath = "../../cache"
	config.Fiddle.InoutPath = "../../inout"
	config.Fiddle.UseMetadata = true

	common.DecodeTomlFile(*configFilename, &config.Fiddle)

	path, err := filepath.Abs(config.Fiddle.ResourcePath)
	if err != nil {
		glog.Fatal(err)
	}
	if err := os.Chdir(path); err != nil {
		glog.Fatal(err)
	}

	codeTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.cpp")))
	gypTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.gyp")))
	indexTemplate = htemplate.Must(htemplate.ParseFiles(
		filepath.Join(path, "templates/index.html"),
		filepath.Join(path, "templates/titlebar.html"),
		filepath.Join(path, "templates/content.html"),
		filepath.Join(path, "templates/header.html"),
		filepath.Join(path, "templates/footer.html"),
	))
	iframeTemplate = htemplate.Must(htemplate.ParseFiles(
		filepath.Join(path, "templates/iframe.html"),
		filepath.Join(path, "templates/content.html"),
		filepath.Join(path, "templates/header.html"),
		filepath.Join(path, "templates/footer.html"),
	))
	recentTemplate = htemplate.Must(htemplate.ParseFiles(
		filepath.Join(path, "templates/recent.html"),
		filepath.Join(path, "templates/titlebar.html"),
		filepath.Join(path, "templates/header.html"),
		filepath.Join(path, "templates/footer.html"),
	))
	workspaceTemplate = htemplate.Must(htemplate.ParseFiles(
		filepath.Join(path, "templates/workspace.html"),
		filepath.Join(path, "templates/titlebar.html"),
		filepath.Join(path, "templates/content.html"),
		filepath.Join(path, "templates/header.html"),
		filepath.Join(path, "templates/footer.html"),
	))

	// The git command returns output of the format:
	//
	//   f672cead70404080a991ebfb86c38316a4589b23 2014-04-27 19:21:51 +0000
	//
	logOutput, err := exec.RunSimple(`git log --format=%H%x20%ai HEAD^..HEAD`)
	if err != nil {
		panic(err)
	}
	logInfo := strings.Split(logOutput, " ")
	gitHash = logInfo[0]
	gitInfo = logInfo[1] + " " + logInfo[2] + " " + logInfo[0][0:6]

	if config.Fiddle.UseMetadata {
		password := metadata.MustGet(PASSWORD_METADATA_KEY)

		// The IP address of the database is found here:
		//    https://console.developers.google.com/project/31977622648/sql/instances/webtry/overview
		// And 3306 is the default port for MySQL.
		db, err = sql.Open("mysql", fmt.Sprintf("webtry:%s@tcp(173.194.83.52:3306)/webtry?parseTime=true", password))
		if err != nil {
			glog.Fatalf("ERROR: Failed to open connection to SQL server: %q\n", err)
		}
	} else {
		// Fallback to sqlite for local use.
		db, err = sql.Open("sqlite3", "./webtry.db")
		if err != nil {
			glog.Errorf("Failed to open: %q\n", err)
			panic(err)
		}
		sql := `CREATE TABLE IF NOT EXISTS source_images (
             id        INTEGER     PRIMARY KEY                NOT NULL,
             image     MEDIUMBLOB  DEFAULT ''                 NOT NULL, -- formatted as a PNG.
             width     INTEGER     DEFAULT 0                  NOT NULL,
             height    INTEGER     DEFAULT 0                  NOT NULL,
             create_ts TIMESTAMP   DEFAULT CURRENT_TIMESTAMP  NOT NULL,
             hidden    INTEGER     DEFAULT 0                  NOT NULL
             )`
		_, err = db.Exec(sql)
		if err != nil {
			glog.Errorf("Creating source_images table failed: %s", err)
		}

		sql = `CREATE TABLE IF NOT EXISTS webtry (
             code               TEXT      DEFAULT ''                 NOT NULL,
             create_ts          TIMESTAMP DEFAULT CURRENT_TIMESTAMP  NOT NULL,
             hash               CHAR(64)  DEFAULT ''                 NOT NULL,
             width              INTEGER   DEFAULT 256                NOT NULL,
             height             INTEGER   DEFAULT 256                NOT NULL,
             source_image_id    INTEGER   DEFAULT 0                  NOT NULL,

             PRIMARY KEY(hash)
            )`
		_, err = db.Exec(sql)
		if err != nil {
			glog.Errorf("Creating webtry table failed: %s", err)
		}

		sql = `CREATE TABLE IF NOT EXISTS workspace (
          name      CHAR(64)  DEFAULT ''                 NOT NULL,
          create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP  NOT NULL,
          PRIMARY KEY(name)
        )`
		_, err = db.Exec(sql)
		if err != nil {
			glog.Errorf("Creating workspace table failed: %s", err)
		}

		sql = `CREATE TABLE IF NOT EXISTS workspacetry (
          name               CHAR(64)  DEFAULT ''                 NOT NULL,
          create_ts          TIMESTAMP DEFAULT CURRENT_TIMESTAMP  NOT NULL,
          hash               CHAR(64)  DEFAULT ''                 NOT NULL,
          width              INTEGER   DEFAULT 256                NOT NULL,
          height             INTEGER   DEFAULT 256                NOT NULL,
          hidden             INTEGER   DEFAULT 0                  NOT NULL,
          source_image_id    INTEGER   DEFAULT 0                  NOT NULL,

          FOREIGN KEY (name)   REFERENCES workspace(name)
        )`
		_, err = db.Exec(sql)
		if err != nil {
			glog.Errorf("Creating workspacetry table failed: %s", err)
		}
	}

	// Ping the database to keep the connection fresh.
	go func() {
		c := time.Tick(1 * time.Minute)
		for _ = range c {
			if err := db.Ping(); err != nil {
				glog.Errorf("Database failed to respond: %q\n", err)
			}
		}
	}()

	common.InitWithMetrics("webtry", graphiteServer)

	writeOutAllSourceImages()
}