Example #1
0
// getBinaryReportsFromGS pulls all files in baseFolder from the skia-fuzzer bucket and
// groups them by fuzz.  It parses these groups of files into a FuzzReportBinary and returns
// the slice of all reports generated in this way.
func getBinaryReportsFromGS(storageService *storage.Service, baseFolder string) ([]fuzz.FuzzReportBinary, error) {
	contents, err := storageService.Objects.List(config.Aggregator.Bucket).Prefix(baseFolder).Fields("nextPageToken", "items(name,size,timeCreated)").MaxResults(100000).Do()
	// Assumption, files are sorted alphabetically and have the structure
	// [baseFolder]/[filetype]/[fuzzname]/[fuzzname][suffix]
	// where suffix is one of _debug.dump, _debug.err, _release.dump or _release.err
	if err != nil {
		return nil, fmt.Errorf("Problem reading from Google Storage: %v", err)
	}

	glog.Infof("Loading %d files from gs://%s/%s", len(contents.Items), config.Aggregator.Bucket, baseFolder)

	reports := make([]fuzz.FuzzReportBinary, 0)

	var debugDump, debugErr, releaseDump, releaseErr string
	isInitialized := false
	currFuzzFolder := "" // will be something like binary_fuzzes/bad/skp/badbeef
	currFuzzName := ""
	currFuzzType := ""
	for _, item := range contents.Items {
		name := item.Name
		if strings.Count(name, "/") <= 3 {
			continue
		}

		if !isInitialized || !strings.HasPrefix(name, currFuzzFolder) {
			if isInitialized {
				reports = append(reports, fuzz.ParseBinaryReport(currFuzzType, currFuzzName, debugDump, debugErr, releaseDump, releaseErr))
			} else {
				isInitialized = true
			}

			parts := strings.Split(name, "/")
			currFuzzFolder = strings.Join(parts[0:4], "/")
			currFuzzType = parts[2]
			currFuzzName = parts[3]
			// reset for next one
			debugDump, debugErr, releaseDump, releaseErr = "", "", "", ""

		}
		if strings.HasSuffix(name, "_debug.dump") {
			debugDump = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name))
		} else if strings.HasSuffix(name, "_debug.err") {
			debugErr = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name))
		} else if strings.HasSuffix(name, "_release.dump") {
			releaseDump = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name))
		} else if strings.HasSuffix(name, "_release.err") {
			releaseErr = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name))
		}
	}

	if currFuzzName != "" {
		reports = append(reports, fuzz.ParseBinaryReport(currFuzzType, currFuzzName, debugDump, debugErr, releaseDump, releaseErr))
	}
	glog.Info("Done loading")
	return reports, nil
}
Example #2
0
// download waits for fuzzPackages to appear on the toDownload channel and then downloads
// the four pieces of the package.  It then parses them into a BinaryFuzzReport and sends
// the binary to the passed in channel.  When there is no more work to be done, this function.
// returns and writes out true to the done channel.
func (g *GSLoader) download(toDownload <-chan fuzzPackage, reports chan<- fuzz.BinaryFuzzReport, wg *sync.WaitGroup) {
	defer wg.Done()
	for job := range toDownload {
		debugDump := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.DebugDumpName))
		debugErr := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.DebugErrName))
		releaseDump := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.ReleaseDumpName))
		releaseErr := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.ReleaseErrName))
		reports <- fuzz.ParseBinaryReport(job.FuzzType, job.FuzzName, debugDump, debugErr, releaseDump, releaseErr)
		atomic.AddInt32(&g.completedCounter, 1)
		if g.completedCounter%100 == 0 {
			glog.Infof("%d fuzzes downloaded", g.completedCounter)
		}
	}
}
// DownloadBinarySeedFiles downloads the seed skp files stored in Google
// Storage to be used by afl-fuzz.  It places them in
// config.Generator.FuzzSamples after cleaning the folder out.
// It returns an error on failure.
func DownloadBinarySeedFiles(storageClient *storage.Client) error {
	if err := os.RemoveAll(config.Generator.FuzzSamples); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Could not clean binary seed path %s: %s", config.Generator.FuzzSamples, err)
	}
	if err := os.MkdirAll(config.Generator.FuzzSamples, 0755); err != nil {
		return fmt.Errorf("Could not create binary seed path %s: %s", config.Generator.FuzzSamples, err)
	}

	err := gs.AllFilesInDir(storageClient, config.GS.Bucket, "skp_samples", func(item *storage.ObjectAttrs) {
		name := item.Name
		// skip the parent folder
		if name == "skp_samples/" {
			return
		}
		content, err := gs.FileContentsFromGS(storageClient, config.GS.Bucket, name)
		if err != nil {
			glog.Errorf("Problem downloading %s from Google Storage, continuing anyway", item.Name)
			return
		}
		fileName := filepath.Join(config.Generator.FuzzSamples, strings.SplitAfter(name, "skp_samples/")[1])
		if err = ioutil.WriteFile(fileName, content, 0644); err != nil && !os.IsExist(err) {
			glog.Errorf("Problem creating binary seed file %s, continuing anyway", fileName)
		}
	})
	return err
}
// download starts a go routine that waits for files to download from Google Storage
// and downloads them to downloadPath.  When it is done (on error or when the channel
// is closed), it signals to the WaitGroup that it is done.
// It also logs the progress on downloading the fuzzes
func (v *VersionUpdater) download(toDownload <-chan string, downloadPath string, wg *sync.WaitGroup) {
	defer wg.Done()
	for file := range toDownload {
		contents, err := gs.FileContentsFromGS(v.storageClient, config.GS.Bucket, file)
		if err != nil {
			glog.Warningf("Problem downloading fuzz %s, continuing anyway: %s", file, err)
		}
		hash := file[strings.LastIndex(file, "/")+1:]
		onDisk := filepath.Join(downloadPath, hash)
		if err = ioutil.WriteFile(onDisk, contents, 0644); err != nil && !os.IsExist(err) {
			glog.Warningf("Problem writing fuzz to %s, continuing anyway: %s", onDisk, err)
		}
		atomic.AddInt32(&completedCounter, 1)
		if completedCounter%100 == 0 {
			glog.Infof("%d fuzzes downloaded", completedCounter)
		}
	}
}
Example #5
0
func fuzzHandler(w http.ResponseWriter, r *http.Request) {
	v := mux.Vars(r)
	kind := v["kind"]
	name := v["name"]
	xs := strings.Split(name, ".")
	hash, ftype := xs[0], xs[1]
	contents, err := gs.FileContentsFromGS(storageClient, config.GS.Bucket, fmt.Sprintf("%s_fuzzes/%s/bad/%s/%s/%s", kind, config.FrontEnd.SkiaVersion.Hash, ftype, hash, hash))
	if err != nil {
		util.ReportError(w, r, err, fmt.Sprintf("Fuzz with name %v not found", v["name"]))
		return
	}
	w.Header().Set("Content-Type", "application/octet-stream")
	w.Header().Set("Content-Disposition", name)
	n, err := w.Write(contents)
	if err != nil || n != len(contents) {
		glog.Errorf("Could only serve %d bytes of fuzz %s, not %d: %s", n, hash, len(contents), err)
		return
	}
}
Example #6
0
func metadataHandler(w http.ResponseWriter, r *http.Request) {
	v := mux.Vars(r)
	ftype := v["type"]
	kind := v["kind"]
	name := v["name"]
	hash := strings.Split(name, "_")[0]

	contents, err := gs.FileContentsFromGS(storageClient, config.GS.Bucket, fmt.Sprintf("%s_fuzzes/%s/bad/%s/%s/%s", kind, config.FrontEnd.SkiaVersion.Hash, ftype, hash, name))
	if err != nil {
		util.ReportError(w, r, err, fmt.Sprintf("Fuzz with name %v not found", v["name"]))
		return
	}
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.Header().Set("Content-Disposition", name)
	n, err := w.Write(contents)
	if err != nil || n != len(contents) {
		glog.Errorf("Could only serve %d bytes of metadata %s, not %d: %s", n, name, len(contents), err)
		return
	}
}