// getBinaryReportsFromGS pulls all files in baseFolder from the skia-fuzzer bucket and // groups them by fuzz. It parses these groups of files into a FuzzReportBinary and returns // the slice of all reports generated in this way. func getBinaryReportsFromGS(storageService *storage.Service, baseFolder string) ([]fuzz.FuzzReportBinary, error) { contents, err := storageService.Objects.List(config.Aggregator.Bucket).Prefix(baseFolder).Fields("nextPageToken", "items(name,size,timeCreated)").MaxResults(100000).Do() // Assumption, files are sorted alphabetically and have the structure // [baseFolder]/[filetype]/[fuzzname]/[fuzzname][suffix] // where suffix is one of _debug.dump, _debug.err, _release.dump or _release.err if err != nil { return nil, fmt.Errorf("Problem reading from Google Storage: %v", err) } glog.Infof("Loading %d files from gs://%s/%s", len(contents.Items), config.Aggregator.Bucket, baseFolder) reports := make([]fuzz.FuzzReportBinary, 0) var debugDump, debugErr, releaseDump, releaseErr string isInitialized := false currFuzzFolder := "" // will be something like binary_fuzzes/bad/skp/badbeef currFuzzName := "" currFuzzType := "" for _, item := range contents.Items { name := item.Name if strings.Count(name, "/") <= 3 { continue } if !isInitialized || !strings.HasPrefix(name, currFuzzFolder) { if isInitialized { reports = append(reports, fuzz.ParseBinaryReport(currFuzzType, currFuzzName, debugDump, debugErr, releaseDump, releaseErr)) } else { isInitialized = true } parts := strings.Split(name, "/") currFuzzFolder = strings.Join(parts[0:4], "/") currFuzzType = parts[2] currFuzzName = parts[3] // reset for next one debugDump, debugErr, releaseDump, releaseErr = "", "", "", "" } if strings.HasSuffix(name, "_debug.dump") { debugDump = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name)) } else if strings.HasSuffix(name, "_debug.err") { debugErr = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name)) } else if strings.HasSuffix(name, "_release.dump") { releaseDump = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name)) } else if strings.HasSuffix(name, "_release.err") { releaseErr = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name)) } } if currFuzzName != "" { reports = append(reports, fuzz.ParseBinaryReport(currFuzzType, currFuzzName, debugDump, debugErr, releaseDump, releaseErr)) } glog.Info("Done loading") return reports, nil }
// download waits for fuzzPackages to appear on the toDownload channel and then downloads // the four pieces of the package. It then parses them into a BinaryFuzzReport and sends // the binary to the passed in channel. When there is no more work to be done, this function. // returns and writes out true to the done channel. func (g *GSLoader) download(toDownload <-chan fuzzPackage, reports chan<- fuzz.BinaryFuzzReport, wg *sync.WaitGroup) { defer wg.Done() for job := range toDownload { debugDump := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.DebugDumpName)) debugErr := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.DebugErrName)) releaseDump := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.ReleaseDumpName)) releaseErr := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.ReleaseErrName)) reports <- fuzz.ParseBinaryReport(job.FuzzType, job.FuzzName, debugDump, debugErr, releaseDump, releaseErr) atomic.AddInt32(&g.completedCounter, 1) if g.completedCounter%100 == 0 { glog.Infof("%d fuzzes downloaded", g.completedCounter) } } }