// getBinaryReportsFromGS pulls all files in baseFolder from the skia-fuzzer bucket and
// groups them by fuzz. It parses these groups of files into a FuzzReportBinary and returns
// the slice of all reports generated in this way.
func getBinaryReportsFromGS(storageService *storage.Service, baseFolder string) ([]fuzz.FuzzReportBinary, error) {
contents, err := storageService.Objects.List(config.Aggregator.Bucket).Prefix(baseFolder).Fields("nextPageToken", "items(name,size,timeCreated)").MaxResults(100000).Do()
// Assumption, files are sorted alphabetically and have the structure
// [baseFolder]/[filetype]/[fuzzname]/[fuzzname][suffix]
// where suffix is one of _debug.dump, _debug.err, _release.dump or _release.err
if err != nil {
return nil, fmt.Errorf("Problem reading from Google Storage: %v", err)
}
glog.Infof("Loading %d files from gs://%s/%s", len(contents.Items), config.Aggregator.Bucket, baseFolder)
reports := make([]fuzz.FuzzReportBinary, 0)
var debugDump, debugErr, releaseDump, releaseErr string
isInitialized := false
currFuzzFolder := "" // will be something like binary_fuzzes/bad/skp/badbeef
currFuzzName := ""
currFuzzType := ""
for _, item := range contents.Items {
name := item.Name
if strings.Count(name, "/") <= 3 {
continue
}
if !isInitialized || !strings.HasPrefix(name, currFuzzFolder) {
if isInitialized {
reports = append(reports, fuzz.ParseBinaryReport(currFuzzType, currFuzzName, debugDump, debugErr, releaseDump, releaseErr))
} else {
isInitialized = true
}
parts := strings.Split(name, "/")
currFuzzFolder = strings.Join(parts[0:4], "/")
currFuzzType = parts[2]
currFuzzName = parts[3]
// reset for next one
debugDump, debugErr, releaseDump, releaseErr = "", "", "", ""
}
if strings.HasSuffix(name, "_debug.dump") {
debugDump = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name))
} else if strings.HasSuffix(name, "_debug.err") {
debugErr = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name))
} else if strings.HasSuffix(name, "_release.dump") {
releaseDump = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name))
} else if strings.HasSuffix(name, "_release.err") {
releaseErr = emptyStringOnError(gs.FileContentsFromGS(storageService, config.Aggregator.Bucket, name))
}
}
if currFuzzName != "" {
reports = append(reports, fuzz.ParseBinaryReport(currFuzzType, currFuzzName, debugDump, debugErr, releaseDump, releaseErr))
}
glog.Info("Done loading")
return reports, nil
}
// download waits for fuzzPackages to appear on the toDownload channel and then downloads
// the four pieces of the package. It then parses them into a BinaryFuzzReport and sends
// the binary to the passed in channel. When there is no more work to be done, this function.
// returns and writes out true to the done channel.
func (g *GSLoader) download(toDownload <-chan fuzzPackage, reports chan<- fuzz.BinaryFuzzReport, wg *sync.WaitGroup) {
defer wg.Done()
for job := range toDownload {
debugDump := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.DebugDumpName))
debugErr := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.DebugErrName))
releaseDump := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.ReleaseDumpName))
releaseErr := emptyStringOnError(gs.FileContentsFromGS(g.storageClient, config.GS.Bucket, job.ReleaseErrName))
reports <- fuzz.ParseBinaryReport(job.FuzzType, job.FuzzName, debugDump, debugErr, releaseDump, releaseErr)
atomic.AddInt32(&g.completedCounter, 1)
if g.completedCounter%100 == 0 {
glog.Infof("%d fuzzes downloaded", g.completedCounter)
}
}
}