// DownloadBinarySeedFiles downloads the seed skp files stored in Google
// Storage to be used by afl-fuzz. It places them in
// config.Generator.FuzzSamples after cleaning the folder out.
// It returns an error on failure.
func DownloadBinarySeedFiles(storageClient *storage.Client) error {
if err := os.RemoveAll(config.Generator.FuzzSamples); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("Could not clean binary seed path %s: %s", config.Generator.FuzzSamples, err)
}
if err := os.MkdirAll(config.Generator.FuzzSamples, 0755); err != nil {
return fmt.Errorf("Could not create binary seed path %s: %s", config.Generator.FuzzSamples, err)
}
err := gs.AllFilesInDir(storageClient, config.GS.Bucket, "skp_samples", func(item *storage.ObjectAttrs) {
name := item.Name
// skip the parent folder
if name == "skp_samples/" {
return
}
content, err := gs.FileContentsFromGS(storageClient, config.GS.Bucket, name)
if err != nil {
glog.Errorf("Problem downloading %s from Google Storage, continuing anyway", item.Name)
return
}
fileName := filepath.Join(config.Generator.FuzzSamples, strings.SplitAfter(name, "skp_samples/")[1])
if err = ioutil.WriteFile(fileName, content, 0644); err != nil && !os.IsExist(err) {
glog.Errorf("Problem creating binary seed file %s, continuing anyway", fileName)
}
})
return err
}
// downloadAllBadAndGreyFuzzes downloads just the fuzzes from a commit in GCS.
// It uses multiple processes to do so and puts them in downloadPath.
func (v *VersionUpdater) downloadAllBadAndGreyFuzzes(commitHash, downloadPath string) (badFuzzNames []string, greyFuzzNames []string, err error) {
toDownload := make(chan string, 100000)
completedCounter = 0
var wg sync.WaitGroup
for i := 0; i < config.Generator.NumDownloadProcesses; i++ {
wg.Add(1)
go v.download(toDownload, downloadPath, &wg)
}
badFilter := func(item *storage.ObjectAttrs) {
name := item.Name
if strings.HasSuffix(name, ".dump") || strings.HasSuffix(name, ".err") {
return
}
fuzzHash := name[strings.LastIndex(name, "/")+1:]
badFuzzNames = append(badFuzzNames, filepath.Join(config.Aggregator.BinaryFuzzPath, fuzzHash))
toDownload <- item.Name
}
greyFilter := func(item *storage.ObjectAttrs) {
name := item.Name
if strings.HasSuffix(item.Name, ".dump") || strings.HasSuffix(item.Name, ".err") {
return
}
fuzzHash := name[strings.LastIndex(name, "/")+1:]
greyFuzzNames = append(greyFuzzNames, filepath.Join(config.Aggregator.BinaryFuzzPath, fuzzHash))
toDownload <- item.Name
}
if err := gs.AllFilesInDir(v.storageClient, config.GS.Bucket, fmt.Sprintf("binary_fuzzes/%s/bad/", commitHash), badFilter); err != nil {
return nil, nil, fmt.Errorf("Problem getting bad fuzzes: %s", err)
}
if err := gs.AllFilesInDir(v.storageClient, config.GS.Bucket, fmt.Sprintf("binary_fuzzes/%s/grey/", commitHash), greyFilter); err != nil {
return nil, nil, fmt.Errorf("Problem getting grey fuzzes: %s", err)
}
close(toDownload)
wg.Wait()
return badFuzzNames, greyFuzzNames, nil
}
// GetAllFuzzNamesInFolder returns all the fuzz names in a given GCS folder. It basically
// returns a list of all files that don't end with a .dump or .err, or error
// if there was a problem.
func GetAllFuzzNamesInFolder(s *storage.Client, name string) (hashes []string, err error) {
filter := func(item *storage.ObjectAttrs) {
name := item.Name
if strings.HasSuffix(name, ".dump") || strings.HasSuffix(name, ".err") {
return
}
fuzzHash := name[strings.LastIndex(name, "/")+1:]
hashes = append(hashes, fuzzHash)
}
if err = gs.AllFilesInDir(s, config.GS.Bucket, name, filter); err != nil {
return hashes, fmt.Errorf("Problem getting fuzzes from folder %s: %s", name, err)
}
return hashes, nil
}
// getMostRecentOldRevision finds the most recently updated revision used.
// It searches the GS bucket under skia_version/old/ An error is returned if there is one.
func (f *FuzzSyncer) getMostRecentOldRevision() (string, error) {
var newestTime time.Time
newestHash := ""
findNewest := func(item *storage.ObjectAttrs) {
glog.Infof("%s: %s", item.Name, item.Updated)
if newestTime.Before(item.Updated) {
newestTime = item.Updated
newestHash = item.Name[strings.LastIndex(item.Name, "/")+1:]
}
}
if err := gs.AllFilesInDir(f.storageClient, config.GS.Bucket, "skia_version/old/", findNewest); err != nil {
return "", err
}
glog.Infof("Most recent old version found to be %s", newestHash)
return newestHash, nil
}
// fetchFuzzPackages scans for all fuzzes in the given folder and returns a
// slice of all of the metadata for each fuzz, as a fuzz package. It returns
// error if it cannot access Google Storage.
func (g *GSLoader) fetchFuzzPackages(baseFolder string) (fuzzPackages []fuzzPackage, err error) {
var debugDump, debugErr, releaseDump, releaseErr string
isInitialized := false
currFuzzFolder := "" // will be something like binary_fuzzes/bad/skp/badbeef
currFuzzName := ""
currFuzzType := ""
// We cannot simply use common.GetAllFuzzNamesInFolder because that loses FuzzType.
err = gs.AllFilesInDir(g.storageClient, config.GS.Bucket, baseFolder, func(item *storage.ObjectAttrs) {
// Assumption, files are sorted alphabetically and have the structure
// [baseFolder]/[filetype]/[fuzzname]/[fuzzname][suffix]
// where suffix is one of _debug.dump, _debug.err, _release.dump or _release.err
name := item.Name
if name == baseFolder || strings.Count(name, "/") <= 4 {
return
}
if !isInitialized || !strings.HasPrefix(name, currFuzzFolder) {
if isInitialized {
fuzzPackages = append(fuzzPackages, fuzzPackage{
FuzzType: currFuzzType,
FuzzName: currFuzzName,
DebugDumpName: debugDump,
DebugErrName: debugErr,
ReleaseDumpName: releaseDump,
ReleaseErrName: releaseErr,
})
} else {
isInitialized = true
}
parts := strings.Split(name, "/")
currFuzzFolder = strings.Join(parts[0:5], "/")
currFuzzType = parts[3]
currFuzzName = parts[4]
// reset for next one
debugDump, debugErr, releaseDump, releaseErr = "", "", "", ""
}
if strings.HasSuffix(name, "_debug.dump") {
debugDump = name
} else if strings.HasSuffix(name, "_debug.err") {
debugErr = name
} else if strings.HasSuffix(name, "_release.dump") {
releaseDump = name
} else if strings.HasSuffix(name, "_release.err") {
releaseErr = name
}
})
if err != nil {
return fuzzPackages, err
}
if currFuzzName != "" {
fuzzPackages = append(fuzzPackages, fuzzPackage{
FuzzType: currFuzzType,
FuzzName: currFuzzName,
DebugDumpName: debugDump,
DebugErrName: debugErr,
ReleaseDumpName: releaseDump,
ReleaseErrName: releaseErr,
})
}
return fuzzPackages, nil
}