// DownloadBinarySeedFiles downloads the seed skp files stored in Google
// Storage to be used by afl-fuzz.  It places them in
// config.Generator.FuzzSamples after cleaning the folder out.
// It returns an error on failure.
func DownloadBinarySeedFiles(storageClient *storage.Client) error {
	if err := os.RemoveAll(config.Generator.FuzzSamples); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Could not clean binary seed path %s: %s", config.Generator.FuzzSamples, err)
	}
	if err := os.MkdirAll(config.Generator.FuzzSamples, 0755); err != nil {
		return fmt.Errorf("Could not create binary seed path %s: %s", config.Generator.FuzzSamples, err)
	}

	err := gs.AllFilesInDir(storageClient, config.GS.Bucket, "skp_samples", func(item *storage.ObjectAttrs) {
		name := item.Name
		// skip the parent folder
		if name == "skp_samples/" {
			return
		}
		content, err := gs.FileContentsFromGS(storageClient, config.GS.Bucket, name)
		if err != nil {
			glog.Errorf("Problem downloading %s from Google Storage, continuing anyway", item.Name)
			return
		}
		fileName := filepath.Join(config.Generator.FuzzSamples, strings.SplitAfter(name, "skp_samples/")[1])
		if err = ioutil.WriteFile(fileName, content, 0644); err != nil && !os.IsExist(err) {
			glog.Errorf("Problem creating binary seed file %s, continuing anyway", fileName)
		}
	})
	return err
}
// downloadAllBadAndGreyFuzzes downloads just the fuzzes from a commit in GCS.
// It uses multiple processes to do so and puts them in downloadPath.
func (v *VersionUpdater) downloadAllBadAndGreyFuzzes(commitHash, downloadPath string) (badFuzzNames []string, greyFuzzNames []string, err error) {
	toDownload := make(chan string, 100000)
	completedCounter = 0

	var wg sync.WaitGroup
	for i := 0; i < config.Generator.NumDownloadProcesses; i++ {
		wg.Add(1)
		go v.download(toDownload, downloadPath, &wg)
	}

	badFilter := func(item *storage.ObjectAttrs) {
		name := item.Name
		if strings.HasSuffix(name, ".dump") || strings.HasSuffix(name, ".err") {
			return
		}
		fuzzHash := name[strings.LastIndex(name, "/")+1:]
		badFuzzNames = append(badFuzzNames, filepath.Join(config.Aggregator.BinaryFuzzPath, fuzzHash))
		toDownload <- item.Name
	}

	greyFilter := func(item *storage.ObjectAttrs) {
		name := item.Name
		if strings.HasSuffix(item.Name, ".dump") || strings.HasSuffix(item.Name, ".err") {
			return
		}
		fuzzHash := name[strings.LastIndex(name, "/")+1:]
		greyFuzzNames = append(greyFuzzNames, filepath.Join(config.Aggregator.BinaryFuzzPath, fuzzHash))
		toDownload <- item.Name
	}

	if err := gs.AllFilesInDir(v.storageClient, config.GS.Bucket, fmt.Sprintf("binary_fuzzes/%s/bad/", commitHash), badFilter); err != nil {
		return nil, nil, fmt.Errorf("Problem getting bad fuzzes: %s", err)
	}

	if err := gs.AllFilesInDir(v.storageClient, config.GS.Bucket, fmt.Sprintf("binary_fuzzes/%s/grey/", commitHash), greyFilter); err != nil {
		return nil, nil, fmt.Errorf("Problem getting grey fuzzes: %s", err)
	}

	close(toDownload)
	wg.Wait()
	return badFuzzNames, greyFuzzNames, nil
}
Esempio n. 3
0
// GetAllFuzzNamesInFolder returns all the fuzz names in a given GCS folder.  It basically
// returns a list of all files that don't end with a .dump or .err, or error
// if there was a problem.
func GetAllFuzzNamesInFolder(s *storage.Client, name string) (hashes []string, err error) {
	filter := func(item *storage.ObjectAttrs) {
		name := item.Name
		if strings.HasSuffix(name, ".dump") || strings.HasSuffix(name, ".err") {
			return
		}
		fuzzHash := name[strings.LastIndex(name, "/")+1:]
		hashes = append(hashes, fuzzHash)
	}

	if err = gs.AllFilesInDir(s, config.GS.Bucket, name, filter); err != nil {
		return hashes, fmt.Errorf("Problem getting fuzzes from folder %s: %s", name, err)
	}
	return hashes, nil
}
Esempio n. 4
0
// getMostRecentOldRevision finds the most recently updated revision used.
// It searches the GS bucket under skia_version/old/  An error is returned if there is one.
func (f *FuzzSyncer) getMostRecentOldRevision() (string, error) {
	var newestTime time.Time
	newestHash := ""
	findNewest := func(item *storage.ObjectAttrs) {
		glog.Infof("%s: %s", item.Name, item.Updated)
		if newestTime.Before(item.Updated) {
			newestTime = item.Updated
			newestHash = item.Name[strings.LastIndex(item.Name, "/")+1:]
		}
	}
	if err := gs.AllFilesInDir(f.storageClient, config.GS.Bucket, "skia_version/old/", findNewest); err != nil {
		return "", err
	}
	glog.Infof("Most recent old version found to be %s", newestHash)
	return newestHash, nil
}
Esempio n. 5
0
// fetchFuzzPackages scans for all fuzzes in the given folder and returns a
// slice of all of the metadata for each fuzz, as a fuzz package.  It returns
// error if it cannot access Google Storage.
func (g *GSLoader) fetchFuzzPackages(baseFolder string) (fuzzPackages []fuzzPackage, err error) {
	var debugDump, debugErr, releaseDump, releaseErr string
	isInitialized := false
	currFuzzFolder := "" // will be something like binary_fuzzes/bad/skp/badbeef
	currFuzzName := ""
	currFuzzType := ""

	// We cannot simply use common.GetAllFuzzNamesInFolder because that loses FuzzType.
	err = gs.AllFilesInDir(g.storageClient, config.GS.Bucket, baseFolder, func(item *storage.ObjectAttrs) {
		// Assumption, files are sorted alphabetically and have the structure
		// [baseFolder]/[filetype]/[fuzzname]/[fuzzname][suffix]
		// where suffix is one of _debug.dump, _debug.err, _release.dump or _release.err
		name := item.Name
		if name == baseFolder || strings.Count(name, "/") <= 4 {
			return
		}

		if !isInitialized || !strings.HasPrefix(name, currFuzzFolder) {
			if isInitialized {
				fuzzPackages = append(fuzzPackages, fuzzPackage{
					FuzzType:        currFuzzType,
					FuzzName:        currFuzzName,
					DebugDumpName:   debugDump,
					DebugErrName:    debugErr,
					ReleaseDumpName: releaseDump,
					ReleaseErrName:  releaseErr,
				})
			} else {
				isInitialized = true
			}

			parts := strings.Split(name, "/")
			currFuzzFolder = strings.Join(parts[0:5], "/")
			currFuzzType = parts[3]
			currFuzzName = parts[4]
			// reset for next one
			debugDump, debugErr, releaseDump, releaseErr = "", "", "", ""
		}
		if strings.HasSuffix(name, "_debug.dump") {
			debugDump = name
		} else if strings.HasSuffix(name, "_debug.err") {
			debugErr = name
		} else if strings.HasSuffix(name, "_release.dump") {
			releaseDump = name
		} else if strings.HasSuffix(name, "_release.err") {
			releaseErr = name
		}
	})
	if err != nil {
		return fuzzPackages, err
	}
	if currFuzzName != "" {
		fuzzPackages = append(fuzzPackages, fuzzPackage{
			FuzzType:        currFuzzType,
			FuzzName:        currFuzzName,
			DebugDumpName:   debugDump,
			DebugErrName:    debugErr,
			ReleaseDumpName: releaseDump,
			ReleaseErrName:  releaseErr,
		})
	}
	return fuzzPackages, nil
}