func (mgr *Manager) saveRepro(crash *Crash, res *repro.Result) {
sig := hash.Hash([]byte(crash.desc))
dir := filepath.Join(mgr.crashdir, sig.String())
if res == nil {
for i := 0; i < maxReproAttempts; i++ {
name := filepath.Join(dir, fmt.Sprintf("repro%v", i))
if _, err := os.Stat(name); err != nil {
ioutil.WriteFile(name, nil, 0660)
break
}
}
return
}
opts := fmt.Sprintf("# %+v\n", res.Opts)
prog := res.Prog.Serialize()
ioutil.WriteFile(filepath.Join(dir, "repro.prog"), append([]byte(opts), prog...), 0660)
if len(mgr.cfg.Tag) > 0 {
ioutil.WriteFile(filepath.Join(dir, "repro.tag"), []byte(mgr.cfg.Tag), 0660)
}
if len(crash.text) > 0 {
ioutil.WriteFile(filepath.Join(dir, "repro.report"), []byte(crash.text), 0660)
}
if res.CRepro {
cprog, err := csource.Write(res.Prog, res.Opts)
if err == nil {
formatted, err := csource.Format(cprog)
if err == nil {
cprog = formatted
}
ioutil.WriteFile(filepath.Join(dir, "repro.cprog"), cprog, 0660)
} else {
Logf(0, "failed to write C source: %v", err)
}
}
}
func newPersistentSet(dir string, verify func(data []byte) bool) *PersistentSet {
ps := &PersistentSet{
dir: dir,
m: make(map[hash.Sig][]byte),
}
os.MkdirAll(dir, 0770)
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
Fatalf("error during dir walk: %v\n", err)
}
if info.IsDir() {
if info.Name() == ".git" {
return filepath.SkipDir // in case corpus is checked in
}
return nil
}
data, err := ioutil.ReadFile(path)
if err != nil {
Fatalf("error during file read: %v\n", err)
return nil
}
sig := hash.Hash(data)
if _, ok := ps.m[sig]; ok {
return nil
}
name := info.Name()
if len(data) == 0 {
// This can happen is master runs on machine-under-test,
// and it has crashed midway.
Logf(0, "removing empty file %v", name)
os.Remove(path)
return nil
}
if _, err := hash.FromString(name); err != nil {
Logf(0, "unknown file in persistent dir %v: %v", dir, name)
return nil
}
if verify != nil && !verify(data) {
os.Remove(path)
return nil
}
if name != sig.String() {
Logf(0, "bad hash in persistent dir %v for file %v, expect %v", dir, name, sig.String())
if err := ioutil.WriteFile(filepath.Join(ps.dir, sig.String()), data, 0660); err != nil {
Fatalf("failed to write file: %v", err)
}
os.Remove(path)
}
ps.m[sig] = data
ps.a = append(ps.a, data)
return nil
})
return ps
}
func (ps *PersistentSet) add(data []byte) bool {
sig := hash.Hash(data)
if _, ok := ps.m[sig]; ok {
return false
}
ps.m[sig] = data
ps.a = append(ps.a, data)
fname := filepath.Join(ps.dir, sig.String())
if err := ioutil.WriteFile(fname, data, 0660); err != nil {
Fatalf("failed to write file: %v", err)
}
return true
}
func (mgr *Manager) needRepro(desc string) bool {
sig := hash.Hash([]byte(desc))
dir := filepath.Join(mgr.crashdir, sig.String())
if _, err := os.Stat(filepath.Join(dir, "repro.prog")); err == nil {
return false
}
for i := 0; i < maxReproAttempts; i++ {
if _, err := os.Stat(filepath.Join(dir, fmt.Sprintf("repro%v", i))); err != nil {
return true
}
}
return false
}
func (mgr *Manager) minimizeCorpus() {
if mgr.cfg.Cover && len(mgr.corpus) != 0 {
// First, sort corpus per call.
type Call struct {
inputs []RpcInput
cov []cover.Cover
}
calls := make(map[string]Call)
for _, inp := range mgr.corpus {
c := calls[inp.Call]
c.inputs = append(c.inputs, inp)
c.cov = append(c.cov, inp.Cover)
calls[inp.Call] = c
}
// Now minimize and build new corpus.
var newCorpus []RpcInput
for _, c := range calls {
for _, idx := range cover.Minimize(c.cov) {
newCorpus = append(newCorpus, c.inputs[idx])
}
}
Logf(1, "minimized corpus: %v -> %v", len(mgr.corpus), len(newCorpus))
mgr.corpus = newCorpus
}
var corpus []*prog.Prog
for _, inp := range mgr.corpus {
p, err := prog.Deserialize(inp.Prog)
if err != nil {
panic(err)
}
corpus = append(corpus, p)
}
mgr.prios = prog.CalculatePriorities(corpus)
// Don't minimize persistent corpus until fuzzers have triaged all inputs from it.
if len(mgr.candidates) == 0 {
hashes := make(map[string]bool)
for _, inp := range mgr.corpus {
sig := hash.Hash(inp.Prog)
hashes[sig.String()] = true
}
for _, h := range mgr.disabledHashes {
hashes[h] = true
}
mgr.persistentCorpus.minimize(hashes)
}
}
func (st *State) addInput(mgr *Manager, input []byte) {
if _, err := prog.CallSet(input); err != nil {
Logf(0, "manager %v: failed to extract call set: %v, program:\n%v", mgr.name, err, string(input))
return
}
sig := hash.Hash(input)
mgr.Corpus[sig] = true
fname := filepath.Join(mgr.dir, "corpus", sig.String())
writeFile(fname, nil)
if st.Corpus[sig] == nil {
st.Corpus[sig] = &Input{
seq: st.seq,
prog: input,
}
fname := filepath.Join(st.dir, "corpus", fmt.Sprintf("%v-%v", sig.String(), st.seq))
writeFile(fname, input)
}
}
func (mgr *Manager) saveCrash(crash *Crash) {
Logf(0, "%v: crash: %v", crash.vmName, crash.desc)
mgr.mu.Lock()
mgr.stats["crashes"]++
mgr.mu.Unlock()
sig := hash.Hash([]byte(crash.desc))
id := sig.String()
dir := filepath.Join(mgr.crashdir, id)
os.MkdirAll(dir, 0700)
if err := ioutil.WriteFile(filepath.Join(dir, "description"), []byte(crash.desc+"\n"), 0660); err != nil {
Logf(0, "failed to write crash: %v", err)
}
// Save up to 100 reports. If we already have 100, overwrite the oldest one.
// Newer reports are generally more useful. Overwriting is also needed
// to be able to understand if a particular bug still happens or already fixed.
oldestI := 0
var oldestTime time.Time
for i := 0; i < 100; i++ {
info, err := os.Stat(filepath.Join(dir, fmt.Sprintf("log%v", i)))
if err != nil {
oldestI = i
break
}
if oldestTime.IsZero() || info.ModTime().Before(oldestTime) {
oldestI = i
oldestTime = info.ModTime()
}
}
ioutil.WriteFile(filepath.Join(dir, fmt.Sprintf("log%v", oldestI)), crash.output, 0660)
if len(mgr.cfg.Tag) > 0 {
ioutil.WriteFile(filepath.Join(dir, fmt.Sprintf("tag%v", oldestI)), []byte(mgr.cfg.Tag), 0660)
}
if len(crash.text) > 0 {
symbolized, err := report.Symbolize(mgr.cfg.Vmlinux, crash.text)
if err != nil {
Logf(0, "failed to symbolize crash: %v", err)
} else {
crash.text = symbolized
}
ioutil.WriteFile(filepath.Join(dir, fmt.Sprintf("report%v", oldestI)), []byte(crash.text), 0660)
}
}
// Make creates State and initializes it from dir.
func Make(dir string) (*State, error) {
st := &State{
dir: dir,
Corpus: make(map[hash.Sig]*Input),
Managers: make(map[string]*Manager),
}
corpusDir := filepath.Join(st.dir, "corpus")
os.MkdirAll(corpusDir, 0700)
inputs, err := ioutil.ReadDir(corpusDir)
if err != nil {
return nil, fmt.Errorf("failed to read %v dir: %v", corpusDir, err)
}
for _, inp := range inputs {
data, err := ioutil.ReadFile(filepath.Join(corpusDir, inp.Name()))
if err != nil {
return nil, err
}
if _, err := prog.CallSet(data); err != nil {
return nil, err
}
parts := strings.Split(inp.Name(), "-")
if len(parts) != 2 {
return nil, fmt.Errorf("bad file in corpus: %v", inp.Name())
}
seq, err := strconv.ParseUint(parts[1], 10, 64)
if err != nil {
return nil, fmt.Errorf("bad file in corpus: %v", inp.Name())
}
sig := hash.Hash(data)
if sig.String() != parts[0] {
return nil, fmt.Errorf("bad file in corpus: %v, want hash %v", inp.Name(), sig.String())
}
st.Corpus[sig] = &Input{
seq: seq,
prog: data,
}
if st.seq < seq {
st.seq = seq
}
}
managersDir := filepath.Join(st.dir, "manager")
os.MkdirAll(managersDir, 0700)
managers, err := ioutil.ReadDir(managersDir)
if err != nil {
return nil, fmt.Errorf("failed to read %v dir: %v", managersDir, err)
}
for _, manager := range managers {
mgr := &Manager{
name: manager.Name(),
}
st.Managers[mgr.name] = mgr
mgr.dir = filepath.Join(managersDir, mgr.name)
seqStr, _ := ioutil.ReadFile(filepath.Join(mgr.dir, "seq"))
mgr.seq, _ = strconv.ParseUint(string(seqStr), 10, 64)
if st.seq < mgr.seq {
st.seq = mgr.seq
}
mgr.Corpus = make(map[hash.Sig]bool)
corpusDir := filepath.Join(mgr.dir, "corpus")
os.MkdirAll(corpusDir, 0700)
corpus, err := ioutil.ReadDir(corpusDir)
if err != nil {
return nil, fmt.Errorf("failed to read %v dir: %v", corpusDir, err)
}
for _, input := range corpus {
sig, err := hash.FromString(input.Name())
if err != nil {
return nil, fmt.Errorf("bad file in corpus: %v", input.Name())
}
mgr.Corpus[sig] = true
}
}
return st, err
}
func RunManager(cfg *config.Config, syscalls map[int]bool) {
crashdir := filepath.Join(cfg.Workdir, "crashes")
os.MkdirAll(crashdir, 0700)
enabledSyscalls := ""
if len(syscalls) != 0 {
buf := new(bytes.Buffer)
for c := range syscalls {
fmt.Fprintf(buf, ",%v", c)
}
enabledSyscalls = buf.String()[1:]
Logf(1, "enabled syscalls: %v", enabledSyscalls)
}
mgr := &Manager{
cfg: cfg,
crashdir: crashdir,
startTime: time.Now(),
stats: make(map[string]uint64),
enabledSyscalls: enabledSyscalls,
corpusCover: make([]cover.Cover, sys.CallCount),
fuzzers: make(map[string]*Fuzzer),
fresh: true,
vmStop: make(chan bool),
}
Logf(0, "loading corpus...")
mgr.persistentCorpus = newPersistentSet(filepath.Join(cfg.Workdir, "corpus"), func(data []byte) bool {
mgr.fresh = false
if _, err := prog.Deserialize(data); err != nil {
Logf(0, "deleting broken program: %v\n%s", err, data)
return false
}
return true
})
for _, data := range mgr.persistentCorpus.a {
p, err := prog.Deserialize(data)
if err != nil {
Fatalf("failed to deserialize program: %v", err)
}
disabled := false
for _, c := range p.Calls {
if !syscalls[c.Meta.ID] {
disabled = true
break
}
}
if disabled {
// This program contains a disabled syscall.
// We won't execute it, but remeber its hash so
// it is not deleted during minimization.
// TODO: use mgr.enabledCalls which accounts for missing devices, etc.
// But it is available only after vm check.
sig := hash.Hash(data)
mgr.disabledHashes = append(mgr.disabledHashes, sig.String())
continue
}
mgr.candidates = append(mgr.candidates, data)
}
Logf(0, "loaded %v programs (%v total)", len(mgr.candidates), len(mgr.persistentCorpus.m))
// Create HTTP server.
mgr.initHttp()
// Create RPC server for fuzzers.
ln, err := net.Listen("tcp", cfg.Rpc)
if err != nil {
Fatalf("failed to listen on %v: %v", cfg.Rpc, err)
}
Logf(0, "serving rpc on tcp://%v", ln.Addr())
mgr.port = ln.Addr().(*net.TCPAddr).Port
s := rpc.NewServer()
s.Register(mgr)
go func() {
for {
conn, err := ln.Accept()
if err != nil {
Logf(0, "failed to accept an rpc connection: %v", err)
continue
}
conn.(*net.TCPConn).SetKeepAlive(true)
conn.(*net.TCPConn).SetKeepAlivePeriod(time.Minute)
go s.ServeCodec(jsonrpc.NewServerCodec(conn))
}
}()
go func() {
for {
time.Sleep(10 * time.Second)
mgr.mu.Lock()
executed := mgr.stats["exec total"]
crashes := mgr.stats["crashes"]
mgr.mu.Unlock()
Logf(0, "executed programs: %v, crashes: %v", executed, crashes)
}
}()
if mgr.cfg.Hub_Addr != "" {
go func() {
for {
time.Sleep(time.Minute)
mgr.hubSync()
}
}()
}
go func() {
c := make(chan os.Signal, 2)
signal.Notify(c, syscall.SIGINT)
<-c
close(vm.Shutdown)
Logf(0, "shutting down...")
<-c
Fatalf("terminating")
}()
mgr.vmLoop()
}
func (mgr *Manager) hubSync() {
mgr.mu.Lock()
defer mgr.mu.Unlock()
if !mgr.vmChecked || len(mgr.candidates) != 0 {
return
}
mgr.minimizeCorpus()
if mgr.hub == nil {
conn, err := rpc.Dial("tcp", mgr.cfg.Hub_Addr)
if err != nil {
Logf(0, "failed to connect to hub at %v: %v", mgr.cfg.Hub_Addr, err)
return
}
mgr.hub = conn
a := &HubConnectArgs{
Name: mgr.cfg.Name,
Key: mgr.cfg.Hub_Key,
Fresh: mgr.fresh,
Calls: mgr.enabledCalls,
}
mgr.hubCorpus = make(map[hash.Sig]bool)
for _, inp := range mgr.corpus {
mgr.hubCorpus[hash.Hash(inp.Prog)] = true
a.Corpus = append(a.Corpus, inp.Prog)
}
if err := mgr.hub.Call("Hub.Connect", a, nil); err != nil {
Logf(0, "Hub.Connect rpc failed: %v", err)
mgr.hub.Close()
mgr.hub = nil
return
}
mgr.fresh = false
Logf(0, "connected to hub at %v, corpus %v", mgr.cfg.Hub_Addr, len(mgr.corpus))
}
a := &HubSyncArgs{
Name: mgr.cfg.Name,
Key: mgr.cfg.Hub_Key,
}
corpus := make(map[hash.Sig]bool)
for _, inp := range mgr.corpus {
sig := hash.Hash(inp.Prog)
corpus[sig] = true
if mgr.hubCorpus[sig] {
continue
}
mgr.hubCorpus[sig] = true
a.Add = append(a.Add, inp.Prog)
}
for sig := range mgr.hubCorpus {
if corpus[sig] {
continue
}
delete(mgr.hubCorpus, sig)
a.Del = append(a.Del, sig.String())
}
r := new(HubSyncRes)
if err := mgr.hub.Call("Hub.Sync", a, r); err != nil {
Logf(0, "Hub.Sync rpc failed: %v", err)
mgr.hub.Close()
mgr.hub = nil
return
}
dropped := 0
for _, inp := range r.Inputs {
_, err := prog.Deserialize(inp)
if err != nil {
dropped++
continue
}
mgr.candidates = append(mgr.candidates, inp)
}
mgr.stats["hub add"] += uint64(len(a.Add))
mgr.stats["hub del"] += uint64(len(a.Del))
mgr.stats["hub drop"] += uint64(dropped)
mgr.stats["hub new"] += uint64(len(r.Inputs) - dropped)
Logf(0, "hub sync: add %v, del %v, drop %v, new %v", len(a.Add), len(a.Del), dropped, len(r.Inputs)-dropped)
}
