func main() {
flag.Parse()
cfg, _, err := config.Parse(*flagConfig)
if err != nil {
Fatalf("%v", err)
}
if len(flag.Args()) != 1 {
Fatalf("usage: syz-crush -config=config.file execution.log")
}
Logf(0, "booting test machines...")
var shutdown uint32
var wg sync.WaitGroup
wg.Add(cfg.Count + 1)
for i := 0; i < cfg.Count; i++ {
i := i
go func() {
defer wg.Done()
for {
vmCfg, err := config.CreateVMConfig(cfg, i)
if atomic.LoadUint32(&shutdown) != 0 {
break
}
if err != nil {
Fatalf("failed to create VM config: %v", err)
}
runInstance(cfg, vmCfg)
if atomic.LoadUint32(&shutdown) != 0 {
break
}
}
}()
}
go func() {
c := make(chan os.Signal, 2)
signal.Notify(c, syscall.SIGINT)
<-c
wg.Done()
atomic.StoreUint32(&shutdown, 1)
close(vm.Shutdown)
Logf(-1, "shutting down...")
atomic.StoreUint32(&shutdown, 1)
<-c
Fatalf("terminating")
}()
wg.Wait()
}
func RunManager(cfg *config.Config, syscalls map[int]bool, suppressions []*regexp.Regexp) {
crashdir := filepath.Join(cfg.Workdir, "crashes")
os.MkdirAll(crashdir, 0700)
enabledSyscalls := ""
if len(syscalls) != 0 {
buf := new(bytes.Buffer)
for c := range syscalls {
fmt.Fprintf(buf, ",%v", c)
}
enabledSyscalls = buf.String()[1:]
Logf(1, "enabled syscalls: %v", enabledSyscalls)
}
mgr := &Manager{
cfg: cfg,
crashdir: crashdir,
startTime: time.Now(),
stats: make(map[string]uint64),
enabledSyscalls: enabledSyscalls,
suppressions: suppressions,
corpusCover: make([]cover.Cover, sys.CallCount),
fuzzers: make(map[string]*Fuzzer),
}
Logf(0, "loading corpus...")
mgr.persistentCorpus = newPersistentSet(filepath.Join(cfg.Workdir, "corpus"), func(data []byte) bool {
if _, err := prog.Deserialize(data); err != nil {
Logf(0, "deleting broken program: %v\n%s", err, data)
return false
}
return true
})
for _, data := range mgr.persistentCorpus.a {
p, err := prog.Deserialize(data)
if err != nil {
Fatalf("failed to deserialize program: %v", err)
}
disabled := false
for _, c := range p.Calls {
if !syscalls[c.Meta.ID] {
disabled = true
break
}
}
if disabled {
// This program contains a disabled syscall.
// We won't execute it, but remeber its hash so
// it is not deleted during minimization.
h := hash(data)
mgr.disabledHashes = append(mgr.disabledHashes, hex.EncodeToString(h[:]))
continue
}
mgr.candidates = append(mgr.candidates, data)
}
Logf(0, "loaded %v programs", len(mgr.persistentCorpus.m))
// Create HTTP server.
mgr.initHttp()
// Create RPC server for fuzzers.
ln, err := net.Listen("tcp", cfg.Rpc)
if err != nil {
Fatalf("failed to listen on %v: %v", cfg.Rpc, err)
}
Logf(0, "serving rpc on tcp://%v", ln.Addr())
mgr.port = ln.Addr().(*net.TCPAddr).Port
s := rpc.NewServer()
s.Register(mgr)
go func() {
for {
conn, err := ln.Accept()
if err != nil {
Logf(0, "failed to accept an rpc connection: %v", err)
continue
}
go s.ServeCodec(jsonrpc.NewServerCodec(conn))
}
}()
Logf(0, "booting test machines...")
var shutdown uint32
var wg sync.WaitGroup
wg.Add(cfg.Count + 1)
for i := 0; i < cfg.Count; i++ {
i := i
go func() {
defer wg.Done()
for {
vmCfg, err := config.CreateVMConfig(cfg, i)
if atomic.LoadUint32(&shutdown) != 0 {
break
}
if err != nil {
Fatalf("failed to create VM config: %v", err)
}
ok := mgr.runInstance(vmCfg, i == 0)
if atomic.LoadUint32(&shutdown) != 0 {
break
}
if !ok {
time.Sleep(10 * time.Second)
}
}
}()
}
go func() {
for {
time.Sleep(10 * time.Second)
mgr.mu.Lock()
executed := mgr.stats["exec total"]
crashes := mgr.stats["crashes"]
mgr.mu.Unlock()
Logf(0, "executed programs: %v, crashes: %v", executed, crashes)
}
}()
go func() {
c := make(chan os.Signal, 2)
signal.Notify(c, syscall.SIGINT)
<-c
wg.Done()
DisableLog() // VMs will fail
atomic.StoreUint32(&mgr.shutdown, 1)
close(vm.Shutdown)
Logf(-1, "shutting down...")
atomic.StoreUint32(&shutdown, 1)
<-c
Fatalf("terminating")
}()
wg.Wait()
}
func main() {
flag.Parse()
cfg, _, _, err := config.Parse(*flagConfig)
if err != nil {
log.Fatalf("%v", err)
}
if *flagCount > 0 {
cfg.Count = *flagCount
}
if len(flag.Args()) != 1 {
log.Fatalf("usage: syz-repro -config=config.file execution.log")
}
data, err := ioutil.ReadFile(flag.Args()[0])
if err != nil {
log.Fatalf("failed to open log file: %v", err)
}
entries := prog.ParseLog(data)
log.Printf("parsed %v programs", len(entries))
crashLoc := vm.CrashRe.FindIndex(data)
if crashLoc == nil {
log.Fatalf("can't find crash message in the log")
}
log.Printf("target crash: '%s'", data[crashLoc[0]:crashLoc[1]])
instances = make(chan VM, cfg.Count)
bootRequests = make(chan bool, cfg.Count)
for i := 0; i < cfg.Count; i++ {
bootRequests <- true
go func() {
for range bootRequests {
vmCfg, err := config.CreateVMConfig(cfg)
if err != nil {
log.Fatalf("failed to create VM config: %v", err)
}
inst, err := vm.Create(cfg.Type, vmCfg)
if err != nil {
log.Fatalf("failed to create VM: %v", err)
}
execprogBin, err := inst.Copy(filepath.Join(cfg.Syzkaller, "bin/syz-execprog"))
if err != nil {
log.Fatalf("failed to copy to VM: %v", err)
}
executorBin, err := inst.Copy(filepath.Join(cfg.Syzkaller, "bin/syz-executor"))
if err != nil {
log.Fatalf("failed to copy to VM: %v", err)
}
instances <- VM{inst, execprogBin, executorBin}
}
}()
}
repro(cfg, entries, crashLoc)
for {
select {
case inst := <-instances:
inst.Close()
default:
return
}
}
}
func Run(crashLog []byte, cfg *config.Config, vmIndexes []int) (*Result, error) {
if len(vmIndexes) == 0 {
return nil, fmt.Errorf("no VMs provided")
}
if _, err := os.Stat(filepath.Join(cfg.Syzkaller, "bin/syz-execprog")); err != nil {
return nil, fmt.Errorf("bin/syz-execprog is missing (run 'make execprog')")
}
entries := prog.ParseLog(crashLog)
if len(entries) == 0 {
return nil, fmt.Errorf("crash log does not contain any programs")
}
crashDesc, _, crashStart, _ := report.Parse(crashLog, cfg.ParsedIgnores)
if crashDesc == "" {
crashStart = len(crashLog) // assuming VM hanged
crashDesc = "hang"
}
Logf(0, "reproducing crash '%v': %v programs, %v VMs", crashDesc, len(entries), len(vmIndexes))
ctx := &context{
cfg: cfg,
crashDesc: crashDesc,
instances: make(chan *instance, len(vmIndexes)),
bootRequests: make(chan int, len(vmIndexes)),
}
var wg sync.WaitGroup
wg.Add(len(vmIndexes))
for _, vmIndex := range vmIndexes {
ctx.bootRequests <- vmIndex
go func() {
defer wg.Done()
for vmIndex := range ctx.bootRequests {
var inst *instance
for try := 0; try < 3; try++ {
vmCfg, err := config.CreateVMConfig(cfg, vmIndex)
if err != nil {
Logf(0, "reproducing crash '%v': failed to create VM config: %v", crashDesc, err)
time.Sleep(10 * time.Second)
continue
}
vmInst, err := vm.Create(cfg.Type, vmCfg)
if err != nil {
Logf(0, "reproducing crash '%v': failed to create VM: %v", crashDesc, err)
time.Sleep(10 * time.Second)
continue
}
execprogBin, err := vmInst.Copy(filepath.Join(cfg.Syzkaller, "bin/syz-execprog"))
if err != nil {
Logf(0, "reproducing crash '%v': failed to copy to VM: %v", crashDesc, err)
vmInst.Close()
time.Sleep(10 * time.Second)
continue
}
executorBin, err := vmInst.Copy(filepath.Join(cfg.Syzkaller, "bin/syz-executor"))
if err != nil {
Logf(0, "reproducing crash '%v': failed to copy to VM: %v", crashDesc, err)
vmInst.Close()
time.Sleep(10 * time.Second)
continue
}
inst = &instance{vmInst, vmIndex, execprogBin, executorBin}
break
}
if inst == nil {
break
}
ctx.instances <- inst
}
}()
}
go func() {
wg.Wait()
close(ctx.instances)
}()
res, err := ctx.repro(entries, crashStart)
close(ctx.bootRequests)
for inst := range ctx.instances {
inst.Close()
}
return res, err
}
func RunManager(cfg *config.Config, syscalls map[int]bool, suppressions []*regexp.Regexp) {
crashdir := filepath.Join(cfg.Workdir, "crashes")
os.MkdirAll(crashdir, 0700)
enabledSyscalls := ""
if len(syscalls) != 0 {
buf := new(bytes.Buffer)
for c := range syscalls {
fmt.Fprintf(buf, ",%v", c)
}
enabledSyscalls = buf.String()[1:]
logf(1, "enabled syscalls: %v", enabledSyscalls)
}
mgr := &Manager{
cfg: cfg,
crashdir: crashdir,
startTime: time.Now(),
stats: make(map[string]uint64),
enabledSyscalls: enabledSyscalls,
suppressions: suppressions,
corpusCover: make([]cover.Cover, sys.CallCount),
fuzzers: make(map[string]*Fuzzer),
}
logf(0, "loading corpus...")
mgr.persistentCorpus = newPersistentSet(filepath.Join(cfg.Workdir, "corpus"), func(data []byte) bool {
if _, err := prog.Deserialize(data); err != nil {
logf(0, "deleting broken program: %v\n%s", err, data)
return false
}
return true
})
for _, data := range mgr.persistentCorpus.a {
p, err := prog.Deserialize(data)
if err != nil {
fatalf("failed to deserialize program: %v", err)
}
disabled := false
for _, c := range p.Calls {
if !syscalls[c.Meta.ID] {
disabled = true
break
}
}
if disabled {
// This program contains a disabled syscall.
// We won't execute it, but remeber its hash so
// it is not deleted during minimization.
h := hash(data)
mgr.disabledHashes = append(mgr.disabledHashes, hex.EncodeToString(h[:]))
continue
}
mgr.candidates = append(mgr.candidates, data)
}
logf(0, "loaded %v programs", len(mgr.persistentCorpus.m))
// Create HTTP server.
mgr.initHttp()
// Create RPC server for fuzzers.
ln, err := net.Listen("tcp", "localhost:0")
if err != nil {
fatalf("failed to listen on localhost:0: %v", err)
}
logf(0, "serving rpc on tcp://%v", ln.Addr())
mgr.port = ln.Addr().(*net.TCPAddr).Port
s := rpc.NewServer()
s.Register(mgr)
go func() {
for {
conn, err := ln.Accept()
if err != nil {
logf(0, "failed to accept an rpc connection: %v", err)
continue
}
go s.ServeCodec(jsonrpc.NewServerCodec(conn))
}
}()
for i := 0; i < cfg.Count; i++ {
first := i == 0
go func() {
for {
vmCfg, err := config.CreateVMConfig(cfg)
if err != nil {
fatalf("failed to create VM config: %v", err)
}
if !mgr.runInstance(vmCfg, first) {
time.Sleep(10 * time.Second)
}
}
}()
}
select {}
}
func (mgr *Manager) vmLoop() {
Logf(0, "booting test machines...")
reproInstances := 4
if reproInstances > mgr.cfg.Count {
reproInstances = mgr.cfg.Count
}
instances := make([]int, mgr.cfg.Count)
for i := range instances {
instances[i] = mgr.cfg.Count - i - 1
}
runDone := make(chan *RunResult, 1)
pendingRepro := make(map[*Crash]bool)
reproducing := make(map[string]bool)
var reproQueue []*Crash
reproDone := make(chan *ReproResult, 1)
stopPending := false
shutdown := vm.Shutdown
for {
for crash := range pendingRepro {
if reproducing[crash.desc] {
continue
}
delete(pendingRepro, crash)
if !mgr.needRepro(crash.desc) {
continue
}
Logf(1, "loop: add to repro queue '%v'", crash.desc)
reproducing[crash.desc] = true
reproQueue = append(reproQueue, crash)
}
Logf(1, "loop: shutdown=%v instances=%v/%v %+v repro: pending=%v reproducing=%v queued=%v",
shutdown == nil, len(instances), mgr.cfg.Count, instances,
len(pendingRepro), len(reproducing), len(reproQueue))
if shutdown == nil {
if len(instances) == mgr.cfg.Count {
return
}
} else {
for len(reproQueue) != 0 && len(instances) >= reproInstances {
last := len(reproQueue) - 1
crash := reproQueue[last]
reproQueue[last] = nil
reproQueue = reproQueue[:last]
vmIndexes := append([]int{}, instances[len(instances)-reproInstances:]...)
instances = instances[:len(instances)-reproInstances]
Logf(1, "loop: starting repro of '%v' on instances %+v", crash.desc, vmIndexes)
go func() {
res, err := repro.Run(crash.output, mgr.cfg, vmIndexes)
reproDone <- &ReproResult{vmIndexes, crash, res, err}
}()
}
for len(reproQueue) == 0 && len(instances) != 0 {
last := len(instances) - 1
idx := instances[last]
instances = instances[:last]
Logf(1, "loop: starting instance %v", idx)
go func() {
vmCfg, err := config.CreateVMConfig(mgr.cfg, idx)
if err != nil {
Fatalf("failed to create VM config: %v", err)
}
crash, err := mgr.runInstance(vmCfg, idx == 0)
runDone <- &RunResult{idx, crash, err}
}()
}
}
var stopRequest chan bool
if len(reproQueue) != 0 && !stopPending {
stopRequest = mgr.vmStop
}
select {
case stopRequest <- true:
Logf(1, "loop: issued stop request")
stopPending = true
case res := <-runDone:
Logf(1, "loop: instance %v finished, crash=%v", res.idx, res.crash != nil)
if res.err != nil && shutdown != nil {
Logf(0, "%v", res.err)
}
stopPending = false
instances = append(instances, res.idx)
// On shutdown qemu crashes with "qemu: terminating on signal 2",
// which we detect as "lost connection". Don't save that as crash.
if shutdown != nil && res.crash != nil && !mgr.isSuppressed(res.crash) {
mgr.saveCrash(res.crash)
if mgr.needRepro(res.crash.desc) {
Logf(1, "loop: add pending repro for '%v'", res.crash.desc)
pendingRepro[res.crash] = true
}
}
case res := <-reproDone:
crepro := false
if res.res != nil {
crepro = res.res.CRepro
}
Logf(1, "loop: repro on instances %+v finished '%v', repro=%v crepro=%v",
res.instances, res.crash.desc, res.res != nil, crepro)
if res.err != nil {
Logf(0, "repro failed: %v", res.err)
}
delete(reproducing, res.crash.desc)
instances = append(instances, res.instances...)
mgr.saveRepro(res.crash, res.res)
case <-shutdown:
Logf(1, "loop: shutting down...")
shutdown = nil
}
}
}
func main() {
flag.Parse()
cfg, _, _, err := config.Parse(*flagConfig)
if err != nil {
Fatalf("%v", err)
}
if *flagCount > 0 {
cfg.Count = *flagCount
}
if _, err := os.Stat(filepath.Join(cfg.Syzkaller, "bin/syz-execprog")); err != nil {
Fatalf("bin/syz-execprog is missing (run 'make execprog')")
}
if len(flag.Args()) != 1 {
Fatalf("usage: syz-repro -config=config.file execution.log")
}
data, err := ioutil.ReadFile(flag.Args()[0])
if err != nil {
Fatalf("failed to open log file: %v", err)
}
entries := prog.ParseLog(data)
Logf(0, "parsed %v programs", len(entries))
crashDesc, _, crashStart, _ := report.Parse(data)
if crashDesc == "" {
crashStart = len(data) // assuming VM hanged
}
instances = make(chan VM, cfg.Count)
bootRequests = make(chan int, cfg.Count)
for i := 0; i < cfg.Count; i++ {
bootRequests <- i
go func() {
for index := range bootRequests {
vmCfg, err := config.CreateVMConfig(cfg, index)
if err != nil {
Fatalf("failed to create VM config: %v", err)
}
inst, err := vm.Create(cfg.Type, vmCfg)
if err != nil {
Fatalf("failed to create VM: %v", err)
}
execprogBin, err := inst.Copy(filepath.Join(cfg.Syzkaller, "bin/syz-execprog"))
if err != nil {
Fatalf("failed to copy to VM: %v", err)
}
executorBin, err := inst.Copy(filepath.Join(cfg.Syzkaller, "bin/syz-executor"))
if err != nil {
Fatalf("failed to copy to VM: %v", err)
}
instances <- VM{inst, index, execprogBin, executorBin}
}
}()
}
go func() {
c := make(chan os.Signal, 2)
signal.Notify(c, syscall.SIGINT)
<-c
close(shutdown)
Logf(-1, "shutting down...")
<-c
Fatalf("terminating")
}()
repro(cfg, entries, crashStart)
exit()
}