func triageInput(env *ipc.Env, inp Input) {
if *flagNoCover {
panic("should not be called when coverage is disabled")
}
call := inp.p.Calls[inp.call].Meta
newCover := cover.Difference(inp.cover, corpusCover[call.CallID])
newCover = cover.Difference(newCover, flakes)
if len(newCover) == 0 {
return
}
if _, ok := corpusHashes[hash(inp.p.Serialize())]; ok {
return
}
minCover := inp.cover
for i := 0; i < 3; i++ {
allCover := execute1(env, inp.p, &statExecTriage)
if len(allCover[inp.call]) == 0 {
// The call was not executed. Happens sometimes, reason unknown.
continue
}
cov := allCover[inp.call]
diff := cover.SymmetricDifference(inp.cover, cov)
if len(diff) != 0 {
flakes = cover.Union(flakes, diff)
}
minCover = cover.Intersection(minCover, cov)
}
stableNewCover := cover.Intersection(newCover, minCover)
if len(stableNewCover) == 0 {
return
}
inp.p, inp.call = prog.Minimize(inp.p, inp.call, func(p1 *prog.Prog, call1 int) bool {
allCover := execute1(env, p1, &statExecMinimize)
if len(allCover[call1]) == 0 {
return false // The call was not executed.
}
cov := allCover[call1]
if len(cover.Intersection(stableNewCover, cov)) != len(stableNewCover) {
return false
}
minCover = cover.Intersection(minCover, cov)
return true
})
inp.cover = minCover
corpusCover[call.CallID] = cover.Union(corpusCover[call.CallID], minCover)
corpus = append(corpus, inp)
data := inp.p.Serialize()
corpusHashes[hash(data)] = struct{}{}
logf(2, "added new input for %v to corpus:\n%s", call.CallName, data)
statNewInput++
a := &NewManagerInputArgs{*flagName, RpcInput{call.CallName, inp.p.Serialize(), inp.call, []uint32(inp.cover)}}
if err := manager.Call("Manager.NewInput", a, nil); err != nil {
panic(err)
}
}
func repro(cfg *config.Config, entries []*prog.LogEntry, crashLoc []int) {
// Cut programs that were executed after crash.
for i, ent := range entries {
if ent.Start > crashLoc[0] {
entries = entries[:i]
break
}
}
// Extract last program on every proc.
procs := make(map[int]int)
for i, ent := range entries {
procs[ent.Proc] = i
}
var indices []int
for _, idx := range procs {
indices = append(indices, idx)
}
sort.Ints(indices)
var suspected []*prog.LogEntry
for i := len(indices) - 1; i >= 0; i-- {
suspected = append(suspected, entries[indices[i]])
}
// Execute the suspected programs.
log.Printf("the suspected programs are:")
for _, ent := range suspected {
log.Printf("on proc %v:\n%s\n", ent.Proc, ent.P.Serialize())
}
var p *prog.Prog
multiplier := 1
for ; p == nil && multiplier <= 100; multiplier *= 10 {
for _, ent := range suspected {
if testProg(cfg, ent.P, multiplier, true, true) {
p = ent.P
break
}
}
}
if p == nil {
log.Printf("no program crashed")
return
}
log.Printf("minimizing program")
p, _ = prog.Minimize(p, -1, func(p1 *prog.Prog, callIndex int) bool {
return testProg(cfg, p1, multiplier, true, true)
})
opts := csource.Options{
Threaded: true,
Collide: true,
}
if testProg(cfg, p, multiplier, true, false) {
opts.Collide = false
if testProg(cfg, p, multiplier, false, false) {
opts.Threaded = false
}
}
src := csource.Write(p, opts)
log.Printf("C source:\n%s\n", src)
srcf, err := fileutil.WriteTempFile(src)
if err != nil {
log.Fatalf("%v", err)
}
bin, err := csource.Build(srcf)
if err != nil {
log.Fatalf("%v", err)
}
defer os.Remove(bin)
testBin(cfg, bin)
}
func (ctx *context) repro(entries []*prog.LogEntry, crashStart int) (*Result, error) {
// Cut programs that were executed after crash.
for i, ent := range entries {
if ent.Start > crashStart {
entries = entries[:i]
break
}
}
// Extract last program on every proc.
procs := make(map[int]int)
for i, ent := range entries {
procs[ent.Proc] = i
}
var indices []int
for _, idx := range procs {
indices = append(indices, idx)
}
sort.Ints(indices)
var suspected []*prog.LogEntry
for i := len(indices) - 1; i >= 0; i-- {
suspected = append(suspected, entries[indices[i]])
}
Logf(2, "reproducing crash '%v': suspecting %v programs", ctx.crashDesc, len(suspected))
opts := csource.Options{
Threaded: true,
Collide: true,
Repeat: true,
Procs: ctx.cfg.Procs,
Sandbox: ctx.cfg.Sandbox,
Repro: true,
}
// Execute the suspected programs.
// We first try to execute each program for 10 seconds, that should detect simple crashes
// (i.e. no races and no hangs). Then we execute each program for 5 minutes
// to catch races and hangs. Note that the max duration must be larger than
// hang/no output detection duration in vm.MonitorExecution, which is currently set to 3 mins.
var res *Result
var duration time.Duration
for _, dur := range []time.Duration{10 * time.Second, 5 * time.Minute} {
for _, ent := range suspected {
crashed, err := ctx.testProg(ent.P, dur, opts, true)
if err != nil {
return nil, err
}
if crashed {
res = &Result{
Prog: ent.P,
Opts: opts,
}
duration = dur * 3 / 2
break
}
}
if res != nil {
break
}
}
if res == nil {
Logf(0, "reproducing crash '%v': no program crashed", ctx.crashDesc)
return nil, nil
}
defer func() {
res.Opts.Repro = false
}()
Logf(2, "reproducing crash '%v': minimizing guilty program", ctx.crashDesc)
res.Prog, _ = prog.Minimize(res.Prog, -1, func(p1 *prog.Prog, callIndex int) bool {
crashed, err := ctx.testProg(p1, duration, res.Opts, false)
if err != nil {
Logf(1, "reproducing crash '%v': minimization failed with %v", ctx.crashDesc, err)
return false
}
return crashed
}, true)
// Try to "minimize" threaded/collide/sandbox/etc to find simpler reproducer.
opts = res.Opts
opts.Collide = false
crashed, err := ctx.testProg(res.Prog, duration, opts, false)
if err != nil {
return res, err
}
if crashed {
res.Opts = opts
opts.Threaded = false
crashed, err := ctx.testProg(res.Prog, duration, opts, false)
if err != nil {
return res, err
}
if crashed {
res.Opts = opts
}
}
if res.Opts.Sandbox == "namespace" {
opts = res.Opts
opts.Sandbox = "none"
crashed, err := ctx.testProg(res.Prog, duration, opts, false)
if err != nil {
return res, err
}
if crashed {
res.Opts = opts
}
}
if res.Opts.Procs > 1 {
opts = res.Opts
opts.Procs = 1
crashed, err := ctx.testProg(res.Prog, duration, opts, false)
if err != nil {
return res, err
}
if crashed {
res.Opts = opts
}
}
if res.Opts.Repeat {
opts = res.Opts
opts.Repeat = false
crashed, err := ctx.testProg(res.Prog, duration, opts, false)
if err != nil {
return res, err
}
if crashed {
res.Opts = opts
}
}
src, err := csource.Write(res.Prog, res.Opts)
if err != nil {
return res, err
}
srcf, err := fileutil.WriteTempFile(src)
if err != nil {
return res, err
}
bin, err := csource.Build(srcf)
if err != nil {
return res, err
}
defer os.Remove(bin)
crashed, err = ctx.testBin(bin, duration, false)
if err != nil {
return res, err
}
res.CRepro = crashed
return res, nil
}
func triageInput(pid int, env *ipc.Env, inp Input) {
if noCover {
panic("should not be called when coverage is disabled")
}
call := inp.p.Calls[inp.call].Meta
coverMu.RLock()
newCover := cover.Difference(inp.cover, corpusCover[call.CallID])
newCover = cover.Difference(newCover, flakes)
coverMu.RUnlock()
if len(newCover) == 0 {
return
}
corpusMu.RLock()
if _, ok := corpusHashes[hash(inp.p.Serialize())]; ok {
corpusMu.RUnlock()
return
}
corpusMu.RUnlock()
minCover := inp.cover
for i := 0; i < 3; i++ {
allCover := execute1(pid, env, inp.p, &statExecTriage)
if len(allCover[inp.call]) == 0 {
// The call was not executed. Happens sometimes, reason unknown.
continue
}
coverMu.RLock()
cov := allCover[inp.call]
diff := cover.SymmetricDifference(inp.cover, cov)
minCover = cover.Intersection(minCover, cov)
updateFlakes := len(diff) != 0 && len(cover.Difference(diff, flakes)) != 0
coverMu.RUnlock()
if updateFlakes {
coverMu.Lock()
flakes = cover.Union(flakes, diff)
coverMu.Unlock()
}
}
stableNewCover := cover.Intersection(newCover, minCover)
if len(stableNewCover) == 0 {
return
}
inp.p, inp.call = prog.Minimize(inp.p, inp.call, func(p1 *prog.Prog, call1 int) bool {
allCover := execute1(pid, env, p1, &statExecMinimize)
coverMu.RLock()
defer coverMu.RUnlock()
if len(allCover[call1]) == 0 {
return false // The call was not executed.
}
cov := allCover[call1]
if len(cover.Intersection(stableNewCover, cov)) != len(stableNewCover) {
return false
}
minCover = cover.Intersection(minCover, cov)
return true
}, false)
inp.cover = minCover
atomic.AddUint64(&statNewInput, 1)
data := inp.p.Serialize()
Logf(2, "added new input for %v to corpus:\n%s", call.CallName, data)
a := &NewInputArgs{*flagName, RpcInput{call.CallName, data, inp.call, []uint32(inp.cover)}}
if err := manager.Call("Manager.NewInput", a, nil); err != nil {
panic(err)
}
corpusMu.Lock()
defer corpusMu.Unlock()
coverMu.Lock()
defer coverMu.Unlock()
corpusCover[call.CallID] = cover.Union(corpusCover[call.CallID], minCover)
corpus = append(corpus, inp.p)
corpusHashes[hash(data)] = struct{}{}
}
