// programLoop runs the program being debugged to completion. When a breakpoint's
// conditions are satisfied, it sends an Update RPC to the Debuglet Controller.
// The function returns when the program exits and all Update RPCs have finished.
func programLoop(c *debuglet.Controller, bs *breakpoints.BreakpointStore, prog debug.Program) {
var wg sync.WaitGroup
for {
// Run the program until it hits a breakpoint or exits.
status, err := prog.Resume()
if err != nil {
break
}
// Get the breakpoints at this address whose conditions were satisfied,
// and remove the ones that aren't logpoints.
bps := bs.BreakpointsAtPC(status.PC)
bps = bpsWithConditionSatisfied(bps, prog)
for _, bp := range bps {
if bp.Action != "LOG" {
bs.RemoveBreakpoint(bp)
}
}
if len(bps) == 0 {
continue
}
// Evaluate expressions and get the stack.
vc := valuecollector.NewCollector(prog, maxCapturedVariables)
needStackFrames := false
for _, bp := range bps {
// If evaluating bp's condition didn't return an error, evaluate bp's
// expressions, and later get the stack frames.
if bp.Status == nil {
bp.EvaluatedExpressions = expressionValues(bp.Expressions, prog, vc)
needStackFrames = true
}
}
var (
stack []*cd.StackFrame
stackFramesStatusMessage *cd.StatusMessage
)
if needStackFrames {
stack, stackFramesStatusMessage = stackFrames(prog, vc)
}
// Read variable values from the program.
variableTable := vc.ReadValues()
// Start a goroutine to send updates to the Debuglet Controller or write
// to logs, concurrently with resuming the program.
// TODO: retry Update on failure.
for _, bp := range bps {
wg.Add(1)
switch bp.Action {
case "LOG":
go func(format string, evaluatedExpressions []*cd.Variable) {
s := valuecollector.LogString(format, evaluatedExpressions, variableTable)
log.Print(s)
wg.Done()
}(bp.LogMessageFormat, bp.EvaluatedExpressions)
bp.Status = nil
bp.EvaluatedExpressions = nil
default:
go func(bp *cd.Breakpoint) {
defer wg.Done()
bp.IsFinalState = true
if bp.Status == nil {
// If evaluating bp's condition didn't return an error, include the
// stack frames, variable table, and any status message produced when
// getting the stack frames.
bp.StackFrames = stack
bp.VariableTable = variableTable
bp.Status = stackFramesStatusMessage
}
if err := c.Update(bp.Id, bp); err != nil {
log.Printf("Failed to send breakpoint update for %s: %s", bp.Id, err)
}
}(bp)
}
}
}
// Wait for all updates to finish before returning.
wg.Wait()
}
// breakpointListLoop repeatedly calls the Debuglet Controller's List RPC, and
// passes the results to the BreakpointStore so it can set and unset breakpoints
// in the program.
//
// After the first List call finishes, ch is closed.
func breakpointListLoop(c *debuglet.Controller, bs *breakpoints.BreakpointStore, first chan bool) {
const (
avgTimeBetweenCalls = time.Second
errorDelay = 5 * time.Second
)
// randomDuration returns a random duration with expected value avg.
randomDuration := func(avg time.Duration) time.Duration {
return time.Duration(rand.Int63n(int64(2*avg + 1)))
}
var consecutiveFailures uint
for {
callStart := time.Now()
resp, err := c.List()
if err != nil && err != debuglet.ErrListUnchanged {
log.Printf("Debuglet controller server error: %v", err)
}
if err == nil {
bs.ProcessBreakpointList(resp.Breakpoints)
}
if first != nil {
// We've finished one call to List and set any breakpoints we received.
close(first)
first = nil
}
// Asynchronously send updates for any breakpoints that caused an error when
// the BreakpointStore tried to process them. We don't wait for the update
// to finish before the program can exit, as we do for normal updates.
errorBps := bs.ErrorBreakpoints()
for _, bp := range errorBps {
go func(bp *cd.Breakpoint) {
if err := c.Update(bp.Id, bp); err != nil {
log.Printf("Failed to send breakpoint update for %s: %s", bp.Id, err)
}
}(bp)
}
// Make the next call not too soon after the one we just did.
delay := randomDuration(avgTimeBetweenCalls)
// If the call returned an error other than ErrListUnchanged, wait longer.
if err != nil && err != debuglet.ErrListUnchanged {
// Wait twice as long after each consecutive failure, to a maximum of 16x.
delay += randomDuration(errorDelay * (1 << consecutiveFailures))
if consecutiveFailures < 4 {
consecutiveFailures++
}
} else {
consecutiveFailures = 0
}
// Sleep until we reach time callStart+delay. If we've already passed that
// time, time.Sleep will return immediately -- this should be the common
// case, since the server will delay responding to List for a while when
// there are no changes to report.
time.Sleep(callStart.Add(delay).Sub(time.Now()))
}
}