// breakpointListLoop repeatedly calls the Debuglet Controller's List RPC, and
// passes the results to the BreakpointStore so it can set and unset breakpoints
// in the program.
//
// After the first List call finishes, ch is closed.
func breakpointListLoop(c *debuglet.Controller, bs *breakpoints.BreakpointStore, first chan bool) {
const (
avgTimeBetweenCalls = time.Second
errorDelay = 5 * time.Second
)
// randomDuration returns a random duration with expected value avg.
randomDuration := func(avg time.Duration) time.Duration {
return time.Duration(rand.Int63n(int64(2*avg + 1)))
}
var consecutiveFailures uint
for {
callStart := time.Now()
resp, err := c.List()
if err != nil && err != debuglet.ErrListUnchanged {
log.Printf("Debuglet controller server error: %v", err)
}
if err == nil {
bs.ProcessBreakpointList(resp.Breakpoints)
}
if first != nil {
// We've finished one call to List and set any breakpoints we received.
close(first)
first = nil
}
// Asynchronously send updates for any breakpoints that caused an error when
// the BreakpointStore tried to process them. We don't wait for the update
// to finish before the program can exit, as we do for normal updates.
errorBps := bs.ErrorBreakpoints()
for _, bp := range errorBps {
go func(bp *cd.Breakpoint) {
if err := c.Update(bp.Id, bp); err != nil {
log.Printf("Failed to send breakpoint update for %s: %s", bp.Id, err)
}
}(bp)
}
// Make the next call not too soon after the one we just did.
delay := randomDuration(avgTimeBetweenCalls)
// If the call returned an error other than ErrListUnchanged, wait longer.
if err != nil && err != debuglet.ErrListUnchanged {
// Wait twice as long after each consecutive failure, to a maximum of 16x.
delay += randomDuration(errorDelay * (1 << consecutiveFailures))
if consecutiveFailures < 4 {
consecutiveFailures++
}
} else {
consecutiveFailures = 0
}
// Sleep until we reach time callStart+delay. If we've already passed that
// time, time.Sleep will return immediately -- this should be the common
// case, since the server will delay responding to List for a while when
// there are no changes to report.
time.Sleep(callStart.Add(delay).Sub(time.Now()))
}
}
