// breakpointListLoop repeatedly calls the Debuglet Controller's List RPC, and // passes the results to the BreakpointStore so it can set and unset breakpoints // in the program. // // After the first List call finishes, ch is closed. func breakpointListLoop(c *debuglet.Controller, bs *breakpoints.BreakpointStore, first chan bool) { const ( avgTimeBetweenCalls = time.Second errorDelay = 5 * time.Second ) // randomDuration returns a random duration with expected value avg. randomDuration := func(avg time.Duration) time.Duration { return time.Duration(rand.Int63n(int64(2*avg + 1))) } var consecutiveFailures uint for { callStart := time.Now() resp, err := c.List() if err != nil && err != debuglet.ErrListUnchanged { log.Printf("Debuglet controller server error: %v", err) } if err == nil { bs.ProcessBreakpointList(resp.Breakpoints) } if first != nil { // We've finished one call to List and set any breakpoints we received. close(first) first = nil } // Asynchronously send updates for any breakpoints that caused an error when // the BreakpointStore tried to process them. We don't wait for the update // to finish before the program can exit, as we do for normal updates. errorBps := bs.ErrorBreakpoints() for _, bp := range errorBps { go func(bp *cd.Breakpoint) { if err := c.Update(bp.Id, bp); err != nil { log.Printf("Failed to send breakpoint update for %s: %s", bp.Id, err) } }(bp) } // Make the next call not too soon after the one we just did. delay := randomDuration(avgTimeBetweenCalls) // If the call returned an error other than ErrListUnchanged, wait longer. if err != nil && err != debuglet.ErrListUnchanged { // Wait twice as long after each consecutive failure, to a maximum of 16x. delay += randomDuration(errorDelay * (1 << consecutiveFailures)) if consecutiveFailures < 4 { consecutiveFailures++ } } else { consecutiveFailures = 0 } // Sleep until we reach time callStart+delay. If we've already passed that // time, time.Sleep will return immediately -- this should be the common // case, since the server will delay responding to List for a while when // there are no changes to report. time.Sleep(callStart.Add(delay).Sub(time.Now())) } }