// newBackend returns a new generic backend.
// It will start monitoring the backend at once
func newBackend(bec BackendConfig, serverHost, healthURL string) *backend {
b := &backend{
ServerHost: serverHost,
HealthURL: healthURL,
}
// Create a transport that is used for health checks.
tr := &http.Transport{
Dial: (&net.Dialer{
Timeout: time.Duration(bec.HealthTimeout),
KeepAlive: 0,
}).Dial,
DisableKeepAlives: true,
DisableCompression: true,
}
b.healthClient = &http.Client{Transport: tr}
// Reset running stats.
b.Stats.Latency = ewma.NewMovingAverage(float64(bec.LatencyAvg))
b.Stats.FailureRate = ewma.NewMovingAverage(10)
// Set up the backend transport.
tr = &http.Transport{
Dial: func(network, addr string) (net.Conn, error) {
return net.DialTimeout(network, addr, time.Duration(bec.DialTimeout))
},
Proxy: http.ProxyFromEnvironment,
}
b.rt = newStatTP(tr)
b.closeMonitor = make(chan struct{}, 0)
go b.startMonitor()
return b
}
// Run runs the given function robustly, catching and restarting on panics.
// The optional options are a rate limit in crashes per second, and a timeout.
// If the function panics more often than the rate limit, for longer than the
// timeout, then Run aborts and re-throws the panic. A third option controls
// whether to print the stack trace for panics that are intercepted.
func Run(function func(), options ...float64) int {
rateLimit, timeout := 1.0, 1.0 // TODO
// We use a moving average to compute the rate of errors per second.
avg := ewma.NewMovingAverage(timeout)
before := time.Now()
var startAboveLimit time.Time
var belowLimit bool = true
var beforeTimeout = true
var totalPanics = 0
var oktorun bool = true
for oktorun {
func() {
defer func() {
localErr := recover()
if localErr == nil {
oktorun = false // The call to f() exited normally.
return
}
totalPanics++
after := time.Now()
duration := after.Sub(before).Seconds()
if duration > 0 {
rate := 1.0 / duration
avg.Add(rate)
// Figure out whether we're above the rate limit and for how long
if avg.Value() > rateLimit {
if belowLimit {
startAboveLimit = after
}
beforeTimeout =
after.Before(startAboveLimit.Add(time.Second * time.Duration(timeout)))
belowLimit = false
} else {
belowLimit = true
}
}
before = after
if !belowLimit && !beforeTimeout {
panic(fmt.Sprintf("giving up after %d errors at %.2f/sec since %s",
totalPanics, avg.Value(), startAboveLimit))
}
if len(options) > 2 && options[2] > 0 {
fmt.Fprintf(os.Stdout, "%v\n%s\n", localErr, debug.Stack())
}
}()
function()
return
}()
}
return totalPanics
}
func consume(queue *lang.Queue) {
e := ewma.NewMovingAverage()
for {
old_len := queue.Len()
time.Sleep(1000 * time.Millisecond)
new_len := queue.Len()
msg_cnt := float64(new_len - old_len)
e.Add(msg_cnt)
str := fmt.Sprintf("Msg cnt: %v | Msg send: %v | ewma: %v", new_len, msg_cnt, e.Value())
fmt.Println(str)
}
}
// NewAccount makes a Account reader for an object
func NewAccount(in io.ReadCloser, obj Object) *Account {
acc := &Account{
in: in,
size: obj.Size(),
name: obj.Remote(),
exit: make(chan struct{}),
avg: ewma.NewMovingAverage(),
lpTime: time.Now(),
}
go acc.averageLoop()
Stats.inProgress.set(acc.name, acc)
return acc
}
// NewRate creates an EWMA rate on the given timescale. Timescales at
// or below 2s are illegal and will cause a panic.
func NewRate(timescale time.Duration) *Rate {
const tickInterval = time.Second
if timescale <= 2*time.Second {
panic(fmt.Sprintf("EWMA with per-second ticks makes no sense on timescale %s", timescale))
}
avgAge := float64(timescale) / float64(2*tickInterval)
return &Rate{
interval: tickInterval,
nextT: now(),
wrapped: ewma.NewMovingAverage(avgAge),
}
}
// NewAccountSizeName makes a Account reader for an io.ReadCloser of
// the given size and name
func NewAccountSizeName(in io.ReadCloser, size int64, name string) *Account {
acc := &Account{
in: in,
size: size,
name: name,
exit: make(chan struct{}),
avg: ewma.NewMovingAverage(),
lpTime: time.Now(),
}
go acc.averageLoop()
Stats.inProgress.set(acc.name, acc)
return acc
}
// Run runs the given function robustly, catching and restarting on panics.
// Takes a RunOptions struct pointer as options, nil to use the default parameters.
func Run(function func(), opts *RunOptions) int {
options := RunOptions{
RateLimit: DefaultRateLimit,
Timeout: DefaultTimeout,
}
if opts != nil {
options = *opts
// Zero values for rate and timeout are mostly useless; so we turn to
// defaults instead.
if options.RateLimit == 0 {
options.RateLimit = DefaultRateLimit
}
if options.Timeout == 0 {
options.Timeout = DefaultTimeout
}
}
// We use a moving average to compute the rate of errors per second.
avg := ewma.NewMovingAverage(options.Timeout.Seconds())
before := time.Now()
var startAboveLimit time.Time
var belowLimit bool = true
var beforeTimeout = true
var totalPanics = 0
var oktorun bool = true
for oktorun {
func() {
defer func() {
localErr := recover()
if localErr == nil {
oktorun = false // The call to f() exited normally.
return
}
totalPanics++
after := time.Now()
duration := after.Sub(before).Seconds()
if duration > 0 {
rate := 1.0 / duration
avg.Add(rate)
// Figure out whether we're above the rate limit and for how long
if avg.Value() > options.RateLimit {
if belowLimit {
startAboveLimit = after
}
beforeTimeout =
after.Before(startAboveLimit.Add(options.Timeout))
belowLimit = false
} else {
belowLimit = true
}
}
before = after
if !belowLimit && !beforeTimeout {
panic(fmt.Sprintf("giving up after %d errors at %.2f/sec since %s",
totalPanics, avg.Value(), startAboveLimit))
}
if options.PrintStack {
log.Printf("[robustly] %v\n%s\n", localErr, debug.Stack())
}
if options.RetryDelay > time.Nanosecond*0 {
time.Sleep(options.RetryDelay)
}
}()
function()
return
}()
}
return totalPanics
}