Пример #1
0
// newBackend returns a new generic backend.
// It will start monitoring the backend at once
func newBackend(bec BackendConfig, serverHost, healthURL string) *backend {
	b := &backend{
		ServerHost: serverHost,
		HealthURL:  healthURL,
	}
	// Create a transport that is used for health checks.
	tr := &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   time.Duration(bec.HealthTimeout),
			KeepAlive: 0,
		}).Dial,
		DisableKeepAlives:  true,
		DisableCompression: true,
	}
	b.healthClient = &http.Client{Transport: tr}

	// Reset running stats.
	b.Stats.Latency = ewma.NewMovingAverage(float64(bec.LatencyAvg))
	b.Stats.FailureRate = ewma.NewMovingAverage(10)

	// Set up the backend transport.
	tr = &http.Transport{
		Dial: func(network, addr string) (net.Conn, error) {
			return net.DialTimeout(network, addr, time.Duration(bec.DialTimeout))
		},
		Proxy: http.ProxyFromEnvironment,
	}

	b.rt = newStatTP(tr)

	b.closeMonitor = make(chan struct{}, 0)
	go b.startMonitor()
	return b
}
Пример #2
0
// Run runs the given function robustly, catching and restarting on panics.
// The optional options are a rate limit in crashes per second, and a timeout.
// If the function panics more often than the rate limit, for longer than the
// timeout, then Run aborts and re-throws the panic. A third option controls
// whether to print the stack trace for panics that are intercepted.
func Run(function func(), options ...float64) int {
	rateLimit, timeout := 1.0, 1.0 // TODO

	// We use a moving average to compute the rate of errors per second.
	avg := ewma.NewMovingAverage(timeout)
	before := time.Now()
	var startAboveLimit time.Time
	var belowLimit bool = true
	var beforeTimeout = true
	var totalPanics = 0
	var oktorun bool = true

	for oktorun {
		func() {
			defer func() {
				localErr := recover()
				if localErr == nil {
					oktorun = false // The call to f() exited normally.
					return
				}

				totalPanics++
				after := time.Now()
				duration := after.Sub(before).Seconds()
				if duration > 0 {
					rate := 1.0 / duration
					avg.Add(rate)

					// Figure out whether we're above the rate limit and for how long
					if avg.Value() > rateLimit {
						if belowLimit {
							startAboveLimit = after
						}
						beforeTimeout =
							after.Before(startAboveLimit.Add(time.Second * time.Duration(timeout)))
						belowLimit = false
					} else {
						belowLimit = true
					}
				}
				before = after

				if !belowLimit && !beforeTimeout {
					panic(fmt.Sprintf("giving up after %d errors at %.2f/sec since %s",
						totalPanics, avg.Value(), startAboveLimit))
				}

				if len(options) > 2 && options[2] > 0 {
					fmt.Fprintf(os.Stdout, "%v\n%s\n", localErr, debug.Stack())
				}
			}()
			function()
			return
		}()

	}
	return totalPanics
}
Пример #3
0
func consume(queue *lang.Queue) {
	e := ewma.NewMovingAverage()
	for {
		old_len := queue.Len()
		time.Sleep(1000 * time.Millisecond)
		new_len := queue.Len()
		msg_cnt := float64(new_len - old_len)
		e.Add(msg_cnt)
		str := fmt.Sprintf("Msg cnt: %v | Msg send: %v | ewma: %v", new_len, msg_cnt, e.Value())
		fmt.Println(str)
	}
}
Пример #4
0
// NewAccount makes a Account reader for an object
func NewAccount(in io.ReadCloser, obj Object) *Account {
	acc := &Account{
		in:     in,
		size:   obj.Size(),
		name:   obj.Remote(),
		exit:   make(chan struct{}),
		avg:    ewma.NewMovingAverage(),
		lpTime: time.Now(),
	}
	go acc.averageLoop()
	Stats.inProgress.set(acc.name, acc)
	return acc
}
Пример #5
0
// NewRate creates an EWMA rate on the given timescale. Timescales at
// or below 2s are illegal and will cause a panic.
func NewRate(timescale time.Duration) *Rate {
	const tickInterval = time.Second
	if timescale <= 2*time.Second {
		panic(fmt.Sprintf("EWMA with per-second ticks makes no sense on timescale %s", timescale))
	}
	avgAge := float64(timescale) / float64(2*tickInterval)

	return &Rate{
		interval: tickInterval,
		nextT:    now(),
		wrapped:  ewma.NewMovingAverage(avgAge),
	}
}
Пример #6
0
// NewAccountSizeName makes a Account reader for an io.ReadCloser of
// the given size and name
func NewAccountSizeName(in io.ReadCloser, size int64, name string) *Account {
	acc := &Account{
		in:     in,
		size:   size,
		name:   name,
		exit:   make(chan struct{}),
		avg:    ewma.NewMovingAverage(),
		lpTime: time.Now(),
	}
	go acc.averageLoop()
	Stats.inProgress.set(acc.name, acc)
	return acc
}
Пример #7
0
// Run runs the given function robustly, catching and restarting on panics.
// Takes a RunOptions struct pointer as options, nil to use the default parameters.
func Run(function func(), opts *RunOptions) int {
	options := RunOptions{
		RateLimit: DefaultRateLimit,
		Timeout:   DefaultTimeout,
	}
	if opts != nil {
		options = *opts

		// Zero values for rate and timeout are mostly useless; so we turn to
		// defaults instead.
		if options.RateLimit == 0 {
			options.RateLimit = DefaultRateLimit
		}
		if options.Timeout == 0 {
			options.Timeout = DefaultTimeout
		}
	}

	// We use a moving average to compute the rate of errors per second.
	avg := ewma.NewMovingAverage(options.Timeout.Seconds())
	before := time.Now()
	var startAboveLimit time.Time
	var belowLimit bool = true
	var beforeTimeout = true
	var totalPanics = 0
	var oktorun bool = true

	for oktorun {
		func() {
			defer func() {
				localErr := recover()
				if localErr == nil {
					oktorun = false // The call to f() exited normally.
					return
				}

				totalPanics++
				after := time.Now()
				duration := after.Sub(before).Seconds()
				if duration > 0 {
					rate := 1.0 / duration
					avg.Add(rate)

					// Figure out whether we're above the rate limit and for how long
					if avg.Value() > options.RateLimit {
						if belowLimit {
							startAboveLimit = after
						}
						beforeTimeout =
							after.Before(startAboveLimit.Add(options.Timeout))
						belowLimit = false
					} else {
						belowLimit = true
					}
				}
				before = after

				if !belowLimit && !beforeTimeout {
					panic(fmt.Sprintf("giving up after %d errors at %.2f/sec since %s",
						totalPanics, avg.Value(), startAboveLimit))
				}

				if options.PrintStack {
					log.Printf("[robustly] %v\n%s\n", localErr, debug.Stack())
				}

				if options.RetryDelay > time.Nanosecond*0 {
					time.Sleep(options.RetryDelay)
				}
			}()
			function()
			return
		}()

	}
	return totalPanics
}