func (s *CircuitBreakerTestSuit) TestOpenCircuit() {
s.circuit = NewDefaultCircuit(defaultOptions)
// This should trip the circuit breaker
for i := 0; i < 100; i++ {
s.circuit.Result(errors.Timeout("code", "description"))
}
s.True(s.circuit.Open())
}
func (s *CircuitBreakerTestSuit) TestOpenToClosedCircuit() {
s.circuit = NewDefaultCircuit(defaultOptions)
// Trip the circuit breaker
for i := 0; i < 100; i++ {
s.circuit.Result(errors.Timeout("code", "description"))
}
s.True(s.circuit.Open())
// Wait and do a successful call. Circuit should be closed.
s.clock.Add(101 * time.Millisecond)
s.False(s.circuit.Open())
s.circuit.Result(nil)
s.False(s.circuit.Open())
}
// doReq sends a request, with timeout options and retries, waits for response and returns it
func (c *client) doReq(req *Request, options ...Options) (*Response, errors.Error) {
if circuitbreaker.Open(req.service, req.endpoint) {
inst.Counter(1.0, fmt.Sprintf("client.error.%s.%s.circuitbroken", req.service, req.endpoint), 1)
log.Warnf("Broken Circuit for %s.%s", req.service, req.endpoint)
return nil, errors.CircuitBroken("com.hailocab.kernel.platform.circuitbreaker", "Circuit is open")
}
retries := c.defaults["retries"].(int)
var timeout time.Duration
timeoutSupplied := false
if len(options) == 1 {
if _, ok := options[0]["retries"]; ok {
retries = options[0]["retries"].(int)
}
if _, ok := options[0]["timeout"]; ok {
timeout = options[0]["timeout"].(time.Duration)
timeoutSupplied = true
}
}
// setup the response channel
rc := make(chan *Response, retries)
c.responses.add(req, rc)
defer c.responses.removeByRequest(req)
instPrefix := fmt.Sprintf("client.%s.%s", req.service, req.endpoint)
tAllRetries := time.Now()
for i := 1; i <= retries+1; i++ {
t := time.Now()
c.RLock()
con := c.listening
c.RUnlock()
if !con {
log.Debug("[Client] not yet listening, establishing now...")
ch := make(chan bool)
go c.listen(ch)
if online := <-ch; !online {
log.Error("[Client] Listener failed")
inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.client.listenfail", 1)
return nil, errors.InternalServerError("com.hailocab.kernel.platform.client.listenfail", "Listener failed")
}
log.Info("[Client] Listener online")
}
// figure out what timeout to use
if !timeoutSupplied {
timeout = c.timeout.Get(req.service, req.endpoint, i)
}
log.Tracef("[Client] Sync request attempt %d for %s using timeout %v", i, req.MessageID(), timeout)
// only bother sending the request if we are listening, otherwise allow to timeout
if err := raven.SendRequest(req, c.instanceID); err != nil {
log.Errorf("[Client] Failed to send request: %v", err)
}
select {
case payload := <-rc:
if payload.IsError() {
inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
errorProto := &pe.PlatformError{}
if err := payload.Unmarshal(errorProto); err != nil {
inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.badresponse", 1)
return nil, errors.BadResponse("com.hailocab.kernel.platform.badresponse", err.Error())
}
err := errors.FromProtobuf(errorProto)
inst.Counter(1.0, fmt.Sprintf("client.error.%s", err.Code()), 1)
circuitbreaker.Result(req.service, req.endpoint, err)
return nil, err
}
inst.Timing(1.0, fmt.Sprintf("%s.success", instPrefix), time.Since(t))
circuitbreaker.Result(req.service, req.endpoint, nil)
return payload, nil
case <-time.After(timeout):
// timeout
log.Errorf("[Client] Timeout talking to %s.%s after %v for %s", req.Service(), req.Endpoint(), timeout, req.MessageID())
inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
c.traceAttemptTimeout(req, i, timeout)
circuitbreaker.Result(req.service, req.endpoint, errors.Timeout("com.hailocab.kernel.platform.timeout",
fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
req.Service(),
req.Endpoint()))
}
}
inst.Timing(1.0, fmt.Sprintf("%s.error.timedOut", instPrefix), time.Since(tAllRetries))
inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.timeout", 1)
return nil, errors.Timeout(
"com.hailocab.kernel.platform.timeout",
fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
req.Service(),
req.Endpoint(),
)
}
func (s *CircuitBreakerTestSuit) TestCircuitConfig() {
service := "com.hailocab.test.cruft"
endpoint := "testendpoint"
// Set default timeout to 100 ms
config.Load(bytes.NewBuffer([]byte(`{
"hailo": {
"platform": {
"circuitbreaker": {
"initialIntervalMs": 100
}
}
}
}`)))
// Let config propagate -- crufty :(
time.Sleep(50 * time.Millisecond)
// Circuit is initially closed
s.False(Open(service, endpoint))
// Trip circuit and check
for i := 0; i < 100; i++ {
Result(service, endpoint, errors.Timeout("code", "description"))
}
s.True(Open(service, endpoint))
// Wait for circuit to half-open
s.clock.Add(51 * time.Millisecond)
s.True(Open(service, endpoint), "Circuit should be open after 51ms")
s.clock.Add(50 * time.Millisecond)
s.False(Open(service, endpoint), "Circuit should be closed after 101ms")
// Set new interval
s.NoError(config.Load(bytes.NewBuffer([]byte(`{
"hailo": {
"platform": {
"circuitbreaker": {
"initialIntervalMs": 50,
"endpoints": {
"com.hailocab.test.cruft": {
"testendpoint": {
"initialIntervalMs": 90
}
}
}
}
}
}
}`))))
// Let config propagate -- crufty :(
time.Sleep(50 * time.Millisecond)
// Check circuit is closed again
s.False(Open(service, endpoint))
// Trip circuit and check
for i := 0; i < 100; i++ {
Result(service, endpoint, errors.Timeout("code", "description"))
}
s.True(Open(service, endpoint))
// Wait for circuit to half-open
s.clock.Add(51 * time.Millisecond)
s.True(Open(service, endpoint), "Circuit should be open after 51ms")
s.clock.Add(40 * time.Millisecond)
s.False(Open(service, endpoint), "Circuit should be closed after 91ms")
}