func (s *CircuitBreakerTestSuit) TestOpenCircuit() {
	s.circuit = NewDefaultCircuit(defaultOptions)

	// This should trip the circuit breaker
	for i := 0; i < 100; i++ {
		s.circuit.Result(errors.Timeout("code", "description"))
	}

	s.True(s.circuit.Open())
}
func (s *CircuitBreakerTestSuit) TestOpenToClosedCircuit() {
	s.circuit = NewDefaultCircuit(defaultOptions)

	// Trip the circuit breaker
	for i := 0; i < 100; i++ {
		s.circuit.Result(errors.Timeout("code", "description"))
	}
	s.True(s.circuit.Open())

	// Wait and do a successful call. Circuit should be closed.
	s.clock.Add(101 * time.Millisecond)
	s.False(s.circuit.Open())
	s.circuit.Result(nil)
	s.False(s.circuit.Open())
}
func TestErrorResponse(t *testing.T) {
	request := &Request{
		delivery: amqp.Delivery{
			ContentType: "application/octetstream",
		},
	}

	e := errors.Timeout("com.something", "Something timed out")
	response, err := ErrorResponse(request, e)

	if err != nil {
		t.Fatalf("Failed to create error response: %v", err)
	}

	if response.MessageType() != "error" {
		t.Errorf(`Wrong message type "%v", expecting "error"`, response.MessageType())
	}
}
Beispiel #4
0
// doReq sends a request, with timeout options and retries, waits for response and returns it
func (c *client) doReq(req *Request, options ...Options) (*Response, errors.Error) {

	if circuitbreaker.Open(req.service, req.endpoint) {
		inst.Counter(1.0, fmt.Sprintf("client.error.%s.%s.circuitbroken", req.service, req.endpoint), 1)
		log.Warnf("Broken Circuit for %s.%s", req.service, req.endpoint)
		return nil, errors.CircuitBroken("com.hailocab.kernel.platform.circuitbreaker", "Circuit is open")
	}

	retries := c.defaults["retries"].(int)
	var timeout time.Duration
	timeoutSupplied := false
	if len(options) == 1 {
		if _, ok := options[0]["retries"]; ok {
			retries = options[0]["retries"].(int)
		}
		if _, ok := options[0]["timeout"]; ok {
			timeout = options[0]["timeout"].(time.Duration)
			timeoutSupplied = true
		}
	}

	// setup the response channel
	rc := make(chan *Response, retries)
	c.responses.add(req, rc)
	defer c.responses.removeByRequest(req)

	instPrefix := fmt.Sprintf("client.%s.%s", req.service, req.endpoint)
	tAllRetries := time.Now()

	for i := 1; i <= retries+1; i++ {
		t := time.Now()

		c.RLock()
		con := c.listening
		c.RUnlock()
		if !con {
			log.Debug("[Client] not yet listening, establishing now...")
			ch := make(chan bool)
			go c.listen(ch)
			if online := <-ch; !online {
				log.Error("[Client] Listener failed")
				inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
				inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.client.listenfail", 1)
				return nil, errors.InternalServerError("com.hailocab.kernel.platform.client.listenfail", "Listener failed")
			}

			log.Info("[Client] Listener online")
		}

		// figure out what timeout to use
		if !timeoutSupplied {
			timeout = c.timeout.Get(req.service, req.endpoint, i)
		}
		log.Tracef("[Client] Sync request attempt %d for %s using timeout %v", i, req.MessageID(), timeout)

		// only bother sending the request if we are listening, otherwise allow to timeout
		if err := raven.SendRequest(req, c.instanceID); err != nil {
			log.Errorf("[Client] Failed to send request: %v", err)
		}

		select {
		case payload := <-rc:
			if payload.IsError() {
				inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))

				errorProto := &pe.PlatformError{}
				if err := payload.Unmarshal(errorProto); err != nil {
					inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.badresponse", 1)
					return nil, errors.BadResponse("com.hailocab.kernel.platform.badresponse", err.Error())
				}

				err := errors.FromProtobuf(errorProto)
				inst.Counter(1.0, fmt.Sprintf("client.error.%s", err.Code()), 1)

				circuitbreaker.Result(req.service, req.endpoint, err)

				return nil, err
			}

			inst.Timing(1.0, fmt.Sprintf("%s.success", instPrefix), time.Since(t))
			circuitbreaker.Result(req.service, req.endpoint, nil)
			return payload, nil
		case <-time.After(timeout):
			// timeout
			log.Errorf("[Client] Timeout talking to %s.%s after %v for %s", req.Service(), req.Endpoint(), timeout, req.MessageID())
			inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
			c.traceAttemptTimeout(req, i, timeout)

			circuitbreaker.Result(req.service, req.endpoint, errors.Timeout("com.hailocab.kernel.platform.timeout",
				fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
				req.Service(),
				req.Endpoint()))
		}
	}

	inst.Timing(1.0, fmt.Sprintf("%s.error.timedOut", instPrefix), time.Since(tAllRetries))
	inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.timeout", 1)

	return nil, errors.Timeout(
		"com.hailocab.kernel.platform.timeout",
		fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
		req.Service(),
		req.Endpoint(),
	)
}
func (s *CircuitBreakerTestSuit) TestCircuitConfig() {
	service := "com.hailocab.test.cruft"
	endpoint := "testendpoint"

	// Set default timeout to 100 ms
	config.Load(bytes.NewBuffer([]byte(`{
		"hailo": {
			"platform": {
				"circuitbreaker": {
					"initialIntervalMs": 100
				}
			}
		}
	}`)))

	// Let config propagate -- crufty :(
	time.Sleep(50 * time.Millisecond)

	// Circuit is initially closed
	s.False(Open(service, endpoint))

	// Trip circuit and check
	for i := 0; i < 100; i++ {
		Result(service, endpoint, errors.Timeout("code", "description"))
	}
	s.True(Open(service, endpoint))

	// Wait for circuit to half-open
	s.clock.Add(51 * time.Millisecond)
	s.True(Open(service, endpoint), "Circuit should be open after 51ms")
	s.clock.Add(50 * time.Millisecond)
	s.False(Open(service, endpoint), "Circuit should be closed after 101ms")

	// Set new interval
	s.NoError(config.Load(bytes.NewBuffer([]byte(`{
		"hailo": {
			"platform": {
				"circuitbreaker": {
					"initialIntervalMs": 50,
					"endpoints": {
						"com.hailocab.test.cruft": {
							"testendpoint": {
								"initialIntervalMs": 90
							}
						}
					}
				}
			}
		}
	}`))))

	// Let config propagate -- crufty :(
	time.Sleep(50 * time.Millisecond)

	// Check circuit is closed again
	s.False(Open(service, endpoint))

	// Trip circuit and check
	for i := 0; i < 100; i++ {
		Result(service, endpoint, errors.Timeout("code", "description"))
	}
	s.True(Open(service, endpoint))

	// Wait for circuit to half-open
	s.clock.Add(51 * time.Millisecond)
	s.True(Open(service, endpoint), "Circuit should be open after 51ms")
	s.clock.Add(40 * time.Millisecond)
	s.False(Open(service, endpoint), "Circuit should be closed after 91ms")
}