// doReq sends a request, with timeout options and retries, waits for response and returns it
func (c *client) doReq(req *Request, options ...Options) (*Response, errors.Error) {
if circuitbreaker.Open(req.service, req.endpoint) {
inst.Counter(1.0, fmt.Sprintf("client.error.%s.%s.circuitbroken", req.service, req.endpoint), 1)
log.Warnf("Broken Circuit for %s.%s", req.service, req.endpoint)
return nil, errors.CircuitBroken("com.hailocab.kernel.platform.circuitbreaker", "Circuit is open")
}
retries := c.defaults["retries"].(int)
var timeout time.Duration
timeoutSupplied := false
if len(options) == 1 {
if _, ok := options[0]["retries"]; ok {
retries = options[0]["retries"].(int)
}
if _, ok := options[0]["timeout"]; ok {
timeout = options[0]["timeout"].(time.Duration)
timeoutSupplied = true
}
}
// setup the response channel
rc := make(chan *Response, retries)
c.responses.add(req, rc)
defer c.responses.removeByRequest(req)
instPrefix := fmt.Sprintf("client.%s.%s", req.service, req.endpoint)
tAllRetries := time.Now()
for i := 1; i <= retries+1; i++ {
t := time.Now()
c.RLock()
con := c.listening
c.RUnlock()
if !con {
log.Debug("[Client] not yet listening, establishing now...")
ch := make(chan bool)
go c.listen(ch)
if online := <-ch; !online {
log.Error("[Client] Listener failed")
inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.client.listenfail", 1)
return nil, errors.InternalServerError("com.hailocab.kernel.platform.client.listenfail", "Listener failed")
}
log.Info("[Client] Listener online")
}
// figure out what timeout to use
if !timeoutSupplied {
timeout = c.timeout.Get(req.service, req.endpoint, i)
}
log.Tracef("[Client] Sync request attempt %d for %s using timeout %v", i, req.MessageID(), timeout)
// only bother sending the request if we are listening, otherwise allow to timeout
if err := raven.SendRequest(req, c.instanceID); err != nil {
log.Errorf("[Client] Failed to send request: %v", err)
}
select {
case payload := <-rc:
if payload.IsError() {
inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
errorProto := &pe.PlatformError{}
if err := payload.Unmarshal(errorProto); err != nil {
inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.badresponse", 1)
return nil, errors.BadResponse("com.hailocab.kernel.platform.badresponse", err.Error())
}
err := errors.FromProtobuf(errorProto)
inst.Counter(1.0, fmt.Sprintf("client.error.%s", err.Code()), 1)
circuitbreaker.Result(req.service, req.endpoint, err)
return nil, err
}
inst.Timing(1.0, fmt.Sprintf("%s.success", instPrefix), time.Since(t))
circuitbreaker.Result(req.service, req.endpoint, nil)
return payload, nil
case <-time.After(timeout):
// timeout
log.Errorf("[Client] Timeout talking to %s.%s after %v for %s", req.Service(), req.Endpoint(), timeout, req.MessageID())
inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
c.traceAttemptTimeout(req, i, timeout)
circuitbreaker.Result(req.service, req.endpoint, errors.Timeout("com.hailocab.kernel.platform.timeout",
fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
req.Service(),
req.Endpoint()))
}
}
inst.Timing(1.0, fmt.Sprintf("%s.error.timedOut", instPrefix), time.Since(tAllRetries))
inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.timeout", 1)
return nil, errors.Timeout(
"com.hailocab.kernel.platform.timeout",
fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
req.Service(),
req.Endpoint(),
)
}
func (suite *multiClientSuite) TestAnyErrorsIgnoring() {
cases := []struct {
errs map[string]errors.Error
types, codes []string
isErr bool
}{
{
errs: map[string]errors.Error{},
types: []string{},
codes: []string{},
isErr: false,
},
// allow nils
{
errs: map[string]errors.Error{},
types: nil,
codes: nil,
isErr: false,
},
// no errors = always false
{
errs: map[string]errors.Error{},
types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
codes: []string{"com.hailocab.service.foo.bar"},
isErr: false,
},
{
errs: map[string]errors.Error{},
types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
codes: []string{},
isErr: false,
},
{
errs: map[string]errors.Error{},
types: []string{},
codes: []string{"com.hailocab.service.foo.bar"},
isErr: false,
},
{
errs: map[string]errors.Error{},
types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
codes: nil,
isErr: false,
},
{
errs: map[string]errors.Error{},
types: nil,
codes: []string{"com.hailocab.service.foo.bar"},
isErr: false,
},
// types
{
errs: map[string]errors.Error{
"foo": errors.BadResponse("foo.bar", "ZOMG"),
},
types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
codes: []string{},
isErr: false,
},
{
errs: map[string]errors.Error{
"foo": errors.BadRequest("foo.bar", "ZOMG"),
},
types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
codes: []string{},
isErr: true,
},
{
errs: map[string]errors.Error{
"foo": errors.BadResponse("foo.bar", "ZOMG"),
"bar": errors.BadRequest("foo.bar", "ZOMG"),
},
types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
codes: []string{},
isErr: true,
},
// codes
{
errs: map[string]errors.Error{
"foo": errors.BadResponse("foo.bar", "ZOMG"),
},
types: nil,
codes: []string{"foo.bar"},
isErr: false,
},
{
errs: map[string]errors.Error{
"foo": errors.BadResponse("foo.bar", "ZOMG"),
},
types: nil,
codes: []string{"foo.bar.baz"},
isErr: true,
},
{
errs: map[string]errors.Error{
"foo": errors.BadResponse("foo.bar", "ZOMG"),
"bar": errors.BadResponse("foo.bar.baz", "ZOMG"),
},
types: nil,
codes: []string{"foo.bar"},
isErr: true,
},
}
cl := &defClient{
errors: &errorsImpl{},
}
for _, tc := range cases {
for uid, err := range tc.errs {
cl.errors.set(uid, &client.Request{}, err, nil)
}
res := cl.AnyErrorsIgnoring(tc.types, tc.codes)
suite.Assertions.Equal(tc.isErr, res, fmt.Sprintf("Wrong result for errors: %v - types: %v - codes: %v",
tc.errs, tc.types, tc.codes))
}
}
// ConfiguredHttpCaller with more explicit configuration options than simple HttpCaller
func ConfiguredHttpCaller(opts Options) Caller {
tp := &httpclient.Transport{
ConnectTimeout: durationOrDefault(opts.ConnectTimeout, 5*time.Second),
RequestTimeout: durationOrDefault(opts.RequestTimeout, 5*time.Second),
ResponseHeaderTimeout: durationOrDefault(opts.ResponseHeaderTimeout, 5*time.Second),
}
if opts.TlsSkipVerify {
tp.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
httpClient := &http.Client{Transport: tp}
return func(req *client.Request, rsp proto.Message) errors.Error {
u, err := url.Parse(opts.BaseUrl)
q := u.Query()
q.Set("session_id", req.SessionID())
q.Set("service", req.Service())
q.Set("endpoint", req.Endpoint())
u.Path = "/rpc"
u.RawQuery = q.Encode()
var httpReq *http.Request
// send JSON req content-type to thin API as form-encoded data
// send proto req content-type directly as bytes, with proto content type
if req.ContentType() == jsonContentType {
values := make(url.Values)
values.Set("service", req.Service())
values.Set("endpoint", req.Endpoint())
values.Set("request", string(req.Payload()))
httpReq, _ = http.NewRequest("POST", u.String(), bytes.NewReader([]byte(values.Encode())))
httpReq.Header.Set("Content-Type", formEncodedContentType)
} else {
httpReq, _ = http.NewRequest("POST", u.String(), bytes.NewReader(req.Payload()))
httpReq.Header.Set("Content-Type", protoContentType)
}
log.Debugf("[Multiclient] HTTP caller - calling '%s' : content-type '%s'", u.String(), req.ContentType())
httpRsp, err := httpClient.Do(httpReq)
if err != nil {
log.Warnf("[Multiclient] HTTP caller error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err)
return errors.InternalServerError("multiclienthttp.postform", fmt.Sprintf("Error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err))
}
if err != nil {
log.Warnf("[Multiclient] HTTP caller error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err)
return errors.InternalServerError("multiclienthttp.postform", fmt.Sprintf("Error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err))
}
defer httpRsp.Body.Close()
rspBody, err := ioutil.ReadAll(httpRsp.Body)
if err != nil {
return errors.BadResponse("multiclienthttp.readresponse", fmt.Sprintf("Error reading response bytes: %v", err))
}
// what status code?
if httpRsp.StatusCode != 200 {
// deal with error
e := &protoerror.PlatformError{}
var err error
if req.ContentType() == jsonContentType {
jsonErr := &errorBody{}
err = json.Unmarshal(rspBody, jsonErr)
e.Code = proto.String(jsonErr.DottedCode)
e.Context = jsonErr.Context
e.Description = proto.String(jsonErr.Payload)
e.HttpCode = proto.Uint32(uint32(httpRsp.StatusCode))
// this conversion is lossy, since the JSON response for errors, as crafted
// by the "thin API", does not currently include the error type, so we have
// to guess from HTTP status code, but there is no distinct code for "BAD_RESPONSE"
switch httpRsp.StatusCode {
case 400:
e.Type = protoerror.PlatformError_BAD_REQUEST.Enum()
case 403:
e.Type = protoerror.PlatformError_FORBIDDEN.Enum()
case 404:
e.Type = protoerror.PlatformError_NOT_FOUND.Enum()
case 500:
e.Type = protoerror.PlatformError_INTERNAL_SERVER_ERROR.Enum()
case 504:
e.Type = protoerror.PlatformError_TIMEOUT.Enum()
}
} else {
err = proto.Unmarshal(rspBody, e)
}
// some issue understanding error rsp
if err != nil {
return errors.BadResponse("multiclienthttp.unmarshalerr", fmt.Sprintf("Error unmarshaling error response '%s': %v", string(rspBody), err))
}
return errors.FromProtobuf(e)
}
// unmarshal response
if req.ContentType() == jsonContentType {
err = json.Unmarshal(rspBody, rsp)
} else {
err = proto.Unmarshal(rspBody, rsp)
}
if err != nil {
return errors.BadResponse("multiclienthttp.unmarshal", fmt.Sprintf("Error unmarshaling response: %v", err))
}
return nil
}
}