// tokenConstrainedMiddleware limits the max concurrent requests handled per caller
func tokenConstrainedMiddleware(ep *Endpoint, h Handler) Handler {
return func(req *Request) (proto.Message, errors.Error) {
callerName := req.From()
if callerName == "" {
callerName = "unknown"
}
tokenBucketName := fmt.Sprintf("server.tokens.%s", callerName)
reqsBucketName := fmt.Sprintf("server.inflightrequests.%s", callerName)
tokC := tokensChan(callerName)
select {
case t := <-tokC:
defer func() {
atomic.AddUint64(&inFlightRequests, ^uint64(0)) // This is actually a subtraction
tokC <- t // Return the token to the pool
}()
nowInFlight := atomic.AddUint64(&inFlightRequests, 1) // Update active request counters
inst.Gauge(1.0, tokenBucketName, len(tokC))
inst.Gauge(1.0, reqsBucketName, int(nowInFlight))
return h(req)
case <-time.After(time.Duration(ep.Mean) * time.Millisecond):
inst.Gauge(1.0, tokenBucketName, len(tokC))
inst.Counter(1.0, "server.error.capacity", 1)
return nil, errors.InternalServerError("com.hailocab.kernel.server.capacity",
fmt.Sprintf("Server %v out of capacity", Name))
}
}
}
// AddScopedReq adds a server-scoped request (from the server request `from`) to our multi-client
// with the `uid` that uniquely identifies the request within the group (for getting response from `Outcome`)
func (c *defClient) AddScopedReq(sr *ScopedReq) MultiClient {
c.Lock()
defer c.Unlock()
if _, exists := c.requests[sr.Uid]; exists {
panic(fmt.Sprintf("Cannot add scoped request with UID '%v' - already exists within this MultiClient", sr.Uid))
}
from := sr.From
if from == nil {
from = c.defaultFromScope
}
var clientReq *client.Request
var err error
// if no from, just use normal client request
if from == nil {
clientReq, err = client.NewRequest(sr.Service, sr.Endpoint, sr.Req)
} else {
clientReq, err = from.ScopedRequest(sr.Service, sr.Endpoint, sr.Req)
}
c.requests[sr.Uid] = clientReq
c.responses[sr.Uid] = sr.Rsp
if err != nil {
c.errors.set(sr.Uid, clientReq,
errors.InternalServerError("com.hailocab.kernel.multirequest.badrequest", err.Error()), from)
} else {
clientReq.SetOptions(sr.Options)
}
return c
}
func (suite *errorsImplSuite) SetupTest() {
suite.Suite.SetupTest()
suite.rawErrs = map[string]errors.Error{
"uid1": errors.InternalServerError("com.foo.uid1", "uid1"),
"uid2": errors.InternalServerError("com.foo.uid2", "uid2"),
"uid3": errors.InternalServerError("com.foo.uid2", "uid3"), // Same service uid as uid2
}
suite.errs = &errorsImpl{}
for uid, err := range suite.rawErrs {
req, reqErr := client.NewJsonRequest(err.Code(), err.Description(), nil)
if reqErr != nil {
panic(reqErr)
}
suite.errs.set(uid, req, err, nil)
}
}
// TestRecoverSessionSad tests sad case (login service has some fatal error)
func (suite *sessionRecoverySuite) TestRecoverSessionSad() {
t := suite.T()
scope := New().(*realScope)
scope.userCache = newTestCache()
mock := multiclient.NewMock()
stub := &multiclient.Stub{
Service: loginService,
Endpoint: readSessionEndpoint,
Error: errors.InternalServerError("com.hailocab.service.login.foo", "Things are foo barred"),
}
mock.Stub(stub)
multiclient.SetCaller(mock.Caller())
err := scope.RecoverSession(testSessId)
if err == nil {
t.Error("Expecting recovery error, because login call should fail.")
}
if scope.IsAuth() {
t.Error("Expecting scope to be IsAuth==false after failed recovery")
}
if scope.HasTriedAuth() {
t.Error("Expecting scope to have HasTriedAuth()==false after _failed_ recovery")
}
// verify we made correct request(s)
if stub.CountCalls() != 1 {
t.Fatalf("Expecting 1 call to readsession; got %v", stub.CountCalls())
}
}
// jsonschemaHandler returns all registered endpoints in json schema format as per ITF draft4
// http://json-schema.org/latest/json-schema-core.html
func jsonschemaHandler(req *Request) (proto.Message, errors.Error) {
// Get all endpoints
request := req.Data().(*jsonschemaproto.Request)
endpoint := request.GetEndpoint()
endpoints := reg.iterate()
schemas := make([]*jsonschema.JsonSchema, 0)
for _, ep := range endpoints {
if endpoint != "" && endpoint != ep.GetName() {
continue
}
schema, err := marshalEndpoint(ep)
if err == nil && schema != nil {
schemas = append(schemas, schema)
}
}
rsp, err := json.Marshal(schemas)
if err != nil {
return nil, errors.InternalServerError("com.hailocab.kernel.marshal.error", fmt.Sprintf("Unable to unmarshal response data: %v", err.Error()))
}
return &jsonschemaproto.Response{
Jsonschema: proto.String(string(rsp)),
}, nil
}
func (m *MockClient) CustomReq(req *Request, options ...Options) (*Response, hailo_errors.Error) {
if matchedRsp := m.getMatchingRequestExpectation(req); matchedRsp != nil {
marshalledJson, err := json.Marshal(*matchedRsp)
if err != nil {
return nil, hailo_errors.InternalServerError("testing.is.fubard", "Can't marshall response to JSON")
}
resp := amqp.Delivery{
ContentType: "application/json",
Body: marshalledJson,
MessageId: "oh-what-a-lovely-test",
CorrelationId: "oh-what-a-lovely-test",
Headers: map[string](interface{}){},
}
return &Response{resp}, nil
} else {
log.Warnf("[Service client mock] Couldn't match response for %s:%s", req.Service(), req.Endpoint())
return nil, hailo_errors.InternalServerError("com.hailocab.kernel.platform.nilresponse", "Nil response")
}
}
func (m *MockClient) Req(req *Request, rsp proto.Message, options ...Options) hailo_errors.Error {
if matchedRsp := m.getMatchingRequestExpectation(req); matchedRsp != nil {
// Marshall to JSON and back again, to get it into rsp (which is passed by value). This is fine; these are
// marshalled and unmarshalled to JSON during normal operation anyway.
marshalledJson, err := json.Marshal(*matchedRsp)
if err != nil {
return hailo_errors.InternalServerError("testing.is.fubard", "Can't marshall response to JSON")
}
err = json.Unmarshal(marshalledJson, rsp)
if err != nil {
return hailo_errors.InternalServerError("testing.is.fubard", "Can't unmarshall response from JSON")
}
log.Tracef("[Service client mock] Matched response to %s:%s", req.Service(), req.Endpoint())
return nil
} else {
log.Warnf("[Service client mock] Couldn't match response for %s:%s", req.Service(), req.Endpoint())
return hailo_errors.InternalServerError("com.hailocab.kernel.platform.nilresponse", "Nil response")
}
}
// unmarshalRequest reads a request's payload into a RequestProtocol object
func (ep *Endpoint) unmarshalRequest(req *Request) (proto.Message, perrors.Error) {
reqProtoT, _ := ep.ProtoTypes()
if reqProtoT == nil { // No registered protocol
return nil, nil
}
result := reflect.New(reqProtoT.Elem()).Interface().(proto.Message)
if err := req.Unmarshal(result); err != nil {
return nil, perrors.InternalServerError(fmt.Sprintf("%s.%s.unmarshal", Name, ep.Name), err.Error())
}
return result, nil
}
func (c *client) Req(req *Request, rsp proto.Message, options ...Options) errors.Error {
// if no options supplied, lookup request options
if len(options) == 0 {
options = []Options{req.GetOptions()}
}
c.traceReq(req)
t := time.Now()
responseMsg, err := c.doReq(req, options...)
if err != nil {
errors.Track(err.Code(), req.From(), req.Service(), req.Endpoint())
return err
}
if responseMsg == nil {
return errors.InternalServerError("com.hailocab.kernel.platform.nilresponse", "Nil response")
}
c.traceRsp(req, responseMsg, err, time.Now().Sub(t))
if marshalError := responseMsg.Unmarshal(rsp); marshalError != nil {
return errors.InternalServerError("com.hailocab.kernel.platform.unmarshal", marshalError.Error())
}
return nil
}
func TestResponder(t *testing.T) {
stub := &Stub{
Service: mockFooService,
Endpoint: mockHealthEndpoint,
Responder: func(invocation int, req *client.Request) (proto.Message, errors.Error) {
if invocation == 1 {
return &hcproto.Response{
Healthchecks: []*hcproto.HealthCheck{
&hcproto.HealthCheck{
Timestamp: proto.Int64(1403629015),
ServiceName: proto.String("foo"),
ServiceVersion: proto.Uint64(1403629015),
Hostname: proto.String("localhost"),
InstanceId: proto.String("foobar"),
HealthCheckId: proto.String("boom"),
IsHealthy: proto.Bool(true),
},
},
}, nil
}
return nil, errors.InternalServerError("only.one.allowed", "First call only works")
},
}
mock := NewMock().Stub(stub)
caller := mock.Caller()
req, _ := client.NewRequest(mockFooService, mockHealthEndpoint, &hcproto.Request{})
rsp := &hcproto.Response{}
e := caller(req, rsp)
assert.Nil(t, e,
"Expecting our mocked call to be intercepted and stubbed response returned, got err: %v", e)
assert.Len(t, rsp.GetHealthchecks(), 1,
"Response does not contain our mocked content: no healthchecks")
// now repeat, and we SHOULD get an error
e = caller(req, rsp)
assert.NotNil(t, e,
"Expecting our mocked call to be intercepted and error response returned on 2nd call")
assert.Equal(t, e.Code(), "only.one.allowed",
"Expecting code 'only.one.allowed', got '%s'", e.Code())
}
// TestAuthUnhappyCase tests when things don't work
func (suite *sessionRecoverySuite) TestAuthUnhappyCaseInvalid() {
t := suite.T()
scope := New().(*realScope)
scope.userCache = newTestCache()
mock := multiclient.NewMock()
stub := &multiclient.Stub{
Service: loginService,
Endpoint: authEndpoint,
Error: errors.InternalServerError("com.hailocab.service.login.auth.foobarred", "It's FOOBARRED"),
}
mock.Stub(stub)
multiclient.SetCaller(mock.Caller())
testMech, testDeviceType := "h2", "cli"
testUsername, testPassword := "******", "Securez1"
testCreds := map[string]string{
"username": testUsername,
"password": testPassword,
}
err := scope.Auth(testMech, testDeviceType, testCreds)
if err == nil {
t.Fatal("Expecting auth error")
}
if err == BadCredentialsError {
t.Error("Error should not bubble up as a BAD CREDENTIALS error")
}
if scope.IsAuth() {
t.Error("Expecting scope to be IsAuth==false after failed auth")
}
if scope.HasTriedAuth() {
t.Error("Expecting scope to have HasTriedAuth()==false after failed auth")
}
if u := scope.AuthUser(); u != nil {
t.Error("Expecting AuthUser()==nil after failed auth")
}
// verify we made correct request(s)
if stub.CountCalls() != 1 {
t.Fatalf("Expecting 1 call to auth; got %v", stub.CountCalls())
}
}
func (e *errorsImpl) Combined() errors.Error {
e.RLock()
defer e.RUnlock()
switch len(e.errs) {
case 0:
return nil
case 1:
for _, re := range e.errs {
return re.err
}
return nil
default:
// Figure out what Scoper to use for the error.
// If each request has the same From, use that, otherwise, use the defaultScoper
scoper, i := e.defaultScoper, 0
for _, re := range e.errs {
if re.scoper != nil && (i == 0 || scoper == nil || re.scoper == scoper) {
scoper = re.scoper
} else {
scoper = e.defaultScoper
}
i++
}
context := ""
if scoper != nil {
context = scoper.Context()
}
if e.suffix != "" {
if context != "" {
context += "."
}
context += e.suffix
}
return errors.InternalServerError(context, e.multiError().Error())
}
}
// Execute runs all requests in parallel, blocking until all have completed
func (c *defClient) Execute() MultiClient {
c.Lock()
defer c.Unlock()
if c.done {
panic("Cannot repeat Execute() on a MultiClient - not supported")
}
c.done = true
completed := make(chan *singleRsp)
firedOff := 0
for uid, req := range c.requests {
// already an err creating req?
if exists := c.errors.ForUid(uid) != nil; exists {
continue
} else if req == nil {
log.Warnf("[Multiclient] Not expecting nil Request within MultiClient")
c.errors.set(uid, req, errors.InternalServerError(
"com.hailocab.kernel.multirequest.badrequest.nil",
fmt.Sprintf("Response for uid %s is nil", uid)), nil)
continue
}
go func(thisReq *client.Request, thisUid string) {
err := c.caller(thisReq, c.responses[thisUid])
completed <- &singleRsp{
uid: thisUid,
err: err,
}
}(req, uid)
firedOff++
}
for i := 0; i < firedOff; i++ {
r := <-completed
if r.err != nil {
c.errors.set(r.uid, c.requests[r.uid], r.err, nil)
}
}
return c
}
// doReq sends a request, with timeout options and retries, waits for response and returns it
func (c *client) doReq(req *Request, options ...Options) (*Response, errors.Error) {
if circuitbreaker.Open(req.service, req.endpoint) {
inst.Counter(1.0, fmt.Sprintf("client.error.%s.%s.circuitbroken", req.service, req.endpoint), 1)
log.Warnf("Broken Circuit for %s.%s", req.service, req.endpoint)
return nil, errors.CircuitBroken("com.hailocab.kernel.platform.circuitbreaker", "Circuit is open")
}
retries := c.defaults["retries"].(int)
var timeout time.Duration
timeoutSupplied := false
if len(options) == 1 {
if _, ok := options[0]["retries"]; ok {
retries = options[0]["retries"].(int)
}
if _, ok := options[0]["timeout"]; ok {
timeout = options[0]["timeout"].(time.Duration)
timeoutSupplied = true
}
}
// setup the response channel
rc := make(chan *Response, retries)
c.responses.add(req, rc)
defer c.responses.removeByRequest(req)
instPrefix := fmt.Sprintf("client.%s.%s", req.service, req.endpoint)
tAllRetries := time.Now()
for i := 1; i <= retries+1; i++ {
t := time.Now()
c.RLock()
con := c.listening
c.RUnlock()
if !con {
log.Debug("[Client] not yet listening, establishing now...")
ch := make(chan bool)
go c.listen(ch)
if online := <-ch; !online {
log.Error("[Client] Listener failed")
inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.client.listenfail", 1)
return nil, errors.InternalServerError("com.hailocab.kernel.platform.client.listenfail", "Listener failed")
}
log.Info("[Client] Listener online")
}
// figure out what timeout to use
if !timeoutSupplied {
timeout = c.timeout.Get(req.service, req.endpoint, i)
}
log.Tracef("[Client] Sync request attempt %d for %s using timeout %v", i, req.MessageID(), timeout)
// only bother sending the request if we are listening, otherwise allow to timeout
if err := raven.SendRequest(req, c.instanceID); err != nil {
log.Errorf("[Client] Failed to send request: %v", err)
}
select {
case payload := <-rc:
if payload.IsError() {
inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
errorProto := &pe.PlatformError{}
if err := payload.Unmarshal(errorProto); err != nil {
inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.badresponse", 1)
return nil, errors.BadResponse("com.hailocab.kernel.platform.badresponse", err.Error())
}
err := errors.FromProtobuf(errorProto)
inst.Counter(1.0, fmt.Sprintf("client.error.%s", err.Code()), 1)
circuitbreaker.Result(req.service, req.endpoint, err)
return nil, err
}
inst.Timing(1.0, fmt.Sprintf("%s.success", instPrefix), time.Since(t))
circuitbreaker.Result(req.service, req.endpoint, nil)
return payload, nil
case <-time.After(timeout):
// timeout
log.Errorf("[Client] Timeout talking to %s.%s after %v for %s", req.Service(), req.Endpoint(), timeout, req.MessageID())
inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
c.traceAttemptTimeout(req, i, timeout)
circuitbreaker.Result(req.service, req.endpoint, errors.Timeout("com.hailocab.kernel.platform.timeout",
fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
req.Service(),
req.Endpoint()))
}
}
inst.Timing(1.0, fmt.Sprintf("%s.error.timedOut", instPrefix), time.Since(tAllRetries))
inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.timeout", 1)
return nil, errors.Timeout(
"com.hailocab.kernel.platform.timeout",
fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
req.Service(),
req.Endpoint(),
)
}
// ConfiguredHttpCaller with more explicit configuration options than simple HttpCaller
func ConfiguredHttpCaller(opts Options) Caller {
tp := &httpclient.Transport{
ConnectTimeout: durationOrDefault(opts.ConnectTimeout, 5*time.Second),
RequestTimeout: durationOrDefault(opts.RequestTimeout, 5*time.Second),
ResponseHeaderTimeout: durationOrDefault(opts.ResponseHeaderTimeout, 5*time.Second),
}
if opts.TlsSkipVerify {
tp.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
httpClient := &http.Client{Transport: tp}
return func(req *client.Request, rsp proto.Message) errors.Error {
u, err := url.Parse(opts.BaseUrl)
q := u.Query()
q.Set("session_id", req.SessionID())
q.Set("service", req.Service())
q.Set("endpoint", req.Endpoint())
u.Path = "/rpc"
u.RawQuery = q.Encode()
var httpReq *http.Request
// send JSON req content-type to thin API as form-encoded data
// send proto req content-type directly as bytes, with proto content type
if req.ContentType() == jsonContentType {
values := make(url.Values)
values.Set("service", req.Service())
values.Set("endpoint", req.Endpoint())
values.Set("request", string(req.Payload()))
httpReq, _ = http.NewRequest("POST", u.String(), bytes.NewReader([]byte(values.Encode())))
httpReq.Header.Set("Content-Type", formEncodedContentType)
} else {
httpReq, _ = http.NewRequest("POST", u.String(), bytes.NewReader(req.Payload()))
httpReq.Header.Set("Content-Type", protoContentType)
}
log.Tracef("[Multiclient] HTTP caller - calling '%s' : content-type '%s'", u.String(), req.ContentType())
httpRsp, err := httpClient.Do(httpReq)
if err != nil {
log.Warnf("[Multiclient] HTTP caller error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err)
return errors.InternalServerError("multiclienthttp.postform", fmt.Sprintf("Error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err))
}
defer httpRsp.Body.Close()
rspBody, err := ioutil.ReadAll(httpRsp.Body)
if err != nil {
return errors.BadResponse("multiclienthttp.readresponse", fmt.Sprintf("Error reading response bytes: %v", err))
}
// what status code?
if httpRsp.StatusCode != 200 {
// deal with error
e := &protoerror.PlatformError{}
var err error
if req.ContentType() == jsonContentType {
jsonErr := &errorBody{}
err = json.Unmarshal(rspBody, jsonErr)
e.Code = proto.String(jsonErr.DottedCode)
e.Context = jsonErr.Context
e.Description = proto.String(jsonErr.Payload)
e.HttpCode = proto.Uint32(uint32(httpRsp.StatusCode))
// this conversion is lossy, since the JSON response for errors, as crafted
// by the "thin API", does not currently include the error type, so we have
// to guess from HTTP status code, but there is no distinct code for "BAD_RESPONSE"
switch httpRsp.StatusCode {
case 400:
e.Type = protoerror.PlatformError_BAD_REQUEST.Enum()
case 403:
e.Type = protoerror.PlatformError_FORBIDDEN.Enum()
case 404:
e.Type = protoerror.PlatformError_NOT_FOUND.Enum()
case 500:
e.Type = protoerror.PlatformError_INTERNAL_SERVER_ERROR.Enum()
case 504:
e.Type = protoerror.PlatformError_TIMEOUT.Enum()
}
} else {
err = proto.Unmarshal(rspBody, e)
}
// some issue understanding error rsp
if err != nil {
return errors.BadResponse("multiclienthttp.unmarshalerr", fmt.Sprintf("Error unmarshaling error response '%s': %v", string(rspBody), err))
}
return errors.FromProtobuf(e)
}
// unmarshal response
if req.ContentType() == jsonContentType {
err = json.Unmarshal(rspBody, rsp)
} else {
err = proto.Unmarshal(rspBody, rsp)
}
if err != nil {
return errors.BadResponse("multiclienthttp.unmarshal", fmt.Sprintf("Error unmarshaling response: %v", err))
}
return nil
}
}
// HandleRequest and send back response
func HandleRequest(req *Request) {
defer func() {
if r := recover(); r != nil {
log.Criticalf("[Server] Panic \"%v\" when handling request: (id: %s, endpoint: %s, content-type: %s,"+
" content-length: %d)", r, req.MessageID(), req.Destination(), req.delivery.ContentType,
len(req.delivery.Body))
inst.Counter(1.0, "runtime.panic", 1)
publishFailure(r)
debug.PrintStack()
}
}()
if len(req.Service()) > 0 && req.Service() != Name {
log.Criticalf(`[Server] Message meant for "%s" not "%s"`, req.Service(), Name)
return
}
reqProcessor:
switch {
case req.isHeartbeat():
if dsc.IsConnected() {
log.Tracef("[Server] Inbound heartbeat from: %s", req.ReplyTo())
dsc.hb.beat()
raven.SendResponse(PongResponse(req), InstanceID)
} else {
log.Warnf("[Server] Not connected but heartbeat from: %s", req.ReplyTo())
}
case req.IsPublication():
log.Tracef("[Server] Inbound publication on topic: %s", req.Topic())
if endpoint, ok := reg.find(req.Topic()); ok { // Match + call handler
if data, err := endpoint.unmarshalRequest(req); err != nil {
log.Warnf("[Server] Failed to unmarshal published message: %s", err.Error())
break reqProcessor
} else {
req.unmarshaledData = data
}
if _, err := endpoint.Handler(req); err != nil {
// don't do anything on error apart from log - it's a pub sub call so no response required
log.Warnf("[Server] Failed to process published message: %v", err)
}
}
default:
log.Tracef("[Server] Inbound message %s from %s", req.MessageID(), req.ReplyTo())
// Match a handler
endpoint, ok := reg.find(req.Endpoint())
if !ok {
if rsp, err := ErrorResponse(req, errors.InternalServerError("com.hailocab.kernel.handler.missing", fmt.Sprintf("No handler registered for %s", req.Destination()))); err != nil {
log.Criticalf("[Server] Unable to build response: %v", err)
} else {
raven.SendResponse(rsp, InstanceID)
}
return
}
// Unmarshal the request data
var (
reqData, rspData proto.Message
err errors.Error
)
if reqData, err = endpoint.unmarshalRequest(req); err == nil {
req.unmarshaledData = reqData
}
// Call handler if no errors so far
if err == nil {
rspData, err = endpoint.Handler(req)
}
// Check response type matches what's registered
if err == nil && rspData != nil {
_, rspProtoT := endpoint.ProtoTypes()
rspDataT := reflect.TypeOf(rspData)
if rspProtoT != nil && rspProtoT != rspDataT {
err = errors.InternalServerError("com.hailocab.kernel.server.mismatchedprotocol",
fmt.Sprintf("Mismatched response protocol. %s != %s", rspDataT.String(), rspProtoT.String()))
}
}
if err != nil {
switch err.Type() {
case errors.ErrorBadRequest, errors.ErrorForbidden, errors.ErrorNotFound:
log.Debugf("[Server] Handler error %s calling %v.%v from %v: %v", err.Type(), req.Service(),
req.Endpoint(), req.From(), err)
case errors.ErrorInternalServer:
go publishError(req, err)
fallthrough
default:
log.Errorf("[Server] Handler error %s calling %v.%v from %v: %v", err.Type(), req.Service(),
req.Endpoint(), req.From(), err)
}
if rsp, err := ErrorResponse(req, err); err != nil {
log.Criticalf("[Server] Unable to build response: %v", err)
} else {
raven.SendResponse(rsp, InstanceID)
}
return
}
if rsp, err := ReplyResponse(req, rspData); err != nil {
if rsp, err2 := ErrorResponse(req, errors.InternalServerError("com.hailocab.kernel.marshal.error", fmt.Sprintf("Could not marshal response %v", err))); err2 != nil {
log.Criticalf("[Server] Unable to build error response: %v", err2)
} else { // Send the error response
raven.SendResponse(rsp, InstanceID)
}
} else { // Send the succesful response
raven.SendResponse(rsp, InstanceID)
}
}
}