示例#1
0
// AddScopedReq adds a server-scoped request (from the server request `from`) to our multi-client
// with the `uid` that uniquely identifies the request within the group (for getting response from `Outcome`)
func (c *defClient) AddScopedReq(sr *ScopedReq) MultiClient {
	c.Lock()
	defer c.Unlock()
	if _, exists := c.requests[sr.Uid]; exists {
		panic(fmt.Sprintf("Cannot add scoped request with UID '%v' - already exists within this MultiClient", sr.Uid))
	}
	from := sr.From
	if from == nil {
		from = c.defaultFromScope
	}

	var clientReq *client.Request
	var err error

	// if no from, just use normal client request
	if from == nil {
		clientReq, err = client.NewRequest(sr.Service, sr.Endpoint, sr.Req)
	} else {
		clientReq, err = from.ScopedRequest(sr.Service, sr.Endpoint, sr.Req)
	}

	c.requests[sr.Uid] = clientReq
	c.responses[sr.Uid] = sr.Rsp
	if err != nil {
		c.errors.set(sr.Uid, clientReq,
			errors.InternalServerError("com.hailocab.kernel.multirequest.badrequest", err.Error()), from)
	} else {
		clientReq.SetOptions(sr.Options)
	}

	return c
}
示例#2
0
// TestRecoverSessionSad tests sad case (login service has some fatal error)
func (suite *sessionRecoverySuite) TestRecoverSessionSad() {
	t := suite.T()

	scope := New().(*realScope)
	scope.userCache = newTestCache()

	mock := multiclient.NewMock()
	stub := &multiclient.Stub{
		Service:  loginService,
		Endpoint: readSessionEndpoint,
		Error:    errors.InternalServerError("com.hailocab.service.login.foo", "Things are foo barred"),
	}
	mock.Stub(stub)
	multiclient.SetCaller(mock.Caller())

	err := scope.RecoverSession(testSessId)
	if err == nil {
		t.Error("Expecting recovery error, because login call should fail.")
	}
	if scope.IsAuth() {
		t.Error("Expecting scope to be IsAuth==false after failed recovery")
	}
	if scope.HasTriedAuth() {
		t.Error("Expecting scope to have HasTriedAuth()==false after _failed_ recovery")
	}

	// verify we made correct request(s)
	if stub.CountCalls() != 1 {
		t.Fatalf("Expecting 1 call to readsession; got %v", stub.CountCalls())
	}
}
示例#3
0
// jsonschemaHandler returns all registered endpoints in json schema format as per ITF draft4
// http://json-schema.org/latest/json-schema-core.html
func jsonschemaHandler(req *Request) (proto.Message, errors.Error) {
	// Get all endpoints
	request := req.Data().(*jsonschemaproto.Request)
	endpoint := request.GetEndpoint()

	endpoints := reg.iterate()
	schemas := make([]*jsonschema.JsonSchema, 0)
	for _, ep := range endpoints {
		if endpoint != "" && endpoint != ep.GetName() {
			continue
		}
		schema, err := marshalEndpoint(ep)
		if err == nil && schema != nil {
			schemas = append(schemas, schema)
		}
	}

	rsp, err := json.Marshal(schemas)
	if err != nil {
		return nil, errors.InternalServerError("com.hailocab.kernel.marshal.error", fmt.Sprintf("Unable to unmarshal response data: %v", err.Error()))
	}

	return &jsonschemaproto.Response{
		Jsonschema: proto.String(string(rsp)),
	}, nil
}
示例#4
0
func (suite *errorsImplSuite) SetupTest() {
	suite.Suite.SetupTest()
	suite.rawErrs = map[string]errors.Error{
		"uid1": errors.InternalServerError("com.foo.uid1", "uid1"),
		"uid2": errors.InternalServerError("com.foo.uid2", "uid2"),
		"uid3": errors.InternalServerError("com.foo.uid2", "uid3"), // Same service uid as uid2
	}
	suite.errs = &errorsImpl{}
	for uid, err := range suite.rawErrs {
		req, reqErr := client.NewJsonRequest(err.Code(), err.Description(), nil)
		if reqErr != nil {
			panic(reqErr)
		}
		suite.errs.set(uid, req, err, nil)
	}
}
示例#5
0
func (m *MockClient) CustomReq(req *Request, options ...Options) (*Response, hailo_errors.Error) {
	if matchedRsp := m.getMatchingRequestExpectation(req); matchedRsp != nil {
		marshalledJson, err := json.Marshal(*matchedRsp)
		if err != nil {
			return nil, hailo_errors.InternalServerError("testing.is.fubard", "Can't marshall response to JSON")
		}

		resp := amqp.Delivery{
			ContentType:   "application/json",
			Body:          marshalledJson,
			MessageId:     "oh-what-a-lovely-test",
			CorrelationId: "oh-what-a-lovely-test",
			Headers:       map[string](interface{}){},
		}
		return &Response{resp}, nil
	} else {
		log.Warnf("[Service client mock] Couldn't match response for %s:%s", req.Service(), req.Endpoint())
		return nil, hailo_errors.InternalServerError("com.hailocab.kernel.platform.nilresponse", "Nil response")
	}
}
示例#6
0
func (m *MockClient) Req(req *Request, rsp proto.Message, options ...Options) hailo_errors.Error {
	if matchedRsp := m.getMatchingRequestExpectation(req); matchedRsp != nil {
		// Marshall to JSON and back again, to get it into rsp (which is passed by value). This is fine; these are
		// marshalled and unmarshalled to JSON during normal operation anyway.
		marshalledJson, err := json.Marshal(*matchedRsp)
		if err != nil {
			return hailo_errors.InternalServerError("testing.is.fubard", "Can't marshall response to JSON")
		}
		err = json.Unmarshal(marshalledJson, rsp)
		if err != nil {
			return hailo_errors.InternalServerError("testing.is.fubard", "Can't unmarshall response from JSON")
		}

		log.Tracef("[Service client mock] Matched response to %s:%s", req.Service(), req.Endpoint())
		return nil
	} else {
		log.Warnf("[Service client mock] Couldn't match response for %s:%s", req.Service(), req.Endpoint())
		return hailo_errors.InternalServerError("com.hailocab.kernel.platform.nilresponse", "Nil response")
	}
}
示例#7
0
文件: endpoint.go 项目: armada-io/h2
// unmarshalRequest reads a request's payload into a RequestProtocol object
func (ep *Endpoint) unmarshalRequest(req *Request) (proto.Message, perrors.Error) {
	reqProtoT, _ := ep.ProtoTypes()

	if reqProtoT == nil { // No registered protocol
		return nil, nil
	}

	result := reflect.New(reqProtoT.Elem()).Interface().(proto.Message)
	if err := req.Unmarshal(result); err != nil {
		return nil, perrors.InternalServerError(fmt.Sprintf("%s.%s.unmarshal", Name, ep.Name), err.Error())
	}

	return result, nil
}
示例#8
0
文件: client.go 项目: armada-io/h2
func (c *client) Req(req *Request, rsp proto.Message, options ...Options) errors.Error {
	// if no options supplied, lookup request options
	if len(options) == 0 {
		options = []Options{req.GetOptions()}
	}

	c.traceReq(req)
	t := time.Now()
	responseMsg, err := c.doReq(req, options...)
	if err != nil {
		errors.Track(err.Code(), req.From(), req.Service(), req.Endpoint())
		return err
	}
	if responseMsg == nil {
		return errors.InternalServerError("com.hailocab.kernel.platform.nilresponse", "Nil response")
	}
	c.traceRsp(req, responseMsg, err, time.Now().Sub(t))

	if marshalError := responseMsg.Unmarshal(rsp); marshalError != nil {
		return errors.InternalServerError("com.hailocab.kernel.platform.unmarshal", marshalError.Error())
	}

	return nil
}
示例#9
0
// TestAuthUnhappyCase tests when things don't work
func (suite *sessionRecoverySuite) TestAuthUnhappyCaseInvalid() {
	t := suite.T()

	scope := New().(*realScope)
	scope.userCache = newTestCache()

	mock := multiclient.NewMock()
	stub := &multiclient.Stub{
		Service:  loginService,
		Endpoint: authEndpoint,
		Error:    errors.InternalServerError("com.hailocab.service.login.auth.foobarred", "It's FOOBARRED"),
	}
	mock.Stub(stub)
	multiclient.SetCaller(mock.Caller())

	testMech, testDeviceType := "h2", "cli"
	testUsername, testPassword := "******", "Securez1"
	testCreds := map[string]string{
		"username": testUsername,
		"password": testPassword,
	}

	err := scope.Auth(testMech, testDeviceType, testCreds)
	if err == nil {
		t.Fatal("Expecting auth error")
	}
	if err == BadCredentialsError {
		t.Error("Error should not bubble up as a BAD CREDENTIALS error")
	}
	if scope.IsAuth() {
		t.Error("Expecting scope to be IsAuth==false after failed auth")
	}
	if scope.HasTriedAuth() {
		t.Error("Expecting scope to have HasTriedAuth()==false after failed auth")
	}
	if u := scope.AuthUser(); u != nil {
		t.Error("Expecting AuthUser()==nil after failed auth")
	}

	// verify we made correct request(s)
	if stub.CountCalls() != 1 {
		t.Fatalf("Expecting 1 call to auth; got %v", stub.CountCalls())
	}
}
示例#10
0
func TestResponder(t *testing.T) {
	stub := &Stub{
		Service:  mockFooService,
		Endpoint: mockHealthEndpoint,
		Responder: func(invocation int, req *client.Request) (proto.Message, errors.Error) {
			if invocation == 1 {
				return &hcproto.Response{
					Healthchecks: []*hcproto.HealthCheck{
						&hcproto.HealthCheck{
							Timestamp:      proto.Int64(1403629015),
							ServiceName:    proto.String("foo"),
							ServiceVersion: proto.Uint64(1403629015),
							Hostname:       proto.String("localhost"),
							InstanceId:     proto.String("foobar"),
							HealthCheckId:  proto.String("boom"),
							IsHealthy:      proto.Bool(true),
						},
					},
				}, nil
			}
			return nil, errors.InternalServerError("only.one.allowed", "First call only works")
		},
	}
	mock := NewMock().Stub(stub)

	caller := mock.Caller()
	req, _ := client.NewRequest(mockFooService, mockHealthEndpoint, &hcproto.Request{})
	rsp := &hcproto.Response{}
	e := caller(req, rsp)

	assert.Nil(t, e,
		"Expecting our mocked call to be intercepted and stubbed response returned, got err: %v", e)

	assert.Len(t, rsp.GetHealthchecks(), 1,
		"Response does not contain our mocked content: no healthchecks")

	// now repeat, and we SHOULD get an error
	e = caller(req, rsp)
	assert.NotNil(t, e,
		"Expecting our mocked call to be intercepted and error response returned on 2nd call")

	assert.Equal(t, e.Code(), "only.one.allowed",
		"Expecting code 'only.one.allowed', got '%s'", e.Code())
}
示例#11
0
文件: errors.go 项目: armada-io/h2
func (e *errorsImpl) Combined() errors.Error {
	e.RLock()
	defer e.RUnlock()

	switch len(e.errs) {
	case 0:
		return nil
	case 1:
		for _, re := range e.errs {
			return re.err
		}
		return nil
	default:
		// Figure out what Scoper to use for the error.
		// If each request has the same From, use that, otherwise, use the defaultScoper
		scoper, i := e.defaultScoper, 0
		for _, re := range e.errs {
			if re.scoper != nil && (i == 0 || scoper == nil || re.scoper == scoper) {
				scoper = re.scoper
			} else {
				scoper = e.defaultScoper
			}
			i++
		}

		context := ""
		if scoper != nil {
			context = scoper.Context()
		}
		if e.suffix != "" {
			if context != "" {
				context += "."
			}
			context += e.suffix
		}
		return errors.InternalServerError(context, e.multiError().Error())
	}
}
示例#12
0
// Execute runs all requests in parallel, blocking until all have completed
func (c *defClient) Execute() MultiClient {
	c.Lock()
	defer c.Unlock()
	if c.done {
		panic("Cannot repeat Execute() on a MultiClient - not supported")
	}
	c.done = true
	completed := make(chan *singleRsp)
	firedOff := 0
	for uid, req := range c.requests {
		// already an err creating req?
		if exists := c.errors.ForUid(uid) != nil; exists {
			continue
		} else if req == nil {
			log.Warnf("[Multiclient] Not expecting nil Request within MultiClient")
			c.errors.set(uid, req, errors.InternalServerError(
				"com.hailocab.kernel.multirequest.badrequest.nil",
				fmt.Sprintf("Response for uid %s is nil", uid)), nil)
			continue
		}
		go func(thisReq *client.Request, thisUid string) {
			err := c.caller(thisReq, c.responses[thisUid])
			completed <- &singleRsp{
				uid: thisUid,
				err: err,
			}
		}(req, uid)
		firedOff++
	}
	for i := 0; i < firedOff; i++ {
		r := <-completed
		if r.err != nil {
			c.errors.set(r.uid, c.requests[r.uid], r.err, nil)
		}
	}
	return c
}
示例#13
0
文件: client.go 项目: armada-io/h2
// doReq sends a request, with timeout options and retries, waits for response and returns it
func (c *client) doReq(req *Request, options ...Options) (*Response, errors.Error) {

	if circuitbreaker.Open(req.service, req.endpoint) {
		inst.Counter(1.0, fmt.Sprintf("client.error.%s.%s.circuitbroken", req.service, req.endpoint), 1)
		log.Warnf("Broken Circuit for %s.%s", req.service, req.endpoint)
		return nil, errors.CircuitBroken("com.hailocab.kernel.platform.circuitbreaker", "Circuit is open")
	}

	retries := c.defaults["retries"].(int)
	var timeout time.Duration
	timeoutSupplied := false
	if len(options) == 1 {
		if _, ok := options[0]["retries"]; ok {
			retries = options[0]["retries"].(int)
		}
		if _, ok := options[0]["timeout"]; ok {
			timeout = options[0]["timeout"].(time.Duration)
			timeoutSupplied = true
		}
	}

	// setup the response channel
	rc := make(chan *Response, retries)
	c.responses.add(req, rc)
	defer c.responses.removeByRequest(req)

	instPrefix := fmt.Sprintf("client.%s.%s", req.service, req.endpoint)
	tAllRetries := time.Now()

	for i := 1; i <= retries+1; i++ {
		t := time.Now()

		c.RLock()
		con := c.listening
		c.RUnlock()
		if !con {
			log.Debug("[Client] not yet listening, establishing now...")
			ch := make(chan bool)
			go c.listen(ch)
			if online := <-ch; !online {
				log.Error("[Client] Listener failed")
				inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
				inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.client.listenfail", 1)
				return nil, errors.InternalServerError("com.hailocab.kernel.platform.client.listenfail", "Listener failed")
			}

			log.Info("[Client] Listener online")
		}

		// figure out what timeout to use
		if !timeoutSupplied {
			timeout = c.timeout.Get(req.service, req.endpoint, i)
		}
		log.Tracef("[Client] Sync request attempt %d for %s using timeout %v", i, req.MessageID(), timeout)

		// only bother sending the request if we are listening, otherwise allow to timeout
		if err := raven.SendRequest(req, c.instanceID); err != nil {
			log.Errorf("[Client] Failed to send request: %v", err)
		}

		select {
		case payload := <-rc:
			if payload.IsError() {
				inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))

				errorProto := &pe.PlatformError{}
				if err := payload.Unmarshal(errorProto); err != nil {
					inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.badresponse", 1)
					return nil, errors.BadResponse("com.hailocab.kernel.platform.badresponse", err.Error())
				}

				err := errors.FromProtobuf(errorProto)
				inst.Counter(1.0, fmt.Sprintf("client.error.%s", err.Code()), 1)

				circuitbreaker.Result(req.service, req.endpoint, err)

				return nil, err
			}

			inst.Timing(1.0, fmt.Sprintf("%s.success", instPrefix), time.Since(t))
			circuitbreaker.Result(req.service, req.endpoint, nil)
			return payload, nil
		case <-time.After(timeout):
			// timeout
			log.Errorf("[Client] Timeout talking to %s.%s after %v for %s", req.Service(), req.Endpoint(), timeout, req.MessageID())
			inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
			c.traceAttemptTimeout(req, i, timeout)

			circuitbreaker.Result(req.service, req.endpoint, errors.Timeout("com.hailocab.kernel.platform.timeout",
				fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
				req.Service(),
				req.Endpoint()))
		}
	}

	inst.Timing(1.0, fmt.Sprintf("%s.error.timedOut", instPrefix), time.Since(tAllRetries))
	inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.timeout", 1)

	return nil, errors.Timeout(
		"com.hailocab.kernel.platform.timeout",
		fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
		req.Service(),
		req.Endpoint(),
	)
}
示例#14
0
文件: server.go 项目: armada-io/h2
// HandleRequest and send back response
func HandleRequest(req *Request) {
	defer func() {
		if r := recover(); r != nil {
			log.Criticalf("[Server] Panic \"%v\" when handling request: (id: %s, endpoint: %s, content-type: %s,"+
				" content-length: %d)", r, req.MessageID(), req.Destination(), req.delivery.ContentType,
				len(req.delivery.Body))
			inst.Counter(1.0, "runtime.panic", 1)
			publishFailure(r)
			debug.PrintStack()
		}
	}()

	if len(req.Service()) > 0 && req.Service() != Name {
		log.Criticalf(`[Server] Message meant for "%s" not "%s"`, req.Service(), Name)
		return
	}

reqProcessor:
	switch {
	case req.isHeartbeat():
		if dsc.IsConnected() {
			log.Tracef("[Server] Inbound heartbeat from: %s", req.ReplyTo())
			dsc.hb.beat()
			raven.SendResponse(PongResponse(req), InstanceID)
		} else {
			log.Warnf("[Server] Not connected but heartbeat from: %s", req.ReplyTo())
		}

	case req.IsPublication():
		log.Tracef("[Server] Inbound publication on topic: %s", req.Topic())

		if endpoint, ok := reg.find(req.Topic()); ok { // Match + call handler
			if data, err := endpoint.unmarshalRequest(req); err != nil {
				log.Warnf("[Server] Failed to unmarshal published message: %s", err.Error())
				break reqProcessor
			} else {
				req.unmarshaledData = data
			}

			if _, err := commonLogHandler(commonLogger, endpoint.instrumentedHandler)(req); err != nil {
				// don't do anything on error apart from log - it's a pub sub call so no response required
				log.Warnf("[Server] Failed to process published message: %v", err)
			}
		}

	default:
		log.Tracef("[Server] Inbound message %s from %s", req.MessageID(), req.ReplyTo())

		// Match a handler
		endpoint, ok := reg.find(req.Endpoint())
		if !ok {
			if rsp, err := ErrorResponse(req, errors.InternalServerError("com.hailocab.kernel.handler.missing", fmt.Sprintf("No handler registered for %s", req.Destination()))); err != nil {
				log.Criticalf("[Server] Unable to build response: %v", err)
			} else {
				raven.SendResponse(rsp, InstanceID)
			}
			return
		}

		// Unmarshal the request data
		var (
			reqData, rspData proto.Message
			err              errors.Error
		)
		if reqData, err = endpoint.unmarshalRequest(req); err == nil {
			req.unmarshaledData = reqData
		}

		// Call handler - constraining the max concurrent requests handled
		if err == nil {
			callerName := req.From()
			if callerName == "" {
				callerName = "unknown"
			}
			tokenBucketName := fmt.Sprintf("server.tokens.%s", callerName)
			reqsBucketName := fmt.Sprintf("server.inflightrequests.%s", callerName)
			tokC := tokensChan(callerName)

			select {
			case t := <-tokC:
				func() {
					defer func() {
						atomic.AddUint64(&inFlightRequests, ^uint64(0)) // This is actually a subtraction
						tokC <- t                                       // Return the token to the pool
					}()

					nowInFlight := atomic.AddUint64(&inFlightRequests, 1) // Update active request counters
					inst.Gauge(1.0, tokenBucketName, len(tokC))
					inst.Gauge(1.0, reqsBucketName, int(nowInFlight))
					rspData, err = commonLogHandler(commonLogger, endpoint.instrumentedHandler)(req)
				}()
			case <-time.After(time.Duration(endpoint.Mean) * time.Millisecond):
				inst.Gauge(1.0, tokenBucketName, len(tokC))
				inst.Counter(1.0, "server.error.capacity", 1)

				err = errors.InternalServerError("com.hailocab.kernel.server.capacity",
					fmt.Sprintf("Server %v out of capacity", Name))
			}
		}

		// Check response type matches what's registered
		if err == nil && rspData != nil {
			_, rspProtoT := endpoint.ProtoTypes()
			rspDataT := reflect.TypeOf(rspData)
			if rspProtoT != nil && rspProtoT != rspDataT {
				err = errors.InternalServerError("com.hailocab.kernel.server.mismatchedprotocol",
					fmt.Sprintf("Mismatched response protocol. %s != %s", rspDataT.String(), rspProtoT.String()))
			}
		}

		if err != nil {
			inst.Counter(1.0, fmt.Sprintf("server.error.%s", err.Code()), 1)

			switch err.Type() {
			case errors.ErrorBadRequest, errors.ErrorForbidden, errors.ErrorNotFound:
				log.Debugf("[Server] Handler error %s calling %v.%v from %v: %v", err.Type(), req.Service(),
					req.Endpoint(), req.From(), err)
			case errors.ErrorInternalServer:
				go publishError(req, err)
				fallthrough
			default:
				log.Errorf("[Server] Handler error %s calling %v.%v from %v: %v", err.Type(), req.Service(),
					req.Endpoint(), req.From(), err)
			}

			if rsp, err := ErrorResponse(req, err); err != nil {
				log.Criticalf("[Server] Unable to build response: %v", err)
			} else {
				raven.SendResponse(rsp, InstanceID)
			}

			return
		}

		if rsp, err := ReplyResponse(req, rspData); err != nil {
			if rsp, err2 := ErrorResponse(req, errors.InternalServerError("com.hailocab.kernel.marshal.error", fmt.Sprintf("Could not marshal response %v", err))); err2 != nil {
				log.Criticalf("[Server] Unable to build error response: %v", err2)
			} else { // Send the error response
				raven.SendResponse(rsp, InstanceID)
			}
		} else { // Send the succesful response
			raven.SendResponse(rsp, InstanceID)
		}
	}
}
示例#15
0
// ConfiguredHttpCaller with more explicit configuration options than simple HttpCaller
func ConfiguredHttpCaller(opts Options) Caller {
	tp := &httpclient.Transport{
		ConnectTimeout:        durationOrDefault(opts.ConnectTimeout, 5*time.Second),
		RequestTimeout:        durationOrDefault(opts.RequestTimeout, 5*time.Second),
		ResponseHeaderTimeout: durationOrDefault(opts.ResponseHeaderTimeout, 5*time.Second),
	}
	if opts.TlsSkipVerify {
		tp.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}

	httpClient := &http.Client{Transport: tp}

	return func(req *client.Request, rsp proto.Message) errors.Error {
		u, err := url.Parse(opts.BaseUrl)

		q := u.Query()
		q.Set("session_id", req.SessionID())
		q.Set("service", req.Service())
		q.Set("endpoint", req.Endpoint())
		u.Path = "/rpc"
		u.RawQuery = q.Encode()

		var httpReq *http.Request

		// send JSON req content-type to thin API as form-encoded data
		// send proto req content-type directly as bytes, with proto content type
		if req.ContentType() == jsonContentType {
			values := make(url.Values)
			values.Set("service", req.Service())
			values.Set("endpoint", req.Endpoint())
			values.Set("request", string(req.Payload()))
			httpReq, _ = http.NewRequest("POST", u.String(), bytes.NewReader([]byte(values.Encode())))
			httpReq.Header.Set("Content-Type", formEncodedContentType)
		} else {
			httpReq, _ = http.NewRequest("POST", u.String(), bytes.NewReader(req.Payload()))
			httpReq.Header.Set("Content-Type", protoContentType)
		}

		log.Debugf("[Multiclient] HTTP caller - calling '%s' : content-type '%s'", u.String(), req.ContentType())

		httpRsp, err := httpClient.Do(httpReq)
		if err != nil {
			log.Warnf("[Multiclient] HTTP caller error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err)
			return errors.InternalServerError("multiclienthttp.postform", fmt.Sprintf("Error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err))
		}

		if err != nil {
			log.Warnf("[Multiclient] HTTP caller error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err)
			return errors.InternalServerError("multiclienthttp.postform", fmt.Sprintf("Error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err))
		}

		defer httpRsp.Body.Close()
		rspBody, err := ioutil.ReadAll(httpRsp.Body)
		if err != nil {
			return errors.BadResponse("multiclienthttp.readresponse", fmt.Sprintf("Error reading response bytes: %v", err))
		}

		// what status code?
		if httpRsp.StatusCode != 200 {
			// deal with error
			e := &protoerror.PlatformError{}
			var err error
			if req.ContentType() == jsonContentType {
				jsonErr := &errorBody{}
				err = json.Unmarshal(rspBody, jsonErr)
				e.Code = proto.String(jsonErr.DottedCode)
				e.Context = jsonErr.Context
				e.Description = proto.String(jsonErr.Payload)
				e.HttpCode = proto.Uint32(uint32(httpRsp.StatusCode))
				// this conversion is lossy, since the JSON response for errors, as crafted
				// by the "thin API", does not currently include the error type, so we have
				// to guess from HTTP status code, but there is no distinct code for "BAD_RESPONSE"
				switch httpRsp.StatusCode {
				case 400:
					e.Type = protoerror.PlatformError_BAD_REQUEST.Enum()
				case 403:
					e.Type = protoerror.PlatformError_FORBIDDEN.Enum()
				case 404:
					e.Type = protoerror.PlatformError_NOT_FOUND.Enum()
				case 500:
					e.Type = protoerror.PlatformError_INTERNAL_SERVER_ERROR.Enum()
				case 504:
					e.Type = protoerror.PlatformError_TIMEOUT.Enum()
				}
			} else {
				err = proto.Unmarshal(rspBody, e)
			}
			// some issue understanding error rsp
			if err != nil {
				return errors.BadResponse("multiclienthttp.unmarshalerr", fmt.Sprintf("Error unmarshaling error response '%s': %v", string(rspBody), err))
			}
			return errors.FromProtobuf(e)
		}

		// unmarshal response
		if req.ContentType() == jsonContentType {
			err = json.Unmarshal(rspBody, rsp)
		} else {
			err = proto.Unmarshal(rspBody, rsp)
		}
		if err != nil {
			return errors.BadResponse("multiclienthttp.unmarshal", fmt.Sprintf("Error unmarshaling response: %v", err))
		}

		return nil
	}
}