Пример #1
0
// doReq sends a request, with timeout options and retries, waits for response and returns it
func (c *client) doReq(req *Request, options ...Options) (*Response, errors.Error) {

	if circuitbreaker.Open(req.service, req.endpoint) {
		inst.Counter(1.0, fmt.Sprintf("client.error.%s.%s.circuitbroken", req.service, req.endpoint), 1)
		log.Warnf("Broken Circuit for %s.%s", req.service, req.endpoint)
		return nil, errors.CircuitBroken("com.hailocab.kernel.platform.circuitbreaker", "Circuit is open")
	}

	retries := c.defaults["retries"].(int)
	var timeout time.Duration
	timeoutSupplied := false
	if len(options) == 1 {
		if _, ok := options[0]["retries"]; ok {
			retries = options[0]["retries"].(int)
		}
		if _, ok := options[0]["timeout"]; ok {
			timeout = options[0]["timeout"].(time.Duration)
			timeoutSupplied = true
		}
	}

	// setup the response channel
	rc := make(chan *Response, retries)
	c.responses.add(req, rc)
	defer c.responses.removeByRequest(req)

	instPrefix := fmt.Sprintf("client.%s.%s", req.service, req.endpoint)
	tAllRetries := time.Now()

	for i := 1; i <= retries+1; i++ {
		t := time.Now()

		c.RLock()
		con := c.listening
		c.RUnlock()
		if !con {
			log.Debug("[Client] not yet listening, establishing now...")
			ch := make(chan bool)
			go c.listen(ch)
			if online := <-ch; !online {
				log.Error("[Client] Listener failed")
				inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
				inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.client.listenfail", 1)
				return nil, errors.InternalServerError("com.hailocab.kernel.platform.client.listenfail", "Listener failed")
			}

			log.Info("[Client] Listener online")
		}

		// figure out what timeout to use
		if !timeoutSupplied {
			timeout = c.timeout.Get(req.service, req.endpoint, i)
		}
		log.Tracef("[Client] Sync request attempt %d for %s using timeout %v", i, req.MessageID(), timeout)

		// only bother sending the request if we are listening, otherwise allow to timeout
		if err := raven.SendRequest(req, c.instanceID); err != nil {
			log.Errorf("[Client] Failed to send request: %v", err)
		}

		select {
		case payload := <-rc:
			if payload.IsError() {
				inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))

				errorProto := &pe.PlatformError{}
				if err := payload.Unmarshal(errorProto); err != nil {
					inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.badresponse", 1)
					return nil, errors.BadResponse("com.hailocab.kernel.platform.badresponse", err.Error())
				}

				err := errors.FromProtobuf(errorProto)
				inst.Counter(1.0, fmt.Sprintf("client.error.%s", err.Code()), 1)

				circuitbreaker.Result(req.service, req.endpoint, err)

				return nil, err
			}

			inst.Timing(1.0, fmt.Sprintf("%s.success", instPrefix), time.Since(t))
			circuitbreaker.Result(req.service, req.endpoint, nil)
			return payload, nil
		case <-time.After(timeout):
			// timeout
			log.Errorf("[Client] Timeout talking to %s.%s after %v for %s", req.Service(), req.Endpoint(), timeout, req.MessageID())
			inst.Timing(1.0, fmt.Sprintf("%s.error", instPrefix), time.Since(t))
			c.traceAttemptTimeout(req, i, timeout)

			circuitbreaker.Result(req.service, req.endpoint, errors.Timeout("com.hailocab.kernel.platform.timeout",
				fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
				req.Service(),
				req.Endpoint()))
		}
	}

	inst.Timing(1.0, fmt.Sprintf("%s.error.timedOut", instPrefix), time.Since(tAllRetries))
	inst.Counter(1.0, "client.error.com.hailocab.kernel.platform.timeout", 1)

	return nil, errors.Timeout(
		"com.hailocab.kernel.platform.timeout",
		fmt.Sprintf("Request timed out talking to %s.%s from %s (most recent timeout %v)", req.Service(), req.Endpoint(), req.From(), timeout),
		req.Service(),
		req.Endpoint(),
	)
}
Пример #2
0
func (suite *multiClientSuite) TestAnyErrorsIgnoring() {
	cases := []struct {
		errs         map[string]errors.Error
		types, codes []string
		isErr        bool
	}{
		{
			errs:  map[string]errors.Error{},
			types: []string{},
			codes: []string{},
			isErr: false,
		},
		// allow nils
		{
			errs:  map[string]errors.Error{},
			types: nil,
			codes: nil,
			isErr: false,
		},
		// no errors = always false
		{
			errs:  map[string]errors.Error{},
			types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
			codes: []string{"com.hailocab.service.foo.bar"},
			isErr: false,
		},
		{
			errs:  map[string]errors.Error{},
			types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
			codes: []string{},
			isErr: false,
		},
		{
			errs:  map[string]errors.Error{},
			types: []string{},
			codes: []string{"com.hailocab.service.foo.bar"},
			isErr: false,
		},
		{
			errs:  map[string]errors.Error{},
			types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
			codes: nil,
			isErr: false,
		},
		{
			errs:  map[string]errors.Error{},
			types: nil,
			codes: []string{"com.hailocab.service.foo.bar"},
			isErr: false,
		},
		// types
		{
			errs: map[string]errors.Error{
				"foo": errors.BadResponse("foo.bar", "ZOMG"),
			},
			types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
			codes: []string{},
			isErr: false,
		},
		{
			errs: map[string]errors.Error{
				"foo": errors.BadRequest("foo.bar", "ZOMG"),
			},
			types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
			codes: []string{},
			isErr: true,
		},
		{
			errs: map[string]errors.Error{
				"foo": errors.BadResponse("foo.bar", "ZOMG"),
				"bar": errors.BadRequest("foo.bar", "ZOMG"),
			},
			types: []string{errors.ErrorForbidden, errors.ErrorBadResponse},
			codes: []string{},
			isErr: true,
		},
		// codes
		{
			errs: map[string]errors.Error{
				"foo": errors.BadResponse("foo.bar", "ZOMG"),
			},
			types: nil,
			codes: []string{"foo.bar"},
			isErr: false,
		},
		{
			errs: map[string]errors.Error{
				"foo": errors.BadResponse("foo.bar", "ZOMG"),
			},
			types: nil,
			codes: []string{"foo.bar.baz"},
			isErr: true,
		},
		{
			errs: map[string]errors.Error{
				"foo": errors.BadResponse("foo.bar", "ZOMG"),
				"bar": errors.BadResponse("foo.bar.baz", "ZOMG"),
			},
			types: nil,
			codes: []string{"foo.bar"},
			isErr: true,
		},
	}

	cl := &defClient{
		errors: &errorsImpl{},
	}

	for _, tc := range cases {
		for uid, err := range tc.errs {
			cl.errors.set(uid, &client.Request{}, err, nil)
		}

		res := cl.AnyErrorsIgnoring(tc.types, tc.codes)
		suite.Assertions.Equal(tc.isErr, res, fmt.Sprintf("Wrong result for errors: %v - types: %v - codes: %v",
			tc.errs, tc.types, tc.codes))
	}
}
Пример #3
0
// ConfiguredHttpCaller with more explicit configuration options than simple HttpCaller
func ConfiguredHttpCaller(opts Options) Caller {
	tp := &httpclient.Transport{
		ConnectTimeout:        durationOrDefault(opts.ConnectTimeout, 5*time.Second),
		RequestTimeout:        durationOrDefault(opts.RequestTimeout, 5*time.Second),
		ResponseHeaderTimeout: durationOrDefault(opts.ResponseHeaderTimeout, 5*time.Second),
	}
	if opts.TlsSkipVerify {
		tp.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}

	httpClient := &http.Client{Transport: tp}

	return func(req *client.Request, rsp proto.Message) errors.Error {
		u, err := url.Parse(opts.BaseUrl)

		q := u.Query()
		q.Set("session_id", req.SessionID())
		q.Set("service", req.Service())
		q.Set("endpoint", req.Endpoint())
		u.Path = "/rpc"
		u.RawQuery = q.Encode()

		var httpReq *http.Request

		// send JSON req content-type to thin API as form-encoded data
		// send proto req content-type directly as bytes, with proto content type
		if req.ContentType() == jsonContentType {
			values := make(url.Values)
			values.Set("service", req.Service())
			values.Set("endpoint", req.Endpoint())
			values.Set("request", string(req.Payload()))
			httpReq, _ = http.NewRequest("POST", u.String(), bytes.NewReader([]byte(values.Encode())))
			httpReq.Header.Set("Content-Type", formEncodedContentType)
		} else {
			httpReq, _ = http.NewRequest("POST", u.String(), bytes.NewReader(req.Payload()))
			httpReq.Header.Set("Content-Type", protoContentType)
		}

		log.Debugf("[Multiclient] HTTP caller - calling '%s' : content-type '%s'", u.String(), req.ContentType())

		httpRsp, err := httpClient.Do(httpReq)
		if err != nil {
			log.Warnf("[Multiclient] HTTP caller error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err)
			return errors.InternalServerError("multiclienthttp.postform", fmt.Sprintf("Error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err))
		}

		if err != nil {
			log.Warnf("[Multiclient] HTTP caller error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err)
			return errors.InternalServerError("multiclienthttp.postform", fmt.Sprintf("Error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err))
		}

		defer httpRsp.Body.Close()
		rspBody, err := ioutil.ReadAll(httpRsp.Body)
		if err != nil {
			return errors.BadResponse("multiclienthttp.readresponse", fmt.Sprintf("Error reading response bytes: %v", err))
		}

		// what status code?
		if httpRsp.StatusCode != 200 {
			// deal with error
			e := &protoerror.PlatformError{}
			var err error
			if req.ContentType() == jsonContentType {
				jsonErr := &errorBody{}
				err = json.Unmarshal(rspBody, jsonErr)
				e.Code = proto.String(jsonErr.DottedCode)
				e.Context = jsonErr.Context
				e.Description = proto.String(jsonErr.Payload)
				e.HttpCode = proto.Uint32(uint32(httpRsp.StatusCode))
				// this conversion is lossy, since the JSON response for errors, as crafted
				// by the "thin API", does not currently include the error type, so we have
				// to guess from HTTP status code, but there is no distinct code for "BAD_RESPONSE"
				switch httpRsp.StatusCode {
				case 400:
					e.Type = protoerror.PlatformError_BAD_REQUEST.Enum()
				case 403:
					e.Type = protoerror.PlatformError_FORBIDDEN.Enum()
				case 404:
					e.Type = protoerror.PlatformError_NOT_FOUND.Enum()
				case 500:
					e.Type = protoerror.PlatformError_INTERNAL_SERVER_ERROR.Enum()
				case 504:
					e.Type = protoerror.PlatformError_TIMEOUT.Enum()
				}
			} else {
				err = proto.Unmarshal(rspBody, e)
			}
			// some issue understanding error rsp
			if err != nil {
				return errors.BadResponse("multiclienthttp.unmarshalerr", fmt.Sprintf("Error unmarshaling error response '%s': %v", string(rspBody), err))
			}
			return errors.FromProtobuf(e)
		}

		// unmarshal response
		if req.ContentType() == jsonContentType {
			err = json.Unmarshal(rspBody, rsp)
		} else {
			err = proto.Unmarshal(rspBody, rsp)
		}
		if err != nil {
			return errors.BadResponse("multiclienthttp.unmarshal", fmt.Sprintf("Error unmarshaling response: %v", err))
		}

		return nil
	}
}