func TestRetryMaxTotalFail(t *testing.T) {
var (
endpoints = sd.FixedSubscriber{} // no endpoints
lb = loadbalancer.NewRoundRobin(endpoints)
retry = loadbalancer.Retry(999, time.Second, lb) // lots of retries
ctx = context.Background()
)
if _, err := retry(ctx, struct{}{}); err == nil {
t.Errorf("expected error, got none") // should fail
}
}
func Example() {
// Let's say this is a service that means to register itself.
// First, we will set up some context.
var (
etcdServer = "http://10.0.0.1:2379" // don't forget schema and port!
prefix = "/services/foosvc/" // known at compile time
instance = "1.2.3.4:8080" // taken from runtime or platform, somehow
key = prefix + instance // should be globally unique
value = "http://" + instance // based on our transport
ctx = context.Background()
)
// Build the client.
client, err := NewClient(ctx, []string{etcdServer}, ClientOptions{})
if err != nil {
panic(err)
}
// Build the registrar.
registrar := NewRegistrar(client, Service{
Key: key,
Value: value,
}, log.NewNopLogger())
// Register our instance.
registrar.Register()
// At the end of our service lifecycle, for example at the end of func main,
// we should make sure to deregister ourselves. This is important! Don't
// accidentally skip this step by invoking a log.Fatal or os.Exit in the
// interim, which bypasses the defer stack.
defer registrar.Deregister()
// It's likely that we'll also want to connect to other services and call
// their methods. We can build a subscriber to listen for changes from etcd
// and build endpoints, wrap it with a load-balancer to pick a single
// endpoint, and finally wrap it with a retry strategy to get something that
// can be used as an endpoint directly.
barPrefix := "/services/barsvc"
subscriber, err := NewSubscriber(client, barPrefix, barFactory, log.NewNopLogger())
if err != nil {
panic(err)
}
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(3, 3*time.Second, balancer)
// And now retry can be used like any other endpoint.
req := struct{}{}
if _, err = retry(ctx, req); err != nil {
panic(err)
}
}
func TestRetryMaxSuccess(t *testing.T) {
var (
endpoints = []endpoint.Endpoint{
func(context.Context, interface{}) (interface{}, error) { return nil, errors.New("error one") },
func(context.Context, interface{}) (interface{}, error) { return nil, errors.New("error two") },
func(context.Context, interface{}) (interface{}, error) { return struct{}{}, nil /* OK */ },
}
subscriber = sd.FixedSubscriber{
0: endpoints[0],
1: endpoints[1],
2: endpoints[2],
}
retries = len(endpoints) // exactly enough retries
lb = loadbalancer.NewRoundRobin(subscriber)
ctx = context.Background()
)
if _, err := loadbalancer.Retry(retries, time.Second, lb)(ctx, struct{}{}); err != nil {
t.Error(err)
}
}
func TestRetryMaxPartialFail(t *testing.T) {
var (
endpoints = []endpoint.Endpoint{
func(context.Context, interface{}) (interface{}, error) { return nil, errors.New("error one") },
func(context.Context, interface{}) (interface{}, error) { return nil, errors.New("error two") },
func(context.Context, interface{}) (interface{}, error) { return struct{}{}, nil /* OK */ },
}
subscriber = sd.FixedSubscriber{
0: endpoints[0],
1: endpoints[1],
2: endpoints[2],
}
retries = len(endpoints) - 1 // not quite enough retries
lb = loadbalancer.NewRoundRobin(subscriber)
ctx = context.Background()
)
if _, err := loadbalancer.Retry(retries, time.Second, lb)(ctx, struct{}{}); err == nil {
t.Errorf("expected error, got none")
}
}
func proxyingMiddleware(instances string, ctx context.Context, logger log.Logger) ServiceMiddleware {
// If instances is empty, don't proxy.
if instances == "" {
logger.Log("proxy_to", "none")
return func(next StringService) StringService { return next }
}
// Set some parameters for our client.
var (
qps = 100 // beyond which we will return an error
maxAttempts = 3 // per request, before giving up
maxTime = 250 * time.Millisecond // wallclock time, before giving up
)
// Otherwise, construct an endpoint for each instance in the list, and add
// it to a fixed set of endpoints. In a real service, rather than doing this
// by hand, you'd probably use package sd's support for your service
// discovery system.
var (
instanceList = split(instances)
subscriber sd.FixedSubscriber
)
logger.Log("proxy_to", fmt.Sprint(instanceList))
for _, instance := range instanceList {
var e endpoint.Endpoint
e = makeUppercaseProxy(ctx, instance)
e = circuitbreaker.Gobreaker(gobreaker.NewCircuitBreaker(gobreaker.Settings{}))(e)
e = ratelimit.NewTokenBucketLimiter(jujuratelimit.NewBucketWithRate(float64(qps), int64(qps)))(e)
subscriber = append(subscriber, e)
}
// Now, build a single, retrying, load-balancing endpoint out of all of
// those individual endpoints.
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(maxAttempts, maxTime, balancer)
// And finally, return the ServiceMiddleware, implemented by proxymw.
return func(next StringService) StringService {
return proxymw{ctx, next, retry}
}
}
func TestRetryTimeout(t *testing.T) {
var (
step = make(chan struct{})
e = func(context.Context, interface{}) (interface{}, error) { <-step; return struct{}{}, nil }
timeout = time.Millisecond
retry = loadbalancer.Retry(999, timeout, loadbalancer.NewRoundRobin(sd.FixedSubscriber{0: e}))
errs = make(chan error, 1)
invoke = func() { _, err := retry(context.Background(), struct{}{}); errs <- err }
)
go func() { step <- struct{}{} }() // queue up a flush of the endpoint
invoke() // invoke the endpoint and trigger the flush
if err := <-errs; err != nil { // that should succeed
t.Error(err)
}
go func() { time.Sleep(10 * timeout); step <- struct{}{} }() // a delayed flush
invoke() // invoke the endpoint
if err := <-errs; err != context.DeadlineExceeded { // that should not succeed
t.Errorf("wanted %v, got none", context.DeadlineExceeded)
}
}
// New returns a service that's load-balanced over instances of restsvc found
// in the provided Consul server. The mechanism of looking up restsvc
// instances in Consul is hard-coded into the client.
func New(consulAddr string, logger log.Logger) (restsvc.Service, error) {
apiclient, err := consulapi.NewClient(&consulapi.Config{
Address: consulAddr,
})
if err != nil {
return nil, err
}
// As the implementer of restsvc, we declare and enforce these
// parameters for all of the restsvc consumers.
var (
consulService = "restsvc"
consulTags = []string{"prod"}
passingOnly = true
retryMax = 3
retryTimeout = 500 * time.Millisecond
)
var (
sdclient = consul.NewClient(apiclient)
endpoints restsvc.Endpoints
)
{
factory := factoryFor(restsvc.MakePostConfigEndpoint)
subscriber := consul.NewSubscriber(sdclient, factory, logger, consulService, consulTags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(retryMax, retryTimeout, balancer)
endpoints.PostConfigEndpoint = retry
}
{
factory := factoryFor(restsvc.MakeGetConfigEndpoint)
subscriber := consul.NewSubscriber(sdclient, factory, logger, consulService, consulTags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(retryMax, retryTimeout, balancer)
endpoints.GetConfigEndpoint = retry
}
{
factory := factoryFor(restsvc.MakePutConfigEndpoint)
subscriber := consul.NewSubscriber(sdclient, factory, logger, consulService, consulTags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(retryMax, retryTimeout, balancer)
endpoints.PutConfigEndpoint = retry
}
{
factory := factoryFor(restsvc.MakePatchConfigEndpoint)
subscriber := consul.NewSubscriber(sdclient, factory, logger, consulService, consulTags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(retryMax, retryTimeout, balancer)
endpoints.PatchConfigEndpoint = retry
}
{
factory := factoryFor(restsvc.MakeDeleteConfigEndpoint)
subscriber := consul.NewSubscriber(sdclient, factory, logger, consulService, consulTags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(retryMax, retryTimeout, balancer)
endpoints.DeleteConfigEndpoint = retry
}
{
factory := factoryFor(restsvc.MakeGetAddressesEndpoint)
subscriber := consul.NewSubscriber(sdclient, factory, logger, consulService, consulTags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(retryMax, retryTimeout, balancer)
endpoints.GetAddressesEndpoint = retry
}
{
factory := factoryFor(restsvc.MakeGetAddressEndpoint)
subscriber := consul.NewSubscriber(sdclient, factory, logger, consulService, consulTags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(retryMax, retryTimeout, balancer)
endpoints.GetAddressEndpoint = retry
}
{
factory := factoryFor(restsvc.MakePostAddressEndpoint)
subscriber := consul.NewSubscriber(sdclient, factory, logger, consulService, consulTags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(retryMax, retryTimeout, balancer)
endpoints.PostAddressEndpoint = retry
}
{
factory := factoryFor(restsvc.MakeDeleteAddressEndpoint)
subscriber := consul.NewSubscriber(sdclient, factory, logger, consulService, consulTags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(retryMax, retryTimeout, balancer)
endpoints.DeleteAddressEndpoint = retry
}
return endpoints, nil
}
func main() {
var (
httpAddr = flag.String("http.addr", ":8000", "Address for HTTP (JSON) server")
consulAddr = flag.String("consul.addr", "", "Consul agent address")
retryMax = flag.Int("retry.max", 3, "per-request retries to different instances")
retryTimeout = flag.Duration("retry.timeout", 500*time.Millisecond, "per-request timeout, including retries")
)
flag.Parse()
// Logging domain.
var logger log.Logger
{
logger = log.NewLogfmtLogger(os.Stderr)
logger = log.NewContext(logger).With("ts", log.DefaultTimestampUTC)
logger = log.NewContext(logger).With("caller", log.DefaultCaller)
}
// Service discovery domain. In this example we use Consul.
var client consulsd.Client
{
consulConfig := api.DefaultConfig()
if len(*consulAddr) > 0 {
consulConfig.Address = *consulAddr
}
consulClient, err := api.NewClient(consulConfig)
if err != nil {
logger.Log("err", err)
os.Exit(1)
}
client = consulsd.NewClient(consulClient)
}
// Transport domain.
tracer := stdopentracing.GlobalTracer() // no-op
ctx := context.Background()
r := mux.NewRouter()
// Now we begin installing the routes. Each route corresponds to a single
// method: sum, concat, uppercase, and count.
// addsvc routes.
{
// Each method gets constructed with a factory. Factories take an
// instance string, and return a specific endpoint. In the factory we
// dial the instance string we get from Consul, and then leverage an
// addsvc client package to construct a complete service. We can then
// leverage the addsvc.Make{Sum,Concat}Endpoint constructors to convert
// the complete service to specific endpoint.
var (
tags = []string{}
passingOnly = true
endpoints = addsvc.Endpoints{}
)
{
factory := addsvcFactory(addsvc.MakeSumEndpoint, tracer, logger)
subscriber := consulsd.NewSubscriber(client, factory, logger, "addsvc", tags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(*retryMax, *retryTimeout, balancer)
endpoints.SumEndpoint = retry
}
{
factory := addsvcFactory(addsvc.MakeConcatEndpoint, tracer, logger)
subscriber := consulsd.NewSubscriber(client, factory, logger, "addsvc", tags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(*retryMax, *retryTimeout, balancer)
endpoints.ConcatEndpoint = retry
}
// Here we leverage the fact that addsvc comes with a constructor for an
// HTTP handler, and just install it under a particular path prefix in
// our router.
r.PathPrefix("addsvc/").Handler(addsvc.MakeHTTPHandler(ctx, endpoints, tracer, logger))
}
// stringsvc routes.
{
// addsvc had lots of nice importable Go packages we could leverage.
// With stringsvc we are not so fortunate, it just has some endpoints
// that we assume will exist. So we have to write that logic here. This
// is by design, so you can see two totally different methods of
// proxying to a remote service.
var (
tags = []string{}
passingOnly = true
uppercase endpoint.Endpoint
count endpoint.Endpoint
)
{
factory := stringsvcFactory(ctx, "GET", "/uppercase")
subscriber := consulsd.NewSubscriber(client, factory, logger, "stringsvc", tags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(*retryMax, *retryTimeout, balancer)
uppercase = retry
}
{
factory := stringsvcFactory(ctx, "GET", "/count")
subscriber := consulsd.NewSubscriber(client, factory, logger, "stringsvc", tags, passingOnly)
balancer := lb.NewRoundRobin(subscriber)
retry := lb.Retry(*retryMax, *retryTimeout, balancer)
count = retry
}
// We can use the transport/http.Server to act as our handler, all we
// have to do provide it with the encode and decode functions for our
// stringsvc methods.
r.Handle("/stringsvc/uppercase", httptransport.NewServer(ctx, uppercase, decodeUppercaseRequest, encodeJSONResponse))
r.Handle("/stringsvc/count", httptransport.NewServer(ctx, count, decodeCountRequest, encodeJSONResponse))
}
// Interrupt handler.
errc := make(chan error)
go func() {
c := make(chan os.Signal)
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
errc <- fmt.Errorf("%s", <-c)
}()
// HTTP transport.
go func() {
logger.Log("transport", "HTTP", "addr", *httpAddr)
errc <- http.ListenAndServe(*httpAddr, r)
}()
// Run!
logger.Log("exit", <-errc)
}