func TestBreaker(t *testing.T) {
// setup
r := mock.NewRegistry()
s := selector.NewSelector(selector.Registry(r))
c := client.NewClient(
// set the selector
client.Selector(s),
// add the breaker wrapper
client.Wrap(NewClientWrapper()),
)
req := c.NewJsonRequest("test.service", "Test.Method", map[string]string{
"foo": "bar",
})
var rsp map[string]interface{}
// Force to point of trip
for i := 0; i < (hystrix.DefaultVolumeThreshold * 2); i++ {
c.Call(context.TODO(), req, rsp)
}
err := c.Call(context.TODO(), req, rsp)
if err == nil {
t.Error("Expecting tripped breaker, got nil error")
}
if err.Error() != "hystrix: circuit open" {
t.Errorf("Expecting tripped breaker, got %v", err)
}
}
func TestLabelSelector(t *testing.T) {
counts := map[string]int{}
r := mock.NewRegistry()
r.Register(®istry.Service{
Name: "bar",
Version: "latest",
Nodes: []*registry.Node{
®istry.Node{
Id: "1",
Metadata: map[string]string{
"key1": "val1",
},
},
®istry.Node{
Id: "2",
Metadata: map[string]string{
"key2": "val2",
},
},
},
})
r.Register(®istry.Service{
Name: "bar",
Version: "1.0.0",
Nodes: []*registry.Node{
®istry.Node{
Id: "3",
Metadata: map[string]string{
"key1": "val1",
},
},
®istry.Node{
Id: "4",
},
},
})
ls := NewSelector(
selector.Registry(r),
Label("key2", "val2"),
Label("key1", "val1"),
)
next, err := ls.Select("bar")
if err != nil {
t.Errorf("Unexpected error calling ls select: %v", err)
}
for i := 0; i < 100; i++ {
node, err := next()
if err != nil {
t.Errorf("Expected node err, got err: %v", err)
}
counts[node.Id]++
}
t.Logf("Label Select Counts %v", counts)
}
func (s *server) proxy() http.Handler {
sel := selector.NewSelector(
selector.Registry((*cmd.DefaultOptions().Registry)),
)
director := func(r *http.Request) {
parts := strings.Split(r.URL.Path, "/")
if len(parts) < 2 {
return
}
if !re.MatchString(parts[1]) {
return
}
next, err := sel.Select(Namespace + "." + parts[1])
if err != nil {
return
}
r.URL.Scheme = "http"
s, err := next()
if err != nil {
return
}
r.URL.Host = fmt.Sprintf("%s:%d", s.Address, s.Port)
r.URL.Path = "/" + strings.Join(parts[2:], "/")
}
return &httputil.ReverseProxy{
Director: director,
}
}
// Proxy is a reverse proxy used by the micro web and api
func Proxy(ns string, ws bool) http.Handler {
return &proxy{
Namespace: ns,
Selector: selector.NewSelector(
selector.Registry((*cmd.DefaultOptions().Registry)),
),
re: regexp.MustCompile("^[a-zA-Z0-9]+$"),
ws: ws,
}
}
// Registry sets the registry for the service
// and the underlying components
func Registry(r registry.Registry) Option {
return func(o *Options) {
o.Registry = r
// Update Client and Server
o.Client.Init(client.Registry(r))
o.Server.Init(server.Registry(r))
// Update Selector
o.Client.Options().Selector.Init(selector.Registry(r))
}
}
func TestGRPCClient(t *testing.T) {
l, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer l.Close()
s := grpc.NewServer()
pb.RegisterGreeterServer(s, &greeterServer{})
go s.Serve(l)
defer s.Stop()
parts := strings.Split(l.Addr().String(), ":")
port, _ := strconv.Atoi(parts[len(parts)-1])
addr := strings.Join(parts[:len(parts)-1], ":")
r := mock.NewRegistry()
r.Register(®istry.Service{
Name: "test",
Version: "test",
Nodes: []*registry.Node{
®istry.Node{
Id: "test-1",
Address: addr,
Port: port,
},
},
})
se := selector.NewSelector(
selector.Registry(r),
)
c := NewClient(
client.Registry(r),
client.Selector(se),
)
req := c.NewRequest("test", "/helloworld.Greeter/SayHello", &pb.HelloRequest{
Name: "John",
})
rsp := pb.HelloReply{}
err = c.Call(context.TODO(), req, &rsp)
if err != nil {
t.Fatal(err)
}
if rsp.Message != "Hello John" {
t.Fatalf("Got unexpected response %v", rsp.Message)
}
}
func newClient(opts ...client.Option) client.Client {
options := client.Options{
CallOptions: client.CallOptions{
Backoff: client.DefaultBackoff,
Retry: client.DefaultRetry,
Retries: client.DefaultRetries,
RequestTimeout: client.DefaultRequestTimeout,
DialTimeout: transport.DefaultDialTimeout,
},
}
for _, o := range opts {
o(&options)
}
if len(options.ContentType) == 0 {
options.ContentType = "application/grpc+proto"
}
if options.Broker == nil {
options.Broker = broker.DefaultBroker
}
if options.Registry == nil {
options.Registry = registry.DefaultRegistry
}
if options.Selector == nil {
options.Selector = selector.NewSelector(
selector.Registry(options.Registry),
)
}
rc := &grpcClient{
once: sync.Once{},
opts: options,
}
c := client.Client(rc)
// wrap in reverse
for i := len(options.Wrappers); i > 0; i-- {
c = options.Wrappers[i-1](c)
}
return c
}
func (s *srv) proxy() http.Handler {
sel := selector.NewSelector(
selector.Registry((*cmd.DefaultOptions().Registry)),
)
director := func(r *http.Request) {
kill := func() {
r.URL.Host = ""
r.URL.Path = ""
r.URL.Scheme = ""
r.Host = ""
r.RequestURI = ""
}
parts := strings.Split(r.URL.Path, "/")
if len(parts) < 2 {
kill()
return
}
if !re.MatchString(parts[1]) {
kill()
return
}
next, err := sel.Select(Namespace + "." + parts[1])
if err != nil {
kill()
return
}
s, err := next()
if err != nil {
kill()
return
}
r.Header.Set(BasePathHeader, "/"+parts[1])
r.URL.Host = fmt.Sprintf("%s:%d", s.Address, s.Port)
r.URL.Path = "/" + strings.Join(parts[2:], "/")
r.URL.Scheme = "http"
r.Host = r.URL.Host
}
return &proxy{
Default: &httputil.ReverseProxy{Director: director},
Director: director,
}
}
func TestDefaultSelector(t *testing.T) {
counts := map[string]int{}
rs := newSelector(selector.Registry(mock.NewRegistry()))
next, err := rs.Select("foo")
if err != nil {
t.Errorf("Unexpected error calling default select: %v", err)
}
for i := 0; i < 100; i++ {
node, err := next()
if err != nil {
t.Errorf("Expected node err, got err: %v", err)
}
counts[node.Id]++
}
t.Logf("Default Counts %v", counts)
}
func TestCacheSelector(t *testing.T) {
counts := map[string]int{}
cache := NewSelector(selector.Registry(mock.NewRegistry()))
next, err := cache.Select("foo")
if err != nil {
t.Errorf("Unexpected error calling cache select: %v", err)
}
for i := 0; i < 100; i++ {
node, err := next()
if err != nil {
t.Errorf("Expected node err, got err: %v", err)
}
counts[node.Id]++
}
t.Logf("Cache Counts %v", counts)
}
func newOptions(options ...Option) Options {
opts := Options{
Codecs: make(map[string]codec.NewCodec),
CallOptions: CallOptions{
Backoff: DefaultBackoff,
Retries: DefaultRetries,
RequestTimeout: DefaultRequestTimeout,
DialTimeout: transport.DefaultDialTimeout,
},
PoolSize: DefaultPoolSize,
PoolTTL: DefaultPoolTTL,
}
for _, o := range options {
o(&opts)
}
if len(opts.ContentType) == 0 {
opts.ContentType = defaultContentType
}
if opts.Broker == nil {
opts.Broker = broker.DefaultBroker
}
if opts.Registry == nil {
opts.Registry = registry.DefaultRegistry
}
if opts.Selector == nil {
opts.Selector = selector.NewSelector(
selector.Registry(opts.Registry),
)
}
if opts.Transport == nil {
opts.Transport = transport.DefaultTransport
}
return opts
}
func (c *cmd) Before(ctx *cli.Context) error {
// Due to logger issues with glog, we need to do this
os.Args = os.Args[:1]
flag.Set("logtostderr", fmt.Sprintf("%v", ctx.Bool("logtostderr")))
flag.Set("alsologtostderr", fmt.Sprintf("%v", ctx.Bool("alsologtostderr")))
flag.Set("stderrthreshold", ctx.String("stderrthreshold"))
flag.Set("log_backtrace_at", ctx.String("log_backtrace_at"))
flag.Set("log_dir", ctx.String("log_dir"))
flag.Set("vmodule", ctx.String("vmodule"))
flag.Set("v", ctx.String("v"))
flag.Parse()
// If flags are set then use them otherwise do nothing
var serverOpts []server.Option
var clientOpts []client.Option
// Set the broker
if len(ctx.String("broker")) > 0 {
if b, ok := c.opts.Brokers[ctx.String("broker")]; ok {
n := b(strings.Split(ctx.String("broker_address"), ","))
*c.opts.Broker = n
} else {
return fmt.Errorf("Broker %s not found", ctx.String("broker"))
}
serverOpts = append(serverOpts, server.Broker(*c.opts.Broker))
clientOpts = append(clientOpts, client.Broker(*c.opts.Broker))
}
// Set the registry
if len(ctx.String("registry")) > 0 {
if r, ok := c.opts.Registries[ctx.String("registry")]; ok {
n := r(strings.Split(ctx.String("registry_address"), ","))
*c.opts.Registry = n
} else {
return fmt.Errorf("Registry %s not found", ctx.String("registry"))
}
serverOpts = append(serverOpts, server.Registry(*c.opts.Registry))
clientOpts = append(clientOpts, client.Registry(*c.opts.Registry))
(*c.opts.Selector).Init(selector.Registry(*c.opts.Registry))
clientOpts = append(clientOpts, client.Selector(*c.opts.Selector))
(*c.opts.Broker).Init(broker.Registry(*c.opts.Registry))
}
// Set the selector
if len(ctx.String("selector")) > 0 {
if s, ok := c.opts.Selectors[ctx.String("selector")]; ok {
n := s(selector.Registry(*c.opts.Registry))
*c.opts.Selector = n
} else {
return fmt.Errorf("Selector %s not found", ctx.String("selector"))
}
// No server option here. Should there be?
clientOpts = append(clientOpts, client.Selector(*c.opts.Selector))
}
// Set the transport
if len(ctx.String("transport")) > 0 {
if t, ok := c.opts.Transports[ctx.String("transport")]; ok {
n := t(strings.Split(ctx.String("transport_address"), ","))
*c.opts.Transport = n
} else {
return fmt.Errorf("Transport %s not found", ctx.String("transport"))
}
serverOpts = append(serverOpts, server.Transport(*c.opts.Transport))
clientOpts = append(clientOpts, client.Transport(*c.opts.Transport))
}
// Parse the server options
metadata := make(map[string]string)
for _, d := range ctx.StringSlice("server_metadata") {
var key, val string
parts := strings.Split(d, "=")
key = parts[0]
if len(parts) > 1 {
val = strings.Join(parts[1:], "=")
}
metadata[key] = val
}
if len(metadata) > 0 {
serverOpts = append(serverOpts, server.Metadata(metadata))
}
if len(ctx.String("server_name")) > 0 {
serverOpts = append(serverOpts, server.Name(ctx.String("server_name")))
}
if len(ctx.String("server_version")) > 0 {
serverOpts = append(serverOpts, server.Version(ctx.String("server_version")))
}
if len(ctx.String("server_id")) > 0 {
serverOpts = append(serverOpts, server.Id(ctx.String("server_id")))
}
if len(ctx.String("server_address")) > 0 {
serverOpts = append(serverOpts, server.Address(ctx.String("server_address")))
}
if len(ctx.String("server_advertise")) > 0 {
serverOpts = append(serverOpts, server.Advertise(ctx.String("server_advertise")))
}
// We have some command line opts for the server.
// Lets set it up
if len(serverOpts) > 0 {
(*c.opts.Server).Init(serverOpts...)
}
// Use an init option?
if len(clientOpts) > 0 {
(*c.opts.Client).Init(clientOpts...)
}
return nil
}
func TestWatcher(t *testing.T) {
r := setupRegistry()
c := cache.NewSelector(selector.Registry(r))
// wait for watcher to get setup
time.Sleep(time.Millisecond)
// check that service is blank
if _, err := c.Select("foo.service"); err != registry.ErrNotFound {
log.Fatal("expected registry.ErrNotFound")
}
// setup svc
svc1 := ®istry.Service{Name: "foo.service", Version: "1"}
register(r, "pod-1", svc1)
time.Sleep(time.Millisecond)
var wg sync.WaitGroup
wg.Add(3)
c.Select("foo.service", selector.WithFilter(func(svcs []*registry.Service) []*registry.Service {
defer wg.Done()
if !hasServices(svcs, []*registry.Service{svc1}) {
t.Fatal("expected services to match")
}
return nil
}))
// setup svc
svc2 := ®istry.Service{Name: "foo.service", Version: "1"}
register(r, "pod-2", svc2)
time.Sleep(time.Millisecond)
c.Select("foo.service", selector.WithFilter(func(svcs []*registry.Service) []*registry.Service {
defer wg.Done()
if !hasNodes(svcs[0].Nodes, []*registry.Node{svc1.Nodes[0], svc2.Nodes[0]}) {
t.Fatal("expected to have same nodes")
}
return nil
}))
// deregister
os.Setenv("HOSTNAME", "pod-1")
r.Deregister(svc1)
time.Sleep(time.Millisecond)
c.Select("foo.service", selector.WithFilter(func(svcs []*registry.Service) []*registry.Service {
defer wg.Done()
if !hasServices(svcs, []*registry.Service{svc2}) {
t.Fatal("expected services to match")
}
return nil
}))
// remove pods
teardownRegistry()
time.Sleep(time.Millisecond)
if _, err := c.Select("foo.service"); err != registry.ErrNotFound {
log.Fatal("expected registry.ErrNotFound")
}
out := make(chan bool)
go func() {
wg.Wait()
close(out)
}()
select {
case <-out:
return
case <-time.After(time.Second):
t.Fatal("expected c.Select() to be called 3 times")
}
}
func TestBlackList(t *testing.T) {
return
r := mock.NewRegistry()
r.Register(®istry.Service{
Name: "test",
Nodes: []*registry.Node{
®istry.Node{
Id: "test-1",
Address: "localhost",
Port: 10001,
},
®istry.Node{
Id: "test-2",
Address: "localhost",
Port: 10002,
},
®istry.Node{
Id: "test-3",
Address: "localhost",
Port: 10002,
},
},
})
rs := newSelector(selector.Registry(r))
next, err := rs.Select("test")
if err != nil {
t.Fatal(err)
}
node, err := next()
if err != nil {
t.Fatal(err)
}
for i := 0; i < 4; i++ {
rs.Mark("test", node, errors.New("error"))
}
next, err = rs.Select("test")
if err != nil {
t.Fatal(err)
}
// still expecting 2 nodes
seen := make(map[string]bool)
for i := 0; i < 10; i++ {
node, err = next()
if err != nil {
t.Fatal(err)
}
seen[node.Id] = true
}
if len(seen) != 2 {
t.Fatalf("Expected seen to be 2 %+v", seen)
}
// blacklist all of it
for i := 0; i < 20; i++ {
node, err = next()
if err != nil {
t.Fatal(err)
}
rs.Mark("test", node, errors.New("error"))
}
next, err = rs.Select("test")
if err != selector.ErrNoneAvailable {
t.Fatalf("Expected %v got %v", selector.ErrNoneAvailable, err)
}
}
func (c *cmd) Before(ctx *cli.Context) error {
// If flags are set then use them otherwise do nothing
var serverOpts []server.Option
var clientOpts []client.Option
// Set the broker
if name := ctx.String("broker"); len(name) > 0 || len(ctx.String("broker_address")) > 0 {
if len(name) == 0 {
name = defaultBroker
}
if b, ok := c.opts.Brokers[name]; ok {
n := b(broker.Addrs(strings.Split(ctx.String("broker_address"), ",")...))
*c.opts.Broker = n
} else {
return fmt.Errorf("Broker %s not found", name)
}
serverOpts = append(serverOpts, server.Broker(*c.opts.Broker))
clientOpts = append(clientOpts, client.Broker(*c.opts.Broker))
}
// Set the registry
if name := ctx.String("registry"); len(name) > 0 || len(ctx.String("registry_address")) > 0 {
if len(name) == 0 {
name = defaultRegistry
}
if r, ok := c.opts.Registries[name]; ok {
n := r(registry.Addrs(strings.Split(ctx.String("registry_address"), ",")...))
*c.opts.Registry = n
} else {
return fmt.Errorf("Registry %s not found", name)
}
serverOpts = append(serverOpts, server.Registry(*c.opts.Registry))
clientOpts = append(clientOpts, client.Registry(*c.opts.Registry))
(*c.opts.Selector).Init(selector.Registry(*c.opts.Registry))
clientOpts = append(clientOpts, client.Selector(*c.opts.Selector))
(*c.opts.Broker).Init(broker.Registry(*c.opts.Registry))
}
// Set the selector
if name := ctx.String("selector"); len(name) > 0 {
if s, ok := c.opts.Selectors[name]; ok {
n := s(selector.Registry(*c.opts.Registry))
*c.opts.Selector = n
} else {
return fmt.Errorf("Selector %s not found", name)
}
// No server option here. Should there be?
clientOpts = append(clientOpts, client.Selector(*c.opts.Selector))
}
// Set the transport
if name := ctx.String("transport"); len(name) > 0 || len(ctx.String("transport_address")) > 0 {
if len(name) == 0 {
name = defaultTransport
}
if t, ok := c.opts.Transports[name]; ok {
n := t(transport.Addrs(strings.Split(ctx.String("transport_address"), ",")...))
*c.opts.Transport = n
} else {
return fmt.Errorf("Transport %s not found", name)
}
serverOpts = append(serverOpts, server.Transport(*c.opts.Transport))
clientOpts = append(clientOpts, client.Transport(*c.opts.Transport))
}
// Parse the server options
metadata := make(map[string]string)
for _, d := range ctx.StringSlice("server_metadata") {
var key, val string
parts := strings.Split(d, "=")
key = parts[0]
if len(parts) > 1 {
val = strings.Join(parts[1:], "=")
}
metadata[key] = val
}
if len(metadata) > 0 {
serverOpts = append(serverOpts, server.Metadata(metadata))
}
if len(ctx.String("server_name")) > 0 {
serverOpts = append(serverOpts, server.Name(ctx.String("server_name")))
}
if len(ctx.String("server_version")) > 0 {
serverOpts = append(serverOpts, server.Version(ctx.String("server_version")))
}
if len(ctx.String("server_id")) > 0 {
serverOpts = append(serverOpts, server.Id(ctx.String("server_id")))
}
if len(ctx.String("server_address")) > 0 {
serverOpts = append(serverOpts, server.Address(ctx.String("server_address")))
}
if len(ctx.String("server_advertise")) > 0 {
serverOpts = append(serverOpts, server.Advertise(ctx.String("server_advertise")))
}
// client opts
if r := ctx.Int("client_retries"); r > 0 {
clientOpts = append(clientOpts, client.Retries(r))
}
if t := ctx.String("client_request_timeout"); len(t) > 0 {
d, err := time.ParseDuration(t)
if err != nil {
return fmt.Errorf("failed to parse client_request_timeout: %v", t)
}
clientOpts = append(clientOpts, client.RequestTimeout(d))
}
if r := ctx.Int("client_pool_size"); r > 0 {
clientOpts = append(clientOpts, client.PoolSize(r))
}
if t := ctx.String("client_pool_ttl"); len(t) > 0 {
d, err := time.ParseDuration(t)
if err != nil {
return fmt.Errorf("failed to parse client_pool_ttl: %v", t)
}
clientOpts = append(clientOpts, client.PoolTTL(d))
}
// We have some command line opts for the server.
// Lets set it up
if len(serverOpts) > 0 {
(*c.opts.Server).Init(serverOpts...)
}
// Use an init option?
if len(clientOpts) > 0 {
(*c.opts.Client).Init(clientOpts...)
}
return nil
}
func TestRPCHandler(t *testing.T) {
r := rmock.NewRegistry()
(*cmd.DefaultOptions().Client).Init(
client.Registry(r),
client.Selector(selector.NewSelector(selector.Registry(r))),
)
(*cmd.DefaultOptions().Server).Init(
server.Name("test"),
server.Registry(r),
)
(*cmd.DefaultOptions().Server).Handle(
(*cmd.DefaultOptions().Server).NewHandler(&TestHandler{t, metadata.Metadata{"Foo": "Bar"}}),
)
if err := server.Start(); err != nil {
t.Fatal(err)
}
if err := server.Register(); err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
request := map[string]string{
"service": "test",
"method": "TestHandler.Exec",
"request": "{}",
}
rb, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
b := bytes.NewBuffer(rb)
req, err := http.NewRequest("POST", "/rpc", b)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Foo", "Bar")
RPC(w, req)
if err := server.Deregister(); err != nil {
t.Fatal(err)
}
if err := server.Stop(); err != nil {
t.Fatal(err)
}
if w.Code != 200 {
t.Fatalf("Expected 200 response got %d %s", w.Code, w.Body.String())
}
}