func NewSocketAndAddress(c *zmq.Context, bind bool, t zmq.Type, name string) (a *zmq.Socket, addr string, err error) {
addr = fmt.Sprintf("inproc://_routerelement_internal-%d", getUniqueId())
if a, err = c.NewSocket(t); err != nil {
goto Error
}
if name != "" {
a.SetIdentity(name)
}
if bind {
if err = a.Bind(addr); err != nil {
goto Error
}
} else {
if err = a.Connect(addr); err != nil {
goto Error
}
}
return
Error:
if a != nil {
a.Close()
a = nil
}
addr = ""
return
}
func (server *Server) monitor(ctx *zmq.Context, addr string) {
sock, err := ctx.NewSocket(zmq.PAIR)
if err != nil {
glog.Fatal(err)
}
if err := sock.Connect(addr); err != nil {
glog.Fatal(err)
}
defer sock.Close()
var lock sync.Mutex
var numClients int
for {
evtType, _, _, err := sock.RecvEvent(0)
if err != nil {
glog.Error(err)
break
}
switch zmq.Event(evtType) {
case zmq.EVENT_ACCEPTED:
lock.Lock()
numClients++
glog.Infof("Client #%d connected", numClients)
lock.Unlock()
case zmq.EVENT_DISCONNECTED:
lock.Lock()
glog.Infof("Client #%d disconnected", numClients)
numClients--
lock.Unlock()
}
}
}
func NewPair(c *zmq.Context) (a *zmq.Socket, b *zmq.Socket, err error) {
addr := fmt.Sprintf("inproc://_routerelement_internal-%d", getUniqueId())
if a, err = c.NewSocket(zmq.PAIR); err != nil {
goto Error
}
if err = a.Bind(addr); err != nil {
goto Error
}
if b, err = c.NewSocket(zmq.PAIR); err != nil {
goto Error
}
if err = b.Connect(addr); err != nil {
goto Error
}
return
Error:
if a != nil {
a.Close()
a = nil
}
if b != nil {
b.Close()
b = nil
}
return
}
// NewClientWithConnection returns a new Client to handle requests to the
// set of services at the other end of the connection.
// An existing connection in the form of a zmq socket together with the zmq
// context they were created with is used
func NewClientWithConnection(ctx *zmq.Context, conn *zmq.Socket) *Client {
// A router socket is the middle-man between client requests and actually sending/receiving on the wire
router, err := ctx.NewSocket(zmq.ROUTER)
if err != nil {
glog.Fatal(err)
}
if err := router.Bind(RouterURL); err != nil {
glog.Fatal(err)
}
client := &Client{
conn: conn,
endpoints: endpoints{
socket: conn,
},
router: router,
ctx: ctx,
}
// Start the proxy in an own routine since it is blocking
go func() {
if err := zmq.Proxy(conn, router, nil); err != nil {
switch zmq.AsErrno(err) {
case zmq.Errno(zmq.ETERM):
glog.Info(err)
case zmq.Errno(syscall.EINTR):
glog.Info(err)
default:
glog.Info(zmq.AsErrno(err))
glog.Info(err)
}
client.Close()
}
}()
// Socket monitor for connect event
monitorURL := "inproc://monitor"
if err := conn.Monitor(monitorURL, zmq.EVENT_CONNECTED|zmq.EVENT_DISCONNECTED); err != nil {
client.Close()
glog.Fatal(err)
}
go client.monitor(monitorURL)
return client
}
func (this *Spout) InitSocket(pull, push string) (err error) {
var reader, writer *zmq.Context
reader, err = zmq.NewContext()
if err != nil {
return
}
this.Component.Reader, err = reader.NewSocket(zmq.PULL)
if err != nil {
return
}
err = this.Component.Reader.Connect("tcp://127.0.0.1:" + pull)
if err != nil {
return
}
writer, err = zmq.NewContext()
if err != nil {
return
}
this.Component.Writer, err = writer.NewSocket(zmq.PUSH)
if err != nil {
return
}
err = this.Component.Writer.Bind("tcp://127.0.0.1:" + push)
return
}
func TestMultipleContexts(t *testing.T) {
chQuit := make(chan interface{})
chErr := make(chan error, 2)
needQuit := false
var sock1, sock2, serv1, serv2 *zmq.Socket
var serv_ctx1, serv_ctx2, ctx1, ctx2 *zmq.Context
var err error
defer func() {
if needQuit {
chQuit <- true
chQuit <- true
<-chErr
<-chErr
}
for _, s := range []*zmq.Socket{sock1, sock2, serv1, serv2} {
if s != nil {
s.SetLinger(0)
s.Close()
}
}
for _, c := range []*zmq.Context{serv_ctx1, serv_ctx2, ctx1, ctx2} {
if c != nil {
c.Term()
}
}
}()
addr1 := "tcp://127.0.0.1:9997"
addr2 := "tcp://127.0.0.1:9998"
serv_ctx1, err = zmq.NewContext()
if err != nil {
t.Fatal("NewContext:", err)
}
serv1, err = serv_ctx1.NewSocket(zmq.REP)
if err != nil {
t.Fatal("NewSocket:", err)
}
err = serv1.Bind(addr1)
if err != nil {
t.Fatal("Bind:", err)
}
serv_ctx2, err = zmq.NewContext()
if err != nil {
t.Fatal("NewContext:", err)
}
serv2, err = serv_ctx2.NewSocket(zmq.REP)
if err != nil {
t.Fatal("NewSocket:", err)
}
err = serv2.Bind(addr2)
if err != nil {
t.Fatal("Bind:", err)
}
new_service := func(sock *zmq.Socket, addr string) {
socket_handler := func(state zmq.State) error {
msg, err := sock.RecvMessage(0)
if err != nil {
return err
}
_, err = sock.SendMessage(addr, msg)
return err
}
quit_handler := func(interface{}) error {
return errors.New("quit")
}
reactor := zmq.NewReactor()
reactor.AddSocket(sock, zmq.POLLIN, socket_handler)
reactor.AddChannel(chQuit, 1, quit_handler)
err = reactor.Run(100 * time.Millisecond)
chErr <- err
}
go new_service(serv1, addr1)
go new_service(serv2, addr2)
needQuit = true
time.Sleep(time.Second)
// default context
sock1, err = zmq.NewSocket(zmq.REQ)
if err != nil {
t.Fatal("NewSocket:", err)
}
sock2, err = zmq.NewSocket(zmq.REQ)
if err != nil {
t.Fatal("NewSocket:", err)
}
err = sock1.Connect(addr1)
if err != nil {
t.Fatal("sock1.Connect:", err)
}
err = sock2.Connect(addr2)
if err != nil {
t.Fatal("sock2.Connect:", err)
}
_, err = sock1.SendMessage(addr1)
if err != nil {
t.Fatal("sock1.SendMessage:", err)
}
_, err = sock2.SendMessage(addr2)
if err != nil {
t.Fatal("sock2.SendMessage:", err)
}
msg, err := sock1.RecvMessage(0)
expected := []string{addr1, addr1}
if err != nil || !arrayEqual(msg, expected) {
t.Errorf("sock1.RecvMessage: expected %v %v, got %v %v", nil, expected, err, msg)
}
msg, err = sock2.RecvMessage(0)
expected = []string{addr2, addr2}
if err != nil || !arrayEqual(msg, expected) {
t.Errorf("sock2.RecvMessage: expected %v %v, got %v %v", nil, expected, err, msg)
}
err = sock1.Close()
sock1 = nil
if err != nil {
t.Fatal("sock1.Close:", err)
}
err = sock2.Close()
sock2 = nil
if err != nil {
t.Fatal("sock2.Close:", err)
}
// non-default contexts
ctx1, err = zmq.NewContext()
if err != nil {
t.Fatal("NewContext:", err)
}
ctx2, err = zmq.NewContext()
if err != nil {
t.Fatal("NewContext:", err)
}
sock1, err = ctx1.NewSocket(zmq.REQ)
if err != nil {
t.Fatal("ctx1.NewSocket:", err)
}
sock2, err = ctx2.NewSocket(zmq.REQ)
if err != nil {
t.Fatal("ctx2.NewSocket:", err)
}
err = sock1.Connect(addr1)
if err != nil {
t.Fatal("sock1.Connect:", err)
}
err = sock2.Connect(addr2)
if err != nil {
t.Fatal("sock2.Connect:", err)
}
_, err = sock1.SendMessage(addr1)
if err != nil {
t.Fatal("sock1.SendMessage:", err)
}
_, err = sock2.SendMessage(addr2)
if err != nil {
t.Fatal("sock2.SendMessage:", err)
}
msg, err = sock1.RecvMessage(0)
expected = []string{addr1, addr1}
if err != nil || !arrayEqual(msg, expected) {
t.Errorf("sock1.RecvMessage: expected %v %v, got %v %v", nil, expected, err, msg)
}
msg, err = sock2.RecvMessage(0)
expected = []string{addr2, addr2}
if err != nil || !arrayEqual(msg, expected) {
t.Errorf("sock2.RecvMessage: expected %v %v, got %v %v", nil, expected, err, msg)
}
err = sock1.Close()
sock1 = nil
if err != nil {
t.Fatal("sock1.Close:", err)
}
err = sock2.Close()
sock2 = nil
if err != nil {
t.Fatal("sock2.Close:", err)
}
err = ctx1.Term()
ctx1 = nil
if err != nil {
t.Fatal("ctx1.Term", nil)
}
err = ctx2.Term()
ctx1 = nil
if err != nil {
t.Fatal("ctx2.Term", nil)
}
needQuit = false
for i := 0; i < 2; i++ {
// close(chQuit) doesn't work because the reactor removes closed channels, instead of acting on them
chQuit <- true
err = <-chErr
if err.Error() != "quit" {
t.Errorf("Expected error value quit, got %v", err)
}
}
}
func TestServiceRouter(t *testing.T) {
numPeers := 50
numServices := 100
var c *zmq.Context
var m *Mirror
var socks []*zmq.Socket = make([]*zmq.Socket, numPeers)
var addr []string = make([]string, numPeers)
var err error
defer func() {
for _, s := range socks {
if s != nil {
s.Close()
}
}
if m != nil {
m.Close()
}
if c != nil {
c.Term()
}
}()
if c, err = zmq.NewContext(); err != nil {
t.Fatalf("Failed to create ZMQ context: %v", err)
}
if m, err = NewMirror(t, c, 1); err != nil {
t.Fatalf("Failed to create a new mirror: %v", err)
}
for np := 0; np < numPeers; np++ {
if socks[np], addr[np], err = svcrouter.NewHalfPair(c, true); err != nil {
t.Fatalf("Failed to create half pair A: %v", err)
}
if err = m.AddPeer(svcrouter.PeerDefinition{
Name: dummyPeerName(np),
ZmqType: zmq.PAIR,
Address: addr[np],
Bind: false,
PeerImpl: &MirrorPeerImpl{},
}); err != nil {
t.Fatalf("Could not add peer %d: %v", np, err)
}
}
svcrouter.Barrier()
r := svcrouter.NewServiceRouter()
t.Log("Adding by Peer/Service")
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
for ns := 0; ns < numServices; ns++ {
serviceType, serviceId := dummyServiceName(ns)
e := svcrouter.NewEndpointEnvelope(serviceType, serviceId, "")
if err := r.AddService(m.RouterElement, peerName, e); err != nil {
t.Fatalf("Could not add service %d to peer %d: %v", ns, np, err)
}
}
}
if err := r.Validate(); err != nil {
t.Fatalf("Failed validation 1: %v", err)
}
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
if l := r.LenPeerEntries(m.RouterElement, peerName); l != numServices {
t.Fatalf("Peer %d has service element mismatch: %d != %d", l, numServices)
}
}
t.Log("Removing by Service/Peer")
for ns := 0; ns < numServices; ns++ {
serviceType, serviceId := dummyServiceName(ns)
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
e := svcrouter.NewEndpointEnvelope(serviceType, serviceId, "")
if err := r.DeleteService(m.RouterElement, peerName, e); err != nil {
t.Fatalf("Could not delete service %d to peer %d: %v", ns, np, err)
}
}
if ns%(numServices/10) == 0 {
if err := r.Validate(); err != nil {
t.Fatalf("Failed validation 2: %v", err)
}
}
}
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
if l := r.LenPeerEntries(m.RouterElement, peerName); l != 0 {
t.Fatalf("Peer %d has service element mismatch: %d != 0", l)
}
}
if err := r.Validate(); err != nil {
t.Fatalf("Failed validation 3: %v", err)
}
t.Log("Adding by Service/Peer")
for ns := 0; ns < numServices; ns++ {
serviceType, serviceId := dummyServiceName(ns)
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
e := svcrouter.NewEndpointEnvelope(serviceType, serviceId, "")
if err := r.AddService(m.RouterElement, peerName, e); err != nil {
t.Fatalf("Could not add service %d to peer %d: %v", ns, np, err)
}
}
}
if err := r.Validate(); err != nil {
t.Fatalf("Failed validation 4: %v", err)
}
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
if l := r.LenPeerEntries(m.RouterElement, peerName); l != numServices {
t.Fatalf("Peer %d has service element mismatch: %d != %d", l, numServices)
}
}
t.Log("Removing by Peer/Service")
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
for ns := 0; ns < numServices; ns++ {
serviceType, serviceId := dummyServiceName(ns)
e := svcrouter.NewEndpointEnvelope(serviceType, serviceId, "")
if err := r.DeleteService(m.RouterElement, peerName, e); err != nil {
t.Fatalf("Could not delete service %d to peer %d: %v", ns, np, err)
}
}
if np%10 == 0 {
if err := r.Validate(); err != nil {
t.Fatalf("Failed validation 5: %v", err)
}
}
}
if err := r.Validate(); err != nil {
t.Fatalf("Failed validation 6: %v", err)
}
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
if l := r.LenPeerEntries(m.RouterElement, peerName); l != 0 {
t.Fatalf("Peer %d has service element mismatch: %d != 0", l)
}
}
t.Log("Adding by Service/Peer")
for ns := 0; ns < numServices; ns++ {
serviceType, serviceId := dummyServiceName(ns)
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
e := svcrouter.NewEndpointEnvelope(serviceType, serviceId, "")
if err := r.AddService(m.RouterElement, peerName, e); err != nil {
t.Fatalf("Could not add service %d to peer %d: %v", ns, np, err)
}
}
}
if err := r.Validate(); err != nil {
t.Fatalf("Failed validation 7: %v", err)
}
for it := 0; it < 10; it++ {
// How exciting. Let's try some routing!
serviceType, serviceId := dummyServiceName(0)
d := make(map[string]int, numPeers)
for np := 0; np < numPeers; np++ {
se := r.GetServerEntry(serviceType, serviceId, "")
d[se.Peer.Name] += 1
}
if len(d) != numPeers {
t.Fatalf("Failed load balance test 1")
}
for np := 0; np < numPeers; np++ {
se := r.GetServerEntry(serviceType, serviceId, "")
if d[se.Peer.Name] != 1 {
t.Fatalf("Failed load balance test 2")
}
d[se.Peer.Name] += 1
}
if len(d) != numPeers {
t.Fatalf("Failed load balance test 3")
}
}
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
if l := r.LenPeerEntries(m.RouterElement, peerName); l != numServices {
t.Fatalf("Peer %d has service element mismatch: %d != %d", l, numServices)
}
}
t.Log("Removing by Peer (all entries)")
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
if err := r.DeleteAllPeerEntries(m.RouterElement, peerName); err != nil {
t.Fatalf("Could not add delete all services from peer %d: %v", np, err)
}
}
for np := 0; np < numPeers; np++ {
peerName := dummyPeerName(np)
if l := r.LenPeerEntries(m.RouterElement, peerName); l != 0 {
t.Fatalf("Peer %d has service element mismatch: %d != 0", l)
}
}
if err := r.Validate(); err != nil {
t.Fatalf("Failed validation final: %v", err)
}
}
func TestRouterElement(t *testing.T) {
var c *zmq.Context
var m *Mirror
var sa, sb *zmq.Socket
var err error
var addra string
var addrb string
defer func() {
if sa != nil {
sa.Close()
}
if sb != nil {
sb.Close()
}
if m != nil {
m.Close()
}
if c != nil {
c.Term()
}
}()
if c, err = zmq.NewContext(); err != nil {
t.Fatalf("Failed to create ZMQ context: %v", err)
}
if m, err = NewMirror(t, c, 1); err != nil {
t.Fatalf("Failed to create a new mirror: %v", err)
}
if sa, addra, err = svcrouter.NewHalfPair(c, true); err != nil {
t.Fatalf("Failed to create half pair A: %v", err)
}
if sb, addrb, err = svcrouter.NewHalfPair(c, true); err != nil {
t.Fatalf("Failed to create half pair B: %v", err)
}
if err = m.AddPeer(svcrouter.PeerDefinition{
Name: "ab",
ZmqType: zmq.PAIR,
Address: addra,
Bind: false,
PeerImpl: &MirrorPeerImpl{},
}); err != nil {
t.Fatalf("Could not add peer A: %v", err)
}
if err = m.AddPeer(svcrouter.PeerDefinition{
Name: "ba",
ZmqType: zmq.PAIR,
Address: addrb,
Bind: false,
PeerImpl: &MirrorPeerImpl{},
}); err != nil {
t.Fatalf("Could not add peer B: %v", err)
}
svcrouter.Barrier()
num := 100
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < num; i++ {
if _, err := sb.RecvMessage(0); err != nil {
t.Fatalf("sb receive error: %v", err)
}
}
}()
msg := [][]byte{[]byte("Hello"), []byte("World")}
for i := 0; i < num; i++ {
if _, err := sa.SendMessage(msg); err != nil {
t.Fatalf("sa send error: %v", err)
}
}
wg.Wait()
}
func TestServiceDispatcher(t *testing.T) {
numPeers := 10
numServices := 10
var c *zmq.Context
var sd *svcrouter.ServiceDispatcher
var socks []*zmq.Socket = make([]*zmq.Socket, numPeers)
var addr []string = make([]string, numPeers)
var killReflectors []chan bool = make([]chan bool, numPeers)
var err error
var wg sync.WaitGroup
defer func() {
for _, s := range socks {
if s != nil {
s.Close()
}
}
if sd != nil {
sd.Close()
}
if c != nil {
c.Term()
}
}()
if c, err = zmq.NewContext(); err != nil {
t.Fatalf("Failed to create ZMQ context: %v", err)
}
if sd, err = svcrouter.NewServiceDispatcher(c, 1); err != nil {
t.Fatalf("Failed to create a new service dispatcher: %v", err)
}
sd.NoEnvRouterPeer = dummyPeerName(0)
sd.EnvRouterPeer = dummyPeerName(2)
for np := 0; np < numPeers; np++ {
if np%2 == 0 {
name := fmt.Sprintf("C%04d st=ST%d si=x", np, (np/2)%2)
if socks[np], addr[np], err = svcrouter.NewSocketAndAddress(c, true, zmq.REQ, name); err != nil {
t.Fatalf("Failed to create half pair A: %v", err)
}
if err = sd.AddPeer(svcrouter.PeerDefinition{
Name: dummyPeerName(np),
ZmqType: zmq.ROUTER,
Address: addr[np],
Bind: false,
}, svcrouter.PT_CLIENTSNOENV); err != nil {
t.Fatalf("Could not add peer %d: %v", np, err)
}
} else {
name := fmt.Sprintf("C%04d st=ST%d si=x", np, ((np-1)/2)%2)
if socks[np], addr[np], err = svcrouter.NewSocketAndAddress(c, true, zmq.REP, name); err != nil {
t.Fatalf("Failed to create half pair A: %v", err)
}
if err = sd.AddPeer(svcrouter.PeerDefinition{
Name: dummyPeerName(np),
ZmqType: zmq.DEALER,
Address: addr[np],
Bind: false,
}, svcrouter.PT_DOWNSTREAMENVREP); err != nil {
t.Fatalf("Could not add peer %d: %v", np, err)
}
killReflectors[np] = reflector(t, socks[np], np, &wg)
}
}
svcrouter.Barrier()
for np := 1; np < numPeers; np += 2 {
for ns := 0; ns < numServices; ns++ {
serviceType := fmt.Sprintf("ST%d", ns)
serviceId := "x"
e := svcrouter.NewEndpointEnvelope(serviceType, serviceId, "")
if err := sd.AddService(sd.RouterElement, dummyPeerName(np), e); err != nil {
t.Fatalf("Could not add service %d to peer %d: %v", ns, np, err)
}
}
}
msg := [][]byte{[]byte("Hello"), []byte("World")}
for it := 0; it < 2; it++ {
if _, err := socks[0].SendMessage(msg); err != nil {
t.Fatalf("socks send error: %v", err)
}
if msg2, err := socks[0].RecvMessageBytes(0); err != nil {
t.Fatalf("socks receive error: %v", err)
} else {
svcrouter.DumpMsg("FINAL", msg2)
if !msgEqual(msg, msg2) {
t.Fatalf("socks messages differ")
}
}
}
for _, v := range killReflectors {
if v != nil {
close(v)
}
}
wg.Wait()
}