// Usage: go test -run=Mesh
func TestMesh(t *testing.T) {
var wg sync.WaitGroup
totalSent = 0
totalRcvd = 0
totalSentTime = 0
totalRcvdTime = 0
sentSince = 0
rcvdSince = 0
subdone = 0
rcvdone = 0
done = make(chan struct{})
for i := 1; i < publishers+1; i++ {
time.Sleep(time.Millisecond * 20)
wg.Add(1)
go startMeshClient(t, i, &wg)
}
wg.Wait()
glog.Infof("Total Sent %d messages in %d ns, %d ns/msg, %d msgs/sec", totalSent, sentSince, int(float64(sentSince)/float64(totalSent)), int(float64(totalSent)/(float64(sentSince)/float64(time.Second))))
glog.Infof("Total Received %d messages in %d ns, %d ns/msg, %d msgs/sec", totalRcvd, rcvdSince, int(float64(rcvdSince)/float64(totalRcvd)), int(float64(totalRcvd)/(float64(rcvdSince)/float64(time.Second))))
}
// ListenAndServe listents to connections on the URI requested, and handles any
// incoming MQTT client sessions. It should not return until Close() is called
// or if there's some critical error that stops the server from running. The URI
// supplied should be of the form "protocol://host:port" that can be parsed by
// url.Parse(). For example, an URI could be "tcp://0.0.0.0:1883".
func (this *Server) ListenAndServe(uri string) error {
defer atomic.CompareAndSwapInt32(&this.running, 1, 0)
if !atomic.CompareAndSwapInt32(&this.running, 0, 1) {
return fmt.Errorf("server/ListenAndServe: Server is already running")
}
this.quit = make(chan struct{})
u, err := url.Parse(uri)
if err != nil {
return err
}
this.ln, err = net.Listen(u.Scheme, u.Host)
if err != nil {
return err
}
defer this.ln.Close()
glog.Infof("server/ListenAndServe: server is ready...")
var tempDelay time.Duration // how long to sleep on accept failure
for {
conn, err := this.ln.Accept()
if err != nil {
// http://zhen.org/blog/graceful-shutdown-of-go-net-dot-listeners/
select {
case <-this.quit:
return nil
default:
}
// Borrowed from go1.3.3/src/pkg/net/http/server.go:1699
if ne, ok := err.(net.Error); ok && ne.Temporary() {
if tempDelay == 0 {
tempDelay = 5 * time.Millisecond
} else {
tempDelay *= 2
}
if max := 1 * time.Second; tempDelay > max {
tempDelay = max
}
glog.Errorf("server/ListenAndServe: Accept error: %v; retrying in %v", err, tempDelay)
time.Sleep(tempDelay)
continue
}
return err
}
go this.handleConnection(conn)
}
}
func startFanPublisher(t testing.TB, cid int, wg *sync.WaitGroup) {
now := time.Now()
runClientTest(t, cid, wg, func(svc *service.Client) {
select {
case <-done:
case <-time.After(time.Second * time.Duration(subscribers)):
glog.Infof("(surgemq%d) Timed out waiting for subscribe response", cid)
return
}
cnt := messages
sent := 0
payload := make([]byte, size)
msg := message.NewPublishMessage()
msg.SetTopic(topic)
msg.SetQoS(qos)
for i := 0; i < cnt; i++ {
binary.BigEndian.PutUint32(payload, uint32(cid*cnt+i))
msg.SetPayload(payload)
err := svc.Publish(msg, nil)
if err != nil {
break
}
sent++
}
since := time.Since(now).Nanoseconds()
statMu.Lock()
totalSent += int64(sent)
totalSentTime += int64(since)
if since > sentSince {
sentSince = since
}
statMu.Unlock()
glog.Debugf("(surgemq%d) Sent %d messages in %d ns, %d ns/msg, %d msgs/sec", cid, sent, since, int(float64(since)/float64(cnt)), int(float64(sent)/(float64(since)/float64(time.Second))))
select {
case <-done2:
case <-time.Tick(time.Second * time.Duration(nap*publishers)):
glog.Errorf("Timed out waiting for messages to be received.")
}
})
}
func startServiceN(t testing.TB, u *url.URL, wg *sync.WaitGroup, ready1, ready2 chan struct{}, cnt int) {
defer wg.Done()
topics.Unregister("mem")
tp := topics.NewMemProvider()
topics.Register("mem", tp)
sessions.Unregister("mem")
sp := sessions.NewMemProvider()
sessions.Register("mem", sp)
ln, err := net.Listen(u.Scheme, u.Host)
require.NoError(t, err)
defer ln.Close()
close(ready1)
svr := &Server{
Authenticator: authenticator,
}
for i := 0; i < cnt; i++ {
conn, err := ln.Accept()
require.NoError(t, err)
_, err = svr.handleConnection(conn)
if authenticator == "mockFailure" {
require.Error(t, err)
return
} else {
require.NoError(t, err)
}
}
<-ready2
for _, svc := range svr.svcs {
glog.Infof("Stopping service %d", svc.id)
svc.stop()
}
}
// Close terminates the server by shutting down all the client connections and closing
// the listener. It will, as best it can, clean up after itself.
func (this *Server) Close() error {
// By closing the quit channel, we are telling the server to stop accepting new
// connection.
close(this.quit)
// We then close the net.Listener, which will force Accept() to return if it's
// blocked waiting for new connections.
this.ln.Close()
for _, svc := range this.svcs {
glog.Infof("Stopping service %d", svc.id)
svc.stop()
}
if this.sessMgr != nil {
this.sessMgr.Close()
}
if this.topicsMgr != nil {
this.topicsMgr.Close()
}
return nil
}
// FIXME: The order of closing here causes panic sometimes. For example, if receiver
// calls this, and closes the buffers, somehow it causes buffer.go:476 to panid.
func (this *service) stop() {
defer func() {
// Let's recover from panic
if r := recover(); r != nil {
glog.Errorf("(%s) Recovering from panic: %v", this.cid(), r)
}
}()
doit := atomic.CompareAndSwapInt64(&this.closed, 0, 1)
if !doit {
return
}
// Close quit channel, effectively telling all the goroutines it's time to quit
if this.done != nil {
glog.Debugf("(%s) closing this.done", this.cid())
close(this.done)
}
// Close the network connection
if this.conn != nil {
glog.Debugf("(%s) closing this.conn", this.cid())
this.conn.Close()
}
this.in.Close()
this.out.Close()
// Wait for all the goroutines to stop.
this.wgStopped.Wait()
glog.Debugf("(%s) Received %d bytes in %d messages.", this.cid(), this.inStat.bytes, this.inStat.msgs)
glog.Debugf("(%s) Sent %d bytes in %d messages.", this.cid(), this.outStat.bytes, this.outStat.msgs)
// Unsubscribe from all the topics for this client, only for the server side though
if !this.client && this.sess != nil {
topics, _, err := this.sess.Topics()
if err != nil {
glog.Errorf("(%s/%d): %v", this.cid(), this.id, err)
} else {
for _, t := range topics {
if err := this.topicsMgr.Unsubscribe([]byte(t), &this.onpub); err != nil {
glog.Errorf("(%s): Error unsubscribing topic %q: %v", this.cid(), t, err)
}
}
}
}
// Publish will message if WillFlag is set. Server side only.
if !this.client && this.sess.Cmsg.WillFlag() {
glog.Infof("(%s) service/stop: connection unexpectedly closed. Sending Will.", this.cid())
this.onPublish(this.sess.Will)
}
// Remove the client topics manager
if this.client {
topics.Unregister(this.sess.ID())
}
// Remove the session from session store if it's suppose to be clean session
if this.sess.Cmsg.CleanSession() && this.sessMgr != nil {
this.sessMgr.Del(this.sess.ID())
}
this.conn = nil
this.in = nil
this.out = nil
}
func startMeshClient(t testing.TB, cid int, wg *sync.WaitGroup) {
runClientTest(t, cid, wg, func(svc *service.Client) {
done2 := make(chan struct{})
cnt := messages
expected := publishers * cnt
received := 0
sent := 0
now := time.Now()
since := time.Since(now).Nanoseconds()
sub := newSubscribeMessage("test", 0)
svc.Subscribe(sub,
func(msg, ack message.Message, err error) error {
subs := atomic.AddInt64(&subdone, 1)
if subs == int64(publishers) {
close(done)
}
return nil
},
func(msg *message.PublishMessage) error {
if received == 0 {
now = time.Now()
}
received++
//glog.Debugf("(surgemq%d) messages received=%d", cid, received)
since = time.Since(now).Nanoseconds()
if received == expected {
close(done2)
}
return nil
})
select {
case <-done:
case <-time.After(time.Second * time.Duration(publishers)):
glog.Infof("(surgemq%d) Timed out waiting for subscribe response", cid)
return
}
payload := make([]byte, size)
msg := message.NewPublishMessage()
msg.SetTopic(topic)
msg.SetQoS(qos)
go func() {
now := time.Now()
for i := 0; i < cnt; i++ {
binary.BigEndian.PutUint32(payload, uint32(cid*cnt+i))
msg.SetPayload(payload)
err := svc.Publish(msg, nil)
if err != nil {
break
}
sent++
}
since := time.Since(now).Nanoseconds()
statMu.Lock()
totalSent += int64(sent)
totalSentTime += int64(since)
if since > sentSince {
sentSince = since
}
statMu.Unlock()
glog.Debugf("(surgemq%d) Sent %d messages in %d ns, %d ns/msg, %d msgs/sec", cid, sent, since, int(float64(since)/float64(cnt)), int(float64(sent)/(float64(since)/float64(time.Second))))
}()
select {
case <-done2:
case <-time.Tick(time.Second * time.Duration(nap*publishers)):
glog.Errorf("Timed out waiting for messages to be received.")
}
statMu.Lock()
totalRcvd += int64(received)
totalRcvdTime += int64(since)
if since > rcvdSince {
rcvdSince = since
}
statMu.Unlock()
glog.Debugf("(surgemq%d) Received %d messages in %d ns, %d ns/msg, %d msgs/sec", cid, received, since, int(float64(since)/float64(cnt)), int(float64(received)/(float64(since)/float64(time.Second))))
})
}
// HandleConnection is for the broker to handle an incoming connection from a client
func (this *Server) handleConnection(c io.Closer) (svc *service, err error) {
if c == nil {
return nil, ErrInvalidConnectionType
}
defer func() {
if err != nil {
c.Close()
}
}()
err = this.checkConfiguration()
if err != nil {
return nil, err
}
conn, ok := c.(net.Conn)
if !ok {
return nil, ErrInvalidConnectionType
}
// To establish a connection, we must
// 1. Read and decode the message.ConnectMessage from the wire
// 2. If no decoding errors, then authenticate using username and password.
// Otherwise, write out to the wire message.ConnackMessage with
// appropriate error.
// 3. If authentication is successful, then either create a new session or
// retrieve existing session
// 4. Write out to the wire a successful message.ConnackMessage message
// Read the CONNECT message from the wire, if error, then check to see if it's
// a CONNACK error. If it's CONNACK error, send the proper CONNACK error back
// to client. Exit regardless of error type.
conn.SetReadDeadline(time.Now().Add(time.Second * time.Duration(this.ConnectTimeout)))
resp := message.NewConnackMessage()
req, err := getConnectMessage(conn)
if err != nil {
if cerr, ok := err.(message.ConnackCode); ok {
//glog.Debugf("request message: %s\nresponse message: %s\nerror : %v", mreq, resp, err)
resp.SetReturnCode(cerr)
resp.SetSessionPresent(false)
writeMessage(conn, resp)
}
return nil, err
}
// Authenticate the user, if error, return error and exit
if err = this.authMgr.Authenticate(string(req.Username()), string(req.Password())); err != nil {
resp.SetReturnCode(message.ErrBadUsernameOrPassword)
resp.SetSessionPresent(false)
writeMessage(conn, resp)
return nil, err
}
if req.KeepAlive() == 0 {
req.SetKeepAlive(minKeepAlive)
}
svc = &service{
id: atomic.AddUint64(&gsvcid, 1),
client: false,
keepAlive: int(req.KeepAlive()),
connectTimeout: this.ConnectTimeout,
ackTimeout: this.AckTimeout,
timeoutRetries: this.TimeoutRetries,
conn: conn,
sessMgr: this.sessMgr,
topicsMgr: this.topicsMgr,
}
err = this.getSession(svc, req, resp)
if err != nil {
return nil, err
}
resp.SetReturnCode(message.ConnectionAccepted)
if err = writeMessage(c, resp); err != nil {
return nil, err
}
svc.inStat.increment(int64(req.Len()))
svc.outStat.increment(int64(resp.Len()))
if err := svc.start(); err != nil {
svc.stop()
return nil, err
}
//this.mu.Lock()
//this.svcs = append(this.svcs, svc)
//this.mu.Unlock()
glog.Infof("(%s) server/handleConnection: Connection established.", svc.cid())
return svc, nil
}
func startFanSubscribers(t testing.TB, cid int, wg *sync.WaitGroup) {
now := time.Now()
runClientTest(t, cid, wg, func(svc *service.Client) {
cnt := messages * publishers
received := 0
since := time.Since(now).Nanoseconds()
sub := newSubscribeMessage("test", 0)
svc.Subscribe(sub,
func(msg, ack message.Message, err error) error {
subs := atomic.AddInt64(&subdone, 1)
if subs == int64(subscribers) {
now = time.Now()
close(done)
}
return nil
},
func(msg *message.PublishMessage) error {
if received == 0 {
now = time.Now()
}
received++
//glog.Debugf("(surgemq%d) messages received=%d", cid, received)
since = time.Since(now).Nanoseconds()
if received == cnt {
rcvd := atomic.AddInt64(&rcvdone, 1)
if rcvd == int64(subscribers) {
close(done2)
}
}
return nil
})
select {
case <-done:
case <-time.After(time.Second * time.Duration(subscribers)):
glog.Infof("(surgemq%d) Timed out waiting for subscribe response", cid)
return
}
select {
case <-done2:
case <-time.Tick(time.Second * time.Duration(nap*publishers)):
glog.Errorf("Timed out waiting for messages to be received.")
}
statMu.Lock()
totalRcvd += int64(received)
totalRcvdTime += int64(since)
if since > rcvdSince {
rcvdSince = since
}
statMu.Unlock()
glog.Debugf("(surgemq%d) Received %d messages in %d ns, %d ns/msg, %d msgs/sec", cid, received, since, int(float64(since)/float64(cnt)), int(float64(received)/(float64(since)/float64(time.Second))))
})
}
