func TestSplitBufferPubOp4(t *testing.T) {
c := &client{subs: hashmap.New()}
pubAll := []byte("PUB foo 11\r\nhello world\r\n")
pub := pubAll[:12]
if err := c.parse(pub); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if !bytes.Equal(c.pa.subject, []byte("foo")) {
t.Fatalf("Unexpected subject: '%s' vs '%s'\n", c.pa.subject, "foo")
}
// Simulate next read of network, make sure pub state is saved
// until msg payload has cleared.
copy(pubAll, "XXXXXXXXXXXX")
if !bytes.Equal(c.pa.subject, []byte("foo")) {
t.Fatalf("Unexpected subject: '%s' vs '%s'\n", c.pa.subject, "foo")
}
if !bytes.Equal(c.pa.reply, []byte("")) {
t.Fatalf("Unexpected reply: '%s' vs '%s'\n", c.pa.reply, "")
}
if !bytes.Equal(c.pa.szb, []byte("11")) {
t.Fatalf("Unexpected size bytes: '%s' vs '%s'\n", c.pa.szb, "11")
}
}
func TestSplitBufferUnsubOp(t *testing.T) {
s := &Server{sl: sublist.New()}
c := &client{srv: s, subs: hashmap.New()}
subop := []byte("SUB foo 1024\r\n")
if err := c.parse(subop); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != OP_START {
t.Fatalf("Expected OP_START state vs %d\n", c.state)
}
unsubop := []byte("UNSUB 1024\r\n")
unsubop1 := unsubop[:8]
unsubop2 := unsubop[8:]
if err := c.parse(unsubop1); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != UNSUB_ARG {
t.Fatalf("Expected UNSUB_ARG state vs %d\n", c.state)
}
if err := c.parse(unsubop2); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != OP_START {
t.Fatalf("Expected OP_START state vs %d\n", c.state)
}
r := s.sl.Match([]byte("foo"))
if r != nil && len(r) != 0 {
t.Fatalf("Should be no subscriptions in results: %+v\n", r)
}
}
// New will create a default sublist
func New() *Sublist {
return &Sublist{
root: newLevel(),
cache: hashmap.New(),
cmax: defaultCacheMax,
stats: stats{since: time.Now()},
}
}
func TestSplitDanglingArgBuf(t *testing.T) {
c := &client{subs: hashmap.New()}
// We test to make sure we do not dangle any argBufs after processing
// since that could lead to performance issues.
// SUB
subop := []byte("SUB foo 1\r\n")
c.parse(subop[:6])
c.parse(subop[6:])
if c.argBuf != nil {
t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf)
}
// UNSUB
unsubop := []byte("UNSUB 1024\r\n")
c.parse(unsubop[:8])
c.parse(unsubop[8:])
if c.argBuf != nil {
t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf)
}
// PUB
pubop := []byte("PUB foo.bar INBOX.22 11\r\nhello world\r\n")
c.parse(pubop[:22])
c.parse(pubop[22:25])
if c.argBuf == nil {
t.Fatal("Expected a nil argBuf!")
}
c.parse(pubop[25:])
if c.argBuf != nil {
t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf)
}
// MINUS_ERR
errop := []byte("-ERR Too Long\r\n")
c.parse(errop[:8])
c.parse(errop[8:])
if c.argBuf != nil {
t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf)
}
// CONNECT_ARG
connop := []byte("CONNECT {\"verbose\":false,\"ssl_required\":false," +
"\"user\":\"test\",\"pedantic\":true,\"pass\":\"pass\"}\r\n")
c.parse(connop[:22])
c.parse(connop[22:])
if c.argBuf != nil {
t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf)
}
}
func TestSplitConnectArg(t *testing.T) {
c := &client{subs: hashmap.New()}
connectAll := []byte("CONNECT {\"verbose\":false,\"ssl_required\":false," +
"\"user\":\"test\",\"pedantic\":true,\"pass\":\"pass\"}\r\n")
argJson := connectAll[8:]
c1 := connectAll[:5]
c2 := connectAll[5:22]
c3 := connectAll[22 : len(connectAll)-2]
c4 := connectAll[len(connectAll)-2:]
if err := c.parse(c1); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.argBuf != nil {
t.Fatalf("Unexpected argBug placeholder.\n")
}
if err := c.parse(c2); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.argBuf == nil {
t.Fatalf("Expected argBug to not be nil.\n")
}
if !bytes.Equal(c.argBuf, argJson[:14]) {
t.Fatalf("argBuf not correct, received %q, wanted %q\n", argJson[:14], c.argBuf)
}
if err := c.parse(c3); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.argBuf == nil {
t.Fatalf("Expected argBug to not be nil.\n")
}
if !bytes.Equal(c.argBuf, argJson[:len(argJson)-2]) {
t.Fatalf("argBuf not correct, received %q, wanted %q\n",
argJson[:len(argJson)-2], c.argBuf)
}
if err := c.parse(c4); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.argBuf != nil {
t.Fatalf("Unexpected argBuf placeholder.\n")
}
}
// Lock should be held
func (c *client) initClient(tlsConn bool) {
s := c.srv
c.cid = atomic.AddUint64(&s.gcid, 1)
c.bw = bufio.NewWriterSize(c.nc, s.opts.BufSize)
c.subs = hashmap.New()
c.debug = (atomic.LoadInt32(&debug) != 0)
c.trace = (atomic.LoadInt32(&trace) != 0)
// This is a scratch buffer used for processMsg()
// The msg header starts with "MSG ",
// in bytes that is [77 83 71 32].
c.msgb = [msgScratchSize]byte{77, 83, 71, 32}
// This is to track pending clients that have data to be flushed
// after we process inbound msgs from our own connection.
c.pcd = make(map[*client]struct{})
// snapshot the string version of the connection
conn := "-"
if ip, ok := c.nc.(*net.TCPConn); ok {
addr := ip.RemoteAddr().(*net.TCPAddr)
conn = fmt.Sprintf("%s:%d", addr.IP, addr.Port)
}
switch c.typ {
case CLIENT:
c.ncs = fmt.Sprintf("%s - cid:%d", conn, c.cid)
case ROUTER:
c.ncs = fmt.Sprintf("%s - rid:%d", conn, c.cid)
}
// No clue why, but this stalls and kills performance on Mac (Mavericks).
//
// if ip, ok := c.nc.(*net.TCPConn); ok {
// ip.SetReadBuffer(s.opts.BufSize)
// ip.SetWriteBuffer(2 * s.opts.BufSize)
// }
if !tlsConn {
// Set the Ping timer
c.setPingTimer()
// Spin up the read loop.
go c.readLoop()
}
}
func TestSplitBufferPubOp5(t *testing.T) {
c := &client{subs: hashmap.New()}
pubAll := []byte("PUB foo 11\r\nhello world\r\n")
// Splits need to be on MSG_END now too, so make sure we check that.
// Split between \r and \n
pub := pubAll[:len(pubAll)-1]
if err := c.parse(pub); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.msgBuf == nil {
t.Fatalf("msgBuf should not be nil!\n")
}
if !bytes.Equal(c.msgBuf, []byte("hello world\r")) {
t.Fatalf("c.msgBuf did not snaphot the msg")
}
}
func TestSplitBufferPubOp2(t *testing.T) {
c := &client{subs: hashmap.New()}
pub := []byte("PUB foo.bar INBOX.22 11\r\nhello world\r\n")
pub1 := pub[:30]
pub2 := pub[30:]
if err := c.parse(pub1); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != MSG_PAYLOAD {
t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state)
}
if err := c.parse(pub2); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != OP_START {
t.Fatalf("Expected OP_START state vs %d\n", c.state)
}
}
func TestSplitBufferSubOp(t *testing.T) {
cli, trash := net.Pipe()
defer cli.Close()
defer trash.Close()
s := &Server{sl: sublist.New()}
c := &client{srv: s, subs: hashmap.New(), nc: cli}
subop := []byte("SUB foo 1\r\n")
subop1 := subop[:6]
subop2 := subop[6:]
if err := c.parse(subop1); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != SUB_ARG {
t.Fatalf("Expected SUB_ARG state vs %d\n", c.state)
}
if err := c.parse(subop2); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != OP_START {
t.Fatalf("Expected OP_START state vs %d\n", c.state)
}
r := s.sl.Match([]byte("foo"))
if r == nil || len(r) != 1 {
t.Fatalf("Did not match subscription properly: %+v\n", r)
}
sub := r[0].(*subscription)
if !bytes.Equal(sub.subject, []byte("foo")) {
t.Fatalf("Subject did not match expected 'foo' : '%s'\n", sub.subject)
}
if !bytes.Equal(sub.sid, []byte("1")) {
t.Fatalf("Sid did not match expected '1' : '%s'\n", sub.sid)
}
if sub.queue != nil {
t.Fatalf("Received a non-nil queue: '%s'\n", sub.queue)
}
}
// Dedup dedups over a window of windowSize Frames a
// stream of frames from r into w. dupsW can be nil. If
// dupsW is supplied, recognized duplicate events will
// be written to this io.Writer. If detectOnly
// is true, we will return a DupDetectedErr at the
// first duplicate, to enable scanning a filesystem.
// With detectOnly set, no dedupped output Frames
// are written.
func Dedup(r io.Reader, w io.Writer, windowSize int, dupsW io.Writer, detectOnly bool) error {
fr := NewFrameReader(r, 1024*1024)
fw := NewFrameWriter(w, 1024*1024)
var dupsWriter *FrameWriter
if dupsW != nil {
dupsWriter = NewFrameWriter(dupsW, 1024*1024)
}
window := make([]*dedup, windowSize)
present := hashmap.New()
defer func() {
fw.Flush()
fw.Sync()
if dupsWriter != nil {
dupsWriter.Flush()
dupsWriter.Sync()
}
}()
var err error
var ptr *dedup
for i := 0; err == nil; i++ {
var frame Frame
_, _, err, _ = fr.NextFrame(&frame)
if err != nil {
if err != io.EOF {
return fmt.Errorf("dedup error from fr.NextFrame(): '%v'", err)
}
} else { // err == nil
// got a frame, check if it is a dup
hash := frame.Blake2b()
p := present.Get(hash)
if p == nil {
// not already seen
if !detectOnly {
fw.Append(&frame)
}
// memorize the new
ptr = &dedup{count: 1, hash: hash}
present.Set(hash, ptr)
} else {
// else skip duplicates, but keep a count so
// our window is always correct. Otherwise
// a duplicate can be missed if masked
// by an even earlier pre-window duplicate.
// e.g. with window size 3 and this sequence
// index: 0 1 2 3 4
// values: [ 1 2 1 3 1 ]
// ^ ^ ^ <-- highlight the duplicates
// Without the count, the dup at index 2
// would be forgotten about when the index 0
// value rolls out of the 'present' hash,
// meaning that the dup at index 4 would
// not be recognized.
if detectOnly {
return NewDupDetectedErr(frame.Stringify(int64(i), false, true, false))
}
ptr = p.(*dedup)
ptr.count++
if dupsWriter != nil {
if !detectOnly {
dupsWriter.Append(&frame)
}
}
}
if i >= windowSize {
// deal with rolling off the last first, so
// we have space to write
last := present.Get(window[i%windowSize].hash).(*dedup)
last.count--
if last.count == 0 {
present.Remove(last.hash)
}
}
// and write our hash into our window ring
window[i%windowSize] = ptr
} // end else err == nil from NextFrame()
if i%1000 == 999 {
fw.Flush()
if dupsWriter != nil {
dupsWriter.Flush()
}
}
}
return nil
}
// Create a new default level. We use FNV1A as the hash
// algortihm for the tokens, which should be short.
func newLevel() *level {
h := hashmap.New()
h.Hash = hash.FNV1A
return &level{nodes: h}
}
func TestSplitBufferPubOp(t *testing.T) {
c := &client{subs: hashmap.New()}
pub := []byte("PUB foo.bar INBOX.22 11\r\nhello world\r")
pub1 := pub[:2]
pub2 := pub[2:9]
pub3 := pub[9:15]
pub4 := pub[15:22]
pub5 := pub[22:25]
pub6 := pub[25:33]
pub7 := pub[33:]
if err := c.parse(pub1); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != OP_PU {
t.Fatalf("Expected OP_PU state vs %d\n", c.state)
}
if err := c.parse(pub2); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != PUB_ARG {
t.Fatalf("Expected OP_PU state vs %d\n", c.state)
}
if err := c.parse(pub3); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != PUB_ARG {
t.Fatalf("Expected OP_PU state vs %d\n", c.state)
}
if err := c.parse(pub4); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != PUB_ARG {
t.Fatalf("Expected PUB_ARG state vs %d\n", c.state)
}
if err := c.parse(pub5); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != MSG_PAYLOAD {
t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state)
}
// Check c.pa
if !bytes.Equal(c.pa.subject, []byte("foo.bar")) {
t.Fatalf("PUB arg subject incorrect: '%s'\n", c.pa.subject)
}
if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) {
t.Fatalf("PUB arg reply subject incorrect: '%s'\n", c.pa.reply)
}
if c.pa.size != 11 {
t.Fatalf("PUB arg msg size incorrect: %d\n", c.pa.size)
}
if err := c.parse(pub6); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != MSG_PAYLOAD {
t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state)
}
if err := c.parse(pub7); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != MSG_END {
t.Fatalf("Expected MSG_END state vs %d\n", c.state)
}
}
func TestSplitBufferMsgOp(t *testing.T) {
c := &client{subs: hashmap.New(), typ: ROUTER}
msg := []byte("MSG foo.bar QRSID:15:3 _INBOX.22 11\r\nhello world\r")
msg1 := msg[:2]
msg2 := msg[2:9]
msg3 := msg[9:15]
msg4 := msg[15:22]
msg5 := msg[22:25]
msg6 := msg[25:37]
msg7 := msg[37:42]
msg8 := msg[42:]
if err := c.parse(msg1); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != OP_MS {
t.Fatalf("Expected OP_MS state vs %d\n", c.state)
}
if err := c.parse(msg2); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != MSG_ARG {
t.Fatalf("Expected MSG_ARG state vs %d\n", c.state)
}
if err := c.parse(msg3); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != MSG_ARG {
t.Fatalf("Expected MSG_ARG state vs %d\n", c.state)
}
if err := c.parse(msg4); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != MSG_ARG {
t.Fatalf("Expected MSG_ARG state vs %d\n", c.state)
}
if err := c.parse(msg5); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != MSG_ARG {
t.Fatalf("Expected MSG_ARG state vs %d\n", c.state)
}
if err := c.parse(msg6); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != MSG_PAYLOAD {
t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state)
}
// Check c.pa
if !bytes.Equal(c.pa.subject, []byte("foo.bar")) {
t.Fatalf("MSG arg subject incorrect: '%s'\n", c.pa.subject)
}
if !bytes.Equal(c.pa.sid, []byte("QRSID:15:3")) {
t.Fatalf("MSG arg sid incorrect: '%s'\n", c.pa.sid)
}
if !bytes.Equal(c.pa.reply, []byte("_INBOX.22")) {
t.Fatalf("MSG arg reply subject incorrect: '%s'\n", c.pa.reply)
}
if c.pa.size != 11 {
t.Fatalf("MSG arg msg size incorrect: %d\n", c.pa.size)
}
if err := c.parse(msg7); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != MSG_PAYLOAD {
t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state)
}
if err := c.parse(msg8); err != nil {
t.Fatalf("Unexpected parse error: %v\n", err)
}
if c.state != MSG_END {
t.Fatalf("Expected MSG_END state vs %d\n", c.state)
}
}