func TestName(t *testing.T) {
debug = testing.Verbose()
d := zx.Dir{
"path": "/a/b",
"name": "b",
"type": "d",
"mode": "0755",
"spath": "/a/b",
"tpath": "/tmp/lfs_test",
"proto": "lfs",
}
preds := []string{
`name=b`,
`! name=b`,
`name!=b`,
}
matches := []bool{true, false, false}
prunes := []bool{false, true, true}
for i, pr := range preds {
p, err := New(pr)
if err != nil {
t.Fatalf("parse %s", err)
}
t.Logf("eval %s\n", p.DebugString())
m, prune, err := p.EvalAt(d, 0)
t.Logf("match %v prune %v sts %v", m, prune, err)
if err != nil || m != matches[i] || prune != prunes[i] {
t.Logf("wrong result %v %v %v", err, m, prune)
t.Fail()
}
}
}
func TestUnbackslash(t *testing.T) {
for _, test := range []struct {
isString bool
in string
out string
}{
{false, "", ""},
{true, "", ""},
// from CSS2 4.1.3 examples:
{true, "\\26 B", "&B"},
{true, "\\000026B", "&B"},
{true, "\\26G", "&G"},
{true, "\\2aG", "*G"},
{true, "\\2AG", "*G"},
{true, "\\2fG", "/G"},
{true, "\\2FG", "/G"},
// standard does not appear to require an even number of digits
{true, "\\026 B", "&B"},
{true, "\\026 B", "& B"},
{true, "\\026", "&"},
{true, "\\", "\\"},
{true, "\\{", "{"},
// Check the special string handling
{true, "a\\\nb", "ab"},
{true, "a\\\r\nb", "ab"},
} {
result := unbackslash(test.in, test.isString)
if result != test.out {
t.Fatalf("Error in TestUnbackslash. In: %q\nOut: %q\nExpected: %q",
test.in, result, test.out)
}
}
}
func TestSimple(t *testing.T) {
value := 1
expected := 2
if value != expected {
t.Fatalf("Expected %v, but %d:", value, expected)
}
}
func TestLoadFile(t *testing.T){
fp := "path-to-file"
h := hive.HiveConfig(...)
e := hdc.LoadFile(h, fp, "table", true)
fatalCheck(t, "LoadFile", e)
q := "select count(*) recordCount from table"
var ms []toolkit.M
e = h.Populate(q, &ms)
fatalCheck(t, "Populate", e)
if ms[0].GetInt("recordCount", 0)==0 {
t.Fatalf("No data imported")
}
}
func TestConnectWithOptions(t *testing.T) {
responseText := "Hello World"
itemID := "theID"
connectStorage := &ConnecterRESTStorage{
connectHandler: &SimpleConnectHandler{
response: responseText,
},
emptyConnectOptions: &SimpleGetOptions{},
}
storage := map[string]rest.Storage{
"simple": &SimpleRESTStorage{},
"simple/connect": connectStorage,
}
handler := handle(storage)
server := httptest.NewServer(handler)
defer server.Close()
resp, err := http.Get(server.URL + "/api/version/namespaces/default/simple/" + itemID + "/connect?param1=value1¶m2=value2")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if resp.StatusCode != http.StatusOK {
t.Errorf("unexpected response: %#v", resp)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if connectStorage.receivedID != itemID {
t.Errorf("Unexpected item id. Expected: %s. Actual: %s.", itemID, connectStorage.receivedID)
}
if string(body) != responseText {
t.Errorf("Unexpected response. Expected: %s. Actual: %s.", responseText, string(body))
}
opts, ok := connectStorage.receivedConnectOptions.(*SimpleGetOptions)
if !ok {
t.Errorf("Unexpected options type: %#v", connectStorage.receivedConnectOptions)
}
if opts.Param1 != "value1" && opts.Param2 != "value2" {
t.Errorf("Unexpected options value: %#v", opts
func TestCliveOutChan(t *testing.T) {
debug = testing.Verbose()
out2 := make(chan face{})
adj := func(c *cmd.Ctx) {
c.ForkEnv()
c.ForkNS()
c.ForkDot()
c.SetOut("out2", out2)
}
c, err := CtxCmd(adj, "eco", "-o", "out2", "-m", "a", "b", "c")
if err != nil {
t.Fatalf("sts %v", err)
}
out := []string{}
for x := range ch.Merge(c.Out, c.Err, out2) {
switch x := x.(type) {
case []byte:
printf("-> [%s]\n", x)
out = append(out, string(x))
default:
t.Fatalf("got type %T", x)
}
}
err = c.Wait()
printf("sts %v\n", err)
if err != nil {
t.Fatalf("did fail")
}
}
func TestLf(t *testing.T) {
app.New() // prevent app.Fatal from calling dbg.Fatal
app.Debug = testing.Verbose()
for i := range tests {
lt := tests[i]
dprintf("run %v\n", lt.args)
out := make(chan interface{})
go func() {
c := app.New()
c.Args = lt.args
defer app.Exiting()
app.DupIO()
app.SetIO(out, 1)
app.Cd("/zx/sys/src/clive")
Run()
}()
outs := []string{}
for x := range out {
d, ok := x.(zx.Dir)
if !ok {
dprintf("got %T %v\n", x, x)
t.Fatalf("not a dir")
}
dprintf("got %T %s\n", d, d["upath"])
outs = append(outs, d["upath"])
}
err := cerror(out)
dprintf("got sts %v\n", err)
if lt.fails && err == nil {
t.Fatalf("didn't fail")
}
if !lt.fails && err != nil {
t.Fatalf("failed: %s", err)
}
if lt.out != nil && strings.Join(lt.out, " ") != strings.Join(outs, " ") {
t.Fatalf("bad output %#v", outs)
}
if lt.out == nil {
dprintf("out: %#v\n", outs)
}
}
}
func TestCliveIn2Chan(t *testing.T) {
debug = testing.Verbose()
in := make(chan face{}, 3)
adj := func(c *cmd.Ctx) {
c.ForkEnv()
c.ForkNS()
c.ForkDot()
c.SetIn("in2", in)
}
c, err := CtxCmd(adj, "eco", "-i", "in2", "-m", "a", "b", "c")
if err != nil {
t.Fatalf("sts %v", err)
}
for _, m := range []string{"x", "y", "z"} {
in <- []byte(m)
}
close(in)
out := []string{}
for x := range ch.Merge(c.Out, c.Err) {
switch x := x.(type) {
case []byte:
printf("-> [%s]\n", x)
out = append(out, string(x))
default:
t.Fatalf("got type %T", x)
}
}
err = c.Wait()
printf("sts %v\n", err)
if err != nil {
t.Fatalf("did fail")
}
if strings.Join(out, "|") != "a|b|c|x|y|z" {
t.Fatalf("bad output")
}
}
func TestOrphaning(t *testing.T) {
gc, clientSet := setup(t)
oldEnableGarbageCollector := registry.EnableGarbageCollector
registry.EnableGarbageCollector = true
defer func() { registry.EnableGarbageCollector = oldEnableGarbageCollector }()
podClient := clientSet.Core().Pods(framework.TestNS)
rcClient := clientSet.Core().ReplicationControllers(framework.TestNS)
// create the RC with the orphan finalizer set
toBeDeletedRC := newOwnerRC(toBeDeletedRCName)
toBeDeletedRC, err := rcClient.Create(toBeDeletedRC)
if err != nil {
t.Fatalf("Failed to create replication controller: %v", err)
}
// these pods should be ophaned.
var podUIDs []types.UID
podsNum := 3
for i := 0; i < podsNum; i++ {
podName := garbageCollectedPodName + strconv.Itoa(i)
pod := newPod(podName, []v1.OwnerReference{{UID: toBeDeletedRC.ObjectMeta.UID, Name: toBeDeletedRCName}})
_, err = podClient.Create(pod)
if err != nil {
t.Fatalf("Failed to create Pod: %v", err)
}
podUIDs = append(podUIDs, pod.ObjectMeta.UID)
}
stopCh := make(chan struct{})
go gc.Run(5, stopCh)
defer close(stopCh)
// we need wait for the gc to observe the creation of the pods, otherwise if
// the deletion of RC is observed before the creation of the pods, the pods
// will not be orphaned.
wait.Poll(5*time.Second, 60*time.Second, func() (bool, error) { return gc.GraphHasUID(podUIDs), nil })
err = rcClient.Delete(toBeDeletedRCName, getOrphanOptions())
if err != nil {
t.Fatalf("Failed to gracefully delete the rc: %v", err)
}
// wait for the garbage collector to drain its queue
if err := wait.Poll(10*time.Second, 300*time.Second, func() (bool, error) {
retu
func TestClientTimeout(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 50 * time.Millisecond
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identify(t, conn)
sub(t, conn, topicName, "ch")
time.Sleep(50 * time.Millisecond)
// depending on timing there may be 1 or 2 hearbeats sent
// just read until we get an error
timer := time.After(100 * time.Millisecond)
for {
select {
case <-timer:
t.Fatalf("test timed out")
default:
_, err := nsq.ReadResponse(conn)
if err != nil {
goto done
}
}
}
done:
}
func TestClientHeartbeat(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 100 * time.Millisecond
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identify(t, conn)
sub(t, conn, topicName, "ch")
err = nsq.Ready(1).Write(conn)
assert.Equal(t, err, nil)
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
assert.Equal(t, data, []byte("_heartbeat_"))
time.Sleep(10 * time.Millisecond)
err = nsq.Nop().Write(conn)
assert.Equal(t, err, nil)
// wait long enough that would have timed out (had we not sent the above cmd)
time.Sleep(50 * time.Millisecond)
err = nsq.Nop().Write(conn)
assert.Equal(t, err, nil)
}
func TestClientHeartbeatDisableSUB(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 200 * time.Millisecond
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identifyHeartbeatInterval(t, conn, -1, nsq.FrameTypeResponse, "OK")
subFail(t, conn, topicName, "ch")
}
func TestClientHeartbeatDisable(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.clientTimeout = 100 * time.Millisecond
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identifyHeartbeatInterval(t, conn, -1, nsq.FrameTypeResponse, "OK")
time.Sleep(150 * time.Millisecond)
err = nsq.Nop().Write(conn)
assert.Equal(t, err, nil)
}
func TestMaxHeartbeatIntervalValid(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.maxHeartbeatInterval = 300 * time.Second
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
hbi := int(options.maxHeartbeatInterval / time.Millisecond)
identifyHeartbeatInterval(t, conn, hbi, nsq.FrameTypeResponse, "OK")
}
func TestMaxHeartbeatIntervalInvalid(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.maxHeartbeatInterval = 300 * time.Second
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
hbi := int(options.maxHeartbeatInterval/time.Millisecond + 1)
identifyHeartbeatInterval(t, conn, hbi, nsq.FrameTypeError, "E_BAD_BODY IDENTIFY heartbeat interval (300001) is invalid")
}
func TestPausing(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix()))
tcpAddr, _ := mustStartNSQd(NewNsqdOptions())
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identify(t, conn)
sub(t, conn, topicName, "ch")
err = nsq.Ready(1).Write(conn)
assert.Equal(t, err, nil)
topic := nsqd.GetTopic(topicName)
msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
channel := topic.GetChannel("ch")
topic.PutMessage(msg)
// receive the first message via the client, finish it, and send new RDY
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
msg, err = nsq.DecodeMessage(data)
assert.Equal(t, msg.Body, []byte("test body"))
err = nsq.Finish(msg.Id).Write(conn)
assert.Equal(t, err, nil)
err = nsq.Ready(1).Write(conn)
assert.Equal(t, err, nil)
// sleep to allow the RDY state to take effect
time.Sleep(50 * time.Millisecond)
// pause the channel... the client shouldn't receive any more messages
channel.Pause()
// sleep to allow the paused state to take effect
time.Sleep(50 * time.Millisecond)
msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body2"))
topic.PutMessage(msg)
// allow the client to possibly get a message, the test would hang indefinitely
// if pausing was not working on the internal clientMsgChan read
time.Sleep(50 * time.Millisecond)
msg = <-channel.clientMsgChan
assert.Equal(t, msg.Body, []byte("test body2"))
// unpause the channel... the client should now be pushed a message
channel.UnPause()
msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body3"))
topic.PutMessage(msg)
resp, _ = nsq.ReadResponse(conn)
_, data, _ = nsq.UnpackResponse(resp)
msg, err = nsq.DecodeMessage(data)
assert.Equal(t, msg.Body, []byte("test body3"))
}
func TestEmptyCommand(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
tcpAddr, _ := mustStartNSQd(NewNsqdOptions())
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
_, err = conn.Write([]byte("\n\n"))
assert.Equal(t, err, nil)
// if we didn't panic here we're good, see issue #120
}
func TestSizeLimits(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
*verbose = true
options.maxMessageSize = 100
options.maxBodySize = 1000
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
topicName := "test_limits_v2" + strconv.Itoa(int(time.Now().Unix()))
identify(t, conn)
sub(t, conn, topicName, "ch")
nsq.Publish(topicName, make([]byte, 95)).Write(conn)
resp, _ := nsq.ReadResponse(conn)
frameType, data, _ := nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
assert.Equal(t, data, []byte("OK"))
nsq.Publish(topicName, make([]byte, 105)).Write(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeError)
assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE PUB message too big 105 > 100"))
// need to reconnect
conn, err = mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
// MPUB body that's valid
mpub := make([][]byte, 0)
for i := 0; i < 5; i++ {
mpub = append(mpub, make([]byte, 100))
}
cmd, _ := nsq.MultiPublish(topicName, mpub)
cmd.Write(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
assert.Equal(t, data, []byte("OK"))
// MPUB body that's invalid (body too big)
mpub = make([][]byte, 0)
for i := 0; i < 11; i++ {
mpub = append(mpub, make([]byte, 100))
}
cmd, _ = nsq.MultiPublish(topicName, mpub)
cmd.Write(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeError)
assert.Equal(t, string(data), fmt.Sprintf("E_BAD_BODY MPUB body too big 1148 > 1000"))
// need to reconnect
conn, err = mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
// MPUB body that's invalid (one of the messages is too big)
mpub = make([][]byte, 0)
for i := 0; i < 5; i++ {
mpub = append(mpub, make([]byte, 101))
}
cmd, _ = nsq.MultiPublish(topicName, mpub)
cmd.Write(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeError)
assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE MPUB message too big 101 > 100"))
}
func TestTouch(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
*verbose = true
options := NewNsqdOptions()
options.msgTimeout = 50 * time.Millisecond
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
topicName := "test_touch" + strconv.Itoa(int(time.Now().Unix()))
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identify(t, conn)
sub(t, conn, topicName, "ch")
topic := nsqd.GetTopic(topicName)
channel := topic.GetChannel("ch")
msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
topic.PutMessage(msg)
err = nsq.Ready(1).Write(conn)
assert.Equal(t, err, nil)
resp, err := nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
frameType, data, err := nsq.UnpackResponse(resp)
msgOut, _ := nsq.DecodeMessage(data)
assert.Equal(t, frameType, nsq.FrameTypeMessage)
assert.Equal(t, msgOut.Id, msg.Id)
time.Sleep(25 * time.Millisecond)
err = nsq.Touch(msg.Id).Write(conn)
assert.Equal(t, err, nil)
time.Sleep(30 * time.Millisecond)
err = nsq.Finish(msg.Id).Write(conn)
assert.Equal(t, err, nil)
assert.Equal(t, channel.timeoutCount, uint64(0))
}
func TestMaxRdyCount(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
*verbose = true
options := NewNsqdOptions()
options.maxRdyCount = 50
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
topicName := "test_max_rdy_count" + strconv.Itoa(int(time.Now().Unix()))
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
topic := nsqd.GetTopic(topicName)
msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
topic.PutMessage(msg)
data := identifyFeatureNegotiation(t, conn)
r := struct {
MaxRdyCount int64 `json:"max_rdy_count"`
}{}
err = json.Unmarshal(data, &r)
assert.Equal(t, err, nil)
assert.Equal(t, r.MaxRdyCount, int64(50))
sub(t, conn, topicName, "ch")
err = nsq.Ready(int(options.maxRdyCount)).Write(conn)
assert.Equal(t, err, nil)
resp, err := nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
frameType, data, err := nsq.UnpackResponse(resp)
msgOut, _ := nsq.DecodeMessage(data)
assert.Equal(t, frameType, nsq.FrameTypeMessage)
assert.Equal(t, msgOut.Id, msg.Id)
err = nsq.Ready(int(options.maxRdyCount) + 1).Write(conn)
assert.Equal(t, err, nil)
resp, err = nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
frameType, data, err = nsq.UnpackResponse(resp)
assert.Equal(t, frameType, int32(1))
assert.Equal(t, string(data), "E_INVALID RDY count 51 out of range 0-50")
}
func BenchmarkProtocolV2Exec(b *testing.B) {
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
p := &ProtocolV2{}
c := NewClientV2(nil)
params := [][]byte{[]byte("NOP")}
b.StartTimer()
for i := 0; i < b.N; i++ {
p.Exec(c, params)
}
}
func benchmarkProtocolV2Pub(b *testing.B, size int) {
var wg sync.WaitGroup
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.memQueueSize = int64(b.N)
tcpAddr, _ := mustStartNSQd(options)
msg := make([]byte, size)
batchSize := 200
batch := make([][]byte, 0)
for i := 0; i < batchSize; i++ {
batch = append(batch, msg)
}
topicName := "bench_v2_pub" + strconv.Itoa(int(time.Now().Unix()))
b.SetBytes(int64(len(msg)))
b.StartTimer()
for j := 0; j < runtime.GOMAXPROCS(0); j++ {
wg.Add(1)
go func() {
conn, err := mustConnectNSQd(tcpAddr)
if err != nil {
panic(err.Error())
}
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
num := b.N / runtime.GOMAXPROCS(0) / batchSize
for i := 0; i < num; i += 1 {
cmd, _ := nsq.MultiPublish(topicName, batch)
err := cmd.Write(rw)
if err != nil {
panic(err.Error())
}
err = rw.Flush()
if err != nil {
panic(err.Error())
}
resp, err := nsq.ReadResponse(rw)
if err != nil {
panic(err.Error())
}
_, data, _ := nsq.UnpackResponse(resp)
if !bytes.Equal(data, []byte("OK")) {
panic("invalid response")
}
}
wg.Done()
}()
}
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func BenchmarkProtocolV2Pub256(b *testing.B) { benchmarkProtocolV2Pub(b, 256) }
func BenchmarkProtocolV2Pub512(b *testing.B) { benchmarkProtocolV2Pub(b, 512) }
func BenchmarkProtocolV2Pub1k(b *testing.B) { benchmarkProtocolV2Pub(b, 1024) }
func BenchmarkProtocolV2Pub2k(b *testing.B) { benchmarkProtocolV2Pub(b, 2*1024) }
func BenchmarkProtocolV2Pub4k(b *testing.B) { benchmarkProtocolV2Pub(b, 4*1024) }
func BenchmarkProtocolV2Pub8k(b *testing.B) { benchmarkProtocolV2Pub(b, 8*1024) }
func BenchmarkProtocolV2Pub16k(b *testing.B) { benchmarkProtocolV2Pub(b, 16*1024) }
func BenchmarkProtocolV2Pub32k(b *testing.B) { benchmarkProtocolV2Pub(b, 32*1024) }
func BenchmarkProtocolV2Pub64k(b *testing.B) { benchmarkProtocolV2Pub(b, 64*1024) }
func BenchmarkProtocolV2Pub128k(b *testing.B) { benchmarkProtocolV2Pub(b, 128*1024) }
func BenchmarkProtocolV2Pub256k(b *testing.B) { benchmarkProtocolV2Pub(b, 256*1024) }
func BenchmarkProtocolV2Pub512k(b *testing.B) { benchmarkProtocolV2Pub(b, 512*1024) }
func BenchmarkProtocolV2Pub1m(b *testing.B) { benchmarkProtocolV2Pub(b, 1024*1024) }
func benchmarkProtocolV2Sub(b *testing.B, size int) {
var wg sync.WaitGroup
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.memQueueSize = int64(b.N)
tcpAddr, _ := mustStartNSQd(options)
msg := make([]byte, size)
topicName := "bench_v2_sub" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
for i := 0; i < b.N; i++ {
msg := nsq.NewMessage(<-nsqd.idChan, msg)
topic.PutMessage(msg)
}
topic.GetChannel("ch")
b.SetBytes(int64(len(msg)))
goChan := make(chan int)
rdyChan := make(chan int)
workers := runtime.GOMAXPROCS(0)
for j := 0; j < workers; j++ {
wg.Add(1)
go func() {
subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
wg.Done()
}()
<-rdyChan
}
b.StartTimer()
close(goChan)
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func subWorker(n int, workers int, tcpAddr *net.TCPAddr, topicName string, rdyChan chan int, goChan chan int) {
conn, err := mustConnectNSQd(tcpAddr)
if err != nil {
panic(err.Error())
}
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
identify(nil, conn)
sub(nil, conn, topicName, "ch")
rdyCount := int(math.Min(math.Max(float64(n/workers), 1), 2500))
rdyChan <- 1
<-goChan
nsq.Ready(rdyCount).Write(rw)
rw.Flush()
num := n / workers
numRdy := num/rdyCount - 1
rdy := rdyCount
for i := 0; i < num; i += 1 {
resp, err := nsq.ReadResponse(rw)
if err != nil {
panic(err.Error())
}
frameType, data, err := nsq.UnpackResponse(resp)
if err != nil {
panic(err.Error())
}
if frameType != nsq.FrameTypeMessage {
panic("got something else")
}
msg, err := nsq.DecodeMessage(data)
if err != nil {
panic(err.Error())
}
nsq.Finish(msg.Id).Write(rw)
rdy--
if rdy == 0 && numRdy > 0 {
nsq.Ready(rdyCount).Write(rw)
rdy = rdyCount
numRdy--
rw.Flush()
}
}
}
func BenchmarkProtocolV2Sub256(b *testing.B) { benchmarkProtocolV2Sub(b, 256) }
func BenchmarkProtocolV2Sub512(b *testing.B) { benchmarkProtocolV2Sub(b, 512) }
func BenchmarkProtocolV2Sub1k(b *testing.B) { benchmarkProtocolV2Sub(b, 1024) }
func BenchmarkProtocolV2Sub2k(b *testing.B) { benchmarkProtocolV2Sub(b, 2*1024) }
func BenchmarkProtocolV2Sub4k(b *testing.B) { benchmarkProtocolV2Sub(b, 4*1024) }
func BenchmarkProtocolV2Sub8k(b *testing.B) { benchmarkProtocolV2Sub(b, 8*1024) }
func BenchmarkProtocolV2Sub16k(b *testing.B) { benchmarkProtocolV2Sub(b, 16*1024) }
func BenchmarkProtocolV2Sub32k(b *testing.B) { benchmarkProtocolV2Sub(b, 32*1024) }
func BenchmarkProtocolV2Sub64k(b *testing.B) { benchmarkProtocolV2Sub(b, 64*1024) }
func BenchmarkProtocolV2Sub128k(b *testing.B) { benchmarkProtocolV2Sub(b, 128*1024) }
func BenchmarkProtocolV2Sub256k(b *testing.B) { benchmarkProtocolV2Sub(b, 256*1024) }
func BenchmarkProtocolV2Sub512k(b *testing.B) { benchmarkProtocolV2Sub(b, 512*1024) }
func BenchmarkProtocolV2Sub1m(b *testing.B) { benchmarkProtocolV2Sub(b, 1024*1024) }
func benchmarkProtocolV2MultiSub(b *testing.B, num int) {
var wg sync.WaitGroup
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.memQueueSize = int64(b.N)
tcpAddr, _ := mustStartNSQd(options)
msg := make([]byte, 256)
b.SetBytes(int64(len(msg) * num))
goChan := make(chan int)
rdyChan := make(chan int)
workers := runtime.GOMAXPROCS(0)
for i := 0; i < num; i++ {
topicName := "bench_v2" + strconv.Itoa(b.N) + "_" + strconv.Itoa(i) + "_" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
for i := 0; i < b.N; i++ {
msg := nsq.NewMessage(<-nsqd.idChan, msg)
topic.PutMessage(msg)
}
topic.GetChannel("ch")
for j := 0; j < workers; j++ {
wg.Add(1)
go func() {
subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
wg.Done()
}()
<-rdyChan
}
}
b.StartTimer()
close(goChan)
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func BenchmarkProtocolV2MultiSub1(b *testing.B) { benchmarkProtocolV2MultiSub(b, 1) }
func BenchmarkProtocolV2MultiSub2(b *testing.B) { benchmarkProtocolV2MultiSub(b, 2) }
func BenchmarkProtocolV2MultiSub4(b *testing.B) { benchmarkProtocolV2MultiSub(b, 4) }
func BenchmarkProtocolV2MultiSub8(b *testing.B) { benchmarkProtocolV2MultiSub(b, 8) }
func BenchmarkProtocolV2MultiSub16(b *testing.B) { benchmarkProtocolV2MultiSub(b, 16) }
func TestNS(t *testing.T) {
ns := mkns(t, testing.Verbose())
defer fstest.RmTree(t, tdir)
d2 := zx.Dir{"path": "x", "name": "x", "proto": "p2"}
err := <-ns.Mount("d", d2, After)
if err == nil {
t.Fatalf("mount: could mount")
}
err = <-ns.Unmount("/d", d2)
if err != nil {
t.Fatalf("unmount dir: %s", err)
}
printf("ns is `%s`\n", ns)
s := ns.String()
ns2, err := Parse(s)
if err != nil {
t.Fatalf("parse: %s", err)
}
printf("ns2 is `%s`\n", ns2)
if ns2.String() != s {
t.Fatalf("parsed ns differs")
}
outs := `path:"/" name:"/" type:"d" mode:"0755" proto:"lfs" spath:"/" tpath:"/tmp/lfs_test"
path:"/a/b" name:"b" type:"d" mode:"0755" proto:"lfs" spath:"/" tpath:"/tmp/lfs_test"
path:"/d" name:"d" type:"p" mode:"0644" proto:"p1"
path:"/d" name:"d" type:"p" mode:"0644" proto:"p2"
path:"/d" name:"d" type:"p" mode:"0644" proto:"p2"
`
if s != outs {
t.Fatalf("bad ns contents")
}
err = <-ns.Unmount("/a/b", nil)
if err != nil {
t.Fatalf("unmount dir: %s", err)
}
printf("ns is `%s`\n", ns)
}
func TestResolve(t *testing.T) {
ns := mkns(t, false)
defer fstest.RmTree(t, tdir)
ns.Debug = testing.Verbose()
ns.DebugFind = testing.Verbose()
for _, r := range resolves {
_, dirs, paths, err := ns.Resolve(r.Path)
printf("sts %v\n", err)
if err!=nil && !r.Fails {
t.Fatalf("failed with %v", err)
}
if err==nil && r.Fails {
t.Fatal("didn't fail")
}
if len(dirs) != len(paths) {
t.Fatal("wrong lengths")
}
printf("dirs:\n")
for _, d := range dirs {
delete(d, "Uid")
delete(d, "Gid")
delete(d, "Wuid")
delete(d, "Sum")
printf("\t`%s`,\n", d)
}
printf("paths:\n")
for _, p := range paths {
printf("\t`%s`,\n", p)
}
for i := 0; i<len(r.Dirs) && i<len(dirs); i++ {
if r.Dirs[i] != dirs[i].String() {
t.Fatalf("bad result [%d]\n\tgot %s\n\twant %s\n",
i, dirs[i], r.Dirs[i])
}
}
if r.Dirs != nil {
if len(dirs) > len(r.Dirs) {
t.Fatalf("unexpected %s", dirs[len(r.Dirs)])
}
if len(dirs) < len(r.Dirs) {
t.Fatalf("did expect %s", r.Dirs[len(dirs)])
}
}
for i := 0; i<len(r.Paths) && i<len(paths); i++ {
if r.Paths[i] != paths[i] {
t.Fatalf("bad result [%d]\n\tgot %s\n\twant %s\n",
i, paths[i], r.Paths[i])
}
}
if r.Paths != nil {
if len(paths) > len(r.Paths) {
t.Fatalf("unexpected %s", paths[len(r.Paths)])
}
if len(paths) < len(r.Paths) {
t.Fatalf("did expect %s", r.Paths[len(paths)])
}
}
}
}
func _(t *testing.T) {
fmt.Printf(<warning descr="Got 1 placeholder(s) for 2 arguments(s)">s</warning>, 1, 2)
fmt.Printf(<warning descr="Value used for formatting text does not appear to be a string">s1</warning>, 1, 2)
fmt.Printf(<warning descr="Value used for formatting text does not appear to be a string">s2</warning>, 1, 2)
fmt.Errorf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
1, 2, 3, 4, 5, 6, 7, 8, 9,
)
fmt.Fprintf(nil, "%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
1, 2, 3, 4, 5, 6, 7, 8, 9,
)
fmt.Fscanf(nil, "%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
1, 2, 3, 4, 5, 6, 7, 8, 9,
)
fmt.Printf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
1, 2, 3, 4, 5, 6, 7, 8, 9,
)
fmt.Scanf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
1, 2, 3, 4, 5, 6, 7, 8, 9,
)
fmt.Sprintf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
1, 2, 3, 4, 5, 6, 7, 8, 9,
)
fmt.Sscanf(nil, "%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
1, 2, 3, 4, 5, 6, 7, 8, 9,
)
log.Fatalf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
1, 2, 3, 4, 5, 6, 7, 8, 9,
)
log.Panicf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
1, 2, 3, 4, 5, 6, 7, 8, 9,
)
log.Printf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
1, 2, 3, 4, 5, 6, 7, 8, 9,
)
fmt.Printf("eq (as-is): %.3f%% score: %v offset: %v descr: [%v]\n", 3.14, 1, 1, "descr")
fmt.Printf("a: %+v", 1)
fmt.Printf("%-4d", 999)
fmt.Printf("a: %%%+v", 1)
fmt.Printf(<warning descr="Got 0 placeholder(s) for 1 arguments(s)">"a: %%%%+v"</warning>, 1)
fmt.Printf("#%02X%02X%02X", 1, 2, 3)
fmt.Printf(<warning descr="Got 3 placeholder(s) for 4 arguments(s)">"#%02X%02X%02X"</warning>, 1, 2, 3, 4)
myFormatVar := "%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d"
log.Printf(myFormatVar, 1, 2, 3, 4, 5, 6, 7, 8, 9)
myWrongFormatVar := "%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d"
log.Printf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatVar</warning>, 1, 2, 3, 4, 5, 6, 7, 8)
log.Printf(myFormatConst, 1, 2, 3, 4, 5, 6, 7, 8, 9)
t.Errorf(myFormatConst, 1, 2, 3, 4, 5, 6, 7, 8, 9)
t.Fatalf(myFormatConst, 1, 2, 3, 4, 5, 6, 7, 8, 9)
t.Logf(myFormatConst, 1, 2, 3, 4, 5, 6, 7, 8, 9)
t.Skipf(myFormatConst, 1, 2, 3, 4, 5, 6, 7, 8, 9)
log.Printf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatConst</warning>, 1, 2, 3, 4, 5, 6, 7, 8)
t.Errorf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatConst</warning>, 1, 2, 3, 4, 5, 6, 7, 8)
t.Fatalf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatConst</warning>, 1, 2, 3, 4, 5, 6, 7, 8)
t.Logf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatConst</warning>, 1, 2, 3, 4, 5, 6, 7, 8)
t.Skipf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatConst</warning>, 1, 2, 3, 4, 5, 6, 7, 8)
printf("%d", 1)
printf("%[2]d %[1]d", 1, 2)
printf("%[2]d %[1]d %d", 1, 2)
printf("%[2]d %[1]d %[2]d", 1, 2)
printf("%d")
myNonFormatFunc := func () int {
return 1
}
log.Printf(<warning descr="Value used for formatting text does not appear to be a string">myNonFormatFunc()</warning>, 1, 2, 3, 4, 5, 6, 7, 8, 9)
log.Printf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">"%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d"</warning>,
1, 2, 3, 4, 5, 6, 7, 8,
)
fmt.Sprintf(<warning descr="Got 1 placeholder(s) for 0 arguments(s)">"%d"</warning>)
log.Printf(<warning descr="Got 7 placeholder(s) for 13 arguments(s)">"%d %d %#[1]x %#x %f %2.f %2.2f %.f %.3f %[3]*.[2]*[1]f %d %d %#[1]x %#x %*[2]d %v % d"</warning>,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
)
fmt.Sprintf(<warning descr="Got 1 placeholder(s) for 2 arguments(s)">"%d"</warning>, 1, 2)
fmt.Print(<warning descr="Possible formatting directive in '\"%[2]*.[1]*[3]d\"'">"%[2]*.[1]*[3]d"</warning>, 2, 3, myNonFormatFunc)
fmt.Print(<warning descr="Possible formatting directive in '\"%[2]*.[1]*[3]d\"'">"%[2]*.[1]*[3]d"</warning>, 2, 3, printf)
fmt.Println("demo<warning descr="Function already ends with new line">\n</warning>", 2, 3, <warning descr="Argument 'myNonFormatFunc' is not a function call">myNonFormatFunc</warning>)
fmt.Println("demo<warning descr="Function already ends with new line">\n</warning>", 2, 3, <warning descr="Argument 'printf' is not a function call">printf</warning>)
fmt.Print("demo\n", 2, 3, <warning descr="Argument 'myNonFormatFunc' is not a function call">myNonFormatFunc</warning>)
fmt.Print("demo\n", 2, 3, <warning descr="Argument 'printf' is not a function call">printf</warning>)
type X struct{ Y, Z int32 }
a := &X{5, 7}
fmt.Println(a, "->", C.sum(*((*C.struct_x)(unsafe.Pointer(a)))))
fmt.Sprintf("asdadad <warning descr="Unrecognized formatting verb '%O' call">%O</warning> asdadad", "demo")
fmt.Printf("%[<warning descr="Index value [0] is not allowed">0</warning>]d", 1)
}
func TestClientTimeout(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 50 * time.Millisecond
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
err = nsq.SendCommand(conn, nsq.Subscribe(topicName, "ch", "TestClientTimeoutV2", "TestClientTimeoutV2"))
assert.Equal(t, err, nil)
time.Sleep(50 * time.Millisecond)
// depending on timing there may be 1 or 2 hearbeats sent
// just read until we get an error
timer := time.After(100 * time.Millisecond)
for {
select {
case <-timer:
t.Fatalf("test timed out")
default:
_, err := nsq.ReadResponse(conn)
if err != nil {
goto done
}
}
}
done:
}
func TestClientHeartbeat(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 100 * time.Millisecond
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
err = nsq.SendCommand(conn, nsq.Subscribe(topicName, "ch", "TestClientHeartbeatV2", "TestClientHeartbeatV2"))
assert.Equal(t, err, nil)
err = nsq.SendCommand(conn, nsq.Ready(1))
assert.Equal(t, err, nil)
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
assert.Equal(t, data, []byte("_heartbeat_"))
time.Sleep(10 * time.Millisecond)
err = nsq.SendCommand(conn, nsq.Nop())
assert.Equal(t, err, nil)
// wait long enough that would have timed out (had we not sent the above cmd)
time.Sleep(50 * time.Millisecond)
err = nsq.SendCommand(conn, nsq.Nop())
assert.Equal(t, err, nil)
}
func TestPausing(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix()))
tcpAddr, _ := mustStartNSQd(NewNsqdOptions())
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
err = nsq.SendCommand(conn, nsq.Subscribe(topicName, "ch", "TestPausing", "TestPausing"))
assert.Equal(t, err, nil)
err = nsq.SendCommand(conn, nsq.Ready(1))
assert.Equal(t, err, nil)
topic := nsqd.GetTopic(topicName)
msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
channel := topic.GetChannel("ch")
topic.PutMessage(msg)
// receive the first message via the client, finish it, and send new RDY
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
msg, err = nsq.DecodeMessage(data)
assert.Equal(t, msg.Body, []byte("test body"))
err = nsq.SendCommand(conn, nsq.Finish(msg.Id))
assert.Equal(t, err, nil)
err = nsq.SendCommand(conn, nsq.Ready(1))
assert.Equal(t, err, nil)
// sleep to allow the RDY state to take effect
time.Sleep(50 * time.Millisecond)
// pause the channel... the client shouldn't receive any more messages
channel.Pause()
// sleep to allow the paused state to take effect
time.Sleep(50 * time.Millisecond)
msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body2"))
topic.PutMessage(msg)
// allow the client to possibly get a message, the test would hang indefinitely
// if pausing was not working on the internal clientMsgChan read
time.Sleep(50 * time.Millisecond)
msg = <-channel.clientMsgChan
assert.Equal(t, msg.Body, []byte("test body2"))
// unpause the channel... the client should now be pushed a message
channel.UnPause()
msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body3"))
topic.PutMessage(msg)
resp, _ = nsq.ReadResponse(conn)
_, data, _ = nsq.UnpackResponse(resp)
msg, err = nsq.DecodeMessage(data)
assert.Equal(t, msg.Body, []byte("test body3"))
}
func BenchmarkProtocolV2Command(b *testing.B) {
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
p := &ProtocolV2{}
c := NewClientV2(nil)
params := [][]byte{[]byte("SUB"), []byte("test"), []byte("ch")}
b.StartTimer()
for i := 0; i < b.N; i++ {
p.Exec(c, params)
}
}
func BenchmarkProtocolV2Data(b *testing.B) {
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
p := &ProtocolV2{}
var cb bytes.Buffer
rw := bufio.NewReadWriter(bufio.NewReader(&cb), bufio.NewWriter(ioutil.Discard))
conn := util.MockConn{rw}
c := NewClientV2(conn)
var buf bytes.Buffer
msg := nsq.NewMessage([]byte("0123456789abcdef"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
b.StartTimer()
for i := 0; i < b.N; i += 1 {
buf.Reset()
msg.Encode(&buf)
p.Send(c, nsq.FrameTypeMessage, buf.Bytes())
}
}
func BenchmarkProtocolV2Pub(b *testing.B) {
var wg sync.WaitGroup
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.memQueueSize = int64(b.N)
tcpAddr, _ := mustStartNSQd(options)
msg := []byte(`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`)
topicName := "bench_v1" + strconv.Itoa(int(time.Now().Unix()))
b.StartTimer()
for j := 0; j < runtime.GOMAXPROCS(0); j++ {
wg.Add(1)
go func() {
conn, err := mustConnectNSQd(tcpAddr)
if err != nil {
b.Fatal(err.Error())
}
for i := 0; i < (b.N / runtime.GOMAXPROCS(0)); i += 1 {
err := nsq.SendCommand(conn, nsq.Publish(topicName, msg))
if err != nil {
b.Fatal(err.Error())
}
resp, err := nsq.ReadResponse(conn)
if err != nil {
b.Fatal(err.Error())
}
_, data, _ := nsq.UnpackResponse(resp)
if !bytes.Equal(data, []byte("OK")) {
b.Fatal("invalid response")
}
b.SetBytes(int64(len(msg)))
}
wg.Done()
}()
}
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func TestClientTimeout(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 50 * time.Millisecond
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
err = nsq.Subscribe(topicName, "ch", "TestClientTimeoutV2", "TestClientTimeoutV2").Write(conn)
assert.Equal(t, err, nil)
time.Sleep(50 * time.Millisecond)
// depending on timing there may be 1 or 2 hearbeats sent
// just read until we get an error
timer := time.After(100 * time.Millisecond)
for {
select {
case <-timer:
t.Fatalf("test timed out")
default:
_, err := nsq.ReadResponse(conn)
if err != nil {
goto done
}
}
}
done:
}
func TestClientHeartbeat(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 100 * time.Millisecond
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
err = nsq.Subscribe(topicName, "ch", "TestClientHeartbeatV2", "TestClientHeartbeatV2").Write(conn)
assert.Equal(t, err, nil)
err = nsq.Ready(1).Write(conn)
assert.Equal(t, err, nil)
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
assert.Equal(t, data, []byte("_heartbeat_"))
time.Sleep(10 * time.Millisecond)
err = nsq.Nop().Write(conn)
assert.Equal(t, err, nil)
// wait long enough that would have timed out (had we not sent the above cmd)
time.Sleep(50 * time.Millisecond)
err = nsq.Nop().Write(conn)
assert.Equal(t, err, nil)
}
func TestPausing(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix()))
tcpAddr, _ := mustStartNSQd(NewNsqdOptions())
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
err = nsq.Subscribe(topicName, "ch", "TestPausing", "TestPausing").Write(conn)
assert.Equal(t, err, nil)
err = nsq.Ready(1).Write(conn)
assert.Equal(t, err, nil)
topic := nsqd.GetTopic(topicName)
msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
channel := topic.GetChannel("ch")
topic.PutMessage(msg)
// receive the first message via the client, finish it, and send new RDY
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
msg, err = nsq.DecodeMessage(data)
assert.Equal(t, msg.Body, []byte("test body"))
err = nsq.Finish(msg.Id).Write(conn)
assert.Equal(t, err, nil)
err = nsq.Ready(1).Write(conn)
assert.Equal(t, err, nil)
// sleep to allow the RDY state to take effect
time.Sleep(50 * time.Millisecond)
// pause the channel... the client shouldn't receive any more messages
channel.Pause()
// sleep to allow the paused state to take effect
time.Sleep(50 * time.Millisecond)
msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body2"))
topic.PutMessage(msg)
// allow the client to possibly get a message, the test would hang indefinitely
// if pausing was not working on the internal clientMsgChan read
time.Sleep(50 * time.Millisecond)
msg = <-channel.clientMsgChan
assert.Equal(t, msg.Body, []byte("test body2"))
// unpause the channel... the client should now be pushed a message
channel.UnPause()
msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body3"))
topic.PutMessage(msg)
resp, _ = nsq.ReadResponse(conn)
_, data, _ = nsq.UnpackResponse(resp)
msg, err = nsq.DecodeMessage(data)
assert.Equal(t, msg.Body, []byte("test body3"))
}
func BenchmarkProtocolV2Exec(b *testing.B) {
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
p := &ProtocolV2{}
c := NewClientV2(nil)
params := [][]byte{[]byte("NOP")}
b.StartTimer()
for i := 0; i < b.N; i++ {
p.Exec(c, params)
}
}
func BenchmarkProtocolV2Data(b *testing.B) {
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
p := &ProtocolV2{}
var cb bytes.Buffer
rw := bufio.NewReadWriter(bufio.NewReader(&cb), bufio.NewWriter(ioutil.Discard))
conn := util.MockConn{rw}
c := NewClientV2(conn)
var buf bytes.Buffer
body := make([]byte, 256)
msg := nsq.NewMessage([]byte("0123456789abcdef"), body)
b.SetBytes(int64(len(body)))
b.StartTimer()
for i := 0; i < b.N; i += 1 {
buf.Reset()
msg.Write(&buf)
p.Send(c, nsq.FrameTypeMessage, buf.Bytes())
}
}
func benchmarkProtocolV2Pub(b *testing.B, size int) {
var wg sync.WaitGroup
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.memQueueSize = int64(b.N)
tcpAddr, _ := mustStartNSQd(options)
msg := make([]byte, size)
batchSize := 200
batch := make([][]byte, 0)
for i := 0; i < batchSize; i++ {
batch = append(batch, msg)
}
topicName := "bench_v2_pub" + strconv.Itoa(int(time.Now().Unix()))
b.SetBytes(int64(len(msg)))
b.StartTimer()
for j := 0; j < runtime.GOMAXPROCS(0); j++ {
wg.Add(1)
go func() {
conn, err := mustConnectNSQd(tcpAddr)
if err != nil {
panic(err.Error())
}
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
num := b.N / runtime.GOMAXPROCS(0) / batchSize
for i := 0; i < num; i += 1 {
cmd, _ := nsq.MultiPublish(topicName, batch)
err := cmd.Write(rw)
if err != nil {
panic(err.Error())
}
err = rw.Flush()
if err != nil {
panic(err.Error())
}
resp, err := nsq.ReadResponse(rw)
if err != nil {
panic(err.Error())
}
_, data, _ := nsq.UnpackResponse(resp)
if !bytes.Equal(data, []byte("OK")) {
panic("invalid response")
}
}
wg.Done()
}()
}
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func BenchmarkProtocolV2Pub256(b *testing.B) { benchmarkProtocolV2Pub(b, 256) }
func BenchmarkProtocolV2Pub512(b *testing.B) { benchmarkProtocolV2Pub(b, 512) }
func BenchmarkProtocolV2Pub1k(b *testing.B) { benchmarkProtocolV2Pub(b, 1024) }
func BenchmarkProtocolV2Pub2k(b *testing.B) { benchmarkProtocolV2Pub(b, 2*1024) }
func BenchmarkProtocolV2Pub4k(b *testing.B) { benchmarkProtocolV2Pub(b, 4*1024) }
func BenchmarkProtocolV2Pub8k(b *testing.B) { benchmarkProtocolV2Pub(b, 8*1024) }
func BenchmarkProtocolV2Pub16k(b *testing.B) { benchmarkProtocolV2Pub(b, 16*1024) }
func BenchmarkProtocolV2Pub32k(b *testing.B) { benchmarkProtocolV2Pub(b, 32*1024) }
func BenchmarkProtocolV2Pub64k(b *testing.B) { benchmarkProtocolV2Pub(b, 64*1024) }
func BenchmarkProtocolV2Pub128k(b *testing.B) { benchmarkProtocolV2Pub(b, 128*1024) }
func BenchmarkProtocolV2Pub256k(b *testing.B) { benchmarkProtocolV2Pub(b, 256*1024) }
func BenchmarkProtocolV2Pub512k(b *testing.B) { benchmarkProtocolV2Pub(b, 512*1024) }
func BenchmarkProtocolV2Pub1m(b *testing.B) { benchmarkProtocolV2Pub(b, 1024*1024) }
func benchmarkProtocolV2Sub(b *testing.B, size int) {
var wg sync.WaitGroup
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.memQueueSize = int64(b.N)
tcpAddr, _ := mustStartNSQd(options)
msg := make([]byte, size)
topicName := "bench_v2_sub" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
for i := 0; i < b.N; i++ {
msg := nsq.NewMessage(<-nsqd.idChan, msg)
topic.PutMessage(msg)
}
topic.GetChannel("ch")
b.SetBytes(int64(len(msg)))
goChan := make(chan int)
rdyChan := make(chan int)
workers := runtime.GOMAXPROCS(0)
for j := 0; j < workers; j++ {
wg.Add(1)
go func(id int) {
subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan, id)
wg.Done()
}(j)
<-rdyChan
}
b.StartTimer()
close(goChan)
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func subWorker(n int, workers int, tcpAddr *net.TCPAddr, topicName string, rdyChan chan int, goChan chan int, id int) {
conn, err := mustConnectNSQd(tcpAddr)
if err != nil {
panic(err.Error())
}
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
nsq.Subscribe(topicName, "ch", "test", "test").Write(rw)
rdyCount := int(math.Min(math.Max(float64(n/workers), 1), 2500))
rdyChan <- 1
<-goChan
nsq.Ready(rdyCount).Write(rw)
rw.Flush()
num := n / workers
numRdy := num/rdyCount - 1
rdy := rdyCount
for i := 0; i < num; i += 1 {
resp, err := nsq.ReadResponse(rw)
if err != nil {
panic(err.Error())
}
frameType, data, err := nsq.UnpackResponse(resp)
if err != nil {
panic(err.Error())
}
if frameType != nsq.FrameTypeMessage {
panic("got something else")
}
msg, err := nsq.DecodeMessage(data)
if err != nil {
panic(err.Error())
}
nsq.Finish(msg.Id).Write(rw)
rdy--
if rdy == 0 && numRdy > 0 {
nsq.Ready(rdyCount).Write(rw)
rdy = rdyCount
numRdy--
rw.Flush()
}
}
}
func BenchmarkProtocolV2Sub256(b *testing.B) { benchmarkProtocolV2Sub(b, 256) }
func BenchmarkProtocolV2Sub512(b *testing.B) { benchmarkProtocolV2Sub(b, 512) }
func BenchmarkProtocolV2Sub1k(b *testing.B) { benchmarkProtocolV2Sub(b, 1024) }
func BenchmarkProtocolV2Sub2k(b *testing.B) { benchmarkProtocolV2Sub(b, 2*1024) }
func BenchmarkProtocolV2Sub4k(b *testing.B) { benchmarkProtocolV2Sub(b, 4*1024) }
func BenchmarkProtocolV2Sub8k(b *testing.B) { benchmarkProtocolV2Sub(b, 8*1024) }
func BenchmarkProtocolV2Sub16k(b *testing.B) { benchmarkProtocolV2Sub(b, 16*1024) }
func BenchmarkProtocolV2Sub32k(b *testing.B) { benchmarkProtocolV2Sub(b, 32*1024) }
func BenchmarkProtocolV2Sub64k(b *testing.B) { benchmarkProtocolV2Sub(b, 64*1024) }
func BenchmarkProtocolV2Sub128k(b *testing.B) { benchmarkProtocolV2Sub(b, 128*1024) }
func BenchmarkProtocolV2Sub256k(b *testing.B) { benchmarkProtocolV2Sub(b, 256*1024) }
func BenchmarkProtocolV2Sub512k(b *testing.B) { benchmarkProtocolV2Sub(b, 512*1024) }
func BenchmarkProtocolV2Sub1m(b *testing.B) { benchmarkProtocolV2Sub(b, 1024*1024) }
func TestMatchChar(t *testing.T) {
tests := []matchTest{
{in: "a", out: []string{"a"}},
{in: "\\\\a", out: []string{"\\\\", "a"}},
{in: "\\\"a", out: []string{"\\\"", "a"}},
{in: "\\\\\"a", out: []string{"\\\\", "a"}},
{in: "a\\\"\\a", out: []string{"a", "\\\"", "a"}},
{in: "a\\\"\\\\a", out: []string{"a", "\\\"", "\\\\", "a"}},
{in: "a\\\\\"\\a", out: []string{"a", "\\\\", "a"}},
}
re := regexp.MustCompile(hstoreChar)
for _, mt := range tests {
matches := re.FindAllStringIndex(mt.in, -1)
checkMatch(t, matches, mt)
}
}
func TestMatchString(t *testing.T) {
tests := []matchTest{
{in: "a", out: []string{}},
{in: "\"a\"", out: []string{"\"a\""}},
{in: "\"id\"=>\"44\", \"foo\"=>\"dfs => somf\", \"name\"=>\"Wash\\\"ington\", \"null\"=>NULL, \"quote\"=>\"\\\"fs ' \"",
out: []string{
"\"id\"",
"\"44\"",
"\"foo\"",
"\"dfs => somf\"",
"\"name\"",
"\"Wash\\\"ington\"",
"\"null\"",
"\"quote\"",
"\"\\\"fs ' \"",
}},
}
re := regexp.MustCompile("\"" + hstoreString + "\"")
for _, mt := range tests {
matches := re.FindAllStringIndex(mt.in, -1)
checkMatch(t, matches, mt)
}
}
func TestMatchKey(t *testing.T) {
tests := []matchTest{
{in: "a", out: []string{}},
{in: "\"a\"", out: []string{"\"a\""}},
{in: "\"id\"=>\"44\", \"foo\"=>\"dfs => somf\", \"name\"=>\"Wash\\\"ington\", \"null\"=>NULL, \"quote\"=>\"\\\"fs ' \"",
out: []string{
"\"id\"",
"\"44\"",
"\"foo\"",
"\"dfs => somf\"",
"\"name\"",
"\"Wash\\\"ington\"",
"\"null\"",
"\"quote\"",
"\"\\\"fs ' \"",
}},
}
re := regexp.MustCompile(hstoreKey)
for _, mt := range tests {
matches := re.FindAllStringIndex(mt.in, -1)
checkMatch(t, matches, mt)
}
}
func TestMatchValue(t *testing.T) {
tests := []matchTest{
{in: "a", out: []string{}},
{in: "\"a\"", out: []string{"\"a\""}},
{in: "\"id\"=>\"44\", \"foo\"=>\"dfs => somf\", \"name\"=>\"Wash\\\"ington\", \"null\"=>NULL, \"quote\"=>\"\\\"fs ' \"",
out: []string{
"\"id\"",
"\"44\"",
"\"foo\"",
"\"dfs => somf\"",
"\"name\"",
"\"Wash\\\"ington\"",
"\"null\"",
"NULL",
"\"quote\"",
"\"\\\"fs ' \"",
}},
}
re := regexp.MustCompile(hstoreValue)
for _, mt := range tests {
matches := re.FindAllStringIndex(mt.in, -1)
checkMatch(t, matches, mt)
}
}
func nullString(v interface{}) sql.NullString {
if v == nil {
return sql.NullString{Valid: false}
}
return sql.NullString{String: v.(string), Valid: true}
}
var fullTests = []matchTest{
{in: "NULL", out: []string{}},
{in: "\"a\" => \"b\"",
out: []string{"\"a\" => \"b\""},
hstore: Hstore{"a": nullString("b")}},
{in: "\"a\" => NULL",
out: []string{"\"a\" => NULL"},
hstore: Hstore{"a": sql.NullString{Valid: false}}},
{in: "\"id\"=>\"44\", \"foo\"=>\"dfs => somf\", \"name\"=>\"Wash\\\"ington\", \"null\"=>NULL, \"quote\"=>\"\\\"fs ' \"",
out: []string{
"\"id\"=>\"44\"",
"\"foo\"=>\"dfs => somf\"",
"\"name\"=>\"Wash\\\"ington\"",
"\"null\"=>NULL",
"\"quote\"=>\"\\\"fs ' \"",
},
hstore: Hstore{
"id": nullString("44"),
"foo": nullString("dfs => somf"),
"name": nullString("Wash\\\"ington"),
"null": nullString(nil),
"quote": nullString("\\\"fs ' "),
},
},
}
func TestParseHstore(t *testing.T) {
for _, mt := range fullTests {
matches := pairExp.FindAllStringIndex(mt.in, -1)
checkMatch(t, matches, mt)
hs := make(Hstore)
err := parseHstore(mt.in, &hs)
if err != nil {
t.Errorf("Error parsing %q: %v", mt.in, err)
}
checkHstore(t, mt.hstore, hs)
}
}
func TestValueAndScan(t *testing.T) {
for _, mt := range fullTests {
valued, err := mt.hstore.Value()
if err != nil {
t.Fatalf("Value failed: %q", err)
}
var scanned Hstore
(&scanned).Scan([]byte(valued.(string)))
checkHstore(t, mt.hstore, scanned)
}
}
func TestDBRoundTrip(t *testing.T) {
db := openTestConn(t)
defer db.Close()
_, err := db.Exec("CREATE EXTENSION IF NOT EXISTS hstore")
fatal(t, err)
_, err = db.Exec("CREATE TEMP TABLE temp (id serial, data hstore)")
fatal(t, err)
for _, mt := range fullTests {
v, err := mt.hstore.Value()
check(t, err)
r, err := db.Exec("INSERT INTO temp (data) VALUES ($1)", v)
check(t, err)
if n, _ := r.RowsAffected(); n != 1 {
t.Fatalf("expected 1 row affected, not %d", n)
}
}
rows, err := db.Query("SELECT data FROM temp ORDER BY id ASC")
check(t, err)
for _, mt := range fullTests {
if !rows.Next() {
t.Errorf("Ran out of rows!")
}
var data Hstore
err = rows.Scan(&data)
check(t, err)
t.Logf("%+v", data)
checkHstore(t, mt.hstore, data)
}
if rows.Next() {
t.Errorf("Too many rows!")
}
}
func TestClientTimeout(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 50 * time.Millisecond
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
err = nsq.SendCommand(conn, nsq.Subscribe(topicName, "ch", "TestClientTimeoutV2", "TestClientTimeoutV2"))
assert.Equal(t, err, nil)
time.Sleep(50 * time.Millisecond)
// depending on timing there may be 1 or 2 hearbeats sent
// just read until we get an error
timer := time.After(100 * time.Millisecond)
for {
select {
case <-timer:
t.Fatalf("test timed out")
default:
_, err := nsq.ReadResponse(conn)
if err != nil {
goto done
}
}
}
done:
}
func TestClientHeartbeat(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 100 * time.Millisecond
tcpAddr, _ := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
err = nsq.SendCommand(conn, nsq.Subscribe(topicName, "ch", "TestClientHeartbeatV2", "TestClientHeartbeatV2"))
assert.Equal(t, err, nil)
err = nsq.SendCommand(conn, nsq.Ready(1))
assert.Equal(t, err, nil)
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
assert.Equal(t, data, []byte("_heartbeat_"))
time.Sleep(10 * time.Millisecond)
err = nsq.SendCommand(conn, nsq.Nop())
assert.Equal(t, err, nil)
// wait long enough that would have timed out (had we not sent the above cmd)
time.Sleep(50 * time.Millisecond)
err = nsq.SendCommand(conn, nsq.Nop())
assert.Equal(t, err, nil)
}
func BenchmarkProtocolV2Command(b *testing.B) {
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
p := &ProtocolV2{}
c := NewClientV2(nil)
params := [][]byte{[]byte("SUB"), []byte("test"), []byte("ch")}
b.StartTimer()
for i := 0; i < b.N; i++ {
p.Exec(c, params)
}
}
func BenchmarkProtocolV2Data(b *testing.B) {
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
p := &ProtocolV2{}
var cb bytes.Buffer
rw := bufio.NewReadWriter(bufio.NewReader(&cb), bufio.NewWriter(ioutil.Discard))
conn := util.MockConn{rw}
c := NewClientV2(conn)
var buf bytes.Buffer
msg := nsq.NewMessage([]byte("0123456789abcdef"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
b.StartTimer()
for i := 0; i < b.N; i += 1 {
buf.Reset()
msg.Encode(&buf)
p.Send(c, nsq.FrameTypeMessage, buf.Bytes())
}
}
func BenchmarkProtocolV2Pub(b *testing.B) {
var wg sync.WaitGroup
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.memQueueSize = int64(b.N)
tcpAddr, _ := mustStartNSQd(options)
msg := []byte(`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`)
topicName := "bench_v1" + strconv.Itoa(int(time.Now().Unix()))
b.StartTimer()
for j := 0; j < runtime.GOMAXPROCS(0); j++ {
wg.Add(1)
go func() {
conn, err := mustConnectNSQd(tcpAddr)
if err != nil {
b.Fatal(err.Error())
}
for i := 0; i < (b.N / runtime.GOMAXPROCS(0)); i += 1 {
err := nsq.SendCommand(conn, nsq.Publish(topicName, msg))
if err != nil {
b.Fatal(err.Error())
}
resp, err := nsq.ReadResponse(conn)
if err != nil {
b.Fatal(err.Error())
}
_, data, _ := nsq.UnpackResponse(resp)
if !bytes.Equal(data, []byte("OK")) {
b.Fatal("invalid response")
}
b.SetBytes(int64(len(msg)))
}
wg.Done()
}()
}
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func TestChromeTermination(t *testing.T) {
auto := testDefaultStartup(t)
doneCh := make(chan struct{})
shutdown := time.NewTimer(time.Second * 4)
timeout := time.NewTimer(time.Second * 10)
terminatedHandler := func(reason string) {
t.Logf("reason: %s\n", reason)
doneCh <- struct{}{}
}
auto.SetTerminationHandler(terminatedHandler)
for {
select {
case <-doneCh:
goto DONE
case <-shutdown.C:
auto.Shutdown()
case <-timeout.C:
t.Fatalf("timed out waiting for termination")
}
}
DONE:
}
func testDefaultStartup(t *testing.T) *AutoGcd {
s := NewSettings(testPath, testRandomDir(t))
s.RemoveUserDir(true)
s.AddStartupFlags(testStartupFlags)
s.SetDebuggerPort(testRandomPort(t))
auto := NewAutoGcd(s)
if err := auto.Start(); err != nil {
t.Fatalf("failed to start chrome: %s\n", err)
}
auto.SetTerminationHandler(nil) // do not want our tests to panic
return auto
}
func testServer() {
testListener, _ = net.Listen("tcp", ":0")
_, testServerPort, _ := net.SplitHostPort(testListener.Addr().String())
testServerAddr = fmt.Sprintf("http://localhost:%s/", testServerPort)
go http.Serve(testListener, http.FileServer(http.Dir("testdata")))
}
func testRandomPort(t *testing.T) string {
l, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatal(err)
}
_, randPort, _ := net.SplitHostPort(l.Addr().String())
l.Close()
return randPort
}
func testRandomDir(t *testing.T) string {
dir, err := ioutil.TempDir(testDir, "autogcd")
if err != nil {
t.Fatalf("error getting temp dir: %s\n", err)
}
return dir
}
func TestClientTimeout(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 50 * time.Millisecond
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identify(t, conn)
sub(t, conn, topicName, "ch")
time.Sleep(50 * time.Millisecond)
// depending on timing there may be 1 or 2 hearbeats sent
// just read until we get an error
timer := time.After(100 * time.Millisecond)
for {
select {
case <-timer:
t.Fatalf("test timed out")
default:
_, err := nsq.ReadResponse(conn)
if err != nil {
goto done
}
}
}
done:
}
func TestClientHeartbeat(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 200 * time.Millisecond
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identify(t, conn)
sub(t, conn, topicName, "ch")
err = nsq.Ready(1).Write(conn)
assert.Equal(t, err, nil)
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
assert.Equal(t, data, []byte("_heartbeat_"))
time.Sleep(20 * time.Millisecond)
err = nsq.Nop().Write(conn)
assert.Equal(t, err, nil)
// wait long enough that would have timed out (had we not sent the above cmd)
time.Sleep(100 * time.Millisecond)
err = nsq.Nop().Write(conn)
assert.Equal(t, err, nil)
}
func TestClientHeartbeatDisableSUB(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))
options := NewNsqdOptions()
options.clientTimeout = 200 * time.Millisecond
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identifyHeartbeatInterval(t, conn, -1, nsq.FrameTypeResponse, "OK")
subFail(t, conn, topicName, "ch")
}
func TestClientHeartbeatDisable(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.clientTimeout = 100 * time.Millisecond
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identifyHeartbeatInterval(t, conn, -1, nsq.FrameTypeResponse, "OK")
time.Sleep(150 * time.Millisecond)
err = nsq.Nop().Write(conn)
assert.Equal(t, err, nil)
}
func TestMaxHeartbeatIntervalValid(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.maxHeartbeatInterval = 300 * time.Second
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
hbi := int(options.maxHeartbeatInterval / time.Millisecond)
identifyHeartbeatInterval(t, conn, hbi, nsq.FrameTypeResponse, "OK")
}
func TestMaxHeartbeatIntervalInvalid(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.maxHeartbeatInterval = 300 * time.Second
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
hbi := int(options.maxHeartbeatInterval/time.Millisecond + 1)
identifyHeartbeatInterval(t, conn, hbi, nsq.FrameTypeError, "E_BAD_BODY IDENTIFY heartbeat interval (300001) is invalid")
}
func TestPausing(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix()))
tcpAddr, _, nsqd := mustStartNSQd(NewNsqdOptions())
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identify(t, conn)
sub(t, conn, topicName, "ch")
err = nsq.Ready(1).Write(conn)
assert.Equal(t, err, nil)
topic := nsqd.GetTopic(topicName)
msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
channel := topic.GetChannel("ch")
topic.PutMessage(msg)
// receive the first message via the client, finish it, and send new RDY
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
msg, err = nsq.DecodeMessage(data)
assert.Equal(t, msg.Body, []byte("test body"))
err = nsq.Finish(msg.Id).Write(conn)
assert.Equal(t, err, nil)
err = nsq.Ready(1).Write(conn)
assert.Equal(t, err, nil)
// sleep to allow the RDY state to take effect
time.Sleep(50 * time.Millisecond)
// pause the channel... the client shouldn't receive any more messages
channel.Pause()
// sleep to allow the paused state to take effect
time.Sleep(50 * time.Millisecond)
msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body2"))
topic.PutMessage(msg)
// allow the client to possibly get a message, the test would hang indefinitely
// if pausing was not working on the internal clientMsgChan read
time.Sleep(50 * time.Millisecond)
msg = <-channel.clientMsgChan
assert.Equal(t, msg.Body, []byte("test body2"))
// unpause the channel... the client should now be pushed a message
channel.UnPause()
msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body3"))
topic.PutMessage(msg)
resp, _ = nsq.ReadResponse(conn)
_, data, _ = nsq.UnpackResponse(resp)
msg, err = nsq.DecodeMessage(data)
assert.Equal(t, msg.Body, []byte("test body3"))
}
func TestEmptyCommand(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
tcpAddr, _, nsqd := mustStartNSQd(NewNsqdOptions())
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
_, err = conn.Write([]byte("\n\n"))
assert.Equal(t, err, nil)
// if we didn't panic here we're good, see issue #120
}
func TestSizeLimits(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
*verbose = true
options.maxMessageSize = 100
options.maxBodySize = 1000
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
topicName := "test_limits_v2" + strconv.Itoa(int(time.Now().Unix()))
identify(t, conn)
sub(t, conn, topicName, "ch")
// PUB that's valid
nsq.Publish(topicName, make([]byte, 95)).Write(conn)
resp, _ := nsq.ReadResponse(conn)
frameType, data, _ := nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
assert.Equal(t, data, []byte("OK"))
// PUB that's invalid (too big)
nsq.Publish(topicName, make([]byte, 105)).Write(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeError)
assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE PUB message too big 105 > 100"))
// need to reconnect
conn, err = mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
// PUB thats empty
nsq.Publish(topicName, make([]byte, 0)).Write(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeError)
assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE PUB invalid message body size 0"))
// need to reconnect
conn, err = mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
// MPUB body that's valid
mpub := make([][]byte, 0)
for i := 0; i < 5; i++ {
mpub = append(mpub, make([]byte, 100))
}
cmd, _ := nsq.MultiPublish(topicName, mpub)
cmd.Write(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
assert.Equal(t, data, []byte("OK"))
// MPUB body that's invalid (body too big)
mpub = make([][]byte, 0)
for i := 0; i < 11; i++ {
mpub = append(mpub, make([]byte, 100))
}
cmd, _ = nsq.MultiPublish(topicName, mpub)
cmd.Write(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeError)
assert.Equal(t, string(data), fmt.Sprintf("E_BAD_BODY MPUB body too big 1148 > 1000"))
// need to reconnect
conn, err = mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
// MPUB that's invalid (one message empty)
mpub = make([][]byte, 0)
for i := 0; i < 5; i++ {
mpub = append(mpub, make([]byte, 100))
}
mpub = append(mpub, make([]byte, 0))
cmd, _ = nsq.MultiPublish(topicName, mpub)
cmd.Write(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeError)
assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE MPUB invalid message(5) body size 0"))
// need to reconnect
conn, err = mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
// MPUB body that's invalid (one of the messages is too big)
mpub = make([][]byte, 0)
for i := 0; i < 5; i++ {
mpub = append(mpub, make([]byte, 101))
}
cmd, _ = nsq.MultiPublish(topicName, mpub)
cmd.Write(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeError)
assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE MPUB message too big 101 > 100"))
}
func TestTouch(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
*verbose = true
options := NewNsqdOptions()
options.msgTimeout = 50 * time.Millisecond
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
topicName := "test_touch" + strconv.Itoa(int(time.Now().Unix()))
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identify(t, conn)
sub(t, conn, topicName, "ch")
topic := nsqd.GetTopic(topicName)
channel := topic.GetChannel("ch")
msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
topic.PutMessage(msg)
err = nsq.Ready(1).Write(conn)
assert.Equal(t, err, nil)
resp, err := nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
frameType, data, err := nsq.UnpackResponse(resp)
msgOut, _ := nsq.DecodeMessage(data)
assert.Equal(t, frameType, nsq.FrameTypeMessage)
assert.Equal(t, msgOut.Id, msg.Id)
time.Sleep(25 * time.Millisecond)
err = nsq.Touch(msg.Id).Write(conn)
assert.Equal(t, err, nil)
time.Sleep(30 * time.Millisecond)
err = nsq.Finish(msg.Id).Write(conn)
assert.Equal(t, err, nil)
assert.Equal(t, channel.timeoutCount, uint64(0))
}
func TestMaxRdyCount(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
*verbose = true
options := NewNsqdOptions()
options.maxRdyCount = 50
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
topicName := "test_max_rdy_count" + strconv.Itoa(int(time.Now().Unix()))
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
topic := nsqd.GetTopic(topicName)
msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
topic.PutMessage(msg)
data := identifyFeatureNegotiation(t, conn, nil)
r := struct {
MaxRdyCount int64 `json:"max_rdy_count"`
}{}
err = json.Unmarshal(data, &r)
assert.Equal(t, err, nil)
assert.Equal(t, r.MaxRdyCount, int64(50))
sub(t, conn, topicName, "ch")
err = nsq.Ready(int(options.maxRdyCount)).Write(conn)
assert.Equal(t, err, nil)
resp, err := nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
frameType, data, err := nsq.UnpackResponse(resp)
msgOut, _ := nsq.DecodeMessage(data)
assert.Equal(t, frameType, nsq.FrameTypeMessage)
assert.Equal(t, msgOut.Id, msg.Id)
err = nsq.Ready(int(options.maxRdyCount) + 1).Write(conn)
assert.Equal(t, err, nil)
resp, err = nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
frameType, data, err = nsq.UnpackResponse(resp)
assert.Equal(t, frameType, int32(1))
assert.Equal(t, string(data), "E_INVALID RDY count 51 out of range 0-50")
}
func TestFatalError(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
tcpAddr, _, nsqd := mustStartNSQd(NewNsqdOptions())
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
_, err = conn.Write([]byte("ASDF\n"))
assert.Equal(t, err, nil)
resp, err := nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
frameType, data, err := nsq.UnpackResponse(resp)
assert.Equal(t, frameType, int32(1))
assert.Equal(t, string(data), "E_INVALID invalid command ASDF")
_, err = nsq.ReadResponse(conn)
assert.NotEqual(t, err, nil)
}
func TestOutputBuffering(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
*verbose = true
options := NewNsqdOptions()
options.maxOutputBufferSize = 512 * 1024
options.maxOutputBufferTimeout = time.Second
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
topicName := "test_output_buffering" + strconv.Itoa(int(time.Now().Unix()))
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
outputBufferSize := 256 * 1024
outputBufferTimeout := 500
topic := nsqd.GetTopic(topicName)
msg := nsq.NewMessage(<-nsqd.idChan, make([]byte, outputBufferSize-1024))
topic.PutMessage(msg)
start := time.Now()
identifyOutputBuffering(t, conn, outputBufferSize, outputBufferTimeout, nsq.FrameTypeResponse, "OK")
sub(t, conn, topicName, "ch")
err = nsq.Ready(10).Write(conn)
assert.Equal(t, err, nil)
resp, err := nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
end := time.Now()
assert.Equal(t, int(end.Sub(start)/time.Millisecond) >= outputBufferTimeout, true)
frameType, data, err := nsq.UnpackResponse(resp)
msgOut, _ := nsq.DecodeMessage(data)
assert.Equal(t, frameType, nsq.FrameTypeMessage)
assert.Equal(t, msgOut.Id, msg.Id)
}
func TestOutputBufferingValidity(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
*verbose = true
options := NewNsqdOptions()
options.maxOutputBufferSize = 512 * 1024
options.maxOutputBufferTimeout = time.Second
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identifyOutputBuffering(t, conn, 512*1024, 1000, nsq.FrameTypeResponse, "OK")
identifyOutputBuffering(t, conn, -1, -1, nsq.FrameTypeResponse, "OK")
identifyOutputBuffering(t, conn, 0, 0, nsq.FrameTypeResponse, "OK")
identifyOutputBuffering(t, conn, 512*1024+1, 0, nsq.FrameTypeError, fmt.Sprintf("E_BAD_BODY IDENTIFY output buffer size (%d) is invalid", 512*1024+1))
conn, err = mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
identifyOutputBuffering(t, conn, 0, 1001, nsq.FrameTypeError, "E_BAD_BODY IDENTIFY output buffer timeout (1001) is invalid")
}
func TestTLS(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
*verbose = true
options := NewNsqdOptions()
options.tlsCert = "./test/cert.pem"
options.tlsKey = "./test/key.pem"
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"tls_v1": true})
r := struct {
TLSv1 bool `json:"tls_v1"`
}{}
err = json.Unmarshal(data, &r)
assert.Equal(t, err, nil)
assert.Equal(t, r.TLSv1, true)
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
}
tlsConn := tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
assert.Equal(t, err, nil)
resp, _ := nsq.ReadResponse(tlsConn)
frameType, data, _ := nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
assert.Equal(t, data, []byte("OK"))
}
func TestDeflate(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
*verbose = true
options := NewNsqdOptions()
options.deflateEnabled = true
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"deflate": true})
r := struct {
Deflate bool `json:"deflate"`
}{}
err = json.Unmarshal(data, &r)
assert.Equal(t, err, nil)
assert.Equal(t, r.Deflate, true)
compressConn := flate.NewReader(conn)
resp, _ := nsq.ReadResponse(compressConn)
frameType, data, _ := nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
assert.Equal(t, data, []byte("OK"))
}
type readWriter struct {
io.Reader
io.Writer
}
func TestSnappy(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
*verbose = true
options := NewNsqdOptions()
options.snappyEnabled = true
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"snappy": true})
r := struct {
Snappy bool `json:"snappy"`
}{}
err = json.Unmarshal(data, &r)
assert.Equal(t, err, nil)
assert.Equal(t, r.Snappy, true)
compressConn := snappystream.NewReader(conn, snappystream.SkipVerifyChecksum)
resp, _ := nsq.ReadResponse(compressConn)
frameType, data, _ := nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
assert.Equal(t, data, []byte("OK"))
msgBody := make([]byte, 128000)
w := snappystream.NewWriter(conn)
rw := readWriter{compressConn, w}
topicName := "test_snappy" + strconv.Itoa(int(time.Now().Unix()))
sub(t, rw, topicName, "ch")
err = nsq.Ready(1).Write(rw)
assert.Equal(t, err, nil)
topic := nsqd.GetTopic(topicName)
msg := nsq.NewMessage(<-nsqd.idChan, msgBody)
topic.PutMessage(msg)
resp, _ = nsq.ReadResponse(compressConn)
frameType, data, _ = nsq.UnpackResponse(resp)
msgOut, _ := nsq.DecodeMessage(data)
assert.Equal(t, frameType, nsq.FrameTypeMessage)
assert.Equal(t, msgOut.Id, msg.Id)
assert.Equal(t, msgOut.Body, msg.Body)
}
func TestTLSDeflate(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
*verbose = true
options := NewNsqdOptions()
options.deflateEnabled = true
options.tlsCert = "./test/cert.pem"
options.tlsKey = "./test/key.pem"
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"tls_v1": true, "deflate": true})
r := struct {
TLSv1 bool `json:"tls_v1"`
Deflate bool `json:"deflate"`
}{}
err = json.Unmarshal(data, &r)
assert.Equal(t, err, nil)
assert.Equal(t, r.TLSv1, true)
assert.Equal(t, r.Deflate, true)
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
}
tlsConn := tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
assert.Equal(t, err, nil)
resp, _ := nsq.ReadResponse(tlsConn)
frameType, data, _ := nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
assert.Equal(t, data, []byte("OK"))
compressConn := flate.NewReader(tlsConn)
resp, _ = nsq.ReadResponse(compressConn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
assert.Equal(t, data, []byte("OK"))
}
func TestSampling(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
rand.Seed(time.Now().UTC().UnixNano())
num := 3000
*verbose = true
options := NewNsqdOptions()
options.maxRdyCount = int64(num)
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"sample_rate": int32(42)})
r := struct {
SampleRate int32 `json:"sample_rate"`
}{}
err = json.Unmarshal(data, &r)
assert.Equal(t, err, nil)
assert.Equal(t, r.SampleRate, int32(42))
topicName := "test_sampling" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
for i := 0; i < num; i++ {
msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
topic.PutMessage(msg)
}
channel := topic.GetChannel("ch")
// let the topic drain into the channel
time.Sleep(50 * time.Millisecond)
sub(t, conn, topicName, "ch")
err = nsq.Ready(num).Write(conn)
assert.Equal(t, err, nil)
doneChan := make(chan int)
go func() {
for {
if channel.Depth() == 0 {
close(doneChan)
return
}
time.Sleep(5 * time.Millisecond)
}
}()
<-doneChan
// within 3%
assert.Equal(t, len(channel.inFlightMessages) <= int(float64(num)*0.45), true)
assert.Equal(t, len(channel.inFlightMessages) >= int(float64(num)*0.39), true)
}
func TestTLSSnappy(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
*verbose = true
options := NewNsqdOptions()
options.snappyEnabled = true
options.tlsCert = "./test/cert.pem"
options.tlsKey = "./test/key.pem"
tcpAddr, _, nsqd := mustStartNSQd(options)
defer nsqd.Exit()
conn, err := mustConnectNSQd(tcpAddr)
assert.Equal(t, err, nil)
data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"tls_v1": true, "snappy": true})
r := struct {
TLSv1 bool `json:"tls_v1"`
Snappy bool `json:"snappy"`
}{}
err = json.Unmarshal(data, &r)
assert.Equal(t, err, nil)
assert.Equal(t, r.TLSv1, true)
assert.Equal(t, r.Snappy, true)
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
}
tlsConn := tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
assert.Equal(t, err, nil)
resp, _ := nsq.ReadResponse(tlsConn)
frameType, data, _ := nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
assert.Equal(t, data, []byte("OK"))
compressConn := snappystream.NewReader(tlsConn, snappystream.SkipVerifyChecksum)
resp, _ = nsq.ReadResponse(compressConn)
frameType, data, _ = nsq.UnpackResponse(resp)
log.Printf("frameType: %d, data: %s", frameType, data)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
assert.Equal(t, data, []byte("OK"))
}
func BenchmarkProtocolV2Exec(b *testing.B) {
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
p := &ProtocolV2{}
c := NewClientV2(0, nil, nil)
params := [][]byte{[]byte("NOP")}
b.StartTimer()
for i := 0; i < b.N; i++ {
p.Exec(c, params)
}
}
func benchmarkProtocolV2Pub(b *testing.B, size int) {
var wg sync.WaitGroup
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.memQueueSize = int64(b.N)
tcpAddr, _, nsqd := mustStartNSQd(options)
msg := make([]byte, size)
batchSize := 200
batch := make([][]byte, 0)
for i := 0; i < batchSize; i++ {
batch = append(batch, msg)
}
topicName := "bench_v2_pub" + strconv.Itoa(int(time.Now().Unix()))
b.SetBytes(int64(len(msg)))
b.StartTimer()
for j := 0; j < runtime.GOMAXPROCS(0); j++ {
wg.Add(1)
go func() {
conn, err := mustConnectNSQd(tcpAddr)
if err != nil {
panic(err.Error())
}
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
num := b.N / runtime.GOMAXPROCS(0) / batchSize
for i := 0; i < num; i += 1 {
cmd, _ := nsq.MultiPublish(topicName, batch)
err := cmd.Write(rw)
if err != nil {
panic(err.Error())
}
err = rw.Flush()
if err != nil {
panic(err.Error())
}
resp, err := nsq.ReadResponse(rw)
if err != nil {
panic(err.Error())
}
_, data, _ := nsq.UnpackResponse(resp)
if !bytes.Equal(data, []byte("OK")) {
panic("invalid response")
}
}
wg.Done()
}()
}
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func BenchmarkProtocolV2Pub256(b *testing.B) { benchmarkProtocolV2Pub(b, 256) }
func BenchmarkProtocolV2Pub512(b *testing.B) { benchmarkProtocolV2Pub(b, 512) }
func BenchmarkProtocolV2Pub1k(b *testing.B) { benchmarkProtocolV2Pub(b, 1024) }
func BenchmarkProtocolV2Pub2k(b *testing.B) { benchmarkProtocolV2Pub(b, 2*1024) }
func BenchmarkProtocolV2Pub4k(b *testing.B) { benchmarkProtocolV2Pub(b, 4*1024) }
func BenchmarkProtocolV2Pub8k(b *testing.B) { benchmarkProtocolV2Pub(b, 8*1024) }
func BenchmarkProtocolV2Pub16k(b *testing.B) { benchmarkProtocolV2Pub(b, 16*1024) }
func BenchmarkProtocolV2Pub32k(b *testing.B) { benchmarkProtocolV2Pub(b, 32*1024) }
func BenchmarkProtocolV2Pub64k(b *testing.B) { benchmarkProtocolV2Pub(b, 64*1024) }
func BenchmarkProtocolV2Pub128k(b *testing.B) { benchmarkProtocolV2Pub(b, 128*1024) }
func BenchmarkProtocolV2Pub256k(b *testing.B) { benchmarkProtocolV2Pub(b, 256*1024) }
func BenchmarkProtocolV2Pub512k(b *testing.B) { benchmarkProtocolV2Pub(b, 512*1024) }
func BenchmarkProtocolV2Pub1m(b *testing.B) { benchmarkProtocolV2Pub(b, 1024*1024) }
func benchmarkProtocolV2Sub(b *testing.B, size int) {
var wg sync.WaitGroup
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.memQueueSize = int64(b.N)
tcpAddr, _, nsqd := mustStartNSQd(options)
msg := make([]byte, size)
topicName := "bench_v2_sub" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
for i := 0; i < b.N; i++ {
msg := nsq.NewMessage(<-nsqd.idChan, msg)
topic.PutMessage(msg)
}
topic.GetChannel("ch")
b.SetBytes(int64(len(msg)))
goChan := make(chan int)
rdyChan := make(chan int)
workers := runtime.GOMAXPROCS(0)
for j := 0; j < workers; j++ {
wg.Add(1)
go func() {
subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
wg.Done()
}()
<-rdyChan
}
b.StartTimer()
close(goChan)
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func subWorker(n int, workers int, tcpAddr *net.TCPAddr, topicName string, rdyChan chan int, goChan chan int) {
conn, err := mustConnectNSQd(tcpAddr)
if err != nil {
panic(err.Error())
}
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
identify(nil, conn)
sub(nil, conn, topicName, "ch")
rdyCount := int(math.Min(math.Max(float64(n/workers), 1), 2500))
rdyChan <- 1
<-goChan
nsq.Ready(rdyCount).Write(rw)
rw.Flush()
num := n / workers
numRdy := num/rdyCount - 1
rdy := rdyCount
for i := 0; i < num; i += 1 {
resp, err := nsq.ReadResponse(rw)
if err != nil {
panic(err.Error())
}
frameType, data, err := nsq.UnpackResponse(resp)
if err != nil {
panic(err.Error())
}
if frameType != nsq.FrameTypeMessage {
panic("got something else")
}
msg, err := nsq.DecodeMessage(data)
if err != nil {
panic(err.Error())
}
nsq.Finish(msg.Id).Write(rw)
rdy--
if rdy == 0 && numRdy > 0 {
nsq.Ready(rdyCount).Write(rw)
rdy = rdyCount
numRdy--
rw.Flush()
}
}
}
func BenchmarkProtocolV2Sub256(b *testing.B) { benchmarkProtocolV2Sub(b, 256) }
func BenchmarkProtocolV2Sub512(b *testing.B) { benchmarkProtocolV2Sub(b, 512) }
func BenchmarkProtocolV2Sub1k(b *testing.B) { benchmarkProtocolV2Sub(b, 1024) }
func BenchmarkProtocolV2Sub2k(b *testing.B) { benchmarkProtocolV2Sub(b, 2*1024) }
func BenchmarkProtocolV2Sub4k(b *testing.B) { benchmarkProtocolV2Sub(b, 4*1024) }
func BenchmarkProtocolV2Sub8k(b *testing.B) { benchmarkProtocolV2Sub(b, 8*1024) }
func BenchmarkProtocolV2Sub16k(b *testing.B) { benchmarkProtocolV2Sub(b, 16*1024) }
func BenchmarkProtocolV2Sub32k(b *testing.B) { benchmarkProtocolV2Sub(b, 32*1024) }
func BenchmarkProtocolV2Sub64k(b *testing.B) { benchmarkProtocolV2Sub(b, 64*1024) }
func BenchmarkProtocolV2Sub128k(b *testing.B) { benchmarkProtocolV2Sub(b, 128*1024) }
func BenchmarkProtocolV2Sub256k(b *testing.B) { benchmarkProtocolV2Sub(b, 256*1024) }
func BenchmarkProtocolV2Sub512k(b *testing.B) { benchmarkProtocolV2Sub(b, 512*1024) }
func BenchmarkProtocolV2Sub1m(b *testing.B) { benchmarkProtocolV2Sub(b, 1024*1024) }
func benchmarkProtocolV2MultiSub(b *testing.B, num int) {
var wg sync.WaitGroup
b.StopTimer()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
options := NewNsqdOptions()
options.memQueueSize = int64(b.N)
tcpAddr, _, nsqd := mustStartNSQd(options)
msg := make([]byte, 256)
b.SetBytes(int64(len(msg) * num))
goChan := make(chan int)
rdyChan := make(chan int)
workers := runtime.GOMAXPROCS(0)
for i := 0; i < num; i++ {
topicName := "bench_v2" + strconv.Itoa(b.N) + "_" + strconv.Itoa(i) + "_" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
for i := 0; i < b.N; i++ {
msg := nsq.NewMessage(<-nsqd.idChan, msg)
topic.PutMessage(msg)
}
topic.GetChannel("ch")
for j := 0; j < workers; j++ {
wg.Add(1)
go func() {
subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
wg.Done()
}()
<-rdyChan
}
}
b.StartTimer()
close(goChan)
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func BenchmarkProtocolV2MultiSub1(b *testing.B) { benchmarkProtocolV2MultiSub(b, 1) }
func BenchmarkProtocolV2MultiSub2(b *testing.B) { benchmarkProtocolV2MultiSub(b, 2) }
func BenchmarkProtocolV2MultiSub4(b *testing.B) { benchmarkProtocolV2MultiSub(b, 4) }
func BenchmarkProtocolV2MultiSub8(b *testing.B) { benchmarkProtocolV2MultiSub(b, 8) }
func BenchmarkProtocolV2MultiSub16(b *testing.B) { benchmarkProtocolV2MultiSub(b, 16) }
func TestClientTimeout(t *testing.T) {
topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.ClientTimeout = 150 * time.Millisecond
opts.Verbose = true
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
identify(t, conn, nil, frameTypeResponse)
sub(t, conn, topicName, "ch")
time.Sleep(150 * time.Millisecond)
// depending on timing there may be 1 or 2 hearbeats sent
// just read until we get an error
timer := time.After(100 * time.Millisecond)
for {
select {
case <-timer:
t.Fatalf("test timed out")
default:
_, err := nsq.ReadResponse(conn)
if err != nil {
goto done
}
}
}
done:
}
func TestClientHeartbeat(t *testing.T) {
topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.ClientTimeout = 200 * time.Millisecond
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
identify(t, conn, nil, frameTypeResponse)
sub(t, conn, topicName, "ch")
_, err = nsq.Ready(1).WriteTo(conn)
equal(t, err, nil)
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
equal(t, data, []byte("_heartbeat_"))
time.Sleep(20 * time.Millisecond)
_, err = nsq.Nop().WriteTo(conn)
equal(t, err, nil)
// wait long enough that would have timed out (had we not sent the above cmd)
time.Sleep(100 * time.Millisecond)
_, err = nsq.Nop().WriteTo(conn)
equal(t, err, nil)
}
func TestClientHeartbeatDisableSUB(t *testing.T) {
topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.ClientTimeout = 200 * time.Millisecond
opts.Verbose = true
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
identify(t, conn, map[string]interface{}{
"heartbeat_interval": -1,
}, frameTypeResponse)
subFail(t, conn, topicName, "ch")
}
func TestClientHeartbeatDisable(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.ClientTimeout = 100 * time.Millisecond
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
identify(t, conn, map[string]interface{}{
"heartbeat_interval": -1,
}, frameTypeResponse)
time.Sleep(150 * time.Millisecond)
_, err = nsq.Nop().WriteTo(conn)
equal(t, err, nil)
}
func TestMaxHeartbeatIntervalValid(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.MaxHeartbeatInterval = 300 * time.Second
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
hbi := int(opts.MaxHeartbeatInterval / time.Millisecond)
identify(t, conn, map[string]interface{}{
"heartbeat_interval": hbi,
}, frameTypeResponse)
}
func TestMaxHeartbeatIntervalInvalid(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.MaxHeartbeatInterval = 300 * time.Second
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
hbi := int(opts.MaxHeartbeatInterval/time.Millisecond + 1)
data := identify(t, conn, map[string]interface{}{
"heartbeat_interval": hbi,
}, frameTypeError)
equal(t, string(data), "E_BAD_BODY IDENTIFY heartbeat interval (300001) is invalid")
}
func TestPausing(t *testing.T) {
topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix()))
opts := NewOptions()
opts.Logger = newTestLogger(t)
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
identify(t, conn, nil, frameTypeResponse)
sub(t, conn, topicName, "ch")
_, err = nsq.Ready(1).WriteTo(conn)
equal(t, err, nil)
topic := nsqd.GetTopic(topicName)
msg := NewMessage(<-nsqd.idChan, []byte("test body"))
channel := topic.GetChannel("ch")
topic.PutMessage(msg)
// receive the first message via the client, finish it, and send new RDY
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
msg, err = decodeMessage(data)
equal(t, msg.Body, []byte("test body"))
_, err = nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(conn)
equal(t, err, nil)
_, err = nsq.Ready(1).WriteTo(conn)
equal(t, err, nil)
// sleep to allow the RDY state to take effect
time.Sleep(50 * time.Millisecond)
// pause the channel... the client shouldn't receive any more messages
channel.Pause()
// sleep to allow the paused state to take effect
time.Sleep(50 * time.Millisecond)
msg = NewMessage(<-nsqd.idChan, []byte("test body2"))
topic.PutMessage(msg)
// allow the client to possibly get a message, the test would hang indefinitely
// if pausing was not working on the internal clientMsgChan read
time.Sleep(50 * time.Millisecond)
msg = <-channel.clientMsgChan
equal(t, msg.Body, []byte("test body2"))
// unpause the channel... the client should now be pushed a message
channel.UnPause()
msg = NewMessage(<-nsqd.idChan, []byte("test body3"))
topic.PutMessage(msg)
resp, _ = nsq.ReadResponse(conn)
_, data, _ = nsq.UnpackResponse(resp)
msg, err = decodeMessage(data)
equal(t, msg.Body, []byte("test body3"))
}
func TestEmptyCommand(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
_, err = conn.Write([]byte("\n\n"))
equal(t, err, nil)
// if we didn't panic here we're good, see issue #120
}
func TestSizeLimits(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.MaxMsgSize = 100
opts.MaxBodySize = 1000
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
topicName := "test_limits_v2" + strconv.Itoa(int(time.Now().Unix()))
identify(t, conn, nil, frameTypeResponse)
sub(t, conn, topicName, "ch")
// PUB that's valid
nsq.Publish(topicName, make([]byte, 95)).WriteTo(conn)
resp, _ := nsq.ReadResponse(conn)
frameType, data, _ := nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
// PUB that's invalid (too big)
nsq.Publish(topicName, make([]byte, 105)).WriteTo(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeError)
equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE PUB message too big 105 > 100"))
// need to reconnect
conn, err = mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
// PUB thats empty
nsq.Publish(topicName, []byte{}).WriteTo(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeError)
equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE PUB invalid message body size 0"))
// need to reconnect
conn, err = mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
// MPUB body that's valid
mpub := make([][]byte, 5)
for i := range mpub {
mpub[i] = make([]byte, 100)
}
cmd, _ := nsq.MultiPublish(topicName, mpub)
cmd.WriteTo(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
// MPUB body that's invalid (body too big)
mpub = make([][]byte, 11)
for i := range mpub {
mpub[i] = make([]byte, 100)
}
cmd, _ = nsq.MultiPublish(topicName, mpub)
cmd.WriteTo(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeError)
equal(t, string(data), fmt.Sprintf("E_BAD_BODY MPUB body too big 1148 > 1000"))
// need to reconnect
conn, err = mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
// MPUB that's invalid (one message empty)
mpub = make([][]byte, 5)
for i := range mpub {
mpub[i] = make([]byte, 100)
}
mpub = append(mpub, []byte{})
cmd, _ = nsq.MultiPublish(topicName, mpub)
cmd.WriteTo(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeError)
equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE MPUB invalid message(5) body size 0"))
// need to reconnect
conn, err = mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
// MPUB body that's invalid (one of the messages is too big)
mpub = make([][]byte, 5)
for i := range mpub {
mpub[i] = make([]byte, 101)
}
cmd, _ = nsq.MultiPublish(topicName, mpub)
cmd.WriteTo(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeError)
equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE MPUB message too big 101 > 100"))
}
func TestDPUB(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
topicName := "test_dpub_v2" + strconv.Itoa(int(time.Now().Unix()))
identify(t, conn, nil, frameTypeResponse)
sub(t, conn, topicName, "ch")
// valid
nsq.DeferredPublish(topicName, time.Second, make([]byte, 100)).WriteTo(conn)
resp, _ := nsq.ReadResponse(conn)
frameType, data, _ := nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
time.Sleep(25 * time.Millisecond)
ch := nsqd.GetTopic(topicName).GetChannel("ch")
ch.Lock()
numDef := len(ch.deferredMessages)
ch.Unlock()
equal(t, numDef, 1)
// duration out of range
nsq.DeferredPublish(topicName, opts.MaxReqTimeout+100*time.Millisecond, make([]byte, 100)).WriteTo(conn)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ = nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeError)
equal(t, string(data), fmt.Sprintf("E_INVALID DPUB timeout 3600100 out of range 0-3600000"))
}
func TestTouch(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.MsgTimeout = 150 * time.Millisecond
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_touch" + strconv.Itoa(int(time.Now().Unix()))
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
identify(t, conn, nil, frameTypeResponse)
sub(t, conn, topicName, "ch")
topic := nsqd.GetTopic(topicName)
channel := topic.GetChannel("ch")
msg := NewMessage(<-nsqd.idChan, []byte("test body"))
topic.PutMessage(msg)
_, err = nsq.Ready(1).WriteTo(conn)
equal(t, err, nil)
resp, err := nsq.ReadResponse(conn)
equal(t, err, nil)
frameType, data, err := nsq.UnpackResponse(resp)
msgOut, _ := decodeMessage(data)
equal(t, frameType, frameTypeMessage)
equal(t, msgOut.ID, msg.ID)
time.Sleep(75 * time.Millisecond)
_, err = nsq.Touch(nsq.MessageID(msg.ID)).WriteTo(conn)
equal(t, err, nil)
time.Sleep(75 * time.Millisecond)
_, err = nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(conn)
equal(t, err, nil)
equal(t, channel.timeoutCount, uint64(0))
}
func TestMaxRdyCount(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.MaxRdyCount = 50
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_max_rdy_count" + strconv.Itoa(int(time.Now().Unix()))
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
topic := nsqd.GetTopic(topicName)
msg := NewMessage(<-nsqd.idChan, []byte("test body"))
topic.PutMessage(msg)
data := identify(t, conn, nil, frameTypeResponse)
r := struct {
MaxRdyCount int64 `json:"max_rdy_count"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.MaxRdyCount, int64(50))
sub(t, conn, topicName, "ch")
_, err = nsq.Ready(int(opts.MaxRdyCount)).WriteTo(conn)
equal(t, err, nil)
resp, err := nsq.ReadResponse(conn)
equal(t, err, nil)
frameType, data, err := nsq.UnpackResponse(resp)
msgOut, _ := decodeMessage(data)
equal(t, frameType, frameTypeMessage)
equal(t, msgOut.ID, msg.ID)
_, err = nsq.Ready(int(opts.MaxRdyCount) + 1).WriteTo(conn)
equal(t, err, nil)
resp, err = nsq.ReadResponse(conn)
equal(t, err, nil)
frameType, data, err = nsq.UnpackResponse(resp)
equal(t, frameType, int32(1))
equal(t, string(data), "E_INVALID RDY count 51 out of range 0-50")
}
func TestFatalError(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
_, err = conn.Write([]byte("ASDF\n"))
equal(t, err, nil)
resp, err := nsq.ReadResponse(conn)
equal(t, err, nil)
frameType, data, err := nsq.UnpackResponse(resp)
equal(t, frameType, int32(1))
equal(t, string(data), "E_INVALID invalid command ASDF")
_, err = nsq.ReadResponse(conn)
nequal(t, err, nil)
}
func TestOutputBuffering(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.MaxOutputBufferSize = 512 * 1024
opts.MaxOutputBufferTimeout = time.Second
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_output_buffering" + strconv.Itoa(int(time.Now().Unix()))
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
outputBufferSize := 256 * 1024
outputBufferTimeout := 500
topic := nsqd.GetTopic(topicName)
msg := NewMessage(<-nsqd.idChan, make([]byte, outputBufferSize-1024))
topic.PutMessage(msg)
start := time.Now()
data := identify(t, conn, map[string]interface{}{
"output_buffer_size": outputBufferSize,
"output_buffer_timeout": outputBufferTimeout,
}, frameTypeResponse)
var decoded map[string]interface{}
json.Unmarshal(data, &decoded)
v, ok := decoded["output_buffer_size"]
equal(t, ok, true)
equal(t, int(v.(float64)), outputBufferSize)
v, ok = decoded["output_buffer_timeout"]
equal(t, int(v.(float64)), outputBufferTimeout)
sub(t, conn, topicName, "ch")
_, err = nsq.Ready(10).WriteTo(conn)
equal(t, err, nil)
resp, err := nsq.ReadResponse(conn)
equal(t, err, nil)
end := time.Now()
equal(t, int(end.Sub(start)/time.Millisecond) >= outputBufferTimeout, true)
frameType, data, err := nsq.UnpackResponse(resp)
msgOut, _ := decodeMessage(data)
equal(t, frameType, frameTypeMessage)
equal(t, msgOut.ID, msg.ID)
}
func TestOutputBufferingValidity(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.MaxOutputBufferSize = 512 * 1024
opts.MaxOutputBufferTimeout = time.Second
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
identify(t, conn, map[string]interface{}{
"output_buffer_size": 512 * 1024,
"output_buffer_timeout": 1000,
}, frameTypeResponse)
identify(t, conn, map[string]interface{}{
"output_buffer_size": -1,
"output_buffer_timeout": -1,
}, frameTypeResponse)
identify(t, conn, map[string]interface{}{
"output_buffer_size": 0,
"output_buffer_timeout": 0,
}, frameTypeResponse)
data := identify(t, conn, map[string]interface{}{
"output_buffer_size": 512*1024 + 1,
"output_buffer_timeout": 0,
}, frameTypeError)
equal(t, string(data), fmt.Sprintf("E_BAD_BODY IDENTIFY output buffer size (%d) is invalid", 512*1024+1))
conn, err = mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data = identify(t, conn, map[string]interface{}{
"output_buffer_size": 0,
"output_buffer_timeout": 1001,
}, frameTypeError)
equal(t, string(data), "E_BAD_BODY IDENTIFY output buffer timeout (1001) is invalid")
}
func TestTLS(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.TLSCert = "./test/certs/server.pem"
opts.TLSKey = "./test/certs/server.key"
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"tls_v1": true,
}, frameTypeResponse)
r := struct {
TLSv1 bool `json:"tls_v1"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.TLSv1, true)
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
}
tlsConn := tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
equal(t, err, nil)
resp, _ := nsq.ReadResponse(tlsConn)
frameType, data, _ := nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
}
func TestTLSRequired(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.TLSCert = "./test/certs/server.pem"
opts.TLSKey = "./test/certs/server.key"
opts.TLSRequired = TLSRequiredExceptHTTP
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_tls_required" + strconv.Itoa(int(time.Now().Unix()))
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
subFail(t, conn, topicName, "ch")
conn, err = mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"tls_v1": true,
}, frameTypeResponse)
r := struct {
TLSv1 bool `json:"tls_v1"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.TLSv1, true)
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
}
tlsConn := tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
equal(t, err, nil)
resp, _ := nsq.ReadResponse(tlsConn)
frameType, data, _ := nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
}
func TestTLSAuthRequire(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.TLSCert = "./test/certs/server.pem"
opts.TLSKey = "./test/certs/server.key"
opts.TLSClientAuthPolicy = "require"
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
// No Certs
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"tls_v1": true,
}, frameTypeResponse)
r := struct {
TLSv1 bool `json:"tls_v1"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.TLSv1, true)
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
}
tlsConn := tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
nequal(t, err, nil)
// With Unsigned Cert
conn, err = mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data = identify(t, conn, map[string]interface{}{
"tls_v1": true,
}, frameTypeResponse)
r = struct {
TLSv1 bool `json:"tls_v1"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.TLSv1, true)
cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem")
equal(t, err, nil)
tlsConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
InsecureSkipVerify: true,
}
tlsConn = tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
equal(t, err, nil)
resp, _ := nsq.ReadResponse(tlsConn)
frameType, data, _ := nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
}
func TestTLSAuthRequireVerify(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.TLSCert = "./test/certs/server.pem"
opts.TLSKey = "./test/certs/server.key"
opts.TLSRootCAFile = "./test/certs/ca.pem"
opts.TLSClientAuthPolicy = "require-verify"
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
// with no cert
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"tls_v1": true,
}, frameTypeResponse)
r := struct {
TLSv1 bool `json:"tls_v1"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.TLSv1, true)
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
}
tlsConn := tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
nequal(t, err, nil)
// with invalid cert
conn, err = mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data = identify(t, conn, map[string]interface{}{
"tls_v1": true,
}, frameTypeResponse)
r = struct {
TLSv1 bool `json:"tls_v1"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.TLSv1, true)
cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem")
equal(t, err, nil)
tlsConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
InsecureSkipVerify: true,
}
tlsConn = tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
nequal(t, err, nil)
// with valid cert
conn, err = mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data = identify(t, conn, map[string]interface{}{
"tls_v1": true,
}, frameTypeResponse)
r = struct {
TLSv1 bool `json:"tls_v1"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.TLSv1, true)
cert, err = tls.LoadX509KeyPair("./test/certs/client.pem", "./test/certs/client.key")
equal(t, err, nil)
tlsConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
InsecureSkipVerify: true,
}
tlsConn = tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
equal(t, err, nil)
resp, _ := nsq.ReadResponse(tlsConn)
frameType, data, _ := nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
}
func TestDeflate(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.DeflateEnabled = true
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"deflate": true,
}, frameTypeResponse)
r := struct {
Deflate bool `json:"deflate"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.Deflate, true)
compressConn := flate.NewReader(conn)
resp, _ := nsq.ReadResponse(compressConn)
frameType, data, _ := nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
}
type readWriter struct {
io.Reader
io.Writer
}
func TestSnappy(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.SnappyEnabled = true
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"snappy": true,
}, frameTypeResponse)
r := struct {
Snappy bool `json:"snappy"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.Snappy, true)
compressConn := snappystream.NewReader(conn, snappystream.SkipVerifyChecksum)
resp, _ := nsq.ReadResponse(compressConn)
frameType, data, _ := nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
msgBody := make([]byte, 128000)
w := snappystream.NewWriter(conn)
rw := readWriter{compressConn, w}
topicName := "test_snappy" + strconv.Itoa(int(time.Now().Unix()))
sub(t, rw, topicName, "ch")
_, err = nsq.Ready(1).WriteTo(rw)
equal(t, err, nil)
topic := nsqd.GetTopic(topicName)
msg := NewMessage(<-nsqd.idChan, msgBody)
topic.PutMessage(msg)
resp, _ = nsq.ReadResponse(compressConn)
frameType, data, _ = nsq.UnpackResponse(resp)
msgOut, _ := decodeMessage(data)
equal(t, frameType, frameTypeMessage)
equal(t, msgOut.ID, msg.ID)
equal(t, msgOut.Body, msg.Body)
}
func TestTLSDeflate(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.DeflateEnabled = true
opts.TLSCert = "./test/certs/cert.pem"
opts.TLSKey = "./test/certs/key.pem"
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"tls_v1": true,
"deflate": true,
}, frameTypeResponse)
r := struct {
TLSv1 bool `json:"tls_v1"`
Deflate bool `json:"deflate"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.TLSv1, true)
equal(t, r.Deflate, true)
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
}
tlsConn := tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
equal(t, err, nil)
resp, _ := nsq.ReadResponse(tlsConn)
frameType, data, _ := nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
compressConn := flate.NewReader(tlsConn)
resp, _ = nsq.ReadResponse(compressConn)
frameType, data, _ = nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
}
func TestSampling(t *testing.T) {
rand.Seed(time.Now().UTC().UnixNano())
num := 10000
sampleRate := 42
slack := 5
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.MaxRdyCount = int64(num)
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"sample_rate": int32(sampleRate),
}, frameTypeResponse)
r := struct {
SampleRate int32 `json:"sample_rate"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.SampleRate, int32(sampleRate))
topicName := "test_sampling" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
for i := 0; i < num; i++ {
msg := NewMessage(<-nsqd.idChan, []byte("test body"))
topic.PutMessage(msg)
}
channel := topic.GetChannel("ch")
// let the topic drain into the channel
time.Sleep(50 * time.Millisecond)
sub(t, conn, topicName, "ch")
_, err = nsq.Ready(num).WriteTo(conn)
equal(t, err, nil)
go func() {
for {
_, err := nsq.ReadResponse(conn)
if err != nil {
return
}
}
}()
doneChan := make(chan int)
go func() {
for {
if channel.Depth() == 0 {
close(doneChan)
return
}
time.Sleep(5 * time.Millisecond)
}
}()
<-doneChan
channel.Lock()
numInFlight := len(channel.inFlightMessages)
channel.Unlock()
equal(t, numInFlight <= int(float64(num)*float64(sampleRate+slack)/100.0), true)
equal(t, numInFlight >= int(float64(num)*float64(sampleRate-slack)/100.0), true)
}
func TestTLSSnappy(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.SnappyEnabled = true
opts.TLSCert = "./test/certs/cert.pem"
opts.TLSKey = "./test/certs/key.pem"
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"tls_v1": true,
"snappy": true,
}, frameTypeResponse)
r := struct {
TLSv1 bool `json:"tls_v1"`
Snappy bool `json:"snappy"`
}{}
err = json.Unmarshal(data, &r)
equal(t, err, nil)
equal(t, r.TLSv1, true)
equal(t, r.Snappy, true)
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
}
tlsConn := tls.Client(conn, tlsConfig)
err = tlsConn.Handshake()
equal(t, err, nil)
resp, _ := nsq.ReadResponse(tlsConn)
frameType, data, _ := nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
compressConn := snappystream.NewReader(tlsConn, snappystream.SkipVerifyChecksum)
resp, _ = nsq.ReadResponse(compressConn)
frameType, data, _ = nsq.UnpackResponse(resp)
t.Logf("frameType: %d, data: %s", frameType, data)
equal(t, frameType, frameTypeResponse)
equal(t, data, []byte("OK"))
}
func TestClientMsgTimeout(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.QueueScanRefreshInterval = 100 * time.Millisecond
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_cmsg_timeout" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
msg := NewMessage(<-nsqd.idChan, make([]byte, 100))
topic.PutMessage(msg)
// without this the race detector thinks there's a write
// to msg.Attempts that races with the read in the protocol's messagePump...
// it does not reflect a realistically possible condition
topic.PutMessage(NewMessage(<-nsqd.idChan, make([]byte, 100)))
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
identify(t, conn, map[string]interface{}{
"msg_timeout": 1000,
}, frameTypeResponse)
sub(t, conn, topicName, "ch")
_, err = nsq.Ready(1).WriteTo(conn)
equal(t, err, nil)
resp, _ := nsq.ReadResponse(conn)
_, data, _ := nsq.UnpackResponse(resp)
msgOut, err := decodeMessage(data)
equal(t, msgOut.ID, msg.ID)
equal(t, msgOut.Body, msg.Body)
_, err = nsq.Ready(0).WriteTo(conn)
equal(t, err, nil)
time.Sleep(1100 * time.Millisecond)
_, err = nsq.Finish(nsq.MessageID(msgOut.ID)).WriteTo(conn)
equal(t, err, nil)
resp, _ = nsq.ReadResponse(conn)
frameType, data, _ := nsq.UnpackResponse(resp)
equal(t, frameType, frameTypeError)
equal(t, string(data),
fmt.Sprintf("E_FIN_FAILED FIN %s failed ID not in flight", msgOut.ID))
}
func TestBadFin(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
identify(t, conn, map[string]interface{}{}, frameTypeResponse)
sub(t, conn, "test_fin", "ch")
fin := nsq.Finish(nsq.MessageID{})
fin.Params[0] = []byte("")
_, err = fin.WriteTo(conn)
equal(t, err, nil)
resp, _ := nsq.ReadResponse(conn)
frameType, data, _ := nsq.UnpackResponse(resp)
equal(t, frameType, frameTypeError)
equal(t, string(data), "E_INVALID Invalid Message ID")
}
func TestClientAuth(t *testing.T) {
authResponse := `{"ttl":1, "authorizations":[]}`
authSecret := "testsecret"
authError := "E_UNAUTHORIZED AUTH No authorizations found"
authSuccess := ""
runAuthTest(t, authResponse, authSecret, authError, authSuccess)
// now one that will succeed
authResponse = `{"ttl":10, "authorizations":
[{"topic":"test", "channels":[".*"], "permissions":["subscribe","publish"]}]
}`
authError = ""
authSuccess = `{"identity":"","identity_url":"","permission_count":1}`
runAuthTest(t, authResponse, authSecret, authError, authSuccess)
}
func runAuthTest(t *testing.T, authResponse, authSecret, authError, authSuccess string) {
var err error
var expectedAuthIP string
expectedAuthTLS := "false"
authd := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Logf("in test auth handler %s", r.RequestURI)
r.ParseForm()
equal(t, r.Form.Get("remote_ip"), expectedAuthIP)
equal(t, r.Form.Get("tls"), expectedAuthTLS)
equal(t, r.Form.Get("secret"), authSecret)
fmt.Fprint(w, authResponse)
}))
defer authd.Close()
addr, err := url.Parse(authd.URL)
equal(t, err, nil)
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
opts.AuthHTTPAddresses = []string{addr.Host}
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
equal(t, err, nil)
defer conn.Close()
expectedAuthIP, _, _ = net.SplitHostPort(conn.LocalAddr().String())
identify(t, conn, map[string]interface{}{
"tls_v1": false,
}, nsq.FrameTypeResponse)
authCmd(t, conn, authSecret, authSuccess)
if authError != "" {
readValidate(t, conn, nsq.FrameTypeError, authError)
} else {
sub(t, conn, "test", "ch")
}
}
func TestIOLoopReturnsClientErrWhenSendFails(t *testing.T) {
fakeConn := test.NewFakeNetConn()
fakeConn.WriteFunc = func(b []byte) (int, error) {
return 0, errors.New("write error")
}
testIOLoopReturnsClientErr(t, fakeConn)
}
func TestIOLoopReturnsClientErrWhenSendSucceeds(t *testing.T) {
fakeConn := test.NewFakeNetConn()
fakeConn.WriteFunc = func(b []byte) (int, error) {
return len(b), nil
}
testIOLoopReturnsClientErr(t, fakeConn)
}
func testIOLoopReturnsClientErr(t *testing.T, fakeConn test.FakeNetConn) {
fakeConn.ReadFunc = func(b []byte) (int, error) {
return copy(b, []byte("INVALID_COMMAND\n")), nil
}
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.Verbose = true
prot := &protocolV2{ctx: &context{nsqd: New(opts)}}
defer prot.ctx.nsqd.Exit()
err := prot.IOLoop(fakeConn)
nequal(t, err, nil)
equal(t, err.Error(), "E_INVALID invalid command INVALID_COMMAND")
nequal(t, err.(*protocol.FatalClientErr), nil)
}
func BenchmarkProtocolV2Exec(b *testing.B) {
b.StopTimer()
opts := NewOptions()
opts.Logger = newTestLogger(b)
nsqd := New(opts)
ctx := &context{nsqd}
p := &protocolV2{ctx}
c := newClientV2(0, nil, ctx)
params := [][]byte{[]byte("NOP")}
b.StartTimer()
for i := 0; i < b.N; i++ {
p.Exec(c, params)
}
}
func benchmarkProtocolV2Pub(b *testing.B, size int) {
var wg sync.WaitGroup
b.StopTimer()
opts := NewOptions()
batchSize := int(opts.MaxBodySize) / (size + 4)
opts.Logger = newTestLogger(b)
opts.MemQueueSize = int64(b.N)
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
msg := make([]byte, size)
batch := make([][]byte, batchSize)
for i := range batch {
batch[i] = msg
}
topicName := "bench_v2_pub" + strconv.Itoa(int(time.Now().Unix()))
b.SetBytes(int64(len(msg)))
b.StartTimer()
for j := 0; j < runtime.GOMAXPROCS(0); j++ {
wg.Add(1)
go func() {
conn, err := mustConnectNSQD(tcpAddr)
if err != nil {
panic(err.Error())
}
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
num := b.N / runtime.GOMAXPROCS(0) / batchSize
for i := 0; i < num; i++ {
cmd, _ := nsq.MultiPublish(topicName, batch)
_, err := cmd.WriteTo(rw)
if err != nil {
panic(err.Error())
}
err = rw.Flush()
if err != nil {
panic(err.Error())
}
resp, err := nsq.ReadResponse(rw)
if err != nil {
panic(err.Error())
}
_, data, _ := nsq.UnpackResponse(resp)
if !bytes.Equal(data, []byte("OK")) {
panic("invalid response")
}
}
wg.Done()
}()
}
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func BenchmarkProtocolV2Pub256(b *testing.B) { benchmarkProtocolV2Pub(b, 256) }
func BenchmarkProtocolV2Pub512(b *testing.B) { benchmarkProtocolV2Pub(b, 512) }
func BenchmarkProtocolV2Pub1k(b *testing.B) { benchmarkProtocolV2Pub(b, 1024) }
func BenchmarkProtocolV2Pub2k(b *testing.B) { benchmarkProtocolV2Pub(b, 2*1024) }
func BenchmarkProtocolV2Pub4k(b *testing.B) { benchmarkProtocolV2Pub(b, 4*1024) }
func BenchmarkProtocolV2Pub8k(b *testing.B) { benchmarkProtocolV2Pub(b, 8*1024) }
func BenchmarkProtocolV2Pub16k(b *testing.B) { benchmarkProtocolV2Pub(b, 16*1024) }
func BenchmarkProtocolV2Pub32k(b *testing.B) { benchmarkProtocolV2Pub(b, 32*1024) }
func BenchmarkProtocolV2Pub64k(b *testing.B) { benchmarkProtocolV2Pub(b, 64*1024) }
func BenchmarkProtocolV2Pub128k(b *testing.B) { benchmarkProtocolV2Pub(b, 128*1024) }
func BenchmarkProtocolV2Pub256k(b *testing.B) { benchmarkProtocolV2Pub(b, 256*1024) }
func BenchmarkProtocolV2Pub512k(b *testing.B) { benchmarkProtocolV2Pub(b, 512*1024) }
func BenchmarkProtocolV2Pub1m(b *testing.B) { benchmarkProtocolV2Pub(b, 1024*1024) }
func benchmarkProtocolV2Sub(b *testing.B, size int) {
var wg sync.WaitGroup
b.StopTimer()
opts := NewOptions()
opts.Logger = newTestLogger(b)
opts.MemQueueSize = int64(b.N)
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
msg := make([]byte, size)
topicName := "bench_v2_sub" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
for i := 0; i < b.N; i++ {
msg := NewMessage(<-nsqd.idChan, msg)
topic.PutMessage(msg)
}
topic.GetChannel("ch")
b.SetBytes(int64(len(msg)))
goChan := make(chan int)
rdyChan := make(chan int)
workers := runtime.GOMAXPROCS(0)
for j := 0; j < workers; j++ {
wg.Add(1)
go func() {
subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
wg.Done()
}()
<-rdyChan
}
b.StartTimer()
close(goChan)
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func subWorker(n int, workers int, tcpAddr *net.TCPAddr, topicName string, rdyChan chan int, goChan chan int) {
conn, err := mustConnectNSQD(tcpAddr)
if err != nil {
panic(err.Error())
}
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriterSize(conn, 65536))
identify(nil, conn, nil, frameTypeResponse)
sub(nil, conn, topicName, "ch")
rdyCount := int(math.Min(math.Max(float64(n/workers), 1), 2500))
rdyChan <- 1
<-goChan
nsq.Ready(rdyCount).WriteTo(rw)
rw.Flush()
num := n / workers
for i := 0; i < num; i++ {
resp, err := nsq.ReadResponse(rw)
if err != nil {
panic(err.Error())
}
frameType, data, err := nsq.UnpackResponse(resp)
if err != nil {
panic(err.Error())
}
if frameType != frameTypeMessage {
panic("got something else")
}
msg, err := decodeMessage(data)
if err != nil {
panic(err.Error())
}
nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(rw)
if (i+1)%rdyCount == 0 || i+1 == num {
if i+1 == num {
nsq.Ready(0).WriteTo(conn)
}
rw.Flush()
}
}
}
func BenchmarkProtocolV2Sub256(b *testing.B) { benchmarkProtocolV2Sub(b, 256) }
func BenchmarkProtocolV2Sub512(b *testing.B) { benchmarkProtocolV2Sub(b, 512) }
func BenchmarkProtocolV2Sub1k(b *testing.B) { benchmarkProtocolV2Sub(b, 1024) }
func BenchmarkProtocolV2Sub2k(b *testing.B) { benchmarkProtocolV2Sub(b, 2*1024) }
func BenchmarkProtocolV2Sub4k(b *testing.B) { benchmarkProtocolV2Sub(b, 4*1024) }
func BenchmarkProtocolV2Sub8k(b *testing.B) { benchmarkProtocolV2Sub(b, 8*1024) }
func BenchmarkProtocolV2Sub16k(b *testing.B) { benchmarkProtocolV2Sub(b, 16*1024) }
func BenchmarkProtocolV2Sub32k(b *testing.B) { benchmarkProtocolV2Sub(b, 32*1024) }
func BenchmarkProtocolV2Sub64k(b *testing.B) { benchmarkProtocolV2Sub(b, 64*1024) }
func BenchmarkProtocolV2Sub128k(b *testing.B) { benchmarkProtocolV2Sub(b, 128*1024) }
func BenchmarkProtocolV2Sub256k(b *testing.B) { benchmarkProtocolV2Sub(b, 256*1024) }
func BenchmarkProtocolV2Sub512k(b *testing.B) { benchmarkProtocolV2Sub(b, 512*1024) }
func BenchmarkProtocolV2Sub1m(b *testing.B) { benchmarkProtocolV2Sub(b, 1024*1024) }
func benchmarkProtocolV2MultiSub(b *testing.B, num int) {
var wg sync.WaitGroup
b.StopTimer()
opts := NewOptions()
opts.Logger = newTestLogger(b)
opts.MemQueueSize = int64(b.N)
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
msg := make([]byte, 256)
b.SetBytes(int64(len(msg) * num))
goChan := make(chan int)
rdyChan := make(chan int)
workers := runtime.GOMAXPROCS(0)
for i := 0; i < num; i++ {
topicName := "bench_v2" + strconv.Itoa(b.N) + "_" + strconv.Itoa(i) + "_" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
for i := 0; i < b.N; i++ {
msg := NewMessage(<-nsqd.idChan, msg)
topic.PutMessage(msg)
}
topic.GetChannel("ch")
for j := 0; j < workers; j++ {
wg.Add(1)
go func() {
subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
wg.Done()
}()
<-rdyChan
}
}
b.StartTimer()
close(goChan)
wg.Wait()
b.StopTimer()
nsqd.Exit()
}
func BenchmarkProtocolV2MultiSub1(b *testing.B) { benchmarkProtocolV2MultiSub(b, 1) }
func BenchmarkProtocolV2MultiSub2(b *testing.B) { benchmarkProtocolV2MultiSub(b, 2) }
func BenchmarkProtocolV2MultiSub4(b *testing.B) { benchmarkProtocolV2MultiSub(b, 4) }
func BenchmarkProtocolV2MultiSub8(b *testing.B) { benchmarkProtocolV2MultiSub(b, 8) }
func BenchmarkProtocolV2MultiSub16(b *testing.B) { benchmarkProtocolV2MultiSub(b, 16) }
func TestProcessKilled(t *testing.T) {
testDefaultStartup(t)
doneCh := make(chan struct{})
shutdown := time.NewTimer(time.Second * 4)
timeout := time.NewTimer(time.Second * 10)
terminatedHandler := func(reason string) {
t.Logf("reason: %s\n", reason)
doneCh <- struct{}{}
}
debugger.SetTerminationHandler(terminatedHandler)
for {
select {
case <-doneCh:
goto DONE
case <-shutdown.C:
debugger.ExitProcess()
case <-timeout.C:
t.Fatalf("timed out waiting for termination")
}
}
DONE:
}
func TestTargetCrashed(t *testing.T) {
testDefaultStartup(t)
defer debugger.ExitProcess()
doneCh := make(chan struct{})
timeout := time.NewTimer(time.Second * 10)
targetCrashedFn := func(targ *ChromeTarget, payload []byte) {
t.Logf("reason: %s\n", string(payload))
doneCh <- struct{}{}
}
tab, err := debugger.NewTab()
if err != nil {
t.Fatalf("error creating new tab")
}
tab.Subscribe("Inspector.targetCrashed", targetCrashedFn)
go func() {
<-timeout.C
t.Fatalf("timed out waiting for crashed to be handled")
}()
_, err = tab.Page.Navigate("chrome://crash")
if err == nil {
t.Fatalf("Navigation should have failed")
}
<-doneCh
}
func TestEvents(t *testing.T) {
testDefaultStartup(t)
defer debugger.ExitProcess()
target, err := debugger.NewTab()
if err != nil {
t.Fatalf("error getting new tab: %s\n", err)
}
console := target.Console
doneCh := make(chan struct{}, 1)
target.Subscribe("Console.messageAdded", func(target *ChromeTarget, v []byte) {
target.Unsubscribe("Console.messageAdded")
msg := &gcdapi.ConsoleMessageAddedEvent{}
err := json.Unmarshal(v, msg)
if err != nil {
t.Fatalf("error unmarshalling event data: %v\n", err)
}
t.Logf("METHOD: %s\n", msg.Method)
eventData := msg.Params.Message
t.Logf("Got event: %v\n", eventData)
t.Logf("Timestamp: %f\n", eventData.Timestamp)
doneCh <- struct{}{}
})
_, err = console.Enable()
if err != nil {
t.Fatalf("error sending enable: %s\n", err)
}
if _, err := target.Page.Navigate(testServerAddr + "console_log.html"); err != nil {
t.Fatalf("error attempting to navigate: %s\n", err)
}
go testTimeoutListener(t, 5, "console message")
<-doneCh
}
func TestSimpleReturn(t *testing.T) {
var ret bool
testDefaultStartup(t)
defer debugger.ExitProcess()
target, err := debugger.NewTab()
if err != nil {
t.Fatalf("error getting new tab: %s\n", err)
}
network := target.Network
if _, err := network.Enable(); err != nil {
t.Fatalf("error enabling network")
}
ret, err = network.CanClearBrowserCache()
if err != nil {
t.Fatalf("error getting response to clearing browser cache: %s\n", err)
}
if !ret {
t.Fatalf("we should have got true for can clear browser cache\n")
}
}
// tests getting a complex object back from inside a fired event that we subscribed to.
func TestComplexReturn(t *testing.T) {
testDefaultStartup(t)
defer debugger.ExitProcess()
wg := &sync.WaitGroup{}
wg.Add(1)
target, err := debugger.NewTab()
if err != nil {
t.Fatalf("error getting new tab: %s\n", err)
}
if _, err := target.Network.Enable(); err != nil {
t.Fatalf("error enabling network %s\n", err)
}
if _, err := target.Page.Enable(); err != nil {
t.Fatalf("error enabling page: %s\n", err)
}
target.Subscribe("Page.loadEventFired", func(target *ChromeTarget, payload []byte) {
var ok bool
t.Logf("page load event fired\n")
cookies, err := target.Network.GetCookies()
if err != nil {
t.Fatalf("error getting cookies!")
}
for _, v := range cookies {
t.Logf("got cookies: %#v\n", v)
if v.Name == "HEYA" {
ok = true
break
}
}
if !ok {
t.Fatalf("error finding our cookie value!")
}
wg.Done()
})
_, err = target.Page.Navigate(testServerAddr + "cookie.html")
if err != nil {
t.Fatalf("error navigating to cookie page: %s\n", err)
}
go testTimeoutListener(t, 7, "waiting for page load to get cookies")
t.Logf("waiting for loadEventFired")
wg.Wait()
}
// UTILITY FUNCTIONS
func testDefaultStartup(t *testing.T) {
debugger = NewChromeDebugger()
debugger.StartProcess(testPath, testRandomTempDir(t), testRandomPort(t))
}
func testServer() {
testListener, _ = net.Listen("tcp", ":0")
_, testServerPort, _ := net.SplitHostPort(testListener.Addr().String())
testServerAddr = fmt.Sprintf("http://localhost:%s/", testServerPort)
go http.Serve(testListener, http.FileServer(http.Dir("testdata/")))
}
func testTimeoutListener(t *testing.T, seconds time.Duration, message string) {
timeout := time.NewTimer(seconds * time.Second)
for {
select {
case <-timeout.C:
t.Fatalf("timed out waiting for %s", message)
}
}
}
func testRandomPort(t *testing.T) string {
l, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatal(err)
}
_, randPort, _ := net.SplitHostPort(l.Addr().String())
l.Close()
return randPort
}
func testRandomTempDir(t *testing.T) string {
dir, err := ioutil.TempDir(testDir, "gcd")
if err != nil {
t.Errorf("error creating temp dir: %s\n", err)
}
return dir
}