Пример #1
0
func TestName(t *testing.T) {
	debug = testing.Verbose()
	d := zx.Dir{
		"path":  "/a/b",
		"name":  "b",
		"type":  "d",
		"mode":  "0755",
		"spath": "/a/b",
		"tpath": "/tmp/lfs_test",
		"proto": "lfs",
	}
	preds := []string{
		`name=b`,
		`! name=b`,
		`name!=b`,
	}
	matches := []bool{true, false, false}
	prunes := []bool{false, true, true}
	for i, pr := range preds {
		p, err := New(pr)
		if err != nil {
			t.Fatalf("parse %s", err)
		}
		t.Logf("eval %s\n", p.DebugString())
		m, prune, err := p.EvalAt(d, 0)
		t.Logf("match %v prune %v sts %v", m, prune, err)
		if err != nil || m != matches[i] || prune != prunes[i] {
			t.Logf("wrong result %v %v %v", err, m, prune)
			t.Fail()
		}
	}
}
Пример #2
0
func TestUnbackslash(t *testing.T) {
	for _, test := range []struct {
		isString bool
		in       string
		out      string
	}{
		{false, "", ""},
		{true, "", ""},
		// from CSS2 4.1.3 examples:
		{true, "\\26 B", "&B"},
		{true, "\\000026B", "&B"},
		{true, "\\26G", "&G"},
		{true, "\\2aG", "*G"},
		{true, "\\2AG", "*G"},
		{true, "\\2fG", "/G"},
		{true, "\\2FG", "/G"},
		// standard does not appear to require an even number of digits
		{true, "\\026 B", "&B"},
		{true, "\\026  B", "& B"},
		{true, "\\026", "&"},
		{true, "\\", "\\"},
		{true, "\\{", "{"},

		// Check the special string handling
		{true, "a\\\nb", "ab"},
		{true, "a\\\r\nb", "ab"},
	} {
		result := unbackslash(test.in, test.isString)
		if result != test.out {
			t.Fatalf("Error in TestUnbackslash. In: %q\nOut: %q\nExpected: %q",
				test.in, result, test.out)
		}
	}
}
Пример #3
0
func TestSimple(t *testing.T) {
	value := 1
	expected := 2
	if value != expected {
		t.Fatalf("Expected %v, but %d:", value, expected)
	}
}
Пример #4
0
func TestLoadFile(t *testing.T){
	fp := "path-to-file"

	h := hive.HiveConfig(...)
	e := hdc.LoadFile(h, fp, "table", true)
	fatalCheck(t, "LoadFile", e)

	q := "select count(*) recordCount from table"
	var ms []toolkit.M
	e = h.Populate(q, &ms)
	fatalCheck(t, "Populate", e)

	if ms[0].GetInt("recordCount", 0)==0 {
		t.Fatalf("No data imported")
	}
}
Пример #5
0
func TestConnectWithOptions(t *testing.T) {
	responseText := "Hello World"
	itemID := "theID"
	connectStorage := &ConnecterRESTStorage{
		connectHandler: &SimpleConnectHandler{
			response: responseText,
		},
		emptyConnectOptions: &SimpleGetOptions{},
	}
	storage := map[string]rest.Storage{
		"simple":         &SimpleRESTStorage{},
		"simple/connect": connectStorage,
	}
	handler := handle(storage)
	server := httptest.NewServer(handler)
	defer server.Close()

	resp, err := http.Get(server.URL + "/api/version/namespaces/default/simple/" + itemID + "/connect?param1=value1&param2=value2")

	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if resp.StatusCode != http.StatusOK {
		t.Errorf("unexpected response: %#v", resp)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	if connectStorage.receivedID != itemID {
		t.Errorf("Unexpected item id. Expected: %s. Actual: %s.", itemID, connectStorage.receivedID)
	}
	if string(body) != responseText {
		t.Errorf("Unexpected response. Expected: %s. Actual: %s.", responseText, string(body))
	}
	opts, ok := connectStorage.receivedConnectOptions.(*SimpleGetOptions)
	if !ok {
		t.Errorf("Unexpected options type: %#v", connectStorage.receivedConnectOptions)
	}
	if opts.Param1 != "value1" && opts.Param2 != "value2" {
		t.Errorf("Unexpected options value: %#v", opts
Пример #6
0
func TestCliveOutChan(t *testing.T) {
	debug = testing.Verbose()
	out2 := make(chan face{})
	adj := func(c *cmd.Ctx) {
		c.ForkEnv()
		c.ForkNS()
		c.ForkDot()
		c.SetOut("out2", out2)
	}
	c, err := CtxCmd(adj, "eco", "-o", "out2", "-m", "a", "b", "c")
	if err != nil {
		t.Fatalf("sts %v", err)
	}
	out := []string{}
	for x := range ch.Merge(c.Out, c.Err, out2) {
		switch x := x.(type) {
		case []byte:
			printf("-> [%s]\n", x)
			out = append(out, string(x))
		default:
			t.Fatalf("got type %T", x)
		}
	}
	err = c.Wait()
	printf("sts %v\n", err)
	if err != nil {
		t.Fatalf("did fail")
	}
}
Пример #7
0
func TestLf(t *testing.T) {
	app.New()	// prevent app.Fatal from calling dbg.Fatal
	app.Debug = testing.Verbose()
	for i := range tests {
		lt := tests[i]
		dprintf("run %v\n", lt.args)
		out := make(chan interface{})
		go func() {
			c := app.New()
			c.Args = lt.args
			defer app.Exiting()
			app.DupIO()
			app.SetIO(out, 1)
			app.Cd("/zx/sys/src/clive")
			Run()
		}()
		outs := []string{}
		for x := range out {
			d, ok := x.(zx.Dir)
			if !ok {
				dprintf("got %T %v\n", x, x)
				t.Fatalf("not a dir")
			}
			dprintf("got %T %s\n", d, d["upath"])
			outs = append(outs, d["upath"])
		}
		err := cerror(out)
		dprintf("got sts %v\n", err)
		if lt.fails && err == nil {
			t.Fatalf("didn't fail")
		}
		if !lt.fails && err != nil {
			t.Fatalf("failed: %s", err)
		}
		if lt.out != nil && strings.Join(lt.out, " ") != strings.Join(outs, " ") {
			t.Fatalf("bad output %#v", outs)
		}
		if lt.out == nil {
			dprintf("out: %#v\n", outs)
		}
	}
}
Пример #8
0
func TestCliveIn2Chan(t *testing.T) {
	debug = testing.Verbose()
	in := make(chan face{}, 3)
	adj := func(c *cmd.Ctx) {
		c.ForkEnv()
		c.ForkNS()
		c.ForkDot()
		c.SetIn("in2", in)
	}
	c, err := CtxCmd(adj, "eco", "-i", "in2", "-m", "a", "b", "c")
	if err != nil {
		t.Fatalf("sts %v", err)
	}
	for _, m := range []string{"x", "y", "z"} {
		in <- []byte(m)
	}
	close(in)
	out := []string{}
	for x := range ch.Merge(c.Out, c.Err) {
		switch x := x.(type) {
		case []byte:
			printf("-> [%s]\n", x)
			out = append(out, string(x))
		default:
			t.Fatalf("got type %T", x)
		}
	}
	err = c.Wait()
	printf("sts %v\n", err)
	if err != nil {
		t.Fatalf("did fail")
	}
	if strings.Join(out, "|") != "a|b|c|x|y|z" {
		t.Fatalf("bad output")
	}
}
Пример #9
0
func TestOrphaning(t *testing.T) {
	gc, clientSet := setup(t)
	oldEnableGarbageCollector := registry.EnableGarbageCollector
	registry.EnableGarbageCollector = true
	defer func() { registry.EnableGarbageCollector = oldEnableGarbageCollector }()
	podClient := clientSet.Core().Pods(framework.TestNS)
	rcClient := clientSet.Core().ReplicationControllers(framework.TestNS)
	// create the RC with the orphan finalizer set
	toBeDeletedRC := newOwnerRC(toBeDeletedRCName)
	toBeDeletedRC, err := rcClient.Create(toBeDeletedRC)
	if err != nil {
		t.Fatalf("Failed to create replication controller: %v", err)
	}

	// these pods should be ophaned.
	var podUIDs []types.UID
	podsNum := 3
	for i := 0; i < podsNum; i++ {
		podName := garbageCollectedPodName + strconv.Itoa(i)
		pod := newPod(podName, []v1.OwnerReference{{UID: toBeDeletedRC.ObjectMeta.UID, Name: toBeDeletedRCName}})
		_, err = podClient.Create(pod)
		if err != nil {
			t.Fatalf("Failed to create Pod: %v", err)
		}
		podUIDs = append(podUIDs, pod.ObjectMeta.UID)
	}
	stopCh := make(chan struct{})
	go gc.Run(5, stopCh)
	defer close(stopCh)

	// we need wait for the gc to observe the creation of the pods, otherwise if
	// the deletion of RC is observed before the creation of the pods, the pods
	// will not be orphaned.
	wait.Poll(5*time.Second, 60*time.Second, func() (bool, error) { return gc.GraphHasUID(podUIDs), nil })

	err = rcClient.Delete(toBeDeletedRCName, getOrphanOptions())
	if err != nil {
		t.Fatalf("Failed to gracefully delete the rc: %v", err)
	}

	// wait for the garbage collector to drain its queue
	if err := wait.Poll(10*time.Second, 300*time.Second, func() (bool, error) {
		retu
Пример #10
0
func TestClientTimeout(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 50 * time.Millisecond
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identify(t, conn)
	sub(t, conn, topicName, "ch")

	time.Sleep(50 * time.Millisecond)

	// depending on timing there may be 1 or 2 hearbeats sent
	// just read until we get an error
	timer := time.After(100 * time.Millisecond)
	for {
		select {
		case <-timer:
			t.Fatalf("test timed out")
		default:
			_, err := nsq.ReadResponse(conn)
			if err != nil {
				goto done
			}
		}
	}
done:
}

func TestClientHeartbeat(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 100 * time.Millisecond
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identify(t, conn)
	sub(t, conn, topicName, "ch")

	err = nsq.Ready(1).Write(conn)
	assert.Equal(t, err, nil)

	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	assert.Equal(t, data, []byte("_heartbeat_"))

	time.Sleep(10 * time.Millisecond)

	err = nsq.Nop().Write(conn)
	assert.Equal(t, err, nil)

	// wait long enough that would have timed out (had we not sent the above cmd)
	time.Sleep(50 * time.Millisecond)

	err = nsq.Nop().Write(conn)
	assert.Equal(t, err, nil)
}

func TestClientHeartbeatDisableSUB(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 200 * time.Millisecond
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identifyHeartbeatInterval(t, conn, -1, nsq.FrameTypeResponse, "OK")
	subFail(t, conn, topicName, "ch")
}

func TestClientHeartbeatDisable(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	options := NewNsqdOptions()
	options.clientTimeout = 100 * time.Millisecond
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identifyHeartbeatInterval(t, conn, -1, nsq.FrameTypeResponse, "OK")

	time.Sleep(150 * time.Millisecond)

	err = nsq.Nop().Write(conn)
	assert.Equal(t, err, nil)
}

func TestMaxHeartbeatIntervalValid(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	options := NewNsqdOptions()
	options.maxHeartbeatInterval = 300 * time.Second
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	hbi := int(options.maxHeartbeatInterval / time.Millisecond)
	identifyHeartbeatInterval(t, conn, hbi, nsq.FrameTypeResponse, "OK")
}

func TestMaxHeartbeatIntervalInvalid(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	options := NewNsqdOptions()
	options.maxHeartbeatInterval = 300 * time.Second
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	hbi := int(options.maxHeartbeatInterval/time.Millisecond + 1)
	identifyHeartbeatInterval(t, conn, hbi, nsq.FrameTypeError, "E_BAD_BODY IDENTIFY heartbeat interval (300001) is invalid")
}

func TestPausing(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix()))

	tcpAddr, _ := mustStartNSQd(NewNsqdOptions())
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identify(t, conn)
	sub(t, conn, topicName, "ch")

	err = nsq.Ready(1).Write(conn)
	assert.Equal(t, err, nil)

	topic := nsqd.GetTopic(topicName)
	msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
	channel := topic.GetChannel("ch")
	topic.PutMessage(msg)

	// receive the first message via the client, finish it, and send new RDY
	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	msg, err = nsq.DecodeMessage(data)
	assert.Equal(t, msg.Body, []byte("test body"))

	err = nsq.Finish(msg.Id).Write(conn)
	assert.Equal(t, err, nil)

	err = nsq.Ready(1).Write(conn)
	assert.Equal(t, err, nil)

	// sleep to allow the RDY state to take effect
	time.Sleep(50 * time.Millisecond)

	// pause the channel... the client shouldn't receive any more messages
	channel.Pause()

	// sleep to allow the paused state to take effect
	time.Sleep(50 * time.Millisecond)

	msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body2"))
	topic.PutMessage(msg)

	// allow the client to possibly get a message, the test would hang indefinitely
	// if pausing was not working on the internal clientMsgChan read
	time.Sleep(50 * time.Millisecond)
	msg = <-channel.clientMsgChan
	assert.Equal(t, msg.Body, []byte("test body2"))

	// unpause the channel... the client should now be pushed a message
	channel.UnPause()

	msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body3"))
	topic.PutMessage(msg)

	resp, _ = nsq.ReadResponse(conn)
	_, data, _ = nsq.UnpackResponse(resp)
	msg, err = nsq.DecodeMessage(data)
	assert.Equal(t, msg.Body, []byte("test body3"))
}

func TestEmptyCommand(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	tcpAddr, _ := mustStartNSQd(NewNsqdOptions())
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	_, err = conn.Write([]byte("\n\n"))
	assert.Equal(t, err, nil)

	// if we didn't panic here we're good, see issue #120
}

func TestSizeLimits(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	options := NewNsqdOptions()
	*verbose = true
	options.maxMessageSize = 100
	options.maxBodySize = 1000
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	topicName := "test_limits_v2" + strconv.Itoa(int(time.Now().Unix()))

	identify(t, conn)
	sub(t, conn, topicName, "ch")

	nsq.Publish(topicName, make([]byte, 95)).Write(conn)
	resp, _ := nsq.ReadResponse(conn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeResponse)
	assert.Equal(t, data, []byte("OK"))

	nsq.Publish(topicName, make([]byte, 105)).Write(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeError)
	assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE PUB message too big 105 > 100"))

	// need to reconnect
	conn, err = mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	// MPUB body that's valid
	mpub := make([][]byte, 0)
	for i := 0; i < 5; i++ {
		mpub = append(mpub, make([]byte, 100))
	}
	cmd, _ := nsq.MultiPublish(topicName, mpub)
	cmd.Write(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeResponse)
	assert.Equal(t, data, []byte("OK"))

	// MPUB body that's invalid (body too big)
	mpub = make([][]byte, 0)
	for i := 0; i < 11; i++ {
		mpub = append(mpub, make([]byte, 100))
	}
	cmd, _ = nsq.MultiPublish(topicName, mpub)
	cmd.Write(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeError)
	assert.Equal(t, string(data), fmt.Sprintf("E_BAD_BODY MPUB body too big 1148 > 1000"))

	// need to reconnect
	conn, err = mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	// MPUB body that's invalid (one of the messages is too big)
	mpub = make([][]byte, 0)
	for i := 0; i < 5; i++ {
		mpub = append(mpub, make([]byte, 101))
	}
	cmd, _ = nsq.MultiPublish(topicName, mpub)
	cmd.Write(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeError)
	assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE MPUB message too big 101 > 100"))
}

func TestTouch(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	*verbose = true
	options := NewNsqdOptions()
	options.msgTimeout = 50 * time.Millisecond
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	topicName := "test_touch" + strconv.Itoa(int(time.Now().Unix()))

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identify(t, conn)
	sub(t, conn, topicName, "ch")

	topic := nsqd.GetTopic(topicName)
	channel := topic.GetChannel("ch")
	msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
	topic.PutMessage(msg)

	err = nsq.Ready(1).Write(conn)
	assert.Equal(t, err, nil)

	resp, err := nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)
	frameType, data, err := nsq.UnpackResponse(resp)
	msgOut, _ := nsq.DecodeMessage(data)
	assert.Equal(t, frameType, nsq.FrameTypeMessage)
	assert.Equal(t, msgOut.Id, msg.Id)

	time.Sleep(25 * time.Millisecond)

	err = nsq.Touch(msg.Id).Write(conn)
	assert.Equal(t, err, nil)

	time.Sleep(30 * time.Millisecond)

	err = nsq.Finish(msg.Id).Write(conn)
	assert.Equal(t, err, nil)

	assert.Equal(t, channel.timeoutCount, uint64(0))
}

func TestMaxRdyCount(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	*verbose = true
	options := NewNsqdOptions()
	options.maxRdyCount = 50
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	topicName := "test_max_rdy_count" + strconv.Itoa(int(time.Now().Unix()))

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	topic := nsqd.GetTopic(topicName)
	msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
	topic.PutMessage(msg)

	data := identifyFeatureNegotiation(t, conn)
	r := struct {
		MaxRdyCount int64 `json:"max_rdy_count"`
	}{}
	err = json.Unmarshal(data, &r)
	assert.Equal(t, err, nil)
	assert.Equal(t, r.MaxRdyCount, int64(50))
	sub(t, conn, topicName, "ch")

	err = nsq.Ready(int(options.maxRdyCount)).Write(conn)
	assert.Equal(t, err, nil)

	resp, err := nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)
	frameType, data, err := nsq.UnpackResponse(resp)
	msgOut, _ := nsq.DecodeMessage(data)
	assert.Equal(t, frameType, nsq.FrameTypeMessage)
	assert.Equal(t, msgOut.Id, msg.Id)

	err = nsq.Ready(int(options.maxRdyCount) + 1).Write(conn)
	assert.Equal(t, err, nil)

	resp, err = nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)
	frameType, data, err = nsq.UnpackResponse(resp)
	assert.Equal(t, frameType, int32(1))
	assert.Equal(t, string(data), "E_INVALID RDY count 51 out of range 0-50")
}

func BenchmarkProtocolV2Exec(b *testing.B) {
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	p := &ProtocolV2{}
	c := NewClientV2(nil)
	params := [][]byte{[]byte("NOP")}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		p.Exec(c, params)
	}
}

func benchmarkProtocolV2Pub(b *testing.B, size int) {
	var wg sync.WaitGroup
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	options := NewNsqdOptions()
	options.memQueueSize = int64(b.N)
	tcpAddr, _ := mustStartNSQd(options)
	msg := make([]byte, size)
	batchSize := 200
	batch := make([][]byte, 0)
	for i := 0; i < batchSize; i++ {
		batch = append(batch, msg)
	}
	topicName := "bench_v2_pub" + strconv.Itoa(int(time.Now().Unix()))
	b.SetBytes(int64(len(msg)))
	b.StartTimer()

	for j := 0; j < runtime.GOMAXPROCS(0); j++ {
		wg.Add(1)
		go func() {
			conn, err := mustConnectNSQd(tcpAddr)
			if err != nil {
				panic(err.Error())
			}
			rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))

			num := b.N / runtime.GOMAXPROCS(0) / batchSize
			for i := 0; i < num; i += 1 {
				cmd, _ := nsq.MultiPublish(topicName, batch)
				err := cmd.Write(rw)
				if err != nil {
					panic(err.Error())
				}
				err = rw.Flush()
				if err != nil {
					panic(err.Error())
				}
				resp, err := nsq.ReadResponse(rw)
				if err != nil {
					panic(err.Error())
				}
				_, data, _ := nsq.UnpackResponse(resp)
				if !bytes.Equal(data, []byte("OK")) {
					panic("invalid response")
				}
			}
			wg.Done()
		}()
	}

	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}

func BenchmarkProtocolV2Pub256(b *testing.B)  { benchmarkProtocolV2Pub(b, 256) }
func BenchmarkProtocolV2Pub512(b *testing.B)  { benchmarkProtocolV2Pub(b, 512) }
func BenchmarkProtocolV2Pub1k(b *testing.B)   { benchmarkProtocolV2Pub(b, 1024) }
func BenchmarkProtocolV2Pub2k(b *testing.B)   { benchmarkProtocolV2Pub(b, 2*1024) }
func BenchmarkProtocolV2Pub4k(b *testing.B)   { benchmarkProtocolV2Pub(b, 4*1024) }
func BenchmarkProtocolV2Pub8k(b *testing.B)   { benchmarkProtocolV2Pub(b, 8*1024) }
func BenchmarkProtocolV2Pub16k(b *testing.B)  { benchmarkProtocolV2Pub(b, 16*1024) }
func BenchmarkProtocolV2Pub32k(b *testing.B)  { benchmarkProtocolV2Pub(b, 32*1024) }
func BenchmarkProtocolV2Pub64k(b *testing.B)  { benchmarkProtocolV2Pub(b, 64*1024) }
func BenchmarkProtocolV2Pub128k(b *testing.B) { benchmarkProtocolV2Pub(b, 128*1024) }
func BenchmarkProtocolV2Pub256k(b *testing.B) { benchmarkProtocolV2Pub(b, 256*1024) }
func BenchmarkProtocolV2Pub512k(b *testing.B) { benchmarkProtocolV2Pub(b, 512*1024) }
func BenchmarkProtocolV2Pub1m(b *testing.B)   { benchmarkProtocolV2Pub(b, 1024*1024) }

func benchmarkProtocolV2Sub(b *testing.B, size int) {
	var wg sync.WaitGroup
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	options := NewNsqdOptions()
	options.memQueueSize = int64(b.N)
	tcpAddr, _ := mustStartNSQd(options)
	msg := make([]byte, size)
	topicName := "bench_v2_sub" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
	topic := nsqd.GetTopic(topicName)
	for i := 0; i < b.N; i++ {
		msg := nsq.NewMessage(<-nsqd.idChan, msg)
		topic.PutMessage(msg)
	}
	topic.GetChannel("ch")
	b.SetBytes(int64(len(msg)))
	goChan := make(chan int)
	rdyChan := make(chan int)
	workers := runtime.GOMAXPROCS(0)
	for j := 0; j < workers; j++ {
		wg.Add(1)
		go func() {
			subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
			wg.Done()
		}()
		<-rdyChan
	}
	b.StartTimer()

	close(goChan)
	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}

func subWorker(n int, workers int, tcpAddr *net.TCPAddr, topicName string, rdyChan chan int, goChan chan int) {
	conn, err := mustConnectNSQd(tcpAddr)
	if err != nil {
		panic(err.Error())
	}
	rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))

	identify(nil, conn)
	sub(nil, conn, topicName, "ch")

	rdyCount := int(math.Min(math.Max(float64(n/workers), 1), 2500))
	rdyChan <- 1
	<-goChan
	nsq.Ready(rdyCount).Write(rw)
	rw.Flush()
	num := n / workers
	numRdy := num/rdyCount - 1
	rdy := rdyCount
	for i := 0; i < num; i += 1 {
		resp, err := nsq.ReadResponse(rw)
		if err != nil {
			panic(err.Error())
		}
		frameType, data, err := nsq.UnpackResponse(resp)
		if err != nil {
			panic(err.Error())
		}
		if frameType != nsq.FrameTypeMessage {
			panic("got something else")
		}
		msg, err := nsq.DecodeMessage(data)
		if err != nil {
			panic(err.Error())
		}
		nsq.Finish(msg.Id).Write(rw)
		rdy--
		if rdy == 0 && numRdy > 0 {
			nsq.Ready(rdyCount).Write(rw)
			rdy = rdyCount
			numRdy--
			rw.Flush()
		}
	}
}

func BenchmarkProtocolV2Sub256(b *testing.B)  { benchmarkProtocolV2Sub(b, 256) }
func BenchmarkProtocolV2Sub512(b *testing.B)  { benchmarkProtocolV2Sub(b, 512) }
func BenchmarkProtocolV2Sub1k(b *testing.B)   { benchmarkProtocolV2Sub(b, 1024) }
func BenchmarkProtocolV2Sub2k(b *testing.B)   { benchmarkProtocolV2Sub(b, 2*1024) }
func BenchmarkProtocolV2Sub4k(b *testing.B)   { benchmarkProtocolV2Sub(b, 4*1024) }
func BenchmarkProtocolV2Sub8k(b *testing.B)   { benchmarkProtocolV2Sub(b, 8*1024) }
func BenchmarkProtocolV2Sub16k(b *testing.B)  { benchmarkProtocolV2Sub(b, 16*1024) }
func BenchmarkProtocolV2Sub32k(b *testing.B)  { benchmarkProtocolV2Sub(b, 32*1024) }
func BenchmarkProtocolV2Sub64k(b *testing.B)  { benchmarkProtocolV2Sub(b, 64*1024) }
func BenchmarkProtocolV2Sub128k(b *testing.B) { benchmarkProtocolV2Sub(b, 128*1024) }
func BenchmarkProtocolV2Sub256k(b *testing.B) { benchmarkProtocolV2Sub(b, 256*1024) }
func BenchmarkProtocolV2Sub512k(b *testing.B) { benchmarkProtocolV2Sub(b, 512*1024) }
func BenchmarkProtocolV2Sub1m(b *testing.B)   { benchmarkProtocolV2Sub(b, 1024*1024) }

func benchmarkProtocolV2MultiSub(b *testing.B, num int) {
	var wg sync.WaitGroup
	b.StopTimer()

	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	options := NewNsqdOptions()
	options.memQueueSize = int64(b.N)
	tcpAddr, _ := mustStartNSQd(options)
	msg := make([]byte, 256)
	b.SetBytes(int64(len(msg) * num))

	goChan := make(chan int)
	rdyChan := make(chan int)
	workers := runtime.GOMAXPROCS(0)
	for i := 0; i < num; i++ {
		topicName := "bench_v2" + strconv.Itoa(b.N) + "_" + strconv.Itoa(i) + "_" + strconv.Itoa(int(time.Now().Unix()))
		topic := nsqd.GetTopic(topicName)
		for i := 0; i < b.N; i++ {
			msg := nsq.NewMessage(<-nsqd.idChan, msg)
			topic.PutMessage(msg)
		}
		topic.GetChannel("ch")

		for j := 0; j < workers; j++ {
			wg.Add(1)
			go func() {
				subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
				wg.Done()
			}()
			<-rdyChan
		}
	}
	b.StartTimer()

	close(goChan)
	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}

func BenchmarkProtocolV2MultiSub1(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 1) }
func BenchmarkProtocolV2MultiSub2(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 2) }
func BenchmarkProtocolV2MultiSub4(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 4) }
func BenchmarkProtocolV2MultiSub8(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 8) }
func BenchmarkProtocolV2MultiSub16(b *testing.B) { benchmarkProtocolV2MultiSub(b, 16) }
Пример #11
0
func TestNS(t *testing.T) {
	ns := mkns(t, testing.Verbose())
	defer fstest.RmTree(t, tdir)
	d2 := zx.Dir{"path": "x", "name": "x", "proto": "p2"}
	err := <-ns.Mount("d", d2, After)
	if err == nil {
		t.Fatalf("mount: could mount")
	}
	err = <-ns.Unmount("/d", d2)
	if err != nil {
		t.Fatalf("unmount dir: %s", err)
	}
	printf("ns is `%s`\n", ns)

	s := ns.String()
	ns2, err := Parse(s)
	if err != nil {
		t.Fatalf("parse: %s", err)
	}
	printf("ns2 is `%s`\n", ns2)
	if ns2.String() != s {
		t.Fatalf("parsed ns differs")
	}

	outs := `path:"/" name:"/" type:"d" mode:"0755" proto:"lfs" spath:"/" tpath:"/tmp/lfs_test"
path:"/a/b" name:"b" type:"d" mode:"0755" proto:"lfs" spath:"/" tpath:"/tmp/lfs_test"
path:"/d" name:"d" type:"p" mode:"0644" proto:"p1"
path:"/d" name:"d" type:"p" mode:"0644" proto:"p2"
path:"/d" name:"d" type:"p" mode:"0644" proto:"p2"
`
	if s != outs {
		t.Fatalf("bad ns contents")
	}
	err = <-ns.Unmount("/a/b", nil)
	if err != nil {
		t.Fatalf("unmount dir: %s", err)
	}
	printf("ns is `%s`\n", ns)
}
Пример #12
0
func TestResolve(t *testing.T) {
	ns := mkns(t, false)
	defer fstest.RmTree(t, tdir)
	ns.Debug = testing.Verbose()
	ns.DebugFind = testing.Verbose()

	for _, r := range resolves {
		_, dirs, paths, err := ns.Resolve(r.Path)
		printf("sts %v\n", err)
		if err!=nil && !r.Fails {
			t.Fatalf("failed with %v", err)
		}
		if err==nil && r.Fails {
			t.Fatal("didn't fail")
		}
		if len(dirs) != len(paths) {
			t.Fatal("wrong lengths")
		}
		printf("dirs:\n")
		for _, d := range dirs {
			delete(d, "Uid")
			delete(d, "Gid")
			delete(d, "Wuid")
			delete(d, "Sum")
			printf("\t`%s`,\n", d)
		}
		printf("paths:\n")
		for _, p := range paths {
			printf("\t`%s`,\n", p)
		}
		for i := 0; i<len(r.Dirs) && i<len(dirs); i++ {
			if r.Dirs[i] != dirs[i].String() {
				t.Fatalf("bad result [%d]\n\tgot %s\n\twant %s\n",
					i, dirs[i], r.Dirs[i])
			}
		}
		if r.Dirs != nil {
			if len(dirs) > len(r.Dirs) {
				t.Fatalf("unexpected %s", dirs[len(r.Dirs)])
			}
			if len(dirs) < len(r.Dirs) {
				t.Fatalf("did expect %s", r.Dirs[len(dirs)])
			}
		}
		for i := 0; i<len(r.Paths) && i<len(paths); i++ {
			if r.Paths[i] != paths[i] {
				t.Fatalf("bad result [%d]\n\tgot %s\n\twant %s\n",
					i, paths[i], r.Paths[i])
			}
		}
		if r.Paths != nil {
			if len(paths) > len(r.Paths) {
				t.Fatalf("unexpected %s", paths[len(r.Paths)])
			}
			if len(paths) < len(r.Paths) {
				t.Fatalf("did expect %s", r.Paths[len(paths)])
			}
		}
	}
}
func _(t *testing.T) {
	fmt.Printf(<warning descr="Got 1 placeholder(s) for 2 arguments(s)">s</warning>, 1, 2)

	fmt.Printf(<warning descr="Value used for formatting text does not appear to be a string">s1</warning>, 1, 2)

	fmt.Printf(<warning descr="Value used for formatting text does not appear to be a string">s2</warning>, 1, 2)

	fmt.Errorf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
		1, 2, 3, 4, 5, 6, 7, 8, 9,
	)
	fmt.Fprintf(nil, "%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
		1, 2, 3, 4, 5, 6, 7, 8, 9,
	)
	fmt.Fscanf(nil, "%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
		1, 2, 3, 4, 5, 6, 7, 8, 9,
	)
	fmt.Printf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
		1, 2, 3, 4, 5, 6, 7, 8, 9,
	)
	fmt.Scanf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
		1, 2, 3, 4, 5, 6, 7, 8, 9,
	)
	fmt.Sprintf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
		1, 2, 3, 4, 5, 6, 7, 8, 9,
	)
	fmt.Sscanf(nil, "%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
		1, 2, 3, 4, 5, 6, 7, 8, 9,
	)
	log.Fatalf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
		1, 2, 3, 4, 5, 6, 7, 8, 9,
	)
	log.Panicf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
		1, 2, 3, 4, 5, 6, 7, 8, 9,
	)
	log.Printf("%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d",
		1, 2, 3, 4, 5, 6, 7, 8, 9,
	)

	fmt.Printf("eq (as-is): %.3f%% score: %v offset: %v descr: [%v]\n", 3.14, 1, 1, "descr")

	fmt.Printf("a: %+v", 1)

	fmt.Printf("%-4d", 999)

	fmt.Printf("a: %%%+v", 1)

	fmt.Printf(<warning descr="Got 0 placeholder(s) for 1 arguments(s)">"a: %%%%+v"</warning>, 1)

	fmt.Printf("#%02X%02X%02X", 1, 2, 3)

	fmt.Printf(<warning descr="Got 3 placeholder(s) for 4 arguments(s)">"#%02X%02X%02X"</warning>, 1, 2, 3, 4)

	myFormatVar := "%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d"
	log.Printf(myFormatVar, 1, 2, 3, 4, 5, 6, 7, 8, 9)

	myWrongFormatVar := "%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d"
	log.Printf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatVar</warning>, 1, 2, 3, 4, 5, 6, 7, 8)

	log.Printf(myFormatConst, 1, 2, 3, 4, 5, 6, 7, 8, 9)
	t.Errorf(myFormatConst, 1, 2, 3, 4, 5, 6, 7, 8, 9)
	t.Fatalf(myFormatConst, 1, 2, 3, 4, 5, 6, 7, 8, 9)
	t.Logf(myFormatConst, 1, 2, 3, 4, 5, 6, 7, 8, 9)
	t.Skipf(myFormatConst, 1, 2, 3, 4, 5, 6, 7, 8, 9)

	log.Printf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatConst</warning>, 1, 2, 3, 4, 5, 6, 7, 8)
	t.Errorf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatConst</warning>, 1, 2, 3, 4, 5, 6, 7, 8)
	t.Fatalf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatConst</warning>, 1, 2, 3, 4, 5, 6, 7, 8)
	t.Logf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatConst</warning>, 1, 2, 3, 4, 5, 6, 7, 8)
	t.Skipf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">myWrongFormatConst</warning>, 1, 2, 3, 4, 5, 6, 7, 8)

	printf("%d", 1)
	printf("%[2]d %[1]d", 1, 2)
	printf("%[2]d %[1]d %d", 1, 2)
	printf("%[2]d %[1]d %[2]d", 1, 2)
	printf("%d")

	myNonFormatFunc := func () int {
		return 1
	}
	log.Printf(<warning descr="Value used for formatting text does not appear to be a string">myNonFormatFunc()</warning>, 1, 2, 3, 4, 5, 6, 7, 8, 9)

	log.Printf(<warning descr="Got 9 placeholder(s) for 8 arguments(s)">"%d %d %#[1]x %#x %2.f %d %2.2f %.f %.3f %[9]*.[2]*[3]f %d %f %#[1]x %#x %[2]d %v % d"</warning>,
		1, 2, 3, 4, 5, 6, 7, 8,
	)
	fmt.Sprintf(<warning descr="Got 1 placeholder(s) for 0 arguments(s)">"%d"</warning>)

	log.Printf(<warning descr="Got 7 placeholder(s) for 13 arguments(s)">"%d %d %#[1]x %#x %f %2.f %2.2f %.f %.3f %[3]*.[2]*[1]f %d %d %#[1]x %#x %*[2]d %v % d"</warning>,
		1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
	)
	fmt.Sprintf(<warning descr="Got 1 placeholder(s) for 2 arguments(s)">"%d"</warning>, 1, 2)

	fmt.Print(<warning descr="Possible formatting directive in '\"%[2]*.[1]*[3]d\"'">"%[2]*.[1]*[3]d"</warning>, 2, 3, myNonFormatFunc)
	fmt.Print(<warning descr="Possible formatting directive in '\"%[2]*.[1]*[3]d\"'">"%[2]*.[1]*[3]d"</warning>, 2, 3, printf)

	fmt.Println("demo<warning descr="Function already ends with new line">\n</warning>", 2, 3, <warning descr="Argument 'myNonFormatFunc' is not a function call">myNonFormatFunc</warning>)
	fmt.Println("demo<warning descr="Function already ends with new line">\n</warning>", 2, 3, <warning descr="Argument 'printf' is not a function call">printf</warning>)

	fmt.Print("demo\n", 2, 3, <warning descr="Argument 'myNonFormatFunc' is not a function call">myNonFormatFunc</warning>)
	fmt.Print("demo\n", 2, 3, <warning descr="Argument 'printf' is not a function call">printf</warning>)

	type X struct{ Y, Z int32 }
	a := &X{5, 7}
	fmt.Println(a, "->", C.sum(*((*C.struct_x)(unsafe.Pointer(a)))))

	fmt.Sprintf("asdadad <warning descr="Unrecognized formatting verb '%O' call">%O</warning> asdadad", "demo")
	fmt.Printf("%[<warning descr="Index value [0] is not allowed">0</warning>]d", 1)
}
Пример #14
0
func TestClientTimeout(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 50 * time.Millisecond
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	err = nsq.SendCommand(conn, nsq.Subscribe(topicName, "ch", "TestClientTimeoutV2", "TestClientTimeoutV2"))
	assert.Equal(t, err, nil)

	time.Sleep(50 * time.Millisecond)

	// depending on timing there may be 1 or 2 hearbeats sent
	// just read until we get an error
	timer := time.After(100 * time.Millisecond)
	for {
		select {
		case <-timer:
			t.Fatalf("test timed out")
		default:
			_, err := nsq.ReadResponse(conn)
			if err != nil {
				goto done
			}
		}
	}
done:
}

func TestClientHeartbeat(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 100 * time.Millisecond
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	err = nsq.SendCommand(conn, nsq.Subscribe(topicName, "ch", "TestClientHeartbeatV2", "TestClientHeartbeatV2"))
	assert.Equal(t, err, nil)

	err = nsq.SendCommand(conn, nsq.Ready(1))
	assert.Equal(t, err, nil)

	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	assert.Equal(t, data, []byte("_heartbeat_"))

	time.Sleep(10 * time.Millisecond)

	err = nsq.SendCommand(conn, nsq.Nop())
	assert.Equal(t, err, nil)

	// wait long enough that would have timed out (had we not sent the above cmd)
	time.Sleep(50 * time.Millisecond)

	err = nsq.SendCommand(conn, nsq.Nop())
	assert.Equal(t, err, nil)
}

func TestPausing(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix()))

	tcpAddr, _ := mustStartNSQd(NewNsqdOptions())
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	err = nsq.SendCommand(conn, nsq.Subscribe(topicName, "ch", "TestPausing", "TestPausing"))
	assert.Equal(t, err, nil)

	err = nsq.SendCommand(conn, nsq.Ready(1))
	assert.Equal(t, err, nil)

	topic := nsqd.GetTopic(topicName)
	msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
	channel := topic.GetChannel("ch")
	topic.PutMessage(msg)

	// receive the first message via the client, finish it, and send new RDY
	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	msg, err = nsq.DecodeMessage(data)
	assert.Equal(t, msg.Body, []byte("test body"))

	err = nsq.SendCommand(conn, nsq.Finish(msg.Id))
	assert.Equal(t, err, nil)

	err = nsq.SendCommand(conn, nsq.Ready(1))
	assert.Equal(t, err, nil)

	// sleep to allow the RDY state to take effect
	time.Sleep(50 * time.Millisecond)

	// pause the channel... the client shouldn't receive any more messages
	channel.Pause()

	// sleep to allow the paused state to take effect
	time.Sleep(50 * time.Millisecond)

	msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body2"))
	topic.PutMessage(msg)

	// allow the client to possibly get a message, the test would hang indefinitely
	// if pausing was not working on the internal clientMsgChan read
	time.Sleep(50 * time.Millisecond)
	msg = <-channel.clientMsgChan
	assert.Equal(t, msg.Body, []byte("test body2"))

	// unpause the channel... the client should now be pushed a message
	channel.UnPause()

	msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body3"))
	topic.PutMessage(msg)

	resp, _ = nsq.ReadResponse(conn)
	_, data, _ = nsq.UnpackResponse(resp)
	msg, err = nsq.DecodeMessage(data)
	assert.Equal(t, msg.Body, []byte("test body3"))
}

func BenchmarkProtocolV2Command(b *testing.B) {
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	p := &ProtocolV2{}
	c := NewClientV2(nil)
	params := [][]byte{[]byte("SUB"), []byte("test"), []byte("ch")}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		p.Exec(c, params)
	}
}

func BenchmarkProtocolV2Data(b *testing.B) {
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	p := &ProtocolV2{}
	var cb bytes.Buffer
	rw := bufio.NewReadWriter(bufio.NewReader(&cb), bufio.NewWriter(ioutil.Discard))
	conn := util.MockConn{rw}
	c := NewClientV2(conn)
	var buf bytes.Buffer
	msg := nsq.NewMessage([]byte("0123456789abcdef"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
	b.StartTimer()

	for i := 0; i < b.N; i += 1 {
		buf.Reset()
		msg.Encode(&buf)
		p.Send(c, nsq.FrameTypeMessage, buf.Bytes())
	}
}

func BenchmarkProtocolV2Pub(b *testing.B) {
	var wg sync.WaitGroup
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	options := NewNsqdOptions()
	options.memQueueSize = int64(b.N)
	tcpAddr, _ := mustStartNSQd(options)
	msg := []byte(`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`)
	topicName := "bench_v1" + strconv.Itoa(int(time.Now().Unix()))
	b.StartTimer()

	for j := 0; j < runtime.GOMAXPROCS(0); j++ {
		wg.Add(1)
		go func() {
			conn, err := mustConnectNSQd(tcpAddr)
			if err != nil {
				b.Fatal(err.Error())
			}
			for i := 0; i < (b.N / runtime.GOMAXPROCS(0)); i += 1 {
				err := nsq.SendCommand(conn, nsq.Publish(topicName, msg))
				if err != nil {
					b.Fatal(err.Error())
				}
				resp, err := nsq.ReadResponse(conn)
				if err != nil {
					b.Fatal(err.Error())
				}
				_, data, _ := nsq.UnpackResponse(resp)
				if !bytes.Equal(data, []byte("OK")) {
					b.Fatal("invalid response")
				}
				b.SetBytes(int64(len(msg)))
			}
			wg.Done()
		}()
	}

	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}
Пример #15
0
func TestClientTimeout(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 50 * time.Millisecond
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	err = nsq.Subscribe(topicName, "ch", "TestClientTimeoutV2", "TestClientTimeoutV2").Write(conn)
	assert.Equal(t, err, nil)

	time.Sleep(50 * time.Millisecond)

	// depending on timing there may be 1 or 2 hearbeats sent
	// just read until we get an error
	timer := time.After(100 * time.Millisecond)
	for {
		select {
		case <-timer:
			t.Fatalf("test timed out")
		default:
			_, err := nsq.ReadResponse(conn)
			if err != nil {
				goto done
			}
		}
	}
done:
}

func TestClientHeartbeat(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 100 * time.Millisecond
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	err = nsq.Subscribe(topicName, "ch", "TestClientHeartbeatV2", "TestClientHeartbeatV2").Write(conn)
	assert.Equal(t, err, nil)

	err = nsq.Ready(1).Write(conn)
	assert.Equal(t, err, nil)

	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	assert.Equal(t, data, []byte("_heartbeat_"))

	time.Sleep(10 * time.Millisecond)

	err = nsq.Nop().Write(conn)
	assert.Equal(t, err, nil)

	// wait long enough that would have timed out (had we not sent the above cmd)
	time.Sleep(50 * time.Millisecond)

	err = nsq.Nop().Write(conn)
	assert.Equal(t, err, nil)
}

func TestPausing(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix()))

	tcpAddr, _ := mustStartNSQd(NewNsqdOptions())
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	err = nsq.Subscribe(topicName, "ch", "TestPausing", "TestPausing").Write(conn)
	assert.Equal(t, err, nil)

	err = nsq.Ready(1).Write(conn)
	assert.Equal(t, err, nil)

	topic := nsqd.GetTopic(topicName)
	msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
	channel := topic.GetChannel("ch")
	topic.PutMessage(msg)

	// receive the first message via the client, finish it, and send new RDY
	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	msg, err = nsq.DecodeMessage(data)
	assert.Equal(t, msg.Body, []byte("test body"))

	err = nsq.Finish(msg.Id).Write(conn)
	assert.Equal(t, err, nil)

	err = nsq.Ready(1).Write(conn)
	assert.Equal(t, err, nil)

	// sleep to allow the RDY state to take effect
	time.Sleep(50 * time.Millisecond)

	// pause the channel... the client shouldn't receive any more messages
	channel.Pause()

	// sleep to allow the paused state to take effect
	time.Sleep(50 * time.Millisecond)

	msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body2"))
	topic.PutMessage(msg)

	// allow the client to possibly get a message, the test would hang indefinitely
	// if pausing was not working on the internal clientMsgChan read
	time.Sleep(50 * time.Millisecond)
	msg = <-channel.clientMsgChan
	assert.Equal(t, msg.Body, []byte("test body2"))

	// unpause the channel... the client should now be pushed a message
	channel.UnPause()

	msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body3"))
	topic.PutMessage(msg)

	resp, _ = nsq.ReadResponse(conn)
	_, data, _ = nsq.UnpackResponse(resp)
	msg, err = nsq.DecodeMessage(data)
	assert.Equal(t, msg.Body, []byte("test body3"))
}

func BenchmarkProtocolV2Exec(b *testing.B) {
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	p := &ProtocolV2{}
	c := NewClientV2(nil)
	params := [][]byte{[]byte("NOP")}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		p.Exec(c, params)
	}
}

func BenchmarkProtocolV2Data(b *testing.B) {
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	p := &ProtocolV2{}
	var cb bytes.Buffer
	rw := bufio.NewReadWriter(bufio.NewReader(&cb), bufio.NewWriter(ioutil.Discard))
	conn := util.MockConn{rw}
	c := NewClientV2(conn)
	var buf bytes.Buffer
	body := make([]byte, 256)
	msg := nsq.NewMessage([]byte("0123456789abcdef"), body)
	b.SetBytes(int64(len(body)))
	b.StartTimer()

	for i := 0; i < b.N; i += 1 {
		buf.Reset()
		msg.Write(&buf)
		p.Send(c, nsq.FrameTypeMessage, buf.Bytes())
	}
}

func benchmarkProtocolV2Pub(b *testing.B, size int) {
	var wg sync.WaitGroup
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	options := NewNsqdOptions()
	options.memQueueSize = int64(b.N)
	tcpAddr, _ := mustStartNSQd(options)
	msg := make([]byte, size)
	batchSize := 200
	batch := make([][]byte, 0)
	for i := 0; i < batchSize; i++ {
		batch = append(batch, msg)
	}
	topicName := "bench_v2_pub" + strconv.Itoa(int(time.Now().Unix()))
	b.SetBytes(int64(len(msg)))
	b.StartTimer()

	for j := 0; j < runtime.GOMAXPROCS(0); j++ {
		wg.Add(1)
		go func() {
			conn, err := mustConnectNSQd(tcpAddr)
			if err != nil {
				panic(err.Error())
			}
			rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))

			num := b.N / runtime.GOMAXPROCS(0) / batchSize
			for i := 0; i < num; i += 1 {
				cmd, _ := nsq.MultiPublish(topicName, batch)
				err := cmd.Write(rw)
				if err != nil {
					panic(err.Error())
				}
				err = rw.Flush()
				if err != nil {
					panic(err.Error())
				}
				resp, err := nsq.ReadResponse(rw)
				if err != nil {
					panic(err.Error())
				}
				_, data, _ := nsq.UnpackResponse(resp)
				if !bytes.Equal(data, []byte("OK")) {
					panic("invalid response")
				}
			}
			wg.Done()
		}()
	}

	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}

func BenchmarkProtocolV2Pub256(b *testing.B)  { benchmarkProtocolV2Pub(b, 256) }
func BenchmarkProtocolV2Pub512(b *testing.B)  { benchmarkProtocolV2Pub(b, 512) }
func BenchmarkProtocolV2Pub1k(b *testing.B)   { benchmarkProtocolV2Pub(b, 1024) }
func BenchmarkProtocolV2Pub2k(b *testing.B)   { benchmarkProtocolV2Pub(b, 2*1024) }
func BenchmarkProtocolV2Pub4k(b *testing.B)   { benchmarkProtocolV2Pub(b, 4*1024) }
func BenchmarkProtocolV2Pub8k(b *testing.B)   { benchmarkProtocolV2Pub(b, 8*1024) }
func BenchmarkProtocolV2Pub16k(b *testing.B)  { benchmarkProtocolV2Pub(b, 16*1024) }
func BenchmarkProtocolV2Pub32k(b *testing.B)  { benchmarkProtocolV2Pub(b, 32*1024) }
func BenchmarkProtocolV2Pub64k(b *testing.B)  { benchmarkProtocolV2Pub(b, 64*1024) }
func BenchmarkProtocolV2Pub128k(b *testing.B) { benchmarkProtocolV2Pub(b, 128*1024) }
func BenchmarkProtocolV2Pub256k(b *testing.B) { benchmarkProtocolV2Pub(b, 256*1024) }
func BenchmarkProtocolV2Pub512k(b *testing.B) { benchmarkProtocolV2Pub(b, 512*1024) }
func BenchmarkProtocolV2Pub1m(b *testing.B)   { benchmarkProtocolV2Pub(b, 1024*1024) }

func benchmarkProtocolV2Sub(b *testing.B, size int) {
	var wg sync.WaitGroup
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	options := NewNsqdOptions()
	options.memQueueSize = int64(b.N)
	tcpAddr, _ := mustStartNSQd(options)
	msg := make([]byte, size)
	topicName := "bench_v2_sub" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
	topic := nsqd.GetTopic(topicName)
	for i := 0; i < b.N; i++ {
		msg := nsq.NewMessage(<-nsqd.idChan, msg)
		topic.PutMessage(msg)
	}
	topic.GetChannel("ch")
	b.SetBytes(int64(len(msg)))
	goChan := make(chan int)
	rdyChan := make(chan int)
	workers := runtime.GOMAXPROCS(0)
	for j := 0; j < workers; j++ {
		wg.Add(1)
		go func(id int) {
			subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan, id)
			wg.Done()
		}(j)
		<-rdyChan
	}
	b.StartTimer()

	close(goChan)
	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}

func subWorker(n int, workers int, tcpAddr *net.TCPAddr, topicName string, rdyChan chan int, goChan chan int, id int) {
	conn, err := mustConnectNSQd(tcpAddr)
	if err != nil {
		panic(err.Error())
	}
	rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
	nsq.Subscribe(topicName, "ch", "test", "test").Write(rw)
	rdyCount := int(math.Min(math.Max(float64(n/workers), 1), 2500))
	rdyChan <- 1
	<-goChan
	nsq.Ready(rdyCount).Write(rw)
	rw.Flush()
	num := n / workers
	numRdy := num/rdyCount - 1
	rdy := rdyCount
	for i := 0; i < num; i += 1 {
		resp, err := nsq.ReadResponse(rw)
		if err != nil {
			panic(err.Error())
		}
		frameType, data, err := nsq.UnpackResponse(resp)
		if err != nil {
			panic(err.Error())
		}
		if frameType != nsq.FrameTypeMessage {
			panic("got something else")
		}
		msg, err := nsq.DecodeMessage(data)
		if err != nil {
			panic(err.Error())
		}
		nsq.Finish(msg.Id).Write(rw)
		rdy--
		if rdy == 0 && numRdy > 0 {
			nsq.Ready(rdyCount).Write(rw)
			rdy = rdyCount
			numRdy--
			rw.Flush()
		}
	}
}

func BenchmarkProtocolV2Sub256(b *testing.B)  { benchmarkProtocolV2Sub(b, 256) }
func BenchmarkProtocolV2Sub512(b *testing.B)  { benchmarkProtocolV2Sub(b, 512) }
func BenchmarkProtocolV2Sub1k(b *testing.B)   { benchmarkProtocolV2Sub(b, 1024) }
func BenchmarkProtocolV2Sub2k(b *testing.B)   { benchmarkProtocolV2Sub(b, 2*1024) }
func BenchmarkProtocolV2Sub4k(b *testing.B)   { benchmarkProtocolV2Sub(b, 4*1024) }
func BenchmarkProtocolV2Sub8k(b *testing.B)   { benchmarkProtocolV2Sub(b, 8*1024) }
func BenchmarkProtocolV2Sub16k(b *testing.B)  { benchmarkProtocolV2Sub(b, 16*1024) }
func BenchmarkProtocolV2Sub32k(b *testing.B)  { benchmarkProtocolV2Sub(b, 32*1024) }
func BenchmarkProtocolV2Sub64k(b *testing.B)  { benchmarkProtocolV2Sub(b, 64*1024) }
func BenchmarkProtocolV2Sub128k(b *testing.B) { benchmarkProtocolV2Sub(b, 128*1024) }
func BenchmarkProtocolV2Sub256k(b *testing.B) { benchmarkProtocolV2Sub(b, 256*1024) }
func BenchmarkProtocolV2Sub512k(b *testing.B) { benchmarkProtocolV2Sub(b, 512*1024) }
func BenchmarkProtocolV2Sub1m(b *testing.B)   { benchmarkProtocolV2Sub(b, 1024*1024) }
Пример #16
0
func TestMatchChar(t *testing.T) {
	tests := []matchTest{
		{in: "a", out: []string{"a"}},
		{in: "\\\\a", out: []string{"\\\\", "a"}},
		{in: "\\\"a", out: []string{"\\\"", "a"}},
		{in: "\\\\\"a", out: []string{"\\\\", "a"}},
		{in: "a\\\"\\a", out: []string{"a", "\\\"", "a"}},
		{in: "a\\\"\\\\a", out: []string{"a", "\\\"", "\\\\", "a"}},
		{in: "a\\\\\"\\a", out: []string{"a", "\\\\", "a"}},
	}

	re := regexp.MustCompile(hstoreChar)
	for _, mt := range tests {
		matches := re.FindAllStringIndex(mt.in, -1)
		checkMatch(t, matches, mt)
	}
}

func TestMatchString(t *testing.T) {
	tests := []matchTest{
		{in: "a", out: []string{}},
		{in: "\"a\"", out: []string{"\"a\""}},
		{in: "\"id\"=>\"44\", \"foo\"=>\"dfs => somf\", \"name\"=>\"Wash\\\"ington\", \"null\"=>NULL, \"quote\"=>\"\\\"fs ' \"",
			out: []string{
				"\"id\"",
				"\"44\"",
				"\"foo\"",
				"\"dfs => somf\"",
				"\"name\"",
				"\"Wash\\\"ington\"",
				"\"null\"",
				"\"quote\"",
				"\"\\\"fs ' \"",
			}},
	}

	re := regexp.MustCompile("\"" + hstoreString + "\"")
	for _, mt := range tests {
		matches := re.FindAllStringIndex(mt.in, -1)
		checkMatch(t, matches, mt)
	}
}

func TestMatchKey(t *testing.T) {
	tests := []matchTest{
		{in: "a", out: []string{}},
		{in: "\"a\"", out: []string{"\"a\""}},
		{in: "\"id\"=>\"44\", \"foo\"=>\"dfs => somf\", \"name\"=>\"Wash\\\"ington\", \"null\"=>NULL, \"quote\"=>\"\\\"fs ' \"",
			out: []string{
				"\"id\"",
				"\"44\"",
				"\"foo\"",
				"\"dfs => somf\"",
				"\"name\"",
				"\"Wash\\\"ington\"",
				"\"null\"",
				"\"quote\"",
				"\"\\\"fs ' \"",
			}},
	}

	re := regexp.MustCompile(hstoreKey)
	for _, mt := range tests {
		matches := re.FindAllStringIndex(mt.in, -1)
		checkMatch(t, matches, mt)
	}
}

func TestMatchValue(t *testing.T) {
	tests := []matchTest{
		{in: "a", out: []string{}},
		{in: "\"a\"", out: []string{"\"a\""}},
		{in: "\"id\"=>\"44\", \"foo\"=>\"dfs => somf\", \"name\"=>\"Wash\\\"ington\", \"null\"=>NULL, \"quote\"=>\"\\\"fs ' \"",
			out: []string{
				"\"id\"",
				"\"44\"",
				"\"foo\"",
				"\"dfs => somf\"",
				"\"name\"",
				"\"Wash\\\"ington\"",
				"\"null\"",
				"NULL",
				"\"quote\"",
				"\"\\\"fs ' \"",
			}},
	}

	re := regexp.MustCompile(hstoreValue)
	for _, mt := range tests {
		matches := re.FindAllStringIndex(mt.in, -1)
		checkMatch(t, matches, mt)
	}
}

func nullString(v interface{}) sql.NullString {
	if v == nil {
		return sql.NullString{Valid: false}
	}
	return sql.NullString{String: v.(string), Valid: true}
}

var fullTests = []matchTest{
	{in: "NULL", out: []string{}},
	{in: "\"a\" => \"b\"",
		out:    []string{"\"a\" => \"b\""},
		hstore: Hstore{"a": nullString("b")}},
	{in: "\"a\"   =>   NULL",
		out:    []string{"\"a\"   =>   NULL"},
		hstore: Hstore{"a": sql.NullString{Valid: false}}},
	{in: "\"id\"=>\"44\", \"foo\"=>\"dfs => somf\", \"name\"=>\"Wash\\\"ington\", \"null\"=>NULL, \"quote\"=>\"\\\"fs ' \"",
		out: []string{
			"\"id\"=>\"44\"",
			"\"foo\"=>\"dfs => somf\"",
			"\"name\"=>\"Wash\\\"ington\"",
			"\"null\"=>NULL",
			"\"quote\"=>\"\\\"fs ' \"",
		},
		hstore: Hstore{
			"id":    nullString("44"),
			"foo":   nullString("dfs => somf"),
			"name":  nullString("Wash\\\"ington"),
			"null":  nullString(nil),
			"quote": nullString("\\\"fs ' "),
		},
	},
}

func TestParseHstore(t *testing.T) {
	for _, mt := range fullTests {
		matches := pairExp.FindAllStringIndex(mt.in, -1)
		checkMatch(t, matches, mt)

		hs := make(Hstore)
		err := parseHstore(mt.in, &hs)
		if err != nil {
			t.Errorf("Error parsing %q: %v", mt.in, err)
		}
		checkHstore(t, mt.hstore, hs)

	}
}

func TestValueAndScan(t *testing.T) {
	for _, mt := range fullTests {
		valued, err := mt.hstore.Value()
		if err != nil {
			t.Fatalf("Value failed: %q", err)
		}
		var scanned Hstore
		(&scanned).Scan([]byte(valued.(string)))
		checkHstore(t, mt.hstore, scanned)
	}
}

func TestDBRoundTrip(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	_, err := db.Exec("CREATE EXTENSION IF NOT EXISTS hstore")
	fatal(t, err)
	_, err = db.Exec("CREATE TEMP TABLE temp (id serial, data hstore)")
	fatal(t, err)

	for _, mt := range fullTests {
		v, err := mt.hstore.Value()
		check(t, err)
		r, err := db.Exec("INSERT INTO temp (data) VALUES ($1)", v)
		check(t, err)

		if n, _ := r.RowsAffected(); n != 1 {
			t.Fatalf("expected 1 row affected, not %d", n)
		}
	}

	rows, err := db.Query("SELECT data FROM temp ORDER BY id ASC")
	check(t, err)

	for _, mt := range fullTests {
		if !rows.Next() {
			t.Errorf("Ran out of rows!")
		}
		var data Hstore
		err = rows.Scan(&data)
		check(t, err)
		t.Logf("%+v", data)
		checkHstore(t, mt.hstore, data)
	}

	if rows.Next() {
		t.Errorf("Too many rows!")
	}

}
Пример #17
0
func TestClientTimeout(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 50 * time.Millisecond
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	err = nsq.SendCommand(conn, nsq.Subscribe(topicName, "ch", "TestClientTimeoutV2", "TestClientTimeoutV2"))
	assert.Equal(t, err, nil)

	time.Sleep(50 * time.Millisecond)

	// depending on timing there may be 1 or 2 hearbeats sent
	// just read until we get an error
	timer := time.After(100 * time.Millisecond)
	for {
		select {
		case <-timer:
			t.Fatalf("test timed out")
		default:
			_, err := nsq.ReadResponse(conn)
			if err != nil {
				goto done
			}
		}
	}
done:
}

func TestClientHeartbeat(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 100 * time.Millisecond
	tcpAddr, _ := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	err = nsq.SendCommand(conn, nsq.Subscribe(topicName, "ch", "TestClientHeartbeatV2", "TestClientHeartbeatV2"))
	assert.Equal(t, err, nil)

	err = nsq.SendCommand(conn, nsq.Ready(1))
	assert.Equal(t, err, nil)

	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	assert.Equal(t, data, []byte("_heartbeat_"))

	time.Sleep(10 * time.Millisecond)

	err = nsq.SendCommand(conn, nsq.Nop())
	assert.Equal(t, err, nil)

	// wait long enough that would have timed out (had we not sent the above cmd)
	time.Sleep(50 * time.Millisecond)

	err = nsq.SendCommand(conn, nsq.Nop())
	assert.Equal(t, err, nil)
}

func BenchmarkProtocolV2Command(b *testing.B) {
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	p := &ProtocolV2{}
	c := NewClientV2(nil)
	params := [][]byte{[]byte("SUB"), []byte("test"), []byte("ch")}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		p.Exec(c, params)
	}
}

func BenchmarkProtocolV2Data(b *testing.B) {
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	p := &ProtocolV2{}
	var cb bytes.Buffer
	rw := bufio.NewReadWriter(bufio.NewReader(&cb), bufio.NewWriter(ioutil.Discard))
	conn := util.MockConn{rw}
	c := NewClientV2(conn)
	var buf bytes.Buffer
	msg := nsq.NewMessage([]byte("0123456789abcdef"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
	b.StartTimer()

	for i := 0; i < b.N; i += 1 {
		buf.Reset()
		msg.Encode(&buf)
		p.Send(c, nsq.FrameTypeMessage, buf.Bytes())
	}
}

func BenchmarkProtocolV2Pub(b *testing.B) {
	var wg sync.WaitGroup
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	options := NewNsqdOptions()
	options.memQueueSize = int64(b.N)
	tcpAddr, _ := mustStartNSQd(options)
	msg := []byte(`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`)
	topicName := "bench_v1" + strconv.Itoa(int(time.Now().Unix()))
	b.StartTimer()

	for j := 0; j < runtime.GOMAXPROCS(0); j++ {
		wg.Add(1)
		go func() {
			conn, err := mustConnectNSQd(tcpAddr)
			if err != nil {
				b.Fatal(err.Error())
			}
			for i := 0; i < (b.N / runtime.GOMAXPROCS(0)); i += 1 {
				err := nsq.SendCommand(conn, nsq.Publish(topicName, msg))
				if err != nil {
					b.Fatal(err.Error())
				}
				resp, err := nsq.ReadResponse(conn)
				if err != nil {
					b.Fatal(err.Error())
				}
				_, data, _ := nsq.UnpackResponse(resp)
				if !bytes.Equal(data, []byte("OK")) {
					b.Fatal("invalid response")
				}
				b.SetBytes(int64(len(msg)))
			}
			wg.Done()
		}()
	}

	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}
Пример #18
0
func TestChromeTermination(t *testing.T) {
	auto := testDefaultStartup(t)
	doneCh := make(chan struct{})
	shutdown := time.NewTimer(time.Second * 4)
	timeout := time.NewTimer(time.Second * 10)
	terminatedHandler := func(reason string) {
		t.Logf("reason: %s\n", reason)
		doneCh <- struct{}{}
	}

	auto.SetTerminationHandler(terminatedHandler)
	for {
		select {
		case <-doneCh:
			goto DONE
		case <-shutdown.C:
			auto.Shutdown()
		case <-timeout.C:
			t.Fatalf("timed out waiting for termination")
		}
	}
DONE:
}

func testDefaultStartup(t *testing.T) *AutoGcd {
	s := NewSettings(testPath, testRandomDir(t))
	s.RemoveUserDir(true)
	s.AddStartupFlags(testStartupFlags)
	s.SetDebuggerPort(testRandomPort(t))
	auto := NewAutoGcd(s)
	if err := auto.Start(); err != nil {
		t.Fatalf("failed to start chrome: %s\n", err)
	}
	auto.SetTerminationHandler(nil) // do not want our tests to panic
	return auto
}

func testServer() {
	testListener, _ = net.Listen("tcp", ":0")
	_, testServerPort, _ := net.SplitHostPort(testListener.Addr().String())
	testServerAddr = fmt.Sprintf("http://localhost:%s/", testServerPort)
	go http.Serve(testListener, http.FileServer(http.Dir("testdata")))
}

func testRandomPort(t *testing.T) string {
	l, err := net.Listen("tcp", ":0")
	if err != nil {
		t.Fatal(err)
	}
	_, randPort, _ := net.SplitHostPort(l.Addr().String())
	l.Close()
	return randPort
}

func testRandomDir(t *testing.T) string {
	dir, err := ioutil.TempDir(testDir, "autogcd")
	if err != nil {
		t.Fatalf("error getting temp dir: %s\n", err)
	}
	return dir
}
Пример #19
0
func TestClientTimeout(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 50 * time.Millisecond
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identify(t, conn)
	sub(t, conn, topicName, "ch")

	time.Sleep(50 * time.Millisecond)

	// depending on timing there may be 1 or 2 hearbeats sent
	// just read until we get an error
	timer := time.After(100 * time.Millisecond)
	for {
		select {
		case <-timer:
			t.Fatalf("test timed out")
		default:
			_, err := nsq.ReadResponse(conn)
			if err != nil {
				goto done
			}
		}
	}
done:
}

func TestClientHeartbeat(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 200 * time.Millisecond
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identify(t, conn)
	sub(t, conn, topicName, "ch")

	err = nsq.Ready(1).Write(conn)
	assert.Equal(t, err, nil)

	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	assert.Equal(t, data, []byte("_heartbeat_"))

	time.Sleep(20 * time.Millisecond)

	err = nsq.Nop().Write(conn)
	assert.Equal(t, err, nil)

	// wait long enough that would have timed out (had we not sent the above cmd)
	time.Sleep(100 * time.Millisecond)

	err = nsq.Nop().Write(conn)
	assert.Equal(t, err, nil)
}

func TestClientHeartbeatDisableSUB(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))

	options := NewNsqdOptions()
	options.clientTimeout = 200 * time.Millisecond
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identifyHeartbeatInterval(t, conn, -1, nsq.FrameTypeResponse, "OK")
	subFail(t, conn, topicName, "ch")
}

func TestClientHeartbeatDisable(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	options := NewNsqdOptions()
	options.clientTimeout = 100 * time.Millisecond
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identifyHeartbeatInterval(t, conn, -1, nsq.FrameTypeResponse, "OK")

	time.Sleep(150 * time.Millisecond)

	err = nsq.Nop().Write(conn)
	assert.Equal(t, err, nil)
}

func TestMaxHeartbeatIntervalValid(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	options := NewNsqdOptions()
	options.maxHeartbeatInterval = 300 * time.Second
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	hbi := int(options.maxHeartbeatInterval / time.Millisecond)
	identifyHeartbeatInterval(t, conn, hbi, nsq.FrameTypeResponse, "OK")
}

func TestMaxHeartbeatIntervalInvalid(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	options := NewNsqdOptions()
	options.maxHeartbeatInterval = 300 * time.Second
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	hbi := int(options.maxHeartbeatInterval/time.Millisecond + 1)
	identifyHeartbeatInterval(t, conn, hbi, nsq.FrameTypeError, "E_BAD_BODY IDENTIFY heartbeat interval (300001) is invalid")
}

func TestPausing(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix()))

	tcpAddr, _, nsqd := mustStartNSQd(NewNsqdOptions())
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identify(t, conn)
	sub(t, conn, topicName, "ch")

	err = nsq.Ready(1).Write(conn)
	assert.Equal(t, err, nil)

	topic := nsqd.GetTopic(topicName)
	msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
	channel := topic.GetChannel("ch")
	topic.PutMessage(msg)

	// receive the first message via the client, finish it, and send new RDY
	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	msg, err = nsq.DecodeMessage(data)
	assert.Equal(t, msg.Body, []byte("test body"))

	err = nsq.Finish(msg.Id).Write(conn)
	assert.Equal(t, err, nil)

	err = nsq.Ready(1).Write(conn)
	assert.Equal(t, err, nil)

	// sleep to allow the RDY state to take effect
	time.Sleep(50 * time.Millisecond)

	// pause the channel... the client shouldn't receive any more messages
	channel.Pause()

	// sleep to allow the paused state to take effect
	time.Sleep(50 * time.Millisecond)

	msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body2"))
	topic.PutMessage(msg)

	// allow the client to possibly get a message, the test would hang indefinitely
	// if pausing was not working on the internal clientMsgChan read
	time.Sleep(50 * time.Millisecond)
	msg = <-channel.clientMsgChan
	assert.Equal(t, msg.Body, []byte("test body2"))

	// unpause the channel... the client should now be pushed a message
	channel.UnPause()

	msg = nsq.NewMessage(<-nsqd.idChan, []byte("test body3"))
	topic.PutMessage(msg)

	resp, _ = nsq.ReadResponse(conn)
	_, data, _ = nsq.UnpackResponse(resp)
	msg, err = nsq.DecodeMessage(data)
	assert.Equal(t, msg.Body, []byte("test body3"))
}

func TestEmptyCommand(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	tcpAddr, _, nsqd := mustStartNSQd(NewNsqdOptions())
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	_, err = conn.Write([]byte("\n\n"))
	assert.Equal(t, err, nil)

	// if we didn't panic here we're good, see issue #120
}

func TestSizeLimits(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	options := NewNsqdOptions()
	*verbose = true
	options.maxMessageSize = 100
	options.maxBodySize = 1000
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	topicName := "test_limits_v2" + strconv.Itoa(int(time.Now().Unix()))

	identify(t, conn)
	sub(t, conn, topicName, "ch")

	// PUB that's valid
	nsq.Publish(topicName, make([]byte, 95)).Write(conn)
	resp, _ := nsq.ReadResponse(conn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeResponse)
	assert.Equal(t, data, []byte("OK"))

	// PUB that's invalid (too big)
	nsq.Publish(topicName, make([]byte, 105)).Write(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeError)
	assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE PUB message too big 105 > 100"))

	// need to reconnect
	conn, err = mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	// PUB thats empty
	nsq.Publish(topicName, make([]byte, 0)).Write(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeError)
	assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE PUB invalid message body size 0"))

	// need to reconnect
	conn, err = mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	// MPUB body that's valid
	mpub := make([][]byte, 0)
	for i := 0; i < 5; i++ {
		mpub = append(mpub, make([]byte, 100))
	}
	cmd, _ := nsq.MultiPublish(topicName, mpub)
	cmd.Write(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeResponse)
	assert.Equal(t, data, []byte("OK"))

	// MPUB body that's invalid (body too big)
	mpub = make([][]byte, 0)
	for i := 0; i < 11; i++ {
		mpub = append(mpub, make([]byte, 100))
	}
	cmd, _ = nsq.MultiPublish(topicName, mpub)
	cmd.Write(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeError)
	assert.Equal(t, string(data), fmt.Sprintf("E_BAD_BODY MPUB body too big 1148 > 1000"))

	// need to reconnect
	conn, err = mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	// MPUB that's invalid (one message empty)
	mpub = make([][]byte, 0)
	for i := 0; i < 5; i++ {
		mpub = append(mpub, make([]byte, 100))
	}
	mpub = append(mpub, make([]byte, 0))
	cmd, _ = nsq.MultiPublish(topicName, mpub)
	cmd.Write(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeError)
	assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE MPUB invalid message(5) body size 0"))

	// need to reconnect
	conn, err = mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	// MPUB body that's invalid (one of the messages is too big)
	mpub = make([][]byte, 0)
	for i := 0; i < 5; i++ {
		mpub = append(mpub, make([]byte, 101))
	}
	cmd, _ = nsq.MultiPublish(topicName, mpub)
	cmd.Write(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeError)
	assert.Equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE MPUB message too big 101 > 100"))
}

func TestTouch(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	*verbose = true
	options := NewNsqdOptions()
	options.msgTimeout = 50 * time.Millisecond
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	topicName := "test_touch" + strconv.Itoa(int(time.Now().Unix()))

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identify(t, conn)
	sub(t, conn, topicName, "ch")

	topic := nsqd.GetTopic(topicName)
	channel := topic.GetChannel("ch")
	msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
	topic.PutMessage(msg)

	err = nsq.Ready(1).Write(conn)
	assert.Equal(t, err, nil)

	resp, err := nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)
	frameType, data, err := nsq.UnpackResponse(resp)
	msgOut, _ := nsq.DecodeMessage(data)
	assert.Equal(t, frameType, nsq.FrameTypeMessage)
	assert.Equal(t, msgOut.Id, msg.Id)

	time.Sleep(25 * time.Millisecond)

	err = nsq.Touch(msg.Id).Write(conn)
	assert.Equal(t, err, nil)

	time.Sleep(30 * time.Millisecond)

	err = nsq.Finish(msg.Id).Write(conn)
	assert.Equal(t, err, nil)

	assert.Equal(t, channel.timeoutCount, uint64(0))
}

func TestMaxRdyCount(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	*verbose = true
	options := NewNsqdOptions()
	options.maxRdyCount = 50
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	topicName := "test_max_rdy_count" + strconv.Itoa(int(time.Now().Unix()))

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	topic := nsqd.GetTopic(topicName)
	msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
	topic.PutMessage(msg)

	data := identifyFeatureNegotiation(t, conn, nil)
	r := struct {
		MaxRdyCount int64 `json:"max_rdy_count"`
	}{}
	err = json.Unmarshal(data, &r)
	assert.Equal(t, err, nil)
	assert.Equal(t, r.MaxRdyCount, int64(50))
	sub(t, conn, topicName, "ch")

	err = nsq.Ready(int(options.maxRdyCount)).Write(conn)
	assert.Equal(t, err, nil)

	resp, err := nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)
	frameType, data, err := nsq.UnpackResponse(resp)
	msgOut, _ := nsq.DecodeMessage(data)
	assert.Equal(t, frameType, nsq.FrameTypeMessage)
	assert.Equal(t, msgOut.Id, msg.Id)

	err = nsq.Ready(int(options.maxRdyCount) + 1).Write(conn)
	assert.Equal(t, err, nil)

	resp, err = nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)
	frameType, data, err = nsq.UnpackResponse(resp)
	assert.Equal(t, frameType, int32(1))
	assert.Equal(t, string(data), "E_INVALID RDY count 51 out of range 0-50")
}

func TestFatalError(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	tcpAddr, _, nsqd := mustStartNSQd(NewNsqdOptions())
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	_, err = conn.Write([]byte("ASDF\n"))
	assert.Equal(t, err, nil)

	resp, err := nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)
	frameType, data, err := nsq.UnpackResponse(resp)
	assert.Equal(t, frameType, int32(1))
	assert.Equal(t, string(data), "E_INVALID invalid command ASDF")

	_, err = nsq.ReadResponse(conn)
	assert.NotEqual(t, err, nil)
}

func TestOutputBuffering(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	*verbose = true
	options := NewNsqdOptions()
	options.maxOutputBufferSize = 512 * 1024
	options.maxOutputBufferTimeout = time.Second
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	topicName := "test_output_buffering" + strconv.Itoa(int(time.Now().Unix()))

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	outputBufferSize := 256 * 1024
	outputBufferTimeout := 500

	topic := nsqd.GetTopic(topicName)
	msg := nsq.NewMessage(<-nsqd.idChan, make([]byte, outputBufferSize-1024))
	topic.PutMessage(msg)

	start := time.Now()
	identifyOutputBuffering(t, conn, outputBufferSize, outputBufferTimeout, nsq.FrameTypeResponse, "OK")
	sub(t, conn, topicName, "ch")

	err = nsq.Ready(10).Write(conn)
	assert.Equal(t, err, nil)

	resp, err := nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)
	end := time.Now()

	assert.Equal(t, int(end.Sub(start)/time.Millisecond) >= outputBufferTimeout, true)

	frameType, data, err := nsq.UnpackResponse(resp)
	msgOut, _ := nsq.DecodeMessage(data)
	assert.Equal(t, frameType, nsq.FrameTypeMessage)
	assert.Equal(t, msgOut.Id, msg.Id)
}

func TestOutputBufferingValidity(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	*verbose = true
	options := NewNsqdOptions()
	options.maxOutputBufferSize = 512 * 1024
	options.maxOutputBufferTimeout = time.Second
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identifyOutputBuffering(t, conn, 512*1024, 1000, nsq.FrameTypeResponse, "OK")
	identifyOutputBuffering(t, conn, -1, -1, nsq.FrameTypeResponse, "OK")
	identifyOutputBuffering(t, conn, 0, 0, nsq.FrameTypeResponse, "OK")
	identifyOutputBuffering(t, conn, 512*1024+1, 0, nsq.FrameTypeError, fmt.Sprintf("E_BAD_BODY IDENTIFY output buffer size (%d) is invalid", 512*1024+1))

	conn, err = mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	identifyOutputBuffering(t, conn, 0, 1001, nsq.FrameTypeError, "E_BAD_BODY IDENTIFY output buffer timeout (1001) is invalid")
}

func TestTLS(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	*verbose = true
	options := NewNsqdOptions()
	options.tlsCert = "./test/cert.pem"
	options.tlsKey = "./test/key.pem"
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"tls_v1": true})
	r := struct {
		TLSv1 bool `json:"tls_v1"`
	}{}
	err = json.Unmarshal(data, &r)
	assert.Equal(t, err, nil)
	assert.Equal(t, r.TLSv1, true)

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
	}
	tlsConn := tls.Client(conn, tlsConfig)

	err = tlsConn.Handshake()
	assert.Equal(t, err, nil)

	resp, _ := nsq.ReadResponse(tlsConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeResponse)
	assert.Equal(t, data, []byte("OK"))
}

func TestDeflate(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	*verbose = true
	options := NewNsqdOptions()
	options.deflateEnabled = true
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"deflate": true})
	r := struct {
		Deflate bool `json:"deflate"`
	}{}
	err = json.Unmarshal(data, &r)
	assert.Equal(t, err, nil)
	assert.Equal(t, r.Deflate, true)

	compressConn := flate.NewReader(conn)
	resp, _ := nsq.ReadResponse(compressConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeResponse)
	assert.Equal(t, data, []byte("OK"))
}

type readWriter struct {
	io.Reader
	io.Writer
}

func TestSnappy(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	*verbose = true
	options := NewNsqdOptions()
	options.snappyEnabled = true
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"snappy": true})
	r := struct {
		Snappy bool `json:"snappy"`
	}{}
	err = json.Unmarshal(data, &r)
	assert.Equal(t, err, nil)
	assert.Equal(t, r.Snappy, true)

	compressConn := snappystream.NewReader(conn, snappystream.SkipVerifyChecksum)
	resp, _ := nsq.ReadResponse(compressConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeResponse)
	assert.Equal(t, data, []byte("OK"))

	msgBody := make([]byte, 128000)
	w := snappystream.NewWriter(conn)

	rw := readWriter{compressConn, w}

	topicName := "test_snappy" + strconv.Itoa(int(time.Now().Unix()))
	sub(t, rw, topicName, "ch")

	err = nsq.Ready(1).Write(rw)
	assert.Equal(t, err, nil)

	topic := nsqd.GetTopic(topicName)
	msg := nsq.NewMessage(<-nsqd.idChan, msgBody)
	topic.PutMessage(msg)

	resp, _ = nsq.ReadResponse(compressConn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	msgOut, _ := nsq.DecodeMessage(data)
	assert.Equal(t, frameType, nsq.FrameTypeMessage)
	assert.Equal(t, msgOut.Id, msg.Id)
	assert.Equal(t, msgOut.Body, msg.Body)
}

func TestTLSDeflate(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	*verbose = true
	options := NewNsqdOptions()
	options.deflateEnabled = true
	options.tlsCert = "./test/cert.pem"
	options.tlsKey = "./test/key.pem"
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"tls_v1": true, "deflate": true})
	r := struct {
		TLSv1   bool `json:"tls_v1"`
		Deflate bool `json:"deflate"`
	}{}
	err = json.Unmarshal(data, &r)
	assert.Equal(t, err, nil)
	assert.Equal(t, r.TLSv1, true)
	assert.Equal(t, r.Deflate, true)

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
	}
	tlsConn := tls.Client(conn, tlsConfig)

	err = tlsConn.Handshake()
	assert.Equal(t, err, nil)

	resp, _ := nsq.ReadResponse(tlsConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeResponse)
	assert.Equal(t, data, []byte("OK"))

	compressConn := flate.NewReader(tlsConn)

	resp, _ = nsq.ReadResponse(compressConn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeResponse)
	assert.Equal(t, data, []byte("OK"))
}

func TestSampling(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	rand.Seed(time.Now().UTC().UnixNano())

	num := 3000

	*verbose = true
	options := NewNsqdOptions()
	options.maxRdyCount = int64(num)
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"sample_rate": int32(42)})
	r := struct {
		SampleRate int32 `json:"sample_rate"`
	}{}
	err = json.Unmarshal(data, &r)
	assert.Equal(t, err, nil)
	assert.Equal(t, r.SampleRate, int32(42))

	topicName := "test_sampling" + strconv.Itoa(int(time.Now().Unix()))
	topic := nsqd.GetTopic(topicName)
	for i := 0; i < num; i++ {
		msg := nsq.NewMessage(<-nsqd.idChan, []byte("test body"))
		topic.PutMessage(msg)
	}
	channel := topic.GetChannel("ch")

	// let the topic drain into the channel
	time.Sleep(50 * time.Millisecond)

	sub(t, conn, topicName, "ch")
	err = nsq.Ready(num).Write(conn)
	assert.Equal(t, err, nil)

	doneChan := make(chan int)
	go func() {
		for {
			if channel.Depth() == 0 {
				close(doneChan)
				return
			}
			time.Sleep(5 * time.Millisecond)
		}
	}()
	<-doneChan
	// within 3%
	assert.Equal(t, len(channel.inFlightMessages) <= int(float64(num)*0.45), true)
	assert.Equal(t, len(channel.inFlightMessages) >= int(float64(num)*0.39), true)
}

func TestTLSSnappy(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	*verbose = true
	options := NewNsqdOptions()
	options.snappyEnabled = true
	options.tlsCert = "./test/cert.pem"
	options.tlsKey = "./test/key.pem"
	tcpAddr, _, nsqd := mustStartNSQd(options)
	defer nsqd.Exit()

	conn, err := mustConnectNSQd(tcpAddr)
	assert.Equal(t, err, nil)

	data := identifyFeatureNegotiation(t, conn, map[string]interface{}{"tls_v1": true, "snappy": true})
	r := struct {
		TLSv1  bool `json:"tls_v1"`
		Snappy bool `json:"snappy"`
	}{}
	err = json.Unmarshal(data, &r)
	assert.Equal(t, err, nil)
	assert.Equal(t, r.TLSv1, true)
	assert.Equal(t, r.Snappy, true)

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
	}
	tlsConn := tls.Client(conn, tlsConfig)

	err = tlsConn.Handshake()
	assert.Equal(t, err, nil)

	resp, _ := nsq.ReadResponse(tlsConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeResponse)
	assert.Equal(t, data, []byte("OK"))

	compressConn := snappystream.NewReader(tlsConn, snappystream.SkipVerifyChecksum)

	resp, _ = nsq.ReadResponse(compressConn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	log.Printf("frameType: %d, data: %s", frameType, data)
	assert.Equal(t, frameType, nsq.FrameTypeResponse)
	assert.Equal(t, data, []byte("OK"))
}

func BenchmarkProtocolV2Exec(b *testing.B) {
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	p := &ProtocolV2{}
	c := NewClientV2(0, nil, nil)
	params := [][]byte{[]byte("NOP")}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		p.Exec(c, params)
	}
}

func benchmarkProtocolV2Pub(b *testing.B, size int) {
	var wg sync.WaitGroup
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	options := NewNsqdOptions()
	options.memQueueSize = int64(b.N)
	tcpAddr, _, nsqd := mustStartNSQd(options)
	msg := make([]byte, size)
	batchSize := 200
	batch := make([][]byte, 0)
	for i := 0; i < batchSize; i++ {
		batch = append(batch, msg)
	}
	topicName := "bench_v2_pub" + strconv.Itoa(int(time.Now().Unix()))
	b.SetBytes(int64(len(msg)))
	b.StartTimer()

	for j := 0; j < runtime.GOMAXPROCS(0); j++ {
		wg.Add(1)
		go func() {
			conn, err := mustConnectNSQd(tcpAddr)
			if err != nil {
				panic(err.Error())
			}
			rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))

			num := b.N / runtime.GOMAXPROCS(0) / batchSize
			for i := 0; i < num; i += 1 {
				cmd, _ := nsq.MultiPublish(topicName, batch)
				err := cmd.Write(rw)
				if err != nil {
					panic(err.Error())
				}
				err = rw.Flush()
				if err != nil {
					panic(err.Error())
				}
				resp, err := nsq.ReadResponse(rw)
				if err != nil {
					panic(err.Error())
				}
				_, data, _ := nsq.UnpackResponse(resp)
				if !bytes.Equal(data, []byte("OK")) {
					panic("invalid response")
				}
			}
			wg.Done()
		}()
	}

	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}

func BenchmarkProtocolV2Pub256(b *testing.B)  { benchmarkProtocolV2Pub(b, 256) }
func BenchmarkProtocolV2Pub512(b *testing.B)  { benchmarkProtocolV2Pub(b, 512) }
func BenchmarkProtocolV2Pub1k(b *testing.B)   { benchmarkProtocolV2Pub(b, 1024) }
func BenchmarkProtocolV2Pub2k(b *testing.B)   { benchmarkProtocolV2Pub(b, 2*1024) }
func BenchmarkProtocolV2Pub4k(b *testing.B)   { benchmarkProtocolV2Pub(b, 4*1024) }
func BenchmarkProtocolV2Pub8k(b *testing.B)   { benchmarkProtocolV2Pub(b, 8*1024) }
func BenchmarkProtocolV2Pub16k(b *testing.B)  { benchmarkProtocolV2Pub(b, 16*1024) }
func BenchmarkProtocolV2Pub32k(b *testing.B)  { benchmarkProtocolV2Pub(b, 32*1024) }
func BenchmarkProtocolV2Pub64k(b *testing.B)  { benchmarkProtocolV2Pub(b, 64*1024) }
func BenchmarkProtocolV2Pub128k(b *testing.B) { benchmarkProtocolV2Pub(b, 128*1024) }
func BenchmarkProtocolV2Pub256k(b *testing.B) { benchmarkProtocolV2Pub(b, 256*1024) }
func BenchmarkProtocolV2Pub512k(b *testing.B) { benchmarkProtocolV2Pub(b, 512*1024) }
func BenchmarkProtocolV2Pub1m(b *testing.B)   { benchmarkProtocolV2Pub(b, 1024*1024) }

func benchmarkProtocolV2Sub(b *testing.B, size int) {
	var wg sync.WaitGroup
	b.StopTimer()
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)
	options := NewNsqdOptions()
	options.memQueueSize = int64(b.N)
	tcpAddr, _, nsqd := mustStartNSQd(options)
	msg := make([]byte, size)
	topicName := "bench_v2_sub" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
	topic := nsqd.GetTopic(topicName)
	for i := 0; i < b.N; i++ {
		msg := nsq.NewMessage(<-nsqd.idChan, msg)
		topic.PutMessage(msg)
	}
	topic.GetChannel("ch")
	b.SetBytes(int64(len(msg)))
	goChan := make(chan int)
	rdyChan := make(chan int)
	workers := runtime.GOMAXPROCS(0)
	for j := 0; j < workers; j++ {
		wg.Add(1)
		go func() {
			subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
			wg.Done()
		}()
		<-rdyChan
	}
	b.StartTimer()

	close(goChan)
	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}

func subWorker(n int, workers int, tcpAddr *net.TCPAddr, topicName string, rdyChan chan int, goChan chan int) {
	conn, err := mustConnectNSQd(tcpAddr)
	if err != nil {
		panic(err.Error())
	}
	rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))

	identify(nil, conn)
	sub(nil, conn, topicName, "ch")

	rdyCount := int(math.Min(math.Max(float64(n/workers), 1), 2500))
	rdyChan <- 1
	<-goChan
	nsq.Ready(rdyCount).Write(rw)
	rw.Flush()
	num := n / workers
	numRdy := num/rdyCount - 1
	rdy := rdyCount
	for i := 0; i < num; i += 1 {
		resp, err := nsq.ReadResponse(rw)
		if err != nil {
			panic(err.Error())
		}
		frameType, data, err := nsq.UnpackResponse(resp)
		if err != nil {
			panic(err.Error())
		}
		if frameType != nsq.FrameTypeMessage {
			panic("got something else")
		}
		msg, err := nsq.DecodeMessage(data)
		if err != nil {
			panic(err.Error())
		}
		nsq.Finish(msg.Id).Write(rw)
		rdy--
		if rdy == 0 && numRdy > 0 {
			nsq.Ready(rdyCount).Write(rw)
			rdy = rdyCount
			numRdy--
			rw.Flush()
		}
	}
}

func BenchmarkProtocolV2Sub256(b *testing.B)  { benchmarkProtocolV2Sub(b, 256) }
func BenchmarkProtocolV2Sub512(b *testing.B)  { benchmarkProtocolV2Sub(b, 512) }
func BenchmarkProtocolV2Sub1k(b *testing.B)   { benchmarkProtocolV2Sub(b, 1024) }
func BenchmarkProtocolV2Sub2k(b *testing.B)   { benchmarkProtocolV2Sub(b, 2*1024) }
func BenchmarkProtocolV2Sub4k(b *testing.B)   { benchmarkProtocolV2Sub(b, 4*1024) }
func BenchmarkProtocolV2Sub8k(b *testing.B)   { benchmarkProtocolV2Sub(b, 8*1024) }
func BenchmarkProtocolV2Sub16k(b *testing.B)  { benchmarkProtocolV2Sub(b, 16*1024) }
func BenchmarkProtocolV2Sub32k(b *testing.B)  { benchmarkProtocolV2Sub(b, 32*1024) }
func BenchmarkProtocolV2Sub64k(b *testing.B)  { benchmarkProtocolV2Sub(b, 64*1024) }
func BenchmarkProtocolV2Sub128k(b *testing.B) { benchmarkProtocolV2Sub(b, 128*1024) }
func BenchmarkProtocolV2Sub256k(b *testing.B) { benchmarkProtocolV2Sub(b, 256*1024) }
func BenchmarkProtocolV2Sub512k(b *testing.B) { benchmarkProtocolV2Sub(b, 512*1024) }
func BenchmarkProtocolV2Sub1m(b *testing.B)   { benchmarkProtocolV2Sub(b, 1024*1024) }

func benchmarkProtocolV2MultiSub(b *testing.B, num int) {
	var wg sync.WaitGroup
	b.StopTimer()

	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	options := NewNsqdOptions()
	options.memQueueSize = int64(b.N)
	tcpAddr, _, nsqd := mustStartNSQd(options)
	msg := make([]byte, 256)
	b.SetBytes(int64(len(msg) * num))

	goChan := make(chan int)
	rdyChan := make(chan int)
	workers := runtime.GOMAXPROCS(0)
	for i := 0; i < num; i++ {
		topicName := "bench_v2" + strconv.Itoa(b.N) + "_" + strconv.Itoa(i) + "_" + strconv.Itoa(int(time.Now().Unix()))
		topic := nsqd.GetTopic(topicName)
		for i := 0; i < b.N; i++ {
			msg := nsq.NewMessage(<-nsqd.idChan, msg)
			topic.PutMessage(msg)
		}
		topic.GetChannel("ch")

		for j := 0; j < workers; j++ {
			wg.Add(1)
			go func() {
				subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
				wg.Done()
			}()
			<-rdyChan
		}
	}
	b.StartTimer()

	close(goChan)
	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}

func BenchmarkProtocolV2MultiSub1(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 1) }
func BenchmarkProtocolV2MultiSub2(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 2) }
func BenchmarkProtocolV2MultiSub4(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 4) }
func BenchmarkProtocolV2MultiSub8(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 8) }
func BenchmarkProtocolV2MultiSub16(b *testing.B) { benchmarkProtocolV2MultiSub(b, 16) }
Пример #20
0
func TestClientTimeout(t *testing.T) {
	topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix()))

	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.ClientTimeout = 150 * time.Millisecond
	opts.Verbose = true
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	identify(t, conn, nil, frameTypeResponse)
	sub(t, conn, topicName, "ch")

	time.Sleep(150 * time.Millisecond)

	// depending on timing there may be 1 or 2 hearbeats sent
	// just read until we get an error
	timer := time.After(100 * time.Millisecond)
	for {
		select {
		case <-timer:
			t.Fatalf("test timed out")
		default:
			_, err := nsq.ReadResponse(conn)
			if err != nil {
				goto done
			}
		}
	}
done:
}

func TestClientHeartbeat(t *testing.T) {
	topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))

	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.ClientTimeout = 200 * time.Millisecond
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	identify(t, conn, nil, frameTypeResponse)
	sub(t, conn, topicName, "ch")

	_, err = nsq.Ready(1).WriteTo(conn)
	equal(t, err, nil)

	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	equal(t, data, []byte("_heartbeat_"))

	time.Sleep(20 * time.Millisecond)

	_, err = nsq.Nop().WriteTo(conn)
	equal(t, err, nil)

	// wait long enough that would have timed out (had we not sent the above cmd)
	time.Sleep(100 * time.Millisecond)

	_, err = nsq.Nop().WriteTo(conn)
	equal(t, err, nil)
}

func TestClientHeartbeatDisableSUB(t *testing.T) {
	topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix()))

	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.ClientTimeout = 200 * time.Millisecond
	opts.Verbose = true
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	identify(t, conn, map[string]interface{}{
		"heartbeat_interval": -1,
	}, frameTypeResponse)
	subFail(t, conn, topicName, "ch")
}

func TestClientHeartbeatDisable(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.ClientTimeout = 100 * time.Millisecond
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	identify(t, conn, map[string]interface{}{
		"heartbeat_interval": -1,
	}, frameTypeResponse)

	time.Sleep(150 * time.Millisecond)

	_, err = nsq.Nop().WriteTo(conn)
	equal(t, err, nil)
}

func TestMaxHeartbeatIntervalValid(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.MaxHeartbeatInterval = 300 * time.Second
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	hbi := int(opts.MaxHeartbeatInterval / time.Millisecond)
	identify(t, conn, map[string]interface{}{
		"heartbeat_interval": hbi,
	}, frameTypeResponse)
}

func TestMaxHeartbeatIntervalInvalid(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.MaxHeartbeatInterval = 300 * time.Second
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	hbi := int(opts.MaxHeartbeatInterval/time.Millisecond + 1)
	data := identify(t, conn, map[string]interface{}{
		"heartbeat_interval": hbi,
	}, frameTypeError)
	equal(t, string(data), "E_BAD_BODY IDENTIFY heartbeat interval (300001) is invalid")
}

func TestPausing(t *testing.T) {
	topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix()))

	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	identify(t, conn, nil, frameTypeResponse)
	sub(t, conn, topicName, "ch")

	_, err = nsq.Ready(1).WriteTo(conn)
	equal(t, err, nil)

	topic := nsqd.GetTopic(topicName)
	msg := NewMessage(<-nsqd.idChan, []byte("test body"))
	channel := topic.GetChannel("ch")
	topic.PutMessage(msg)

	// receive the first message via the client, finish it, and send new RDY
	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	msg, err = decodeMessage(data)
	equal(t, msg.Body, []byte("test body"))

	_, err = nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(conn)
	equal(t, err, nil)

	_, err = nsq.Ready(1).WriteTo(conn)
	equal(t, err, nil)

	// sleep to allow the RDY state to take effect
	time.Sleep(50 * time.Millisecond)

	// pause the channel... the client shouldn't receive any more messages
	channel.Pause()

	// sleep to allow the paused state to take effect
	time.Sleep(50 * time.Millisecond)

	msg = NewMessage(<-nsqd.idChan, []byte("test body2"))
	topic.PutMessage(msg)

	// allow the client to possibly get a message, the test would hang indefinitely
	// if pausing was not working on the internal clientMsgChan read
	time.Sleep(50 * time.Millisecond)
	msg = <-channel.clientMsgChan
	equal(t, msg.Body, []byte("test body2"))

	// unpause the channel... the client should now be pushed a message
	channel.UnPause()

	msg = NewMessage(<-nsqd.idChan, []byte("test body3"))
	topic.PutMessage(msg)

	resp, _ = nsq.ReadResponse(conn)
	_, data, _ = nsq.UnpackResponse(resp)
	msg, err = decodeMessage(data)
	equal(t, msg.Body, []byte("test body3"))
}

func TestEmptyCommand(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	_, err = conn.Write([]byte("\n\n"))
	equal(t, err, nil)

	// if we didn't panic here we're good, see issue #120
}

func TestSizeLimits(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.MaxMsgSize = 100
	opts.MaxBodySize = 1000
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	topicName := "test_limits_v2" + strconv.Itoa(int(time.Now().Unix()))

	identify(t, conn, nil, frameTypeResponse)
	sub(t, conn, topicName, "ch")

	// PUB that's valid
	nsq.Publish(topicName, make([]byte, 95)).WriteTo(conn)
	resp, _ := nsq.ReadResponse(conn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))

	// PUB that's invalid (too big)
	nsq.Publish(topicName, make([]byte, 105)).WriteTo(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeError)
	equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE PUB message too big 105 > 100"))

	// need to reconnect
	conn, err = mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	// PUB thats empty
	nsq.Publish(topicName, []byte{}).WriteTo(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeError)
	equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE PUB invalid message body size 0"))

	// need to reconnect
	conn, err = mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	// MPUB body that's valid
	mpub := make([][]byte, 5)
	for i := range mpub {
		mpub[i] = make([]byte, 100)
	}
	cmd, _ := nsq.MultiPublish(topicName, mpub)
	cmd.WriteTo(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))

	// MPUB body that's invalid (body too big)
	mpub = make([][]byte, 11)
	for i := range mpub {
		mpub[i] = make([]byte, 100)
	}
	cmd, _ = nsq.MultiPublish(topicName, mpub)
	cmd.WriteTo(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeError)
	equal(t, string(data), fmt.Sprintf("E_BAD_BODY MPUB body too big 1148 > 1000"))

	// need to reconnect
	conn, err = mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	// MPUB that's invalid (one message empty)
	mpub = make([][]byte, 5)
	for i := range mpub {
		mpub[i] = make([]byte, 100)
	}
	mpub = append(mpub, []byte{})
	cmd, _ = nsq.MultiPublish(topicName, mpub)
	cmd.WriteTo(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeError)
	equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE MPUB invalid message(5) body size 0"))

	// need to reconnect
	conn, err = mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	// MPUB body that's invalid (one of the messages is too big)
	mpub = make([][]byte, 5)
	for i := range mpub {
		mpub[i] = make([]byte, 101)
	}
	cmd, _ = nsq.MultiPublish(topicName, mpub)
	cmd.WriteTo(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeError)
	equal(t, string(data), fmt.Sprintf("E_BAD_MESSAGE MPUB message too big 101 > 100"))
}

func TestDPUB(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	topicName := "test_dpub_v2" + strconv.Itoa(int(time.Now().Unix()))

	identify(t, conn, nil, frameTypeResponse)
	sub(t, conn, topicName, "ch")

	// valid
	nsq.DeferredPublish(topicName, time.Second, make([]byte, 100)).WriteTo(conn)
	resp, _ := nsq.ReadResponse(conn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))

	time.Sleep(25 * time.Millisecond)

	ch := nsqd.GetTopic(topicName).GetChannel("ch")
	ch.Lock()
	numDef := len(ch.deferredMessages)
	ch.Unlock()
	equal(t, numDef, 1)

	// duration out of range
	nsq.DeferredPublish(topicName, opts.MaxReqTimeout+100*time.Millisecond, make([]byte, 100)).WriteTo(conn)
	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeError)
	equal(t, string(data), fmt.Sprintf("E_INVALID DPUB timeout 3600100 out of range 0-3600000"))
}

func TestTouch(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.MsgTimeout = 150 * time.Millisecond
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	topicName := "test_touch" + strconv.Itoa(int(time.Now().Unix()))

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	identify(t, conn, nil, frameTypeResponse)
	sub(t, conn, topicName, "ch")

	topic := nsqd.GetTopic(topicName)
	channel := topic.GetChannel("ch")
	msg := NewMessage(<-nsqd.idChan, []byte("test body"))
	topic.PutMessage(msg)

	_, err = nsq.Ready(1).WriteTo(conn)
	equal(t, err, nil)

	resp, err := nsq.ReadResponse(conn)
	equal(t, err, nil)
	frameType, data, err := nsq.UnpackResponse(resp)
	msgOut, _ := decodeMessage(data)
	equal(t, frameType, frameTypeMessage)
	equal(t, msgOut.ID, msg.ID)

	time.Sleep(75 * time.Millisecond)

	_, err = nsq.Touch(nsq.MessageID(msg.ID)).WriteTo(conn)
	equal(t, err, nil)

	time.Sleep(75 * time.Millisecond)

	_, err = nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(conn)
	equal(t, err, nil)

	equal(t, channel.timeoutCount, uint64(0))
}

func TestMaxRdyCount(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.MaxRdyCount = 50
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	topicName := "test_max_rdy_count" + strconv.Itoa(int(time.Now().Unix()))

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	topic := nsqd.GetTopic(topicName)
	msg := NewMessage(<-nsqd.idChan, []byte("test body"))
	topic.PutMessage(msg)

	data := identify(t, conn, nil, frameTypeResponse)
	r := struct {
		MaxRdyCount int64 `json:"max_rdy_count"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.MaxRdyCount, int64(50))
	sub(t, conn, topicName, "ch")

	_, err = nsq.Ready(int(opts.MaxRdyCount)).WriteTo(conn)
	equal(t, err, nil)

	resp, err := nsq.ReadResponse(conn)
	equal(t, err, nil)
	frameType, data, err := nsq.UnpackResponse(resp)
	msgOut, _ := decodeMessage(data)
	equal(t, frameType, frameTypeMessage)
	equal(t, msgOut.ID, msg.ID)

	_, err = nsq.Ready(int(opts.MaxRdyCount) + 1).WriteTo(conn)
	equal(t, err, nil)

	resp, err = nsq.ReadResponse(conn)
	equal(t, err, nil)
	frameType, data, err = nsq.UnpackResponse(resp)
	equal(t, frameType, int32(1))
	equal(t, string(data), "E_INVALID RDY count 51 out of range 0-50")
}

func TestFatalError(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	_, err = conn.Write([]byte("ASDF\n"))
	equal(t, err, nil)

	resp, err := nsq.ReadResponse(conn)
	equal(t, err, nil)
	frameType, data, err := nsq.UnpackResponse(resp)
	equal(t, frameType, int32(1))
	equal(t, string(data), "E_INVALID invalid command ASDF")

	_, err = nsq.ReadResponse(conn)
	nequal(t, err, nil)
}

func TestOutputBuffering(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.MaxOutputBufferSize = 512 * 1024
	opts.MaxOutputBufferTimeout = time.Second
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	topicName := "test_output_buffering" + strconv.Itoa(int(time.Now().Unix()))

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	outputBufferSize := 256 * 1024
	outputBufferTimeout := 500

	topic := nsqd.GetTopic(topicName)
	msg := NewMessage(<-nsqd.idChan, make([]byte, outputBufferSize-1024))
	topic.PutMessage(msg)

	start := time.Now()
	data := identify(t, conn, map[string]interface{}{
		"output_buffer_size":    outputBufferSize,
		"output_buffer_timeout": outputBufferTimeout,
	}, frameTypeResponse)
	var decoded map[string]interface{}
	json.Unmarshal(data, &decoded)
	v, ok := decoded["output_buffer_size"]
	equal(t, ok, true)
	equal(t, int(v.(float64)), outputBufferSize)
	v, ok = decoded["output_buffer_timeout"]
	equal(t, int(v.(float64)), outputBufferTimeout)
	sub(t, conn, topicName, "ch")

	_, err = nsq.Ready(10).WriteTo(conn)
	equal(t, err, nil)

	resp, err := nsq.ReadResponse(conn)
	equal(t, err, nil)
	end := time.Now()

	equal(t, int(end.Sub(start)/time.Millisecond) >= outputBufferTimeout, true)

	frameType, data, err := nsq.UnpackResponse(resp)
	msgOut, _ := decodeMessage(data)
	equal(t, frameType, frameTypeMessage)
	equal(t, msgOut.ID, msg.ID)
}

func TestOutputBufferingValidity(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.MaxOutputBufferSize = 512 * 1024
	opts.MaxOutputBufferTimeout = time.Second
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	identify(t, conn, map[string]interface{}{
		"output_buffer_size":    512 * 1024,
		"output_buffer_timeout": 1000,
	}, frameTypeResponse)
	identify(t, conn, map[string]interface{}{
		"output_buffer_size":    -1,
		"output_buffer_timeout": -1,
	}, frameTypeResponse)
	identify(t, conn, map[string]interface{}{
		"output_buffer_size":    0,
		"output_buffer_timeout": 0,
	}, frameTypeResponse)
	data := identify(t, conn, map[string]interface{}{
		"output_buffer_size":    512*1024 + 1,
		"output_buffer_timeout": 0,
	}, frameTypeError)
	equal(t, string(data), fmt.Sprintf("E_BAD_BODY IDENTIFY output buffer size (%d) is invalid", 512*1024+1))

	conn, err = mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data = identify(t, conn, map[string]interface{}{
		"output_buffer_size":    0,
		"output_buffer_timeout": 1001,
	}, frameTypeError)
	equal(t, string(data), "E_BAD_BODY IDENTIFY output buffer timeout (1001) is invalid")
}

func TestTLS(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.TLSCert = "./test/certs/server.pem"
	opts.TLSKey = "./test/certs/server.key"
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data := identify(t, conn, map[string]interface{}{
		"tls_v1": true,
	}, frameTypeResponse)
	r := struct {
		TLSv1 bool `json:"tls_v1"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.TLSv1, true)

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
	}
	tlsConn := tls.Client(conn, tlsConfig)

	err = tlsConn.Handshake()
	equal(t, err, nil)

	resp, _ := nsq.ReadResponse(tlsConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))
}

func TestTLSRequired(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.TLSCert = "./test/certs/server.pem"
	opts.TLSKey = "./test/certs/server.key"
	opts.TLSRequired = TLSRequiredExceptHTTP

	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	topicName := "test_tls_required" + strconv.Itoa(int(time.Now().Unix()))

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	subFail(t, conn, topicName, "ch")

	conn, err = mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data := identify(t, conn, map[string]interface{}{
		"tls_v1": true,
	}, frameTypeResponse)
	r := struct {
		TLSv1 bool `json:"tls_v1"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.TLSv1, true)

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
	}
	tlsConn := tls.Client(conn, tlsConfig)

	err = tlsConn.Handshake()
	equal(t, err, nil)

	resp, _ := nsq.ReadResponse(tlsConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))
}

func TestTLSAuthRequire(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.TLSCert = "./test/certs/server.pem"
	opts.TLSKey = "./test/certs/server.key"
	opts.TLSClientAuthPolicy = "require"

	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	// No Certs
	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data := identify(t, conn, map[string]interface{}{
		"tls_v1": true,
	}, frameTypeResponse)
	r := struct {
		TLSv1 bool `json:"tls_v1"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.TLSv1, true)
	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
	}
	tlsConn := tls.Client(conn, tlsConfig)
	err = tlsConn.Handshake()
	nequal(t, err, nil)

	// With Unsigned Cert
	conn, err = mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data = identify(t, conn, map[string]interface{}{
		"tls_v1": true,
	}, frameTypeResponse)
	r = struct {
		TLSv1 bool `json:"tls_v1"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.TLSv1, true)

	cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem")
	equal(t, err, nil)
	tlsConfig = &tls.Config{
		Certificates:       []tls.Certificate{cert},
		InsecureSkipVerify: true,
	}
	tlsConn = tls.Client(conn, tlsConfig)
	err = tlsConn.Handshake()
	equal(t, err, nil)

	resp, _ := nsq.ReadResponse(tlsConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))

}

func TestTLSAuthRequireVerify(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.TLSCert = "./test/certs/server.pem"
	opts.TLSKey = "./test/certs/server.key"
	opts.TLSRootCAFile = "./test/certs/ca.pem"
	opts.TLSClientAuthPolicy = "require-verify"

	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	// with no cert
	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data := identify(t, conn, map[string]interface{}{
		"tls_v1": true,
	}, frameTypeResponse)
	r := struct {
		TLSv1 bool `json:"tls_v1"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.TLSv1, true)
	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
	}
	tlsConn := tls.Client(conn, tlsConfig)
	err = tlsConn.Handshake()
	nequal(t, err, nil)

	// with invalid cert
	conn, err = mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data = identify(t, conn, map[string]interface{}{
		"tls_v1": true,
	}, frameTypeResponse)
	r = struct {
		TLSv1 bool `json:"tls_v1"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.TLSv1, true)
	cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem")
	equal(t, err, nil)
	tlsConfig = &tls.Config{
		Certificates:       []tls.Certificate{cert},
		InsecureSkipVerify: true,
	}
	tlsConn = tls.Client(conn, tlsConfig)
	err = tlsConn.Handshake()
	nequal(t, err, nil)

	// with valid cert
	conn, err = mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data = identify(t, conn, map[string]interface{}{
		"tls_v1": true,
	}, frameTypeResponse)
	r = struct {
		TLSv1 bool `json:"tls_v1"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.TLSv1, true)
	cert, err = tls.LoadX509KeyPair("./test/certs/client.pem", "./test/certs/client.key")
	equal(t, err, nil)
	tlsConfig = &tls.Config{
		Certificates:       []tls.Certificate{cert},
		InsecureSkipVerify: true,
	}
	tlsConn = tls.Client(conn, tlsConfig)
	err = tlsConn.Handshake()
	equal(t, err, nil)

	resp, _ := nsq.ReadResponse(tlsConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))
}

func TestDeflate(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.DeflateEnabled = true
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data := identify(t, conn, map[string]interface{}{
		"deflate": true,
	}, frameTypeResponse)
	r := struct {
		Deflate bool `json:"deflate"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.Deflate, true)

	compressConn := flate.NewReader(conn)
	resp, _ := nsq.ReadResponse(compressConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))
}

type readWriter struct {
	io.Reader
	io.Writer
}

func TestSnappy(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.SnappyEnabled = true
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data := identify(t, conn, map[string]interface{}{
		"snappy": true,
	}, frameTypeResponse)
	r := struct {
		Snappy bool `json:"snappy"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.Snappy, true)

	compressConn := snappystream.NewReader(conn, snappystream.SkipVerifyChecksum)
	resp, _ := nsq.ReadResponse(compressConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))

	msgBody := make([]byte, 128000)
	w := snappystream.NewWriter(conn)

	rw := readWriter{compressConn, w}

	topicName := "test_snappy" + strconv.Itoa(int(time.Now().Unix()))
	sub(t, rw, topicName, "ch")

	_, err = nsq.Ready(1).WriteTo(rw)
	equal(t, err, nil)

	topic := nsqd.GetTopic(topicName)
	msg := NewMessage(<-nsqd.idChan, msgBody)
	topic.PutMessage(msg)

	resp, _ = nsq.ReadResponse(compressConn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	msgOut, _ := decodeMessage(data)
	equal(t, frameType, frameTypeMessage)
	equal(t, msgOut.ID, msg.ID)
	equal(t, msgOut.Body, msg.Body)
}

func TestTLSDeflate(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.DeflateEnabled = true
	opts.TLSCert = "./test/certs/cert.pem"
	opts.TLSKey = "./test/certs/key.pem"
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data := identify(t, conn, map[string]interface{}{
		"tls_v1":  true,
		"deflate": true,
	}, frameTypeResponse)
	r := struct {
		TLSv1   bool `json:"tls_v1"`
		Deflate bool `json:"deflate"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.TLSv1, true)
	equal(t, r.Deflate, true)

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
	}
	tlsConn := tls.Client(conn, tlsConfig)

	err = tlsConn.Handshake()
	equal(t, err, nil)

	resp, _ := nsq.ReadResponse(tlsConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))

	compressConn := flate.NewReader(tlsConn)

	resp, _ = nsq.ReadResponse(compressConn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))
}

func TestSampling(t *testing.T) {
	rand.Seed(time.Now().UTC().UnixNano())

	num := 10000
	sampleRate := 42
	slack := 5

	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.MaxRdyCount = int64(num)
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data := identify(t, conn, map[string]interface{}{
		"sample_rate": int32(sampleRate),
	}, frameTypeResponse)
	r := struct {
		SampleRate int32 `json:"sample_rate"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.SampleRate, int32(sampleRate))

	topicName := "test_sampling" + strconv.Itoa(int(time.Now().Unix()))
	topic := nsqd.GetTopic(topicName)
	for i := 0; i < num; i++ {
		msg := NewMessage(<-nsqd.idChan, []byte("test body"))
		topic.PutMessage(msg)
	}
	channel := topic.GetChannel("ch")

	// let the topic drain into the channel
	time.Sleep(50 * time.Millisecond)

	sub(t, conn, topicName, "ch")
	_, err = nsq.Ready(num).WriteTo(conn)
	equal(t, err, nil)

	go func() {
		for {
			_, err := nsq.ReadResponse(conn)
			if err != nil {
				return
			}
		}
	}()

	doneChan := make(chan int)
	go func() {
		for {
			if channel.Depth() == 0 {
				close(doneChan)
				return
			}
			time.Sleep(5 * time.Millisecond)
		}
	}()
	<-doneChan

	channel.Lock()
	numInFlight := len(channel.inFlightMessages)
	channel.Unlock()

	equal(t, numInFlight <= int(float64(num)*float64(sampleRate+slack)/100.0), true)
	equal(t, numInFlight >= int(float64(num)*float64(sampleRate-slack)/100.0), true)
}

func TestTLSSnappy(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.SnappyEnabled = true
	opts.TLSCert = "./test/certs/cert.pem"
	opts.TLSKey = "./test/certs/key.pem"
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data := identify(t, conn, map[string]interface{}{
		"tls_v1": true,
		"snappy": true,
	}, frameTypeResponse)
	r := struct {
		TLSv1  bool `json:"tls_v1"`
		Snappy bool `json:"snappy"`
	}{}
	err = json.Unmarshal(data, &r)
	equal(t, err, nil)
	equal(t, r.TLSv1, true)
	equal(t, r.Snappy, true)

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
	}
	tlsConn := tls.Client(conn, tlsConfig)

	err = tlsConn.Handshake()
	equal(t, err, nil)

	resp, _ := nsq.ReadResponse(tlsConn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))

	compressConn := snappystream.NewReader(tlsConn, snappystream.SkipVerifyChecksum)

	resp, _ = nsq.ReadResponse(compressConn)
	frameType, data, _ = nsq.UnpackResponse(resp)
	t.Logf("frameType: %d, data: %s", frameType, data)
	equal(t, frameType, frameTypeResponse)
	equal(t, data, []byte("OK"))
}

func TestClientMsgTimeout(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.QueueScanRefreshInterval = 100 * time.Millisecond
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	topicName := "test_cmsg_timeout" + strconv.Itoa(int(time.Now().Unix()))
	topic := nsqd.GetTopic(topicName)
	msg := NewMessage(<-nsqd.idChan, make([]byte, 100))
	topic.PutMessage(msg)

	// without this the race detector thinks there's a write
	// to msg.Attempts that races with the read in the protocol's messagePump...
	// it does not reflect a realistically possible condition
	topic.PutMessage(NewMessage(<-nsqd.idChan, make([]byte, 100)))

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	identify(t, conn, map[string]interface{}{
		"msg_timeout": 1000,
	}, frameTypeResponse)
	sub(t, conn, topicName, "ch")

	_, err = nsq.Ready(1).WriteTo(conn)
	equal(t, err, nil)

	resp, _ := nsq.ReadResponse(conn)
	_, data, _ := nsq.UnpackResponse(resp)
	msgOut, err := decodeMessage(data)
	equal(t, msgOut.ID, msg.ID)
	equal(t, msgOut.Body, msg.Body)

	_, err = nsq.Ready(0).WriteTo(conn)
	equal(t, err, nil)

	time.Sleep(1100 * time.Millisecond)

	_, err = nsq.Finish(nsq.MessageID(msgOut.ID)).WriteTo(conn)
	equal(t, err, nil)

	resp, _ = nsq.ReadResponse(conn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	equal(t, frameType, frameTypeError)
	equal(t, string(data),
		fmt.Sprintf("E_FIN_FAILED FIN %s failed ID not in flight", msgOut.ID))
}

func TestBadFin(t *testing.T) {
	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	identify(t, conn, map[string]interface{}{}, frameTypeResponse)
	sub(t, conn, "test_fin", "ch")

	fin := nsq.Finish(nsq.MessageID{})
	fin.Params[0] = []byte("")
	_, err = fin.WriteTo(conn)
	equal(t, err, nil)

	resp, _ := nsq.ReadResponse(conn)
	frameType, data, _ := nsq.UnpackResponse(resp)
	equal(t, frameType, frameTypeError)
	equal(t, string(data), "E_INVALID Invalid Message ID")
}

func TestClientAuth(t *testing.T) {
	authResponse := `{"ttl":1, "authorizations":[]}`
	authSecret := "testsecret"
	authError := "E_UNAUTHORIZED AUTH No authorizations found"
	authSuccess := ""
	runAuthTest(t, authResponse, authSecret, authError, authSuccess)

	// now one that will succeed
	authResponse = `{"ttl":10, "authorizations":
		[{"topic":"test", "channels":[".*"], "permissions":["subscribe","publish"]}]
	}`
	authError = ""
	authSuccess = `{"identity":"","identity_url":"","permission_count":1}`
	runAuthTest(t, authResponse, authSecret, authError, authSuccess)

}

func runAuthTest(t *testing.T, authResponse, authSecret, authError, authSuccess string) {
	var err error
	var expectedAuthIP string
	expectedAuthTLS := "false"

	authd := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		t.Logf("in test auth handler %s", r.RequestURI)
		r.ParseForm()
		equal(t, r.Form.Get("remote_ip"), expectedAuthIP)
		equal(t, r.Form.Get("tls"), expectedAuthTLS)
		equal(t, r.Form.Get("secret"), authSecret)
		fmt.Fprint(w, authResponse)
	}))
	defer authd.Close()

	addr, err := url.Parse(authd.URL)
	equal(t, err, nil)

	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.AuthHTTPAddresses = []string{addr.Host}
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	expectedAuthIP, _, _ = net.SplitHostPort(conn.LocalAddr().String())

	identify(t, conn, map[string]interface{}{
		"tls_v1": false,
	}, nsq.FrameTypeResponse)

	authCmd(t, conn, authSecret, authSuccess)
	if authError != "" {
		readValidate(t, conn, nsq.FrameTypeError, authError)
	} else {
		sub(t, conn, "test", "ch")
	}

}

func TestIOLoopReturnsClientErrWhenSendFails(t *testing.T) {
	fakeConn := test.NewFakeNetConn()
	fakeConn.WriteFunc = func(b []byte) (int, error) {
		return 0, errors.New("write error")
	}

	testIOLoopReturnsClientErr(t, fakeConn)
}

func TestIOLoopReturnsClientErrWhenSendSucceeds(t *testing.T) {
	fakeConn := test.NewFakeNetConn()
	fakeConn.WriteFunc = func(b []byte) (int, error) {
		return len(b), nil
	}

	testIOLoopReturnsClientErr(t, fakeConn)
}

func testIOLoopReturnsClientErr(t *testing.T, fakeConn test.FakeNetConn) {
	fakeConn.ReadFunc = func(b []byte) (int, error) {
		return copy(b, []byte("INVALID_COMMAND\n")), nil
	}

	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true

	prot := &protocolV2{ctx: &context{nsqd: New(opts)}}
	defer prot.ctx.nsqd.Exit()

	err := prot.IOLoop(fakeConn)

	nequal(t, err, nil)
	equal(t, err.Error(), "E_INVALID invalid command INVALID_COMMAND")
	nequal(t, err.(*protocol.FatalClientErr), nil)
}

func BenchmarkProtocolV2Exec(b *testing.B) {
	b.StopTimer()
	opts := NewOptions()
	opts.Logger = newTestLogger(b)
	nsqd := New(opts)
	ctx := &context{nsqd}
	p := &protocolV2{ctx}
	c := newClientV2(0, nil, ctx)
	params := [][]byte{[]byte("NOP")}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		p.Exec(c, params)
	}
}

func benchmarkProtocolV2Pub(b *testing.B, size int) {
	var wg sync.WaitGroup
	b.StopTimer()
	opts := NewOptions()
	batchSize := int(opts.MaxBodySize) / (size + 4)
	opts.Logger = newTestLogger(b)
	opts.MemQueueSize = int64(b.N)
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	msg := make([]byte, size)
	batch := make([][]byte, batchSize)
	for i := range batch {
		batch[i] = msg
	}
	topicName := "bench_v2_pub" + strconv.Itoa(int(time.Now().Unix()))
	b.SetBytes(int64(len(msg)))
	b.StartTimer()

	for j := 0; j < runtime.GOMAXPROCS(0); j++ {
		wg.Add(1)
		go func() {
			conn, err := mustConnectNSQD(tcpAddr)
			if err != nil {
				panic(err.Error())
			}
			rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))

			num := b.N / runtime.GOMAXPROCS(0) / batchSize
			for i := 0; i < num; i++ {
				cmd, _ := nsq.MultiPublish(topicName, batch)
				_, err := cmd.WriteTo(rw)
				if err != nil {
					panic(err.Error())
				}
				err = rw.Flush()
				if err != nil {
					panic(err.Error())
				}
				resp, err := nsq.ReadResponse(rw)
				if err != nil {
					panic(err.Error())
				}
				_, data, _ := nsq.UnpackResponse(resp)
				if !bytes.Equal(data, []byte("OK")) {
					panic("invalid response")
				}
			}
			wg.Done()
		}()
	}

	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}

func BenchmarkProtocolV2Pub256(b *testing.B)  { benchmarkProtocolV2Pub(b, 256) }
func BenchmarkProtocolV2Pub512(b *testing.B)  { benchmarkProtocolV2Pub(b, 512) }
func BenchmarkProtocolV2Pub1k(b *testing.B)   { benchmarkProtocolV2Pub(b, 1024) }
func BenchmarkProtocolV2Pub2k(b *testing.B)   { benchmarkProtocolV2Pub(b, 2*1024) }
func BenchmarkProtocolV2Pub4k(b *testing.B)   { benchmarkProtocolV2Pub(b, 4*1024) }
func BenchmarkProtocolV2Pub8k(b *testing.B)   { benchmarkProtocolV2Pub(b, 8*1024) }
func BenchmarkProtocolV2Pub16k(b *testing.B)  { benchmarkProtocolV2Pub(b, 16*1024) }
func BenchmarkProtocolV2Pub32k(b *testing.B)  { benchmarkProtocolV2Pub(b, 32*1024) }
func BenchmarkProtocolV2Pub64k(b *testing.B)  { benchmarkProtocolV2Pub(b, 64*1024) }
func BenchmarkProtocolV2Pub128k(b *testing.B) { benchmarkProtocolV2Pub(b, 128*1024) }
func BenchmarkProtocolV2Pub256k(b *testing.B) { benchmarkProtocolV2Pub(b, 256*1024) }
func BenchmarkProtocolV2Pub512k(b *testing.B) { benchmarkProtocolV2Pub(b, 512*1024) }
func BenchmarkProtocolV2Pub1m(b *testing.B)   { benchmarkProtocolV2Pub(b, 1024*1024) }

func benchmarkProtocolV2Sub(b *testing.B, size int) {
	var wg sync.WaitGroup
	b.StopTimer()
	opts := NewOptions()
	opts.Logger = newTestLogger(b)
	opts.MemQueueSize = int64(b.N)
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	msg := make([]byte, size)
	topicName := "bench_v2_sub" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
	topic := nsqd.GetTopic(topicName)
	for i := 0; i < b.N; i++ {
		msg := NewMessage(<-nsqd.idChan, msg)
		topic.PutMessage(msg)
	}
	topic.GetChannel("ch")
	b.SetBytes(int64(len(msg)))
	goChan := make(chan int)
	rdyChan := make(chan int)
	workers := runtime.GOMAXPROCS(0)
	for j := 0; j < workers; j++ {
		wg.Add(1)
		go func() {
			subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
			wg.Done()
		}()
		<-rdyChan
	}
	b.StartTimer()

	close(goChan)
	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}

func subWorker(n int, workers int, tcpAddr *net.TCPAddr, topicName string, rdyChan chan int, goChan chan int) {
	conn, err := mustConnectNSQD(tcpAddr)
	if err != nil {
		panic(err.Error())
	}
	rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriterSize(conn, 65536))

	identify(nil, conn, nil, frameTypeResponse)
	sub(nil, conn, topicName, "ch")

	rdyCount := int(math.Min(math.Max(float64(n/workers), 1), 2500))
	rdyChan <- 1
	<-goChan
	nsq.Ready(rdyCount).WriteTo(rw)
	rw.Flush()
	num := n / workers
	for i := 0; i < num; i++ {
		resp, err := nsq.ReadResponse(rw)
		if err != nil {
			panic(err.Error())
		}
		frameType, data, err := nsq.UnpackResponse(resp)
		if err != nil {
			panic(err.Error())
		}
		if frameType != frameTypeMessage {
			panic("got something else")
		}
		msg, err := decodeMessage(data)
		if err != nil {
			panic(err.Error())
		}
		nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(rw)
		if (i+1)%rdyCount == 0 || i+1 == num {
			if i+1 == num {
				nsq.Ready(0).WriteTo(conn)
			}
			rw.Flush()
		}
	}
}

func BenchmarkProtocolV2Sub256(b *testing.B)  { benchmarkProtocolV2Sub(b, 256) }
func BenchmarkProtocolV2Sub512(b *testing.B)  { benchmarkProtocolV2Sub(b, 512) }
func BenchmarkProtocolV2Sub1k(b *testing.B)   { benchmarkProtocolV2Sub(b, 1024) }
func BenchmarkProtocolV2Sub2k(b *testing.B)   { benchmarkProtocolV2Sub(b, 2*1024) }
func BenchmarkProtocolV2Sub4k(b *testing.B)   { benchmarkProtocolV2Sub(b, 4*1024) }
func BenchmarkProtocolV2Sub8k(b *testing.B)   { benchmarkProtocolV2Sub(b, 8*1024) }
func BenchmarkProtocolV2Sub16k(b *testing.B)  { benchmarkProtocolV2Sub(b, 16*1024) }
func BenchmarkProtocolV2Sub32k(b *testing.B)  { benchmarkProtocolV2Sub(b, 32*1024) }
func BenchmarkProtocolV2Sub64k(b *testing.B)  { benchmarkProtocolV2Sub(b, 64*1024) }
func BenchmarkProtocolV2Sub128k(b *testing.B) { benchmarkProtocolV2Sub(b, 128*1024) }
func BenchmarkProtocolV2Sub256k(b *testing.B) { benchmarkProtocolV2Sub(b, 256*1024) }
func BenchmarkProtocolV2Sub512k(b *testing.B) { benchmarkProtocolV2Sub(b, 512*1024) }
func BenchmarkProtocolV2Sub1m(b *testing.B)   { benchmarkProtocolV2Sub(b, 1024*1024) }

func benchmarkProtocolV2MultiSub(b *testing.B, num int) {
	var wg sync.WaitGroup
	b.StopTimer()

	opts := NewOptions()
	opts.Logger = newTestLogger(b)
	opts.MemQueueSize = int64(b.N)
	tcpAddr, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	msg := make([]byte, 256)
	b.SetBytes(int64(len(msg) * num))

	goChan := make(chan int)
	rdyChan := make(chan int)
	workers := runtime.GOMAXPROCS(0)
	for i := 0; i < num; i++ {
		topicName := "bench_v2" + strconv.Itoa(b.N) + "_" + strconv.Itoa(i) + "_" + strconv.Itoa(int(time.Now().Unix()))
		topic := nsqd.GetTopic(topicName)
		for i := 0; i < b.N; i++ {
			msg := NewMessage(<-nsqd.idChan, msg)
			topic.PutMessage(msg)
		}
		topic.GetChannel("ch")

		for j := 0; j < workers; j++ {
			wg.Add(1)
			go func() {
				subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan)
				wg.Done()
			}()
			<-rdyChan
		}
	}
	b.StartTimer()

	close(goChan)
	wg.Wait()

	b.StopTimer()
	nsqd.Exit()
}

func BenchmarkProtocolV2MultiSub1(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 1) }
func BenchmarkProtocolV2MultiSub2(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 2) }
func BenchmarkProtocolV2MultiSub4(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 4) }
func BenchmarkProtocolV2MultiSub8(b *testing.B)  { benchmarkProtocolV2MultiSub(b, 8) }
func BenchmarkProtocolV2MultiSub16(b *testing.B) { benchmarkProtocolV2MultiSub(b, 16) }
Пример #21
0
func TestProcessKilled(t *testing.T) {
	testDefaultStartup(t)
	doneCh := make(chan struct{})
	shutdown := time.NewTimer(time.Second * 4)
	timeout := time.NewTimer(time.Second * 10)
	terminatedHandler := func(reason string) {
		t.Logf("reason: %s\n", reason)
		doneCh <- struct{}{}
	}
	debugger.SetTerminationHandler(terminatedHandler)
	for {
		select {
		case <-doneCh:
			goto DONE
		case <-shutdown.C:
			debugger.ExitProcess()
		case <-timeout.C:
			t.Fatalf("timed out waiting for termination")
		}
	}
DONE:
}

func TestTargetCrashed(t *testing.T) {
	testDefaultStartup(t)
	defer debugger.ExitProcess()

	doneCh := make(chan struct{})
	timeout := time.NewTimer(time.Second * 10)

	targetCrashedFn := func(targ *ChromeTarget, payload []byte) {
		t.Logf("reason: %s\n", string(payload))
		doneCh <- struct{}{}
	}

	tab, err := debugger.NewTab()
	if err != nil {
		t.Fatalf("error creating new tab")
	}

	tab.Subscribe("Inspector.targetCrashed", targetCrashedFn)
	go func() {
		<-timeout.C
		t.Fatalf("timed out waiting for crashed to be handled")
	}()

	_, err = tab.Page.Navigate("chrome://crash")
	if err == nil {
		t.Fatalf("Navigation should have failed")
	}

	<-doneCh
}

func TestEvents(t *testing.T) {
	testDefaultStartup(t)
	defer debugger.ExitProcess()

	target, err := debugger.NewTab()
	if err != nil {
		t.Fatalf("error getting new tab: %s\n", err)
	}
	console := target.Console

	doneCh := make(chan struct{}, 1)
	target.Subscribe("Console.messageAdded", func(target *ChromeTarget, v []byte) {
		target.Unsubscribe("Console.messageAdded")
		msg := &gcdapi.ConsoleMessageAddedEvent{}
		err := json.Unmarshal(v, msg)
		if err != nil {
			t.Fatalf("error unmarshalling event data: %v\n", err)
		}
		t.Logf("METHOD: %s\n", msg.Method)
		eventData := msg.Params.Message
		t.Logf("Got event: %v\n", eventData)
		t.Logf("Timestamp: %f\n", eventData.Timestamp)
		doneCh <- struct{}{}
	})

	_, err = console.Enable()
	if err != nil {
		t.Fatalf("error sending enable: %s\n", err)
	}

	if _, err := target.Page.Navigate(testServerAddr + "console_log.html"); err != nil {
		t.Fatalf("error attempting to navigate: %s\n", err)
	}

	go testTimeoutListener(t, 5, "console message")

	<-doneCh
}

func TestSimpleReturn(t *testing.T) {
	var ret bool
	testDefaultStartup(t)
	defer debugger.ExitProcess()

	target, err := debugger.NewTab()
	if err != nil {
		t.Fatalf("error getting new tab: %s\n", err)
	}
	network := target.Network
	if _, err := network.Enable(); err != nil {
		t.Fatalf("error enabling network")
	}
	ret, err = network.CanClearBrowserCache()
	if err != nil {
		t.Fatalf("error getting response to clearing browser cache: %s\n", err)
	}
	if !ret {
		t.Fatalf("we should have got true for can clear browser cache\n")
	}
}

// tests getting a complex object back from inside a fired event that we subscribed to.
func TestComplexReturn(t *testing.T) {
	testDefaultStartup(t)
	defer debugger.ExitProcess()

	wg := &sync.WaitGroup{}
	wg.Add(1)
	target, err := debugger.NewTab()
	if err != nil {
		t.Fatalf("error getting new tab: %s\n", err)
	}
	if _, err := target.Network.Enable(); err != nil {
		t.Fatalf("error enabling network %s\n", err)
	}

	if _, err := target.Page.Enable(); err != nil {
		t.Fatalf("error enabling page: %s\n", err)
	}

	target.Subscribe("Page.loadEventFired", func(target *ChromeTarget, payload []byte) {
		var ok bool
		t.Logf("page load event fired\n")
		cookies, err := target.Network.GetCookies()
		if err != nil {
			t.Fatalf("error getting cookies!")
		}
		for _, v := range cookies {
			t.Logf("got cookies: %#v\n", v)
			if v.Name == "HEYA" {
				ok = true
				break
			}
		}
		if !ok {
			t.Fatalf("error finding our cookie value!")
		}
		wg.Done()
	})

	_, err = target.Page.Navigate(testServerAddr + "cookie.html")
	if err != nil {
		t.Fatalf("error navigating to cookie page: %s\n", err)
	}

	go testTimeoutListener(t, 7, "waiting for page load to get cookies")
	t.Logf("waiting for loadEventFired")
	wg.Wait()
}

// UTILITY FUNCTIONS

func testDefaultStartup(t *testing.T) {
	debugger = NewChromeDebugger()
	debugger.StartProcess(testPath, testRandomTempDir(t), testRandomPort(t))
}

func testServer() {
	testListener, _ = net.Listen("tcp", ":0")
	_, testServerPort, _ := net.SplitHostPort(testListener.Addr().String())
	testServerAddr = fmt.Sprintf("http://localhost:%s/", testServerPort)
	go http.Serve(testListener, http.FileServer(http.Dir("testdata/")))
}

func testTimeoutListener(t *testing.T, seconds time.Duration, message string) {
	timeout := time.NewTimer(seconds * time.Second)
	for {
		select {
		case <-timeout.C:
			t.Fatalf("timed out waiting for %s", message)
		}
	}
}

func testRandomPort(t *testing.T) string {
	l, err := net.Listen("tcp", ":0")
	if err != nil {
		t.Fatal(err)
	}
	_, randPort, _ := net.SplitHostPort(l.Addr().String())
	l.Close()
	return randPort
}

func testRandomTempDir(t *testing.T) string {
	dir, err := ioutil.TempDir(testDir, "gcd")
	if err != nil {
		t.Errorf("error creating temp dir: %s\n", err)
	}
	return dir
}