func TestHTTPPauseTopicPOST(t *testing.T) {
dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t)
defer os.RemoveAll(dataPath)
defer nsqds[0].Exit()
defer nsqlookupds[0].Exit()
defer nsqadmin1.Exit()
topicName := "test_pause_topic_post" + strconv.Itoa(int(time.Now().Unix()))
nsqds[0].GetTopic(topicName)
time.Sleep(100 * time.Millisecond)
client := http.Client{}
url := fmt.Sprintf("http://%s/api/topics/%s", nsqadmin1.RealHTTPAddr(), topicName)
body, _ := json.Marshal(map[string]interface{}{
"action": "pause",
})
req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body))
resp, err := client.Do(req)
test.Nil(t, err)
body, _ = ioutil.ReadAll(resp.Body)
test.Equal(t, 200, resp.StatusCode)
resp.Body.Close()
url = fmt.Sprintf("http://%s/api/topics/%s", nsqadmin1.RealHTTPAddr(), topicName)
body, _ = json.Marshal(map[string]interface{}{
"action": "unpause",
})
req, _ = http.NewRequest("POST", url, bytes.NewBuffer(body))
resp, err = client.Do(req)
test.Nil(t, err)
test.Equal(t, 200, resp.StatusCode)
resp.Body.Close()
}
func TestHTTPmput(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
_, httpAddr, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_http_mput" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
msg := []byte("test message")
msgs := make([][]byte, 4)
for i := range msgs {
msgs[i] = msg
}
buf := bytes.NewBuffer(bytes.Join(msgs, []byte("\n")))
url := fmt.Sprintf("http://%s/mput?topic=%s", httpAddr, topicName)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Nil(t, err)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, "OK", string(body))
time.Sleep(5 * time.Millisecond)
test.Equal(t, int64(4), topic.Depth())
}
func TestHTTPTopicsGET(t *testing.T) {
dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t)
defer os.RemoveAll(dataPath)
defer nsqds[0].Exit()
defer nsqlookupds[0].Exit()
defer nsqadmin1.Exit()
topicName := "test_topics_get" + strconv.Itoa(int(time.Now().Unix()))
nsqds[0].GetTopic(topicName)
time.Sleep(100 * time.Millisecond)
client := http.Client{}
url := fmt.Sprintf("http://%s/api/topics", nsqadmin1.RealHTTPAddr())
req, _ := http.NewRequest("GET", url, nil)
resp, err := client.Do(req)
test.Nil(t, err)
test.Equal(t, 200, resp.StatusCode)
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
t.Logf("%s", body)
tr := TopicsDoc{}
err = json.Unmarshal(body, &tr)
test.Nil(t, err)
test.Equal(t, 1, len(tr.Topics))
test.Equal(t, topicName, tr.Topics[0])
}
func TestHTTPpubDefer(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
_, httpAddr, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_http_pub_defer" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
ch := topic.GetChannel("ch")
buf := bytes.NewBuffer([]byte("test message"))
url := fmt.Sprintf("http://%s/pub?topic=%s&defer=%d", httpAddr, topicName, 1000)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Nil(t, err)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, "OK", string(body))
time.Sleep(5 * time.Millisecond)
ch.deferredMutex.Lock()
numDef := len(ch.deferredMessages)
ch.deferredMutex.Unlock()
test.Equal(t, 1, numDef)
}
func TestInactiveNodes(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.InactiveProducerTimeout = 200 * time.Millisecond
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)}
topicName := "inactive_nodes"
conn := mustConnectLookupd(t, tcpAddr)
defer conn.Close()
identify(t, conn)
nsq.Register(topicName, "channel1").WriteTo(conn)
_, err := nsq.ReadResponse(conn)
test.Nil(t, err)
ci := clusterinfo.New(nil, http_api.NewClient(nil, ConnectTimeout, RequestTimeout))
producers, _ := ci.GetLookupdProducers(lookupdHTTPAddrs)
test.Equal(t, 1, len(producers))
test.Equal(t, 1, len(producers[0].Topics))
test.Equal(t, topicName, producers[0].Topics[0].Topic)
test.Equal(t, false, producers[0].Topics[0].Tombstoned)
time.Sleep(250 * time.Millisecond)
producers, _ = ci.GetLookupdProducers(lookupdHTTPAddrs)
test.Equal(t, 0, len(producers))
}
func TestPauseMetadata(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
_, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
// avoid concurrency issue of async PersistMetadata() calls
atomic.StoreInt32(&nsqd.isLoading, 1)
topicName := "pause_metadata" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
channel := topic.GetChannel("ch")
atomic.StoreInt32(&nsqd.isLoading, 0)
nsqd.PersistMetadata()
var isPaused = func(n *NSQD, topicIndex int, channelIndex int) bool {
m, _ := getMetadata(n)
return m.Topics[topicIndex].Channels[channelIndex].Paused
}
test.Equal(t, false, isPaused(nsqd, 0, 0))
channel.Pause()
test.Equal(t, false, isPaused(nsqd, 0, 0))
nsqd.PersistMetadata()
test.Equal(t, true, isPaused(nsqd, 0, 0))
channel.UnPause()
test.Equal(t, true, isPaused(nsqd, 0, 0))
nsqd.PersistMetadata()
test.Equal(t, false, isPaused(nsqd, 0, 0))
}
func TestHTTPmputBinary(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
_, httpAddr, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_http_mput_bin" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
mpub := make([][]byte, 5)
for i := range mpub {
mpub[i] = make([]byte, 100)
}
cmd, _ := nsq.MultiPublish(topicName, mpub)
buf := bytes.NewBuffer(cmd.Body)
url := fmt.Sprintf("http://%s/mput?topic=%s&binary=true", httpAddr, topicName)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Nil(t, err)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, "OK", string(body))
time.Sleep(5 * time.Millisecond)
test.Equal(t, int64(5), topic.Depth())
}
// exercise the basic operations of the V2 protocol
func TestBasicV2(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.ClientTimeout = 60 * time.Second
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_v2" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
msg := NewMessage(<-nsqd.idChan, []byte("test body"))
topic.PutMessage(msg)
conn, err := mustConnectNSQD(tcpAddr)
test.Nil(t, err)
defer conn.Close()
identify(t, conn, nil, frameTypeResponse)
sub(t, conn, topicName, "ch")
_, err = nsq.Ready(1).WriteTo(conn)
test.Nil(t, err)
resp, err := nsq.ReadResponse(conn)
test.Nil(t, err)
frameType, data, err := nsq.UnpackResponse(resp)
msgOut, _ := decodeMessage(data)
test.Equal(t, frameTypeMessage, frameType)
test.Equal(t, msg.ID, msgOut.ID)
test.Equal(t, msg.Body, msgOut.Body)
test.Equal(t, uint16(1), msgOut.Attempts)
}
func TestRemove(t *testing.T) {
c := 100
pq := newInFlightPqueue(c)
msgs := make(map[MessageID]*Message)
for i := 0; i < c; i++ {
m := &Message{pri: int64(rand.Intn(100000000))}
copy(m.ID[:], fmt.Sprintf("%016d", m.pri))
msgs[m.ID] = m
pq.Push(m)
}
for i := 0; i < 10; i++ {
idx := rand.Intn((c - 1) - i)
var fm *Message
for _, m := range msgs {
if m.index == idx {
fm = m
break
}
}
rm := pq.Remove(idx)
test.Equal(t, fmt.Sprintf("%s", fm.ID), fmt.Sprintf("%s", rm.ID))
}
lastPriority := pq.Pop().pri
for i := 0; i < (c - 10 - 1); i++ {
msg := pq.Pop()
test.Equal(t, true, lastPriority <= msg.pri)
lastPriority = msg.pri
}
}
func TestHTTPconfig(t *testing.T) {
lopts := nsqlookupd.NewOptions()
lopts.Logger = test.NewTestLogger(t)
_, _, lookupd1 := mustStartNSQLookupd(lopts)
defer lookupd1.Exit()
_, _, lookupd2 := mustStartNSQLookupd(lopts)
defer lookupd2.Exit()
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
_, httpAddr, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
url := fmt.Sprintf("http://%s/config/nsqlookupd_tcp_addresses", httpAddr)
resp, err := http.Get(url)
test.Nil(t, err)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, 200, resp.StatusCode)
test.Equal(t, "[]", string(body))
client := http.Client{}
addrs := fmt.Sprintf(`["%s","%s"]`, lookupd1.RealTCPAddr().String(), lookupd2.RealTCPAddr().String())
url = fmt.Sprintf("http://%s/config/nsqlookupd_tcp_addresses", httpAddr)
req, err := http.NewRequest("PUT", url, bytes.NewBuffer([]byte(addrs)))
test.Nil(t, err)
resp, err = client.Do(req)
test.Nil(t, err)
defer resp.Body.Close()
body, _ = ioutil.ReadAll(resp.Body)
test.Equal(t, 200, resp.StatusCode)
test.Equal(t, addrs, string(body))
}
func testIOLoopReturnsClientErr(t *testing.T, fakeConn test.FakeNetConn) {
fakeConn.ReadFunc = func(b []byte) (int, error) {
return copy(b, []byte("INVALID_COMMAND\n")), nil
}
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.Verbose = true
prot := &LookupProtocolV1{ctx: &Context{nsqlookupd: New(opts)}}
errChan := make(chan error)
testIOLoop := func() {
errChan <- prot.IOLoop(fakeConn)
defer prot.ctx.nsqlookupd.Exit()
}
go testIOLoop()
var err error
var timeout bool
select {
case err = <-errChan:
case <-time.After(2 * time.Second):
timeout = true
}
test.Equal(t, false, timeout)
test.NotNil(t, err)
test.Equal(t, "E_INVALID invalid command INVALID_COMMAND", err.Error())
test.NotNil(t, err.(*protocol.FatalClientErr))
}
func TestChannelEmptyConsumer(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, _ := mustConnectNSQD(tcpAddr)
defer conn.Close()
topicName := "test_channel_empty" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
channel := topic.GetChannel("channel")
client := newClientV2(0, conn, &context{nsqd})
client.SetReadyCount(25)
channel.AddClient(client.ID, client)
for i := 0; i < 25; i++ {
msg := NewMessage(<-nsqd.idChan, []byte("test"))
channel.StartInFlightTimeout(msg, 0, opts.MsgTimeout)
client.SendingMessage()
}
for _, cl := range channel.clients {
stats := cl.Stats()
test.Equal(t, int64(25), stats.InFlightCount)
}
channel.Empty()
for _, cl := range channel.clients {
stats := cl.Stats()
test.Equal(t, int64(0), stats.InFlightCount)
}
}
func TestTLSRequireVerifyExceptHTTP(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.Verbose = true
opts.TLSCert = "./test/certs/server.pem"
opts.TLSKey = "./test/certs/server.key"
opts.TLSRootCAFile = "./test/certs/ca.pem"
opts.TLSClientAuthPolicy = "require-verify"
opts.TLSRequired = TLSRequiredExceptHTTP
_, httpAddr, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_http_req_verf_except_http" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
// no cert
buf := bytes.NewBuffer([]byte("test message"))
url := fmt.Sprintf("http://%s/put?topic=%s", httpAddr, topicName)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Nil(t, err)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, "OK", string(body))
time.Sleep(5 * time.Millisecond)
test.Equal(t, int64(1), topic.Depth())
}
func TestStats(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
tcpAddr, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_stats" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
msg := NewMessage(<-nsqd.idChan, []byte("test body"))
topic.PutMessage(msg)
conn, err := mustConnectNSQD(tcpAddr)
test.Nil(t, err)
defer conn.Close()
identify(t, conn, nil, frameTypeResponse)
sub(t, conn, topicName, "ch")
stats := nsqd.GetStats()
t.Logf("stats: %+v", stats)
test.Equal(t, 1, len(stats))
test.Equal(t, 1, len(stats[0].Channels))
test.Equal(t, 1, len(stats[0].Channels[0].Clients))
}
func TestTLSHTTPClient(t *testing.T) {
nsqdOpts := nsqd.NewOptions()
nsqdOpts.Verbose = true
nsqdOpts.TLSCert = "./test/server.pem"
nsqdOpts.TLSKey = "./test/server-key.pem"
nsqdOpts.TLSRootCAFile = "./test/ca.pem"
nsqdOpts.TLSClientAuthPolicy = "require-verify"
_, nsqdHTTPAddr, nsqd := mustStartNSQD(nsqdOpts)
defer os.RemoveAll(nsqdOpts.DataPath)
defer nsqd.Exit()
opts := NewOptions()
opts.HTTPAddress = "127.0.0.1:0"
opts.NSQDHTTPAddresses = []string{nsqdHTTPAddr.String()}
opts.HTTPClientTLSRootCAFile = "./test/ca.pem"
opts.HTTPClientTLSCert = "./test/client.pem"
opts.HTTPClientTLSKey = "./test/client-key.pem"
nsqadmin := New(opts)
nsqadmin.Main()
defer nsqadmin.Exit()
httpAddr := nsqadmin.RealHTTPAddr()
u := url.URL{
Scheme: "http",
Host: httpAddr.String(),
Path: "/api/nodes/" + nsqdHTTPAddr.String(),
}
resp, err := http.Get(u.String())
defer resp.Body.Close()
test.Equal(t, nil, err)
test.Equal(t, resp.StatusCode < 500, true)
}
func TestHTTPconfig(t *testing.T) {
dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t)
defer os.RemoveAll(dataPath)
defer nsqds[0].Exit()
defer nsqlookupds[0].Exit()
defer nsqadmin1.Exit()
lopts := nsqlookupd.NewOptions()
lopts.Logger = test.NewTestLogger(t)
_, _, lookupd1 := mustStartNSQLookupd(lopts)
defer lookupd1.Exit()
_, _, lookupd2 := mustStartNSQLookupd(lopts)
defer lookupd2.Exit()
url := fmt.Sprintf("http://%s/config/nsqlookupd_http_addresses", nsqadmin1.RealHTTPAddr())
resp, err := http.Get(url)
test.Nil(t, err)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, 200, resp.StatusCode)
origaddrs := fmt.Sprintf(`["%s"]`, nsqlookupds[0].RealHTTPAddr().String())
test.Equal(t, origaddrs, string(body))
client := http.Client{}
addrs := fmt.Sprintf(`["%s","%s"]`, lookupd1.RealHTTPAddr().String(), lookupd2.RealHTTPAddr().String())
url = fmt.Sprintf("http://%s/config/nsqlookupd_http_addresses", nsqadmin1.RealHTTPAddr())
req, err := http.NewRequest("PUT", url, bytes.NewBuffer([]byte(addrs)))
test.Nil(t, err)
resp, err = client.Do(req)
test.Nil(t, err)
defer resp.Body.Close()
body, _ = ioutil.ReadAll(resp.Body)
test.Equal(t, 200, resp.StatusCode)
test.Equal(t, addrs, string(body))
}
func TestHTTPEmptyChannelPOST(t *testing.T) {
dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t)
defer os.RemoveAll(dataPath)
defer nsqds[0].Exit()
defer nsqlookupds[0].Exit()
defer nsqadmin1.Exit()
topicName := "test_empty_channel_post" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqds[0].GetTopic(topicName)
channel := topic.GetChannel("ch")
channel.PutMessage(nsqd.NewMessage(nsqd.MessageID{}, []byte("1234")))
time.Sleep(100 * time.Millisecond)
test.Equal(t, int64(1), channel.Depth())
client := http.Client{}
url := fmt.Sprintf("http://%s/api/topics/%s/ch", nsqadmin1.RealHTTPAddr(), topicName)
body, _ := json.Marshal(map[string]interface{}{
"action": "empty",
})
req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body))
resp, err := client.Do(req)
test.Nil(t, err)
body, _ = ioutil.ReadAll(resp.Body)
test.Equal(t, 200, resp.StatusCode)
resp.Body.Close()
test.Equal(t, int64(0), channel.Depth())
}
func readValidate(t *testing.T, conn io.Reader, f int32, d string) []byte {
resp, err := nsq.ReadResponse(conn)
test.Nil(t, err)
frameType, data, err := nsq.UnpackResponse(resp)
test.Nil(t, err)
test.Equal(t, f, frameType)
test.Equal(t, d, string(data))
return data
}
func TestChannelEmpty(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
_, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_channel_empty" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
channel := topic.GetChannel("channel")
msgs := make([]*Message, 0, 25)
for i := 0; i < 25; i++ {
msg := NewMessage(<-nsqd.idChan, []byte("test"))
channel.StartInFlightTimeout(msg, 0, opts.MsgTimeout)
msgs = append(msgs, msg)
}
channel.RequeueMessage(0, msgs[len(msgs)-1].ID, 100*time.Millisecond)
test.Equal(t, 24, len(channel.inFlightMessages))
test.Equal(t, 24, len(channel.inFlightPQ))
test.Equal(t, 1, len(channel.deferredMessages))
test.Equal(t, 1, len(channel.deferredPQ))
channel.Empty()
test.Equal(t, 0, len(channel.inFlightMessages))
test.Equal(t, 0, len(channel.inFlightPQ))
test.Equal(t, 0, len(channel.deferredMessages))
test.Equal(t, 0, len(channel.deferredPQ))
test.Equal(t, int64(0), channel.Depth())
}
func TestClientAttributes(t *testing.T) {
userAgent := "Test User Agent"
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.Verbose = true
opts.SnappyEnabled = true
tcpAddr, httpAddr, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
test.Nil(t, err)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"snappy": true,
"user_agent": userAgent,
}, frameTypeResponse)
resp := struct {
Snappy bool `json:"snappy"`
UserAgent string `json:"user_agent"`
}{}
err = json.Unmarshal(data, &resp)
test.Nil(t, err)
test.Equal(t, true, resp.Snappy)
r := snappystream.NewReader(conn, snappystream.SkipVerifyChecksum)
w := snappystream.NewWriter(conn)
readValidate(t, r, frameTypeResponse, "OK")
topicName := "test_client_attributes" + strconv.Itoa(int(time.Now().Unix()))
sub(t, readWriter{r, w}, topicName, "ch")
var d struct {
Topics []struct {
Channels []struct {
Clients []struct {
UserAgent string `json:"user_agent"`
Snappy bool `json:"snappy"`
} `json:"clients"`
} `json:"channels"`
} `json:"topics"`
}
endpoint := fmt.Sprintf("http://127.0.0.1:%d/stats?format=json", httpAddr.Port)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &d)
test.Nil(t, err)
test.Equal(t, userAgent, d.Topics[0].Channels[0].Clients[0].UserAgent)
test.Equal(t, true, d.Topics[0].Channels[0].Clients[0].Snappy)
}
func TestHTTPNodesGET(t *testing.T) {
dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t)
defer os.RemoveAll(dataPath)
defer nsqds[0].Exit()
defer nsqlookupds[0].Exit()
defer nsqadmin1.Exit()
time.Sleep(100 * time.Millisecond)
client := http.Client{}
url := fmt.Sprintf("http://%s/api/nodes", nsqadmin1.RealHTTPAddr())
req, _ := http.NewRequest("GET", url, nil)
resp, err := client.Do(req)
test.Nil(t, err)
test.Equal(t, 200, resp.StatusCode)
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
hostname, _ := os.Hostname()
t.Logf("%s", body)
ns := NodesDoc{}
err = json.Unmarshal(body, &ns)
test.Nil(t, err)
test.Equal(t, 1, len(ns.Nodes))
testNode := ns.Nodes[0]
test.Equal(t, hostname, testNode.Hostname)
test.Equal(t, "127.0.0.1", testNode.BroadcastAddress)
test.Equal(t, nsqds[0].RealTCPAddr().Port, testNode.TCPPort)
test.Equal(t, nsqds[0].RealHTTPAddr().Port, testNode.HTTPPort)
test.Equal(t, version.Binary, testNode.Version)
test.Equal(t, 0, len(testNode.Topics))
}
func TestChannelHealth(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.MemQueueSize = 2
_, httpAddr, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topic := nsqd.GetTopic("test")
channel := topic.GetChannel("channel")
channel.backend = &errorBackendQueue{}
msg := NewMessage(<-nsqd.idChan, make([]byte, 100))
err := channel.PutMessage(msg)
test.Nil(t, err)
msg = NewMessage(<-nsqd.idChan, make([]byte, 100))
err = channel.PutMessage(msg)
test.Nil(t, err)
msg = NewMessage(<-nsqd.idChan, make([]byte, 100))
err = channel.PutMessage(msg)
test.NotNil(t, err)
url := fmt.Sprintf("http://%s/ping", httpAddr)
resp, err := http.Get(url)
test.Nil(t, err)
test.Equal(t, 500, resp.StatusCode)
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
test.Equal(t, "NOK - never gonna happen", string(body))
channel.backend = &errorRecoveredBackendQueue{}
msg = NewMessage(<-nsqd.idChan, make([]byte, 100))
err = channel.PutMessage(msg)
test.Nil(t, err)
resp, err = http.Get(url)
test.Nil(t, err)
test.Equal(t, 200, resp.StatusCode)
body, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close()
test.Equal(t, "OK", string(body))
}
func subFail(t *testing.T, conn io.ReadWriter, topicName string, channelName string) {
_, err := nsq.Subscribe(topicName, channelName).WriteTo(conn)
test.Nil(t, err)
resp, err := nsq.ReadResponse(conn)
frameType, _, err := nsq.UnpackResponse(resp)
test.Equal(t, frameTypeError, frameType)
}
func TestPriorityQueue(t *testing.T) {
c := 100
pq := newInFlightPqueue(c)
for i := 0; i < c+1; i++ {
pq.Push(&Message{clientID: int64(i), pri: int64(i)})
}
test.Equal(t, c+1, len(pq))
test.Equal(t, c*2, cap(pq))
for i := 0; i < c+1; i++ {
msg := pq.Pop()
test.Equal(t, int64(i), msg.clientID)
}
test.Equal(t, c/4, cap(pq))
}
func TestTombstoneRecover(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.TombstoneLifetime = 50 * time.Millisecond
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
topicName := "tombstone_recover"
topicName2 := topicName + "2"
conn := mustConnectLookupd(t, tcpAddr)
defer conn.Close()
identify(t, conn)
nsq.Register(topicName, "channel1").WriteTo(conn)
_, err := nsq.ReadResponse(conn)
test.Nil(t, err)
nsq.Register(topicName2, "channel2").WriteTo(conn)
_, err = nsq.ReadResponse(conn)
test.Nil(t, err)
endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s:%d",
httpAddr, topicName, HostAddr, HTTPPort)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(endpoint)
test.Nil(t, err)
pr := ProducersDoc{}
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).NegotiateV1(endpoint, &pr)
test.Nil(t, err)
test.Equal(t, 0, len(pr.Producers))
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName2)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).NegotiateV1(endpoint, &pr)
test.Nil(t, err)
test.Equal(t, 1, len(pr.Producers))
time.Sleep(75 * time.Millisecond)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).NegotiateV1(endpoint, &pr)
test.Nil(t, err)
test.Equal(t, 1, len(pr.Producers))
}
func TestEphemeralTopicsAndChannels(t *testing.T) {
// ephemeral topics/channels are lazily removed after the last channel/client is removed
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.MemQueueSize = 100
_, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
topicName := "ephemeral_topic" + strconv.Itoa(int(time.Now().Unix())) + "#ephemeral"
doneExitChan := make(chan int)
exitChan := make(chan int)
go func() {
<-exitChan
nsqd.Exit()
doneExitChan <- 1
}()
body := []byte("an_ephemeral_message")
topic := nsqd.GetTopic(topicName)
ephemeralChannel := topic.GetChannel("ch1#ephemeral")
client := newClientV2(0, nil, &context{nsqd})
ephemeralChannel.AddClient(client.ID, client)
msg := NewMessage(<-nsqd.idChan, body)
topic.PutMessage(msg)
msg = <-ephemeralChannel.memoryMsgChan
test.Equal(t, body, msg.Body)
ephemeralChannel.RemoveClient(client.ID)
time.Sleep(100 * time.Millisecond)
topic.Lock()
numChannels := len(topic.channelMap)
topic.Unlock()
test.Equal(t, 0, numChannels)
nsqd.Lock()
numTopics := len(nsqd.topicMap)
nsqd.Unlock()
test.Equal(t, 0, numTopics)
exitChan <- 1
<-doneExitChan
}
func TestPing(t *testing.T) {
dataPath, nsqds, nsqlookupd1 := bootstrapNSQCluster(t)
defer os.RemoveAll(dataPath)
defer nsqds[0].Exit()
defer nsqlookupd1.Exit()
client := http.Client{}
url := fmt.Sprintf("http://%s/ping", nsqlookupd1.RealHTTPAddr())
req, _ := http.NewRequest("GET", url, nil)
resp, err := client.Do(req)
test.Nil(t, err)
test.Equal(t, 200, resp.StatusCode)
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
test.Equal(t, []byte("OK"), body)
}
func TestInFlightWorker(t *testing.T) {
count := 250
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.MsgTimeout = 100 * time.Millisecond
opts.QueueScanRefreshInterval = 100 * time.Millisecond
_, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_in_flight_worker" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
channel := topic.GetChannel("channel")
for i := 0; i < count; i++ {
msg := NewMessage(<-nsqd.idChan, []byte("test"))
channel.StartInFlightTimeout(msg, 0, opts.MsgTimeout)
}
channel.Lock()
inFlightMsgs := len(channel.inFlightMessages)
channel.Unlock()
test.Equal(t, count, inFlightMsgs)
channel.inFlightMutex.Lock()
inFlightPQMsgs := len(channel.inFlightPQ)
channel.inFlightMutex.Unlock()
test.Equal(t, count, inFlightPQMsgs)
// the in flight worker has a resolution of 100ms so we need to wait
// at least that much longer than our msgTimeout (in worst case)
time.Sleep(4 * opts.MsgTimeout)
channel.Lock()
inFlightMsgs = len(channel.inFlightMessages)
channel.Unlock()
test.Equal(t, 0, inFlightMsgs)
channel.inFlightMutex.Lock()
inFlightPQMsgs = len(channel.inFlightPQ)
channel.inFlightMutex.Unlock()
test.Equal(t, 0, inFlightPQMsgs)
}
func TestHTTPSRequire(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.Verbose = true
opts.TLSCert = "./test/certs/server.pem"
opts.TLSKey = "./test/certs/server.key"
opts.TLSClientAuthPolicy = "require"
_, httpAddr, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "test_http_put_req" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopic(topicName)
buf := bytes.NewBuffer([]byte("test message"))
url := fmt.Sprintf("http://%s/put?topic=%s", httpAddr, topicName)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Equal(t, 403, resp.StatusCode)
httpsAddr := nsqd.httpsListener.Addr().(*net.TCPAddr)
cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem")
test.Nil(t, err)
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{cert},
InsecureSkipVerify: true,
MinVersion: 0,
}
transport := &http.Transport{
TLSClientConfig: tlsConfig,
}
client := &http.Client{Transport: transport}
buf = bytes.NewBuffer([]byte("test message"))
url = fmt.Sprintf("https://%s/put?topic=%s", httpsAddr, topicName)
resp, err = client.Post(url, "application/octet-stream", buf)
test.Nil(t, err)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, "OK", string(body))
time.Sleep(5 * time.Millisecond)
test.Equal(t, int64(1), topic.Depth())
}
func TestDiskQueueCorruption(t *testing.T) {
l := test.NewTestLogger(t)
dqName := "test_disk_queue_corruption" + strconv.Itoa(int(time.Now().Unix()))
tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano()))
if err != nil {
panic(err)
}
defer os.RemoveAll(tmpDir)
// require a non-zero message length for the corrupt (len 0) test below
dq := newDiskQueue(dqName, tmpDir, 1000, 10, 1<<10, 5, 2*time.Second, l)
defer dq.Close()
msg := make([]byte, 123) // 127 bytes per message, 8 (1016 bytes) messages per file
for i := 0; i < 25; i++ {
dq.Put(msg)
}
test.Equal(t, int64(25), dq.Depth())
// corrupt the 2nd file
dqFn := dq.(*diskQueue).fileName(1)
os.Truncate(dqFn, 500) // 3 valid messages, 5 corrupted
for i := 0; i < 19; i++ { // 1 message leftover in 4th file
test.Equal(t, msg, <-dq.ReadChan())
}
// corrupt the 4th (current) file
dqFn = dq.(*diskQueue).fileName(3)
os.Truncate(dqFn, 100)
dq.Put(msg) // in 5th file
test.Equal(t, msg, <-dq.ReadChan())
// write a corrupt (len 0) message at the 5th (current) file
dq.(*diskQueue).writeFile.Write([]byte{0, 0, 0, 0})
// force a new 6th file - put into 5th, then readOne errors, then put into 6th
dq.Put(msg)
dq.Put(msg)
test.Equal(t, msg, <-dq.ReadChan())
}