func TestHTTPmpubBinary(t *testing.T) {
opts := nsqd.NewOptions()
opts.Logger = newTestLogger(t)
_, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
topicName := "test_http_mpub_bin" + strconv.Itoa(int(time.Now().Unix()))
nsqd.GetTopicIgnPart(topicName)
mpub := make([][]byte, 5)
for i := range mpub {
mpub[i] = make([]byte, 100)
}
cmd, _ := nsq.MultiPublish(topicName, mpub)
buf := bytes.NewBuffer(cmd.Body)
url := fmt.Sprintf("http://%s/mpub?topic=%s&binary=true", httpAddr, topicName)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, string(body), "OK")
time.Sleep(5 * time.Millisecond)
}
func TestStats(t *testing.T) {
opts := nsqdNs.NewOptions()
opts.Logger = newTestLogger(t)
tcpAddr, _, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
topicName := "test_stats" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopicIgnPart(topicName)
msg := nsqdNs.NewMessage(0, []byte("test body"))
topic.PutMessage(msg)
conn, err := mustConnectNSQD(tcpAddr)
test.Equal(t, err, nil)
defer conn.Close()
identify(t, conn, nil, frameTypeResponse)
sub(t, conn, topicName, "ch")
stats := nsqd.GetStats(false)
t.Logf("stats: %+v", stats)
test.Equal(t, len(stats), 1)
test.Equal(t, len(stats[0].Channels), 1)
test.Equal(t, len(stats[0].Channels[0].Clients), 1)
}
func TestHTTPmpubEmpty(t *testing.T) {
opts := nsqd.NewOptions()
opts.Logger = newTestLogger(t)
_, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
topicName := "test_http_mpub_empty" + strconv.Itoa(int(time.Now().Unix()))
nsqd.GetTopicIgnPart(topicName)
msg := []byte("test message")
msgs := make([][]byte, 4)
for i := range msgs {
msgs[i] = msg
}
buf := bytes.NewBuffer(bytes.Join(msgs, []byte("\n")))
_, err := buf.Write([]byte("\n"))
test.Equal(t, err, nil)
url := fmt.Sprintf("http://%s/mpub?topic=%s", httpAddr, topicName)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, string(body), "OK")
time.Sleep(5 * time.Millisecond)
}
func TestHTTPconfig(t *testing.T) {
lopts := nsqlookupd.NewOptions()
lopts.Logger = newTestLogger(t)
nsqlookupd.SetLogger(lopts)
_, _, lookupd1 := mustStartNSQLookupd(lopts)
defer lookupd1.Exit()
_, _, lookupd2 := mustStartNSQLookupd(lopts)
defer lookupd2.Exit()
opts := nsqd.NewOptions()
opts.Logger = newTestLogger(t)
_, httpAddr, _, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
url := fmt.Sprintf("http://%s/config/nsqlookupd_tcp_addresses", httpAddr)
resp, err := http.Get(url)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, resp.StatusCode, 200)
test.Equal(t, string(body), "[]")
client := http.Client{}
addrs := fmt.Sprintf(`["%s","%s"]`, lookupd1.RealTCPAddr().String(), lookupd2.RealTCPAddr().String())
url = fmt.Sprintf("http://%s/config/nsqlookupd_tcp_addresses", httpAddr)
req, err := http.NewRequest("PUT", url, bytes.NewBuffer([]byte(addrs)))
test.Equal(t, err, nil)
resp, err = client.Do(req)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ = ioutil.ReadAll(resp.Body)
test.Equal(t, resp.StatusCode, 200)
test.Equal(t, string(body), addrs)
}
func BenchmarkFetchNodeHourlyPubsize(b *testing.B) {
b.StopTimer()
lgr := newTestLogger(b)
nsqdOpts := nsqd.NewOptions()
nsqdOpts.TCPAddress = "127.0.0.1:0"
nsqdOpts.HTTPAddress = "127.0.0.1:0"
nsqdOpts.BroadcastAddress = "127.0.0.1"
nsqdOpts.Logger = lgr
_, _, nsqd1, nsqd1Srv := mustStartNSQD(nsqdOpts)
b.Logf("nsqd started")
time.Sleep(100 * time.Millisecond)
var topicNames []string
for i := 0; i < 1000; i++ {
//create sample topics
topicName := fmt.Sprintf("Topic-Benchmark%04d", i)
nsqd1.GetTopic(topicName, 0)
topicNames = append(topicNames, topicName)
b.Logf("Topic %s created.", topicName)
}
type TopicHourlyPubsizeStat struct {
TopicName string `json:"topic_name"`
TopicPartition string `json:"topic_partition"`
HourlyPubsize int64 `json:"hourly_pub_size"`
}
type NodeHourlyPubsizeStats struct {
TopicHourlyPubsizeList []*TopicHourlyPubsizeStat `json:"node_hourly_pub_size_stats"`
}
b.StartTimer()
for i := 0; i < b.N; i++ {
endpoint := fmt.Sprintf("http://%s/message/historystats", nsqd1Srv.ctx.httpAddr.String())
req, err := http.NewRequest("GET", endpoint, nil)
if err != nil {
b.Errorf("Fail to get message history from %s - %s", endpoint, err)
b.Fail()
}
client := http.Client{}
resp, err := client.Do(req)
var nodeHourlyPubsizeStats NodeHourlyPubsizeStats
body, _ := ioutil.ReadAll(resp.Body)
json.Unmarshal(body, &nodeHourlyPubsizeStats)
resp.Body.Close()
b.Logf("Node topics hourly pub size list: %d", len(nodeHourlyPubsizeStats.TopicHourlyPubsizeList))
}
b.StopTimer()
//cleanup
for _, topicName := range topicNames {
err := nsqd1.DeleteExistingTopic(topicName, 0)
if err != nil {
b.Logf("Fail to delete topic: %s", topicName)
}
}
}
func TestTLSRequireVerifyExceptHTTP(t *testing.T) {
opts := nsqd.NewOptions()
opts.Logger = newTestLogger(t)
opts.LogLevel = 3
opts.TLSCert = "./test/certs/server.pem"
opts.TLSKey = "./test/certs/server.key"
opts.TLSRootCAFile = "./test/certs/ca.pem"
opts.TLSClientAuthPolicy = "require-verify"
opts.TLSRequired = TLSRequiredExceptHTTP
_, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
topicName := "test_http_req_verf_except_http" + strconv.Itoa(int(time.Now().Unix()))
nsqd.GetTopicIgnPart(topicName)
// no cert
buf := bytes.NewBuffer([]byte("test message"))
url := fmt.Sprintf("http://%s/pub?topic=%s", httpAddr, topicName)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, string(body), "OK")
time.Sleep(5 * time.Millisecond)
}
func TestHTTPgetStatusText(t *testing.T) {
opts := nsqd.NewOptions()
opts.Logger = newTestLogger(t)
_, httpAddr, _, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
url := fmt.Sprintf("http://%s/stats?format=text", httpAddr)
resp, err := http.Get(url)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, resp.StatusCode, 200)
test.NotNil(t, body)
}
func TestHTTPpub(t *testing.T) {
opts := nsqd.NewOptions()
opts.LogLevel = 2
opts.Logger = newTestLogger(t)
//opts.Logger = &levellogger.GLogger{}
tcpAddr, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
topicName := "test_http_pub" + strconv.Itoa(int(time.Now().Unix()))
_ = nsqd.GetTopicIgnPart(topicName)
conn, err := mustConnectNSQD(tcpAddr)
test.Equal(t, err, nil)
identify(t, conn, nil, frameTypeResponse)
sub(t, conn, topicName, "ch")
buf := bytes.NewBuffer([]byte("test message"))
url := fmt.Sprintf("http://%s/pub?topic=%s", httpAddr, topicName)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, string(body), "OK")
time.Sleep(5 * time.Millisecond)
_, err = nsq.Ready(1).WriteTo(conn)
test.Equal(t, err, nil)
// sleep to allow the RDY state to take effect
time.Sleep(50 * time.Millisecond)
for {
resp, _ := nsq.ReadResponse(conn)
frameType, data, err := nsq.UnpackResponse(resp)
test.Nil(t, err)
test.NotEqual(t, frameTypeError, frameType)
if frameType == frameTypeResponse {
t.Logf("got response data: %v", string(data))
continue
}
msgOut, err := nsq.DecodeMessage(data)
test.Equal(t, []byte("test message"), msgOut.Body)
_, err = nsq.Finish(msgOut.ID).WriteTo(conn)
test.Nil(t, err)
break
}
conn.Close()
}
func TestHTTPpubtrace(t *testing.T) {
opts := nsqd.NewOptions()
opts.LogLevel = 2
opts.Logger = newTestLogger(t)
//opts.Logger = &levellogger.GLogger{}
_, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
topicName := "test_http_pub_trace" + strconv.Itoa(int(time.Now().Unix()))
_ = nsqd.GetTopicIgnPart(topicName)
buf := bytes.NewBuffer([]byte("test message"))
rawurl := fmt.Sprintf("http://%s/pubtrace?topic=%s", httpAddr, topicName)
resp, err := http.Post(rawurl, "application/octet-stream", buf)
test.Equal(t, err, nil)
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
test.Equal(t, resp.StatusCode, 400)
test.Equal(t, string(body), `{"message":"INVALID_TRACE_ID"}`)
time.Sleep(time.Second)
// the buffer will be drained by the http post
// so we need refill the buffer.
buf = bytes.NewBuffer([]byte("test message 2"))
rawurl = fmt.Sprintf("http://%s/pubtrace?topic=%s&partition=0&trace_id=11", httpAddr, topicName)
resp, err = http.Post(rawurl, "application/octet-stream", buf)
test.Equal(t, err, nil)
body, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close()
test.Equal(t, resp.StatusCode, 200)
type tmpResp struct {
Status string `json:"status"`
ID uint64 `json:"id"`
TraceID string `json:"trace_id"`
QueueOffset uint64 `json:"queue_offset"`
DataRawSize uint32 `json:"rawsize"`
}
var ret tmpResp
json.Unmarshal(body, &ret)
test.Equal(t, ret.Status, "OK")
test.Equal(t, ret.TraceID, "11")
time.Sleep(5 * time.Millisecond)
}
func TestClientAttributes(t *testing.T) {
userAgent := "Test User Agent"
opts := nsqdNs.NewOptions()
opts.Logger = newTestLogger(t)
opts.LogLevel = 3
opts.SnappyEnabled = true
tcpAddr, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
conn, err := mustConnectNSQD(tcpAddr)
test.Equal(t, err, nil)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"snappy": true,
"user_agent": userAgent,
}, frameTypeResponse)
resp := struct {
Snappy bool `json:"snappy"`
UserAgent string `json:"user_agent"`
}{}
err = json.Unmarshal(data, &resp)
test.Equal(t, err, nil)
test.Equal(t, resp.Snappy, true)
r := snappystream.NewReader(conn, snappystream.SkipVerifyChecksum)
w := snappystream.NewWriter(conn)
readValidate(t, r, frameTypeResponse, "OK")
topicName := "test_client_attributes" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopicIgnPart(topicName)
topic.GetChannel("ch")
sub(t, readWriter{r, w}, topicName, "ch")
testURL := fmt.Sprintf("http://127.0.0.1:%d/stats?format=json", httpAddr.Port)
statsData, err := API(testURL)
test.Equal(t, err, nil)
client := statsData.Get("topics").GetIndex(0).Get("channels").GetIndex(0).Get("clients").GetIndex(0)
test.Equal(t, client.Get("user_agent").MustString(), userAgent)
test.Equal(t, client.Get("snappy").MustBool(), true)
}
func TestReconfigure(t *testing.T) {
lopts := nsqlookupd.NewOptions()
lopts.Logger = newTestLogger(t)
nsqlookupd.SetLogger(lopts)
_, _, lookupd1 := mustStartNSQLookupd(lopts)
defer lookupd1.Exit()
_, _, lookupd2 := mustStartNSQLookupd(lopts)
defer lookupd2.Exit()
_, _, lookupd3 := mustStartNSQLookupd(lopts)
defer lookupd3.Exit()
opts := nsqdNs.NewOptions()
opts.Logger = newTestLogger(t)
_, _, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
time.Sleep(50 * time.Millisecond)
newOpts := *opts
newOpts.NSQLookupdTCPAddresses = []string{lookupd1.RealTCPAddr().String()}
nsqd.SwapOpts(&newOpts)
nsqd.TriggerOptsNotification()
test.Equal(t, len(nsqd.GetOpts().NSQLookupdTCPAddresses), 1)
time.Sleep(50 * time.Millisecond)
numLookupPeers := len(nsqdServer.lookupPeers.Load().([]*clusterinfo.LookupPeer))
test.Equal(t, numLookupPeers, 1)
newOpts = *opts
newOpts.NSQLookupdTCPAddresses = []string{lookupd2.RealTCPAddr().String(), lookupd3.RealTCPAddr().String()}
nsqd.SwapOpts(&newOpts)
nsqd.TriggerOptsNotification()
test.Equal(t, len(nsqd.GetOpts().NSQLookupdTCPAddresses), 2)
time.Sleep(50 * time.Millisecond)
var lookupPeers []string
for _, lp := range nsqdServer.lookupPeers.Load().([]*clusterinfo.LookupPeer) {
lookupPeers = append(lookupPeers, lp.String())
}
test.Equal(t, len(lookupPeers), 2)
test.Equal(t, lookupPeers, newOpts.NSQLookupdTCPAddresses)
}
func TestHTTPSRequire(t *testing.T) {
opts := nsqd.NewOptions()
opts.Logger = newTestLogger(t)
//opts.LogLevel = 2
//opts.Logger = &levellogger.GLogger{}
opts.LogLevel = 3
opts.TLSCert = "./test/certs/server.pem"
opts.TLSKey = "./test/certs/server.key"
opts.TLSClientAuthPolicy = "require"
_, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
topicName := "test_http_pub_req" + strconv.Itoa(int(time.Now().Unix()))
nsqd.GetTopicIgnPart(topicName)
buf := bytes.NewBuffer([]byte("test message"))
url := fmt.Sprintf("http://%s/pub?topic=%s", httpAddr, topicName)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Equal(t, resp.StatusCode, 403)
httpsAddr := nsqdServer.httpsListener.Addr().(*net.TCPAddr)
cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem")
test.Equal(t, err, nil)
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{cert},
InsecureSkipVerify: true,
MinVersion: 0,
}
transport := &http.Transport{
TLSClientConfig: tlsConfig,
}
client := &http.Client{Transport: transport}
buf = bytes.NewBuffer([]byte("test message"))
url = fmt.Sprintf("https://%s/pub?topic=%s", httpsAddr, topicName)
resp, err = client.Post(url, "application/octet-stream", buf)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, string(body), "OK")
time.Sleep(5 * time.Millisecond)
}
func bootstrapNSQCluster(t *testing.T) (string, []*nsqd.NSQD, []*nsqdserver.NsqdServer, []*nsqlookupd.NSQLookupd, *NSQAdmin) {
lgr := newTestLogger(t)
nsqlookupdOpts := nsqlookupd.NewOptions()
nsqlookupdOpts.TCPAddress = "127.0.0.1:0"
nsqlookupdOpts.HTTPAddress = "127.0.0.1:0"
nsqlookupdOpts.BroadcastAddress = "127.0.0.1"
nsqlookupdOpts.Logger = lgr
nsqlookupd.SetLogger(nsqlookupdOpts)
nsqlookupd1 := nsqlookupd.New(nsqlookupdOpts)
go nsqlookupd1.Main()
// wait http server
time.Sleep(time.Second)
if nsqlookupd1.RealHTTPAddr().String() == "" {
t.Fatal("lookupd should not empty")
}
nsqdOpts := nsqd.NewOptions()
nsqdOpts.TCPAddress = "127.0.0.1:0"
nsqdOpts.HTTPAddress = "127.0.0.1:0"
nsqdOpts.BroadcastAddress = "127.0.0.1"
nsqdOpts.NSQLookupdTCPAddresses = []string{nsqlookupd1.RealTCPAddr().String()}
nsqdOpts.Logger = lgr
tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano()))
if err != nil {
panic(err)
}
nsqdOpts.DataPath = tmpDir
nsqd1, nsqd1Server := nsqdserver.NewNsqdServer(nsqdOpts)
go nsqd1Server.Main()
t.Log("nsqd started")
nsqadminOpts := NewOptions()
nsqadminOpts.HTTPAddress = "127.0.0.1:0"
nsqadminOpts.NSQLookupdHTTPAddresses = []string{nsqlookupd1.RealHTTPAddr().String()}
nsqadminOpts.Logger = lgr
nsqadmin1 := New(nsqadminOpts)
go nsqadmin1.Main()
time.Sleep(100 * time.Millisecond)
return tmpDir, []*nsqd.NSQD{nsqd1}, []*nsqdserver.NsqdServer{nsqd1Server}, []*nsqlookupd.NSQLookupd{nsqlookupd1}, nsqadmin1
}
func TestHTTPgetStatusJSON(t *testing.T) {
testTime := time.Now()
opts := nsqd.NewOptions()
opts.Logger = newTestLogger(t)
_, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
testTime = nsqd.GetStartTime()
expectedJSON := fmt.Sprintf(`{"status_code":200,"status_txt":"OK","data":{"version":"%v","health":"OK","start_time":%v,"topics":[]}}`, version.Binary, testTime.Unix())
url := fmt.Sprintf("http://%s/stats?format=json", httpAddr)
resp, err := http.Get(url)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, resp.StatusCode, 200)
test.Equal(t, string(body), expectedJSON)
}
func BenchmarkHTTPpub(b *testing.B) {
var wg sync.WaitGroup
b.StopTimer()
opts := nsqd.NewOptions()
opts.Logger = newTestLogger(b)
opts.MemQueueSize = int64(b.N)
_, httpAddr, nsqdData, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
msg := make([]byte, 256)
topicName := "bench_http_pub" + strconv.Itoa(int(time.Now().Unix()))
url := fmt.Sprintf("http://%s/pub?topic=%s", httpAddr, topicName)
client := &http.Client{}
nsqdData.GetTopic(topicName, 0)
b.SetBytes(int64(len(msg)))
b.StartTimer()
for j := 0; j < runtime.GOMAXPROCS(0); j++ {
wg.Add(1)
go func() {
num := b.N / runtime.GOMAXPROCS(0)
for i := 0; i < num; i++ {
buf := bytes.NewBuffer(msg)
req, _ := http.NewRequest("POST", url, buf)
resp, err := client.Do(req)
if err != nil {
panic(err.Error())
}
body, _ := ioutil.ReadAll(resp.Body)
if !bytes.Equal(body, []byte("OK")) {
panic("bad response:" + string(body))
}
resp.Body.Close()
}
wg.Done()
}()
}
wg.Wait()
b.StopTimer()
nsqdServer.Exit()
}
func TestConfigFlagParsing(t *testing.T) {
opts := nsqd.NewOptions()
flagSet := nsqdFlagSet(opts)
flagSet.Parse([]string{})
var cfg config
f, err := os.Open("../../contrib/nsqd.cfg.example")
if err != nil {
t.Fatalf("%s", err)
}
toml.DecodeReader(f, &cfg)
cfg.Validate()
options.Resolve(opts, flagSet, cfg)
nsqd.New(opts)
if opts.TLSMinVersion != tls.VersionTLS10 {
t.Errorf("min %#v not expected %#v", opts.TLSMinVersion, tls.VersionTLS10)
}
}
func TestHTTPpubEmpty(t *testing.T) {
opts := nsqd.NewOptions()
opts.Logger = newTestLogger(t)
_, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
topicName := "test_http_pub_empty" + strconv.Itoa(int(time.Now().Unix()))
nsqd.GetTopicIgnPart(topicName)
buf := bytes.NewBuffer([]byte(""))
url := fmt.Sprintf("http://%s/pub?topic=%s", httpAddr, topicName)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, resp.StatusCode, 500)
test.Equal(t, string(body), `{"status_code":500,"status_txt":"MSG_EMPTY","data":null}`)
time.Sleep(5 * time.Millisecond)
}
func TestHTTPpubpartition(t *testing.T) {
opts := nsqd.NewOptions()
opts.LogLevel = 2
opts.Logger = newTestLogger(t)
//opts.Logger = &levellogger.GLogger{}
_, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
topicName := "test_http_pub_partition" + strconv.Itoa(int(time.Now().Unix()))
_ = nsqd.GetTopicIgnPart(topicName)
buf := bytes.NewBuffer([]byte("test message"))
// should failed pub to not exist partition
url := fmt.Sprintf("http://%s/pub?topic=%s&partition=2", httpAddr, topicName)
resp, err := http.Post(url, "application/octet-stream", buf)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.NotEqual(t, string(body), "OK")
time.Sleep(5 * time.Millisecond)
}
func TestHTTPerrors(t *testing.T) {
opts := nsqd.NewOptions()
opts.Logger = newTestLogger(t)
_, httpAddr, _, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
url := fmt.Sprintf("http://%s/stats", httpAddr)
resp, err := http.Post(url, "text/plain", nil)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
test.Equal(t, resp.StatusCode, 405)
test.Equal(t, string(body), `{"message":"METHOD_NOT_ALLOWED"}`)
url = fmt.Sprintf("http://%s/not_found", httpAddr)
resp, err = http.Get(url)
test.Equal(t, err, nil)
defer resp.Body.Close()
body, _ = ioutil.ReadAll(resp.Body)
test.Equal(t, resp.StatusCode, 404)
test.Equal(t, string(body), `{"message":"NOT_FOUND"}`)
}
func TestChannelEmptyConsumer(t *testing.T) {
opts := nsqdNs.NewOptions()
opts.Logger = newTestLogger(t)
tcpAddr, _, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
conn, _ := mustConnectNSQD(tcpAddr)
defer conn.Close()
topicName := "test_channel_empty" + strconv.Itoa(int(time.Now().Unix()))
topic := nsqd.GetTopicIgnPart(topicName)
channel := topic.GetChannel("channel")
client := nsqdNs.NewClientV2(0, conn, opts, nil)
client.SetReadyCount(25)
channel.AddClient(client.ID, client)
for i := 0; i < 25; i++ {
msg := nsqdNs.NewMessage(nsqdNs.MessageID(i), []byte("test"))
channel.StartInFlightTimeout(msg, 0, "", opts.MsgTimeout)
client.SendingMessage()
}
for _, cl := range channel.GetClients() {
stats := cl.Stats()
test.Equal(t, stats.InFlightCount, int64(25))
}
channel.SetConsumeOffset(channel.GetChannelEnd().Offset(), channel.GetChannelEnd().TotalMsgCnt(), true)
time.Sleep(time.Second)
for _, cl := range channel.GetClients() {
stats := cl.Stats()
test.Equal(t, stats.InFlightCount, int64(0))
}
}
func TestCluster(t *testing.T) {
lopts := nsqlookupd.NewOptions()
lopts.Logger = newTestLogger(t)
lopts.BroadcastAddress = "127.0.0.1"
nsqlookupd.SetLogger(lopts)
_, _, lookupd := mustStartNSQLookupd(lopts)
opts := nsqdNs.NewOptions()
opts.Logger = newTestLogger(t)
opts.NSQLookupdTCPAddresses = []string{lookupd.RealTCPAddr().String()}
opts.BroadcastAddress = "127.0.0.1"
tcpAddr, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
topicName := "cluster_test" + strconv.Itoa(int(time.Now().Unix()))
partitionStr := "0"
hostname, err := os.Hostname()
test.Equal(t, err, nil)
nsqd.GetTopicIgnPart(topicName)
url := fmt.Sprintf("http://%s/channel/create?topic=%s&channel=ch", httpAddr, topicName)
_, err = http_api.NewClient(nil).POSTV1(url)
test.Equal(t, err, nil)
// allow some time for nsqd to push info to nsqlookupd
time.Sleep(350 * time.Millisecond)
endpoint := fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr())
data, err := API(endpoint)
test.Equal(t, err, nil)
t.Logf("debug data: %v", data)
topicData := data.Get("topic:" + topicName)
producers, _ := topicData.Array()
test.Equal(t, len(producers), 1)
producer := topicData.GetIndex(0)
test.Equal(t, producer.Get("hostname").MustString(), hostname)
test.Equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
test.Equal(t, producer.Get("tcp_port").MustInt(), tcpAddr.Port)
test.Equal(t, producer.Get("tombstoned").MustBool(), false)
channelData := data.Get("channel:" + topicName + ":" + partitionStr)
producers, _ = channelData.Array()
test.Equal(t, len(producers), 1)
producer = topicData.GetIndex(0)
test.Equal(t, producer.Get("hostname").MustString(), hostname)
test.Equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
test.Equal(t, producer.Get("tcp_port").MustInt(), tcpAddr.Port)
test.Equal(t, producer.Get("tombstoned").MustBool(), false)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName)
data, err = API(endpoint)
producers, _ = data.Get("producers").Array()
test.Equal(t, len(producers), 1)
producer = data.Get("producers").GetIndex(0)
test.Equal(t, producer.Get("hostname").MustString(), hostname)
test.Equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
test.Equal(t, producer.Get("tcp_port").MustInt(), tcpAddr.Port)
channels, _ := data.Get("channels").Array()
test.Equal(t, len(channels), 1)
channel := channels[0].(string)
test.Equal(t, channel, "ch")
nsqd.DeleteExistingTopic(topicName, 0)
// allow some time for nsqd to push info to nsqlookupd
time.Sleep(350 * time.Millisecond)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName)
data, err = API(endpoint)
test.Equal(t, err, nil)
producers, _ = data.Get("producers").Array()
test.Equal(t, len(producers), 0)
endpoint = fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr())
data, err = API(endpoint)
test.Equal(t, err, nil)
producers, _ = data.Get("topic:" + topicName).Array()
test.Equal(t, len(producers), 0)
producers, _ = data.Get("channel:" + topicName + ":" + partitionStr).Array()
test.Equal(t, len(producers), 0)
}
func TestNSQDStatsFilter(t *testing.T) {
t.Logf("Starts nsqd...")
nsqdOpts := nsqd.NewOptions()
nsqdOpts.Logger = newTestLogger(t)
tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano()))
if err != nil {
panic(err)
}
nsqdOpts.DataPath = tmpDir
_, nsqdHTTPAddr, nsqd, _ := mustStartNSQD(nsqdOpts)
t.Logf("nsqd started")
//delete existing topics if there is any
topicCnt := 0
for _, topic := range nsqd.GetStats(false) {
parNum, _ := strconv.Atoi(topic.TopicPartition)
nsqd.DeleteExistingTopic(topic.TopicName, parNum)
topicCnt++
}
t.Logf("%d topic(s) deleted.", topicCnt)
topicName := fmt.Sprintf("test_nsqd_stats_filter%d", time.Now().UnixNano())
nsqd.GetTopic(topicName, 0)
defer nsqd.DeleteExistingTopic(topicName, 0)
topicNameAnother := fmt.Sprintf("test_nsqd_stats_filter_another%d", time.Now().UnixNano())
nsqd.GetTopic(topicNameAnother, 0)
topic_another, err := nsqd.GetExistingTopic(topicNameAnother, 0)
topic_another.DisableForSlave()
defer nsqd.DeleteExistingTopic(topicNameAnother, 0)
time.Sleep(500 * time.Millisecond)
client := http.Client{}
url := fmt.Sprintf("http://%s/stats?format=json", nsqdHTTPAddr)
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Accept", "application/vnd.nsq; version=1.0")
resp, err := client.Do(req)
test.Assert(t, err == nil, "error in first response.")
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
type NSQDStats struct {
TopicStats []map[string]interface{} `json:"topics"`
Version string `json:"version"`
Health string `json:"health"`
StartTime int64 `json:"start_time"`
}
var stats NSQDStats
json.Unmarshal(body, &stats)
t.Logf("topic len in first response: %d", len(stats.TopicStats))
test.Assert(t, len(stats.TopicStats) == 2, "topic number does not match.")
url = fmt.Sprintf("http://%s/stats?format=json&leaderOnly=true", nsqdHTTPAddr)
req, _ = http.NewRequest("GET", url, nil)
req.Header.Add("Accept", "application/vnd.nsq; version=1.0")
resp, err = client.Do(req)
test.Assert(t, err == nil, "error in second response.")
body, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close()
var statFiltered NSQDStats
json.Unmarshal(body, &statFiltered)
t.Logf("topic len: %d", len(statFiltered.TopicStats))
test.Assert(t, len(statFiltered.TopicStats) == 1, "topic number is not empty.")
}
func TestHTTPV1TopicChannel(t *testing.T) {
opts := nsqd.NewOptions()
opts.Logger = newTestLogger(t)
_, httpAddr, nsqd, nsqdServer := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqdServer.Exit()
topicName := "test_http_topic_channel2" + strconv.Itoa(int(time.Now().Unix()))
topicPart := 0
channelName := "ch2"
nsqd.GetTopicIgnPart(topicName)
url := fmt.Sprintf("http://%s/channel/create?topic=%s&channel=%s", httpAddr, topicName, channelName)
resp, err := http.Post(url, "application/json", nil)
test.Equal(t, err, nil)
test.Equal(t, resp.StatusCode, 200)
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
test.Equal(t, string(body), "")
test.Equal(t, resp.Header.Get("X-NSQ-Content-Type"), "nsq; version=1.0")
topic, err := nsqd.GetExistingTopic(topicName, topicPart)
test.Equal(t, err, nil)
test.NotNil(t, topic)
channel, err := topic.GetExistingChannel(channelName)
test.Equal(t, err, nil)
test.NotNil(t, channel)
url = fmt.Sprintf("http://%s/channel/pause?topic=%s&channel=%s", httpAddr, topicName, channelName)
resp, err = http.Post(url, "application/json", nil)
test.Equal(t, err, nil)
test.Equal(t, resp.StatusCode, 200)
body, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close()
test.Equal(t, string(body), "")
test.Equal(t, resp.Header.Get("X-NSQ-Content-Type"), "nsq; version=1.0")
test.Equal(t, channel.IsPaused(), true)
url = fmt.Sprintf("http://%s/channel/unpause?topic=%s&channel=%s", httpAddr, topicName, channelName)
resp, err = http.Post(url, "application/json", nil)
test.Equal(t, err, nil)
test.Equal(t, resp.StatusCode, 200)
body, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close()
test.Equal(t, string(body), "")
test.Equal(t, resp.Header.Get("X-NSQ-Content-Type"), "nsq; version=1.0")
test.Equal(t, channel.IsPaused(), false)
url = fmt.Sprintf("http://%s/channel/delete?topic=%s&channel=%s", httpAddr, topicName, channelName)
resp, err = http.Post(url, "application/json", nil)
test.Equal(t, err, nil)
test.Equal(t, resp.StatusCode, 200)
body, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close()
test.Equal(t, string(body), "")
test.Equal(t, resp.Header.Get("X-NSQ-Content-Type"), "nsq; version=1.0")
_, err = topic.GetExistingChannel(channelName)
test.NotNil(t, err)
nsqd.DeleteExistingTopic(topicName, topicPart)
_, err = nsqd.GetExistingTopic(topicName, topicPart)
test.NotNil(t, err)
}
