func API(endpoint string) (data *simplejson.Json, err error) {
d := make(map[string]interface{})
err = http_api.NewClient(nil).NegotiateV1(endpoint, &d)
data = simplejson.New()
data.SetPath(nil, d)
return
}
func TestInactiveNodes(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.InactiveProducerTimeout = 200 * time.Millisecond
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)}
topicName := "inactive_nodes"
conn := mustConnectLookupd(t, tcpAddr)
defer conn.Close()
identify(t, conn, "ip.address", 5000, 5555, "fake-version")
nsq.Register(topicName, "channel1").WriteTo(conn)
_, err := nsq.ReadResponse(conn)
equal(t, err, nil)
ci := clusterinfo.New(nil, http_api.NewClient(nil))
producers, _ := ci.GetLookupdProducers(lookupdHTTPAddrs)
equal(t, len(producers), 1)
equal(t, len(producers[0].Topics), 1)
equal(t, producers[0].Topics[0].Topic, topicName)
equal(t, producers[0].Topics[0].Tombstoned, false)
time.Sleep(250 * time.Millisecond)
producers, _ = ci.GetLookupdProducers(lookupdHTTPAddrs)
equal(t, len(producers), 0)
}
func TestInactiveNodes(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.InactiveProducerTimeout = 200 * time.Millisecond
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)}
topicName := "inactive_nodes"
conn := mustConnectLookupd(t, tcpAddr)
defer conn.Close()
identify(t, conn)
nsq.Register(topicName, "channel1").WriteTo(conn)
_, err := nsq.ReadResponse(conn)
test.Nil(t, err)
ci := clusterinfo.New(nil, http_api.NewClient(nil, ConnectTimeout, RequestTimeout))
producers, _ := ci.GetLookupdProducers(lookupdHTTPAddrs)
test.Equal(t, 1, len(producers))
test.Equal(t, 1, len(producers[0].Topics))
test.Equal(t, topicName, producers[0].Topics[0].Topic)
test.Equal(t, false, producers[0].Topics[0].Tombstoned)
time.Sleep(250 * time.Millisecond)
producers, _ = ci.GetLookupdProducers(lookupdHTTPAddrs)
test.Equal(t, 0, len(producers))
}
func TestTombstoneRecover(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.TombstoneLifetime = 50 * time.Millisecond
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
topicName := "tombstone_recover"
topicName2 := topicName + "2"
conn := mustConnectLookupd(t, tcpAddr)
defer conn.Close()
identify(t, conn)
nsq.Register(topicName, "channel1").WriteTo(conn)
_, err := nsq.ReadResponse(conn)
test.Nil(t, err)
nsq.Register(topicName2, "channel2").WriteTo(conn)
_, err = nsq.ReadResponse(conn)
test.Nil(t, err)
endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s:%d",
httpAddr, topicName, HostAddr, HTTPPort)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(endpoint)
test.Nil(t, err)
pr := ProducersDoc{}
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).NegotiateV1(endpoint, &pr)
test.Nil(t, err)
test.Equal(t, 0, len(pr.Producers))
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName2)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).NegotiateV1(endpoint, &pr)
test.Nil(t, err)
test.Equal(t, 1, len(pr.Producers))
time.Sleep(75 * time.Millisecond)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).NegotiateV1(endpoint, &pr)
test.Nil(t, err)
test.Equal(t, 1, len(pr.Producers))
}
func New(opts *Options) *NSQD {
n := &NSQD{
flag: flagHealthy,
startTime: time.Now(),
topicMap: make(map[string]*Topic),
idChan: make(chan MessageID, 4096),
exitChan: make(chan int),
notifyChan: make(chan interface{}),
optsNotificationChan: make(chan struct{}, 1),
ci: clusterinfo.New(opts.Logger, http_api.NewClient(nil)),
}
n.swapOpts(opts)
if opts.MaxDeflateLevel < 1 || opts.MaxDeflateLevel > 9 {
n.logf("FATAL: --max-deflate-level must be [1,9]")
os.Exit(1)
}
if opts.ID < 0 || opts.ID >= 1024 {
n.logf("FATAL: --worker-id must be [0,1024)")
os.Exit(1)
}
if opts.StatsdPrefix != "" {
_, port, err := net.SplitHostPort(opts.HTTPAddress)
if err != nil {
n.logf("ERROR: failed to parse HTTP address (%s) - %s", opts.HTTPAddress, err)
os.Exit(1)
}
statsdHostKey := statsd.HostKey(net.JoinHostPort(opts.BroadcastAddress, port))
prefixWithHost := strings.Replace(opts.StatsdPrefix, "%s", statsdHostKey, -1)
if prefixWithHost[len(prefixWithHost)-1] != '.' {
prefixWithHost += "."
}
opts.StatsdPrefix = prefixWithHost
}
if opts.TLSClientAuthPolicy != "" && opts.TLSRequired == TLSNotRequired {
opts.TLSRequired = TLSRequired
}
tlsConfig, err := buildTLSConfig(opts)
if err != nil {
n.logf("FATAL: failed to build TLS config - %s", err)
os.Exit(1)
}
if tlsConfig == nil && opts.TLSRequired != TLSNotRequired {
n.logf("FATAL: cannot require TLS client connections without TLS key and cert")
os.Exit(1)
}
n.tlsConfig = tlsConfig
n.logf(version.String("nsqd"))
n.logf("ID: %d", opts.ID)
return n
}
func NewHTTPServer(ctx *Context) *httpServer {
log := http_api.Log(ctx.nsqadmin.getOpts().Logger)
client := http_api.NewClient(ctx.nsqadmin.httpClientTLSConfig, ctx.nsqadmin.getOpts().HTTPClientConnectTimeout,
ctx.nsqadmin.getOpts().HTTPClientRequestTimeout)
router := httprouter.New()
router.HandleMethodNotAllowed = true
router.PanicHandler = http_api.LogPanicHandler(ctx.nsqadmin.getOpts().Logger)
router.NotFound = http_api.LogNotFoundHandler(ctx.nsqadmin.getOpts().Logger)
router.MethodNotAllowed = http_api.LogMethodNotAllowedHandler(ctx.nsqadmin.getOpts().Logger)
s := &httpServer{
ctx: ctx,
router: router,
client: client,
ci: clusterinfo.New(ctx.nsqadmin.getOpts().Logger, client),
}
router.Handle("GET", "/ping", http_api.Decorate(s.pingHandler, log, http_api.PlainText))
router.Handle("GET", "/", http_api.Decorate(s.indexHandler, log))
router.Handle("GET", "/topics", http_api.Decorate(s.indexHandler, log))
router.Handle("GET", "/topics/:topic", http_api.Decorate(s.indexHandler, log))
router.Handle("GET", "/topics/:topic/:channel", http_api.Decorate(s.indexHandler, log))
router.Handle("GET", "/nodes", http_api.Decorate(s.indexHandler, log))
router.Handle("GET", "/nodes/:node", http_api.Decorate(s.indexHandler, log))
router.Handle("GET", "/counter", http_api.Decorate(s.indexHandler, log))
router.Handle("GET", "/lookup", http_api.Decorate(s.indexHandler, log))
router.Handle("GET", "/static/:asset", http_api.Decorate(s.staticAssetHandler, log, http_api.PlainText))
router.Handle("GET", "/fonts/:asset", http_api.Decorate(s.staticAssetHandler, log, http_api.PlainText))
if s.ctx.nsqadmin.getOpts().ProxyGraphite {
proxy := NewSingleHostReverseProxy(ctx.nsqadmin.graphiteURL, ctx.nsqadmin.getOpts().HTTPClientConnectTimeout,
ctx.nsqadmin.getOpts().HTTPClientRequestTimeout)
router.Handler("GET", "/render", proxy)
}
// v1 endpoints
router.Handle("GET", "/api/topics", http_api.Decorate(s.topicsHandler, log, http_api.V1))
router.Handle("GET", "/api/topics/:topic", http_api.Decorate(s.topicHandler, log, http_api.V1))
router.Handle("GET", "/api/topics/:topic/:channel", http_api.Decorate(s.channelHandler, log, http_api.V1))
router.Handle("GET", "/api/nodes", http_api.Decorate(s.nodesHandler, log, http_api.V1))
router.Handle("GET", "/api/nodes/:node", http_api.Decorate(s.nodeHandler, log, http_api.V1))
router.Handle("POST", "/api/topics", http_api.Decorate(s.createTopicChannelHandler, log, http_api.V1))
router.Handle("POST", "/api/topics/:topic", http_api.Decorate(s.topicActionHandler, log, http_api.V1))
router.Handle("POST", "/api/topics/:topic/:channel", http_api.Decorate(s.channelActionHandler, log, http_api.V1))
router.Handle("DELETE", "/api/nodes/:node", http_api.Decorate(s.tombstoneNodeForTopicHandler, log, http_api.V1))
router.Handle("DELETE", "/api/topics/:topic", http_api.Decorate(s.deleteTopicHandler, log, http_api.V1))
router.Handle("DELETE", "/api/topics/:topic/:channel", http_api.Decorate(s.deleteChannelHandler, log, http_api.V1))
router.Handle("GET", "/api/counter", http_api.Decorate(s.counterHandler, log, http_api.V1))
router.Handle("GET", "/api/graphite", http_api.Decorate(s.graphiteHandler, log, http_api.V1))
router.Handle("GET", "/config/:opt", http_api.Decorate(s.doConfig, log, http_api.V1))
router.Handle("PUT", "/config/:opt", http_api.Decorate(s.doConfig, log, http_api.V1))
return s
}
func TestClientAttributes(t *testing.T) {
userAgent := "Test User Agent"
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.Verbose = true
opts.SnappyEnabled = true
tcpAddr, httpAddr, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
conn, err := mustConnectNSQD(tcpAddr)
test.Nil(t, err)
defer conn.Close()
data := identify(t, conn, map[string]interface{}{
"snappy": true,
"user_agent": userAgent,
}, frameTypeResponse)
resp := struct {
Snappy bool `json:"snappy"`
UserAgent string `json:"user_agent"`
}{}
err = json.Unmarshal(data, &resp)
test.Nil(t, err)
test.Equal(t, true, resp.Snappy)
r := snappystream.NewReader(conn, snappystream.SkipVerifyChecksum)
w := snappystream.NewWriter(conn)
readValidate(t, r, frameTypeResponse, "OK")
topicName := "test_client_attributes" + strconv.Itoa(int(time.Now().Unix()))
sub(t, readWriter{r, w}, topicName, "ch")
var d struct {
Topics []struct {
Channels []struct {
Clients []struct {
UserAgent string `json:"user_agent"`
Snappy bool `json:"snappy"`
} `json:"clients"`
} `json:"channels"`
} `json:"topics"`
}
endpoint := fmt.Sprintf("http://127.0.0.1:%d/stats?format=json", httpAddr.Port)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &d)
test.Nil(t, err)
test.Equal(t, userAgent, d.Topics[0].Channels[0].Clients[0].UserAgent)
test.Equal(t, true, d.Topics[0].Channels[0].Clients[0].Snappy)
}
func TestTombstonedNodes(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)}
topicName := "inactive_nodes"
conn := mustConnectLookupd(t, tcpAddr)
defer conn.Close()
identify(t, conn, "ip.address", 5000, 5555, "fake-version")
nsq.Register(topicName, "channel1").WriteTo(conn)
_, err := nsq.ReadResponse(conn)
equal(t, err, nil)
ci := clusterinfo.New(nil, http_api.NewClient(nil))
producers, _ := ci.GetLookupdProducers(lookupdHTTPAddrs)
equal(t, len(producers), 1)
equal(t, len(producers[0].Topics), 1)
equal(t, producers[0].Topics[0].Topic, topicName)
equal(t, producers[0].Topics[0].Tombstoned, false)
endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s",
httpAddr, topicName, "ip.address:5555")
err = http_api.NewClient(nil).POSTV1(endpoint)
equal(t, err, nil)
producers, _ = ci.GetLookupdProducers(lookupdHTTPAddrs)
equal(t, len(producers), 1)
equal(t, len(producers[0].Topics), 1)
equal(t, producers[0].Topics[0].Topic, topicName)
equal(t, producers[0].Topics[0].Tombstoned, true)
}
func TestTombstonedNodes(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)}
topicName := "inactive_nodes"
conn := mustConnectLookupd(t, tcpAddr)
defer conn.Close()
identify(t, conn)
nsq.Register(topicName, "channel1").WriteTo(conn)
_, err := nsq.ReadResponse(conn)
test.Nil(t, err)
ci := clusterinfo.New(nil, http_api.NewClient(nil, ConnectTimeout, RequestTimeout))
producers, _ := ci.GetLookupdProducers(lookupdHTTPAddrs)
test.Equal(t, 1, len(producers))
test.Equal(t, 1, len(producers[0].Topics))
test.Equal(t, topicName, producers[0].Topics[0].Topic)
test.Equal(t, false, producers[0].Topics[0].Tombstoned)
endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s:%d",
httpAddr, topicName, HostAddr, HTTPPort)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(endpoint)
test.Nil(t, err)
producers, _ = ci.GetLookupdProducers(lookupdHTTPAddrs)
test.Equal(t, 1, len(producers))
test.Equal(t, 1, len(producers[0].Topics))
test.Equal(t, topicName, producers[0].Topics[0].Topic)
test.Equal(t, true, producers[0].Topics[0].Tombstoned)
}
func TestTombstoneRecover(t *testing.T) {
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.TombstoneLifetime = 50 * time.Millisecond
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
topicName := "tombstone_recover"
topicName2 := topicName + "2"
conn := mustConnectLookupd(t, tcpAddr)
defer conn.Close()
identify(t, conn, "ip.address", 5000, 5555, "fake-version")
nsq.Register(topicName, "channel1").WriteTo(conn)
_, err := nsq.ReadResponse(conn)
equal(t, err, nil)
nsq.Register(topicName2, "channel2").WriteTo(conn)
_, err = nsq.ReadResponse(conn)
equal(t, err, nil)
endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s",
httpAddr, topicName, "ip.address:5555")
err = http_api.NewClient(nil).POSTV1(endpoint)
equal(t, err, nil)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
data, err := API(endpoint)
equal(t, err, nil)
producers, _ := data.Get("producers").Array()
equal(t, len(producers), 0)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName2)
data, err = API(endpoint)
equal(t, err, nil)
producers, _ = data.Get("producers").Array()
equal(t, len(producers), 1)
time.Sleep(75 * time.Millisecond)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
data, err = API(endpoint)
equal(t, err, nil)
producers, _ = data.Get("producers").Array()
equal(t, len(producers), 1)
}
func TestChannelUnregister(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
topics := nsqlookupd.DB.FindRegistrations("topic", "*", "*")
test.Equal(t, 0, len(topics))
topicName := "channel_unregister"
conn := mustConnectLookupd(t, tcpAddr)
defer conn.Close()
identify(t, conn)
nsq.Register(topicName, "ch1").WriteTo(conn)
v, err := nsq.ReadResponse(conn)
test.Nil(t, err)
test.Equal(t, []byte("OK"), v)
topics = nsqlookupd.DB.FindRegistrations("topic", topicName, "")
test.Equal(t, 1, len(topics))
channels := nsqlookupd.DB.FindRegistrations("channel", topicName, "*")
test.Equal(t, 1, len(channels))
nsq.UnRegister(topicName, "ch1").WriteTo(conn)
v, err = nsq.ReadResponse(conn)
test.Nil(t, err)
test.Equal(t, []byte("OK"), v)
topics = nsqlookupd.DB.FindRegistrations("topic", topicName, "")
test.Equal(t, 1, len(topics))
// we should still have mention of the topic even though there is no producer
// (ie. we haven't *deleted* the channel, just unregistered as a producer)
channels = nsqlookupd.DB.FindRegistrations("channel", topicName, "*")
test.Equal(t, 1, len(channels))
pr := ProducersDoc{}
endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).NegotiateV1(endpoint, &pr)
test.Nil(t, err)
t.Logf("got %v", pr)
test.Equal(t, 1, len(pr.Producers))
}
func QueryAuthd(authd, remoteIP, tlsEnabled, authSecret string,
connectTimeout time.Duration, requestTimeout time.Duration) (*State, error) {
v := url.Values{}
v.Set("remote_ip", remoteIP)
v.Set("tls", tlsEnabled)
v.Set("secret", authSecret)
endpoint := fmt.Sprintf("http://%s/auth?%s", authd, v.Encode())
var authState State
client := http_api.NewClient(nil, connectTimeout, requestTimeout)
if err := client.GETV1(endpoint, &authState); err != nil {
return nil, err
}
// validation on response
for _, auth := range authState.Authorizations {
for _, p := range auth.Permissions {
switch p {
case "subscribe", "publish":
default:
return nil, fmt.Errorf("unknown permission %s", p)
}
}
if _, err := regexp.Compile(auth.Topic); err != nil {
return nil, fmt.Errorf("unable to compile topic %q %s", auth.Topic, err)
}
for _, channel := range auth.Channels {
if _, err := regexp.Compile(channel); err != nil {
return nil, fmt.Errorf("unable to compile channel %q %s", channel, err)
}
}
}
if authState.TTL <= 0 {
return nil, fmt.Errorf("invalid TTL %d (must be >0)", authState.TTL)
}
authState.Expires = time.Now().Add(time.Duration(authState.TTL) * time.Second)
return &authState, nil
}
func (t *TopicDiscoverer) syncTopics(addrs []string, pattern string) {
newTopics, err := clusterinfo.New(nil, http_api.NewClient(nil)).GetLookupdTopics(addrs)
if err != nil {
log.Printf("ERROR: could not retrieve topic list: %s", err)
}
for _, topic := range newTopics {
if _, ok := t.topics[topic]; !ok {
if !t.allowTopicName(pattern, topic) {
log.Println("Skipping topic ", topic, "as it didn't match required pattern:", pattern)
continue
}
logger, err := newConsumerFileLogger(topic, t.cfg)
if err != nil {
log.Printf("ERROR: couldn't create logger for new topic %s: %s", topic, err)
continue
}
t.topics[topic] = logger
go t.startTopicRouter(logger)
}
}
}
func TestCluster(t *testing.T) {
lopts := nsqlookupd.NewOptions()
lopts.Logger = newTestLogger(t)
lopts.BroadcastAddress = "127.0.0.1"
_, _, lookupd := mustStartNSQLookupd(lopts)
opts := NewOptions()
opts.Logger = newTestLogger(t)
opts.NSQLookupdTCPAddresses = []string{lookupd.RealTCPAddr().String()}
opts.BroadcastAddress = "127.0.0.1"
_, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "cluster_test" + strconv.Itoa(int(time.Now().Unix()))
hostname, err := os.Hostname()
equal(t, err, nil)
url := fmt.Sprintf("http://%s/topic/create?topic=%s", nsqd.RealHTTPAddr(), topicName)
err = http_api.NewClient(nil).POSTV1(url)
equal(t, err, nil)
url = fmt.Sprintf("http://%s/channel/create?topic=%s&channel=ch", nsqd.RealHTTPAddr(), topicName)
err = http_api.NewClient(nil).POSTV1(url)
equal(t, err, nil)
// allow some time for nsqd to push info to nsqlookupd
time.Sleep(350 * time.Millisecond)
endpoint := fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr())
data, err := API(endpoint)
equal(t, err, nil)
topicData := data.Get("topic:" + topicName + ":")
producers, _ := topicData.Array()
equal(t, len(producers), 1)
producer := topicData.GetIndex(0)
equal(t, producer.Get("hostname").MustString(), hostname)
equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
equal(t, producer.Get("tcp_port").MustInt(), nsqd.RealTCPAddr().Port)
equal(t, producer.Get("tombstoned").MustBool(), false)
channelData := data.Get("channel:" + topicName + ":ch")
producers, _ = channelData.Array()
equal(t, len(producers), 1)
producer = topicData.GetIndex(0)
equal(t, producer.Get("hostname").MustString(), hostname)
equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
equal(t, producer.Get("tcp_port").MustInt(), nsqd.RealTCPAddr().Port)
equal(t, producer.Get("tombstoned").MustBool(), false)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName)
data, err = API(endpoint)
producers, _ = data.Get("producers").Array()
equal(t, len(producers), 1)
producer = data.Get("producers").GetIndex(0)
equal(t, producer.Get("hostname").MustString(), hostname)
equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
equal(t, producer.Get("tcp_port").MustInt(), nsqd.RealTCPAddr().Port)
channels, _ := data.Get("channels").Array()
equal(t, len(channels), 1)
channel := channels[0].(string)
equal(t, channel, "ch")
url = fmt.Sprintf("http://%s/topic/delete?topic=%s", nsqd.RealHTTPAddr(), topicName)
err = http_api.NewClient(nil).POSTV1(url)
equal(t, err, nil)
// allow some time for nsqd to push info to nsqlookupd
time.Sleep(350 * time.Millisecond)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName)
data, err = API(endpoint)
equal(t, err, nil)
producers, _ = data.Get("producers").Array()
equal(t, len(producers), 0)
endpoint = fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr())
data, err = API(endpoint)
equal(t, err, nil)
producers, _ = data.Get("topic:" + topicName + ":").Array()
equal(t, len(producers), 0)
producers, _ = data.Get("channel:" + topicName + ":ch").Array()
equal(t, len(producers), 0)
}
func TestBasicLookupd(t *testing.T) {
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
topics := nsqlookupd.DB.FindRegistrations("topic", "*", "*")
test.Equal(t, 0, len(topics))
topicName := "connectmsg"
conn := mustConnectLookupd(t, tcpAddr)
identify(t, conn)
nsq.Register(topicName, "channel1").WriteTo(conn)
v, err := nsq.ReadResponse(conn)
test.Nil(t, err)
test.Equal(t, []byte("OK"), v)
pr := ProducersDoc{}
endpoint := fmt.Sprintf("http://%s/nodes", httpAddr)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).NegotiateV1(endpoint, &pr)
test.Nil(t, err)
t.Logf("got %v", pr)
test.Equal(t, 1, len(pr.Producers))
topics = nsqlookupd.DB.FindRegistrations("topic", topicName, "")
test.Equal(t, 1, len(topics))
producers := nsqlookupd.DB.FindProducers("topic", topicName, "")
test.Equal(t, 1, len(producers))
producer := producers[0]
test.Equal(t, HostAddr, producer.peerInfo.BroadcastAddress)
test.Equal(t, HostAddr, producer.peerInfo.Hostname)
test.Equal(t, TCPPort, producer.peerInfo.TCPPort)
test.Equal(t, HTTPPort, producer.peerInfo.HTTPPort)
tr := TopicsDoc{}
endpoint = fmt.Sprintf("http://%s/topics", httpAddr)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).NegotiateV1(endpoint, &tr)
test.Nil(t, err)
t.Logf("got %v", tr)
test.Equal(t, 1, len(tr.Topics))
lr := LookupDoc{}
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).NegotiateV1(endpoint, &lr)
test.Nil(t, err)
t.Logf("got %v", lr)
test.Equal(t, 1, len(lr.Channels))
test.Equal(t, 1, len(lr.Producers))
for _, p := range lr.Producers {
test.Equal(t, TCPPort, p.TCPPort)
test.Equal(t, HTTPPort, p.HTTPPort)
test.Equal(t, HostAddr, p.BroadcastAddress)
test.Equal(t, NSQDVersion, p.Version)
}
conn.Close()
time.Sleep(10 * time.Millisecond)
// now there should be no producers, but still topic/channel entries
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).NegotiateV1(endpoint, &lr)
test.Nil(t, err)
test.Equal(t, 1, len(lr.Channels))
test.Equal(t, 0, len(lr.Producers))
}
func TestCluster(t *testing.T) {
lopts := nsqlookupd.NewOptions()
lopts.Logger = test.NewTestLogger(t)
lopts.BroadcastAddress = "127.0.0.1"
_, _, lookupd := mustStartNSQLookupd(lopts)
opts := NewOptions()
opts.Logger = test.NewTestLogger(t)
opts.NSQLookupdTCPAddresses = []string{lookupd.RealTCPAddr().String()}
opts.BroadcastAddress = "127.0.0.1"
_, _, nsqd := mustStartNSQD(opts)
defer os.RemoveAll(opts.DataPath)
defer nsqd.Exit()
topicName := "cluster_test" + strconv.Itoa(int(time.Now().Unix()))
hostname, err := os.Hostname()
test.Nil(t, err)
url := fmt.Sprintf("http://%s/topic/create?topic=%s", nsqd.RealHTTPAddr(), topicName)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(url)
test.Nil(t, err)
url = fmt.Sprintf("http://%s/channel/create?topic=%s&channel=ch", nsqd.RealHTTPAddr(), topicName)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(url)
test.Nil(t, err)
// allow some time for nsqd to push info to nsqlookupd
time.Sleep(350 * time.Millisecond)
var d map[string][]struct {
Hostname string `json:"hostname"`
BroadcastAddress string `json:"broadcast_address"`
TCPPort int `json:"tcp_port"`
Tombstoned bool `json:"tombstoned"`
}
endpoint := fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr())
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &d)
test.Nil(t, err)
topicData := d["topic:"+topicName+":"]
test.Equal(t, 1, len(topicData))
test.Equal(t, hostname, topicData[0].Hostname)
test.Equal(t, "127.0.0.1", topicData[0].BroadcastAddress)
test.Equal(t, nsqd.RealTCPAddr().Port, topicData[0].TCPPort)
test.Equal(t, false, topicData[0].Tombstoned)
channelData := d["channel:"+topicName+":ch"]
test.Equal(t, 1, len(channelData))
test.Equal(t, hostname, channelData[0].Hostname)
test.Equal(t, "127.0.0.1", channelData[0].BroadcastAddress)
test.Equal(t, nsqd.RealTCPAddr().Port, channelData[0].TCPPort)
test.Equal(t, false, channelData[0].Tombstoned)
var lr struct {
Producers []struct {
Hostname string `json:"hostname"`
BroadcastAddress string `json:"broadcast_address"`
TCPPort int `json:"tcp_port"`
} `json:"producers"`
Channels []string `json:"channels"`
}
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &lr)
test.Nil(t, err)
test.Equal(t, 1, len(lr.Producers))
test.Equal(t, hostname, lr.Producers[0].Hostname)
test.Equal(t, "127.0.0.1", lr.Producers[0].BroadcastAddress)
test.Equal(t, nsqd.RealTCPAddr().Port, lr.Producers[0].TCPPort)
test.Equal(t, 1, len(lr.Channels))
test.Equal(t, "ch", lr.Channels[0])
url = fmt.Sprintf("http://%s/topic/delete?topic=%s", nsqd.RealHTTPAddr(), topicName)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(url)
test.Nil(t, err)
// allow some time for nsqd to push info to nsqlookupd
time.Sleep(350 * time.Millisecond)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName)
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &lr)
test.Nil(t, err)
test.Equal(t, 0, len(lr.Producers))
var dd map[string][]interface{}
endpoint = fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr())
err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &dd)
test.Nil(t, err)
test.Equal(t, 0, len(dd["topic:"+topicName+":"]))
test.Equal(t, 0, len(dd["channel:"+topicName+":ch"]))
}
func New(opts *Options) *NSQD {
//缓存消息的磁盘路径
dataPath := opts.DataPath
if opts.DataPath == "" {
cwd, _ := os.Getwd()
dataPath = cwd
}
n := &NSQD{
startTime: time.Now(),
topicMap: make(map[string]*Topic),
idChan: make(chan MessageID, 4096),
exitChan: make(chan int),
notifyChan: make(chan interface{}),
optsNotificationChan: make(chan struct{}, 1),
//WEB接听
ci: clusterinfo.New(opts.Logger, http_api.NewClient(nil)),
dl: dirlock.New(dataPath),
}
// 设置原子操作对象(配置信息) --???--是否还可以正常方式读写 (sync/atomic) (原子操作的效率要比锁来的高)
n.swapOpts(opts)
n.errValue.Store(errStore{})
//阻止数据库文件的外来操作
err := n.dl.Lock()
if err != nil {
n.logf("FATAL: --data-path=%s in use (possibly by another instance of nsqd)", dataPath)
os.Exit(1)
}
//-max-deflate-level=6: 最大的压缩比率等级
if opts.MaxDeflateLevel < 1 || opts.MaxDeflateLevel > 9 {
n.logf("FATAL: --max-deflate-level must be [1,9]")
os.Exit(1)
}
//-worker-id=0: 进程的唯一码(默认是主机名的哈希值)
if opts.ID < 0 || opts.ID >= 1024 {
n.logf("FATAL: --worker-id must be [0,1024)")
os.Exit(1)
}
//-statsd-prefix="nsq.%s": 发送给统计keys 的前缀(%s for host replacement)
if opts.StatsdPrefix != "" { //生成显示前缀
var port string
//-http-address="0.0.0.0:4151": 为 HTTP 客户端监听 <addr>:<port>
_, port, err = net.SplitHostPort(opts.HTTPAddress)
if err != nil {
n.logf("ERROR: failed to parse HTTP address (%s) - %s", opts.HTTPAddress, err)
os.Exit(1)
}
statsdHostKey := statsd.HostKey(net.JoinHostPort(opts.BroadcastAddress, port))
prefixWithHost := strings.Replace(opts.StatsdPrefix, "%s", statsdHostKey, -1)
if prefixWithHost[len(prefixWithHost)-1] != '.' {
prefixWithHost += "."
}
opts.StatsdPrefix = prefixWithHost
}
//TLS安全文件传输协议
if opts.TLSClientAuthPolicy != "" && opts.TLSRequired == TLSNotRequired {
opts.TLSRequired = TLSRequired
}
tlsConfig, err := buildTLSConfig(opts)
if err != nil {
n.logf("FATAL: failed to build TLS config - %s", err)
os.Exit(1)
}
if tlsConfig == nil && opts.TLSRequired != TLSNotRequired {
n.logf("FATAL: cannot require TLS client connections without TLS key and cert")
os.Exit(1)
}
n.tlsConfig = tlsConfig
n.logf(version.String("nsqd"))
n.logf("ID: %d", opts.ID)
return n
}
func statLoop(interval time.Duration, topic string, channel string,
nsqdTCPAddrs []string, lookupdHTTPAddrs []string) {
ci := clusterinfo.New(nil, http_api.NewClient(nil))
var o *clusterinfo.ChannelStats
for i := 0; !countNum.isSet || countNum.value >= i; i++ {
var producers clusterinfo.Producers
var err error
if len(lookupdHTTPAddrs) != 0 {
producers, err = ci.GetLookupdTopicProducers(topic, lookupdHTTPAddrs)
} else {
producers, err = ci.GetNSQDTopicProducers(topic, nsqdHTTPAddrs)
}
if err != nil {
log.Fatalf("ERROR: failed to get topic producers - %s", err)
}
_, allChannelStats, err := ci.GetNSQDStats(producers, topic)
if err != nil {
log.Fatalf("ERROR: failed to get nsqd stats - %s", err)
}
c, ok := allChannelStats[channel]
if !ok {
log.Fatalf("ERROR: failed to find channel(%s) in stats metadata for topic(%s)", channel, topic)
}
if i%25 == 0 {
fmt.Printf("%s+%s+%s\n",
"------rate------",
"----------------depth----------------",
"--------------metadata---------------")
fmt.Printf("%7s %7s | %7s %7s %7s %5s %5s | %7s %7s %12s %7s\n",
"ingress", "egress",
"total", "mem", "disk", "inflt",
"def", "req", "t-o", "msgs", "clients")
}
if o == nil {
o = c
time.Sleep(interval)
continue
}
// TODO: paused
fmt.Printf("%7d %7d | %7d %7d %7d %5d %5d | %7d %7d %12d %7d\n",
int64(float64(c.MessageCount-o.MessageCount)/interval.Seconds()),
int64(float64(c.MessageCount-o.MessageCount-(c.Depth-o.Depth))/interval.Seconds()),
c.Depth,
c.MemoryDepth,
c.BackendDepth,
c.InFlightCount,
c.DeferredCount,
c.RequeueCount,
c.TimeoutCount,
c.MessageCount,
c.ClientCount)
o = c
time.Sleep(interval)
}
os.Exit(0)
}
func main() {
cfg := nsq.NewConfig()
// TODO: remove, deprecated
flag.Var(&nsq.ConfigFlag{cfg}, "reader-opt", "(deprecated) use --consumer-opt")
flag.Var(&nsq.ConfigFlag{cfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)")
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_file v%s\n", version.Binary)
return
}
if *channel == "" {
log.Fatal("--channel is required")
}
var topicsFromNSQLookupd bool
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatal("--nsqd-tcp-address or --lookupd-http-address required.")
}
if len(nsqdTCPAddrs) != 0 && len(lookupdHTTPAddrs) != 0 {
log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if *gzipLevel < 1 || *gzipLevel > 9 {
log.Fatalf("invalid --gzip-level value (%d), should be 1-9", *gzipLevel)
}
// TODO: remove, deprecated
if hasArg("gzip-compression") {
log.Printf("WARNING: --gzip-compression is deprecated in favor of --gzip-level")
switch *gzipCompression {
case 1:
*gzipLevel = gzip.BestSpeed
case 2:
*gzipLevel = gzip.BestCompression
case 3:
*gzipLevel = gzip.DefaultCompression
default:
log.Fatalf("invalid --gzip-compression value (%d), should be 1,2,3", *gzipCompression)
}
}
cfg.UserAgent = fmt.Sprintf("nsq_to_file/%s go-nsq/%s", version.Binary, nsq.VERSION)
cfg.MaxInFlight = *maxInFlight
discoverer := newTopicDiscoverer(cfg)
signal.Notify(discoverer.hupChan, syscall.SIGHUP)
signal.Notify(discoverer.termChan, syscall.SIGINT, syscall.SIGTERM)
if len(topics) < 1 {
if len(lookupdHTTPAddrs) < 1 {
log.Fatal("use --topic to list at least one topic to subscribe to or specify at least one --lookupd-http-address to subscribe to all its topics")
}
topicsFromNSQLookupd = true
var err error
topics, err = clusterinfo.New(nil, http_api.NewClient(nil)).GetLookupdTopics(lookupdHTTPAddrs)
if err != nil {
log.Fatalf("ERROR: could not retrieve topic list: %s", err)
}
}
for _, topic := range topics {
if !discoverer.allowTopicName(*topicPattern, topic) {
log.Println("Skipping topic", topic, "as it didn't match required pattern:", *topicPattern)
continue
}
logger, err := newConsumerFileLogger(topic, cfg)
if err != nil {
log.Fatalf("ERROR: couldn't create logger for topic %s: %s", topic, err)
}
discoverer.topics[topic] = logger
go discoverer.startTopicRouter(logger)
}
discoverer.watch(lookupdHTTPAddrs, topicsFromNSQLookupd, *topicPattern)
}
func New(opts *Options) *NSQD {
dataPath := opts.DataPath
if opts.DataPath == "" {
cwd, _ := os.Getwd()
dataPath = cwd
}
n := &NSQD{
startTime: time.Now(),
topicMap: make(map[string]*Topic),
idChan: make(chan MessageID, 4096),
exitChan: make(chan int),
notifyChan: make(chan interface{}),
optsNotificationChan: make(chan struct{}, 1),
ci: clusterinfo.New(opts.Logger, http_api.NewClient(nil, opts.HTTPClientConnectTimeout, opts.HTTPClientRequestTimeout)),
dl: dirlock.New(dataPath),
}
n.swapOpts(opts)
n.errValue.Store(errStore{})
err := n.dl.Lock()
if err != nil {
n.logf("FATAL: --data-path=%s in use (possibly by another instance of nsqd)", dataPath)
os.Exit(1)
}
if opts.MaxDeflateLevel < 1 || opts.MaxDeflateLevel > 9 {
n.logf("FATAL: --max-deflate-level must be [1,9]")
os.Exit(1)
}
if opts.ID < 0 || opts.ID >= 1024 {
n.logf("FATAL: --worker-id must be [0,1024)")
os.Exit(1)
}
if opts.StatsdPrefix != "" {
var port string
_, port, err = net.SplitHostPort(opts.HTTPAddress)
if err != nil {
n.logf("ERROR: failed to parse HTTP address (%s) - %s", opts.HTTPAddress, err)
os.Exit(1)
}
statsdHostKey := statsd.HostKey(net.JoinHostPort(opts.BroadcastAddress, port))
prefixWithHost := strings.Replace(opts.StatsdPrefix, "%s", statsdHostKey, -1)
if prefixWithHost[len(prefixWithHost)-1] != '.' {
prefixWithHost += "."
}
opts.StatsdPrefix = prefixWithHost
}
if opts.TLSClientAuthPolicy != "" && opts.TLSRequired == TLSNotRequired {
opts.TLSRequired = TLSRequired
}
tlsConfig, err := buildTLSConfig(opts)
if err != nil {
n.logf("FATAL: failed to build TLS config - %s", err)
os.Exit(1)
}
if tlsConfig == nil && opts.TLSRequired != TLSNotRequired {
n.logf("FATAL: cannot require TLS client connections without TLS key and cert")
os.Exit(1)
}
n.tlsConfig = tlsConfig
n.logf(version.String("nsqd"))
n.logf("ID: %d", opts.ID)
return n
}