func TestInactiveNodes(t *testing.T) {
opts := NewNSQLookupdOptions()
opts.Logger = newTestLogger(t)
opts.InactiveProducerTimeout = 200 * time.Millisecond
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)}
topicName := "inactive_nodes"
conn := mustConnectLookupd(t, tcpAddr)
defer conn.Close()
identify(t, conn, "ip.address", 5000, 5555, "fake-version")
nsq.Register(topicName, "channel1").WriteTo(conn)
_, err := nsq.ReadResponse(conn)
equal(t, err, nil)
producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs)
equal(t, len(producers), 1)
equal(t, len(producers[0].Topics), 1)
equal(t, producers[0].Topics[0].Topic, topicName)
equal(t, producers[0].Topics[0].Tombstoned, false)
time.Sleep(250 * time.Millisecond)
producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs)
equal(t, len(producers), 0)
}
func (s *httpServer) nodesHandler(w http.ResponseWriter, req *http.Request) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: failed to parse request params - %s", err)
http.Error(w, "INVALID_REQUEST", 500)
return
}
producers, _ := lookupd.GetLookupdProducers(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
p := struct {
Title string
Version string
GraphOptions *GraphOptions
Producers []*lookupd.Producer
Lookupd []string
}{
Title: "NSQ Nodes",
Version: version.Binary,
GraphOptions: NewGraphOptions(w, req, reqParams, s.ctx),
Producers: producers,
Lookupd: s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses,
}
err = templates.T.ExecuteTemplate(w, "nodes.html", p)
if err != nil {
s.ctx.nsqadmin.logf("Template Error %s", err)
http.Error(w, "Template Error", 500)
}
}
func (s *httpServer) nodesHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
producers, _ := lookupd.GetLookupdProducers(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
p := struct {
Title string
Version string
GraphOptions *GraphOptions
Producers []*lookupd.Producer
Lookupd []string
}{
Title: "NSQ Nodes",
Version: version.Binary,
GraphOptions: NewGraphOptions(w, req, reqParams, s.ctx),
Producers: producers,
Lookupd: s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses,
}
err = templates.T.ExecuteTemplate(w, "nodes.html", p)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: executing template - %s", err)
return nil, http_api.Err{500, "INTERNAL_ERROR"}
}
return nil, nil
}
// this endpoint works by giving out an ID that maps to a stats dictionary
// The initial request is the number of messages processed since each nsqd started up.
// Subsequent requsts pass that ID and get an updated delta based on each individual channel/nsqd message count
// That ID must be re-requested or it will be expired.
func (s *httpServer) counterDataHandler(w http.ResponseWriter, req *http.Request) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: failed to parse request params - %s", err)
http_api.Respond(w, 500, "INVALID_REQUEST", nil)
return
}
statsID, _ := reqParams.Get("id")
now := time.Now()
if statsID == "" {
// make a new one
statsID = fmt.Sprintf("%d.%d", now.Unix(), now.UnixNano())
}
stats, ok := s.counters[statsID]
if !ok {
stats = make(map[string]int64)
}
newStats := make(map[string]int64)
newStats["time"] = now.Unix()
producers, _ := lookupd.GetLookupdProducers(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
addresses := make([]string, len(producers))
for i, p := range producers {
addresses[i] = p.HTTPAddress()
}
_, channelStats, _ := lookupd.GetNSQDStats(addresses, "")
var newMessages int64
var totalMessages int64
for _, channelStats := range channelStats {
for _, hostChannelStats := range channelStats.HostStats {
key := fmt.Sprintf("%s:%s:%s", channelStats.TopicName, channelStats.ChannelName, hostChannelStats.HostAddress)
d, ok := stats[key]
if ok && d <= hostChannelStats.MessageCount {
newMessages += (hostChannelStats.MessageCount - d)
}
totalMessages += hostChannelStats.MessageCount
newStats[key] = hostChannelStats.MessageCount
}
}
s.counters[statsID] = newStats
data := make(map[string]interface{})
data["new_messages"] = newMessages
data["total_messages"] = totalMessages
data["id"] = statsID
http_api.Respond(w, 200, "OK", data)
}
func TestTombstonedNodes(t *testing.T) {
opts := NewNSQLookupdOptions()
opts.Logger = newTestLogger(t)
tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
defer nsqlookupd.Exit()
lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)}
topicName := "inactive_nodes"
conn := mustConnectLookupd(t, tcpAddr)
defer conn.Close()
identify(t, conn, "ip.address", 5000, 5555, "fake-version")
nsq.Register(topicName, "channel1").WriteTo(conn)
_, err := nsq.ReadResponse(conn)
equal(t, err, nil)
producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs)
equal(t, len(producers), 1)
equal(t, len(producers[0].Topics), 1)
equal(t, producers[0].Topics[0].Topic, topicName)
equal(t, producers[0].Topics[0].Tombstoned, false)
endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s",
httpAddr, topicName, "ip.address:5555")
_, err = http_api.NegotiateV1("POST", endpoint, nil)
equal(t, err, nil)
producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs)
equal(t, len(producers), 1)
equal(t, len(producers[0].Topics), 1)
equal(t, producers[0].Topics[0].Topic, topicName)
equal(t, producers[0].Topics[0].Tombstoned, true)
}
// this endpoint works by giving out an ID that maps to a stats dictionary
// The initial request is the number of messages processed since each nsqd started up.
// Subsequent requsts pass that ID and get an updated delta based on each individual channel/nsqd message count
// That ID must be re-requested or it will be expired.
func (s *httpServer) counterDataHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
statsID, _ := reqParams.Get("id")
now := time.Now()
if statsID == "" {
// make a new one
statsID = fmt.Sprintf("%d.%d", now.Unix(), now.UnixNano())
}
stats, ok := s.counters[statsID]
if !ok {
stats = make(map[string]int64)
}
newStats := make(map[string]int64)
newStats["time"] = now.Unix()
producers, _ := lookupd.GetLookupdProducers(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
addresses := make([]string, len(producers))
for i, p := range producers {
addresses[i] = p.HTTPAddress()
}
_, channelStats, _ := lookupd.GetNSQDStats(addresses, "")
var newMessages int64
var totalMessages int64
for _, channelStats := range channelStats {
for _, hostChannelStats := range channelStats.HostStats {
key := fmt.Sprintf("%s:%s:%s", channelStats.TopicName, channelStats.ChannelName, hostChannelStats.HostAddress)
d, ok := stats[key]
if ok && d <= hostChannelStats.MessageCount {
newMessages += (hostChannelStats.MessageCount - d)
}
totalMessages += hostChannelStats.MessageCount
newStats[key] = hostChannelStats.MessageCount
}
}
s.counters[statsID] = newStats
return struct {
NewMessages int64 `json:"new_messages"`
TotalMessages int64 `json:"total_messages"`
ID string `json:"id"`
}{newMessages, totalMessages, statsID}, nil
}
func (s *httpServer) performVersionNegotiatedRequestsToNSQD(
nsqlookupdAddrs []string, nsqdAddrs []string,
deprecatedURI string, v1URI string, queryString string) {
var err error
// get producer structs in one set of up-front requests
// so we can negotiate versions
//
// (this returns an empty list if there are no nsqlookupd configured)
producers, _ := lookupd.GetLookupdProducers(nsqlookupdAddrs)
for _, addr := range nsqdAddrs {
var nodeVer semver.Version
uri := deprecatedURI
producer := producerSearch(producers, addr)
if producer != nil {
nodeVer = producer.VersionObj
} else {
// we couldn't find the node in our list
// so ask it for a version directly
nodeVer, err = lookupd.GetVersion(addr)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: failed to get nsqd %s version - %s", addr, err)
}
}
if nodeVer.NE(semver.Version{}) && nodeVer.GTE(v1EndpointVersion) {
uri = v1URI
}
endpoint := fmt.Sprintf("http://%s/%s?%s", addr, uri, queryString)
s.ctx.nsqadmin.logf("NSQD: querying %s", endpoint)
_, err := http_api.NegotiateV1("POST", endpoint, nil)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: nsqd %s - %s", endpoint, err)
continue
}
}
}
func (s *httpServer) nodeHandler(w http.ResponseWriter, req *http.Request) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: failed to parse request params - %s", err)
http.Error(w, "INVALID_REQUEST", 500)
return
}
var urlRegex = regexp.MustCompile(`^/node/(.*)$`)
matches := urlRegex.FindStringSubmatch(req.URL.Path)
if len(matches) == 0 {
http.Error(w, "INVALID_NODE", 500)
return
}
parts := strings.Split(matches[1], "/")
node := parts[0]
found := false
for _, n := range s.ctx.nsqadmin.opts.NSQDHTTPAddresses {
if node == n {
found = true
break
}
}
producers, _ := lookupd.GetLookupdProducers(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
for _, p := range producers {
if node == fmt.Sprintf("%s:%d", p.BroadcastAddress, p.HTTPPort) {
found = true
break
}
}
if !found {
http.Error(w, "INVALID_NODE", 500)
return
}
topicStats, channelStats, _ := lookupd.GetNSQDStats([]string{node}, "")
numClients := int64(0)
numMessages := int64(0)
for _, ts := range topicStats {
for _, cs := range ts.Channels {
numClients += int64(len(cs.Clients))
}
numMessages += ts.MessageCount
}
p := struct {
Title string
Version string
GraphOptions *GraphOptions
Node Node
TopicStats []*lookupd.TopicStats
ChannelStats map[string]*lookupd.ChannelStats
NumMessages int64
NumClients int64
}{
Title: "NSQ Node - " + node,
Version: version.Binary,
GraphOptions: NewGraphOptions(w, req, reqParams, s.ctx),
Node: Node(node),
TopicStats: topicStats,
ChannelStats: channelStats,
NumMessages: numMessages,
NumClients: numClients,
}
err = templates.T.ExecuteTemplate(w, "node.html", p)
if err != nil {
s.ctx.nsqadmin.logf("Template Error %s", err)
http.Error(w, "Template Error", 500)
}
}
func (s *httpServer) nodeHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
node := ps.ByName("node")
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
found := false
for _, n := range s.ctx.nsqadmin.opts.NSQDHTTPAddresses {
if node == n {
found = true
break
}
}
producers, _ := lookupd.GetLookupdProducers(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
for _, p := range producers {
if node == fmt.Sprintf("%s:%d", p.BroadcastAddress, p.HTTPPort) {
found = true
break
}
}
if !found {
return nil, http_api.Err{404, "NOT_FOUND"}
}
topicStats, channelStats, _ := lookupd.GetNSQDStats([]string{node}, "")
numClients := int64(0)
numMessages := int64(0)
for _, ts := range topicStats {
for _, cs := range ts.Channels {
numClients += int64(len(cs.Clients))
}
numMessages += ts.MessageCount
}
p := struct {
Title string
Version string
GraphOptions *GraphOptions
Node Node
TopicStats []*lookupd.TopicStats
ChannelStats map[string]*lookupd.ChannelStats
NumMessages int64
NumClients int64
}{
Title: "NSQ Node - " + node,
Version: version.Binary,
GraphOptions: NewGraphOptions(w, req, reqParams, s.ctx),
Node: Node(node),
TopicStats: topicStats,
ChannelStats: channelStats,
NumMessages: numMessages,
NumClients: numClients,
}
err = templates.T.ExecuteTemplate(w, "node.html", p)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: executing template - %s", err)
return nil, http_api.Err{500, "INTERNAL_ERROR"}
}
return nil, nil
}