func TestTombstonedNodes(t *testing.T) { log.SetOutput(ioutil.Discard) defer log.SetOutput(os.Stdout) tcpAddr, httpAddr := mustStartLookupd() defer lookupd.Exit() lookupd.inactiveProducerTimeout = 50 * time.Millisecond lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)} topicName := "inactive_nodes" conn := mustConnectLookupd(t, tcpAddr) identify(t, conn, "ip.address", 5000, 5555, "fake-version") nsq.Register(topicName, "channel1").Write(conn) _, err := nsq.ReadResponse(conn) assert.Equal(t, err, nil) producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs) assert.Equal(t, len(producers), 1) assert.Equal(t, len(producers[0].Topics), 1) assert.Equal(t, producers[0].Topics[0].Topic, topicName) assert.Equal(t, producers[0].Topics[0].Tombstoned, false) endpoint := fmt.Sprintf("http://%s/tombstone_topic_producer?topic=%s&node=%s", httpAddr, topicName, "ip.address:5555") _, err = nsq.ApiRequest(endpoint) assert.Equal(t, err, nil) producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs) assert.Equal(t, len(producers), 1) assert.Equal(t, len(producers[0].Topics), 1) assert.Equal(t, producers[0].Topics[0].Topic, topicName) assert.Equal(t, producers[0].Topics[0].Tombstoned, true) }
func TestInactiveNodes(t *testing.T) { opts := NewNSQLookupdOptions() opts.Logger = newTestLogger(t) opts.InactiveProducerTimeout = 200 * time.Millisecond tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts) defer nsqlookupd.Exit() lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)} topicName := "inactive_nodes" conn := mustConnectLookupd(t, tcpAddr) defer conn.Close() identify(t, conn, "ip.address", 5000, 5555, "fake-version") nsq.Register(topicName, "channel1").WriteTo(conn) _, err := nsq.ReadResponse(conn) equal(t, err, nil) producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs) equal(t, len(producers), 1) equal(t, len(producers[0].Topics), 1) equal(t, producers[0].Topics[0].Topic, topicName) equal(t, producers[0].Topics[0].Tombstoned, false) time.Sleep(250 * time.Millisecond) producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs) equal(t, len(producers), 0) }
func TestInactiveNodes(t *testing.T) { log.SetOutput(ioutil.Discard) defer log.SetOutput(os.Stdout) tcpAddr, httpAddr, nsqlookupd := mustStartLookupd() defer nsqlookupd.Exit() nsqlookupd.inactiveProducerTimeout = 50 * time.Millisecond lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)} topicName := "inactive_nodes" conn := mustConnectLookupd(t, tcpAddr) identify(t, conn, "ip.address", 5000, 5555, "fake-version") nsq.Register(topicName, "channel1").Write(conn) _, err := nsq.ReadResponse(conn) assert.Equal(t, err, nil) producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs) assert.Equal(t, len(producers), 1) assert.Equal(t, len(producers[0].Topics), 1) assert.Equal(t, producers[0].Topics[0].Topic, topicName) assert.Equal(t, producers[0].Topics[0].Tombstoned, false) time.Sleep(55 * time.Millisecond) producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs) assert.Equal(t, len(producers), 0) }
func TestTombstonedNodes(t *testing.T) { log.SetOutput(ioutil.Discard) defer log.SetOutput(os.Stdout) options := NewNSQLookupdOptions() tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(options) defer nsqlookupd.Exit() lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)} topicName := "inactive_nodes" conn := mustConnectLookupd(t, tcpAddr) identify(t, conn, "ip.address", 5000, 5555, "fake-version") go func() { for { time.Sleep(5 * time.Millisecond) nsq.Ping().WriteTo(conn) _, err := nsq.ReadResponse(conn) if err != nil { return } } }() nsq.Register(topicName, "channel1").WriteTo(conn) _, err := nsq.ReadResponse(conn) assert.Equal(t, err, nil) producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs) assert.Equal(t, len(producers), 1) assert.Equal(t, len(producers[0].Topics), 1) assert.Equal(t, producers[0].Topics[0].Topic, topicName) assert.Equal(t, producers[0].Topics[0].Tombstoned, false) endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s", httpAddr, topicName, "ip.address:5555") _, err = util.APIRequestNegotiateV1("POST", endpoint, nil) assert.Equal(t, err, nil) producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs) assert.Equal(t, len(producers), 1) assert.Equal(t, len(producers[0].Topics), 1) assert.Equal(t, producers[0].Topics[0].Topic, topicName) assert.Equal(t, producers[0].Topics[0].Tombstoned, true) conn.Close() }
func (s *httpServer) nodesHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } producers, _ := lookupd.GetLookupdProducers(s.context.nsqadmin.options.NSQLookupdHTTPAddresses) p := struct { Title string Version string GraphOptions *GraphOptions Producers []*lookupd.Producer Lookupd []string }{ Title: "NSQ Nodes", Version: util.BINARY_VERSION, GraphOptions: NewGraphOptions(w, req, reqParams, s.context), Producers: producers, Lookupd: s.context.nsqadmin.options.NSQLookupdHTTPAddresses, } err = templates.T.ExecuteTemplate(w, "nodes.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }
// this endpoint works by giving out an ID that maps to a stats dictionary // The initial request is the number of messages processed since each nsqd started up. // Subsequent requsts pass that ID and get an updated delta based on each individual channel/nsqd message count // That ID must be re-requested or it will be expired. func (s *httpServer) counterDataHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } statsID, _ := reqParams.Get("id") now := time.Now() if statsID == "" { // make a new one statsID = fmt.Sprintf("%d.%d", now.Unix(), now.UnixNano()) } stats, ok := s.counters[statsID] if !ok { stats = make(map[string]int64) } newStats := make(map[string]int64) newStats["time"] = now.Unix() producers, _ := lookupd.GetLookupdProducers(s.context.nsqadmin.options.NSQLookupdHTTPAddresses) addresses := make([]string, len(producers)) for i, p := range producers { addresses[i] = p.HTTPAddress() } _, channelStats, _ := lookupd.GetNSQDStats(addresses, "") var newMessages int64 var totalMessages int64 for _, channelStats := range channelStats { for _, hostChannelStats := range channelStats.HostStats { key := fmt.Sprintf("%s:%s:%s", channelStats.TopicName, channelStats.ChannelName, hostChannelStats.HostAddress) d, ok := stats[key] if ok && d <= hostChannelStats.MessageCount { newMessages += (hostChannelStats.MessageCount - d) } totalMessages += hostChannelStats.MessageCount newStats[key] = hostChannelStats.MessageCount } } s.counters[statsID] = newStats data := make(map[string]interface{}) data["new_messages"] = newMessages data["total_messages"] = totalMessages data["id"] = statsID util.ApiResponse(w, 200, "OK", data) }
func TestTombstonedNodes(t *testing.T) { opts := NewNSQLookupdOptions() opts.Logger = newTestLogger(t) tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts) defer nsqlookupd.Exit() lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)} topicName := "inactive_nodes" conn := mustConnectLookupd(t, tcpAddr) defer conn.Close() identify(t, conn, "ip.address", 5000, 5555, "fake-version") nsq.Register(topicName, "channel1").WriteTo(conn) _, err := nsq.ReadResponse(conn) equal(t, err, nil) producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs) equal(t, len(producers), 1) equal(t, len(producers[0].Topics), 1) equal(t, producers[0].Topics[0].Topic, topicName) equal(t, producers[0].Topics[0].Tombstoned, false) endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s", httpAddr, topicName, "ip.address:5555") _, err = util.APIRequestNegotiateV1("POST", endpoint, nil) equal(t, err, nil) producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs) equal(t, len(producers), 1) equal(t, len(producers[0].Topics), 1) equal(t, producers[0].Topics[0].Topic, topicName) equal(t, producers[0].Topics[0].Tombstoned, true) }
func (s *httpServer) performVersionNegotiatedRequestsToNSQD( nsqlookupdAddrs []string, nsqdAddrs []string, deprecatedURI string, v1URI string, queryString string) { var err error // get producer structs in one set of up-front requests // so we can negotiate versions // // (this returns an empty list if there are no nsqlookupd configured) producers, _ := lookupd.GetLookupdProducers(nsqlookupdAddrs) for _, addr := range nsqdAddrs { var nodeVer *semver.Version uri := deprecatedURI producer := producerSearch(producers, addr) if producer != nil { nodeVer = producer.VersionObj } else { // we couldn't find the node in our list // so ask it for a version directly nodeVer, err = lookupd.GetVersion(addr) if err != nil { s.ctx.nsqadmin.logf("ERROR: failed to get nsqd %s version - %s", addr, err) } } if nodeVer != nil && !nodeVer.Less(v1EndpointVersion) { uri = v1URI } endpoint := fmt.Sprintf("http://%s/%s?%s", addr, uri, queryString) s.ctx.nsqadmin.logf("NSQD: querying %s", endpoint) _, err := util.APIRequestNegotiateV1("POST", endpoint, nil) if err != nil { s.ctx.nsqadmin.logf("ERROR: nsqd %s - %s", endpoint, err) continue } } }
func (s *httpServer) nodeHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } var urlRegex = regexp.MustCompile(`^/node/(.*)$`) matches := urlRegex.FindStringSubmatch(req.URL.Path) if len(matches) == 0 { http.Error(w, "INVALID_NODE", 500) return } parts := strings.Split(matches[1], "/") node := parts[0] found := false for _, n := range s.context.nsqadmin.options.NSQDHTTPAddresses { if node == n { found = true break } } producers, _ := lookupd.GetLookupdProducers(s.context.nsqadmin.options.NSQLookupdHTTPAddresses) for _, p := range producers { if node == fmt.Sprintf("%s:%d", p.BroadcastAddress, p.HttpPort) { found = true break } } if !found { http.Error(w, "INVALID_NODE", 500) return } topicStats, channelStats, _ := lookupd.GetNSQDStats([]string{node}, "") numClients := int64(0) numMessages := int64(0) for _, ts := range topicStats { for _, cs := range ts.Channels { numClients += int64(len(cs.Clients)) } numMessages += ts.MessageCount } p := struct { Title string Version string GraphOptions *GraphOptions Node Node TopicStats []*lookupd.TopicStats ChannelStats map[string]*lookupd.ChannelStats NumMessages int64 NumClients int64 }{ Title: "NSQ Node - " + node, Version: util.BINARY_VERSION, GraphOptions: NewGraphOptions(w, req, reqParams, s.context), Node: Node(node), TopicStats: topicStats, ChannelStats: channelStats, NumMessages: numMessages, NumClients: numClients, } err = templates.T.ExecuteTemplate(w, "node.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }