func statLoop(interval time.Duration, topic string, channel string, nsqdTCPAddrs []string, lookupdHTTPAddrs []string) { i := 0 for { var producers []string var err error log.SetOutput(ioutil.Discard) if len(lookupdHTTPAddrs) != 0 { producers, err = lookupd.GetLookupdTopicProducers(topic, lookupdHTTPAddrs) } else { producers, err = lookupd.GetNSQDTopicProducers(topic, nsqdHTTPAddrs) } log.SetOutput(os.Stdout) if err != nil { log.Fatalf("ERROR: failed to get topic producers - %s", err.Error()) } log.SetOutput(ioutil.Discard) _, allChannelStats, err := lookupd.GetNSQDStats(producers, topic) log.SetOutput(os.Stdout) if err != nil { log.Fatalf("ERROR: failed to get nsqd stats - %s", err.Error()) } c, ok := allChannelStats[channel] if !ok { log.Fatalf("ERROR: failed to find channel(%s) in stats metadata for topic(%s)", channel, topic) } if i%25 == 0 { fmt.Printf("-----------depth------------+--------------metadata---------------\n") fmt.Printf("%7s %7s %5s %5s | %7s %7s %12s %7s\n", "mem", "disk", "inflt", "def", "req", "t-o", "msgs", "clients") } // TODO: paused fmt.Printf("%7d %7d %5d %5d | %7d %7d %12d %7d\n", c.Depth, c.BackendDepth, c.InFlightCount, c.DeferredCount, c.RequeueCount, c.TimeoutCount, c.MessageCount, c.ClientCount) time.Sleep(interval) i++ } }
// this endpoint works by giving out an ID that maps to a stats dictionary // The initial request is the number of messages processed since each nsqd started up. // Subsequent requsts pass that ID and get an updated delta based on each individual channel/nsqd message count // That ID must be re-requested or it will be expired. func counterDataHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } statsID, _ := reqParams.Get("id") now := time.Now() if statsID == "" { // make a new one statsID = fmt.Sprintf("%d.%d", now.Unix(), now.UnixNano()) } stats, ok := globalCounters[statsID] if !ok { stats = make(map[string]int64) } newStats := make(map[string]int64) newStats["time"] = now.Unix() producers, _ := lookupd.GetLookupdProducers(lookupdHTTPAddrs) addresses := make([]string, len(producers)) for i, p := range producers { addresses[i] = p.HTTPAddress() } _, channelStats, _ := lookupd.GetNSQDStats(addresses, "") var newMessages int64 var totalMessages int64 for _, channelStats := range channelStats { for _, hostChannelStats := range channelStats.HostStats { key := fmt.Sprintf("%s:%s:%s", channelStats.TopicName, channelStats.ChannelName, hostChannelStats.HostAddress) d, ok := stats[key] if ok && d <= hostChannelStats.MessageCount { newMessages += (hostChannelStats.MessageCount - d) } totalMessages += hostChannelStats.MessageCount newStats[key] = hostChannelStats.MessageCount } } globalCounters[statsID] = newStats data := make(map[string]interface{}) data["new_messages"] = newMessages data["total_messages"] = totalMessages data["id"] = statsID util.ApiResponse(w, 200, "OK", data) }
func channelHandler(w http.ResponseWriter, req *http.Request, topicName string, channelName string) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } producers := getProducers(topicName) _, allChannelStats, _ := lookupd.GetNSQDStats(producers, topicName) channelStats := allChannelStats[channelName] hasE2eLatency := len(channelStats.E2eProcessingLatency.Percentiles) > 0 p := struct { Title string GraphOptions *GraphOptions Version string Topic string Channel string TopicProducers []string ChannelStats *lookupd.ChannelStats HasE2eLatency bool }{ Title: fmt.Sprintf("NSQ %s / %s", topicName, channelName), GraphOptions: NewGraphOptions(w, req, reqParams), Version: util.BINARY_VERSION, Topic: topicName, Channel: channelName, TopicProducers: producers, ChannelStats: channelStats, HasE2eLatency: hasE2eLatency, } err = templates.ExecuteTemplate(w, "channel.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }
func nodeHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } var urlRegex = regexp.MustCompile(`^/node/(.*)$`) matches := urlRegex.FindStringSubmatch(req.URL.Path) if len(matches) == 0 { http.Error(w, "INVALID_NODE", 500) return } parts := strings.Split(matches[1], "/") node := parts[0] found := false for _, n := range nsqdHTTPAddrs { if node == n { found = true break } } producers, _ := lookupd.GetLookupdProducers(lookupdHTTPAddrs) for _, p := range producers { if node == fmt.Sprintf("%s:%d", p.BroadcastAddress, p.HttpPort) { found = true break } } if !found { http.Error(w, "INVALID_NODE", 500) return } topicStats, channelStats, _ := lookupd.GetNSQDStats([]string{node}, "") numClients := int64(0) numMessages := int64(0) for _, ts := range topicStats { for _, cs := range ts.Channels { numClients += int64(len(cs.Clients)) } numMessages += ts.MessageCount } p := struct { Title string Version string GraphOptions *GraphOptions Node Node TopicStats []*lookupd.TopicStats ChannelStats map[string]*lookupd.ChannelStats NumMessages int64 NumClients int64 }{ Title: "NSQ Node - " + node, Version: util.BINARY_VERSION, GraphOptions: NewGraphOptions(w, req, reqParams), Node: Node(node), TopicStats: topicStats, ChannelStats: channelStats, NumMessages: numMessages, NumClients: numClients, } err = templates.ExecuteTemplate(w, "node.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }
func topicHandler(w http.ResponseWriter, req *http.Request) { var urlRegex = regexp.MustCompile(`^/topic/(.*)$`) matches := urlRegex.FindStringSubmatch(req.URL.Path) if len(matches) == 0 { http.Error(w, "INVALID_TOPIC", 500) return } parts := strings.Split(matches[1], "/") topicName := parts[0] if !nsq.IsValidTopicName(topicName) { http.Error(w, "INVALID_TOPIC", 500) return } if len(parts) == 2 { channelName := parts[1] if !nsq.IsValidChannelName(channelName) { http.Error(w, "INVALID_CHANNEL", 500) } else { channelHandler(w, req, topicName, channelName) } return } reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } producers := getProducers(topicName) topicStats, channelStats, _ := lookupd.GetNSQDStats(producers, topicName) globalTopicStats := &lookupd.TopicStats{HostAddress: "Total"} for _, t := range topicStats { globalTopicStats.Add(t) } hasE2eLatency := len(globalTopicStats.E2eProcessingLatency.Percentiles) > 0 p := struct { Title string GraphOptions *GraphOptions Version string Topic string TopicProducers []string TopicStats []*lookupd.TopicStats GlobalTopicStats *lookupd.TopicStats ChannelStats map[string]*lookupd.ChannelStats HasE2eLatency bool }{ Title: fmt.Sprintf("NSQ %s", topicName), GraphOptions: NewGraphOptions(w, req, reqParams), Version: util.BINARY_VERSION, Topic: topicName, TopicProducers: producers, TopicStats: topicStats, GlobalTopicStats: globalTopicStats, ChannelStats: channelStats, HasE2eLatency: hasE2eLatency, } err = templates.ExecuteTemplate(w, "topic.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }