func (s *httpServer) deleteChannelHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } topicName, channelName, err := util.GetTopicChannelArgs(reqParams) if err != nil { util.ApiResponse(w, 500, err.Error(), nil) return } topic, err := s.context.nsqd.GetExistingTopic(topicName) if err != nil { util.ApiResponse(w, 500, "INVALID_TOPIC", nil) return } err = topic.DeleteExistingChannel(channelName) if err != nil { util.ApiResponse(w, 500, "INVALID_CHANNEL", nil) return } util.ApiResponse(w, 200, "OK", nil) }
func nodesHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } producers, _ := lookupd.GetLookupdProducers(lookupdHTTPAddrs) p := struct { Title string Version string GraphOptions *GraphOptions Producers []*lookupd.Producer Lookupd []string }{ Title: "NSQ Nodes", Version: util.BINARY_VERSION, GraphOptions: NewGraphOptions(w, req, reqParams), Producers: producers, Lookupd: lookupdHTTPAddrs, } err = templates.ExecuteTemplate(w, "nodes.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }
func (s *httpServer) lookupHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } topicName, err := reqParams.Get("topic") if err != nil { util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil) return } registration := s.context.nsqlookupd.DB.FindRegistrations("topic", topicName, "") if len(registration) == 0 { util.ApiResponse(w, 500, "INVALID_ARG_TOPIC", nil) return } channels := s.context.nsqlookupd.DB.FindRegistrations("channel", topicName, "*").SubKeys() producers := s.context.nsqlookupd.DB.FindProducers("topic", topicName, "") producers = producers.FilterByActive(s.context.nsqlookupd.inactiveProducerTimeout, s.context.nsqlookupd.tombstoneLifetime) data := make(map[string]interface{}) data["channels"] = channels data["producers"] = producers.PeerInfo() util.ApiResponse(w, 200, "OK", data) }
func (s *httpServer) tombstoneTopicProducerHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } topicName, err := reqParams.Get("topic") if err != nil { util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil) return } node, err := reqParams.Get("node") if err != nil { util.ApiResponse(w, 500, "MISSING_ARG_NODE", nil) return } log.Printf("DB: setting tombstone for producer@%s of topic(%s)", node, topicName) producers := s.context.nsqlookupd.DB.FindProducers("topic", topicName, "") for _, p := range producers { thisNode := fmt.Sprintf("%s:%d", p.peerInfo.BroadcastAddress, p.peerInfo.HttpPort) if thisNode == node { p.Tombstone() } } util.ApiResponse(w, 200, "OK", nil) }
func (s *httpServer) emptyTopicHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } topicName, err := reqParams.Get("topic") if err != nil { util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil) return } if !nsq.IsValidTopicName(topicName) { util.ApiResponse(w, 500, "INVALID_TOPIC", nil) return } topic, err := s.context.nsqd.GetExistingTopic(topicName) if err != nil { util.ApiResponse(w, 500, "INVALID_TOPIC", nil) return } err = topic.Empty() if err != nil { util.ApiResponse(w, 500, "INTERNAL_ERROR", nil) return } util.ApiResponse(w, 200, "OK", nil) }
func indexHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } var topics []string if len(lookupdHTTPAddrs) != 0 { topics, _ = lookupd.GetLookupdTopics(lookupdHTTPAddrs) } else { topics, _ = lookupd.GetNSQDTopics(nsqdHTTPAddrs) } p := struct { Title string GraphOptions *GraphOptions Topics Topics Version string }{ Title: "NSQ", GraphOptions: NewGraphOptions(w, req, reqParams), Topics: TopicsFromStrings(topics), Version: util.BINARY_VERSION, } err = templates.ExecuteTemplate(w, "index.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }
func (s *httpServer) deleteChannelHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } topicName, channelName, err := util.GetTopicChannelArgs(reqParams) if err != nil { util.ApiResponse(w, 500, err.Error(), nil) return } registrations := s.context.nsqlookupd.DB.FindRegistrations("channel", topicName, channelName) if len(registrations) == 0 { util.ApiResponse(w, 404, "NOT_FOUND", nil) return } log.Printf("DB: removing channel(%s) from topic(%s)", channelName, topicName) for _, registration := range registrations { s.context.nsqlookupd.DB.RemoveRegistration(registration) } util.ApiResponse(w, 200, "OK", nil) }
func (s *httpServer) deleteTopicHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } topicName, err := reqParams.Get("topic") if err != nil { util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil) return } registrations := s.context.nsqlookupd.DB.FindRegistrations("channel", topicName, "*") for _, registration := range registrations { log.Printf("DB: removing channel(%s) from topic(%s)", registration.SubKey, topicName) s.context.nsqlookupd.DB.RemoveRegistration(registration) } registrations = s.context.nsqlookupd.DB.FindRegistrations("topic", topicName, "") for _, registration := range registrations { log.Printf("DB: removing topic(%s)", topicName) s.context.nsqlookupd.DB.RemoveRegistration(registration) } util.ApiResponse(w, 200, "OK", nil) }
func graphiteDataHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } metric, err := reqParams.Get("metric") if err != nil { log.Printf("ERROR: missing metric param - %s", err.Error()) http.Error(w, "MISSING_METRIC_PARAM", 500) return } target, err := reqParams.Get("target") if err != nil { log.Printf("ERROR: missing target param - %s", err.Error()) http.Error(w, "MISSING_TARGET_PARAM", 500) return } var queryFunc func(string) string var formatJsonResponseFunc func([]byte) ([]byte, error) switch metric { case "rate": queryFunc = rateQuery formatJsonResponseFunc = parseRateResponse default: log.Printf("ERROR: unknown metric value %s", metric) http.Error(w, "INVALID_METRIC_PARAM", 500) return } query := queryFunc(target) url := *graphiteUrl + query log.Printf("GRAPHITE: %s", url) response, err := GraphiteGet(url) if err != nil { log.Printf("ERROR: graphite request failed %s", err.Error()) http.Error(w, "GRAPHITE_FAILED", 500) return } resp, err := formatJsonResponseFunc(response) if err != nil { log.Printf("ERROR: response formating failed - %s", err.Error()) http.Error(w, "INVALID_GRAPHITE_RESPONSE", 500) return } w.Header().Set("Content-Type", "application/json") w.Write(resp) return }
// this endpoint works by giving out an ID that maps to a stats dictionary // The initial request is the number of messages processed since each nsqd started up. // Subsequent requsts pass that ID and get an updated delta based on each individual channel/nsqd message count // That ID must be re-requested or it will be expired. func counterDataHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } statsID, _ := reqParams.Get("id") now := time.Now() if statsID == "" { // make a new one statsID = fmt.Sprintf("%d.%d", now.Unix(), now.UnixNano()) } stats, ok := globalCounters[statsID] if !ok { stats = make(map[string]int64) } newStats := make(map[string]int64) newStats["time"] = now.Unix() producers, _ := lookupd.GetLookupdProducers(lookupdHTTPAddrs) addresses := make([]string, len(producers)) for i, p := range producers { addresses[i] = p.HTTPAddress() } _, channelStats, _ := lookupd.GetNSQDStats(addresses, "") var newMessages int64 var totalMessages int64 for _, channelStats := range channelStats { for _, hostChannelStats := range channelStats.HostStats { key := fmt.Sprintf("%s:%s:%s", channelStats.TopicName, channelStats.ChannelName, hostChannelStats.HostAddress) d, ok := stats[key] if ok && d <= hostChannelStats.MessageCount { newMessages += (hostChannelStats.MessageCount - d) } totalMessages += hostChannelStats.MessageCount newStats[key] = hostChannelStats.MessageCount } } globalCounters[statsID] = newStats data := make(map[string]interface{}) data["new_messages"] = newMessages data["total_messages"] = totalMessages data["id"] = statsID util.ApiResponse(w, 200, "OK", data) }
func (s *httpServer) channelsHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } topicName, err := reqParams.Get("topic") if err != nil { util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil) return } channels := s.context.nsqlookupd.DB.FindRegistrations("channel", topicName, "*").SubKeys() data := make(map[string]interface{}) data["channels"] = channels util.ApiResponse(w, 200, "OK", data) }
func channelHandler(w http.ResponseWriter, req *http.Request, topicName string, channelName string) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } producers := getProducers(topicName) _, allChannelStats, _ := lookupd.GetNSQDStats(producers, topicName) channelStats := allChannelStats[channelName] hasE2eLatency := len(channelStats.E2eProcessingLatency.Percentiles) > 0 p := struct { Title string GraphOptions *GraphOptions Version string Topic string Channel string TopicProducers []string ChannelStats *lookupd.ChannelStats HasE2eLatency bool }{ Title: fmt.Sprintf("NSQ %s / %s", topicName, channelName), GraphOptions: NewGraphOptions(w, req, reqParams), Version: util.BINARY_VERSION, Topic: topicName, Channel: channelName, TopicProducers: producers, ChannelStats: channelStats, HasE2eLatency: hasE2eLatency, } err = templates.ExecuteTemplate(w, "channel.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }
func lookupHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } channels := make(map[string][]string) allTopics, _ := lookupd.GetLookupdTopics(lookupdHTTPAddrs) for _, topicName := range allTopics { var producers []string producers, _ = lookupd.GetLookupdTopicProducers(topicName, lookupdHTTPAddrs) if len(producers) == 0 { topicChannels, _ := lookupd.GetLookupdTopicChannels(topicName, lookupdHTTPAddrs) channels[topicName] = topicChannels } } p := struct { Title string GraphOptions *GraphOptions TopicMap map[string][]string Lookupd []string Version string }{ Title: "NSQ Lookup", GraphOptions: NewGraphOptions(w, req, reqParams), TopicMap: channels, Lookupd: lookupdHTTPAddrs, Version: util.BINARY_VERSION, } err = templates.ExecuteTemplate(w, "lookup.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }
func (s *httpServer) createChannelHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } topicName, channelName, err := util.GetTopicChannelArgs(reqParams) if err != nil { util.ApiResponse(w, 500, err.Error(), nil) return } log.Printf("DB: adding channel(%s) in topic(%s)", channelName, topicName) key := Registration{"channel", topicName, channelName} s.context.nsqlookupd.DB.AddRegistration(key) log.Printf("DB: adding topic(%s)", topicName) key = Registration{"topic", topicName, ""} s.context.nsqlookupd.DB.AddRegistration(key) util.ApiResponse(w, 200, "OK", nil) }
func (s *httpServer) pauseChannelHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } topicName, channelName, err := util.GetTopicChannelArgs(reqParams) if err != nil { util.ApiResponse(w, 500, err.Error(), nil) return } topic, err := s.context.nsqd.GetExistingTopic(topicName) if err != nil { util.ApiResponse(w, 500, "INVALID_TOPIC", nil) return } channel, err := topic.GetExistingChannel(channelName) if err != nil { util.ApiResponse(w, 500, "INVALID_CHANNEL", nil) return } if strings.HasPrefix(req.URL.Path, "/pause") { err = channel.Pause() } else { err = channel.UnPause() } if err != nil { log.Printf("ERROR: failure in %s - %s", req.URL.Path, err.Error()) } util.ApiResponse(w, 200, "OK", nil) }
func counterHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } p := struct { Title string Version string GraphOptions *GraphOptions Target counterTarget }{ Title: "NSQ Message Counts", Version: util.BINARY_VERSION, GraphOptions: NewGraphOptions(w, req, reqParams), Target: counterTarget{}, } err = templates.ExecuteTemplate(w, "counter.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }
func (s *httpServer) createTopicHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } topicName, err := reqParams.Get("topic") if err != nil { util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil) return } if !nsq.IsValidTopicName(topicName) { util.ApiResponse(w, 500, "INVALID_TOPIC", nil) return } log.Printf("DB: adding topic(%s)", topicName) key := Registration{"topic", topicName, ""} s.context.nsqlookupd.DB.AddRegistration(key) util.ApiResponse(w, 200, "OK", nil) }
func (s *StreamServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { path := req.URL.Path if path == "/stats" { StatsHandler(w, req) return } if path != "/sub" { w.WriteHeader(404) return } reqParams, err := util.NewReqParams(req) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } topicName, channelName, err := util.GetTopicChannelArgs(reqParams) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } hj, ok := w.(http.Hijacker) if !ok { http.Error(w, "httpserver doesn't support hijacking", http.StatusInternalServerError) return } conn, bufrw, err := hj.Hijack() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } r, err := nsq.NewReader(topicName, channelName, *authenticationPassword) r.SetMaxInFlight(*maxInFlight) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } sr := &StreamReader{ topic: topicName, channel: channelName, reader: r, req: req, conn: conn, bufrw: bufrw, // TODO: latency writer connectTime: time.Now(), } s.Set(sr) log.Printf("[%s] new connection", conn.RemoteAddr().String()) bufrw.WriteString("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n") bufrw.Flush() r.AddHandler(sr) // TODO: handle the error cases better (ie. at all :) ) errors := ConnectToNSQAndLookupd(r, nsqdTCPAddrs, lookupdHTTPAddrs) log.Printf("connected to NSQ %v", errors) // this read allows us to detect clients that disconnect go func(rw *bufio.ReadWriter) { b, err := rw.ReadByte() if err != nil { log.Printf("got connection err %s", err.Error()) } else { log.Printf("unexpected data on request socket (%s); closing", b) } sr.reader.Stop() }(bufrw) go sr.HeartbeatLoop() }
func topicHandler(w http.ResponseWriter, req *http.Request) { var urlRegex = regexp.MustCompile(`^/topic/(.*)$`) matches := urlRegex.FindStringSubmatch(req.URL.Path) if len(matches) == 0 { http.Error(w, "INVALID_TOPIC", 500) return } parts := strings.Split(matches[1], "/") topicName := parts[0] if !nsq.IsValidTopicName(topicName) { http.Error(w, "INVALID_TOPIC", 500) return } if len(parts) == 2 { channelName := parts[1] if !nsq.IsValidChannelName(channelName) { http.Error(w, "INVALID_CHANNEL", 500) } else { channelHandler(w, req, topicName, channelName) } return } reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } producers := getProducers(topicName) topicStats, channelStats, _ := lookupd.GetNSQDStats(producers, topicName) globalTopicStats := &lookupd.TopicStats{HostAddress: "Total"} for _, t := range topicStats { globalTopicStats.Add(t) } hasE2eLatency := len(globalTopicStats.E2eProcessingLatency.Percentiles) > 0 p := struct { Title string GraphOptions *GraphOptions Version string Topic string TopicProducers []string TopicStats []*lookupd.TopicStats GlobalTopicStats *lookupd.TopicStats ChannelStats map[string]*lookupd.ChannelStats HasE2eLatency bool }{ Title: fmt.Sprintf("NSQ %s", topicName), GraphOptions: NewGraphOptions(w, req, reqParams), Version: util.BINARY_VERSION, Topic: topicName, TopicProducers: producers, TopicStats: topicStats, GlobalTopicStats: globalTopicStats, ChannelStats: channelStats, HasE2eLatency: hasE2eLatency, } err = templates.ExecuteTemplate(w, "topic.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }
func nodeHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) http.Error(w, "INVALID_REQUEST", 500) return } var urlRegex = regexp.MustCompile(`^/node/(.*)$`) matches := urlRegex.FindStringSubmatch(req.URL.Path) if len(matches) == 0 { http.Error(w, "INVALID_NODE", 500) return } parts := strings.Split(matches[1], "/") node := parts[0] found := false for _, n := range nsqdHTTPAddrs { if node == n { found = true break } } producers, _ := lookupd.GetLookupdProducers(lookupdHTTPAddrs) for _, p := range producers { if node == fmt.Sprintf("%s:%d", p.BroadcastAddress, p.HttpPort) { found = true break } } if !found { http.Error(w, "INVALID_NODE", 500) return } topicStats, channelStats, _ := lookupd.GetNSQDStats([]string{node}, "") numClients := int64(0) numMessages := int64(0) for _, ts := range topicStats { for _, cs := range ts.Channels { numClients += int64(len(cs.Clients)) } numMessages += ts.MessageCount } p := struct { Title string Version string GraphOptions *GraphOptions Node Node TopicStats []*lookupd.TopicStats ChannelStats map[string]*lookupd.ChannelStats NumMessages int64 NumClients int64 }{ Title: "NSQ Node - " + node, Version: util.BINARY_VERSION, GraphOptions: NewGraphOptions(w, req, reqParams), Node: Node(node), TopicStats: topicStats, ChannelStats: channelStats, NumMessages: numMessages, NumClients: numClients, } err = templates.ExecuteTemplate(w, "node.html", p) if err != nil { log.Printf("Template Error %s", err.Error()) http.Error(w, "Template Error", 500) } }
func (s *httpServer) statsHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } formatString, _ := reqParams.Get("format") jsonFormat := formatString == "json" now := time.Now() if !jsonFormat { io.WriteString(w, fmt.Sprintf("%s\n", util.Version("nsqd"))) } stats := s.context.nsqd.getStats() if jsonFormat { util.ApiResponse(w, 200, "OK", struct { Topics []TopicStats `json:"topics"` }{stats}) } else { if len(stats) == 0 { io.WriteString(w, "\nNO_TOPICS\n") return } for _, t := range stats { io.WriteString(w, fmt.Sprintf("\n[%-15s] depth: %-5d be-depth: %-5d msgs: %-8d e2e%%: %s\n", t.TopicName, t.Depth, t.BackendDepth, t.MessageCount, t.E2eProcessingLatency)) for _, c := range t.Channels { var pausedPrefix string if c.Paused { pausedPrefix = " *P " } else { pausedPrefix = " " } io.WriteString(w, fmt.Sprintf("%s[%-25s] depth: %-5d be-depth: %-5d inflt: %-4d def: %-4d re-q: %-5d timeout: %-5d msgs: %-8d e2e%%: %s\n", pausedPrefix, c.ChannelName, c.Depth, c.BackendDepth, c.InFlightCount, c.DeferredCount, c.RequeueCount, c.TimeoutCount, c.MessageCount, c.E2eProcessingLatency)) for _, client := range c.Clients { connectTime := time.Unix(client.ConnectTime, 0) // truncate to the second duration := time.Duration(int64(now.Sub(connectTime).Seconds())) * time.Second _, port, _ := net.SplitHostPort(client.RemoteAddress) io.WriteString(w, fmt.Sprintf(" [%s %-21s] state: %d inflt: %-4d rdy: %-4d fin: %-8d re-q: %-8d msgs: %-8d connected: %s\n", client.Version, fmt.Sprintf("%s:%s", client.Name, port), client.State, client.InFlightCount, client.ReadyCount, client.FinishCount, client.RequeueCount, client.MessageCount, duration, )) } } } } }