func (s *httpServer) doTombstoneTopicProducer(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } node, err := reqParams.Get("node") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_NODE"} } s.ctx.nsqlookupd.logf("DB: setting tombstone for producer@%s of topic(%s)", node, topicName) producers := s.ctx.nsqlookupd.DB.FindProducers("topic", topicName, "") for _, p := range producers { thisNode := fmt.Sprintf("%s:%d", p.peerInfo.BroadcastAddress, p.peerInfo.HTTPPort) if thisNode == node { p.Tombstone() } } return nil, nil }
func (s *httpServer) doPauseTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err) return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } topic, err := s.ctx.nsqd.GetExistingTopic(topicName) if err != nil { return nil, http_api.Err{404, "TOPIC_NOT_FOUND"} } if strings.Contains(req.URL.Path, "unpause") { err = topic.UnPause() } else { err = topic.Pause() } if err != nil { s.ctx.nsqd.logf("ERROR: failure in %s - %s", req.URL.Path, err) return nil, http_api.Err{500, "INTERNAL_ERROR"} } // pro-actively persist metadata so in case of process failure // nsqd won't suddenly (un)pause a topic s.ctx.nsqd.Lock() s.ctx.nsqd.PersistMetadata() s.ctx.nsqd.Unlock() return nil, nil }
func (s *httpServer) doEmptyTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err) return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } if !protocol.IsValidTopicName(topicName) { return nil, http_api.Err{400, "INVALID_TOPIC"} } topic, err := s.ctx.nsqd.GetExistingTopic(topicName) if err != nil { return nil, http_api.Err{404, "TOPIC_NOT_FOUND"} } err = topic.Empty() if err != nil { return nil, http_api.Err{500, "INTERNAL_ERROR"} } return nil, nil }
func (s *httpServer) doDeleteTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } registrations := s.ctx.nsqlookupd.DB.FindRegistrations("channel", topicName, "*") for _, registration := range registrations { s.ctx.nsqlookupd.logf("DB: removing channel(%s) from topic(%s)", registration.SubKey, topicName) s.ctx.nsqlookupd.DB.RemoveRegistration(registration) } registrations = s.ctx.nsqlookupd.DB.FindRegistrations("topic", topicName, "") for _, registration := range registrations { s.ctx.nsqlookupd.logf("DB: removing topic(%s)", topicName) s.ctx.nsqlookupd.DB.RemoveRegistration(registration) } return nil, nil }
func (s *httpServer) doLookup(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } registration := s.ctx.nsqlookupd.DB.FindRegistrations("topic", topicName, "") if len(registration) == 0 { return nil, http_api.Err{404, "TOPIC_NOT_FOUND"} } channels := s.ctx.nsqlookupd.DB.FindRegistrations("channel", topicName, "*").SubKeys() producers := s.ctx.nsqlookupd.DB.FindProducers("topic", topicName, "") producers = producers.FilterByActive(s.ctx.nsqlookupd.opts.InactiveProducerTimeout, s.ctx.nsqlookupd.opts.TombstoneLifetime) return map[string]interface{}{ "channels": channels, "producers": producers.PeerInfo(), }, nil }
func (s *httpServer) doStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err) return nil, http_api.Err{400, "INVALID_REQUEST"} } formatString, _ := reqParams.Get("format") topicName, _ := reqParams.Get("topic") channelName, _ := reqParams.Get("channel") jsonFormat := formatString == "json" stats := s.ctx.nsqd.GetStats() health := s.ctx.nsqd.GetHealth() startTime := s.ctx.nsqd.GetStartTime() uptime := time.Since(startTime) // If we WERE given a topic-name, remove stats for all the other topics: if len(topicName) > 0 { // Find the desired-topic-index: for _, topicStats := range stats { if topicStats.TopicName == topicName { // If we WERE given a channel-name, remove stats for all the other channels: if len(channelName) > 0 { // Find the desired-channel: for _, channelStats := range topicStats.Channels { if channelStats.ChannelName == channelName { topicStats.Channels = []ChannelStats{channelStats} // We've got the channel we were looking for: break } } } // We've got the topic we were looking for: stats = []TopicStats{topicStats} break } } } if !jsonFormat { return s.printStats(stats, health, startTime, uptime), nil } return struct { Version string `json:"version"` Health string `json:"health"` StartTime int64 `json:"start_time"` Topics []TopicStats `json:"topics"` }{version.Binary, health, startTime.Unix(), stats}, nil }
func (s *httpServer) topicsHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, err.Error()} } var topics []string if len(s.ctx.nsqadmin.getOpts().NSQLookupdHTTPAddresses) != 0 { topics, err = s.ci.GetLookupdTopics(s.ctx.nsqadmin.getOpts().NSQLookupdHTTPAddresses) } else { topics, err = s.ci.GetNSQDTopics(s.ctx.nsqadmin.getOpts().NSQDHTTPAddresses) } if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.ctx.nsqadmin.logf("ERROR: failed to get topics - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.ctx.nsqadmin.logf("WARNING: %s", err) messages = append(messages, pe.Error()) } inactive, _ := reqParams.Get("inactive") if inactive == "true" { topicChannelMap := make(map[string][]string) if len(s.ctx.nsqadmin.getOpts().NSQLookupdHTTPAddresses) == 0 { goto respond } for _, topicName := range topics { producers, _ := s.ci.GetLookupdTopicProducers( topicName, s.ctx.nsqadmin.getOpts().NSQLookupdHTTPAddresses) if len(producers) == 0 { topicChannels, _ := s.ci.GetLookupdTopicChannels( topicName, s.ctx.nsqadmin.getOpts().NSQLookupdHTTPAddresses) topicChannelMap[topicName] = topicChannels } } respond: return struct { Topics map[string][]string `json:"topics"` Message string `json:"message"` }{topicChannelMap, maybeWarnMsg(messages)}, nil } return struct { Topics []string `json:"topics"` Message string `json:"message"` }{topics, maybeWarnMsg(messages)}, nil }
func (s *httpServer) graphiteHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } metric, err := reqParams.Get("metric") if err != nil || metric != "rate" { return nil, http_api.Err{400, "INVALID_ARG_METRIC"} } target, err := reqParams.Get("target") if err != nil { return nil, http_api.Err{400, "INVALID_ARG_TARGET"} } params := url.Values{} params.Set("from", fmt.Sprintf("-%dsec", s.ctx.nsqadmin.getOpts().StatsdInterval*2/time.Second)) params.Set("until", fmt.Sprintf("-%dsec", s.ctx.nsqadmin.getOpts().StatsdInterval/time.Second)) params.Set("format", "json") params.Set("target", target) query := fmt.Sprintf("/render?%s", params.Encode()) url := s.ctx.nsqadmin.getOpts().GraphiteURL + query s.ctx.nsqadmin.logf("GRAPHITE: %s", url) var response []struct { Target string `json:"target"` DataPoints [][]*float64 `json:"datapoints"` } err = s.client.GETV1(url, &response) if err != nil { s.ctx.nsqadmin.logf("ERROR: graphite request failed - %s", err) return nil, http_api.Err{500, "INTERNAL_ERROR"} } var rateStr string rate := *response[0].DataPoints[0][0] if rate < 0 { rateStr = "N/A" } else { rateDivisor := s.ctx.nsqadmin.getOpts().StatsdInterval / time.Second rateStr = fmt.Sprintf("%.2f", rate/float64(rateDivisor)) } return struct { Rate string `json:"rate"` }{rateStr}, nil }
func (s *httpServer) doChannels(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } channels := s.ctx.nsqlookupd.DB.FindRegistrations("channel", topicName, "*").SubKeys() return map[string]interface{}{ "channels": channels, }, nil }
func (s *httpServer) doDeleteTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err) return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } err = s.ctx.nsqd.DeleteExistingTopic(topicName) if err != nil { return nil, http_api.Err{404, "TOPIC_NOT_FOUND"} } return nil, nil }
func (s *httpServer) getExistingTopicFromQuery(req *http.Request) (*http_api.ReqParams, *Topic, string, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err) return nil, nil, "", http_api.Err{400, "INVALID_REQUEST"} } topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams) if err != nil { return nil, nil, "", http_api.Err{400, err.Error()} } topic, err := s.ctx.nsqd.GetExistingTopic(topicName) if err != nil { return nil, nil, "", http_api.Err{404, "TOPIC_NOT_FOUND"} } return reqParams, topic, channelName, err }
func (s *httpServer) doCreateChannel(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams) if err != nil { return nil, http_api.Err{400, err.Error()} } s.ctx.nsqlookupd.logf("DB: adding channel(%s) in topic(%s)", channelName, topicName) key := Registration{"channel", topicName, channelName} s.ctx.nsqlookupd.DB.AddRegistration(key) s.ctx.nsqlookupd.logf("DB: adding topic(%s)", topicName) key = Registration{"topic", topicName, ""} s.ctx.nsqlookupd.DB.AddRegistration(key) return nil, nil }
func (s *httpServer) doCreateTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } if !protocol.IsValidTopicName(topicName) { return nil, http_api.Err{400, "INVALID_ARG_TOPIC"} } s.ctx.nsqlookupd.logf("DB: adding topic(%s)", topicName) key := Registration{"topic", topicName, ""} s.ctx.nsqlookupd.DB.AddRegistration(key) return nil, nil }
func (s *httpServer) doTopics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, err.Error()} } var topics []string if len(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses) != 0 { topics, _ = s.ci.GetLookupdTopics(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses) } else { topics, _ = s.ci.GetNSQDTopics(s.ctx.nsqadmin.opts.NSQDHTTPAddresses) } inactive, _ := reqParams.Get("inactive") if inactive == "true" { topicChannelMap := make(map[string][]string) if len(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses) == 0 { goto respond } for _, topicName := range topics { producerList, _ := s.ci.GetLookupdTopicProducers( topicName, s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses) if len(producerList) == 0 { topicChannels, _ := s.ci.GetLookupdTopicChannels( topicName, s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses) topicChannelMap[topicName] = topicChannels } } respond: return struct { Topics map[string][]string `json:"topics"` }{topicChannelMap}, nil } return struct { Topics []string `json:"topics"` }{topics}, nil }
func (s *httpServer) doDeleteChannel(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams) if err != nil { return nil, http_api.Err{400, err.Error()} } registrations := s.ctx.nsqlookupd.DB.FindRegistrations("channel", topicName, channelName) if len(registrations) == 0 { return nil, http_api.Err{404, "CHANNEL_NOT_FOUND"} } s.ctx.nsqlookupd.logf("DB: removing channel(%s) from topic(%s)", channelName, topicName) for _, registration := range registrations { s.ctx.nsqlookupd.DB.RemoveRegistration(registration) } return nil, nil }
func (s *StreamServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { path := req.URL.Path if path == "/stats" { StatsHandler(w, req) return } if path != "/sub" { w.WriteHeader(404) return } reqParams, err := http_api.NewReqParams(req) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } hj, ok := w.(http.Hijacker) if !ok { http.Error(w, "httpserver doesn't support hijacking", http.StatusInternalServerError) return } conn, bufrw, err := hj.Hijack() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } cfg := nsq.NewConfig() cfg.UserAgent = fmt.Sprintf("nsq_pubsub/%s go-nsq/%s", version.Binary, nsq.VERSION) cfg.MaxInFlight = *maxInFlight r, err := nsq.NewConsumer(topicName, channelName, cfg) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } r.SetLogger(log.New(os.Stderr, "", log.LstdFlags), nsq.LogLevelInfo) sr := &StreamReader{ topic: topicName, channel: channelName, consumer: r, req: req, conn: conn, bufrw: bufrw, // TODO: latency writer connectTime: time.Now(), } s.Set(sr) log.Printf("[%s] new connection", conn.RemoteAddr().String()) bufrw.WriteString("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n") bufrw.Flush() r.AddHandler(sr) // TODO: handle the error cases better (ie. at all :) ) errors := ConnectToNSQAndLookupd(r, nsqdTCPAddrs, lookupdHTTPAddrs) log.Printf("connected to NSQ %v", errors) // this read allows us to detect clients that disconnect go func(rw *bufio.ReadWriter) { b, err := rw.ReadByte() if err != nil { log.Printf("got connection err %s", err.Error()) } else { log.Printf("unexpected data on request socket (%c); closing", b) } sr.consumer.Stop() }(bufrw) go sr.HeartbeatLoop() }