func (s *httpServer) doTombstoneTopicProducer(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, err := reqParams.Get("topic")
if err != nil {
return nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
}
node, err := reqParams.Get("node")
if err != nil {
return nil, http_api.Err{400, "MISSING_ARG_NODE"}
}
s.ctx.nsqlookupd.logf("DB: setting tombstone for producer@%s of topic(%s)", node, topicName)
producers := s.ctx.nsqlookupd.DB.FindProducers("topic", topicName, "")
for _, p := range producers {
thisNode := fmt.Sprintf("%s:%d", p.peerInfo.BroadcastAddress, p.peerInfo.HTTPPort)
if thisNode == node {
p.Tombstone()
}
}
return nil, nil
}
func (s *httpServer) doPauseTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err)
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, err := reqParams.Get("topic")
if err != nil {
return nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
}
topic, err := s.ctx.nsqd.GetExistingTopic(topicName)
if err != nil {
return nil, http_api.Err{404, "TOPIC_NOT_FOUND"}
}
if strings.Contains(req.URL.Path, "unpause") {
err = topic.UnPause()
} else {
err = topic.Pause()
}
if err != nil {
s.ctx.nsqd.logf("ERROR: failure in %s - %s", req.URL.Path, err)
return nil, http_api.Err{500, "INTERNAL_ERROR"}
}
// pro-actively persist metadata so in case of process failure
// nsqd won't suddenly (un)pause a topic
s.ctx.nsqd.Lock()
s.ctx.nsqd.PersistMetadata()
s.ctx.nsqd.Unlock()
return nil, nil
}
func (s *httpServer) doEmptyTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err)
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, err := reqParams.Get("topic")
if err != nil {
return nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
}
if !protocol.IsValidTopicName(topicName) {
return nil, http_api.Err{400, "INVALID_TOPIC"}
}
topic, err := s.ctx.nsqd.GetExistingTopic(topicName)
if err != nil {
return nil, http_api.Err{404, "TOPIC_NOT_FOUND"}
}
err = topic.Empty()
if err != nil {
return nil, http_api.Err{500, "INTERNAL_ERROR"}
}
return nil, nil
}
func (s *httpServer) doDeleteTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, err := reqParams.Get("topic")
if err != nil {
return nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
}
registrations := s.ctx.nsqlookupd.DB.FindRegistrations("channel", topicName, "*")
for _, registration := range registrations {
s.ctx.nsqlookupd.logf("DB: removing channel(%s) from topic(%s)", registration.SubKey, topicName)
s.ctx.nsqlookupd.DB.RemoveRegistration(registration)
}
registrations = s.ctx.nsqlookupd.DB.FindRegistrations("topic", topicName, "")
for _, registration := range registrations {
s.ctx.nsqlookupd.logf("DB: removing topic(%s)", topicName)
s.ctx.nsqlookupd.DB.RemoveRegistration(registration)
}
return nil, nil
}
func (s *httpServer) doLookup(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, err := reqParams.Get("topic")
if err != nil {
return nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
}
registration := s.ctx.nsqlookupd.DB.FindRegistrations("topic", topicName, "")
if len(registration) == 0 {
return nil, http_api.Err{404, "TOPIC_NOT_FOUND"}
}
channels := s.ctx.nsqlookupd.DB.FindRegistrations("channel", topicName, "*").SubKeys()
producers := s.ctx.nsqlookupd.DB.FindProducers("topic", topicName, "")
producers = producers.FilterByActive(s.ctx.nsqlookupd.opts.InactiveProducerTimeout,
s.ctx.nsqlookupd.opts.TombstoneLifetime)
return map[string]interface{}{
"channels": channels,
"producers": producers.PeerInfo(),
}, nil
}
func (s *httpServer) doStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err)
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
formatString, _ := reqParams.Get("format")
topicName, _ := reqParams.Get("topic")
channelName, _ := reqParams.Get("channel")
jsonFormat := formatString == "json"
stats := s.ctx.nsqd.GetStats()
health := s.ctx.nsqd.GetHealth()
startTime := s.ctx.nsqd.GetStartTime()
uptime := time.Since(startTime)
// If we WERE given a topic-name, remove stats for all the other topics:
if len(topicName) > 0 {
// Find the desired-topic-index:
for _, topicStats := range stats {
if topicStats.TopicName == topicName {
// If we WERE given a channel-name, remove stats for all the other channels:
if len(channelName) > 0 {
// Find the desired-channel:
for _, channelStats := range topicStats.Channels {
if channelStats.ChannelName == channelName {
topicStats.Channels = []ChannelStats{channelStats}
// We've got the channel we were looking for:
break
}
}
}
// We've got the topic we were looking for:
stats = []TopicStats{topicStats}
break
}
}
}
if !jsonFormat {
return s.printStats(stats, health, startTime, uptime), nil
}
return struct {
Version string `json:"version"`
Health string `json:"health"`
StartTime int64 `json:"start_time"`
Topics []TopicStats `json:"topics"`
}{version.Binary, health, startTime.Unix(), stats}, nil
}
func (s *httpServer) topicsHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
var messages []string
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, err.Error()}
}
var topics []string
if len(s.ctx.nsqadmin.getOpts().NSQLookupdHTTPAddresses) != 0 {
topics, err = s.ci.GetLookupdTopics(s.ctx.nsqadmin.getOpts().NSQLookupdHTTPAddresses)
} else {
topics, err = s.ci.GetNSQDTopics(s.ctx.nsqadmin.getOpts().NSQDHTTPAddresses)
}
if err != nil {
pe, ok := err.(clusterinfo.PartialErr)
if !ok {
s.ctx.nsqadmin.logf("ERROR: failed to get topics - %s", err)
return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)}
}
s.ctx.nsqadmin.logf("WARNING: %s", err)
messages = append(messages, pe.Error())
}
inactive, _ := reqParams.Get("inactive")
if inactive == "true" {
topicChannelMap := make(map[string][]string)
if len(s.ctx.nsqadmin.getOpts().NSQLookupdHTTPAddresses) == 0 {
goto respond
}
for _, topicName := range topics {
producers, _ := s.ci.GetLookupdTopicProducers(
topicName, s.ctx.nsqadmin.getOpts().NSQLookupdHTTPAddresses)
if len(producers) == 0 {
topicChannels, _ := s.ci.GetLookupdTopicChannels(
topicName, s.ctx.nsqadmin.getOpts().NSQLookupdHTTPAddresses)
topicChannelMap[topicName] = topicChannels
}
}
respond:
return struct {
Topics map[string][]string `json:"topics"`
Message string `json:"message"`
}{topicChannelMap, maybeWarnMsg(messages)}, nil
}
return struct {
Topics []string `json:"topics"`
Message string `json:"message"`
}{topics, maybeWarnMsg(messages)}, nil
}
func (s *httpServer) graphiteHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
metric, err := reqParams.Get("metric")
if err != nil || metric != "rate" {
return nil, http_api.Err{400, "INVALID_ARG_METRIC"}
}
target, err := reqParams.Get("target")
if err != nil {
return nil, http_api.Err{400, "INVALID_ARG_TARGET"}
}
params := url.Values{}
params.Set("from", fmt.Sprintf("-%dsec", s.ctx.nsqadmin.getOpts().StatsdInterval*2/time.Second))
params.Set("until", fmt.Sprintf("-%dsec", s.ctx.nsqadmin.getOpts().StatsdInterval/time.Second))
params.Set("format", "json")
params.Set("target", target)
query := fmt.Sprintf("/render?%s", params.Encode())
url := s.ctx.nsqadmin.getOpts().GraphiteURL + query
s.ctx.nsqadmin.logf("GRAPHITE: %s", url)
var response []struct {
Target string `json:"target"`
DataPoints [][]*float64 `json:"datapoints"`
}
err = s.client.GETV1(url, &response)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: graphite request failed - %s", err)
return nil, http_api.Err{500, "INTERNAL_ERROR"}
}
var rateStr string
rate := *response[0].DataPoints[0][0]
if rate < 0 {
rateStr = "N/A"
} else {
rateDivisor := s.ctx.nsqadmin.getOpts().StatsdInterval / time.Second
rateStr = fmt.Sprintf("%.2f", rate/float64(rateDivisor))
}
return struct {
Rate string `json:"rate"`
}{rateStr}, nil
}
func (s *httpServer) doChannels(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, err := reqParams.Get("topic")
if err != nil {
return nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
}
channels := s.ctx.nsqlookupd.DB.FindRegistrations("channel", topicName, "*").SubKeys()
return map[string]interface{}{
"channels": channels,
}, nil
}
func (s *httpServer) doDeleteTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err)
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, err := reqParams.Get("topic")
if err != nil {
return nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
}
err = s.ctx.nsqd.DeleteExistingTopic(topicName)
if err != nil {
return nil, http_api.Err{404, "TOPIC_NOT_FOUND"}
}
return nil, nil
}
func (s *httpServer) getExistingTopicFromQuery(req *http.Request) (*http_api.ReqParams, *Topic, string, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err)
return nil, nil, "", http_api.Err{400, "INVALID_REQUEST"}
}
topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams)
if err != nil {
return nil, nil, "", http_api.Err{400, err.Error()}
}
topic, err := s.ctx.nsqd.GetExistingTopic(topicName)
if err != nil {
return nil, nil, "", http_api.Err{404, "TOPIC_NOT_FOUND"}
}
return reqParams, topic, channelName, err
}
func (s *httpServer) doCreateChannel(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams)
if err != nil {
return nil, http_api.Err{400, err.Error()}
}
s.ctx.nsqlookupd.logf("DB: adding channel(%s) in topic(%s)", channelName, topicName)
key := Registration{"channel", topicName, channelName}
s.ctx.nsqlookupd.DB.AddRegistration(key)
s.ctx.nsqlookupd.logf("DB: adding topic(%s)", topicName)
key = Registration{"topic", topicName, ""}
s.ctx.nsqlookupd.DB.AddRegistration(key)
return nil, nil
}
func (s *httpServer) doCreateTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, err := reqParams.Get("topic")
if err != nil {
return nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
}
if !protocol.IsValidTopicName(topicName) {
return nil, http_api.Err{400, "INVALID_ARG_TOPIC"}
}
s.ctx.nsqlookupd.logf("DB: adding topic(%s)", topicName)
key := Registration{"topic", topicName, ""}
s.ctx.nsqlookupd.DB.AddRegistration(key)
return nil, nil
}
func (s *httpServer) doTopics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, err.Error()}
}
var topics []string
if len(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses) != 0 {
topics, _ = s.ci.GetLookupdTopics(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
} else {
topics, _ = s.ci.GetNSQDTopics(s.ctx.nsqadmin.opts.NSQDHTTPAddresses)
}
inactive, _ := reqParams.Get("inactive")
if inactive == "true" {
topicChannelMap := make(map[string][]string)
if len(s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses) == 0 {
goto respond
}
for _, topicName := range topics {
producerList, _ := s.ci.GetLookupdTopicProducers(
topicName, s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
if len(producerList) == 0 {
topicChannels, _ := s.ci.GetLookupdTopicChannels(
topicName, s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
topicChannelMap[topicName] = topicChannels
}
}
respond:
return struct {
Topics map[string][]string `json:"topics"`
}{topicChannelMap}, nil
}
return struct {
Topics []string `json:"topics"`
}{topics}, nil
}
func (s *httpServer) doDeleteChannel(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams)
if err != nil {
return nil, http_api.Err{400, err.Error()}
}
registrations := s.ctx.nsqlookupd.DB.FindRegistrations("channel", topicName, channelName)
if len(registrations) == 0 {
return nil, http_api.Err{404, "CHANNEL_NOT_FOUND"}
}
s.ctx.nsqlookupd.logf("DB: removing channel(%s) from topic(%s)", channelName, topicName)
for _, registration := range registrations {
s.ctx.nsqlookupd.DB.RemoveRegistration(registration)
}
return nil, nil
}
func (s *StreamServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
path := req.URL.Path
if path == "/stats" {
StatsHandler(w, req)
return
}
if path != "/sub" {
w.WriteHeader(404)
return
}
reqParams, err := http_api.NewReqParams(req)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
hj, ok := w.(http.Hijacker)
if !ok {
http.Error(w, "httpserver doesn't support hijacking", http.StatusInternalServerError)
return
}
conn, bufrw, err := hj.Hijack()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
cfg := nsq.NewConfig()
cfg.UserAgent = fmt.Sprintf("nsq_pubsub/%s go-nsq/%s", version.Binary, nsq.VERSION)
cfg.MaxInFlight = *maxInFlight
r, err := nsq.NewConsumer(topicName, channelName, cfg)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
r.SetLogger(log.New(os.Stderr, "", log.LstdFlags), nsq.LogLevelInfo)
sr := &StreamReader{
topic: topicName,
channel: channelName,
consumer: r,
req: req,
conn: conn,
bufrw: bufrw, // TODO: latency writer
connectTime: time.Now(),
}
s.Set(sr)
log.Printf("[%s] new connection", conn.RemoteAddr().String())
bufrw.WriteString("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n")
bufrw.Flush()
r.AddHandler(sr)
// TODO: handle the error cases better (ie. at all :) )
errors := ConnectToNSQAndLookupd(r, nsqdTCPAddrs, lookupdHTTPAddrs)
log.Printf("connected to NSQ %v", errors)
// this read allows us to detect clients that disconnect
go func(rw *bufio.ReadWriter) {
b, err := rw.ReadByte()
if err != nil {
log.Printf("got connection err %s", err.Error())
} else {
log.Printf("unexpected data on request socket (%c); closing", b)
}
sr.consumer.Stop()
}(bufrw)
go sr.HeartbeatLoop()
}