func deleteTopicHandler(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
log.Printf("ERROR: invalid %s to POST only method", req.Method)
http.Error(w, "INVALID_REQUEST", 500)
return
}
reqParams := &util.PostParams{req}
topicName, err := reqParams.Get("topic")
if err != nil {
http.Error(w, "MISSING_ARG_TOPIC", 500)
return
}
rd, _ := reqParams.Get("rd")
if !strings.HasPrefix(rd, "/") {
rd = "/"
}
// for topic removal, you need to get all the producers *first*
var producers []string
if len(lookupdHTTPAddrs) != 0 {
producers, _ = getLookupdTopicProducers(topicName, lookupdHTTPAddrs)
} else {
producers, _ = getNSQDTopicProducers(topicName, nsqdHTTPAddrs)
}
// remove the topic from all the lookupds
for _, addr := range lookupdHTTPAddrs {
endpoint := fmt.Sprintf("http://%s/delete_topic?topic=%s", addr, url.QueryEscape(topicName))
log.Printf("LOOKUPD: querying %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
continue
}
}
// now remove the topic from all the producers
for _, addr := range producers {
endpoint := fmt.Sprintf("http://%s/delete_topic?topic=%s", addr, url.QueryEscape(topicName))
log.Printf("NSQD: querying %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
continue
}
}
NotifyAdminAction("delete_topic", topicName, "", req)
http.Redirect(w, req, rd, 302)
}
func deleteChannelHandler(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
log.Printf("ERROR: invalid %s to POST only method", req.Method)
http.Error(w, "INVALID_REQUEST", 500)
return
}
reqParams := &util.PostParams{req}
topicName, channelName, err := util.GetTopicChannelArgs(reqParams)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
rd, _ := reqParams.Get("rd")
if !strings.HasPrefix(rd, "/") {
rd = fmt.Sprintf("/topic/%s", url.QueryEscape(topicName))
}
for _, addr := range lookupdHTTPAddrs {
endpoint := fmt.Sprintf("http://%s/delete_channel?topic=%s&channel=%s",
addr, url.QueryEscape(topicName), url.QueryEscape(channelName))
log.Printf("LOOKUPD: querying %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
continue
}
}
var producers []string
if len(lookupdHTTPAddrs) != 0 {
producers, _ = getLookupdTopicProducers(topicName, lookupdHTTPAddrs)
} else {
producers, _ = getNSQDTopicProducers(topicName, nsqdHTTPAddrs)
}
for _, addr := range producers {
endpoint := fmt.Sprintf("http://%s/delete_channel?topic=%s&channel=%s",
addr, url.QueryEscape(topicName), url.QueryEscape(channelName))
log.Printf("NSQD: querying %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
continue
}
}
NotifyAdminAction("delete_channel", topicName, channelName, req)
http.Redirect(w, req, rd, 302)
}
func tombstoneTopicProducerHandler(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
log.Printf("ERROR: invalid %s to POST only method", req.Method)
http.Error(w, "INVALID_REQUEST", 500)
return
}
reqParams := &util.PostParams{req}
topicName, err := reqParams.Get("topic")
if err != nil {
http.Error(w, "MISSING_ARG_TOPIC", 500)
return
}
node, err := reqParams.Get("node")
if err != nil {
http.Error(w, "MISSING_ARG_NODE", 500)
return
}
rd, _ := reqParams.Get("rd")
if !strings.HasPrefix(rd, "/") {
rd = "/"
}
// tombstone the topic on all the lookupds
for _, addr := range lookupdHTTPAddrs {
endpoint := fmt.Sprintf("http://%s/tombstone_topic_producer?topic=%s&node=%s",
addr, url.QueryEscape(topicName), url.QueryEscape(node))
log.Printf("LOOKUPD: querying %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
}
}
// delete the topic on the producer
endpoint := fmt.Sprintf("http://%s/delete_topic?topic=%s", node, url.QueryEscape(topicName))
log.Printf("NSQD: querying %s", endpoint)
_, err = nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
}
NotifyAdminAction("tombstone_topic_producer", topicName, "", req)
http.Redirect(w, req, rd, 302)
}
func getLookupdTopics(lookupdHTTPAddrs []string) ([]string, error) {
success := false
allTopics := make([]string, 0)
var lock sync.Mutex
var wg sync.WaitGroup
for _, addr := range lookupdHTTPAddrs {
wg.Add(1)
endpoint := fmt.Sprintf("http://%s/topics", addr)
log.Printf("LOOKUPD: querying %s", endpoint)
go func(endpoint string) {
data, err := nsq.ApiRequest(endpoint)
lock.Lock()
defer lock.Unlock()
defer wg.Done()
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
return
}
success = true
// {"data":{"topics":["test"]}}
// TODO: convert this to a StringArray() function in simplejson
topics, _ := data.Get("topics").Array()
allTopics = util.StringUnion(allTopics, topics)
}(endpoint)
}
wg.Wait()
sort.Strings(allTopics)
if success == false {
return nil, errors.New("unable to query any lookupd")
}
return allTopics, nil
}
func emptyTopicHandler(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
log.Printf("ERROR: invalid %s to POST only method", req.Method)
http.Error(w, "INVALID_REQUEST", 500)
return
}
reqParams := &util.PostParams{req}
topicName, err := reqParams.Get("topic")
if err != nil {
http.Error(w, "MISSING_ARG_TOPIC", 500)
return
}
producers := getProducers(topicName)
for _, addr := range producers {
endpoint := fmt.Sprintf("http://%s/empty_topic?topic=%s", addr, url.QueryEscape(topicName))
log.Printf("NSQD: calling %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
continue
}
}
NotifyAdminAction("empty_topic", topicName, "", req)
http.Redirect(w, req, fmt.Sprintf("/topic/%s", url.QueryEscape(topicName)), 302)
}
// GetLookupdTopicChannels returns a []string containing a union of the channels
// from all the given lookupd for the given topic
func GetLookupdTopicChannels(topic string, lookupdHTTPAddrs []string) ([]string, error) {
success := false
allChannels := make([]string, 0)
var lock sync.Mutex
var wg sync.WaitGroup
for _, addr := range lookupdHTTPAddrs {
wg.Add(1)
endpoint := fmt.Sprintf("http://%s/channels?topic=%s", addr, url.QueryEscape(topic))
log.Printf("LOOKUPD: querying %s", endpoint)
go func(endpoint string) {
data, err := nsq.ApiRequest(endpoint)
lock.Lock()
defer lock.Unlock()
defer wg.Done()
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
return
}
success = true
// {"data":{"channels":["test"]}}
channels, _ := data.Get("channels").Array()
allChannels = util.StringUnion(allChannels, channels)
}(endpoint)
}
wg.Wait()
sort.Strings(allChannels)
if success == false {
return nil, errors.New("unable to query any lookupd")
}
return allChannels, nil
}
// GetNSQDTopics returns a []string containing all the topics
// produced by the given nsqd
func GetNSQDTopics(nsqdHTTPAddrs []string) ([]string, error) {
topics := make([]string, 0)
var lock sync.Mutex
var wg sync.WaitGroup
success := false
for _, addr := range nsqdHTTPAddrs {
wg.Add(1)
endpoint := fmt.Sprintf("http://%s/stats?format=json", addr)
log.Printf("NSQD: querying %s", endpoint)
go func(endpoint string) {
data, err := nsq.ApiRequest(endpoint)
lock.Lock()
defer lock.Unlock()
defer wg.Done()
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
return
}
success = true
topicList, _ := data.Get("topics").Array()
for _, topicInfo := range topicList {
topicInfo := topicInfo.(map[string]interface{})
topicName := topicInfo["topic_name"].(string)
topics = util.StringAdd(topics, topicName)
}
}(endpoint)
}
wg.Wait()
sort.Strings(topics)
if success == false {
return nil, errors.New("unable to query any nsqd")
}
return topics, nil
}
func TestTombstonedNodes(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
tcpAddr, httpAddr := mustStartLookupd()
defer lookupd.Exit()
lookupd.inactiveProducerTimeout = 50 * time.Millisecond
lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)}
topicName := "inactive_nodes"
conn := mustConnectLookupd(t, tcpAddr)
identify(t, conn, "ip.address", 5000, 5555, "fake-version")
nsq.Register(topicName, "channel1").Write(conn)
_, err := nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs)
assert.Equal(t, len(producers), 1)
assert.Equal(t, len(producers[0].Topics), 1)
assert.Equal(t, producers[0].Topics[0].Topic, topicName)
assert.Equal(t, producers[0].Topics[0].Tombstoned, false)
endpoint := fmt.Sprintf("http://%s/tombstone_topic_producer?topic=%s&node=%s", httpAddr, topicName, "ip.address:5555")
_, err = nsq.ApiRequest(endpoint)
assert.Equal(t, err, nil)
producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs)
assert.Equal(t, len(producers), 1)
assert.Equal(t, len(producers[0].Topics), 1)
assert.Equal(t, producers[0].Topics[0].Topic, topicName)
assert.Equal(t, producers[0].Topics[0].Tombstoned, true)
}
func pauseChannelHandler(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
log.Printf("ERROR: invalid %s to POST only method", req.Method)
http.Error(w, "INVALID_REQUEST", 500)
return
}
reqParams := &util.PostParams{req}
topicName, channelName, err := util.GetTopicChannelArgs(reqParams)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
producers := getProducers(topicName)
for _, addr := range producers {
endpoint := fmt.Sprintf("http://%s%s?topic=%s&channel=%s",
addr, req.URL.Path, url.QueryEscape(topicName), url.QueryEscape(channelName))
log.Printf("NSQD: calling %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
continue
}
}
NotifyAdminAction(strings.TrimLeft(req.URL.Path, "/"), topicName, channelName, req)
http.Redirect(w, req, fmt.Sprintf("/topic/%s/%s", url.QueryEscape(topicName), url.QueryEscape(channelName)), 302)
}
func GetChannelsForTopic(topic string, lookupdHTTPAddrs []string) ([]string, error) {
channels := make([]string, 0)
var lock sync.Mutex
var wg sync.WaitGroup
success := false
for _, addr := range lookupdHTTPAddrs {
wg.Add(1)
endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", addr, url.QueryEscape(topic))
log.Printf("LOOKUPD: querying %s", endpoint)
go func(endpiont string) {
data, err := nsq.ApiRequest(endpoint)
lock.Lock()
defer lock.Unlock()
defer wg.Done()
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
return
}
success = true
c, _ := data.Get("channels").Array()
channels = StringUnion(channels, c)
}(endpoint)
}
wg.Wait()
if success == false {
return nil, errors.New("unable to query any lookupd")
}
return channels, nil
}
func TestTombstoneRecover(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
tcpAddr, httpAddr := mustStartLookupd()
defer lookupd.Exit()
lookupd.tombstoneLifetime = 50 * time.Millisecond
topicName := "tombstone_recover"
topicName2 := topicName + "2"
conn := mustConnectLookupd(t, tcpAddr)
identify(t, conn, "ip.address", 5000, 5555, "fake-version")
nsq.Register(topicName, "channel1").Write(conn)
_, err := nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
nsq.Register(topicName2, "channel2").Write(conn)
_, err = nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
endpoint := fmt.Sprintf("http://%s/tombstone_topic_producer?topic=%s&node=%s", httpAddr, topicName, "ip.address:5555")
_, err = nsq.ApiRequest(endpoint)
assert.Equal(t, err, nil)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
data, err := nsq.ApiRequest(endpoint)
assert.Equal(t, err, nil)
producers, _ := data.Get("producers").Array()
assert.Equal(t, len(producers), 0)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName2)
data, err = nsq.ApiRequest(endpoint)
assert.Equal(t, err, nil)
producers, _ = data.Get("producers").Array()
assert.Equal(t, len(producers), 1)
time.Sleep(55 * time.Millisecond)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
data, err = nsq.ApiRequest(endpoint)
assert.Equal(t, err, nil)
producers, _ = data.Get("producers").Array()
assert.Equal(t, len(producers), 1)
}
func TestChannelUnregister(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
tcpAddr, httpAddr := mustStartLookupd()
defer lookupd.Exit()
topics := lookupd.DB.FindRegistrations("topic", "*", "*")
assert.Equal(t, len(topics), 0)
topicName := "channel_unregister"
conn := mustConnectLookupd(t, tcpAddr)
tcpPort := 5000
httpPort := 5555
identify(t, conn, "ip.address", tcpPort, httpPort, "fake-version")
nsq.Register(topicName, "ch1").Write(conn)
v, err := nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
assert.Equal(t, v, []byte("OK"))
topics = lookupd.DB.FindRegistrations("topic", topicName, "")
assert.Equal(t, len(topics), 1)
channels := lookupd.DB.FindRegistrations("channel", topicName, "*")
assert.Equal(t, len(channels), 1)
nsq.UnRegister(topicName, "ch1").Write(conn)
v, err = nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
assert.Equal(t, v, []byte("OK"))
topics = lookupd.DB.FindRegistrations("topic", topicName, "")
assert.Equal(t, len(topics), 1)
// we should still have mention of the topic even though there is no producer
// (ie. we haven't *deleted* the channel, just unregistered as a producer)
channels = lookupd.DB.FindRegistrations("channel", topicName, "*")
assert.Equal(t, len(channels), 1)
endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
data, err := nsq.ApiRequest(endpoint)
assert.Equal(t, err, nil)
returnedProducers, err := data.Get("producers").Array()
assert.Equal(t, err, nil)
assert.Equal(t, len(returnedProducers), 1)
}
// GetLookupdTopicProducers returns a []string of the broadcast_address:http_port of all the
// producers for a given topic by unioning the results returned from the given lookupd
func GetLookupdTopicProducers(topic string, lookupdHTTPAddrs []string) ([]string, error) {
success := false
allSources := make([]string, 0)
var lock sync.Mutex
var wg sync.WaitGroup
for _, addr := range lookupdHTTPAddrs {
wg.Add(1)
endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", addr, url.QueryEscape(topic))
log.Printf("LOOKUPD: querying %s", endpoint)
go func(endpoint string) {
data, err := nsq.ApiRequest(endpoint)
lock.Lock()
defer lock.Unlock()
defer wg.Done()
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
return
}
success = true
producers := data.Get("producers")
producersArray, _ := producers.Array()
for i := range producersArray {
producer := producers.GetIndex(i)
address := producer.Get("address").MustString() //TODO: remove for 1.0
broadcastAddress := producer.Get("broadcast_address").MustString()
if broadcastAddress == "" {
broadcastAddress = address
}
httpPort := producer.Get("http_port").MustInt()
key := fmt.Sprintf("%s:%d", broadcastAddress, httpPort)
allSources = util.StringAdd(allSources, key)
}
}(endpoint)
}
wg.Wait()
if success == false {
return nil, errors.New("unable to query any lookupd")
}
return allSources, nil
}
func TestBasicLookupd(t *testing.T) {
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
tcpAddr, httpAddr := mustStartLookupd()
defer lookupd.Exit()
topics := lookupd.DB.FindRegistrations("topic", "*", "*")
assert.Equal(t, len(topics), 0)
topicName := "connectmsg"
conn := mustConnectLookupd(t, tcpAddr)
tcpPort := 5000
httpPort := 5555
identify(t, conn, "ip.address", tcpPort, httpPort, "fake-version")
nsq.Register(topicName, "channel1").Write(conn)
v, err := nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
assert.Equal(t, v, []byte("OK"))
endpoint := fmt.Sprintf("http://%s/nodes", httpAddr)
data, err := nsq.ApiRequest(endpoint)
log.Printf("got %v", data)
returnedProducers, err := data.Get("producers").Array()
assert.Equal(t, err, nil)
assert.Equal(t, len(returnedProducers), 1)
topics = lookupd.DB.FindRegistrations("topic", topicName, "")
assert.Equal(t, len(topics), 1)
producers := lookupd.DB.FindProducers("topic", topicName, "")
assert.Equal(t, len(producers), 1)
producer := producers[0]
assert.Equal(t, producer.peerInfo.Address, "ip.address") //TODO: remove for 1.0
assert.Equal(t, producer.peerInfo.BroadcastAddress, "ip.address")
assert.Equal(t, producer.peerInfo.Hostname, "ip.address")
assert.Equal(t, producer.peerInfo.TcpPort, tcpPort)
assert.Equal(t, producer.peerInfo.HttpPort, httpPort)
endpoint = fmt.Sprintf("http://%s/topics", httpAddr)
data, err = nsq.ApiRequest(endpoint)
assert.Equal(t, err, nil)
returnedTopics, err := data.Get("topics").Array()
log.Printf("got returnedTopics %v", returnedTopics)
assert.Equal(t, err, nil)
assert.Equal(t, len(returnedTopics), 1)
endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
data, err = nsq.ApiRequest(endpoint)
assert.Equal(t, err, nil)
returnedChannels, err := data.Get("channels").Array()
assert.Equal(t, err, nil)
assert.Equal(t, len(returnedChannels), 1)
returnedProducers, err = data.Get("producers").Array()
log.Printf("got returnedProducers %v", returnedProducers)
assert.Equal(t, err, nil)
assert.Equal(t, len(returnedProducers), 1)
for i := range returnedProducers {
producer := data.Get("producers").GetIndex(i)
log.Printf("producer %v", producer)
assert.Equal(t, err, nil)
port, err := producer.Get("tcp_port").Int()
assert.Equal(t, err, nil)
assert.Equal(t, port, tcpPort)
port, err = producer.Get("http_port").Int()
assert.Equal(t, err, nil)
assert.Equal(t, port, httpPort)
address, err := producer.Get("address").String() //TODO: remove for 1.0
broadcastaddress, err := producer.Get("broadcast_address").String()
assert.Equal(t, err, nil)
assert.Equal(t, address, "ip.address")
assert.Equal(t, broadcastaddress, "ip.address")
ver, err := producer.Get("version").String()
assert.Equal(t, err, nil)
assert.Equal(t, ver, "fake-version")
}
conn.Close()
time.Sleep(10 * time.Millisecond)
// now there should be no producers, but still topic/channel entries
data, err = nsq.ApiRequest(endpoint)
assert.Equal(t, err, nil)
returnedChannels, err = data.Get("channels").Array()
assert.Equal(t, err, nil)
assert.Equal(t, len(returnedChannels), 1)
returnedProducers, err = data.Get("producers").Array()
assert.Equal(t, err, nil)
assert.Equal(t, len(returnedProducers), 0)
}
// GetNSQDStats returns aggregate topic and channel stats from the given NSQD instances
//
// if selectedTopic is empty, this will return stats for *all* topic/channels
// and the ChannelStats dict will be keyed by topic + ':' + channel
func GetNSQDStats(nsqdHTTPAddrs []string, selectedTopic string) ([]*TopicStats, map[string]*ChannelStats, error) {
topicStats := make([]*TopicStats, 0)
channelStats := make(map[string]*ChannelStats)
success := false
var lock sync.Mutex
var wg sync.WaitGroup
for _, addr := range nsqdHTTPAddrs {
wg.Add(1)
endpoint := fmt.Sprintf("http://%s/stats?format=json", addr)
log.Printf("NSQD: querying %s", endpoint)
go func(endpoint string, addr string) {
data, err := nsq.ApiRequest(endpoint)
lock.Lock()
defer lock.Unlock()
defer wg.Done()
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
return
}
success = true
topics, _ := data.Get("topics").Array()
for _, topicInfo := range topics {
topicInfo := topicInfo.(map[string]interface{})
topicName := topicInfo["topic_name"].(string)
if selectedTopic != "" && topicName != selectedTopic {
continue
}
depth := int64(topicInfo["depth"].(float64))
backendDepth := int64(topicInfo["backend_depth"].(float64))
h := &TopicStats{
HostAddress: addr,
Depth: depth,
BackendDepth: backendDepth,
MemoryDepth: depth - backendDepth,
MessageCount: int64(topicInfo["message_count"].(float64)),
ChannelCount: len(topicInfo["channels"].([]interface{})),
Topic: topicName,
}
topicStats = append(topicStats, h)
channels := topicInfo["channels"].([]interface{})
for _, c := range channels {
c := c.(map[string]interface{})
channelName := c["channel_name"].(string)
channelStatsKey := channelName
if selectedTopic == "" {
channelStatsKey = fmt.Sprintf("%s:%s", topicName, channelName)
}
channel, ok := channelStats[channelStatsKey]
if !ok {
channel = &ChannelStats{
ChannelName: channelName,
Topic: topicName,
}
channelStats[channelStatsKey] = channel
}
h := &ChannelStats{HostAddress: addr, ChannelName: channelName, Topic: topicName}
depth := int64(c["depth"].(float64))
backendDepth := int64(c["backend_depth"].(float64))
h.Depth = depth
var paused bool
pausedInterface, ok := c["paused"]
if ok {
paused = pausedInterface.(bool)
}
h.Paused = paused
h.BackendDepth = backendDepth
h.MemoryDepth = depth - backendDepth
h.InFlightCount = int64(c["in_flight_count"].(float64))
h.DeferredCount = int64(c["deferred_count"].(float64))
h.MessageCount = int64(c["message_count"].(float64))
h.RequeueCount = int64(c["requeue_count"].(float64))
h.TimeoutCount = int64(c["timeout_count"].(float64))
clients := c["clients"].([]interface{})
// TODO: this is sort of wrong; clients should be de-duped
// client A that connects to NSQD-a and NSQD-b should only be counted once. right?
h.ClientCount = len(clients)
channel.AddHostStats(h)
// "clients": [
// {
// "version": "V2",
// "remote_address": "127.0.0.1:49700",
// "name": "jehiah-air",
// "state": 3,
// "ready_count": 1000,
// "in_flight_count": 0,
// "message_count": 0,
// "finish_count": 0,
// "requeue_count": 0,
// "connect_ts": 1347150965
// }
// ]
for _, client := range clients {
client := client.(map[string]interface{})
connected := time.Unix(int64(client["connect_ts"].(float64)), 0)
connectedDuration := time.Now().Sub(connected).Seconds()
clientInfo := &ClientInfo{
HostAddress: addr,
ClientVersion: client["version"].(string),
ClientIdentifier: fmt.Sprintf("%s:%s", client["name"].(string), strings.Split(client["remote_address"].(string), ":")[1]),
ConnectedDuration: time.Duration(int64(connectedDuration)) * time.Second, // truncate to second
InFlightCount: int(client["in_flight_count"].(float64)),
ReadyCount: int(client["ready_count"].(float64)),
FinishCount: int64(client["finish_count"].(float64)),
RequeueCount: int64(client["requeue_count"].(float64)),
MessageCount: int64(client["message_count"].(float64)),
}
channel.Clients = append(channel.Clients, clientInfo)
}
sort.Sort(ClientsByHost{channel.Clients})
}
}
sort.Sort(TopicStatsByHost{topicStats})
}(endpoint, addr)
}
wg.Wait()
if success == false {
return nil, nil, errors.New("unable to query any nsqd")
}
return topicStats, channelStats, nil
}
// GetLookupdProducers returns a slice of pointers to Producer structs
// containing metadata for each node connected to given lookupds
func GetLookupdProducers(lookupdHTTPAddrs []string) ([]*Producer, error) {
success := false
allProducers := make(map[string]*Producer, 0)
output := make([]*Producer, 0)
maxVersion, _ := semver.Parse("0.0.0")
var lock sync.Mutex
var wg sync.WaitGroup
for _, addr := range lookupdHTTPAddrs {
wg.Add(1)
endpoint := fmt.Sprintf("http://%s/nodes", addr)
log.Printf("LOOKUPD: querying %s", endpoint)
go func(addr string, endpoint string) {
data, err := nsq.ApiRequest(endpoint)
lock.Lock()
defer lock.Unlock()
defer wg.Done()
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
return
}
success = true
producers := data.Get("producers")
producersArray, _ := producers.Array()
for i := range producersArray {
producer := producers.GetIndex(i)
remoteAddress := producer.Get("remote_address").MustString()
if remoteAddress == "" {
remoteAddress = "NA"
}
address := producer.Get("address").MustString() //TODO: remove for 1.0
hostname := producer.Get("hostname").MustString()
broadcastAddress := producer.Get("broadcast_address").MustString()
if broadcastAddress == "" {
broadcastAddress = address
}
httpPort := producer.Get("http_port").MustInt()
tcpPort := producer.Get("tcp_port").MustInt()
key := fmt.Sprintf("%s:%d:%d", broadcastAddress, httpPort, tcpPort)
p, ok := allProducers[key]
if !ok {
topicList, _ := producer.Get("topics").Array()
var topics []string
for _, t := range topicList {
topics = append(topics, t.(string))
}
sort.Strings(topics)
version := producer.Get("version").MustString("unknown")
versionObj, err := semver.Parse(version)
if err != nil {
versionObj = maxVersion
}
if maxVersion.Less(versionObj) {
maxVersion = versionObj
}
p = &Producer{
Address: address, //TODO: remove for 1.0
Hostname: hostname,
BroadcastAddress: broadcastAddress,
TcpPort: tcpPort,
HttpPort: httpPort,
Version: version,
VersionObj: versionObj,
Topics: topics,
}
allProducers[key] = p
output = append(output, p)
}
p.RemoteAddresses = append(p.RemoteAddresses, fmt.Sprintf("%s/%s", addr, remoteAddress))
}
}(addr, endpoint)
}
wg.Wait()
for _, producer := range allProducers {
if producer.VersionObj.Less(maxVersion) {
producer.OutOfDate = true
}
}
sort.Sort(ProducersByHost{output})
if success == false {
return nil, errors.New("unable to query any lookupd")
}
return output, nil
}
func createTopicChannelHandler(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
log.Printf("ERROR: invalid %s to POST only method", req.Method)
http.Error(w, "INVALID_REQUEST", 500)
return
}
reqParams := &util.PostParams{req}
topicName, err := reqParams.Get("topic")
if err != nil || !nsq.IsValidTopicName(topicName) {
http.Error(w, "INVALID_TOPIC", 500)
return
}
channelName, err := reqParams.Get("channel")
if err != nil || (len(channelName) > 0 && !nsq.IsValidChannelName(channelName)) {
http.Error(w, "INVALID_CHANNEL", 500)
return
}
for _, addr := range lookupdHTTPAddrs {
endpoint := fmt.Sprintf("http://%s/create_topic?topic=%s", addr, url.QueryEscape(topicName))
log.Printf("LOOKUPD: querying %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
continue
}
}
NotifyAdminAction("create_topic", topicName, "", req)
if len(channelName) > 0 {
for _, addr := range lookupdHTTPAddrs {
endpoint := fmt.Sprintf("http://%s/create_channel?topic=%s&channel=%s",
addr, url.QueryEscape(topicName), url.QueryEscape(channelName))
log.Printf("LOOKUPD: querying %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
continue
}
}
// TODO: we can remove this when we push new channel information from nsqlookupd -> nsqd
producers, _ := getLookupdTopicProducers(topicName, lookupdHTTPAddrs)
for _, addr := range producers {
endpoint := fmt.Sprintf("http://%s/create_channel?topic=%s&channel=%s",
addr, url.QueryEscape(topicName), url.QueryEscape(channelName))
log.Printf("NSQD: querying %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
continue
}
}
NotifyAdminAction("create_channel", topicName, channelName, req)
}
http.Redirect(w, req, "/lookup", 302)
}