Example #1
0
File: http.go Project: jmanero/nsq
func deleteTopicHandler(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		log.Printf("ERROR: invalid %s to POST only method", req.Method)
		http.Error(w, "INVALID_REQUEST", 500)
		return
	}
	reqParams := &util.PostParams{req}

	topicName, err := reqParams.Get("topic")
	if err != nil {
		http.Error(w, "MISSING_ARG_TOPIC", 500)
		return
	}

	rd, _ := reqParams.Get("rd")
	if !strings.HasPrefix(rd, "/") {
		rd = "/"
	}

	// for topic removal, you need to get all the producers *first*
	var producers []string
	if len(lookupdHTTPAddrs) != 0 {
		producers, _ = getLookupdTopicProducers(topicName, lookupdHTTPAddrs)
	} else {
		producers, _ = getNSQDTopicProducers(topicName, nsqdHTTPAddrs)
	}

	// remove the topic from all the lookupds
	for _, addr := range lookupdHTTPAddrs {
		endpoint := fmt.Sprintf("http://%s/delete_topic?topic=%s", addr, url.QueryEscape(topicName))
		log.Printf("LOOKUPD: querying %s", endpoint)

		_, err := nsq.ApiRequest(endpoint)
		if err != nil {
			log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
			continue
		}
	}

	// now remove the topic from all the producers
	for _, addr := range producers {
		endpoint := fmt.Sprintf("http://%s/delete_topic?topic=%s", addr, url.QueryEscape(topicName))
		log.Printf("NSQD: querying %s", endpoint)
		_, err := nsq.ApiRequest(endpoint)
		if err != nil {
			log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
			continue
		}
	}

	NotifyAdminAction("delete_topic", topicName, "", req)

	http.Redirect(w, req, rd, 302)
}
Example #2
0
File: http.go Project: jmanero/nsq
func deleteChannelHandler(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		log.Printf("ERROR: invalid %s to POST only method", req.Method)
		http.Error(w, "INVALID_REQUEST", 500)
		return
	}
	reqParams := &util.PostParams{req}

	topicName, channelName, err := util.GetTopicChannelArgs(reqParams)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	rd, _ := reqParams.Get("rd")
	if !strings.HasPrefix(rd, "/") {
		rd = fmt.Sprintf("/topic/%s", url.QueryEscape(topicName))
	}

	for _, addr := range lookupdHTTPAddrs {
		endpoint := fmt.Sprintf("http://%s/delete_channel?topic=%s&channel=%s",
			addr, url.QueryEscape(topicName), url.QueryEscape(channelName))
		log.Printf("LOOKUPD: querying %s", endpoint)

		_, err := nsq.ApiRequest(endpoint)
		if err != nil {
			log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
			continue
		}
	}

	var producers []string
	if len(lookupdHTTPAddrs) != 0 {
		producers, _ = getLookupdTopicProducers(topicName, lookupdHTTPAddrs)
	} else {
		producers, _ = getNSQDTopicProducers(topicName, nsqdHTTPAddrs)
	}

	for _, addr := range producers {
		endpoint := fmt.Sprintf("http://%s/delete_channel?topic=%s&channel=%s",
			addr, url.QueryEscape(topicName), url.QueryEscape(channelName))
		log.Printf("NSQD: querying %s", endpoint)
		_, err := nsq.ApiRequest(endpoint)
		if err != nil {
			log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
			continue
		}
	}

	NotifyAdminAction("delete_channel", topicName, channelName, req)

	http.Redirect(w, req, rd, 302)
}
Example #3
0
File: http.go Project: jmanero/nsq
func tombstoneTopicProducerHandler(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		log.Printf("ERROR: invalid %s to POST only method", req.Method)
		http.Error(w, "INVALID_REQUEST", 500)
		return
	}
	reqParams := &util.PostParams{req}

	topicName, err := reqParams.Get("topic")
	if err != nil {
		http.Error(w, "MISSING_ARG_TOPIC", 500)
		return
	}

	node, err := reqParams.Get("node")
	if err != nil {
		http.Error(w, "MISSING_ARG_NODE", 500)
		return
	}

	rd, _ := reqParams.Get("rd")
	if !strings.HasPrefix(rd, "/") {
		rd = "/"
	}

	// tombstone the topic on all the lookupds
	for _, addr := range lookupdHTTPAddrs {
		endpoint := fmt.Sprintf("http://%s/tombstone_topic_producer?topic=%s&node=%s",
			addr, url.QueryEscape(topicName), url.QueryEscape(node))
		log.Printf("LOOKUPD: querying %s", endpoint)
		_, err := nsq.ApiRequest(endpoint)
		if err != nil {
			log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
		}
	}

	// delete the topic on the producer
	endpoint := fmt.Sprintf("http://%s/delete_topic?topic=%s", node, url.QueryEscape(topicName))
	log.Printf("NSQD: querying %s", endpoint)
	_, err = nsq.ApiRequest(endpoint)
	if err != nil {
		log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
	}

	NotifyAdminAction("tombstone_topic_producer", topicName, "", req)

	http.Redirect(w, req, rd, 302)
}
Example #4
0
func getLookupdTopics(lookupdHTTPAddrs []string) ([]string, error) {
	success := false
	allTopics := make([]string, 0)
	var lock sync.Mutex
	var wg sync.WaitGroup
	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/topics", addr)
		log.Printf("LOOKUPD: querying %s", endpoint)

		go func(endpoint string) {
			data, err := nsq.ApiRequest(endpoint)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true
			// {"data":{"topics":["test"]}}
			// TODO: convert this to a StringArray() function in simplejson
			topics, _ := data.Get("topics").Array()
			allTopics = util.StringUnion(allTopics, topics)
		}(endpoint)
	}
	wg.Wait()
	sort.Strings(allTopics)
	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}
	return allTopics, nil
}
Example #5
0
func emptyTopicHandler(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		log.Printf("ERROR: invalid %s to POST only method", req.Method)
		http.Error(w, "INVALID_REQUEST", 500)
		return
	}
	reqParams := &util.PostParams{req}

	topicName, err := reqParams.Get("topic")
	if err != nil {
		http.Error(w, "MISSING_ARG_TOPIC", 500)
		return
	}

	producers := getProducers(topicName)
	for _, addr := range producers {
		endpoint := fmt.Sprintf("http://%s/empty_topic?topic=%s", addr, url.QueryEscape(topicName))
		log.Printf("NSQD: calling %s", endpoint)

		_, err := nsq.ApiRequest(endpoint)
		if err != nil {
			log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
			continue
		}
	}

	NotifyAdminAction("empty_topic", topicName, "", req)

	http.Redirect(w, req, fmt.Sprintf("/topic/%s", url.QueryEscape(topicName)), 302)
}
Example #6
0
// GetLookupdTopicChannels returns a []string containing a union of the channels
// from all the given lookupd for the given topic
func GetLookupdTopicChannels(topic string, lookupdHTTPAddrs []string) ([]string, error) {
	success := false
	allChannels := make([]string, 0)
	var lock sync.Mutex
	var wg sync.WaitGroup
	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/channels?topic=%s", addr, url.QueryEscape(topic))
		log.Printf("LOOKUPD: querying %s", endpoint)
		go func(endpoint string) {
			data, err := nsq.ApiRequest(endpoint)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true
			// {"data":{"channels":["test"]}}
			channels, _ := data.Get("channels").Array()
			allChannels = util.StringUnion(allChannels, channels)
		}(endpoint)
	}
	wg.Wait()
	sort.Strings(allChannels)
	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}
	return allChannels, nil
}
Example #7
0
// GetNSQDTopics returns a []string containing all the topics
// produced by the given nsqd
func GetNSQDTopics(nsqdHTTPAddrs []string) ([]string, error) {
	topics := make([]string, 0)
	var lock sync.Mutex
	var wg sync.WaitGroup
	success := false
	for _, addr := range nsqdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/stats?format=json", addr)
		log.Printf("NSQD: querying %s", endpoint)

		go func(endpoint string) {
			data, err := nsq.ApiRequest(endpoint)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true
			topicList, _ := data.Get("topics").Array()
			for _, topicInfo := range topicList {
				topicInfo := topicInfo.(map[string]interface{})
				topicName := topicInfo["topic_name"].(string)
				topics = util.StringAdd(topics, topicName)
			}
		}(endpoint)
	}
	wg.Wait()
	sort.Strings(topics)
	if success == false {
		return nil, errors.New("unable to query any nsqd")
	}
	return topics, nil
}
Example #8
0
func TestTombstonedNodes(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	tcpAddr, httpAddr := mustStartLookupd()
	defer lookupd.Exit()
	lookupd.inactiveProducerTimeout = 50 * time.Millisecond

	lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)}

	topicName := "inactive_nodes"

	conn := mustConnectLookupd(t, tcpAddr)
	identify(t, conn, "ip.address", 5000, 5555, "fake-version")

	nsq.Register(topicName, "channel1").Write(conn)
	_, err := nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)

	producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs)
	assert.Equal(t, len(producers), 1)
	assert.Equal(t, len(producers[0].Topics), 1)
	assert.Equal(t, producers[0].Topics[0].Topic, topicName)
	assert.Equal(t, producers[0].Topics[0].Tombstoned, false)

	endpoint := fmt.Sprintf("http://%s/tombstone_topic_producer?topic=%s&node=%s", httpAddr, topicName, "ip.address:5555")
	_, err = nsq.ApiRequest(endpoint)
	assert.Equal(t, err, nil)

	producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs)
	assert.Equal(t, len(producers), 1)
	assert.Equal(t, len(producers[0].Topics), 1)
	assert.Equal(t, producers[0].Topics[0].Topic, topicName)
	assert.Equal(t, producers[0].Topics[0].Tombstoned, true)
}
Example #9
0
func pauseChannelHandler(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		log.Printf("ERROR: invalid %s to POST only method", req.Method)
		http.Error(w, "INVALID_REQUEST", 500)
		return
	}
	reqParams := &util.PostParams{req}

	topicName, channelName, err := util.GetTopicChannelArgs(reqParams)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	producers := getProducers(topicName)
	for _, addr := range producers {
		endpoint := fmt.Sprintf("http://%s%s?topic=%s&channel=%s",
			addr, req.URL.Path, url.QueryEscape(topicName), url.QueryEscape(channelName))
		log.Printf("NSQD: calling %s", endpoint)

		_, err := nsq.ApiRequest(endpoint)
		if err != nil {
			log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
			continue
		}
	}

	NotifyAdminAction(strings.TrimLeft(req.URL.Path, "/"), topicName, channelName, req)

	http.Redirect(w, req, fmt.Sprintf("/topic/%s/%s", url.QueryEscape(topicName), url.QueryEscape(channelName)), 302)
}
Example #10
0
func GetChannelsForTopic(topic string, lookupdHTTPAddrs []string) ([]string, error) {
	channels := make([]string, 0)
	var lock sync.Mutex
	var wg sync.WaitGroup
	success := false
	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", addr, url.QueryEscape(topic))
		log.Printf("LOOKUPD: querying %s", endpoint)
		go func(endpiont string) {
			data, err := nsq.ApiRequest(endpoint)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true
			c, _ := data.Get("channels").Array()
			channels = StringUnion(channels, c)
		}(endpoint)
	}
	wg.Wait()
	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}
	return channels, nil
}
Example #11
0
func TestTombstoneRecover(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	tcpAddr, httpAddr := mustStartLookupd()
	defer lookupd.Exit()
	lookupd.tombstoneLifetime = 50 * time.Millisecond

	topicName := "tombstone_recover"
	topicName2 := topicName + "2"

	conn := mustConnectLookupd(t, tcpAddr)
	identify(t, conn, "ip.address", 5000, 5555, "fake-version")

	nsq.Register(topicName, "channel1").Write(conn)
	_, err := nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)

	nsq.Register(topicName2, "channel2").Write(conn)
	_, err = nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)

	endpoint := fmt.Sprintf("http://%s/tombstone_topic_producer?topic=%s&node=%s", httpAddr, topicName, "ip.address:5555")
	_, err = nsq.ApiRequest(endpoint)
	assert.Equal(t, err, nil)

	endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
	data, err := nsq.ApiRequest(endpoint)
	assert.Equal(t, err, nil)
	producers, _ := data.Get("producers").Array()
	assert.Equal(t, len(producers), 0)

	endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName2)
	data, err = nsq.ApiRequest(endpoint)
	assert.Equal(t, err, nil)
	producers, _ = data.Get("producers").Array()
	assert.Equal(t, len(producers), 1)

	time.Sleep(55 * time.Millisecond)

	endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
	data, err = nsq.ApiRequest(endpoint)
	assert.Equal(t, err, nil)
	producers, _ = data.Get("producers").Array()
	assert.Equal(t, len(producers), 1)
}
Example #12
0
func TestChannelUnregister(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	tcpAddr, httpAddr := mustStartLookupd()
	defer lookupd.Exit()

	topics := lookupd.DB.FindRegistrations("topic", "*", "*")
	assert.Equal(t, len(topics), 0)

	topicName := "channel_unregister"

	conn := mustConnectLookupd(t, tcpAddr)
	tcpPort := 5000
	httpPort := 5555
	identify(t, conn, "ip.address", tcpPort, httpPort, "fake-version")

	nsq.Register(topicName, "ch1").Write(conn)
	v, err := nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)
	assert.Equal(t, v, []byte("OK"))

	topics = lookupd.DB.FindRegistrations("topic", topicName, "")
	assert.Equal(t, len(topics), 1)

	channels := lookupd.DB.FindRegistrations("channel", topicName, "*")
	assert.Equal(t, len(channels), 1)

	nsq.UnRegister(topicName, "ch1").Write(conn)
	v, err = nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)
	assert.Equal(t, v, []byte("OK"))

	topics = lookupd.DB.FindRegistrations("topic", topicName, "")
	assert.Equal(t, len(topics), 1)

	// we should still have mention of the topic even though there is no producer
	// (ie. we haven't *deleted* the channel, just unregistered as a producer)
	channels = lookupd.DB.FindRegistrations("channel", topicName, "*")
	assert.Equal(t, len(channels), 1)

	endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
	data, err := nsq.ApiRequest(endpoint)
	assert.Equal(t, err, nil)
	returnedProducers, err := data.Get("producers").Array()
	assert.Equal(t, err, nil)
	assert.Equal(t, len(returnedProducers), 1)
}
Example #13
0
// GetLookupdTopicProducers returns a []string of the broadcast_address:http_port of all the
// producers for a given topic by unioning the results returned from the given lookupd
func GetLookupdTopicProducers(topic string, lookupdHTTPAddrs []string) ([]string, error) {
	success := false
	allSources := make([]string, 0)
	var lock sync.Mutex
	var wg sync.WaitGroup

	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)

		endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", addr, url.QueryEscape(topic))
		log.Printf("LOOKUPD: querying %s", endpoint)

		go func(endpoint string) {
			data, err := nsq.ApiRequest(endpoint)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true
			producers := data.Get("producers")
			producersArray, _ := producers.Array()
			for i := range producersArray {
				producer := producers.GetIndex(i)
				address := producer.Get("address").MustString() //TODO: remove for 1.0
				broadcastAddress := producer.Get("broadcast_address").MustString()
				if broadcastAddress == "" {
					broadcastAddress = address
				}
				httpPort := producer.Get("http_port").MustInt()
				key := fmt.Sprintf("%s:%d", broadcastAddress, httpPort)
				allSources = util.StringAdd(allSources, key)
			}
		}(endpoint)
	}
	wg.Wait()
	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}
	return allSources, nil
}
Example #14
0
func TestBasicLookupd(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	tcpAddr, httpAddr := mustStartLookupd()
	defer lookupd.Exit()

	topics := lookupd.DB.FindRegistrations("topic", "*", "*")
	assert.Equal(t, len(topics), 0)

	topicName := "connectmsg"

	conn := mustConnectLookupd(t, tcpAddr)
	tcpPort := 5000
	httpPort := 5555
	identify(t, conn, "ip.address", tcpPort, httpPort, "fake-version")

	nsq.Register(topicName, "channel1").Write(conn)
	v, err := nsq.ReadResponse(conn)
	assert.Equal(t, err, nil)
	assert.Equal(t, v, []byte("OK"))

	endpoint := fmt.Sprintf("http://%s/nodes", httpAddr)
	data, err := nsq.ApiRequest(endpoint)
	log.Printf("got %v", data)
	returnedProducers, err := data.Get("producers").Array()
	assert.Equal(t, err, nil)
	assert.Equal(t, len(returnedProducers), 1)

	topics = lookupd.DB.FindRegistrations("topic", topicName, "")
	assert.Equal(t, len(topics), 1)

	producers := lookupd.DB.FindProducers("topic", topicName, "")
	assert.Equal(t, len(producers), 1)
	producer := producers[0]

	assert.Equal(t, producer.peerInfo.Address, "ip.address") //TODO: remove for 1.0
	assert.Equal(t, producer.peerInfo.BroadcastAddress, "ip.address")
	assert.Equal(t, producer.peerInfo.Hostname, "ip.address")
	assert.Equal(t, producer.peerInfo.TcpPort, tcpPort)
	assert.Equal(t, producer.peerInfo.HttpPort, httpPort)

	endpoint = fmt.Sprintf("http://%s/topics", httpAddr)
	data, err = nsq.ApiRequest(endpoint)
	assert.Equal(t, err, nil)
	returnedTopics, err := data.Get("topics").Array()
	log.Printf("got returnedTopics %v", returnedTopics)
	assert.Equal(t, err, nil)
	assert.Equal(t, len(returnedTopics), 1)

	endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
	data, err = nsq.ApiRequest(endpoint)
	assert.Equal(t, err, nil)
	returnedChannels, err := data.Get("channels").Array()
	assert.Equal(t, err, nil)
	assert.Equal(t, len(returnedChannels), 1)

	returnedProducers, err = data.Get("producers").Array()
	log.Printf("got returnedProducers %v", returnedProducers)
	assert.Equal(t, err, nil)
	assert.Equal(t, len(returnedProducers), 1)
	for i := range returnedProducers {
		producer := data.Get("producers").GetIndex(i)
		log.Printf("producer %v", producer)
		assert.Equal(t, err, nil)
		port, err := producer.Get("tcp_port").Int()
		assert.Equal(t, err, nil)
		assert.Equal(t, port, tcpPort)
		port, err = producer.Get("http_port").Int()
		assert.Equal(t, err, nil)
		assert.Equal(t, port, httpPort)
		address, err := producer.Get("address").String() //TODO: remove for 1.0
		broadcastaddress, err := producer.Get("broadcast_address").String()

		assert.Equal(t, err, nil)
		assert.Equal(t, address, "ip.address")
		assert.Equal(t, broadcastaddress, "ip.address")
		ver, err := producer.Get("version").String()
		assert.Equal(t, err, nil)
		assert.Equal(t, ver, "fake-version")
	}

	conn.Close()
	time.Sleep(10 * time.Millisecond)

	// now there should be no producers, but still topic/channel entries
	data, err = nsq.ApiRequest(endpoint)
	assert.Equal(t, err, nil)
	returnedChannels, err = data.Get("channels").Array()
	assert.Equal(t, err, nil)
	assert.Equal(t, len(returnedChannels), 1)
	returnedProducers, err = data.Get("producers").Array()
	assert.Equal(t, err, nil)
	assert.Equal(t, len(returnedProducers), 0)
}
Example #15
0
// GetNSQDStats returns aggregate topic and channel stats from the given NSQD instances
//
// if selectedTopic is empty, this will return stats for *all* topic/channels
// and the ChannelStats dict will be keyed by topic + ':' + channel
func GetNSQDStats(nsqdHTTPAddrs []string, selectedTopic string) ([]*TopicStats, map[string]*ChannelStats, error) {
	topicStats := make([]*TopicStats, 0)
	channelStats := make(map[string]*ChannelStats)
	success := false
	var lock sync.Mutex
	var wg sync.WaitGroup
	for _, addr := range nsqdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/stats?format=json", addr)
		log.Printf("NSQD: querying %s", endpoint)

		go func(endpoint string, addr string) {
			data, err := nsq.ApiRequest(endpoint)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true
			topics, _ := data.Get("topics").Array()
			for _, topicInfo := range topics {
				topicInfo := topicInfo.(map[string]interface{})
				topicName := topicInfo["topic_name"].(string)
				if selectedTopic != "" && topicName != selectedTopic {
					continue
				}
				depth := int64(topicInfo["depth"].(float64))
				backendDepth := int64(topicInfo["backend_depth"].(float64))
				h := &TopicStats{
					HostAddress:  addr,
					Depth:        depth,
					BackendDepth: backendDepth,
					MemoryDepth:  depth - backendDepth,
					MessageCount: int64(topicInfo["message_count"].(float64)),
					ChannelCount: len(topicInfo["channels"].([]interface{})),
					Topic:        topicName,
				}
				topicStats = append(topicStats, h)

				channels := topicInfo["channels"].([]interface{})
				for _, c := range channels {
					c := c.(map[string]interface{})
					channelName := c["channel_name"].(string)
					channelStatsKey := channelName
					if selectedTopic == "" {
						channelStatsKey = fmt.Sprintf("%s:%s", topicName, channelName)
					}
					channel, ok := channelStats[channelStatsKey]
					if !ok {
						channel = &ChannelStats{
							ChannelName: channelName,
							Topic:       topicName,
						}
						channelStats[channelStatsKey] = channel
					}
					h := &ChannelStats{HostAddress: addr, ChannelName: channelName, Topic: topicName}
					depth := int64(c["depth"].(float64))
					backendDepth := int64(c["backend_depth"].(float64))
					h.Depth = depth
					var paused bool
					pausedInterface, ok := c["paused"]
					if ok {
						paused = pausedInterface.(bool)
					}
					h.Paused = paused
					h.BackendDepth = backendDepth
					h.MemoryDepth = depth - backendDepth
					h.InFlightCount = int64(c["in_flight_count"].(float64))
					h.DeferredCount = int64(c["deferred_count"].(float64))
					h.MessageCount = int64(c["message_count"].(float64))
					h.RequeueCount = int64(c["requeue_count"].(float64))
					h.TimeoutCount = int64(c["timeout_count"].(float64))
					clients := c["clients"].([]interface{})
					// TODO: this is sort of wrong; clients should be de-duped
					// client A that connects to NSQD-a and NSQD-b should only be counted once. right?
					h.ClientCount = len(clients)
					channel.AddHostStats(h)

					// "clients": [
					//   {
					//     "version": "V2",
					//     "remote_address": "127.0.0.1:49700",
					//     "name": "jehiah-air",
					//     "state": 3,
					//     "ready_count": 1000,
					//     "in_flight_count": 0,
					//     "message_count": 0,
					//     "finish_count": 0,
					//     "requeue_count": 0,
					//     "connect_ts": 1347150965
					//   }
					// ]
					for _, client := range clients {
						client := client.(map[string]interface{})
						connected := time.Unix(int64(client["connect_ts"].(float64)), 0)
						connectedDuration := time.Now().Sub(connected).Seconds()
						clientInfo := &ClientInfo{
							HostAddress:       addr,
							ClientVersion:     client["version"].(string),
							ClientIdentifier:  fmt.Sprintf("%s:%s", client["name"].(string), strings.Split(client["remote_address"].(string), ":")[1]),
							ConnectedDuration: time.Duration(int64(connectedDuration)) * time.Second, // truncate to second
							InFlightCount:     int(client["in_flight_count"].(float64)),
							ReadyCount:        int(client["ready_count"].(float64)),
							FinishCount:       int64(client["finish_count"].(float64)),
							RequeueCount:      int64(client["requeue_count"].(float64)),
							MessageCount:      int64(client["message_count"].(float64)),
						}
						channel.Clients = append(channel.Clients, clientInfo)
					}
					sort.Sort(ClientsByHost{channel.Clients})
				}
			}
			sort.Sort(TopicStatsByHost{topicStats})
		}(endpoint, addr)
	}
	wg.Wait()
	if success == false {
		return nil, nil, errors.New("unable to query any nsqd")
	}
	return topicStats, channelStats, nil
}
Example #16
0
// GetLookupdProducers returns a slice of pointers to Producer structs
// containing metadata for each node connected to given lookupds
func GetLookupdProducers(lookupdHTTPAddrs []string) ([]*Producer, error) {
	success := false
	allProducers := make(map[string]*Producer, 0)
	output := make([]*Producer, 0)
	maxVersion, _ := semver.Parse("0.0.0")
	var lock sync.Mutex
	var wg sync.WaitGroup

	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/nodes", addr)
		log.Printf("LOOKUPD: querying %s", endpoint)
		go func(addr string, endpoint string) {
			data, err := nsq.ApiRequest(endpoint)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true

			producers := data.Get("producers")
			producersArray, _ := producers.Array()
			for i := range producersArray {
				producer := producers.GetIndex(i)
				remoteAddress := producer.Get("remote_address").MustString()
				if remoteAddress == "" {
					remoteAddress = "NA"
				}
				address := producer.Get("address").MustString() //TODO: remove for 1.0
				hostname := producer.Get("hostname").MustString()
				broadcastAddress := producer.Get("broadcast_address").MustString()
				if broadcastAddress == "" {
					broadcastAddress = address
				}
				httpPort := producer.Get("http_port").MustInt()
				tcpPort := producer.Get("tcp_port").MustInt()
				key := fmt.Sprintf("%s:%d:%d", broadcastAddress, httpPort, tcpPort)
				p, ok := allProducers[key]
				if !ok {
					topicList, _ := producer.Get("topics").Array()
					var topics []string
					for _, t := range topicList {
						topics = append(topics, t.(string))
					}
					sort.Strings(topics)
					version := producer.Get("version").MustString("unknown")
					versionObj, err := semver.Parse(version)
					if err != nil {
						versionObj = maxVersion
					}
					if maxVersion.Less(versionObj) {
						maxVersion = versionObj
					}
					p = &Producer{
						Address:          address, //TODO: remove for 1.0
						Hostname:         hostname,
						BroadcastAddress: broadcastAddress,
						TcpPort:          tcpPort,
						HttpPort:         httpPort,
						Version:          version,
						VersionObj:       versionObj,
						Topics:           topics,
					}
					allProducers[key] = p
					output = append(output, p)
				}
				p.RemoteAddresses = append(p.RemoteAddresses, fmt.Sprintf("%s/%s", addr, remoteAddress))
			}
		}(addr, endpoint)
	}
	wg.Wait()
	for _, producer := range allProducers {
		if producer.VersionObj.Less(maxVersion) {
			producer.OutOfDate = true
		}
	}
	sort.Sort(ProducersByHost{output})
	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}
	return output, nil
}
Example #17
0
File: http.go Project: jmanero/nsq
func createTopicChannelHandler(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		log.Printf("ERROR: invalid %s to POST only method", req.Method)
		http.Error(w, "INVALID_REQUEST", 500)
		return
	}
	reqParams := &util.PostParams{req}

	topicName, err := reqParams.Get("topic")
	if err != nil || !nsq.IsValidTopicName(topicName) {
		http.Error(w, "INVALID_TOPIC", 500)
		return
	}

	channelName, err := reqParams.Get("channel")
	if err != nil || (len(channelName) > 0 && !nsq.IsValidChannelName(channelName)) {
		http.Error(w, "INVALID_CHANNEL", 500)
		return
	}

	for _, addr := range lookupdHTTPAddrs {
		endpoint := fmt.Sprintf("http://%s/create_topic?topic=%s", addr, url.QueryEscape(topicName))
		log.Printf("LOOKUPD: querying %s", endpoint)
		_, err := nsq.ApiRequest(endpoint)
		if err != nil {
			log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
			continue
		}
	}

	NotifyAdminAction("create_topic", topicName, "", req)

	if len(channelName) > 0 {
		for _, addr := range lookupdHTTPAddrs {
			endpoint := fmt.Sprintf("http://%s/create_channel?topic=%s&channel=%s",
				addr, url.QueryEscape(topicName), url.QueryEscape(channelName))
			log.Printf("LOOKUPD: querying %s", endpoint)
			_, err := nsq.ApiRequest(endpoint)
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				continue
			}
		}

		// TODO: we can remove this when we push new channel information from nsqlookupd -> nsqd
		producers, _ := getLookupdTopicProducers(topicName, lookupdHTTPAddrs)
		for _, addr := range producers {
			endpoint := fmt.Sprintf("http://%s/create_channel?topic=%s&channel=%s",
				addr, url.QueryEscape(topicName), url.QueryEscape(channelName))
			log.Printf("NSQD: querying %s", endpoint)
			_, err := nsq.ApiRequest(endpoint)
			if err != nil {
				log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
				continue
			}
		}
		NotifyAdminAction("create_channel", topicName, channelName, req)
	}

	http.Redirect(w, req, "/lookup", 302)
}