Пример #1
0
func TestTombstoneRecover(t *testing.T) {
	opts := NewNSQLookupdOptions()
	opts.Logger = newTestLogger(t)
	opts.TombstoneLifetime = 50 * time.Millisecond
	tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
	defer nsqlookupd.Exit()

	topicName := "tombstone_recover"
	topicName2 := topicName + "2"

	conn := mustConnectLookupd(t, tcpAddr)
	defer conn.Close()

	identify(t, conn, "ip.address", 5000, 5555, "fake-version")

	nsq.Register(topicName, "channel1").WriteTo(conn)
	_, err := nsq.ReadResponse(conn)
	equal(t, err, nil)

	nsq.Register(topicName2, "channel2").WriteTo(conn)
	_, err = nsq.ReadResponse(conn)
	equal(t, err, nil)

	endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s",
		httpAddr, topicName, "ip.address:5555")
	_, err = http_api.NegotiateV1("POST", endpoint, nil)
	equal(t, err, nil)

	endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
	data, err := http_api.NegotiateV1("GET", endpoint, nil)
	equal(t, err, nil)
	producers, _ := data.Get("producers").Array()
	equal(t, len(producers), 0)

	endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName2)
	data, err = http_api.NegotiateV1("GET", endpoint, nil)
	equal(t, err, nil)
	producers, _ = data.Get("producers").Array()
	equal(t, len(producers), 1)

	time.Sleep(75 * time.Millisecond)

	endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
	data, err = http_api.NegotiateV1("GET", endpoint, nil)
	equal(t, err, nil)
	producers, _ = data.Get("producers").Array()
	equal(t, len(producers), 1)
}
Пример #2
0
func API(endpoint string) (data *simplejson.Json, err error) {
	d := make(map[string]interface{})
	err = http_api.NegotiateV1(endpoint, &d)
	data = simplejson.New()
	data.SetPath(nil, d)
	return
}
Пример #3
0
// GetLookupdTopicChannels returns a []string containing a union of the channels
// from all the given lookupd for the given topic
func GetLookupdTopicChannels(topic string, lookupdHTTPAddrs []string) ([]string, error) {
	success := false
	var allChannels []string
	var lock sync.Mutex
	var wg sync.WaitGroup
	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/channels?topic=%s", addr, url.QueryEscape(topic))
		log.Printf("LOOKUPD: querying %s", endpoint)
		go func(endpoint string) {
			data, err := http_api.NegotiateV1("GET", endpoint, nil)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true
			// {"data":{"channels":["test"]}}
			channels, _ := data.Get("channels").StringArray()
			allChannels = stringy.Union(allChannels, channels)
		}(endpoint)
	}
	wg.Wait()
	sort.Strings(allChannels)
	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}
	return allChannels, nil
}
Пример #4
0
// GetNSQDTopics returns a []string containing all the topics
// produced by the given nsqd
func GetNSQDTopics(nsqdHTTPAddrs []string) ([]string, error) {
	var topics []string
	var lock sync.Mutex
	var wg sync.WaitGroup
	success := false
	for _, addr := range nsqdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/stats?format=json", addr)
		log.Printf("NSQD: querying %s", endpoint)

		go func(endpoint string) {
			data, err := http_api.NegotiateV1("GET", endpoint, nil)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true
			topicList, _ := data.Get("topics").Array()
			for i := range topicList {
				topicInfo := data.Get("topics").GetIndex(i)
				topics = stringy.Add(topics, topicInfo.Get("topic_name").MustString())
			}
		}(endpoint)
	}
	wg.Wait()
	sort.Strings(topics)
	if success == false {
		return nil, errors.New("unable to query any nsqd")
	}
	return topics, nil
}
Пример #5
0
// GetLookupdProducers returns a ProducerList metadata for each node connected to the given lookupds
func (c *ClusterInfo) GetLookupdProducers(lookupdHTTPAddrs []string) (ProducerList, error) {
	var success bool
	var output []*Producer
	var lock sync.Mutex
	var wg sync.WaitGroup

	allProducers := make(map[string]*Producer)
	maxVersion, _ := semver.Parse("0.0.0")

	type respType struct {
		Producers []*Producer `json:"producers"`
	}

	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/nodes", addr)
		c.logf("LOOKUPD: querying %s", endpoint)
		go func(addr string, endpoint string) {
			defer wg.Done()

			var resp respType
			err := http_api.NegotiateV1(endpoint, &resp)
			if err != nil {
				c.logf("ERROR: lookupd %s - %s", endpoint, err)
				return
			}

			lock.Lock()
			defer lock.Unlock()
			success = true
			for _, producer := range resp.Producers {
				key := producer.TCPAddress()
				p, ok := allProducers[key]
				if !ok {
					if maxVersion.LT(producer.VersionObj) {
						maxVersion = producer.VersionObj
					}
					sort.Sort(producer.Topics)
					p = producer
					allProducers[key] = p
					output = append(output, p)
				}
				p.RemoteAddresses = append(p.RemoteAddresses, fmt.Sprintf("%s/%s", addr, producer.Address()))
			}
		}(addr, endpoint)
	}
	wg.Wait()

	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}

	for _, producer := range allProducers {
		if producer.VersionObj.LT(maxVersion) {
			producer.OutOfDate = true
		}
	}
	sort.Sort(ProducersByHost{output})
	return output, nil
}
Пример #6
0
func (s *httpServer) deleteChannelHandler(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		s.ctx.nsqadmin.logf("ERROR: invalid %s to POST only method", req.Method)
		http.Error(w, "INVALID_REQUEST", 500)
		return
	}
	reqParams := &http_api.PostParams{req}

	topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	rd, _ := reqParams.Get("rd")
	if !strings.HasPrefix(rd, "/") {
		rd = fmt.Sprintf("/topic/%s", url.QueryEscape(topicName))
	}

	for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses {
		nsqlookupdVersion, err := lookupd.GetVersion(addr)
		if err != nil {
			s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err)
		}

		uri := "delete_channel"
		if !nsqlookupdVersion.LT(v1EndpointVersion) {
			uri = "channel/delete"
		}

		endpoint := fmt.Sprintf("http://%s/%s?topic=%s&channel=%s",
			addr, uri,
			url.QueryEscape(topicName),
			url.QueryEscape(channelName))
		s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint)
		_, err = http_api.NegotiateV1("POST", endpoint, nil)
		if err != nil {
			s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err)
			continue
		}
	}

	producerAddrs := s.getProducers(topicName)
	s.performVersionNegotiatedRequestsToNSQD(
		s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses,
		producerAddrs,
		"delete_channel",
		"channel/delete",
		fmt.Sprintf("topic=%s&channel=%s",
			url.QueryEscape(topicName), url.QueryEscape(channelName)))

	s.notifyAdminAction("delete_channel", topicName, channelName, "", req)

	http.Redirect(w, req, rd, 302)
}
Пример #7
0
func (s *httpServer) deleteTopicHandler(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		s.ctx.nsqadmin.logf("ERROR: invalid %s to POST only method", req.Method)
		http.Error(w, "INVALID_REQUEST", 500)
		return
	}
	reqParams := &http_api.PostParams{req}

	topicName, err := reqParams.Get("topic")
	if err != nil {
		http.Error(w, "MISSING_ARG_TOPIC", 500)
		return
	}

	rd, _ := reqParams.Get("rd")
	if !strings.HasPrefix(rd, "/") {
		rd = "/"
	}

	// for topic removal, you need to get all the producers *first*
	producerAddrs := s.getProducers(topicName)

	// remove the topic from all the lookupds
	for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses {
		nsqlookupdVersion, err := lookupd.GetVersion(addr)
		if err != nil {
			s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err)
		}

		uri := "delete_topic"
		if !nsqlookupdVersion.LT(v1EndpointVersion) {
			uri = "topic/delete"
		}

		endpoint := fmt.Sprintf("http://%s/%s?topic=%s", addr, uri, url.QueryEscape(topicName))
		s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint)
		_, err = http_api.NegotiateV1("POST", endpoint, nil)
		if err != nil {
			s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err)
			continue
		}
	}

	s.performVersionNegotiatedRequestsToNSQD(
		s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses,
		producerAddrs,
		"delete_topic",
		"topic/delete",
		fmt.Sprintf("topic=%s", url.QueryEscape(topicName)))

	s.notifyAdminAction("delete_topic", topicName, "", "", req)

	http.Redirect(w, req, rd, 302)
}
Пример #8
0
// GetVersion returns a semver.Version object by querying /info
func GetVersion(addr string) (semver.Version, error) {
	endpoint := fmt.Sprintf("http://%s/info", addr)
	log.Printf("version negotiation %s", endpoint)
	info, err := http_api.NegotiateV1("GET", endpoint, nil)
	if err != nil {
		log.Printf("ERROR: %s - %s", endpoint, err)
		return semver.Version{}, err
	}
	version := info.Get("version").MustString("unknown")
	return semver.Parse(version)
}
Пример #9
0
func TestChannelUnregister(t *testing.T) {
	opts := NewNSQLookupdOptions()
	opts.Logger = newTestLogger(t)
	tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
	defer nsqlookupd.Exit()

	topics := nsqlookupd.DB.FindRegistrations("topic", "*", "*")
	equal(t, len(topics), 0)

	topicName := "channel_unregister"

	conn := mustConnectLookupd(t, tcpAddr)
	defer conn.Close()

	tcpPort := 5000
	httpPort := 5555
	identify(t, conn, "ip.address", tcpPort, httpPort, "fake-version")

	nsq.Register(topicName, "ch1").WriteTo(conn)
	v, err := nsq.ReadResponse(conn)
	equal(t, err, nil)
	equal(t, v, []byte("OK"))

	topics = nsqlookupd.DB.FindRegistrations("topic", topicName, "")
	equal(t, len(topics), 1)

	channels := nsqlookupd.DB.FindRegistrations("channel", topicName, "*")
	equal(t, len(channels), 1)

	nsq.UnRegister(topicName, "ch1").WriteTo(conn)
	v, err = nsq.ReadResponse(conn)
	equal(t, err, nil)
	equal(t, v, []byte("OK"))

	topics = nsqlookupd.DB.FindRegistrations("topic", topicName, "")
	equal(t, len(topics), 1)

	// we should still have mention of the topic even though there is no producer
	// (ie. we haven't *deleted* the channel, just unregistered as a producer)
	channels = nsqlookupd.DB.FindRegistrations("channel", topicName, "*")
	equal(t, len(channels), 1)

	endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
	data, err := http_api.NegotiateV1("GET", endpoint, nil)
	equal(t, err, nil)
	returnedProducers, err := data.Get("producers").Array()
	equal(t, err, nil)
	equal(t, len(returnedProducers), 1)
}
Пример #10
0
// GetVersion returns a semver.Version object by querying /info
func (c *ClusterInfo) GetVersion(addr string) (semver.Version, error) {
	endpoint := fmt.Sprintf("http://%s/info", addr)
	c.logf("version negotiation %s", endpoint)
	var resp struct {
		Version string `json:'version'`
	}
	err := http_api.NegotiateV1(endpoint, &resp)
	if err != nil {
		c.logf("ERROR: %s - %s", endpoint, err)
		return semver.Version{}, err
	}
	if resp.Version == "" {
		resp.Version = "unknown"
	}
	return semver.Parse(resp.Version)
}
Пример #11
0
Файл: data.go Проект: rpau/nsq
// GetNSQDTopicProducers returns a []string containing the addresses of all the nsqd
// that produce the given topic out of the given nsqd
func (c *ClusterInfo) GetNSQDTopicProducers(topic string, nsqdHTTPAddrs []string) ([]string, error) {
	var addresses []string
	var lock sync.Mutex
	var wg sync.WaitGroup
	success := false

	type respType struct {
		Topics []struct {
			Name string `json:"topic_name"`
		} `json:"topics"`
	}

	for _, addr := range nsqdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/stats?format=json", addr)
		c.logf("NSQD: querying %s", endpoint)

		go func(endpoint, addr string) {
			defer wg.Done()

			var resp respType
			err := http_api.NegotiateV1(endpoint, &resp)
			lock.Lock()
			defer lock.Unlock()
			if err != nil {
				c.logf("ERROR: nsqd %s - %s", endpoint, err.Error())
				return
			}
			success = true

			for _, t := range resp.Topics {
				if t.Name == topic {
					addresses = append(addresses, addr)
					return
				}
			}
		}(endpoint, addr)
	}
	wg.Wait()
	if success == false {
		return nil, errors.New("unable to query any nsqd")
	}
	return addresses, nil
}
Пример #12
0
func TestClientAttributes(t *testing.T) {
	userAgent := "Test User Agent"

	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.Verbose = true
	opts.SnappyEnabled = true
	tcpAddr, httpAddr, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	conn, err := mustConnectNSQD(tcpAddr)
	equal(t, err, nil)
	defer conn.Close()

	data := identify(t, conn, map[string]interface{}{
		"snappy":     true,
		"user_agent": userAgent,
	}, frameTypeResponse)
	resp := struct {
		Snappy    bool   `json:"snappy"`
		UserAgent string `json:"user_agent"`
	}{}
	err = json.Unmarshal(data, &resp)
	equal(t, err, nil)
	equal(t, resp.Snappy, true)

	r := snappystream.NewReader(conn, snappystream.SkipVerifyChecksum)
	w := snappystream.NewWriter(conn)
	readValidate(t, r, frameTypeResponse, "OK")

	topicName := "test_client_attributes" + strconv.Itoa(int(time.Now().Unix()))
	sub(t, readWriter{r, w}, topicName, "ch")

	testURL := fmt.Sprintf("http://127.0.0.1:%d/stats?format=json", httpAddr.Port)

	statsData, err := http_api.NegotiateV1("GET", testURL, nil)
	equal(t, err, nil)

	client := statsData.Get("topics").GetIndex(0).Get("channels").GetIndex(0).Get("clients").GetIndex(0)
	equal(t, client.Get("user_agent").MustString(), userAgent)
	equal(t, client.Get("snappy").MustBool(), true)
}
Пример #13
0
// GetNSQDTopics returns a []string containing all the topics produced by the given nsqd
func (c *ClusterInfo) GetNSQDTopics(nsqdHTTPAddrs []string) ([]string, error) {
	var success bool
	var topics []string
	var lock sync.Mutex
	var wg sync.WaitGroup

	type respType struct {
		Topics []struct {
			Name string `json:"topic_name"`
		} `json:"topics"`
	}

	for _, addr := range nsqdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/stats?format=json", addr)
		c.logf("NSQD: querying %s", endpoint)
		go func(endpoint string) {
			defer wg.Done()

			var resp respType
			err := http_api.NegotiateV1(endpoint, &resp)
			if err != nil {
				c.logf("ERROR: lookupd %s - %s", endpoint, err)
				return
			}

			lock.Lock()
			defer lock.Unlock()
			success = true
			for _, topic := range resp.Topics {
				topics = stringy.Add(topics, topic.Name)
			}
		}(endpoint)
	}
	wg.Wait()

	if success == false {
		return nil, errors.New("unable to query any nsqd")
	}

	sort.Strings(topics)
	return topics, nil
}
Пример #14
0
// GetLookupdTopics returns a []string containing a union of all the topics
// from all the given lookupd
func (c *ClusterInfo) GetLookupdTopics(lookupdHTTPAddrs []string) ([]string, error) {
	var success bool
	var allTopics []string
	var lock sync.Mutex
	var wg sync.WaitGroup

	type respType struct {
		Topics []string `json:"topics"`
	}

	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/topics", addr)
		c.logf("LOOKUPD: querying %s", endpoint)
		go func(endpoint string) {
			defer wg.Done()

			var resp respType
			err := http_api.NegotiateV1(endpoint, &resp)
			if err != nil {
				c.logf("ERROR: lookupd %s - %s", endpoint, err)
				return
			}

			lock.Lock()
			defer lock.Unlock()
			success = true
			allTopics = append(allTopics, resp.Topics...)
		}(endpoint)
	}
	wg.Wait()

	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}

	allTopics = stringy.Uniq(allTopics)
	sort.Strings(allTopics)
	return allTopics, nil
}
Пример #15
0
Файл: data.go Проект: rpau/nsq
// GetLookupdTopicProducers returns a []string of the broadcast_address:http_port of all the
// producers for a given topic by unioning the results returned from the given lookupd
func (c *ClusterInfo) GetLookupdTopicProducers(topic string, lookupdHTTPAddrs []string) ([]string, error) {
	success := false
	var allSources []string
	var lock sync.Mutex
	var wg sync.WaitGroup

	type respType struct {
		Producers []Producer `json:"producers"`
	}

	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)

		endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", addr, url.QueryEscape(topic))
		c.logf("LOOKUPD: querying %s", endpoint)

		go func(endpoint string) {
			defer wg.Done()

			var resp respType
			err := http_api.NegotiateV1(endpoint, &resp)
			lock.Lock()
			defer lock.Unlock()
			if err != nil {
				c.logf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true

			for _, producer := range resp.Producers {
				allSources = stringy.Add(allSources, producer.HTTPAddress())
			}
		}(endpoint)
	}
	wg.Wait()
	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}
	return allSources, nil
}
Пример #16
0
// GetLookupdTopicProducers returns a []string of the broadcast_address:http_port of all the
// producers for a given topic by unioning the results returned from the given lookupd
func GetLookupdTopicProducers(topic string, lookupdHTTPAddrs []string) ([]string, error) {
	success := false
	var allSources []string
	var lock sync.Mutex
	var wg sync.WaitGroup

	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)

		endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", addr, url.QueryEscape(topic))
		log.Printf("LOOKUPD: querying %s", endpoint)

		go func(endpoint string) {
			data, err := http_api.NegotiateV1("GET", endpoint, nil)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true
			producers := data.Get("producers")
			producersArray, _ := producers.Array()
			for i := range producersArray {
				producer := producers.GetIndex(i)
				broadcastAddress := producer.Get("broadcast_address").MustString()
				httpPort := producer.Get("http_port").MustInt()
				key := net.JoinHostPort(broadcastAddress, strconv.Itoa(httpPort))
				allSources = stringy.Add(allSources, key)
			}
		}(endpoint)
	}
	wg.Wait()
	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}
	return allSources, nil
}
Пример #17
0
func (s *httpServer) performVersionNegotiatedRequestsToNSQD(
	nsqlookupdAddrs []string, nsqdAddrs []string,
	deprecatedURI string, v1URI string, queryString string) {
	var err error
	// get producer structs in one set of up-front requests
	// so we can negotiate versions
	//
	// (this returns an empty list if there are no nsqlookupd configured)
	producers, _ := lookupd.GetLookupdProducers(nsqlookupdAddrs)
	for _, addr := range nsqdAddrs {
		var nodeVer semver.Version

		uri := deprecatedURI
		producer := producerSearch(producers, addr)
		if producer != nil {
			nodeVer = producer.VersionObj
		} else {
			// we couldn't find the node in our list
			// so ask it for a version directly
			nodeVer, err = lookupd.GetVersion(addr)
			if err != nil {
				s.ctx.nsqadmin.logf("ERROR: failed to get nsqd %s version - %s", addr, err)
			}
		}

		if nodeVer.NE(semver.Version{}) && nodeVer.GTE(v1EndpointVersion) {
			uri = v1URI
		}

		endpoint := fmt.Sprintf("http://%s/%s?%s", addr, uri, queryString)
		s.ctx.nsqadmin.logf("NSQD: querying %s", endpoint)
		_, err := http_api.NegotiateV1("POST", endpoint, nil)
		if err != nil {
			s.ctx.nsqadmin.logf("ERROR: nsqd %s - %s", endpoint, err)
			continue
		}
	}
}
Пример #18
0
Файл: data.go Проект: rpau/nsq
// GetLookupdTopicChannels returns a []string containing a union of the channels
// from all the given lookupd for the given topic
func (c *ClusterInfo) GetLookupdTopicChannels(topic string, lookupdHTTPAddrs []string) ([]string, error) {
	success := false
	var allChannels []string
	var lock sync.Mutex
	var wg sync.WaitGroup

	type respType struct {
		Channels []string `json:"channels"`
	}

	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/channels?topic=%s", addr, url.QueryEscape(topic))
		c.logf("LOOKUPD: querying %s", endpoint)
		go func(endpoint string) {
			defer wg.Done()

			var resp respType
			err := http_api.NegotiateV1(endpoint, &resp)
			lock.Lock()
			defer lock.Unlock()
			if err != nil {
				c.logf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true
			allChannels = append(allChannels, resp.Channels...)
		}(endpoint)
	}
	wg.Wait()
	allChannels = stringy.Uniq(allChannels)
	sort.Strings(allChannels)
	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}
	return allChannels, nil
}
Пример #19
0
// GetLookupdTopicProducers returns a ProducerList of all the nsqd for a given topic by
// unioning the nodes returned from the given lookupd
func (c *ClusterInfo) GetLookupdTopicProducers(topic string, lookupdHTTPAddrs []string) (ProducerList, error) {
	var success bool
	var producerList ProducerList
	var lock sync.Mutex
	var wg sync.WaitGroup

	type respType struct {
		Producers ProducerList `json:"producers"`
	}

	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", addr, url.QueryEscape(topic))
		c.logf("LOOKUPD: querying %s", endpoint)
		go func(endpoint string) {
			defer wg.Done()

			var resp respType
			err := http_api.NegotiateV1(endpoint, &resp)
			if err != nil {
				c.logf("ERROR: lookupd %s - %s", endpoint, err)
				return
			}

			lock.Lock()
			defer lock.Unlock()
			success = true
			producerList = append(producerList, resp.Producers...)
		}(endpoint)
	}
	wg.Wait()

	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}
	return producerList, nil
}
Пример #20
0
func TestTombstonedNodes(t *testing.T) {
	opts := NewNSQLookupdOptions()
	opts.Logger = newTestLogger(t)
	tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
	defer nsqlookupd.Exit()

	lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)}

	topicName := "inactive_nodes"

	conn := mustConnectLookupd(t, tcpAddr)
	defer conn.Close()

	identify(t, conn, "ip.address", 5000, 5555, "fake-version")

	nsq.Register(topicName, "channel1").WriteTo(conn)
	_, err := nsq.ReadResponse(conn)
	equal(t, err, nil)

	producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs)
	equal(t, len(producers), 1)
	equal(t, len(producers[0].Topics), 1)
	equal(t, producers[0].Topics[0].Topic, topicName)
	equal(t, producers[0].Topics[0].Tombstoned, false)

	endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s",
		httpAddr, topicName, "ip.address:5555")
	_, err = http_api.NegotiateV1("POST", endpoint, nil)
	equal(t, err, nil)

	producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs)
	equal(t, len(producers), 1)
	equal(t, len(producers[0].Topics), 1)
	equal(t, producers[0].Topics[0].Topic, topicName)
	equal(t, producers[0].Topics[0].Tombstoned, true)
}
Пример #21
0
// GetLookupdProducers returns a slice of pointers to Producer structs
// containing metadata for each node connected to given lookupds
func GetLookupdProducers(lookupdHTTPAddrs []string) ([]*Producer, error) {
	success := false
	allProducers := make(map[string]*Producer)
	var output []*Producer
	maxVersion, _ := semver.Parse("0.0.0")
	var lock sync.Mutex
	var wg sync.WaitGroup

	for _, addr := range lookupdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/nodes", addr)
		log.Printf("LOOKUPD: querying %s", endpoint)
		go func(addr string, endpoint string) {
			data, err := http_api.NegotiateV1("GET", endpoint, nil)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()
			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true

			producers := data.Get("producers")
			producersArray, _ := producers.Array()
			for i := range producersArray {
				producer := producers.GetIndex(i)
				remoteAddress := producer.Get("remote_address").MustString()
				if remoteAddress == "" {
					remoteAddress = "NA"
				}
				hostname := producer.Get("hostname").MustString()
				broadcastAddress := producer.Get("broadcast_address").MustString()
				httpPort := producer.Get("http_port").MustInt()
				tcpPort := producer.Get("tcp_port").MustInt()
				key := fmt.Sprintf("%s:%d:%d", broadcastAddress, httpPort, tcpPort)
				p, ok := allProducers[key]
				if !ok {
					var tombstones []bool
					var topics ProducerTopics

					topicList, _ := producer.Get("topics").Array()
					tombstoneList, err := producer.Get("tombstones").Array()
					if err != nil {
						// backwards compatibility with nsqlookupd < v0.2.22
						tombstones = make([]bool, len(topicList))
					} else {
						for _, t := range tombstoneList {
							tombstones = append(tombstones, t.(bool))
						}
					}

					for i, t := range topicList {
						topics = append(topics, ProducerTopic{
							Topic:      t.(string),
							Tombstoned: tombstones[i],
						})
					}

					sort.Sort(topics)

					version := producer.Get("version").MustString("unknown")
					versionObj, err := semver.Parse(version)
					if err != nil {
						versionObj = maxVersion
					}
					if maxVersion.LT(versionObj) {
						maxVersion = versionObj
					}

					p = &Producer{
						Hostname:         hostname,
						BroadcastAddress: broadcastAddress,
						TCPPort:          tcpPort,
						HTTPPort:         httpPort,
						Version:          version,
						VersionObj:       versionObj,
						Topics:           topics,
					}
					allProducers[key] = p
					output = append(output, p)
				}
				p.RemoteAddresses = append(p.RemoteAddresses, fmt.Sprintf("%s/%s", addr, remoteAddress))
			}
		}(addr, endpoint)
	}
	wg.Wait()
	for _, producer := range allProducers {
		if producer.VersionObj.LT(maxVersion) {
			producer.OutOfDate = true
		}
	}
	sort.Sort(ProducersByHost{output})
	if success == false {
		return nil, errors.New("unable to query any lookupd")
	}
	return output, nil
}
Пример #22
0
// GetNSQDStats returns aggregate topic and channel stats from the given NSQD instances
//
// if selectedTopic is empty, this will return stats for *all* topic/channels
// and the ChannelStats dict will be keyed by topic + ':' + channel
func GetNSQDStats(nsqdHTTPAddrs []string, selectedTopic string) ([]*TopicStats, map[string]*ChannelStats, error) {
	var lock sync.Mutex
	var wg sync.WaitGroup

	var topicStatsList TopicStatsList
	channelStatsMap := make(map[string]*ChannelStats)

	success := false
	for _, addr := range nsqdHTTPAddrs {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/stats?format=json", addr)
		log.Printf("NSQD: querying %s", endpoint)

		go func(endpoint string, addr string) {
			data, err := http_api.NegotiateV1("GET", endpoint, nil)
			lock.Lock()
			defer lock.Unlock()
			defer wg.Done()

			if err != nil {
				log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
				return
			}
			success = true

			topics, _ := data.Get("topics").Array()
			for i := range topics {
				t := data.Get("topics").GetIndex(i)

				topicName := t.Get("topic_name").MustString()
				if selectedTopic != "" && topicName != selectedTopic {
					continue
				}
				depth := t.Get("depth").MustInt64()
				backendDepth := t.Get("backend_depth").MustInt64()
				channels := t.Get("channels").MustArray()

				e2eProcessingLatency := quantile.E2eProcessingLatencyAggregateFromJSON(t.Get("e2e_processing_latency"), topicName, "", addr)

				topicStats := &TopicStats{
					HostAddress:  addr,
					TopicName:    topicName,
					Depth:        depth,
					BackendDepth: backendDepth,
					MemoryDepth:  depth - backendDepth,
					MessageCount: t.Get("message_count").MustInt64(),
					ChannelCount: len(channels),
					Paused:       t.Get("paused").MustBool(),

					E2eProcessingLatency: e2eProcessingLatency,
				}
				topicStatsList = append(topicStatsList, topicStats)

				for j := range channels {
					c := t.Get("channels").GetIndex(j)

					channelName := c.Get("channel_name").MustString()
					key := channelName
					if selectedTopic == "" {
						key = fmt.Sprintf("%s:%s", topicName, channelName)
					}

					channelStats, ok := channelStatsMap[key]
					if !ok {
						channelStats = &ChannelStats{
							HostAddress: addr,
							TopicName:   topicName,
							ChannelName: channelName,
						}
						channelStatsMap[key] = channelStats
					}

					depth := c.Get("depth").MustInt64()
					backendDepth := c.Get("backend_depth").MustInt64()
					clients := c.Get("clients").MustArray()

					e2eProcessingLatency := quantile.E2eProcessingLatencyAggregateFromJSON(c.Get("e2e_processing_latency"), topicName, channelName, addr)

					hostChannelStats := &ChannelStats{
						HostAddress:   addr,
						TopicName:     topicName,
						ChannelName:   channelName,
						Depth:         depth,
						BackendDepth:  backendDepth,
						MemoryDepth:   depth - backendDepth,
						Paused:        c.Get("paused").MustBool(),
						InFlightCount: c.Get("in_flight_count").MustInt64(),
						DeferredCount: c.Get("deferred_count").MustInt64(),
						MessageCount:  c.Get("message_count").MustInt64(),
						RequeueCount:  c.Get("requeue_count").MustInt64(),
						TimeoutCount:  c.Get("timeout_count").MustInt64(),

						E2eProcessingLatency: e2eProcessingLatency,
						// TODO: this is sort of wrong; clients should be de-duped
						// client A that connects to NSQD-a and NSQD-b should only be counted once. right?
						ClientCount: len(clients),
					}
					channelStats.Add(hostChannelStats)

					for k := range clients {
						client := c.Get("clients").GetIndex(k)

						connected := time.Unix(client.Get("connect_ts").MustInt64(), 0)
						connectedDuration := time.Now().Sub(connected).Seconds()

						clientID := client.Get("clientID").MustString()
						if clientID == "" {
							// TODO: deprecated, remove in 1.0
							name := client.Get("name").MustString()
							remoteAddressParts := strings.Split(client.Get("remote_address").MustString(), ":")
							port := remoteAddressParts[len(remoteAddressParts)-1]
							if len(remoteAddressParts) < 2 {
								port = "NA"
							}
							clientID = fmt.Sprintf("%s:%s", name, port)
						}

						clientStats := &ClientStats{
							HostAddress:       addr,
							RemoteAddress:     client.Get("remote_address").MustString(),
							Version:           client.Get("version").MustString(),
							ClientID:          clientID,
							Hostname:          client.Get("hostname").MustString(),
							UserAgent:         client.Get("user_agent").MustString(),
							ConnectedDuration: time.Duration(int64(connectedDuration)) * time.Second, // truncate to second
							InFlightCount:     client.Get("in_flight_count").MustInt(),
							ReadyCount:        client.Get("ready_count").MustInt(),
							FinishCount:       client.Get("finish_count").MustInt64(),
							RequeueCount:      client.Get("requeue_count").MustInt64(),
							MessageCount:      client.Get("message_count").MustInt64(),
							SampleRate:        int32(client.Get("sample_rate").MustInt()),
							Deflate:           client.Get("deflate").MustBool(),
							Snappy:            client.Get("snappy").MustBool(),
							Authed:            client.Get("authed").MustBool(),
							AuthIdentity:      client.Get("auth_identity").MustString(),
							AuthIdentityURL:   client.Get("auth_identity_url").MustString(),

							TLS:                           client.Get("tls").MustBool(),
							CipherSuite:                   client.Get("tls_cipher_suite").MustString(),
							TLSVersion:                    client.Get("tls_version").MustString(),
							TLSNegotiatedProtocol:         client.Get("tls_negotiated_protocol").MustString(),
							TLSNegotiatedProtocolIsMutual: client.Get("tls_negotiated_protocol_is_mutual").MustBool(),
						}
						hostChannelStats.Clients = append(hostChannelStats.Clients, clientStats)
						channelStats.Clients = append(channelStats.Clients, clientStats)
					}
					sort.Sort(ClientsByHost{hostChannelStats.Clients})
					sort.Sort(ClientsByHost{channelStats.Clients})

					topicStats.Channels = append(topicStats.Channels, hostChannelStats)
				}
			}
			sort.Sort(TopicStatsByHost{topicStatsList})
		}(endpoint, addr)
	}
	wg.Wait()
	if success == false {
		return nil, nil, errors.New("unable to query any nsqd")
	}
	return topicStatsList, channelStatsMap, nil
}
Пример #23
0
func (s *httpServer) tombstoneTopicProducerHandler(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		s.ctx.nsqadmin.logf("ERROR: invalid %s to POST only method", req.Method)
		http.Error(w, "INVALID_REQUEST", 500)
		return
	}
	reqParams := &http_api.PostParams{req}

	topicName, err := reqParams.Get("topic")
	if err != nil {
		http.Error(w, "MISSING_ARG_TOPIC", 500)
		return
	}

	node, err := reqParams.Get("node")
	if err != nil {
		http.Error(w, "MISSING_ARG_NODE", 500)
		return
	}

	rd, _ := reqParams.Get("rd")
	if !strings.HasPrefix(rd, "/") {
		rd = "/"
	}

	// tombstone the topic on all the lookupds
	for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses {
		nsqlookupdVersion, err := lookupd.GetVersion(addr)
		if err != nil {
			s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err)
		}

		uri := "tombstone_topic_producer"
		if !nsqlookupdVersion.LT(v1EndpointVersion) {
			uri = "topic/tombstone"
		}

		endpoint := fmt.Sprintf("http://%s/%s?topic=%s&node=%s",
			addr, uri,
			url.QueryEscape(topicName), url.QueryEscape(node))
		s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint)
		_, err = http_api.NegotiateV1("POST", endpoint, nil)
		if err != nil {
			s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err)
		}
	}

	nsqdVersion, err := lookupd.GetVersion(node)
	if err != nil {
		s.ctx.nsqadmin.logf("ERROR: failed to get nsqd %s version - %s", node, err)
	}

	uri := "delete_topic"
	if !nsqdVersion.LT(v1EndpointVersion) {
		uri = "topic/delete"
	}

	// delete the topic on the producer
	endpoint := fmt.Sprintf("http://%s/%s?topic=%s", node,
		uri, url.QueryEscape(topicName))
	s.ctx.nsqadmin.logf("NSQD: querying %s", endpoint)
	_, err = http_api.NegotiateV1("POST", endpoint, nil)
	if err != nil {
		s.ctx.nsqadmin.logf("ERROR: nsqd %s - %s", endpoint, err)
	}

	s.notifyAdminAction("tombstone_topic_producer", topicName, "", node, req)

	http.Redirect(w, req, rd, 302)
}
Пример #24
0
func (s *httpServer) createTopicChannelHandler(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		s.ctx.nsqadmin.logf("ERROR: invalid %s to POST only method", req.Method)
		http.Error(w, "INVALID_REQUEST", 500)
		return
	}
	reqParams := &http_api.PostParams{req}

	topicName, err := reqParams.Get("topic")
	if err != nil || !protocol.IsValidTopicName(topicName) {
		http.Error(w, "INVALID_TOPIC", 500)
		return
	}

	channelName, err := reqParams.Get("channel")
	if err != nil || (len(channelName) > 0 && !protocol.IsValidChannelName(channelName)) {
		http.Error(w, "INVALID_CHANNEL", 500)
		return
	}

	for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses {
		nsqlookupdVersion, err := lookupd.GetVersion(addr)
		if err != nil {
			s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err)
		}

		uri := "create_topic"
		if !nsqlookupdVersion.LT(v1EndpointVersion) {
			uri = "topic/create"
		}

		endpoint := fmt.Sprintf("http://%s/%s?topic=%s", addr,
			uri, url.QueryEscape(topicName))
		s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint)
		_, err = http_api.NegotiateV1("POST", endpoint, nil)
		if err != nil {
			s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err)
			continue
		}

		if len(channelName) > 0 {
			uri := "create_channel"
			if !nsqlookupdVersion.LT(v1EndpointVersion) {
				uri = "channel/create"
			}
			endpoint := fmt.Sprintf("http://%s/%s?topic=%s&channel=%s",
				addr, uri,
				url.QueryEscape(topicName),
				url.QueryEscape(channelName))
			s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint)
			_, err := http_api.NegotiateV1("POST", endpoint, nil)
			if err != nil {
				s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err)
				continue
			}
		}
	}

	s.notifyAdminAction("create_topic", topicName, "", "", req)

	if len(channelName) > 0 {
		// TODO: we can remove this when we push new channel information from nsqlookupd -> nsqd
		producerAddrs, _ := lookupd.GetLookupdTopicProducers(topicName,
			s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)

		s.performVersionNegotiatedRequestsToNSQD(
			s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses,
			producerAddrs,
			"create_channel",
			"channel/create",
			fmt.Sprintf("topic=%s&channel=%s",
				url.QueryEscape(topicName), url.QueryEscape(channelName)))

		s.notifyAdminAction("create_channel", topicName, channelName, "", req)
	}

	http.Redirect(w, req, "/lookup", 302)
}
Пример #25
0
func TestBasicLookupd(t *testing.T) {
	opts := NewNSQLookupdOptions()
	opts.Logger = newTestLogger(t)
	tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts)
	defer nsqlookupd.Exit()

	topics := nsqlookupd.DB.FindRegistrations("topic", "*", "*")
	equal(t, len(topics), 0)

	topicName := "connectmsg"

	conn := mustConnectLookupd(t, tcpAddr)

	tcpPort := 5000
	httpPort := 5555
	identify(t, conn, "ip.address", tcpPort, httpPort, "fake-version")

	nsq.Register(topicName, "channel1").WriteTo(conn)
	v, err := nsq.ReadResponse(conn)
	equal(t, err, nil)
	equal(t, v, []byte("OK"))

	endpoint := fmt.Sprintf("http://%s/nodes", httpAddr)
	data, err := http_api.NegotiateV1("GET", endpoint, nil)
	t.Logf("got %v", data)
	returnedProducers, err := data.Get("producers").Array()
	equal(t, err, nil)
	equal(t, len(returnedProducers), 1)

	topics = nsqlookupd.DB.FindRegistrations("topic", topicName, "")
	equal(t, len(topics), 1)

	producers := nsqlookupd.DB.FindProducers("topic", topicName, "")
	equal(t, len(producers), 1)
	producer := producers[0]

	equal(t, producer.peerInfo.BroadcastAddress, "ip.address")
	equal(t, producer.peerInfo.Hostname, "ip.address")
	equal(t, producer.peerInfo.TCPPort, tcpPort)
	equal(t, producer.peerInfo.HTTPPort, httpPort)

	endpoint = fmt.Sprintf("http://%s/topics", httpAddr)
	data, err = http_api.NegotiateV1("GET", endpoint, nil)
	equal(t, err, nil)
	returnedTopics, err := data.Get("topics").Array()
	t.Logf("got returnedTopics %v", returnedTopics)
	equal(t, err, nil)
	equal(t, len(returnedTopics), 1)

	endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName)
	data, err = http_api.NegotiateV1("GET", endpoint, nil)
	equal(t, err, nil)
	returnedChannels, err := data.Get("channels").Array()
	equal(t, err, nil)
	equal(t, len(returnedChannels), 1)

	returnedProducers, err = data.Get("producers").Array()
	t.Logf("got returnedProducers %v", returnedProducers)
	equal(t, err, nil)
	equal(t, len(returnedProducers), 1)
	for i := range returnedProducers {
		producer := data.Get("producers").GetIndex(i)
		t.Logf("producer %v", producer)

		port, err := producer.Get("tcp_port").Int()
		equal(t, err, nil)
		equal(t, port, tcpPort)

		port, err = producer.Get("http_port").Int()
		equal(t, err, nil)
		equal(t, port, httpPort)

		broadcastaddress, err := producer.Get("broadcast_address").String()
		equal(t, err, nil)
		equal(t, broadcastaddress, "ip.address")

		ver, err := producer.Get("version").String()
		equal(t, err, nil)
		equal(t, ver, "fake-version")
	}

	conn.Close()
	time.Sleep(10 * time.Millisecond)

	// now there should be no producers, but still topic/channel entries
	data, err = http_api.NegotiateV1("GET", endpoint, nil)
	equal(t, err, nil)
	returnedChannels, err = data.Get("channels").Array()
	equal(t, err, nil)
	equal(t, len(returnedChannels), 1)
	returnedProducers, err = data.Get("producers").Array()
	equal(t, err, nil)
	equal(t, len(returnedProducers), 0)
}
Пример #26
0
func TestNsqdToLookupd(t *testing.T) {
	topicName := "cluster_test" + strconv.Itoa(int(time.Now().Unix()))

	hostname, err := os.Hostname()
	equal(t, err, nil)

	url := fmt.Sprintf("http://127.0.0.1:4151/topic/create?topic=%s", topicName)
	_, err = http_api.NegotiateV1("POST", url, nil)
	equal(t, err, nil)

	url = fmt.Sprintf("http://127.0.0.1:4151/channel/create?topic=%s&channel=ch", topicName)
	_, err = http_api.NegotiateV1("POST", url, nil)
	equal(t, err, nil)

	// allow some time for nsqd to push info to nsqlookupd
	time.Sleep(350 * time.Millisecond)

	data, err := http_api.NegotiateV1("GET", "http://127.0.0.1:4161/debug", nil)
	equal(t, err, nil)

	topicData := data.Get("topic:" + topicName + ":")
	producers, _ := topicData.Array()
	equal(t, len(producers), 1)

	producer := topicData.GetIndex(0)
	equal(t, producer.Get("hostname").MustString(), hostname)
	equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
	equal(t, producer.Get("tcp_port").MustInt(), 4150)
	equal(t, producer.Get("tombstoned").MustBool(), false)

	channelData := data.Get("channel:" + topicName + ":ch")
	producers, _ = channelData.Array()
	equal(t, len(producers), 1)

	producer = topicData.GetIndex(0)
	equal(t, producer.Get("hostname").MustString(), hostname)
	equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
	equal(t, producer.Get("tcp_port").MustInt(), 4150)
	equal(t, producer.Get("tombstoned").MustBool(), false)

	data, err = http_api.NegotiateV1("GET", "http://127.0.0.1:4161/lookup?topic="+topicName, nil)
	equal(t, err, nil)

	producers, _ = data.Get("producers").Array()
	equal(t, len(producers), 1)

	producer = data.Get("producers").GetIndex(0)
	equal(t, producer.Get("hostname").MustString(), hostname)
	equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
	equal(t, producer.Get("tcp_port").MustInt(), 4150)

	channels, _ := data.Get("channels").Array()
	equal(t, len(channels), 1)

	channel := channels[0].(string)
	equal(t, channel, "ch")

	data, err = http_api.NegotiateV1("POST", "http://127.0.0.1:4151/topic/delete?topic="+topicName, nil)
	equal(t, err, nil)

	// allow some time for nsqd to push info to nsqlookupd
	time.Sleep(350 * time.Millisecond)

	data, err = http_api.NegotiateV1("GET", "http://127.0.0.1:4161/lookup?topic="+topicName, nil)
	equal(t, err, nil)

	producers, _ = data.Get("producers").Array()
	equal(t, len(producers), 0)

	data, err = http_api.NegotiateV1("GET", "http://127.0.0.1:4161/debug", nil)
	equal(t, err, nil)

	producers, _ = data.Get("topic:" + topicName + ":").Array()
	equal(t, len(producers), 0)

	producers, _ = data.Get("channel:" + topicName + ":ch").Array()
	equal(t, len(producers), 0)
}
Пример #27
0
// GetNSQDStats returns aggregate topic and channel stats from the given NSQD instances
//
// if selectedTopic is empty, this will return stats for *all* topic/channels
// and the ChannelStats dict will be keyed by topic + ':' + channel
func (c *ClusterInfo) GetNSQDStats(producerList ProducerList, selectedTopic string) ([]*TopicStats, map[string]*ChannelStats, error) {
	var lock sync.Mutex
	var wg sync.WaitGroup
	var topicStatsList TopicStatsList

	channelStatsMap := make(map[string]*ChannelStats)

	type respType struct {
		Topics []*TopicStats `json:"topics"`
	}

	success := false
	for _, p := range producerList {
		wg.Add(1)
		endpoint := fmt.Sprintf("http://%s/stats?format=json", p.HTTPAddress())
		c.logf("NSQD: querying %s", endpoint)
		go func(endpoint string, p *Producer) {
			defer wg.Done()

			var resp respType
			err := http_api.NegotiateV1(endpoint, &resp)
			if err != nil {
				c.logf("ERROR: lookupd %s - %s", endpoint, err)
				return
			}

			lock.Lock()
			defer lock.Unlock()
			success = true
			for _, topic := range resp.Topics {
				topic.Node = p.HTTPAddress()
				topic.Hostname = p.Hostname
				if selectedTopic != "" && topic.TopicName != selectedTopic {
					continue
				}
				topicStatsList = append(topicStatsList, topic)

				for _, channel := range topic.Channels {
					channel.Node = p.HTTPAddress()
					channel.Hostname = p.Hostname
					channel.TopicName = topic.TopicName
					key := channel.ChannelName
					if selectedTopic == "" {
						key = fmt.Sprintf("%s:%s", topic.TopicName, channel.ChannelName)
					}
					channelStats, ok := channelStatsMap[key]
					if !ok {
						channelStats = &ChannelStats{
							Node:        p.HTTPAddress(),
							TopicName:   topic.TopicName,
							ChannelName: channel.ChannelName,
						}
						channelStatsMap[key] = channelStats
					}
					for _, c := range channel.Clients {
						c.Node = p.HTTPAddress()
					}
					channelStats.Add(channel)
					channelStats.Clients = append(channelStats.Clients, channel.Clients...)
				}
			}
		}(endpoint, p)
	}
	wg.Wait()

	if success == false {
		return nil, nil, errors.New("unable to query any nsqd")
	}

	sort.Sort(TopicStatsByHost{topicStatsList})
	return topicStatsList, channelStatsMap, nil
}
Пример #28
0
// GetNSQDTopicProducers returns a ProducerList containing the addresses of all the nsqd
// that produce the given topic
func (c *ClusterInfo) GetNSQDTopicProducers(topic string, nsqdHTTPAddrs []string) (ProducerList, error) {
	var success bool
	var producerList ProducerList
	var lock sync.Mutex
	var wg sync.WaitGroup

	type infoRespType struct {
		Version          string `json:"version"`
		BroadcastAddress string `json:"broadcast_address"`
		Hostname         string `json:"hostname"`
		HTTPPort         int    `json:"http_port"`
		TCPPort          int    `json:"tcp_port"`
	}

	type statsRespType struct {
		Topics []struct {
			Name string `json:"topic_name"`
		} `json:"topics"`
	}

	for _, addr := range nsqdHTTPAddrs {
		wg.Add(1)
		go func(addr string) {
			defer wg.Done()

			endpoint := fmt.Sprintf("http://%s/stats?format=json", addr)
			c.logf("NSQD: querying %s", endpoint)

			var statsResp statsRespType
			err := http_api.NegotiateV1(endpoint, &statsResp)
			if err != nil {
				c.logf("ERROR: nsqd %s - %s", endpoint, err)
				return
			}

			var producerTopics ProducerTopics
			for _, t := range statsResp.Topics {
				producerTopics = append(producerTopics, ProducerTopic{Topic: t.Name})
			}

			lock.Lock()
			success = true
			lock.Unlock()

			for _, t := range statsResp.Topics {
				if t.Name == topic {
					endpoint := fmt.Sprintf("http://%s/info", addr)
					c.logf("NSQD: querying %s", endpoint)

					var infoResp infoRespType
					err := http_api.NegotiateV1(endpoint, &infoResp)
					if err != nil {
						c.logf("ERROR: nsqd %s - %s", endpoint, err)
						return
					}

					version, err := semver.Parse(infoResp.Version)
					if err != nil {
						version, _ = semver.Parse("0.0.0")
					}

					lock.Lock()
					defer lock.Unlock()
					producerList = append(producerList, &Producer{
						Version:          infoResp.Version,
						VersionObj:       version,
						BroadcastAddress: infoResp.BroadcastAddress,
						Hostname:         infoResp.Hostname,
						HTTPPort:         infoResp.HTTPPort,
						TCPPort:          infoResp.TCPPort,
						Topics:           producerTopics,
					})

					return
				}
			}
		}(addr)
	}
	wg.Wait()

	if success == false {
		return nil, errors.New("unable to query any nsqd")
	}
	return producerList, nil
}
Пример #29
0
func TestCluster(t *testing.T) {
	lopts := nsqlookupd.NewOptions()
	lopts.Logger = newTestLogger(t)
	lopts.BroadcastAddress = "127.0.0.1"
	_, _, lookupd := mustStartNSQLookupd(lopts)

	opts := NewOptions()
	opts.Logger = newTestLogger(t)
	opts.NSQLookupdTCPAddresses = []string{lookupd.RealTCPAddr().String()}
	opts.BroadcastAddress = "127.0.0.1"
	_, _, nsqd := mustStartNSQD(opts)
	defer os.RemoveAll(opts.DataPath)
	defer nsqd.Exit()

	topicName := "cluster_test" + strconv.Itoa(int(time.Now().Unix()))

	hostname, err := os.Hostname()
	equal(t, err, nil)

	url := fmt.Sprintf("http://%s/topic/create?topic=%s", nsqd.RealHTTPAddr(), topicName)
	_, err = http_api.NegotiateV1("POST", url, nil)
	equal(t, err, nil)

	url = fmt.Sprintf("http://%s/channel/create?topic=%s&channel=ch", nsqd.RealHTTPAddr(), topicName)
	_, err = http_api.NegotiateV1("POST", url, nil)
	equal(t, err, nil)

	// allow some time for nsqd to push info to nsqlookupd
	time.Sleep(350 * time.Millisecond)

	data, err := http_api.NegotiateV1("GET", fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr()), nil)
	equal(t, err, nil)

	topicData := data.Get("topic:" + topicName + ":")
	producers, _ := topicData.Array()
	equal(t, len(producers), 1)

	producer := topicData.GetIndex(0)
	equal(t, producer.Get("hostname").MustString(), hostname)
	equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
	equal(t, producer.Get("tcp_port").MustInt(), nsqd.RealTCPAddr().Port)
	equal(t, producer.Get("tombstoned").MustBool(), false)

	channelData := data.Get("channel:" + topicName + ":ch")
	producers, _ = channelData.Array()
	equal(t, len(producers), 1)

	producer = topicData.GetIndex(0)
	equal(t, producer.Get("hostname").MustString(), hostname)
	equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
	equal(t, producer.Get("tcp_port").MustInt(), nsqd.RealTCPAddr().Port)
	equal(t, producer.Get("tombstoned").MustBool(), false)

	data, err = http_api.NegotiateV1("GET", fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName), nil)
	equal(t, err, nil)

	producers, _ = data.Get("producers").Array()
	equal(t, len(producers), 1)

	producer = data.Get("producers").GetIndex(0)
	equal(t, producer.Get("hostname").MustString(), hostname)
	equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1")
	equal(t, producer.Get("tcp_port").MustInt(), nsqd.RealTCPAddr().Port)

	channels, _ := data.Get("channels").Array()
	equal(t, len(channels), 1)

	channel := channels[0].(string)
	equal(t, channel, "ch")

	data, err = http_api.NegotiateV1("POST", fmt.Sprintf("http://%s/topic/delete?topic=%s", nsqd.RealHTTPAddr(), topicName), nil)
	equal(t, err, nil)

	// allow some time for nsqd to push info to nsqlookupd
	time.Sleep(350 * time.Millisecond)

	data, err = http_api.NegotiateV1("GET", fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName), nil)
	equal(t, err, nil)

	producers, _ = data.Get("producers").Array()
	equal(t, len(producers), 0)

	data, err = http_api.NegotiateV1("GET", fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr()), nil)
	equal(t, err, nil)

	producers, _ = data.Get("topic:" + topicName + ":").Array()
	equal(t, len(producers), 0)

	producers, _ = data.Get("channel:" + topicName + ":ch").Array()
	equal(t, len(producers), 0)
}