func TestTombstonedNodes(t *testing.T) { log.SetOutput(ioutil.Discard) defer log.SetOutput(os.Stdout) tcpAddr, httpAddr := mustStartLookupd() defer lookupd.Exit() lookupd.inactiveProducerTimeout = 50 * time.Millisecond lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)} topicName := "inactive_nodes" conn := mustConnectLookupd(t, tcpAddr) identify(t, conn, "ip.address", 5000, 5555, "fake-version") nsq.Register(topicName, "channel1").Write(conn) _, err := nsq.ReadResponse(conn) assert.Equal(t, err, nil) producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs) assert.Equal(t, len(producers), 1) assert.Equal(t, len(producers[0].Topics), 1) assert.Equal(t, producers[0].Topics[0].Topic, topicName) assert.Equal(t, producers[0].Topics[0].Tombstoned, false) endpoint := fmt.Sprintf("http://%s/tombstone_topic_producer?topic=%s&node=%s", httpAddr, topicName, "ip.address:5555") _, err = nsq.ApiRequest(endpoint) assert.Equal(t, err, nil) producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs) assert.Equal(t, len(producers), 1) assert.Equal(t, len(producers[0].Topics), 1) assert.Equal(t, producers[0].Topics[0].Topic, topicName) assert.Equal(t, producers[0].Topics[0].Tombstoned, true) }
func TestInactiveNodes(t *testing.T) { log.SetOutput(ioutil.Discard) defer log.SetOutput(os.Stdout) tcpAddr, httpAddr, nsqlookupd := mustStartLookupd() defer nsqlookupd.Exit() nsqlookupd.inactiveProducerTimeout = 50 * time.Millisecond lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)} topicName := "inactive_nodes" conn := mustConnectLookupd(t, tcpAddr) identify(t, conn, "ip.address", 5000, 5555, "fake-version") nsq.Register(topicName, "channel1").Write(conn) _, err := nsq.ReadResponse(conn) assert.Equal(t, err, nil) producers, _ := lookuputil.GetLookupdProducers(lookupdHTTPAddrs) assert.Equal(t, len(producers), 1) assert.Equal(t, len(producers[0].Topics), 1) assert.Equal(t, producers[0].Topics[0].Topic, topicName) assert.Equal(t, producers[0].Topics[0].Tombstoned, false) time.Sleep(55 * time.Millisecond) producers, _ = lookuputil.GetLookupdProducers(lookupdHTTPAddrs) assert.Equal(t, len(producers), 0) }
func TestTombstoneRecover(t *testing.T) { log.SetOutput(ioutil.Discard) defer log.SetOutput(os.Stdout) tcpAddr, httpAddr := mustStartLookupd() defer lookupd.Exit() lookupd.tombstoneLifetime = 50 * time.Millisecond topicName := "tombstone_recover" topicName2 := topicName + "2" conn := mustConnectLookupd(t, tcpAddr) identify(t, conn, "ip.address", 5000, 5555, "fake-version") nsq.Register(topicName, "channel1").Write(conn) _, err := nsq.ReadResponse(conn) assert.Equal(t, err, nil) nsq.Register(topicName2, "channel2").Write(conn) _, err = nsq.ReadResponse(conn) assert.Equal(t, err, nil) endpoint := fmt.Sprintf("http://%s/tombstone_topic_producer?topic=%s&node=%s", httpAddr, topicName, "ip.address:5555") _, err = nsq.ApiRequest(endpoint) assert.Equal(t, err, nil) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) data, err := nsq.ApiRequest(endpoint) assert.Equal(t, err, nil) producers, _ := data.Get("producers").Array() assert.Equal(t, len(producers), 0) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName2) data, err = nsq.ApiRequest(endpoint) assert.Equal(t, err, nil) producers, _ = data.Get("producers").Array() assert.Equal(t, len(producers), 1) time.Sleep(55 * time.Millisecond) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) data, err = nsq.ApiRequest(endpoint) assert.Equal(t, err, nil) producers, _ = data.Get("producers").Array() assert.Equal(t, len(producers), 1) }
func TestChannelUnregister(t *testing.T) { log.SetOutput(ioutil.Discard) defer log.SetOutput(os.Stdout) tcpAddr, httpAddr := mustStartLookupd() defer lookupd.Exit() topics := lookupd.DB.FindRegistrations("topic", "*", "*") assert.Equal(t, len(topics), 0) topicName := "channel_unregister" conn := mustConnectLookupd(t, tcpAddr) tcpPort := 5000 httpPort := 5555 identify(t, conn, "ip.address", tcpPort, httpPort, "fake-version") nsq.Register(topicName, "ch1").Write(conn) v, err := nsq.ReadResponse(conn) assert.Equal(t, err, nil) assert.Equal(t, v, []byte("OK")) topics = lookupd.DB.FindRegistrations("topic", topicName, "") assert.Equal(t, len(topics), 1) channels := lookupd.DB.FindRegistrations("channel", topicName, "*") assert.Equal(t, len(channels), 1) nsq.UnRegister(topicName, "ch1").Write(conn) v, err = nsq.ReadResponse(conn) assert.Equal(t, err, nil) assert.Equal(t, v, []byte("OK")) topics = lookupd.DB.FindRegistrations("topic", topicName, "") assert.Equal(t, len(topics), 1) // we should still have mention of the topic even though there is no producer // (ie. we haven't *deleted* the channel, just unregistered as a producer) channels = lookupd.DB.FindRegistrations("channel", topicName, "*") assert.Equal(t, len(channels), 1) endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) data, err := nsq.ApiRequest(endpoint) assert.Equal(t, err, nil) returnedProducers, err := data.Get("producers").Array() assert.Equal(t, err, nil) assert.Equal(t, len(returnedProducers), 1) }
func TestBasicLookupd(t *testing.T) { log.SetOutput(ioutil.Discard) defer log.SetOutput(os.Stdout) tcpAddr, httpAddr := mustStartLookupd() defer lookupd.Exit() topics := lookupd.DB.FindRegistrations("topic", "*", "*") assert.Equal(t, len(topics), 0) topicName := "connectmsg" conn := mustConnectLookupd(t, tcpAddr) tcpPort := 5000 httpPort := 5555 identify(t, conn, "ip.address", tcpPort, httpPort, "fake-version") nsq.Register(topicName, "channel1").Write(conn) v, err := nsq.ReadResponse(conn) assert.Equal(t, err, nil) assert.Equal(t, v, []byte("OK")) endpoint := fmt.Sprintf("http://%s/nodes", httpAddr) data, err := nsq.ApiRequest(endpoint) log.Printf("got %v", data) returnedProducers, err := data.Get("producers").Array() assert.Equal(t, err, nil) assert.Equal(t, len(returnedProducers), 1) topics = lookupd.DB.FindRegistrations("topic", topicName, "") assert.Equal(t, len(topics), 1) producers := lookupd.DB.FindProducers("topic", topicName, "") assert.Equal(t, len(producers), 1) producer := producers[0] assert.Equal(t, producer.peerInfo.Address, "ip.address") //TODO: remove for 1.0 assert.Equal(t, producer.peerInfo.BroadcastAddress, "ip.address") assert.Equal(t, producer.peerInfo.Hostname, "ip.address") assert.Equal(t, producer.peerInfo.TcpPort, tcpPort) assert.Equal(t, producer.peerInfo.HttpPort, httpPort) endpoint = fmt.Sprintf("http://%s/topics", httpAddr) data, err = nsq.ApiRequest(endpoint) assert.Equal(t, err, nil) returnedTopics, err := data.Get("topics").Array() log.Printf("got returnedTopics %v", returnedTopics) assert.Equal(t, err, nil) assert.Equal(t, len(returnedTopics), 1) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) data, err = nsq.ApiRequest(endpoint) assert.Equal(t, err, nil) returnedChannels, err := data.Get("channels").Array() assert.Equal(t, err, nil) assert.Equal(t, len(returnedChannels), 1) returnedProducers, err = data.Get("producers").Array() log.Printf("got returnedProducers %v", returnedProducers) assert.Equal(t, err, nil) assert.Equal(t, len(returnedProducers), 1) for i := range returnedProducers { producer := data.Get("producers").GetIndex(i) log.Printf("producer %v", producer) assert.Equal(t, err, nil) port, err := producer.Get("tcp_port").Int() assert.Equal(t, err, nil) assert.Equal(t, port, tcpPort) port, err = producer.Get("http_port").Int() assert.Equal(t, err, nil) assert.Equal(t, port, httpPort) address, err := producer.Get("address").String() //TODO: remove for 1.0 broadcastaddress, err := producer.Get("broadcast_address").String() assert.Equal(t, err, nil) assert.Equal(t, address, "ip.address") assert.Equal(t, broadcastaddress, "ip.address") ver, err := producer.Get("version").String() assert.Equal(t, err, nil) assert.Equal(t, ver, "fake-version") } conn.Close() time.Sleep(10 * time.Millisecond) // now there should be no producers, but still topic/channel entries data, err = nsq.ApiRequest(endpoint) assert.Equal(t, err, nil) returnedChannels, err = data.Get("channels").Array() assert.Equal(t, err, nil) assert.Equal(t, len(returnedChannels), 1) returnedProducers, err = data.Get("producers").Array() assert.Equal(t, err, nil) assert.Equal(t, len(returnedProducers), 0) }
func (n *NSQd) lookupLoop() { syncTopicChan := make(chan *nsq.LookupPeer) hostname, err := os.Hostname() if err != nil { log.Fatalf("ERROR: failed to get hostname - %s", err.Error()) } for _, host := range n.lookupdTCPAddrs { log.Printf("LOOKUP: adding peer %s", host) lookupPeer := nsq.NewLookupPeer(host, func(lp *nsq.LookupPeer) { ci := make(map[string]interface{}) ci["version"] = util.BINARY_VERSION ci["tcp_port"] = n.tcpAddr.Port ci["http_port"] = n.httpAddr.Port ci["address"] = hostname //TODO: drop for 1.0 ci["hostname"] = hostname ci["broadcast_address"] = n.options.broadcastAddress cmd, err := nsq.Identify(ci) if err != nil { lp.Close() return } resp, err := lp.Command(cmd) if err != nil { log.Printf("LOOKUPD(%s): ERROR %s - %s", lp, cmd, err.Error()) } else if bytes.Equal(resp, []byte("E_INVALID")) { log.Printf("LOOKUPD(%s): lookupd returned %s", lp, resp) } else { err = json.Unmarshal(resp, &lp.Info) if err != nil { log.Printf("LOOKUPD(%s): ERROR parsing response - %v", lp, resp) } else { log.Printf("LOOKUPD(%s): peer info %+v", lp, lp.Info) } } go func() { syncTopicChan <- lp }() }) lookupPeer.Command(nil) // start the connection n.lookupPeers = append(n.lookupPeers, lookupPeer) } // for announcements, lookupd determines the host automatically ticker := time.Tick(15 * time.Second) for { select { case <-ticker: // send a heartbeat and read a response (read detects closed conns) for _, lookupPeer := range n.lookupPeers { log.Printf("LOOKUPD(%s): sending heartbeat", lookupPeer) cmd := nsq.Ping() _, err := lookupPeer.Command(cmd) if err != nil { log.Printf("LOOKUPD(%s): ERROR %s - %s", lookupPeer, cmd, err.Error()) } } case val := <-n.notifyChan: var cmd *nsq.Command var branch string switch val.(type) { case *Channel: // notify all nsqlookupds that a new channel exists, or that it's removed branch = "channel" channel := val.(*Channel) if channel.Exiting() == true { cmd = nsq.UnRegister(channel.topicName, channel.name) } else { cmd = nsq.Register(channel.topicName, channel.name) } case *Topic: // notify all nsqlookupds that a new topic exists, or that it's removed branch = "topic" topic := val.(*Topic) if topic.Exiting() == true { cmd = nsq.UnRegister(topic.name, "") } else { cmd = nsq.Register(topic.name, "") } } for _, lookupPeer := range n.lookupPeers { log.Printf("LOOKUPD(%s): %s %s", lookupPeer, branch, cmd) _, err := lookupPeer.Command(cmd) if err != nil { log.Printf("LOOKUPD(%s): ERROR %s - %s", lookupPeer, cmd, err.Error()) } } case lookupPeer := <-syncTopicChan: commands := make([]*nsq.Command, 0) // build all the commands first so we exit the lock(s) as fast as possible nsqd.RLock() for _, topic := range nsqd.topicMap { topic.RLock() if len(topic.channelMap) == 0 { commands = append(commands, nsq.Register(topic.name, "")) } else { for _, channel := range topic.channelMap { commands = append(commands, nsq.Register(channel.topicName, channel.name)) } } topic.RUnlock() } nsqd.RUnlock() for _, cmd := range commands { log.Printf("LOOKUPD(%s): %s", lookupPeer, cmd) _, err := lookupPeer.Command(cmd) if err != nil { log.Printf("LOOKUPD(%s): ERROR %s - %s", lookupPeer, cmd, err.Error()) break } } case <-n.exitChan: goto exit } } exit: log.Printf("LOOKUP: closing") }