func identify(t *testing.T, conn net.Conn) {
ci := make(map[string]interface{})
ci["short_id"] = "test"
ci["long_id"] = "test"
cmd, _ := nsq.Identify(ci)
err := cmd.Write(conn)
assert.Equal(t, err, nil)
readValidate(t, conn, nsq.FrameTypeResponse, "OK")
}
func identifyHeartbeatInterval(t *testing.T, conn net.Conn, interval int, f int32, d string) {
ci := make(map[string]interface{})
ci["short_id"] = "test"
ci["long_id"] = "test"
ci["heartbeat_interval"] = interval
cmd, _ := nsq.Identify(ci)
err := cmd.Write(conn)
assert.Equal(t, err, nil)
readValidate(t, conn, f, d)
}
func identifyDisabledHearbeat(t *testing.T, conn net.Conn) {
ci := make(map[string]interface{})
ci["short_id"] = "test"
ci["long_id"] = "test"
ci["heartbeat_interval"] = -1
cmd, _ := nsq.Identify(ci)
err := cmd.Write(conn)
assert.Equal(t, err, nil)
readValidateOK(t, conn)
}
func identifyOutputBuffering(t *testing.T, conn net.Conn, size int, timeout int, f int32, d string) {
ci := make(map[string]interface{})
ci["short_id"] = "test"
ci["long_id"] = "test"
ci["output_buffer_size"] = size
ci["output_buffer_timeout"] = timeout
cmd, _ := nsq.Identify(ci)
err := cmd.Write(conn)
assert.Equal(t, err, nil)
readValidate(t, conn, f, d)
}
func identify(t *testing.T, conn net.Conn, address string, tcpPort int, httpPort int, version string) {
ci := make(map[string]interface{})
ci["tcp_port"] = tcpPort
ci["http_port"] = httpPort
ci["address"] = address //TODO: remove for 1.0
ci["broadcast_address"] = address
ci["hostname"] = address
ci["version"] = version
cmd, _ := nsq.Identify(ci)
err := cmd.Write(conn)
assert.Equal(t, err, nil)
_, err = nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
}
func subWorker(n int, workers int, tcpAddr string, topic string, channel string, rdyChan chan int, goChan chan int, id int) {
conn, err := net.DialTimeout("tcp", tcpAddr, time.Second)
if err != nil {
panic(err.Error())
}
conn.Write(nsq.MagicV2)
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
ci := make(map[string]interface{})
ci["short_id"] = "test"
ci["long_id"] = "test"
cmd, _ := nsq.Identify(ci)
cmd.Write(rw)
nsq.Subscribe(topic, channel).Write(rw)
rdyCount := int(math.Min(math.Max(float64(n/workers), 1), 2500))
rdyChan <- 1
<-goChan
nsq.Ready(rdyCount).Write(rw)
rw.Flush()
nsq.ReadResponse(rw)
nsq.ReadResponse(rw)
num := n / workers
numRdy := num/rdyCount - 1
rdy := rdyCount
for i := 0; i < num; i += 1 {
resp, err := nsq.ReadResponse(rw)
if err != nil {
panic(err.Error())
}
frameType, data, err := nsq.UnpackResponse(resp)
if err != nil {
panic(err.Error())
}
if frameType == nsq.FrameTypeError {
panic("got something else")
}
msg, err := nsq.DecodeMessage(data)
if err != nil {
panic(err.Error())
}
nsq.Finish(msg.Id).Write(rw)
rdy--
if rdy == 0 && numRdy > 0 {
nsq.Ready(rdyCount).Write(rw)
rdy = rdyCount
numRdy--
rw.Flush()
}
}
}
func identifyFeatureNegotiation(t *testing.T, conn net.Conn) []byte {
ci := make(map[string]interface{})
ci["short_id"] = "test"
ci["long_id"] = "test"
ci["feature_negotiation"] = true
cmd, _ := nsq.Identify(ci)
err := cmd.Write(conn)
assert.Equal(t, err, nil)
resp, err := nsq.ReadResponse(conn)
assert.Equal(t, err, nil)
frameType, data, err := nsq.UnpackResponse(resp)
assert.Equal(t, err, nil)
assert.Equal(t, frameType, nsq.FrameTypeResponse)
return data
}
func (n *NSQd) lookupLoop() {
syncTopicChan := make(chan *nsq.LookupPeer)
hostname, err := os.Hostname()
if err != nil {
log.Fatalf("ERROR: failed to get hostname - %s", err.Error())
}
for _, host := range n.lookupdTCPAddrs {
log.Printf("LOOKUP: adding peer %s", host)
lookupPeer := nsq.NewLookupPeer(host, func(lp *nsq.LookupPeer) {
ci := make(map[string]interface{})
ci["version"] = util.BINARY_VERSION
ci["tcp_port"] = n.tcpAddr.Port
ci["http_port"] = n.httpAddr.Port
ci["address"] = hostname //TODO: drop for 1.0
ci["hostname"] = hostname
ci["broadcast_address"] = n.options.broadcastAddress
cmd, err := nsq.Identify(ci)
if err != nil {
lp.Close()
return
}
resp, err := lp.Command(cmd)
if err != nil {
log.Printf("LOOKUPD(%s): ERROR %s - %s", lp, cmd, err.Error())
} else if bytes.Equal(resp, []byte("E_INVALID")) {
log.Printf("LOOKUPD(%s): lookupd returned %s", lp, resp)
} else {
err = json.Unmarshal(resp, &lp.Info)
if err != nil {
log.Printf("LOOKUPD(%s): ERROR parsing response - %v", lp, resp)
} else {
log.Printf("LOOKUPD(%s): peer info %+v", lp, lp.Info)
}
}
go func() {
syncTopicChan <- lp
}()
})
lookupPeer.Command(nil) // start the connection
n.lookupPeers = append(n.lookupPeers, lookupPeer)
}
// for announcements, lookupd determines the host automatically
ticker := time.Tick(15 * time.Second)
for {
select {
case <-ticker:
// send a heartbeat and read a response (read detects closed conns)
for _, lookupPeer := range n.lookupPeers {
log.Printf("LOOKUPD(%s): sending heartbeat", lookupPeer)
cmd := nsq.Ping()
_, err := lookupPeer.Command(cmd)
if err != nil {
log.Printf("LOOKUPD(%s): ERROR %s - %s", lookupPeer, cmd, err.Error())
}
}
case val := <-n.notifyChan:
var cmd *nsq.Command
var branch string
switch val.(type) {
case *Channel:
// notify all nsqlookupds that a new channel exists, or that it's removed
branch = "channel"
channel := val.(*Channel)
if channel.Exiting() == true {
cmd = nsq.UnRegister(channel.topicName, channel.name)
} else {
cmd = nsq.Register(channel.topicName, channel.name)
}
case *Topic:
// notify all nsqlookupds that a new topic exists, or that it's removed
branch = "topic"
topic := val.(*Topic)
if topic.Exiting() == true {
cmd = nsq.UnRegister(topic.name, "")
} else {
cmd = nsq.Register(topic.name, "")
}
}
for _, lookupPeer := range n.lookupPeers {
log.Printf("LOOKUPD(%s): %s %s", lookupPeer, branch, cmd)
_, err := lookupPeer.Command(cmd)
if err != nil {
log.Printf("LOOKUPD(%s): ERROR %s - %s", lookupPeer, cmd, err.Error())
}
}
case lookupPeer := <-syncTopicChan:
commands := make([]*nsq.Command, 0)
// build all the commands first so we exit the lock(s) as fast as possible
nsqd.RLock()
for _, topic := range nsqd.topicMap {
topic.RLock()
if len(topic.channelMap) == 0 {
commands = append(commands, nsq.Register(topic.name, ""))
} else {
for _, channel := range topic.channelMap {
commands = append(commands, nsq.Register(channel.topicName, channel.name))
}
}
topic.RUnlock()
}
nsqd.RUnlock()
for _, cmd := range commands {
log.Printf("LOOKUPD(%s): %s", lookupPeer, cmd)
_, err := lookupPeer.Command(cmd)
if err != nil {
log.Printf("LOOKUPD(%s): ERROR %s - %s", lookupPeer, cmd, err.Error())
break
}
}
case <-n.exitChan:
goto exit
}
}
exit:
log.Printf("LOOKUP: closing")
}