// test channel/topic names
func TestChannelTopicNames(t *testing.T) {
equal(t, protocol.IsValidChannelName("test"), true)
equal(t, protocol.IsValidChannelName("test-with_period."), true)
equal(t, protocol.IsValidChannelName("test#ephemeral"), true)
equal(t, protocol.IsValidTopicName("test"), true)
equal(t, protocol.IsValidTopicName("test-with_period."), true)
equal(t, protocol.IsValidTopicName("test#ephemeral"), true)
equal(t, protocol.IsValidTopicName("test:ephemeral"), false)
}
func (s *httpServer) doEmptyTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err)
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, err := reqParams.Get("topic")
if err != nil {
return nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
}
if !protocol.IsValidTopicName(topicName) {
return nil, http_api.Err{400, "INVALID_TOPIC"}
}
topic, err := s.ctx.nsqd.GetExistingTopic(topicName)
if err != nil {
return nil, http_api.Err{404, "TOPIC_NOT_FOUND"}
}
err = topic.Empty()
if err != nil {
return nil, http_api.Err{500, "INTERNAL_ERROR"}
}
return nil, nil
}
func (s *httpServer) tombstoneTopicNode(req *http.Request, topicName string, node string) error {
if !protocol.IsValidTopicName(topicName) {
return errors.New("INVALID_TOPIC")
}
if node == "" {
return errors.New("INVALID_NODE")
}
// tombstone the topic on all the lookupds
for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses {
nsqlookupdVersion, err := s.ci.GetVersion(addr)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err)
}
uri := "tombstone_topic_producer"
if !nsqlookupdVersion.LT(v1EndpointVersion) {
uri = "topic/tombstone"
}
endpoint := fmt.Sprintf("http://%s/%s?topic=%s&node=%s",
addr, uri,
url.QueryEscape(topicName), url.QueryEscape(node))
s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint)
err = http_api.POSTV1(endpoint)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err)
}
}
nsqdVersion, err := s.ci.GetVersion(node)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: failed to get nsqd %s version - %s", node, err)
}
uri := "delete_topic"
if !nsqdVersion.LT(v1EndpointVersion) {
uri = "topic/delete"
}
// delete the topic on the producer
endpoint := fmt.Sprintf("http://%s/%s?topic=%s", node,
uri, url.QueryEscape(topicName))
s.ctx.nsqadmin.logf("NSQD: querying %s", endpoint)
err = http_api.POSTV1(endpoint)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: nsqd %s - %s", endpoint, err)
}
s.notifyAdminAction("tombstone_topic_producer", topicName, "", node, req)
return nil
}
func (p *protocolV2) MPUB(client *clientV2, params [][]byte) ([]byte, error) {
var err error
if len(params) < 2 {
return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "MPUB insufficient number of parameters")
}
topicName := string(params[1])
if !protocol.IsValidTopicName(topicName) {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC",
fmt.Sprintf("E_BAD_TOPIC MPUB topic name %q is not valid", topicName))
}
bodyLen, err := readLen(client.Reader, client.lenSlice)
if err != nil {
return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "MPUB failed to read body size")
}
if bodyLen <= 0 {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY",
fmt.Sprintf("MPUB invalid body size %d", bodyLen))
}
if int64(bodyLen) > p.ctx.nsqd.getOpts().MaxBodySize {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY",
fmt.Sprintf("MPUB body too big %d > %d", bodyLen, p.ctx.nsqd.getOpts().MaxBodySize))
}
messages, err := readMPUB(client.Reader, client.lenSlice, p.ctx.nsqd.idChan,
p.ctx.nsqd.getOpts().MaxMsgSize)
if err != nil {
return nil, err
}
if err := p.CheckAuth(client, "MPUB", topicName, ""); err != nil {
return nil, err
}
topic := p.ctx.nsqd.GetTopic(topicName)
// if we've made it this far we've validated all the input,
// the only possible error is that the topic is exiting during
// this next call (and no messages will be queued in that case)
err = topic.PutMessages(messages)
if err != nil {
return nil, protocol.NewFatalClientErr(err, "E_MPUB_FAILED", "MPUB failed "+err.Error())
}
return okBytes, nil
}
func (p *protocolV2) PUB(client *clientV2, params [][]byte) ([]byte, error) {
var err error
if len(params) < 2 {
return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "PUB insufficient number of parameters")
}
topicName := string(params[1])
if !protocol.IsValidTopicName(topicName) {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC",
fmt.Sprintf("PUB topic name %q is not valid", topicName))
}
bodyLen, err := readLen(client.Reader, client.lenSlice)
if err != nil {
return nil, protocol.NewFatalClientErr(err, "E_BAD_MESSAGE", "PUB failed to read message body size")
}
if bodyLen <= 0 {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_MESSAGE",
fmt.Sprintf("PUB invalid message body size %d", bodyLen))
}
if int64(bodyLen) > p.ctx.nsqd.getOpts().MaxMsgSize {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_MESSAGE",
fmt.Sprintf("PUB message too big %d > %d", bodyLen, p.ctx.nsqd.getOpts().MaxMsgSize))
}
messageBody := make([]byte, bodyLen)
_, err = io.ReadFull(client.Reader, messageBody)
if err != nil {
return nil, protocol.NewFatalClientErr(err, "E_BAD_MESSAGE", "PUB failed to read message body")
}
if err := p.CheckAuth(client, "PUB", topicName, ""); err != nil {
return nil, err
}
topic := p.ctx.nsqd.GetTopic(topicName)
msg := NewMessage(<-p.ctx.nsqd.idChan, messageBody)
err = topic.PutMessage(msg)
if err != nil {
return nil, protocol.NewFatalClientErr(err, "E_PUB_FAILED", "PUB failed "+err.Error())
}
return okBytes, nil
}
func (s *httpServer) getTopicFromQuery(req *http.Request) (url.Values, *Topic, error) {
reqParams, err := url.ParseQuery(req.URL.RawQuery)
if err != nil {
s.ctx.nsqd.logf("ERROR: failed to parse request params - %s", err)
return nil, nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicNames, ok := reqParams["topic"]
if !ok {
return nil, nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
}
topicName := topicNames[0]
if !protocol.IsValidTopicName(topicName) {
return nil, nil, http_api.Err{400, "INVALID_TOPIC"}
}
return reqParams, s.ctx.nsqd.GetTopic(topicName), nil
}
func getTopicChan(command string, params []string) (string, string, error) {
if len(params) == 0 {
return "", "", protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("%s insufficient number of params", command))
}
topicName := params[0]
var channelName string
if len(params) >= 2 {
channelName = params[1]
}
if !protocol.IsValidTopicName(topicName) {
return "", "", protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("%s topic name '%s' is not valid", command, topicName))
}
if channelName != "" && !protocol.IsValidChannelName(channelName) {
return "", "", protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL", fmt.Sprintf("%s channel name '%s' is not valid", command, channelName))
}
return topicName, channelName, nil
}
func (s *httpServer) doCreateTopic(req *http.Request) (interface{}, error) {
reqParams, err := http_api.NewReqParams(req)
if err != nil {
return nil, http_api.Err{400, "INVALID_REQUEST"}
}
topicName, err := reqParams.Get("topic")
if err != nil {
return nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
}
if !protocol.IsValidTopicName(topicName) {
return nil, http_api.Err{400, "INVALID_ARG_TOPIC"}
}
s.ctx.nsqlookupd.logf("DB: adding topic(%s)", topicName)
key := Registration{"topic", topicName, ""}
s.ctx.nsqlookupd.DB.AddRegistration(key)
return nil, nil
}
func GetTopicChannelArgs(rp getter) (string, string, error) {
topicName, err := rp.Get("topic")
if err != nil {
return "", "", errors.New("MISSING_ARG_TOPIC")
}
if !protocol.IsValidTopicName(topicName) {
return "", "", errors.New("INVALID_ARG_TOPIC")
}
channelName, err := rp.Get("channel")
if err != nil {
return "", "", errors.New("MISSING_ARG_CHANNEL")
}
if !protocol.IsValidChannelName(channelName) {
return "", "", errors.New("INVALID_ARG_CHANNEL")
}
return topicName, channelName, nil
}
func (p *protocolV2) SUB(client *clientV2, params [][]byte) ([]byte, error) {
if atomic.LoadInt32(&client.State) != stateInit {
return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot SUB in current state")
}
if client.HeartbeatInterval <= 0 {
return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot SUB with heartbeats disabled")
}
if len(params) < 3 {
return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "SUB insufficient number of parameters")
}
topicName := string(params[1])
if !protocol.IsValidTopicName(topicName) {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC",
fmt.Sprintf("SUB topic name %q is not valid", topicName))
}
channelName := string(params[2])
if !protocol.IsValidChannelName(channelName) {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL",
fmt.Sprintf("SUB channel name %q is not valid", channelName))
}
if err := p.CheckAuth(client, "SUB", topicName, channelName); err != nil {
return nil, err
}
topic := p.ctx.nsqd.GetTopic(topicName)
channel := topic.GetChannel(channelName)
channel.AddClient(client.ID, client)
atomic.StoreInt32(&client.State, stateSubscribed)
client.Channel = channel
// update message pump
client.SubEventChan <- channel
return okBytes, nil
}
func (s *httpServer) createTopicChannelHandler(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
s.ctx.nsqadmin.logf("ERROR: invalid %s to POST only method", req.Method)
http.Error(w, "INVALID_REQUEST", 500)
return
}
reqParams := &http_api.PostParams{req}
topicName, err := reqParams.Get("topic")
if err != nil || !protocol.IsValidTopicName(topicName) {
http.Error(w, "INVALID_TOPIC", 500)
return
}
channelName, err := reqParams.Get("channel")
if err != nil || (len(channelName) > 0 && !protocol.IsValidChannelName(channelName)) {
http.Error(w, "INVALID_CHANNEL", 500)
return
}
for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses {
nsqlookupdVersion, err := lookupd.GetVersion(addr)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err)
}
uri := "create_topic"
if !nsqlookupdVersion.LT(v1EndpointVersion) {
uri = "topic/create"
}
endpoint := fmt.Sprintf("http://%s/%s?topic=%s", addr,
uri, url.QueryEscape(topicName))
s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint)
_, err = http_api.NegotiateV1("POST", endpoint, nil)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err)
continue
}
if len(channelName) > 0 {
uri := "create_channel"
if !nsqlookupdVersion.LT(v1EndpointVersion) {
uri = "channel/create"
}
endpoint := fmt.Sprintf("http://%s/%s?topic=%s&channel=%s",
addr, uri,
url.QueryEscape(topicName),
url.QueryEscape(channelName))
s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint)
_, err := http_api.NegotiateV1("POST", endpoint, nil)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err)
continue
}
}
}
s.notifyAdminAction("create_topic", topicName, "", "", req)
if len(channelName) > 0 {
// TODO: we can remove this when we push new channel information from nsqlookupd -> nsqd
producerAddrs, _ := lookupd.GetLookupdTopicProducers(topicName,
s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
s.performVersionNegotiatedRequestsToNSQD(
s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses,
producerAddrs,
"create_channel",
"channel/create",
fmt.Sprintf("topic=%s&channel=%s",
url.QueryEscape(topicName), url.QueryEscape(channelName)))
s.notifyAdminAction("create_channel", topicName, channelName, "", req)
}
http.Redirect(w, req, "/lookup", 302)
}
func (s *httpServer) topicHandler(w http.ResponseWriter, req *http.Request) {
var urlRegex = regexp.MustCompile(`^/topic/(.*)$`)
matches := urlRegex.FindStringSubmatch(req.URL.Path)
if len(matches) == 0 {
http.Error(w, "INVALID_TOPIC", 500)
return
}
parts := strings.Split(matches[1], "/")
topicName := parts[0]
if !protocol.IsValidTopicName(topicName) {
http.Error(w, "INVALID_TOPIC", 500)
return
}
if len(parts) == 2 {
channelName := parts[1]
if !protocol.IsValidChannelName(channelName) {
http.Error(w, "INVALID_CHANNEL", 500)
} else {
s.channelHandler(w, req, topicName, channelName)
}
return
}
reqParams, err := http_api.NewReqParams(req)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: failed to parse request params - %s", err)
http.Error(w, "INVALID_REQUEST", 500)
return
}
producers := s.getProducers(topicName)
topicStats, channelStats, _ := lookupd.GetNSQDStats(producers, topicName)
globalTopicStats := &lookupd.TopicStats{HostAddress: "Total"}
for _, t := range topicStats {
globalTopicStats.Add(t)
}
hasE2eLatency := globalTopicStats.E2eProcessingLatency != nil &&
len(globalTopicStats.E2eProcessingLatency.Percentiles) > 0
var firstTopic *lookupd.TopicStats
if len(topicStats) > 0 {
firstTopic = topicStats[0]
}
p := struct {
Title string
GraphOptions *GraphOptions
Version string
Topic string
TopicProducers []string
TopicStats []*lookupd.TopicStats
FirstTopic *lookupd.TopicStats
GlobalTopicStats *lookupd.TopicStats
ChannelStats map[string]*lookupd.ChannelStats
HasE2eLatency bool
}{
Title: fmt.Sprintf("NSQ %s", topicName),
GraphOptions: NewGraphOptions(w, req, reqParams, s.ctx),
Version: version.Binary,
Topic: topicName,
TopicProducers: producers,
TopicStats: topicStats,
FirstTopic: firstTopic,
GlobalTopicStats: globalTopicStats,
ChannelStats: channelStats,
HasE2eLatency: hasE2eLatency,
}
err = templates.T.ExecuteTemplate(w, "topic.html", p)
if err != nil {
s.ctx.nsqadmin.logf("Template Error %s", err)
http.Error(w, "Template Error", 500)
}
}
func (p *protocolV2) DPUB(client *clientV2, params [][]byte) ([]byte, error) {
var err error
if len(params) < 3 {
return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "DPUB insufficient number of parameters")
}
topicName := string(params[1])
if !protocol.IsValidTopicName(topicName) {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC",
fmt.Sprintf("DPUB topic name %q is not valid", topicName))
}
timeoutMs, err := protocol.ByteToBase10(params[2])
if err != nil {
return nil, protocol.NewFatalClientErr(err, "E_INVALID",
fmt.Sprintf("DPUB could not parse timeout %s", params[2]))
}
timeoutDuration := time.Duration(timeoutMs) * time.Millisecond
if timeoutDuration < 0 || timeoutDuration > p.ctx.nsqd.getOpts().MaxReqTimeout {
return nil, protocol.NewFatalClientErr(nil, "E_INVALID",
fmt.Sprintf("DPUB timeout %d out of range 0-%d",
timeoutMs, p.ctx.nsqd.getOpts().MaxReqTimeout/time.Millisecond))
}
bodyLen, err := readLen(client.Reader, client.lenSlice)
if err != nil {
return nil, protocol.NewFatalClientErr(err, "E_BAD_MESSAGE", "DPUB failed to read message body size")
}
if bodyLen <= 0 {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_MESSAGE",
fmt.Sprintf("DPUB invalid message body size %d", bodyLen))
}
if int64(bodyLen) > p.ctx.nsqd.getOpts().MaxMsgSize {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_MESSAGE",
fmt.Sprintf("DPUB message too big %d > %d", bodyLen, p.ctx.nsqd.getOpts().MaxMsgSize))
}
messageBody := make([]byte, bodyLen)
_, err = io.ReadFull(client.Reader, messageBody)
if err != nil {
return nil, protocol.NewFatalClientErr(err, "E_BAD_MESSAGE", "DPUB failed to read message body")
}
if err := p.CheckAuth(client, "DPUB", topicName, ""); err != nil {
return nil, err
}
topic := p.ctx.nsqd.GetTopic(topicName)
msg := NewMessage(<-p.ctx.nsqd.idChan, messageBody)
msg.deferred = timeoutDuration
err = topic.PutMessage(msg)
if err != nil {
return nil, protocol.NewFatalClientErr(err, "E_DPUB_FAILED", "DPUB failed "+err.Error())
}
return okBytes, nil
}
func (s *httpServer) createTopicChannel(req *http.Request, topicName string, channelName string) error {
if !protocol.IsValidTopicName(topicName) {
return errors.New("INVALID_TOPIC")
}
if len(channelName) > 0 && !protocol.IsValidChannelName(channelName) {
return errors.New("INVALID_CHANNEL")
}
for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses {
nsqlookupdVersion, err := s.ci.GetVersion(addr)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err)
}
uri := "create_topic"
if !nsqlookupdVersion.LT(v1EndpointVersion) {
uri = "topic/create"
}
endpoint := fmt.Sprintf("http://%s/%s?topic=%s", addr,
uri, url.QueryEscape(topicName))
s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint)
err = http_api.POSTV1(endpoint)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err)
continue
}
if len(channelName) > 0 {
uri := "create_channel"
if !nsqlookupdVersion.LT(v1EndpointVersion) {
uri = "channel/create"
}
endpoint := fmt.Sprintf("http://%s/%s?topic=%s&channel=%s",
addr, uri,
url.QueryEscape(topicName),
url.QueryEscape(channelName))
s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint)
err := http_api.POSTV1(endpoint)
if err != nil {
s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err)
continue
}
}
}
s.notifyAdminAction("create_topic", topicName, "", "", req)
if len(channelName) > 0 {
// TODO: we can remove this when we push new channel information from nsqlookupd -> nsqd
producerList, _ := s.ci.GetLookupdTopicProducers(topicName,
s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
s.performVersionNegotiatedRequestsToNSQD(
s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses,
producerList.HTTPAddrs(),
"create_channel",
"channel/create",
fmt.Sprintf("topic=%s&channel=%s",
url.QueryEscape(topicName), url.QueryEscape(channelName)))
s.notifyAdminAction("create_channel", topicName, channelName, "", req)
}
return nil
}
func (n *NSQD) LoadMetadata() {
n.setFlag(flagLoading, true)
defer n.setFlag(flagLoading, false)
fn := fmt.Sprintf(path.Join(n.getOpts().DataPath, "nsqd.%d.dat"), n.getOpts().ID)
data, err := ioutil.ReadFile(fn)
if err != nil {
if !os.IsNotExist(err) {
n.logf("ERROR: failed to read channel metadata from %s - %s", fn, err)
}
return
}
js, err := simplejson.NewJson(data)
if err != nil {
n.logf("ERROR: failed to parse metadata - %s", err)
return
}
topics, err := js.Get("topics").Array()
if err != nil {
n.logf("ERROR: failed to parse metadata - %s", err)
return
}
for ti := range topics {
topicJs := js.Get("topics").GetIndex(ti)
topicName, err := topicJs.Get("name").String()
if err != nil {
n.logf("ERROR: failed to parse metadata - %s", err)
return
}
if !protocol.IsValidTopicName(topicName) {
n.logf("WARNING: skipping creation of invalid topic %s", topicName)
continue
}
topic := n.GetTopic(topicName)
paused, _ := topicJs.Get("paused").Bool()
if paused {
topic.Pause()
}
channels, err := topicJs.Get("channels").Array()
if err != nil {
n.logf("ERROR: failed to parse metadata - %s", err)
return
}
for ci := range channels {
channelJs := topicJs.Get("channels").GetIndex(ci)
channelName, err := channelJs.Get("name").String()
if err != nil {
n.logf("ERROR: failed to parse metadata - %s", err)
return
}
if !protocol.IsValidChannelName(channelName) {
n.logf("WARNING: skipping creation of invalid channel %s", channelName)
continue
}
channel := topic.GetChannel(channelName)
paused, _ = channelJs.Get("paused").Bool()
if paused {
channel.Pause()
}
}
}
}
func main() {
var selectedMode int
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_nsq v%s\n", version.Binary)
return
}
if *topic == "" || *channel == "" {
log.Fatal("--topic and --channel are required")
}
if *destTopic == "" {
*destTopic = *topic
}
if !protocol.IsValidTopicName(*topic) {
log.Fatal("--topic is invalid")
}
if !protocol.IsValidTopicName(*destTopic) {
log.Fatal("--destination-topic is invalid")
}
if !protocol.IsValidChannelName(*channel) {
log.Fatal("--channel is invalid")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatal("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(destNsqdTCPAddrs) == 0 {
log.Fatal("--destination-nsqd-tcp-address required")
}
switch *mode {
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool":
selectedMode = ModeHostPool
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
defaultUA := fmt.Sprintf("nsq_to_nsq/%s go-nsq/%s", version.Binary, nsq.VERSION)
cCfg := nsq.NewConfig()
cCfg.UserAgent = defaultUA
err := app.ParseOpts(cCfg, consumerOpts)
if err != nil {
log.Fatal(err)
}
cCfg.MaxInFlight = *maxInFlight
// TODO: remove, deprecated
if hasArg("max-backoff-duration") {
log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --consumer-opt=max_backoff_duration,X")
cCfg.MaxBackoffDuration = *maxBackoffDuration
}
pCfg := nsq.NewConfig()
pCfg.UserAgent = defaultUA
err = app.ParseOpts(pCfg, producerOpts)
if err != nil {
log.Fatal(err)
}
consumer, err := nsq.NewConsumer(*topic, *channel, cCfg)
if err != nil {
log.Fatal(err)
}
producers := make(map[string]*nsq.Producer)
for _, addr := range destNsqdTCPAddrs {
producer, err := nsq.NewProducer(addr, pCfg)
if err != nil {
log.Fatalf("failed creating producer %s", err)
}
producers[addr] = producer
}
perAddressStatus := make(map[string]*timer_metrics.TimerMetrics)
if len(destNsqdTCPAddrs) == 1 {
// disable since there is only one address
perAddressStatus[destNsqdTCPAddrs[0]] = timer_metrics.NewTimerMetrics(0, "")
} else {
for _, a := range destNsqdTCPAddrs {
perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery,
fmt.Sprintf("[%s]:", a))
}
}
handler := &PublishHandler{
addresses: destNsqdTCPAddrs,
producers: producers,
mode: selectedMode,
hostPool: hostpool.New(destNsqdTCPAddrs),
respChan: make(chan *nsq.ProducerTransaction, len(destNsqdTCPAddrs)),
perAddressStatus: perAddressStatus,
timermetrics: timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"),
}
consumer.AddConcurrentHandlers(handler, len(destNsqdTCPAddrs))
for i := 0; i < len(destNsqdTCPAddrs); i++ {
go handler.responder()
}
err = consumer.ConnectToNSQDs(nsqdTCPAddrs)
if err != nil {
log.Fatal(err)
}
err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs)
if err != nil {
log.Fatal(err)
}
for {
select {
case <-consumer.StopChan:
return
case <-termChan:
consumer.Stop()
}
}
}