// test channel/topic names func TestChannelTopicNames(t *testing.T) { test.Equal(t, true, protocol.IsValidChannelName("test")) test.Equal(t, true, protocol.IsValidChannelName("test-with_period.")) test.Equal(t, true, protocol.IsValidChannelName("test#ephemeral")) test.Equal(t, true, protocol.IsValidTopicName("test")) test.Equal(t, true, protocol.IsValidTopicName("test-with_period.")) test.Equal(t, true, protocol.IsValidTopicName("test#ephemeral")) test.Equal(t, false, protocol.IsValidTopicName("test:ephemeral")) }
// test channel/topic names func TestChannelTopicNames(t *testing.T) { equal(t, protocol.IsValidChannelName("test"), true) equal(t, protocol.IsValidChannelName("test-with_period."), true) equal(t, protocol.IsValidChannelName("test#ephemeral"), true) equal(t, protocol.IsValidTopicName("test"), true) equal(t, protocol.IsValidTopicName("test-with_period."), true) equal(t, protocol.IsValidTopicName("test#ephemeral"), true) equal(t, protocol.IsValidTopicName("test:ephemeral"), false) }
func (n *NSQD) LoadMetadata() { atomic.StoreInt32(&n.isLoading, 1) defer atomic.StoreInt32(&n.isLoading, 0) fn := fmt.Sprintf(path.Join(n.getOpts().DataPath, "nsqd.%d.dat"), n.getOpts().ID) data, err := ioutil.ReadFile(fn) if err != nil { if !os.IsNotExist(err) { n.logf("ERROR: failed to read channel metadata from %s - %s", fn, err) } return } var m meta err = json.Unmarshal(data, &m) if err != nil { n.logf("ERROR: failed to parse metadata - %s", err) return } for _, t := range m.Topics { if !protocol.IsValidTopicName(t.Name) { n.logf("WARNING: skipping creation of invalid topic %s", t.Name) continue } topic := n.GetTopic(t.Name) if t.Paused { topic.Pause() } for _, c := range t.Channels { if !protocol.IsValidChannelName(c.Name) { n.logf("WARNING: skipping creation of invalid channel %s", c.Name) continue } channel := topic.GetChannel(c.Name) if c.Paused { channel.Pause() } } } }
func (s *httpServer) createTopicChannelHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string var body struct { Topic string `json:"topic"` Channel string `json:"channel"` } err := json.NewDecoder(req.Body).Decode(&body) if err != nil { return nil, http_api.Err{400, err.Error()} } if !protocol.IsValidTopicName(body.Topic) { return nil, http_api.Err{400, "INVALID_TOPIC"} } if len(body.Channel) > 0 && !protocol.IsValidChannelName(body.Channel) { return nil, http_api.Err{400, "INVALID_CHANNEL"} } err = s.ci.CreateTopicChannel(body.Topic, body.Channel, s.ctx.nsqadmin.getOpts().NSQLookupdHTTPAddresses) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.ctx.nsqadmin.logf("ERROR: failed to create topic/channel - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.ctx.nsqadmin.logf("WARNING: %s", err) messages = append(messages, pe.Error()) } s.notifyAdminAction("create_topic", body.Topic, "", "", req) if len(body.Channel) > 0 { s.notifyAdminAction("create_channel", body.Topic, body.Channel, "", req) } return struct { Message string `json:"message"` }{maybeWarnMsg(messages)}, nil }
func getTopicChan(command string, params []string) (string, string, error) { if len(params) == 0 { return "", "", protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("%s insufficient number of params", command)) } topicName := params[0] var channelName string if len(params) >= 2 { channelName = params[1] } if !protocol.IsValidTopicName(topicName) { return "", "", protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("%s topic name '%s' is not valid", command, topicName)) } if channelName != "" && !protocol.IsValidChannelName(channelName) { return "", "", protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL", fmt.Sprintf("%s channel name '%s' is not valid", command, channelName)) } return topicName, channelName, nil }
func GetTopicChannelArgs(rp getter) (string, string, error) { topicName, err := rp.Get("topic") if err != nil { return "", "", errors.New("MISSING_ARG_TOPIC") } if !protocol.IsValidTopicName(topicName) { return "", "", errors.New("INVALID_ARG_TOPIC") } channelName, err := rp.Get("channel") if err != nil { return "", "", errors.New("MISSING_ARG_CHANNEL") } if !protocol.IsValidChannelName(channelName) { return "", "", errors.New("INVALID_ARG_CHANNEL") } return topicName, channelName, nil }
func (p *protocolV2) SUB(client *clientV2, params [][]byte) ([]byte, error) { if atomic.LoadInt32(&client.State) != stateInit { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot SUB in current state") } if client.HeartbeatInterval <= 0 { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot SUB with heartbeats disabled") } if len(params) < 3 { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "SUB insufficient number of parameters") } topicName := string(params[1]) if !protocol.IsValidTopicName(topicName) { return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("SUB topic name %q is not valid", topicName)) } channelName := string(params[2]) if !protocol.IsValidChannelName(channelName) { return nil, protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL", fmt.Sprintf("SUB channel name %q is not valid", channelName)) } if err := p.CheckAuth(client, "SUB", topicName, channelName); err != nil { return nil, err } topic := p.ctx.nsqd.GetTopic(topicName) channel := topic.GetChannel(channelName) channel.AddClient(client.ID, client) atomic.StoreInt32(&client.State, stateSubscribed) client.Channel = channel // update message pump client.SubEventChan <- channel return okBytes, nil }
func main() { var selectedMode int cCfg := nsq.NewConfig() pCfg := nsq.NewConfig() // TODO: remove, deprecated flag.Var(&nsq.ConfigFlag{cCfg}, "reader-opt", "(deprecated) use --consumer-opt") flag.Var(&nsq.ConfigFlag{cCfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, see http://godoc.org/github.com/nsqio/go-nsq#Config)") flag.Var(&nsq.ConfigFlag{pCfg}, "producer-opt", "option to passthrough to nsq.Producer (may be given multiple times, see http://godoc.org/github.com/nsqio/go-nsq#Config)") flag.Parse() if *showVersion { fmt.Printf("nsq_to_nsq v%s\n", version.Binary) return } if *topic == "" || *channel == "" { log.Fatal("--topic and --channel are required") } if *destTopic == "" { *destTopic = *topic } if !protocol.IsValidTopicName(*topic) { log.Fatal("--topic is invalid") } if !protocol.IsValidTopicName(*destTopic) { log.Fatal("--destination-topic is invalid") } if !protocol.IsValidChannelName(*channel) { log.Fatal("--channel is invalid") } if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 { log.Fatal("--nsqd-tcp-address or --lookupd-http-address required") } if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 { log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both") } if len(destNsqdTCPAddrs) == 0 { log.Fatal("--destination-nsqd-tcp-address required") } switch *mode { case "round-robin": selectedMode = ModeRoundRobin case "hostpool", "epsilon-greedy": selectedMode = ModeHostPool } termChan := make(chan os.Signal, 1) signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM) defaultUA := fmt.Sprintf("nsq_to_nsq/%s go-nsq/%s", version.Binary, nsq.VERSION) cCfg.UserAgent = defaultUA cCfg.MaxInFlight = *maxInFlight // TODO: remove, deprecated if hasArg("max-backoff-duration") { log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --consumer-opt=max_backoff_duration,X") cCfg.MaxBackoffDuration = *maxBackoffDuration } consumer, err := nsq.NewConsumer(*topic, *channel, cCfg) if err != nil { log.Fatal(err) } pCfg.UserAgent = defaultUA producers := make(map[string]*nsq.Producer) for _, addr := range destNsqdTCPAddrs { producer, err := nsq.NewProducer(addr, pCfg) if err != nil { log.Fatalf("failed creating producer %s", err) } producers[addr] = producer } perAddressStatus := make(map[string]*timer_metrics.TimerMetrics) if len(destNsqdTCPAddrs) == 1 { // disable since there is only one address perAddressStatus[destNsqdTCPAddrs[0]] = timer_metrics.NewTimerMetrics(0, "") } else { for _, a := range destNsqdTCPAddrs { perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery, fmt.Sprintf("[%s]:", a)) } } hostPool := hostpool.New(destNsqdTCPAddrs) if *mode == "epsilon-greedy" { hostPool = hostpool.NewEpsilonGreedy(destNsqdTCPAddrs, 0, &hostpool.LinearEpsilonValueCalculator{}) } handler := &PublishHandler{ addresses: destNsqdTCPAddrs, producers: producers, mode: selectedMode, hostPool: hostPool, respChan: make(chan *nsq.ProducerTransaction, len(destNsqdTCPAddrs)), perAddressStatus: perAddressStatus, timermetrics: timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"), } consumer.AddConcurrentHandlers(handler, len(destNsqdTCPAddrs)) for i := 0; i < len(destNsqdTCPAddrs); i++ { go handler.responder() } err = consumer.ConnectToNSQDs(nsqdTCPAddrs) if err != nil { log.Fatal(err) } err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs) if err != nil { log.Fatal(err) } for { select { case <-consumer.StopChan: return case <-termChan: consumer.Stop() } } }
func (n *NSQD) LoadMetadata() { atomic.StoreInt32(&n.isLoading, 1) defer atomic.StoreInt32(&n.isLoading, 0) fn := fmt.Sprintf(path.Join(n.getOpts().DataPath, "nsqd.%d.dat"), n.getOpts().ID) data, err := ioutil.ReadFile(fn) if err != nil { if !os.IsNotExist(err) { n.logf("ERROR: failed to read channel metadata from %s - %s", fn, err) } return } js, err := simplejson.NewJson(data) if err != nil { n.logf("ERROR: failed to parse metadata - %s", err) return } topics, err := js.Get("topics").Array() if err != nil { n.logf("ERROR: failed to parse metadata - %s", err) return } for ti := range topics { topicJs := js.Get("topics").GetIndex(ti) topicName, err := topicJs.Get("name").String() if err != nil { n.logf("ERROR: failed to parse metadata - %s", err) return } if !protocol.IsValidTopicName(topicName) { n.logf("WARNING: skipping creation of invalid topic %s", topicName) continue } topic := n.GetTopic(topicName) paused, _ := topicJs.Get("paused").Bool() if paused { topic.Pause() } channels, err := topicJs.Get("channels").Array() if err != nil { n.logf("ERROR: failed to parse metadata - %s", err) return } for ci := range channels { channelJs := topicJs.Get("channels").GetIndex(ci) channelName, err := channelJs.Get("name").String() if err != nil { n.logf("ERROR: failed to parse metadata - %s", err) return } if !protocol.IsValidChannelName(channelName) { n.logf("WARNING: skipping creation of invalid channel %s", channelName) continue } channel := topic.GetChannel(channelName) paused, _ = channelJs.Get("paused").Bool() if paused { channel.Pause() } } } }
func (s *httpServer) createTopicChannel(req *http.Request, topicName string, channelName string) error { if !protocol.IsValidTopicName(topicName) { return errors.New("INVALID_TOPIC") } if len(channelName) > 0 && !protocol.IsValidChannelName(channelName) { return errors.New("INVALID_CHANNEL") } for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses { nsqlookupdVersion, err := s.ci.GetVersion(addr) if err != nil { s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err) } uri := "create_topic" if !nsqlookupdVersion.LT(v1EndpointVersion) { uri = "topic/create" } endpoint := fmt.Sprintf("http://%s/%s?topic=%s", addr, uri, url.QueryEscape(topicName)) s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint) err = s.client.POSTV1(endpoint) if err != nil { s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err) continue } if len(channelName) > 0 { uri := "create_channel" if !nsqlookupdVersion.LT(v1EndpointVersion) { uri = "channel/create" } endpoint := fmt.Sprintf("http://%s/%s?topic=%s&channel=%s", addr, uri, url.QueryEscape(topicName), url.QueryEscape(channelName)) s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint) err := s.client.POSTV1(endpoint) if err != nil { s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err) continue } } } s.notifyAdminAction("create_topic", topicName, "", "", req) if len(channelName) > 0 { // TODO: we can remove this when we push new channel information from nsqlookupd -> nsqd producerList, _ := s.ci.GetLookupdTopicProducers(topicName, s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses) s.performVersionNegotiatedRequestsToNSQD( s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses, producerList.HTTPAddrs(), "create_channel", "channel/create", fmt.Sprintf("topic=%s&channel=%s", url.QueryEscape(topicName), url.QueryEscape(channelName))) s.notifyAdminAction("create_channel", topicName, channelName, "", req) } return nil }