func NewHTTPServer(ctx *Context, logging bool) http.Handler { h := &httpServer{ ctx: ctx, GetMetrics: timer_metrics.NewTimerMetrics(1500, "/get"), MgetMetrics: timer_metrics.NewTimerMetrics(1500, "/mget"), } if logging { return LoggingHandler(os.Stdout, h) } return h }
func main() { var selectedMode int cCfg := nsq.NewConfig() pCfg := nsq.NewConfig() // TODO: remove, deprecated flag.Var(&nsq.ConfigFlag{cCfg}, "reader-opt", "(deprecated) use --consumer-opt") flag.Var(&nsq.ConfigFlag{cCfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, see http://godoc.org/github.com/nsqio/go-nsq#Config)") flag.Var(&nsq.ConfigFlag{pCfg}, "producer-opt", "option to passthrough to nsq.Producer (may be given multiple times, see http://godoc.org/github.com/nsqio/go-nsq#Config)") flag.Parse() if *showVersion { fmt.Printf("nsq_to_nsq v%s\n", version.Binary) return } if *topic == "" || *channel == "" { log.Fatal("--topic and --channel are required") } if *destTopic == "" { *destTopic = *topic } if !protocol.IsValidTopicName(*topic) { log.Fatal("--topic is invalid") } if !protocol.IsValidTopicName(*destTopic) { log.Fatal("--destination-topic is invalid") } if !protocol.IsValidChannelName(*channel) { log.Fatal("--channel is invalid") } if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 { log.Fatal("--nsqd-tcp-address or --lookupd-http-address required") } if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 { log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both") } if len(destNsqdTCPAddrs) == 0 { log.Fatal("--destination-nsqd-tcp-address required") } switch *mode { case "round-robin": selectedMode = ModeRoundRobin case "hostpool", "epsilon-greedy": selectedMode = ModeHostPool } termChan := make(chan os.Signal, 1) signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM) defaultUA := fmt.Sprintf("nsq_to_nsq/%s go-nsq/%s", version.Binary, nsq.VERSION) cCfg.UserAgent = defaultUA cCfg.MaxInFlight = *maxInFlight // TODO: remove, deprecated if hasArg("max-backoff-duration") { log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --consumer-opt=max_backoff_duration,X") cCfg.MaxBackoffDuration = *maxBackoffDuration } consumer, err := nsq.NewConsumer(*topic, *channel, cCfg) if err != nil { log.Fatal(err) } pCfg.UserAgent = defaultUA producers := make(map[string]*nsq.Producer) for _, addr := range destNsqdTCPAddrs { producer, err := nsq.NewProducer(addr, pCfg) if err != nil { log.Fatalf("failed creating producer %s", err) } producers[addr] = producer } perAddressStatus := make(map[string]*timer_metrics.TimerMetrics) if len(destNsqdTCPAddrs) == 1 { // disable since there is only one address perAddressStatus[destNsqdTCPAddrs[0]] = timer_metrics.NewTimerMetrics(0, "") } else { for _, a := range destNsqdTCPAddrs { perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery, fmt.Sprintf("[%s]:", a)) } } hostPool := hostpool.New(destNsqdTCPAddrs) if *mode == "epsilon-greedy" { hostPool = hostpool.NewEpsilonGreedy(destNsqdTCPAddrs, 0, &hostpool.LinearEpsilonValueCalculator{}) } handler := &PublishHandler{ addresses: destNsqdTCPAddrs, producers: producers, mode: selectedMode, hostPool: hostPool, respChan: make(chan *nsq.ProducerTransaction, len(destNsqdTCPAddrs)), perAddressStatus: perAddressStatus, timermetrics: timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"), } consumer.AddConcurrentHandlers(handler, len(destNsqdTCPAddrs)) for i := 0; i < len(destNsqdTCPAddrs); i++ { go handler.responder() } err = consumer.ConnectToNSQDs(nsqdTCPAddrs) if err != nil { log.Fatal(err) } err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs) if err != nil { log.Fatal(err) } for { select { case <-consumer.StopChan: return case <-termChan: consumer.Stop() } } }
func main() { cfg := nsq.NewConfig() // TODO: remove, deprecated flag.Var(&nsq.ConfigFlag{cfg}, "reader-opt", "(deprecated) use --consumer-opt") flag.Var(&nsq.ConfigFlag{cfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/absolute8511/go-nsq#Config)") var publisher Publisher var addresses app.StringArray var selectedMode int flag.Parse() if *showVersion { fmt.Printf("nsq_to_http v%s\n", version.Binary) return } if *topic == "" || *channel == "" { log.Fatal("--topic and --channel are required") } if *contentType != flag.Lookup("content-type").DefValue { if len(postAddrs) == 0 { log.Fatal("--content-type only used with --post") } if len(*contentType) == 0 { log.Fatal("--content-type requires a value when used") } } if len(lookupdHTTPAddrs) == 0 { log.Fatal("--nsqd-tcp-address or --lookupd-http-address required") } if len(getAddrs) == 0 && len(postAddrs) == 0 { log.Fatal("--get or --post required") } if len(getAddrs) > 0 && len(postAddrs) > 0 { log.Fatal("use --get or --post not both") } if len(getAddrs) > 0 { for _, get := range getAddrs { if strings.Count(get, "%s") != 1 { log.Fatal("invalid GET address - must be a printf string") } } } switch *mode { case "multicast": log.Printf("WARNING: multicast mode is deprecated in favor of using separate nsq_to_http on different channels (and will be dropped in a future release)") selectedMode = ModeAll case "round-robin": selectedMode = ModeRoundRobin case "hostpool", "epsilon-greedy": selectedMode = ModeHostPool } // TODO: remove, deprecated if hasArg("--round-robin") { log.Printf("WARNING: --round-robin is deprecated in favor of --mode=round-robin") selectedMode = ModeRoundRobin } // TODO: remove, deprecated if hasArg("throttle-fraction") { log.Printf("WARNING: --throttle-fraction is deprecatedin favor of --sample=X") *sample = *throttleFraction } if *sample > 1.0 || *sample < 0.0 { log.Fatal("ERROR: --sample must be between 0.0 and 1.0") } // TODO: remove, deprecated if hasArg("http-timeout-ms") { log.Printf("WARNING: --http-timeout-ms is deprecated in favor of --http-timeout=X") *httpTimeout = time.Duration(*httpTimeoutMs) * time.Millisecond } termChan := make(chan os.Signal, 1) signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM) if len(postAddrs) > 0 { publisher = &PostPublisher{} addresses = postAddrs } else { publisher = &GetPublisher{} addresses = getAddrs } cfg.UserAgent = fmt.Sprintf("nsq_to_http/%s go-nsq/%s", version.Binary, nsq.VERSION) cfg.MaxInFlight = *maxInFlight // TODO: remove, deprecated if hasArg("max-backoff-duration") { log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --consumer-opt=max_backoff_duration,X") cfg.MaxBackoffDuration = *maxBackoffDuration } consumer, err := nsq.NewConsumer(*topic, *channel, cfg) if err != nil { log.Fatal(err) } perAddressStatus := make(map[string]*timer_metrics.TimerMetrics) if len(addresses) == 1 { // disable since there is only one address perAddressStatus[addresses[0]] = timer_metrics.NewTimerMetrics(0, "") } else { for _, a := range addresses { perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery, fmt.Sprintf("[%s]:", a)) } } hostPool := hostpool.New(addresses) if *mode == "epsilon-greedy" { hostPool = hostpool.NewEpsilonGreedy(addresses, 0, &hostpool.LinearEpsilonValueCalculator{}) } handler := &PublishHandler{ Publisher: publisher, addresses: addresses, mode: selectedMode, hostPool: hostPool, perAddressStatus: perAddressStatus, timermetrics: timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"), } consumer.AddConcurrentHandlers(handler, *numPublishers) err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs) if err != nil { log.Fatal(err) } for { select { case <-consumer.StopChan: return case <-termChan: consumer.Stop() } } }