func NewHTTPServer(ctx *Context, logging bool) http.Handler {
h := &httpServer{
ctx: ctx,
GetMetrics: timer_metrics.NewTimerMetrics(1500, "/get"),
MgetMetrics: timer_metrics.NewTimerMetrics(1500, "/mget"),
}
if logging {
return LoggingHandler(os.Stdout, h)
}
return h
}
func main() {
var selectedMode int
cCfg := nsq.NewConfig()
pCfg := nsq.NewConfig()
// TODO: remove, deprecated
flag.Var(&nsq.ConfigFlag{cCfg}, "reader-opt", "(deprecated) use --consumer-opt")
flag.Var(&nsq.ConfigFlag{cCfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, see http://godoc.org/github.com/nsqio/go-nsq#Config)")
flag.Var(&nsq.ConfigFlag{pCfg}, "producer-opt", "option to passthrough to nsq.Producer (may be given multiple times, see http://godoc.org/github.com/nsqio/go-nsq#Config)")
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_nsq v%s\n", version.Binary)
return
}
if *topic == "" || *channel == "" {
log.Fatal("--topic and --channel are required")
}
if *destTopic == "" {
*destTopic = *topic
}
if !protocol.IsValidTopicName(*topic) {
log.Fatal("--topic is invalid")
}
if !protocol.IsValidTopicName(*destTopic) {
log.Fatal("--destination-topic is invalid")
}
if !protocol.IsValidChannelName(*channel) {
log.Fatal("--channel is invalid")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatal("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(destNsqdTCPAddrs) == 0 {
log.Fatal("--destination-nsqd-tcp-address required")
}
switch *mode {
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool", "epsilon-greedy":
selectedMode = ModeHostPool
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
defaultUA := fmt.Sprintf("nsq_to_nsq/%s go-nsq/%s", version.Binary, nsq.VERSION)
cCfg.UserAgent = defaultUA
cCfg.MaxInFlight = *maxInFlight
// TODO: remove, deprecated
if hasArg("max-backoff-duration") {
log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --consumer-opt=max_backoff_duration,X")
cCfg.MaxBackoffDuration = *maxBackoffDuration
}
consumer, err := nsq.NewConsumer(*topic, *channel, cCfg)
if err != nil {
log.Fatal(err)
}
pCfg.UserAgent = defaultUA
producers := make(map[string]*nsq.Producer)
for _, addr := range destNsqdTCPAddrs {
producer, err := nsq.NewProducer(addr, pCfg)
if err != nil {
log.Fatalf("failed creating producer %s", err)
}
producers[addr] = producer
}
perAddressStatus := make(map[string]*timer_metrics.TimerMetrics)
if len(destNsqdTCPAddrs) == 1 {
// disable since there is only one address
perAddressStatus[destNsqdTCPAddrs[0]] = timer_metrics.NewTimerMetrics(0, "")
} else {
for _, a := range destNsqdTCPAddrs {
perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery,
fmt.Sprintf("[%s]:", a))
}
}
hostPool := hostpool.New(destNsqdTCPAddrs)
if *mode == "epsilon-greedy" {
hostPool = hostpool.NewEpsilonGreedy(destNsqdTCPAddrs, 0, &hostpool.LinearEpsilonValueCalculator{})
}
handler := &PublishHandler{
addresses: destNsqdTCPAddrs,
producers: producers,
mode: selectedMode,
hostPool: hostPool,
respChan: make(chan *nsq.ProducerTransaction, len(destNsqdTCPAddrs)),
perAddressStatus: perAddressStatus,
timermetrics: timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"),
}
consumer.AddConcurrentHandlers(handler, len(destNsqdTCPAddrs))
for i := 0; i < len(destNsqdTCPAddrs); i++ {
go handler.responder()
}
err = consumer.ConnectToNSQDs(nsqdTCPAddrs)
if err != nil {
log.Fatal(err)
}
err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs)
if err != nil {
log.Fatal(err)
}
for {
select {
case <-consumer.StopChan:
return
case <-termChan:
consumer.Stop()
}
}
}
func main() {
cfg := nsq.NewConfig()
// TODO: remove, deprecated
flag.Var(&nsq.ConfigFlag{cfg}, "reader-opt", "(deprecated) use --consumer-opt")
flag.Var(&nsq.ConfigFlag{cfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/absolute8511/go-nsq#Config)")
var publisher Publisher
var addresses app.StringArray
var selectedMode int
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_http v%s\n", version.Binary)
return
}
if *topic == "" || *channel == "" {
log.Fatal("--topic and --channel are required")
}
if *contentType != flag.Lookup("content-type").DefValue {
if len(postAddrs) == 0 {
log.Fatal("--content-type only used with --post")
}
if len(*contentType) == 0 {
log.Fatal("--content-type requires a value when used")
}
}
if len(lookupdHTTPAddrs) == 0 {
log.Fatal("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(getAddrs) == 0 && len(postAddrs) == 0 {
log.Fatal("--get or --post required")
}
if len(getAddrs) > 0 && len(postAddrs) > 0 {
log.Fatal("use --get or --post not both")
}
if len(getAddrs) > 0 {
for _, get := range getAddrs {
if strings.Count(get, "%s") != 1 {
log.Fatal("invalid GET address - must be a printf string")
}
}
}
switch *mode {
case "multicast":
log.Printf("WARNING: multicast mode is deprecated in favor of using separate nsq_to_http on different channels (and will be dropped in a future release)")
selectedMode = ModeAll
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool", "epsilon-greedy":
selectedMode = ModeHostPool
}
// TODO: remove, deprecated
if hasArg("--round-robin") {
log.Printf("WARNING: --round-robin is deprecated in favor of --mode=round-robin")
selectedMode = ModeRoundRobin
}
// TODO: remove, deprecated
if hasArg("throttle-fraction") {
log.Printf("WARNING: --throttle-fraction is deprecatedin favor of --sample=X")
*sample = *throttleFraction
}
if *sample > 1.0 || *sample < 0.0 {
log.Fatal("ERROR: --sample must be between 0.0 and 1.0")
}
// TODO: remove, deprecated
if hasArg("http-timeout-ms") {
log.Printf("WARNING: --http-timeout-ms is deprecated in favor of --http-timeout=X")
*httpTimeout = time.Duration(*httpTimeoutMs) * time.Millisecond
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
if len(postAddrs) > 0 {
publisher = &PostPublisher{}
addresses = postAddrs
} else {
publisher = &GetPublisher{}
addresses = getAddrs
}
cfg.UserAgent = fmt.Sprintf("nsq_to_http/%s go-nsq/%s", version.Binary, nsq.VERSION)
cfg.MaxInFlight = *maxInFlight
// TODO: remove, deprecated
if hasArg("max-backoff-duration") {
log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --consumer-opt=max_backoff_duration,X")
cfg.MaxBackoffDuration = *maxBackoffDuration
}
consumer, err := nsq.NewConsumer(*topic, *channel, cfg)
if err != nil {
log.Fatal(err)
}
perAddressStatus := make(map[string]*timer_metrics.TimerMetrics)
if len(addresses) == 1 {
// disable since there is only one address
perAddressStatus[addresses[0]] = timer_metrics.NewTimerMetrics(0, "")
} else {
for _, a := range addresses {
perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery,
fmt.Sprintf("[%s]:", a))
}
}
hostPool := hostpool.New(addresses)
if *mode == "epsilon-greedy" {
hostPool = hostpool.NewEpsilonGreedy(addresses, 0, &hostpool.LinearEpsilonValueCalculator{})
}
handler := &PublishHandler{
Publisher: publisher,
addresses: addresses,
mode: selectedMode,
hostPool: hostPool,
perAddressStatus: perAddressStatus,
timermetrics: timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"),
}
consumer.AddConcurrentHandlers(handler, *numPublishers)
err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs)
if err != nil {
log.Fatal(err)
}
for {
select {
case <-consumer.StopChan:
return
case <-termChan:
consumer.Stop()
}
}
}