func newReaderFileLogger(topic string) (*ReaderFileLogger, error) {
f, err := NewFileLogger(*gzipEnabled, *gzipLevel, *filenameFormat, topic)
if err != nil {
return nil, err
}
cfg := nsq.NewConfig()
err = util.ParseReaderOpts(cfg, readerOpts)
if err != nil {
return nil, err
}
cfg.Set("max_in_flight", *maxInFlight)
// TODO: remove, deprecated
if hasArg("verbose") {
log.Printf("WARNING: --verbose is deprecated in favor of --reader-opt=verbose")
cfg.Set("verbose", true)
}
r, err := nsq.NewConsumer(topic, *channel, cfg)
if err != nil {
return nil, err
}
r.SetLogger(log.New(os.Stderr, "", log.LstdFlags), nsq.LogLevelInfo)
r.SetHandler(f)
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQD(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToNSQLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
return &ReaderFileLogger{
R: r,
F: f,
}, nil
}
func newReaderFileLogger(topic string) (*ReaderFileLogger, error) {
f, err := NewFileLogger(*gzipEnabled, *gzipLevel, *filenameFormat, topic)
if err != nil {
return nil, err
}
r, err := nsq.NewReader(topic, *channel)
if err != nil {
return nil, err
}
err = util.ParseReaderOpts(r, readerOpts)
if err != nil {
return nil, err
}
r.SetMaxInFlight(*maxInFlight)
r.AddAsyncHandler(f)
// TODO: remove, deprecated
if hasArg("verbose") {
log.Printf("WARNING: --verbose is deprecated in favor of --reader-opt=verbose")
r.Configure("verbose", true)
}
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQ(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
return &ReaderFileLogger{
R: r,
F: f,
}, nil
}
func newReaderFileLogger(topic string) (*ReaderFileLogger, error) {
f, err := NewFileLogger(*gzipEnabled, *gzipLevel, *filenameFormat, topic)
if err != nil {
return nil, err
}
cfg := nsq.NewConfig()
cfg.UserAgent = fmt.Sprintf("nsq_to_file/%s go-nsq/%s", util.BINARY_VERSION, nsq.VERSION)
err = util.ParseReaderOpts(cfg, readerOpts)
if err != nil {
return nil, err
}
cfg.MaxInFlight = *maxInFlight
r, err := nsq.NewConsumer(topic, *channel, cfg)
if err != nil {
return nil, err
}
r.SetLogger(log.New(os.Stderr, "", log.LstdFlags), nsq.LogLevelInfo)
r.AddHandler(f)
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQD(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToNSQLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
return &ReaderFileLogger{
R: r,
F: f,
}, nil
}
func main() {
logger = log.New(os.Stderr, "", log.LstdFlags)
// handle stopping
stopChan := make(chan bool)
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
// parse flags
flag.Parse()
if len(*topic) == 0 {
fatal(true, "Must specify a valid topic")
}
if len(*delimiter) != 1 {
fatal(true, "Delimiter must be a single byte")
}
// prepare the configuration
wcfg := nsq.NewConfig()
wcfg.UserAgent = fmt.Sprintf("to_nsq/%s go-nsq/%s", util.BINARY_VERSION, nsq.VERSION)
if err := util.ParseReaderOpts(wcfg, readerOpts); err != nil {
fatal(true, err)
}
// make the producers
producers := make(map[string]*nsq.Producer)
for _, addr := range destNsqdTCPAddrs {
producer, err := nsq.NewProducer(addr, wcfg)
if err != nil {
fatal(true, "failed creating producer", err)
}
producer.SetLogger(logger, nsq.LogLevelInfo)
producers[addr] = producer
defer producer.Stop()
}
if len(producers) == 0 {
fatal(true, "Must specify at least one nsqd-tcp-address")
}
var fatalErr error
// scan the input
r := bufio.NewReader(os.Stdin)
delim := (*delimiter)[0]
go func() {
for {
if err := readAndPublish(r, delim, producers); err != nil {
if err != io.EOF {
fatalErr = err
}
stopChan <- true // stop
break
}
}
}()
// wait for things to finish
select {
case <-termChan:
case <-stopChan:
}
// if a fatal error occurred - report it
if fatalErr != nil {
fatal(false, fatalErr)
}
}
func main() {
var selectedMode int
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_nsq v%s\n", util.BINARY_VERSION)
return
}
if *topic == "" || *channel == "" {
log.Fatalf("--topic and --channel are required")
}
if *destTopic == "" {
*destTopic = *topic
}
if !util.IsValidTopicName(*topic) {
log.Fatalf("--topic is invalid")
}
if !util.IsValidTopicName(*destTopic) {
log.Fatalf("--destination-topic is invalid")
}
if !util.IsValidChannelName(*channel) {
log.Fatalf("--channel is invalid")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(destNsqdTCPAddrs) == 0 {
log.Fatalf("--destination-nsqd-tcp-address required")
}
switch *mode {
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool":
selectedMode = ModeHostPool
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
cfg := nsq.NewConfig()
err := util.ParseReaderOpts(cfg, readerOpts)
if err != nil {
log.Fatalf(err.Error())
}
cfg.Set("max_in_flight", *maxInFlight)
// TODO: remove, deprecated
if hasArg("verbose") {
log.Printf("WARNING: --verbose is deprecated in favor of --reader-opt=verbose")
cfg.Set("verbose", true)
}
// TODO: remove, deprecated
if hasArg("max-backoff-duration") {
log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --reader-opt=max_backoff_duration=X")
cfg.Set("max_backoff_duration", *maxBackoffDuration)
}
r, err := nsq.NewConsumer(*topic, *channel, cfg)
if err != nil {
log.Fatalf(err.Error())
}
r.SetLogger(log.New(os.Stderr, "", log.LstdFlags), nsq.LogLevelInfo)
wcfg := nsq.NewConfig()
wcfg.Set("heartbeat_interval", nsq.DefaultClientTimeout/2)
producers := make(map[string]*nsq.Producer)
for _, addr := range destNsqdTCPAddrs {
producers[addr] = nsq.NewProducer(addr, wcfg)
}
handler := &PublishHandler{
addresses: destNsqdTCPAddrs,
producers: producers,
mode: selectedMode,
reqs: make(Durations, 0, *statusEvery),
hostPool: hostpool.New(destNsqdTCPAddrs),
respChan: make(chan *nsq.ProducerTransaction, len(destNsqdTCPAddrs)),
}
r.SetConcurrentHandlers(handler, len(destNsqdTCPAddrs))
for i := 0; i < len(destNsqdTCPAddrs); i++ {
go handler.responder()
}
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQD(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToNSQLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for {
select {
case <-r.StopChan:
return
case <-termChan:
r.Stop()
}
}
}
func main() {
var publisher Publisher
var addresses util.StringArray
var selectedMode int
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_http v%s\n", util.BINARY_VERSION)
return
}
if *topic == "" || *channel == "" {
log.Fatalf("--topic and --channel are required")
}
if *contentType != flag.Lookup("content-type").DefValue {
if len(postAddrs) == 0 {
log.Fatalf("--content-type only used with --post")
}
if len(*contentType) == 0 {
log.Fatalf("--content-type requires a value when used")
}
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(getAddrs) == 0 && len(postAddrs) == 0 {
log.Fatalf("--get or --post required")
}
if len(getAddrs) > 0 && len(postAddrs) > 0 {
log.Fatalf("use --get or --post not both")
}
if len(getAddrs) > 0 {
for _, get := range getAddrs {
if strings.Count(get, "%s") != 1 {
log.Fatal("invalid GET address - must be a printf string")
}
}
}
switch *mode {
case "multicast":
log.Printf("WARNING: multicast mode is deprecated in favor of using separate nsq_to_http on different channels (and will be dropped in a future release)")
selectedMode = ModeAll
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool":
selectedMode = ModeHostPool
}
// TODO: remove, deprecated
if hasArg("--round-robin") {
log.Printf("WARNING: --round-robin is deprecated in favor of --mode=round-robin")
selectedMode = ModeRoundRobin
}
// TODO: remove, deprecated
if hasArg("throttle-fraction") {
log.Printf("WARNING: --throttle-fraction is deprecatedin favor of --sample=X")
*sample = *throttleFraction
}
if *sample > 1.0 || *sample < 0.0 {
log.Fatalf("ERROR: --sample must be between 0.0 and 1.0")
}
// TODO: remove, deprecated
if hasArg("http-timeout-ms") {
log.Printf("WARNING: --http-timeout-ms is deprecated in favor of --http-timeout=X")
*httpTimeout = time.Duration(*httpTimeoutMs) * time.Millisecond
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
if len(postAddrs) > 0 {
publisher = &PostPublisher{}
addresses = postAddrs
} else {
publisher = &GetPublisher{}
addresses = getAddrs
}
cfg := nsq.NewConfig()
cfg.UserAgent = fmt.Sprintf("nsq_to_http/%s go-nsq/%s", util.BINARY_VERSION, nsq.VERSION)
err := util.ParseReaderOpts(cfg, readerOpts)
if err != nil {
log.Fatalf(err.Error())
}
cfg.MaxInFlight = *maxInFlight
// TODO: remove, deprecated
if hasArg("max-backoff-duration") {
log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --reader-opt=max_backoff_duration=X")
cfg.MaxBackoffDuration = *maxBackoffDuration
}
r, err := nsq.NewConsumer(*topic, *channel, cfg)
if err != nil {
log.Fatalf(err.Error())
}
perAddressStatus := make(map[string]*timermetrics.TimerMetrics)
if len(addresses) == 1 {
// disable since there is only one address
perAddressStatus[addresses[0]] = timermetrics.NewTimerMetrics(0, "")
} else {
for _, a := range addresses {
perAddressStatus[a] = timermetrics.NewTimerMetrics(*statusEvery,
fmt.Sprintf("[%s]:", a))
}
}
handler := &PublishHandler{
Publisher: publisher,
addresses: addresses,
mode: selectedMode,
hostPool: hostpool.New(addresses),
perAddressStatus: perAddressStatus,
timermetrics: timermetrics.NewTimerMetrics(*statusEvery, "[aggregate]:"),
}
r.AddConcurrentHandlers(handler, *numPublishers)
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQD(addrString)
if err != nil {
log.Fatalf("%s", err)
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToNSQLookupd(addrString)
if err != nil {
log.Fatalf("%s", err)
}
}
for {
select {
case <-r.StopChan:
return
case <-termChan:
r.Stop()
}
}
}
func main() {
flag.Parse()
if *showVersion {
fmt.Printf("nsq_tail v%s\n", util.BINARY_VERSION)
return
}
if *channel == "" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
}
if *topic == "" {
log.Fatalf("--topic is required")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
// Don't ask for more messages than we want
if *totalMessages > 0 && *totalMessages < *maxInFlight {
*maxInFlight = *totalMessages
}
cfg := nsq.NewConfig()
cfg.MaxInFlight = *maxInFlight
err := util.ParseReaderOpts(cfg, readerOpts)
if err != nil {
log.Fatalf(err.Error())
}
r, err := nsq.NewConsumer(*topic, *channel, cfg)
if err != nil {
log.Fatalf(err.Error())
}
r.SetLogger(log.New(os.Stderr, "", log.LstdFlags), nsq.LogLevelInfo)
r.SetHandler(&TailHandler{totalMessages: *totalMessages})
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQD(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToNSQLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for {
select {
case <-r.StopChan:
return
case <-sigChan:
r.Stop()
}
}
}
func main() {
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_file v%s\n", util.BINARY_VERSION)
return
}
if *topic == "" || *channel == "" {
log.Fatalf("--topic and --channel are required")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required.")
}
if len(nsqdTCPAddrs) != 0 && len(lookupdHTTPAddrs) != 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if *gzipLevel < 1 || *gzipLevel > 9 {
log.Fatalf("invalid --gzip-level value (%d), should be 1-9", *gzipLevel)
}
// TODO: remove, deprecated
if hasArg("gzip-compression") {
log.Printf("WARNING: --gzip-compression is deprecated in favor of --gzip-level")
switch *gzipCompression {
case 1:
*gzipLevel = gzip.BestSpeed
case 2:
*gzipLevel = gzip.BestCompression
case 3:
*gzipLevel = gzip.DefaultCompression
default:
log.Fatalf("invalid --gzip-compression value (%d), should be 1,2,3", *gzipCompression)
}
}
hupChan := make(chan os.Signal, 1)
termChan := make(chan os.Signal, 1)
signal.Notify(hupChan, syscall.SIGHUP)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
f, err := NewFileLogger(*gzipEnabled, *gzipLevel, *filenameFormat)
if err != nil {
log.Fatal(err.Error())
}
r, err := nsq.NewReader(*topic, *channel)
if err != nil {
log.Fatalf(err.Error())
}
err = util.ParseReaderOpts(r, readerOpts)
if err != nil {
log.Fatalf(err.Error())
}
r.SetMaxInFlight(*maxInFlight)
r.AddAsyncHandler(f)
// TODO: remove, deprecated
if hasArg("verbose") {
log.Printf("WARNING: --verbose is deprecated in favor of --reader-opt=verbose")
r.Configure("verbose", true)
}
go f.router(r, termChan, hupChan)
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQ(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
<-f.ExitChan
}
func main() {
var publisher Publisher
var addresses util.StringArray
var selectedMode int
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_http v%s\n", util.BINARY_VERSION)
return
}
if *topic == "" || *channel == "" {
log.Fatalf("--topic and --channel are required")
}
if *contentType != flag.Lookup("content-type").DefValue {
if len(postAddrs) == 0 {
log.Fatalf("--content-type only used with --post")
}
if len(*contentType) == 0 {
log.Fatalf("--content-type requires a value when used")
}
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(getAddrs) == 0 && len(postAddrs) == 0 {
log.Fatalf("--get or --post required")
}
if len(getAddrs) > 0 && len(postAddrs) > 0 {
log.Fatalf("use --get or --post not both")
}
if len(getAddrs) > 0 {
for _, get := range getAddrs {
if strings.Count(get, "%s") != 1 {
log.Fatal("invalid GET address - must be a printf string")
}
}
}
switch *mode {
case "multicast":
log.Printf("WARNING: multicast mode is deprecated in favor of using separate nsq_to_http on different channels (and will be dropped in a future release)")
selectedMode = ModeAll
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool":
selectedMode = ModeHostPool
}
// TODO: remove, deprecated
if hasArg("--round-robin") {
log.Printf("WARNING: --round-robin is deprecated in favor of --mode=round-robin")
selectedMode = ModeRoundRobin
}
// TODO: remove, deprecated
if hasArg("throttle-fraction") {
log.Printf("WARNING: --throttle-fraction is deprecatedin favor of --sample=X")
*sample = *throttleFraction
}
if *sample > 1.0 || *sample < 0.0 {
log.Fatalf("ERROR: --sample must be between 0.0 and 1.0")
}
// TODO: remove, deprecated
if hasArg("http-timeout-ms") {
log.Printf("WARNING: --http-timeout-ms is deprecated in favor of --http-timeout=X")
*httpTimeout = time.Duration(*httpTimeoutMs) * time.Millisecond
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
if len(postAddrs) > 0 {
publisher = &PostPublisher{}
addresses = postAddrs
} else {
publisher = &GetPublisher{}
addresses = getAddrs
}
r, err := nsq.NewReader(*topic, *channel)
if err != nil {
log.Fatalf(err.Error())
}
err = util.ParseReaderOpts(r, readerOpts)
if err != nil {
log.Fatalf(err.Error())
}
r.SetMaxInFlight(*maxInFlight)
// TODO: remove, deprecated
if hasArg("verbose") {
log.Printf("WARNING: --verbose is deprecated in favor of --reader-opt=verbose")
r.Configure("verbose", true)
}
// TODO: remove, deprecated
if hasArg("max-backoff-duration") {
log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --reader-opt=max_backoff_duration=X")
r.Configure("max_backoff_duration", *maxBackoffDuration)
}
for i := 0; i < *numPublishers; i++ {
handler := &PublishHandler{
Publisher: publisher,
addresses: addresses,
mode: selectedMode,
reqs: make(Durations, 0, *statusEvery),
id: i,
hostPool: hostpool.New(addresses),
}
r.AddHandler(handler)
}
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQ(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for {
select {
case <-r.ExitChan:
return
case <-termChan:
r.Stop()
}
}
}
func main() {
var selectedMode int
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_nsq v%s\n", util.BINARY_VERSION)
return
}
if *topic == "" || *channel == "" {
log.Fatalf("--topic and --channel are required")
}
if *destTopic == "" {
*destTopic = *topic
}
if !nsq.IsValidTopicName(*topic) {
log.Fatalf("--topic is invalid")
}
if !nsq.IsValidTopicName(*destTopic) {
log.Fatalf("--destination-topic is invalid")
}
if !nsq.IsValidChannelName(*channel) {
log.Fatalf("--channel is invalid")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(destNsqdTCPAddrs) == 0 {
log.Fatalf("--destination-nsqd-tcp-address required")
}
switch *mode {
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool":
selectedMode = ModeHostPool
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
r, err := nsq.NewReader(*topic, *channel)
if err != nil {
log.Fatalf(err.Error())
}
err = util.ParseReaderOpts(r, readerOpts)
if err != nil {
log.Fatalf(err.Error())
}
r.SetMaxInFlight(*maxInFlight)
// TODO: remove, deprecated
if hasArg("verbose") {
log.Printf("WARNING: --verbose is deprecated in favor of --reader-opt=verbose")
r.Configure("verbose", true)
}
// TODO: remove, deprecated
if hasArg("max-backoff-duration") {
log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --reader-opt=max_backoff_duration=X")
r.Configure("max_backoff_duration", *maxBackoffDuration)
}
writers := make(map[string]*nsq.Writer)
for _, addr := range destNsqdTCPAddrs {
writer := nsq.NewWriter(addr)
writer.HeartbeatInterval = nsq.DefaultClientTimeout / 2
writers[addr] = writer
}
for i := 0; i < len(destNsqdTCPAddrs); i++ {
respChan := make(chan *nsq.WriterTransaction)
handler := &PublishHandler{
addresses: destNsqdTCPAddrs,
writers: writers,
mode: selectedMode,
reqs: make(Durations, 0, *statusEvery),
id: i,
hostPool: hostpool.New(destNsqdTCPAddrs),
respChan: respChan,
}
r.AddAsyncHandler(handler)
go handler.responder()
}
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQ(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for {
select {
case <-r.ExitChan:
return
case <-termChan:
r.Stop()
}
}
}
func main() {
flag.Parse()
if *showVersion {
fmt.Printf("nsq_tail v%s\n", util.BINARY_VERSION)
return
}
if *channel == "" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
}
if *topic == "" {
log.Fatalf("--topic is required")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
r, err := nsq.NewReader(*topic, *channel)
if err != nil {
log.Fatalf(err.Error())
}
err = util.ParseReaderOpts(r, readerOpts)
if err != nil {
log.Fatalf(err.Error())
}
r.SetMaxInFlight(*maxInFlight)
r.AddHandler(&TailHandler{})
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQ(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for {
select {
case <-r.ExitChan:
return
case <-sigChan:
r.Stop()
}
}
}
func main() {
var selectedMode int
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_nsq v%s\n", util.BINARY_VERSION)
return
}
if *topic == "" || *channel == "" {
log.Fatalf("--topic and --channel are required")
}
if *destTopic == "" {
*destTopic = *topic
}
if !util.IsValidTopicName(*topic) {
log.Fatalf("--topic is invalid")
}
if !util.IsValidTopicName(*destTopic) {
log.Fatalf("--destination-topic is invalid")
}
if !util.IsValidChannelName(*channel) {
log.Fatalf("--channel is invalid")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(destNsqdTCPAddrs) == 0 {
log.Fatalf("--destination-nsqd-tcp-address required")
}
switch *mode {
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool":
selectedMode = ModeHostPool
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
cfg := nsq.NewConfig()
cfg.UserAgent = fmt.Sprintf("nsq_to_nsq/%s go-nsq/%s", util.BINARY_VERSION, nsq.VERSION)
err := util.ParseReaderOpts(cfg, readerOpts)
if err != nil {
log.Fatalf(err.Error())
}
cfg.MaxInFlight = *maxInFlight
// TODO: remove, deprecated
if hasArg("max-backoff-duration") {
log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --reader-opt=max_backoff_duration=X")
cfg.MaxBackoffDuration = *maxBackoffDuration
}
r, err := nsq.NewConsumer(*topic, *channel, cfg)
if err != nil {
log.Fatalf("%s", err)
}
r.SetLogger(log.New(os.Stderr, "", log.LstdFlags), nsq.LogLevelInfo)
wcfg := nsq.NewConfig()
cfg.UserAgent = fmt.Sprintf("nsq_to_nsq/%s go-nsq/%s", util.BINARY_VERSION, nsq.VERSION)
producers := make(map[string]*nsq.Producer)
for _, addr := range destNsqdTCPAddrs {
producer, err := nsq.NewProducer(addr, wcfg)
if err != nil {
log.Fatalf("failed creating producer %s", err)
}
producers[addr] = producer
}
perAddressStatus := make(map[string]*timermetrics.TimerMetrics)
if len(destNsqdTCPAddrs) == 1 {
// disable since there is only one address
perAddressStatus[destNsqdTCPAddrs[0]] = timermetrics.NewTimerMetrics(0, "")
} else {
for _, a := range destNsqdTCPAddrs {
perAddressStatus[a] = timermetrics.NewTimerMetrics(*statusEvery,
fmt.Sprintf("[%s]:", a))
}
}
handler := &PublishHandler{
addresses: destNsqdTCPAddrs,
producers: producers,
mode: selectedMode,
hostPool: hostpool.New(destNsqdTCPAddrs),
respChan: make(chan *nsq.ProducerTransaction, len(destNsqdTCPAddrs)),
perAddressStatus: perAddressStatus,
timermetrics: timermetrics.NewTimerMetrics(*statusEvery, "[aggregate]:"),
}
r.AddConcurrentHandlers(handler, len(destNsqdTCPAddrs))
for i := 0; i < len(destNsqdTCPAddrs); i++ {
go handler.responder()
}
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQD(addrString)
if err != nil {
log.Fatalf("%s", err)
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToNSQLookupd(addrString)
if err != nil {
log.Fatalf("%s", err)
}
}
for {
select {
case <-r.StopChan:
return
case <-termChan:
r.Stop()
}
}
}
