func createTestReaderOrFail(t *testing.T, handleMsg func(message *nsq.Message, finish chan *nsq.FinishedMessage)) *CallbackHandler {
addr := "127.0.0.1:4150"
q, err := nsq.NewReader(test_topic, "ch")
if err != nil {
t.Fatalf("couldn't create reader: %v", err.Error())
}
q.VerboseLogging = true
handler := &CallbackHandler{
t: t,
q: q,
handleMsg: handleMsg,
}
q.AddAsyncHandler(handler)
err = q.ConnectToNSQ(addr)
if err != nil {
t.Fatal(err.Error())
}
return handler
}
// MustReader calls nsq.NewReader and panics if nsq.NewReader
// returned an error. nsq.NewReader only fails on invalid
// topic and channel names.
func MustReader(topic, channel string) *nsq.Reader {
r, err := nsq.NewReader(topic, channel)
if err != nil {
panic(err)
}
return r
}
// AttachHandler creates a new nsq.Reader for a topic and attaches the
// given handler to it.
func AttachHandler(topic, channel string, lookupd string, handler nsq.Handler) error {
mountReader, err := nsq.NewReader(topic, channel)
if err != nil {
return err
}
mountReader.AddHandler(handler)
return mountReader.ConnectToLookupd(lookupd)
}
func main() {
flag.Parse()
if *showVersion {
fmt.Printf("nsq_tail v%s\n", util.BINARY_VERSION)
return
}
if *topic == "" || *channel == "" {
log.Fatalf("--topic and --channel are required")
}
if *maxInFlight < 0 {
log.Fatalf("--max-in-flight must be > 0")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
r, err := nsq.NewReader(*topic, *channel)
if err != nil {
log.Fatalf(err.Error())
}
r.SetMaxInFlight(*maxInFlight)
r.AddHandler(&TailHandler{})
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQ(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for {
select {
case <-r.ExitChan:
return
case <-sigChan:
r.Stop()
}
}
}
func main() {
flag.Parse()
if *topic == "" || *channel == "" {
log.Fatalf("--topic and --channel are required")
}
if *maxInFlight < 0 {
log.Fatalf("--max-in-flight must be > 0")
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
r, err := nsq.NewReader(*topic, *channel)
if err != nil {
log.Fatalf(err.Error())
}
r.SetMaxInFlight(*maxInFlight)
msg_handler := MsgHandler{make(chan Msg)}
r.AddHandler(&msg_handler)
lookupdlist := strings.Split(*lookupdHTTPAddrs, ",")
exitchan := make(chan int)
go tcp_server(*port, msg_handler.msg_chan, exitchan)
for _, addrString := range lookupdlist {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
select {
case <-r.ExitChan:
log.Println("reader exited")
case <-sigChan:
r.Stop()
exitchan <- 1
log.Println("stop all")
}
time.Sleep(time.Second)
}
func main() {
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_file v%s\n", util.BINARY_VERSION)
return
}
if *topic == "" || *channel == "" {
log.Fatalf("--topic and --channel are required")
}
if *maxInFlight < 0 {
log.Fatalf("--max-in-flight must be > 0")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required.")
}
if len(nsqdTCPAddrs) != 0 && len(lookupdHTTPAddrs) != 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if *gzipCompression < 1 || *gzipCompression > 3 {
log.Fatalf("invalid --gzip-compresion value (%v). should be 1,2 or 3", *gzipCompression)
}
hupChan := make(chan os.Signal, 1)
termChan := make(chan os.Signal, 1)
signal.Notify(hupChan, syscall.SIGHUP)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
f, err := NewFileLogger(*gzipEnabled, *gzipCompression, *filenameFormat)
if err != nil {
log.Fatal(err.Error())
}
r, err := nsq.NewReader(*topic, *channel)
if err != nil {
log.Fatalf(err.Error())
}
r.SetMaxInFlight(*maxInFlight)
r.VerboseLogging = *verbose
if *tlsEnabled {
r.TLSv1 = true
r.TLSConfig = &tls.Config{
InsecureSkipVerify: *tlsInsecureSkipVerify,
}
}
r.AddAsyncHandler(f)
go f.router(r, termChan, hupChan)
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQ(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
<-f.ExitChan
}
func main() {
flag.Parse()
if *showVersion {
fmt.Printf("nsq_tail v%s\n", util.BINARY_VERSION)
return
}
if *channel == "" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
}
if *topic == "" {
log.Fatalf("--topic is required")
}
if *maxInFlight < 0 {
log.Fatalf("--max-in-flight must be > 0")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
r, err := nsq.NewReader(*topic, *channel)
if err != nil {
log.Fatalf(err.Error())
}
r.SetMaxInFlight(*maxInFlight)
r.AddHandler(&TailHandler{})
if *tlsEnabled {
r.TLSv1 = true
r.TLSConfig = &tls.Config{
InsecureSkipVerify: *tlsInsecureSkipVerify,
}
}
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQ(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for {
select {
case <-r.ExitChan:
return
case <-sigChan:
r.Stop()
}
}
}
func main() {
var publisher Publisher
var addresses util.StringArray
var mode int
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_http v%s\n", util.BINARY_VERSION)
return
}
if *topic == "" || *channel == "" {
log.Fatalf("--topic and --channel are required")
}
if *maxInFlight < 0 {
log.Fatalf("--max-in-flight must be > 0")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(getAddrs) == 0 && len(postAddrs) == 0 {
log.Fatalf("--get or --post required")
}
if len(getAddrs) > 0 && len(postAddrs) > 0 {
log.Fatalf("use --get or --post not both")
}
if len(getAddrs) > 0 {
for _, get := range getAddrs {
if strings.Count(get, "%s") != 1 {
log.Fatal("invalid GET address - must be a printf string")
}
}
}
if *roundRobin {
mode = ModeRoundRobin
}
if *throttleFraction > 1.0 || *throttleFraction < 0.0 {
log.Fatalf("Throttle fraction must be between 0.0 and 1.0")
}
hupChan := make(chan os.Signal, 1)
termChan := make(chan os.Signal, 1)
signal.Notify(hupChan, syscall.SIGHUP)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
if len(postAddrs) > 0 {
publisher = &PostPublisher{}
addresses = postAddrs
} else {
publisher = &GetPublisher{}
addresses = getAddrs
}
r, err := nsq.NewReader(*topic, *channel)
if err != nil {
log.Fatalf(err.Error())
}
r.SetMaxInFlight(*maxInFlight)
r.VerboseLogging = *verbose
for i := 0; i < *numPublishers; i++ {
handler := &PublishHandler{
Publisher: publisher,
addresses: addresses,
mode: mode,
reqs: make(Durations, 0, *statusEvery),
id: i,
}
r.AddHandler(handler)
}
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQ(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
select {
case <-r.ExitChan:
case <-hupChan:
r.Stop()
case <-termChan:
r.Stop()
}
}
func (s *StreamServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
path := req.URL.Path
if path == "/stats" {
StatsHandler(w, req)
return
}
if path != "/sub" {
w.WriteHeader(404)
return
}
reqParams, err := util.NewReqParams(req)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
topicName, channelName, err := util.GetTopicChannelArgs(reqParams)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
hj, ok := w.(http.Hijacker)
if !ok {
http.Error(w, "httpserver doesn't support hijacking", http.StatusInternalServerError)
return
}
conn, bufrw, err := hj.Hijack()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
r, err := nsq.NewReader(topicName, channelName)
r.SetMaxInFlight(*maxInFlight)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
sr := &StreamReader{
topic: topicName,
channel: channelName,
reader: r,
req: req,
conn: conn,
bufrw: bufrw, // TODO: latency writer
connectTime: time.Now(),
}
s.Set(sr)
log.Printf("[%s] new connection", conn.RemoteAddr().String())
bufrw.WriteString("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n")
bufrw.Flush()
r.AddHandler(sr)
// TODO: handle the error cases better (ie. at all :) )
errors := ConnectToNSQAndLookupd(r, nsqdTCPAddrs, lookupdHTTPAddrs)
log.Printf("connected to NSQ %v", errors)
// this read allows us to detect clients that disconnect
go func(rw *bufio.ReadWriter) {
b, err := rw.ReadByte()
if err != nil {
log.Printf("got connection err %s", err.Error())
} else {
log.Printf("unexpected data on request socket (%s); closing", b)
}
sr.reader.Stop()
}(bufrw)
go sr.HeartbeatLoop()
}
func main() {
var publisher Publisher
var addresses util.StringArray
var selectedMode int
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_http v%s\n", util.BINARY_VERSION)
return
}
if *topic == "" || *channel == "" {
log.Fatalf("--topic and --channel are required")
}
if *maxInFlight < 0 {
log.Fatalf("--max-in-flight must be > 0")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(getAddrs) == 0 && len(postAddrs) == 0 {
log.Fatalf("--get or --post required")
}
if len(getAddrs) > 0 && len(postAddrs) > 0 {
log.Fatalf("use --get or --post not both")
}
if len(getAddrs) > 0 {
for _, get := range getAddrs {
if strings.Count(get, "%s") != 1 {
log.Fatal("invalid GET address - must be a printf string")
}
}
}
if *roundRobin {
log.Printf("WARNING: the use of the --round-robin flag is deprecated in favor of --mode=round-robin (and will be dropped in a future release)")
selectedMode = ModeRoundRobin
}
if *roundRobin && *mode != "" {
log.Fatalf("ERROR: cannot use both --round-robin and --mode flags")
}
switch *mode {
case "multicast":
log.Printf("WARNING: multicast mode is deprecated in favor of using separate nsq_to_http on different channels (and will be dropped in a future release)")
selectedMode = ModeAll
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool":
selectedMode = ModeHostPool
}
if *throttleFraction > 1.0 || *throttleFraction < 0.0 {
log.Fatalf("ERROR: --throttle-fraction must be between 0.0 and 1.0")
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
if len(postAddrs) > 0 {
publisher = &PostPublisher{}
addresses = postAddrs
} else {
publisher = &GetPublisher{}
addresses = getAddrs
}
r, err := nsq.NewReader(*topic, *channel)
if err != nil {
log.Fatalf(err.Error())
}
r.SetMaxInFlight(*maxInFlight)
r.SetMaxBackoffDuration(*maxBackoffDuration)
r.VerboseLogging = *verbose
for i := 0; i < *numPublishers; i++ {
handler := &PublishHandler{
Publisher: publisher,
addresses: addresses,
mode: selectedMode,
reqs: make(Durations, 0, *statusEvery),
id: i,
hostPool: hostpool.New(addresses),
}
r.AddHandler(handler)
}
for _, addrString := range nsqdTCPAddrs {
err := r.ConnectToNSQ(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
for _, addrString := range lookupdHTTPAddrs {
log.Printf("lookupd addr %s", addrString)
err := r.ConnectToLookupd(addrString)
if err != nil {
log.Fatalf(err.Error())
}
}
select {
case <-r.ExitChan:
case <-termChan:
}
r.Stop()
}
//read log from nsq
func read_nsq(lookupd_addr string, topic string, logchan string, lreader *logreader) {
reader, _ := nsq.NewReader(topic, logchan)
reader.AddAsyncHandler(lreader)
reader.ConnectToLookupd(lookupd_addr)
}