func main() { flagSet.Parse(os.Args[1:]) if *showVersion { fmt.Println(util.Version("nsqlookupd")) return } signalChan := make(chan os.Signal, 1) exitChan := make(chan int) go func() { <-signalChan exitChan <- 1 }() signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) var cfg map[string]interface{} if *config != "" { _, err := toml.DecodeFile(*config, &cfg) if err != nil { log.Fatalf("ERROR: failed to load config file %s - %s", *config, err.Error()) } } options := nsqlookupd.NewNSQLookupdOptions() util.ResolveOptions(options, flagSet, cfg) daemon := nsqlookupd.NewNSQLookupd(options) log.Println(util.Version("nsqlookupd")) daemon.Main() <-exitChan daemon.Exit() }
func (p *program) Start(s service.Service) error { flagSet.Parse(os.Args[1:]) if *showVersion { fmt.Println(util.Version("nsqlookupd")) os.Exit(0) return nil } var cfg map[string]interface{} if *config != "" { _, err := toml.DecodeFile(*config, &cfg) if err != nil { log.Fatalf("ERROR: failed to load config file %s - %s", *config, err.Error()) } } opts := nsqlookupd.NewNSQLookupdOptions() options.Resolve(opts, flagSet, cfg) p.daemon = nsqlookupd.NewNSQLookupd(opts) p.daemon.Main() return nil }
func main() { flagSet.Parse(os.Args[1:]) rand.Seed(time.Now().UTC().UnixNano()) if *showVersion { fmt.Println(util.Version("nsqd")) return } signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) var cfg map[string]interface{} if *config != "" { _, err := toml.DecodeFile(*config, &cfg) if err != nil { log.Fatalf("ERROR: failed to load config file %s - %s", *config, err.Error()) } } opts := nsqd.NewNSQDOptions() options.Resolve(opts, flagSet, cfg) nsqd := nsqd.NewNSQD(opts) nsqd.LoadMetadata() err := nsqd.PersistMetadata() if err != nil { log.Fatalf("ERROR: failed to persist metadata - %s", err.Error()) } nsqd.Main() <-signalChan nsqd.Exit() }
func (s *httpServer) doTextStats(stats []TopicStats, w http.ResponseWriter) { now := time.Now() io.WriteString(w, fmt.Sprintf("%s\n", util.Version("nsqd"))) if len(stats) == 0 { io.WriteString(w, "\nNO_TOPICS\n") return } for _, t := range stats { var pausedPrefix string if t.Paused { pausedPrefix = "*P " } else { pausedPrefix = " " } io.WriteString(w, fmt.Sprintf("\n%s[%-15s] depth: %-5d be-depth: %-5d msgs: %-8d e2e%%: %s\n", pausedPrefix, t.TopicName, t.Depth, t.BackendDepth, t.MessageCount, t.E2eProcessingLatency)) for _, c := range t.Channels { if c.Paused { pausedPrefix = " *P " } else { pausedPrefix = " " } io.WriteString(w, fmt.Sprintf("%s[%-25s] depth: %-5d be-depth: %-5d inflt: %-4d def: %-4d re-q: %-5d timeout: %-5d msgs: %-8d e2e%%: %s\n", pausedPrefix, c.ChannelName, c.Depth, c.BackendDepth, c.InFlightCount, c.DeferredCount, c.RequeueCount, c.TimeoutCount, c.MessageCount, c.E2eProcessingLatency)) for _, client := range c.Clients { connectTime := time.Unix(client.ConnectTime, 0) // truncate to the second duration := time.Duration(int64(now.Sub(connectTime).Seconds())) * time.Second _, port, _ := net.SplitHostPort(client.RemoteAddress) io.WriteString(w, fmt.Sprintf(" [%s %-21s] state: %d inflt: %-4d rdy: %-4d fin: %-8d re-q: %-8d msgs: %-8d connected: %s\n", client.Version, fmt.Sprintf("%s:%s", client.Name, port), client.State, client.InFlightCount, client.ReadyCount, client.FinishCount, client.RequeueCount, client.MessageCount, duration, )) } } } }
func main() { flag.Parse() if *broadcastAddress == "" { hostname, err := os.Hostname() if err != nil { log.Fatal(err) } *broadcastAddress = hostname } if *showVersion { fmt.Println(util.Version("nsqlookupd")) return } signalChan := make(chan os.Signal, 1) exitChan := make(chan int) go func() { <-signalChan exitChan <- 1 }() signal.Notify(signalChan, os.Interrupt) tcpAddr, err := net.ResolveTCPAddr("tcp", *tcpAddress) if err != nil { log.Fatal(err) } httpAddr, err := net.ResolveTCPAddr("tcp", *httpAddress) if err != nil { log.Fatal(err) } log.Println(util.Version("nsqlookupd")) nsqlookupd := NewNSQLookupd() nsqlookupd.tcpAddr = tcpAddr nsqlookupd.httpAddr = httpAddr nsqlookupd.broadcastAddress = *broadcastAddress nsqlookupd.inactiveProducerTimeout = *inactiveProducerTimeout nsqlookupd.tombstoneLifetime = *tombstoneLifetime nsqlookupd.Main() <-exitChan nsqlookupd.Exit() }
func main() { flagSet.Parse(os.Args[1:]) if *showVersion { fmt.Println(util.Version("nsqadmin")) return } if *templateDir != "" { log.Printf("WARNING: --template-dir is deprecated and will be removed in the next release (templates are now compiled into the binary)") } exitChan := make(chan int) signalChan := make(chan os.Signal, 1) go func() { <-signalChan exitChan <- 1 }() signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) var cfg map[string]interface{} if *config != "" { _, err := toml.DecodeFile(*config, &cfg) if err != nil { log.Fatalf("ERROR: failed to load config file %s - %s", *config, err.Error()) } } opts := NewNSQAdminOptions() options.Resolve(opts, flagSet, cfg) nsqadmin := NewNSQAdmin(opts) log.Println(util.Version("nsqadmin")) nsqadmin.Main() <-exitChan nsqadmin.Exit() }
func NewNSQLookupd(opts *nsqlookupdOptions) *NSQLookupd { tcpAddr, err := net.ResolveTCPAddr("tcp", opts.TCPAddress) if err != nil { log.Fatal(err) } httpAddr, err := net.ResolveTCPAddr("tcp", opts.HTTPAddress) if err != nil { log.Fatal(err) } n := &NSQLookupd{ opts: opts, tcpAddr: tcpAddr, httpAddr: httpAddr, DB: NewRegistrationDB(), } n.opts.Logger.Output(2, util.Version("nsqlookupd")) return n }
func NewNSQLookupd(opts *nsqlookupdOptions) *NSQLookupd { n := &NSQLookupd{ opts: opts, DB: NewRegistrationDB(), } tcpAddr, err := net.ResolveTCPAddr("tcp", opts.TCPAddress) if err != nil { n.logf("FATAL: failed to resolve TCP address (%s) - %s", opts.TCPAddress, err) os.Exit(1) } n.tcpAddr = tcpAddr httpAddr, err := net.ResolveTCPAddr("tcp", opts.HTTPAddress) if err != nil { n.logf("FATAL: failed to resolve HTTP address (%s) - %s", opts.HTTPAddress, err) os.Exit(1) } n.httpAddr = httpAddr n.logf(util.Version("nsqlookupd")) return n }
func NewNSQAdmin(opts *nsqadminOptions) *NSQAdmin { if len(opts.NSQDHTTPAddresses) == 0 && len(opts.NSQLookupdHTTPAddresses) == 0 { log.Fatalf("--nsqd-http-address or --lookupd-http-address required.") } if len(opts.NSQDHTTPAddresses) != 0 && len(opts.NSQLookupdHTTPAddresses) != 0 { log.Fatalf("use --nsqd-http-address or --lookupd-http-address not both") } httpAddr, err := net.ResolveTCPAddr("tcp", opts.HTTPAddress) if err != nil { log.Fatal(err) } n := &NSQAdmin{ opts: opts, httpAddr: httpAddr, notifications: make(chan *AdminAction), } n.opts.Logger.Output(2, util.Version("nsqlookupd")) return n }
func NewNSQAdmin(opts *nsqadminOptions) *NSQAdmin { n := &NSQAdmin{ opts: opts, notifications: make(chan *AdminAction), } if len(opts.NSQDHTTPAddresses) == 0 && len(opts.NSQLookupdHTTPAddresses) == 0 { n.logf("--nsqd-http-address or --lookupd-http-address required.") os.Exit(1) } if len(opts.NSQDHTTPAddresses) != 0 && len(opts.NSQLookupdHTTPAddresses) != 0 { n.logf("use --nsqd-http-address or --lookupd-http-address not both") os.Exit(1) } httpAddr, err := net.ResolveTCPAddr("tcp", opts.HTTPAddress) if err != nil { n.logf("FATAL: failed to resolve HTTP address (%s) - %s", opts.HTTPAddress, err) os.Exit(1) } n.httpAddr = httpAddr if opts.ProxyGraphite { url, err := url.Parse(opts.GraphiteURL) if err != nil { n.logf("FATAL: failed to parse --graphite-url='%s' - %s", opts.GraphiteURL, err) os.Exit(1) } n.graphiteURL = url } n.logf(util.Version("nsqlookupd")) return n }
func main() { flag.Parse() hostname, err := os.Hostname() if err != nil { log.Fatal(err) } if *showVersion { fmt.Println(util.Version("nsqd")) return } if *workerId == 0 { h := md5.New() io.WriteString(h, hostname) *workerId = int64(crc32.ChecksumIEEE(h.Sum(nil)) % 1024) } tcpAddr, err := net.ResolveTCPAddr("tcp", *tcpAddress) if err != nil { log.Fatal(err) } httpAddr, err := net.ResolveTCPAddr("tcp", *httpAddress) if err != nil { log.Fatal(err) } if *broadcastAddress == "" { *broadcastAddress = hostname } log.Println(util.Version("nsqd")) log.Printf("worker id %d", *workerId) exitChan := make(chan int) signalChan := make(chan os.Signal, 1) go func() { <-signalChan exitChan <- 1 }() signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) if *statsdAddress != "" { underHostname := fmt.Sprintf("%s_%d", strings.Replace(hostname, ".", "_", -1), httpAddr.Port) prefix := fmt.Sprintf("nsq.%s.", underHostname) go statsdLoop(*statsdAddress, prefix, *statsdInterval) } // for backwards compatibility if --msg-timeout only // contains numbers then default to ms var msgTimeoutDuration time.Duration if regexp.MustCompile(`^[0-9]+$`).MatchString(*msgTimeout) { intMsgTimeout, err := strconv.Atoi(*msgTimeout) if err != nil { log.Fatalf("ERROR: failed to Atoi --msg-timeout %s - %s", *msgTimeout, err.Error()) } msgTimeoutDuration = time.Duration(intMsgTimeout) * time.Millisecond } else { msgTimeoutDuration, err = time.ParseDuration(*msgTimeout) if err != nil { log.Fatalf("ERROR: failed to ParseDuration --msg-timeout %s - %s", *msgTimeout, err.Error()) } } options := NewNsqdOptions() options.maxRdyCount = *maxRdyCount options.maxMessageSize = *maxMessageSize options.maxBodySize = *maxBodySize options.memQueueSize = *memQueueSize options.dataPath = *dataPath options.maxBytesPerFile = *maxBytesPerFile options.syncEvery = *syncEvery options.msgTimeout = msgTimeoutDuration options.maxMsgTimeout = *maxMsgTimeout options.broadcastAddress = *broadcastAddress options.maxHeartbeatInterval = *maxHeartbeatInterval nsqd = NewNSQd(*workerId, options) nsqd.tcpAddr = tcpAddr nsqd.httpAddr = httpAddr nsqd.lookupdTCPAddrs = lookupdTCPAddrs nsqd.LoadMetadata() err = nsqd.PersistMetadata() if err != nil { log.Fatalf("ERROR: failed to persist metadata - %s", err.Error()) } nsqd.Main() <-exitChan nsqd.Exit() }
func (s *httpServer) statsHandler(w http.ResponseWriter, req *http.Request) { reqParams, err := util.NewReqParams(req) if err != nil { log.Printf("ERROR: failed to parse request params - %s", err.Error()) util.ApiResponse(w, 500, "INVALID_REQUEST", nil) return } formatString, _ := reqParams.Get("format") jsonFormat := formatString == "json" now := time.Now() if !jsonFormat { io.WriteString(w, fmt.Sprintf("%s\n", util.Version("nsqd"))) } stats := s.context.nsqd.getStats() if jsonFormat { util.ApiResponse(w, 200, "OK", struct { Topics []TopicStats `json:"topics"` }{stats}) } else { if len(stats) == 0 { io.WriteString(w, "\nNO_TOPICS\n") return } for _, t := range stats { io.WriteString(w, fmt.Sprintf("\n[%-15s] depth: %-5d be-depth: %-5d msgs: %-8d\n", t.TopicName, t.Depth, t.BackendDepth, t.MessageCount)) for _, c := range t.Channels { var pausedPrefix string if c.Paused { pausedPrefix = " *P " } else { pausedPrefix = " " } io.WriteString(w, fmt.Sprintf("%s[%-25s] depth: %-5d be-depth: %-5d inflt: %-4d def: %-4d re-q: %-5d timeout: %-5d msgs: %-8d\n", pausedPrefix, c.ChannelName, c.Depth, c.BackendDepth, c.InFlightCount, c.DeferredCount, c.RequeueCount, c.TimeoutCount, c.MessageCount)) for _, client := range c.Clients { connectTime := time.Unix(client.ConnectTime, 0) // truncate to the second duration := time.Duration(int64(now.Sub(connectTime).Seconds())) * time.Second _, port, _ := net.SplitHostPort(client.RemoteAddress) io.WriteString(w, fmt.Sprintf(" [%s %-21s] state: %d inflt: %-4d rdy: %-4d fin: %-8d re-q: %-8d msgs: %-8d connected: %s\n", client.Version, fmt.Sprintf("%s:%s", client.Name, port), client.State, client.InFlightCount, client.ReadyCount, client.FinishCount, client.RequeueCount, client.MessageCount, duration, )) } } } } }
func main() { var waitGroup util.WaitGroupWrapper flag.Parse() if *showVersion { fmt.Println(util.Version("nsqadmin")) return } if *templateDir == "" { for _, defaultPath := range []string{"templates", "/usr/local/share/nsqadmin/templates"} { if info, err := os.Stat(defaultPath); err == nil && info.IsDir() { *templateDir = defaultPath break } } } if *templateDir == "" { log.Fatalf("--template-dir must be specified (or install the templates to /usr/local/share/nsqadmin/templates)") } if len(nsqdHTTPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 { log.Fatalf("--nsqd-http-address or --lookupd-http-address required.") } if len(nsqdHTTPAddrs) != 0 && len(lookupdHTTPAddrs) != 0 { log.Fatalf("use --nsqd-http-address or --lookupd-http-address not both") } log.Println(util.Version("nsqadmin")) exitChan := make(chan int) signalChan := make(chan os.Signal, 1) go func() { <-signalChan exitChan <- 1 }() signal.Notify(signalChan, os.Interrupt) httpAddr, err := net.ResolveTCPAddr("tcp", *httpAddress) if err != nil { log.Fatal(err) } httpListener, err := net.Listen("tcp", httpAddr.String()) if err != nil { log.Fatalf("FATAL: listen (%s) failed - %s", httpAddr, err.Error()) } waitGroup.Wrap(func() { httpServer(httpListener) }) notifications = make(chan *AdminAction) waitGroup.Wrap(func() { HandleAdminActions() }) <-exitChan httpListener.Close() close(notifications) waitGroup.Wait() }
func main() { flag.Parse() hostname, err := os.Hostname() if err != nil { log.Fatal(err) } if *showVersion { fmt.Println(util.Version("nsqd")) return } if *workerId == 0 { h := md5.New() io.WriteString(h, hostname) *workerId = int64(crc32.ChecksumIEEE(h.Sum(nil)) % 1024) } tcpAddr, err := net.ResolveTCPAddr("tcp", *tcpAddress) if err != nil { log.Fatal(err) } httpAddr, err := net.ResolveTCPAddr("tcp", *httpAddress) if err != nil { log.Fatal(err) } if *broadcastAddress == "" { *broadcastAddress = hostname } log.Println(util.Version("nsqd")) log.Printf("worker id %d", *workerId) exitChan := make(chan int) signalChan := make(chan os.Signal, 1) go func() { <-signalChan exitChan <- 1 }() signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) // flagToDuration will fatally error if it is invalid msgTimeoutDuration := flagToDuration(*msgTimeout, time.Millisecond, "--msg-timeout") options := NewNsqdOptions() options.maxRdyCount = *maxRdyCount options.maxMessageSize = *maxMessageSize options.maxBodySize = *maxBodySize options.memQueueSize = *memQueueSize options.dataPath = *dataPath options.maxBytesPerFile = *maxBytesPerFile options.syncEvery = *syncEvery options.syncTimeout = *syncTimeout options.msgTimeout = msgTimeoutDuration options.maxMsgTimeout = *maxMsgTimeout options.broadcastAddress = *broadcastAddress options.maxHeartbeatInterval = *maxHeartbeatInterval options.maxOutputBufferSize = *maxOutputBufferSize options.maxOutputBufferTimeout = *maxOutputBufferTimeout options.tlsCert = *tlsCert options.tlsKey = *tlsKey if *statsdAddress != "" { // flagToDuration will fatally error if it is invalid options.statsdInterval = flagToDuration(*statsdInterval, time.Second, "--statsd-interval") undered := fmt.Sprintf("%s_%d", strings.Replace(*broadcastAddress, ".", "_", -1), httpAddr.Port) options.statsdPrefix = fmt.Sprintf("nsq.%s.", undered) options.statsdAddress = *statsdAddress } nsqd := NewNSQd(*workerId, options) nsqd.tcpAddr = tcpAddr nsqd.httpAddr = httpAddr nsqd.lookupdTCPAddrs = lookupdTCPAddrs nsqd.LoadMetadata() err = nsqd.PersistMetadata() if err != nil { log.Fatalf("ERROR: failed to persist metadata - %s", err.Error()) } nsqd.Main() <-exitChan nsqd.Exit() }
func NewNSQD(opts *nsqdOptions) *NSQD { n := &NSQD{ opts: opts, healthy: 1, topicMap: make(map[string]*Topic), idChan: make(chan MessageID, 4096), exitChan: make(chan int), notifyChan: make(chan interface{}), } if opts.MaxDeflateLevel < 1 || opts.MaxDeflateLevel > 9 { n.logf("FATAL: --max-deflate-level must be [1,9]") os.Exit(1) } if opts.ID < 0 || opts.ID >= 4096 { n.logf("FATAL: --worker-id must be [0,4096)") os.Exit(1) } tcpAddr, err := net.ResolveTCPAddr("tcp", opts.TCPAddress) if err != nil { n.logf("FATAL: failed to resolve TCP address (%s) - %s", opts.TCPAddress, err) os.Exit(1) } n.tcpAddr = tcpAddr httpAddr, err := net.ResolveTCPAddr("tcp", opts.HTTPAddress) if err != nil { n.logf("FATAL: failed to resolve HTTP address (%s) - %s", opts.HTTPAddress, err) os.Exit(1) } n.httpAddr = httpAddr if opts.HTTPSAddress != "" { httpsAddr, err := net.ResolveTCPAddr("tcp", opts.HTTPSAddress) if err != nil { n.logf("FATAL: failed to resolve HTTPS address (%s) - %s", opts.HTTPSAddress, err) os.Exit(1) } n.httpsAddr = httpsAddr } if opts.StatsdPrefix != "" { statsdHostKey := util.StatsdHostKey(net.JoinHostPort(opts.BroadcastAddress, strconv.Itoa(httpAddr.Port))) prefixWithHost := strings.Replace(opts.StatsdPrefix, "%s", statsdHostKey, -1) if prefixWithHost[len(prefixWithHost)-1] != '.' { prefixWithHost += "." } opts.StatsdPrefix = prefixWithHost } if opts.TLSClientAuthPolicy != "" { opts.TLSRequired = true } tlsConfig, err := buildTLSConfig(opts) if err != nil { n.logf("FATAL: failed to build TLS config - %s", err) os.Exit(1) } if tlsConfig == nil && n.opts.TLSRequired { n.logf("FATAL: cannot require TLS client connections without TLS key and cert") os.Exit(1) } n.tlsConfig = tlsConfig n.waitGroup.Wrap(func() { n.idPump() }) n.logf(util.Version("nsqd")) n.logf("ID: %d", n.opts.ID) return n }
func main() { flag.Parse() hostname, err := os.Hostname() if err != nil { log.Fatal(err) } if *showVersion { fmt.Println(util.Version("nsqd")) return } if *workerId == 0 { h := md5.New() io.WriteString(h, hostname) *workerId = int64(crc32.ChecksumIEEE(h.Sum(nil)) % 1024) } tcpAddr, err := net.ResolveTCPAddr("tcp", *tcpAddress) if err != nil { log.Fatal(err) } httpAddr, err := net.ResolveTCPAddr("tcp", *httpAddress) if err != nil { log.Fatal(err) } if *broadcastAddress == "" { *broadcastAddress = hostname } log.Println(util.Version("nsqd")) log.Printf("worker id %d", *workerId) exitChan := make(chan int) signalChan := make(chan os.Signal, 1) go func() { <-signalChan exitChan <- 1 }() signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) // flagToDuration will fatally error if it is invalid msgTimeoutDuration := flagToDuration(*msgTimeout, time.Millisecond, "--msg-timeout") if *maxDeflateLevel < 1 || *maxDeflateLevel > 9 { log.Fatalf("--max-deflate-level must be [1,9]") } options := NewNsqdOptions() options.maxRdyCount = *maxRdyCount options.maxMessageSize = *maxMessageSize options.maxBodySize = *maxBodySize options.memQueueSize = *memQueueSize options.dataPath = *dataPath options.maxBytesPerFile = *maxBytesPerFile options.syncEvery = *syncEvery options.syncTimeout = *syncTimeout options.msgTimeout = msgTimeoutDuration options.maxMsgTimeout = *maxMsgTimeout options.broadcastAddress = *broadcastAddress options.maxHeartbeatInterval = *maxHeartbeatInterval options.maxOutputBufferSize = *maxOutputBufferSize options.maxOutputBufferTimeout = *maxOutputBufferTimeout options.tlsCert = *tlsCert options.tlsKey = *tlsKey options.deflateEnabled = *deflateEnabled options.maxDeflateLevel = *maxDeflateLevel options.snappyEnabled = *snappyEnabled options.e2eProcessingLatencyWindowTime = *e2eProcessingLatencyWindowTime options.e2eProcessingLatencyPercentiles = e2eProcessingLatencyPercentiles if *statsdAddress != "" { // flagToDuration will fatally error if it is invalid options.statsdInterval = flagToDuration(*statsdInterval, time.Second, "--statsd-interval") statsdHostKey := util.StatsdHostKey(net.JoinHostPort(*broadcastAddress, strconv.Itoa(httpAddr.Port))) prefixWithHost := strings.Replace(*statsdPrefix, "%s", statsdHostKey, -1) if prefixWithHost[len(prefixWithHost)-1] != '.' { prefixWithHost += "." } options.statsdPrefix = prefixWithHost options.statsdAddress = *statsdAddress } nsqd := NewNSQd(*workerId, options) nsqd.tcpAddr = tcpAddr nsqd.httpAddr = httpAddr nsqd.lookupdTCPAddrs = lookupdTCPAddrs nsqd.LoadMetadata() err = nsqd.PersistMetadata() if err != nil { log.Fatalf("ERROR: failed to persist metadata - %s", err.Error()) } nsqd.Main() <-exitChan nsqd.Exit() }