func (s *httpServer) indexHandler(w http.ResponseWriter, req *http.Request) {
reqParams, err := util.NewReqParams(req)
if err != nil {
log.Printf("ERROR: failed to parse request params - %s", err.Error())
http.Error(w, "INVALID_REQUEST", 500)
return
}
var topics []string
if len(s.context.nsqadmin.options.NSQLookupdHTTPAddresses) != 0 {
topics, _ = lookupd.GetLookupdTopics(s.context.nsqadmin.options.NSQLookupdHTTPAddresses)
} else {
topics, _ = lookupd.GetNSQDTopics(s.context.nsqadmin.options.NSQDHTTPAddresses)
}
p := struct {
Title string
GraphOptions *GraphOptions
Topics Topics
Version string
}{
Title: "NSQ",
GraphOptions: NewGraphOptions(w, req, reqParams, s.context),
Topics: TopicsFromStrings(topics),
Version: util.BINARY_VERSION,
}
err = templates.T.ExecuteTemplate(w, "index.html", p)
if err != nil {
log.Printf("Template Error %s", err.Error())
http.Error(w, "Template Error", 500)
}
}
func (n *NSQHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
var topics []string
topicStats := make(map[string]interface{})
topics, _ = lookupd.GetLookupdTopics(n.NSQLookupdHTTPAddresses)
for _, topic := range topics {
topicStats[topic] = getTopicStats(topic, n.NSQLookupdHTTPAddresses)
}
content, _ := json.Marshal(topicStats)
w.Write(content)
}
func (t *TopicDiscoverer) syncTopics(addrs []string) {
newTopics, err := lookupd.GetLookupdTopics(addrs)
if err != nil {
log.Print("ERROR: could not retrieve topic list: %s", err)
}
for _, topic := range newTopics {
if _, ok := t.topics[topic]; !ok {
logger, err := newReaderFileLogger(topic)
if err != nil {
log.Printf("ERROR: couldn't create logger for new topic %s: %s", topic, err.Error())
continue
}
t.topics[topic] = logger
go t.startTopicRouter(logger)
}
}
}
// TopicExists checks if the given topic name exists in the producers
// registered with the provided lookupd instances
func TopicExists(topic string, lookupdHTTPAddr []string) (exists bool, err error) {
var topics []string
topics, err = lookupd.GetLookupdTopics(lookupdHTTPAddr)
if err != nil {
log.Printf("ERROR: Could not query lookupd topic existance: %s", err)
return
}
sort.Strings(topics)
found := sort.SearchStrings(topics, topic)
if found == len(topics) || topics[found] != topic {
exists = false
} else {
exists = true
}
return
}
func (s *ConsServer) updateReader(sqlChan chan SqlItem) {
topics, err := lookupd.GetLookupdTopics(s.opts.NsqlookupdHttpAddr)
if err != nil {
log.Fatal("ERR, lookupdlGetLookupdTopics, %s, %s", s.opts.NsqlookupdHttpAddr, err)
}
for i := 0; i < len(s.jsonconf.Log_list); i++ {
table := s.jsonconf.Log_list[i].Table
found := false
for j := 0; j < len(topics); j++ {
if table == topics[j] {
found = true
break
}
}
if !found {
log.Printf("WAR: topic %s not in NSQD", table)
continue
}
idx := -1
for k := 0; k < len(s.readers); k++ {
if s.readers[k].table == table {
idx = k
break
}
}
if idx != -1 {
if !s.readers[idx].IsRun() {
log.Printf("INF, restart reader[%s]", table)
s.readers[idx].Start(s.jsonconf, s.opts.MaxInFlight, s.opts.NsqlookupdHttpAddr, sqlChan)
}
continue
}
r := NewlogReader(table, s.opts.Duration)
s.readers = append(s.readers, r)
r.Start(s.jsonconf, s.opts.MaxInFlight, s.opts.NsqlookupdHttpAddr, sqlChan)
log.Printf("INF: new reader[%s]", table)
}
}
func (s *httpServer) lookupHandler(w http.ResponseWriter, req *http.Request) {
reqParams, err := util.NewReqParams(req)
if err != nil {
log.Printf("ERROR: failed to parse request params - %s", err.Error())
http.Error(w, "INVALID_REQUEST", 500)
return
}
channels := make(map[string][]string)
allTopics, _ := lookupd.GetLookupdTopics(s.context.nsqadmin.options.NSQLookupdHTTPAddresses)
for _, topicName := range allTopics {
var producers []string
producers, _ = lookupd.GetLookupdTopicProducers(topicName, s.context.nsqadmin.options.NSQLookupdHTTPAddresses)
if len(producers) == 0 {
topicChannels, _ := lookupd.GetLookupdTopicChannels(topicName, s.context.nsqadmin.options.NSQLookupdHTTPAddresses)
channels[topicName] = topicChannels
}
}
p := struct {
Title string
GraphOptions *GraphOptions
TopicMap map[string][]string
Lookupd []string
Version string
}{
Title: "NSQ Lookup",
GraphOptions: NewGraphOptions(w, req, reqParams, s.context),
TopicMap: channels,
Lookupd: s.context.nsqadmin.options.NSQLookupdHTTPAddresses,
Version: util.BINARY_VERSION,
}
err = templates.T.ExecuteTemplate(w, "lookup.html", p)
if err != nil {
log.Printf("Template Error %s", err.Error())
http.Error(w, "Template Error", 500)
}
}
func main() {
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_file v%s\n", util.BINARY_VERSION)
return
}
if *channel == "" {
log.Fatalf("--channel is required")
}
var topicsFromNSQLookupd bool
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required.")
}
if len(nsqdTCPAddrs) != 0 && len(lookupdHTTPAddrs) != 0 {
log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if *gzipLevel < 1 || *gzipLevel > 9 {
log.Fatalf("invalid --gzip-level value (%d), should be 1-9", *gzipLevel)
}
// TODO: remove, deprecated
if hasArg("gzip-compression") {
log.Printf("WARNING: --gzip-compression is deprecated in favor of --gzip-level")
switch *gzipCompression {
case 1:
*gzipLevel = gzip.BestSpeed
case 2:
*gzipLevel = gzip.BestCompression
case 3:
*gzipLevel = gzip.DefaultCompression
default:
log.Fatalf("invalid --gzip-compression value (%d), should be 1,2,3", *gzipCompression)
}
}
discoverer := newTopicDiscoverer()
signal.Notify(discoverer.hupChan, syscall.SIGHUP)
signal.Notify(discoverer.termChan, syscall.SIGINT, syscall.SIGTERM)
if len(topics) < 1 {
if len(lookupdHTTPAddrs) < 1 {
log.Fatalf("use --topic to list at least one topic to subscribe to or specify at least one --lookupd-http-address to subscribe to all its topics")
}
topicsFromNSQLookupd = true
var err error
topics, err = lookupd.GetLookupdTopics(lookupdHTTPAddrs)
if err != nil {
log.Fatalf("ERROR: could not retrieve topic list: %s", err)
}
}
for _, topic := range topics {
logger, err := newReaderFileLogger(topic)
if err != nil {
log.Fatalf("ERROR: couldn't create logger for topic %s: %s", topic, err.Error())
}
discoverer.topics[topic] = logger
go discoverer.startTopicRouter(logger)
}
discoverer.watch(lookupdHTTPAddrs, topicsFromNSQLookupd)
}