func main() {
flag.Parse()
// Only try and parse the conf file if it exists
if _, err := os.Stat(*confFile); err == nil {
conf, err := globalconf.NewWithOptions(&globalconf.Options{Filename: *confFile})
if err != nil {
log.Fatal(4, err.Error())
}
conf.ParseAll()
}
log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
if *showVersion {
fmt.Println("nsq_probe_events_to_elasticsearch")
return
}
if *channel == "" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
}
if *topic == "" {
log.Fatal(4, "--topic is required")
}
if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
log.Fatal(4, "--nsqd-tcp-address or --lookupd-http-address required")
}
if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
log.Fatal(4, "use --nsqd-tcp-address or --lookupd-http-address not both")
}
hostname, err := os.Hostname()
if err != nil {
log.Fatal(4, err.Error())
}
metrics, err := helper.New(true, *statsdAddr, *statsdType, "nsq_probe_events_to_elasticsearch", strings.Replace(hostname, ".", "_", -1))
if err != nil {
log.Fatal(4, err.Error())
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
eventsToEsOK = metrics.NewCount("events_to_es.ok")
eventsToEsFail = metrics.NewCount("events_to_es.fail")
messagesSize = metrics.NewMeter("message_size", 0)
msgsAge = metrics.NewMeter("message_age", 0)
esPutDuration = metrics.NewTimer("es_put_duration", 0)
msgsHandleOK = metrics.NewCount("handle.ok")
msgsHandleFail = metrics.NewCount("handle.fail")
cfg := nsq.NewConfig()
cfg.UserAgent = "nsq_probe_events_to_elasticsearch"
err = app.ParseOpts(cfg, *consumerOpts)
if err != nil {
log.Fatal(4, err.Error())
}
cfg.MaxInFlight = *maxInFlight
consumer, err := insq.NewConsumer(*topic, *channel, cfg, "%s", metrics)
if err != nil {
log.Fatal(4, err.Error())
}
handler, err := NewESHandler()
if err != nil {
log.Fatal(4, err.Error())
}
consumer.AddConcurrentHandlers(handler, 80)
nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
nsqdAdds = []string{}
}
err = consumer.ConnectToNSQDs(nsqdAdds)
if err != nil {
log.Fatal(4, err.Error())
}
log.Info("connected to nsqd")
lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
lookupdAdds = []string{}
}
err = consumer.ConnectToNSQLookupds(lookupdAdds)
if err != nil {
log.Fatal(4, err.Error())
}
go func() {
log.Info("INFO starting listener for http/debug on %s", *listenAddr)
httperr := http.ListenAndServe(*listenAddr, nil)
if httperr != nil {
log.Info(httperr.Error())
}
}()
for {
select {
case <-consumer.StopChan:
return
case <-sigChan:
consumer.Stop()
}
}
}
func main() {
flag.Parse()
log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
if *showVersion {
fmt.Println("nsq_metrics_to_stdout")
return
}
if *channel == "" || *channel == "stdout<random-number>#ephemeral" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("stdout%06d#ephemeral", rand.Int()%999999)
}
if *topic == "" {
log.Fatal(4, "--topic is required")
}
if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
log.Fatal(4, "--nsqd-tcp-address or --lookupd-http-address required")
}
if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
log.Fatal(4, "use --nsqd-tcp-address or --lookupd-http-address not both")
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
cfg := nsq.NewConfig()
cfg.UserAgent = "nsq_metrics_to_stdout"
err := app.ParseOpts(cfg, *consumerOpts)
if err != nil {
log.Fatal(4, err.Error())
}
cfg.MaxInFlight = *maxInFlight
consumer, err := nsq.NewConsumer(*topic, *channel, cfg)
if err != nil {
log.Fatal(4, err.Error())
}
handler, err := NewStdoutHandler()
if err != nil {
log.Fatal(4, err.Error())
}
consumer.AddHandler(handler)
nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
nsqdAdds = []string{}
}
err = consumer.ConnectToNSQDs(nsqdAdds)
if err != nil {
log.Fatal(4, err.Error())
}
log.Info("connected to nsqd")
lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
lookupdAdds = []string{}
}
err = consumer.ConnectToNSQLookupds(lookupdAdds)
if err != nil {
log.Fatal(4, err.Error())
}
go func() {
log.Info("starting listener for http/debug on %s", *listenAddr)
log.Info("%s", http.ListenAndServe(*listenAddr, nil))
}()
for {
select {
case <-consumer.StopChan:
return
case <-sigChan:
consumer.Stop()
<-consumer.StopChan
}
}
}
func main() {
flag.Parse()
// Only try and parse the conf file if it exists
if _, err := os.Stat(*confFile); err == nil {
conf, err := globalconf.NewWithOptions(&globalconf.Options{Filename: *confFile})
if err != nil {
log.Fatal(4, "error with configuration file: %s", err)
os.Exit(1)
}
conf.ParseAll()
}
log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
if *showVersion {
fmt.Println("metrics_tank")
return
}
if *instance == "" {
log.Fatal(0, "instance can't be empty")
}
hostname, err := os.Hostname()
if err != nil {
log.Fatal(0, "failed to lookup hostname. %s", err)
}
stats, err := helper.New(true, *statsdAddr, *statsdType, "metric_tank", strings.Replace(hostname, ".", "_", -1))
if err != nil {
log.Fatal(0, "failed to initialize statsd. %s", err)
}
if *channel == "" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("metric_tank%06d#ephemeral", rand.Int()%999999)
}
if *topic == "" {
log.Fatal(0, "--topic is required")
}
if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
log.Fatal(0, "--nsqd-tcp-address or --lookupd-http-address required")
}
if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
log.Fatal(0, "use --nsqd-tcp-address or --lookupd-http-address not both")
}
// set default cassandra address if none is set.
if *cassandraAddrs == "" {
*cassandraAddrs = "localhost"
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
cfg := nsq.NewConfig()
cfg.UserAgent = "metrics_tank"
err = app.ParseOpts(cfg, *consumerOpts)
if err != nil {
log.Fatal(0, "failed to parse nsq consumer options. %s", err)
}
cfg.MaxInFlight = *maxInFlight
consumer, err := insq.NewConsumer(*topic, *channel, cfg, "%s", stats)
if err != nil {
log.Fatal(0, "Failed to create NSQ consumer. %s", err)
}
initMetrics(stats)
err = InitCassandra()
if err != nil {
log.Fatal(4, "failed to initialize cassandra. %s", err)
}
set := strings.Split(*aggSettings, ",")
finalSettings := make([]aggSetting, 0)
for _, v := range set {
if v == "" {
continue
}
fields := strings.Split(v, ":")
if len(fields) != 3 {
log.Fatal(0, "bad agg settings")
}
aggSpan, err := strconv.Atoi(fields[0])
if err != nil {
log.Fatal(0, "bad agg settings", err)
}
aggChunkSpan, err := strconv.Atoi(fields[1])
if err != nil {
log.Fatal(0, "bad agg settings", err)
}
aggNumChunks, err := strconv.Atoi(fields[2])
if err != nil {
log.Fatal(0, "bad agg settings", err)
}
finalSettings = append(finalSettings, aggSetting{uint32(aggSpan), uint32(aggChunkSpan), uint32(aggNumChunks)})
}
metrics = NewAggMetrics(uint32(*chunkSpan), uint32(*numChunks), uint32(*chunkMaxStale), uint32(*metricMaxStale), finalSettings)
handler := NewHandler(metrics)
consumer.AddConcurrentHandlers(handler, *concurrency)
nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
nsqdAdds = []string{}
}
err = consumer.ConnectToNSQDs(nsqdAdds)
if err != nil {
log.Fatal(4, "failed to connect to NSQDs. %s", err)
}
log.Info("connected to nsqd")
lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
lookupdAdds = []string{}
}
err = consumer.ConnectToNSQLookupds(lookupdAdds)
if err != nil {
log.Fatal(4, "failed to connect to NSQLookupds. %s", err)
}
go func() {
m := &runtime.MemStats{}
for range time.Tick(time.Duration(1) * time.Second) {
runtime.ReadMemStats(m)
alloc.Value(int64(m.Alloc))
totalAlloc.Value(int64(m.TotalAlloc))
sysBytes.Value(int64(m.Sys))
}
}()
go func() {
http.HandleFunc("/get", Get)
log.Info("starting listener for metrics and http/debug on %s", *listenAddr)
log.Info("%s", http.ListenAndServe(*listenAddr, nil))
}()
for {
select {
case <-consumer.StopChan:
err := metrics.Persist()
if err != nil {
log.Error(3, "failed to persist aggmetrics. %s", err)
}
log.Info("closing cassandra session.")
cSession.Close()
log.Info("terminating.")
log.Close()
return
case <-sigChan:
log.Info("Shutting down")
consumer.Stop()
}
}
}
