func initLogging(args *CommandLineArgs) {
// Get and check log mode.
LogModes = strings.Split(Cfg.Section("log").Key("mode").MustString("console"), ",")
LogsPath = makeAbsolute(Cfg.Section("paths").Key("logs").String(), HomePath)
LogConfigs = make([]util.DynMap, len(LogModes))
for i, mode := range LogModes {
mode = strings.TrimSpace(mode)
sec, err := Cfg.GetSection("log." + mode)
if err != nil {
log.Fatal(4, "Unknown log mode: %s", mode)
}
// Log level.
levelName := Cfg.Section("log."+mode).Key("level").In("Trace",
[]string{"Trace", "Debug", "Info", "Warn", "Error", "Critical"})
level, ok := logLevels[levelName]
if !ok {
log.Fatal(4, "Unknown log level: %s", levelName)
}
// Generate log configuration.
switch mode {
case "console":
LogConfigs[i] = util.DynMap{"level": level}
case "file":
logPath := sec.Key("file_name").MustString(filepath.Join(LogsPath, "grafana.log"))
os.MkdirAll(filepath.Dir(logPath), os.ModePerm)
LogConfigs[i] = util.DynMap{
"level": level,
"filename": logPath,
"rotate": sec.Key("log_rotate").MustBool(true),
"maxlines": sec.Key("max_lines").MustInt(1000000),
"maxsize": 1 << uint(sec.Key("max_size_shift").MustInt(28)),
"daily": sec.Key("daily_rotate").MustBool(true),
"maxdays": sec.Key("max_days").MustInt(7),
}
case "conn":
LogConfigs[i] = util.DynMap{
"level": level,
"reconnectOnMsg": sec.Key("reconnect_on_msg").MustBool(),
"reconnect": sec.Key("reconnect").MustBool(),
"net": sec.Key("protocol").In("tcp", []string{"tcp", "unix", "udp"}),
"addr": sec.Key("addr").MustString(":7020"),
}
case "smtp":
LogConfigs[i] = util.DynMap{
"level": level,
"user": sec.Key("user").MustString("*****@*****.**"),
"passwd": sec.Key("passwd").MustString("******"),
"host": sec.Key("host").MustString("127.0.0.1:25"),
"receivers": sec.Key("receivers").MustString("[]"),
"subject": sec.Key("subject").MustString("Diagnostic message from serve"),
}
case "database":
LogConfigs[i] = util.DynMap{
"level": level,
"driver": sec.Key("driver").String(),
"conn": sec.Key("conn").String(),
}
}
cfgJsonBytes, _ := json.Marshal(LogConfigs[i])
log.NewLogger(Cfg.Section("log").Key("buffer_len").MustInt64(10000), mode, string(cfgJsonBytes))
}
}
func init() {
IsWindows = runtime.GOOS == "windows"
log.NewLogger(0, "console", `{"level": 0}`)
}
func main() {
flag.Parse()
log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
if *showVersion {
fmt.Println("nsq_metrics_to_stdout")
return
}
if *channel == "" || *channel == "stdout<random-number>#ephemeral" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("stdout%06d#ephemeral", rand.Int()%999999)
}
if *topic == "" {
log.Fatal(4, "--topic is required")
}
if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
log.Fatal(4, "--nsqd-tcp-address or --lookupd-http-address required")
}
if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
log.Fatal(4, "use --nsqd-tcp-address or --lookupd-http-address not both")
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
cfg := nsq.NewConfig()
cfg.UserAgent = "nsq_metrics_to_stdout"
err := app.ParseOpts(cfg, *consumerOpts)
if err != nil {
log.Fatal(4, err.Error())
}
cfg.MaxInFlight = *maxInFlight
consumer, err := nsq.NewConsumer(*topic, *channel, cfg)
if err != nil {
log.Fatal(4, err.Error())
}
handler, err := NewStdoutHandler()
if err != nil {
log.Fatal(4, err.Error())
}
consumer.AddHandler(handler)
nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
nsqdAdds = []string{}
}
err = consumer.ConnectToNSQDs(nsqdAdds)
if err != nil {
log.Fatal(4, err.Error())
}
log.Info("connected to nsqd")
lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
lookupdAdds = []string{}
}
err = consumer.ConnectToNSQLookupds(lookupdAdds)
if err != nil {
log.Fatal(4, err.Error())
}
go func() {
log.Info("starting listener for http/debug on %s", *listenAddr)
log.Info("%s", http.ListenAndServe(*listenAddr, nil))
}()
for {
select {
case <-consumer.StopChan:
return
case <-sigChan:
consumer.Stop()
<-consumer.StopChan
}
}
}
func init() {
IsWindows = runtime.GOOS == "windows"
log.NewLogger(0, "console", `{"level": 0, "formatting":true}`)
}
func main() {
flag.Parse()
// Only try and parse the conf file if it exists
if _, err := os.Stat(*confFile); err == nil {
conf, err := globalconf.NewWithOptions(&globalconf.Options{Filename: *confFile})
if err != nil {
log.Fatal(4, err.Error())
}
conf.ParseAll()
}
log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
if *showVersion {
fmt.Println("nsq_probe_events_to_elasticsearch")
return
}
if *channel == "" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
}
if *topic == "" {
log.Fatal(4, "--topic is required")
}
if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
log.Fatal(4, "--nsqd-tcp-address or --lookupd-http-address required")
}
if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
log.Fatal(4, "use --nsqd-tcp-address or --lookupd-http-address not both")
}
hostname, err := os.Hostname()
if err != nil {
log.Fatal(4, err.Error())
}
metrics, err := helper.New(true, *statsdAddr, *statsdType, "nsq_probe_events_to_elasticsearch", strings.Replace(hostname, ".", "_", -1))
if err != nil {
log.Fatal(4, err.Error())
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
eventsToEsOK = metrics.NewCount("events_to_es.ok")
eventsToEsFail = metrics.NewCount("events_to_es.fail")
messagesSize = metrics.NewMeter("message_size", 0)
msgsAge = metrics.NewMeter("message_age", 0)
esPutDuration = metrics.NewTimer("es_put_duration", 0)
msgsHandleOK = metrics.NewCount("handle.ok")
msgsHandleFail = metrics.NewCount("handle.fail")
cfg := nsq.NewConfig()
cfg.UserAgent = "nsq_probe_events_to_elasticsearch"
err = app.ParseOpts(cfg, *consumerOpts)
if err != nil {
log.Fatal(4, err.Error())
}
cfg.MaxInFlight = *maxInFlight
consumer, err := insq.NewConsumer(*topic, *channel, cfg, "%s", metrics)
if err != nil {
log.Fatal(4, err.Error())
}
handler, err := NewESHandler()
if err != nil {
log.Fatal(4, err.Error())
}
consumer.AddConcurrentHandlers(handler, 80)
nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
nsqdAdds = []string{}
}
err = consumer.ConnectToNSQDs(nsqdAdds)
if err != nil {
log.Fatal(4, err.Error())
}
log.Info("connected to nsqd")
lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
lookupdAdds = []string{}
}
err = consumer.ConnectToNSQLookupds(lookupdAdds)
if err != nil {
log.Fatal(4, err.Error())
}
go func() {
log.Info("INFO starting listener for http/debug on %s", *listenAddr)
httperr := http.ListenAndServe(*listenAddr, nil)
if httperr != nil {
log.Info(httperr.Error())
}
}()
for {
select {
case <-consumer.StopChan:
return
case <-sigChan:
consumer.Stop()
}
}
}
func main() {
flag.Parse()
// Only try and parse the conf file if it exists
if _, err := os.Stat(*confFile); err == nil {
conf, err := globalconf.NewWithOptions(&globalconf.Options{Filename: *confFile})
if err != nil {
log.Fatal(4, "error with configuration file: %s", err)
os.Exit(1)
}
conf.ParseAll()
}
log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
if *showVersion {
fmt.Println("metrics_tank")
return
}
if *instance == "" {
log.Fatal(0, "instance can't be empty")
}
hostname, err := os.Hostname()
if err != nil {
log.Fatal(0, "failed to lookup hostname. %s", err)
}
stats, err := helper.New(true, *statsdAddr, *statsdType, "metric_tank", strings.Replace(hostname, ".", "_", -1))
if err != nil {
log.Fatal(0, "failed to initialize statsd. %s", err)
}
if *channel == "" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("metric_tank%06d#ephemeral", rand.Int()%999999)
}
if *topic == "" {
log.Fatal(0, "--topic is required")
}
if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
log.Fatal(0, "--nsqd-tcp-address or --lookupd-http-address required")
}
if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
log.Fatal(0, "use --nsqd-tcp-address or --lookupd-http-address not both")
}
// set default cassandra address if none is set.
if *cassandraAddrs == "" {
*cassandraAddrs = "localhost"
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
cfg := nsq.NewConfig()
cfg.UserAgent = "metrics_tank"
err = app.ParseOpts(cfg, *consumerOpts)
if err != nil {
log.Fatal(0, "failed to parse nsq consumer options. %s", err)
}
cfg.MaxInFlight = *maxInFlight
consumer, err := insq.NewConsumer(*topic, *channel, cfg, "%s", stats)
if err != nil {
log.Fatal(0, "Failed to create NSQ consumer. %s", err)
}
initMetrics(stats)
err = InitCassandra()
if err != nil {
log.Fatal(4, "failed to initialize cassandra. %s", err)
}
set := strings.Split(*aggSettings, ",")
finalSettings := make([]aggSetting, 0)
for _, v := range set {
if v == "" {
continue
}
fields := strings.Split(v, ":")
if len(fields) != 3 {
log.Fatal(0, "bad agg settings")
}
aggSpan, err := strconv.Atoi(fields[0])
if err != nil {
log.Fatal(0, "bad agg settings", err)
}
aggChunkSpan, err := strconv.Atoi(fields[1])
if err != nil {
log.Fatal(0, "bad agg settings", err)
}
aggNumChunks, err := strconv.Atoi(fields[2])
if err != nil {
log.Fatal(0, "bad agg settings", err)
}
finalSettings = append(finalSettings, aggSetting{uint32(aggSpan), uint32(aggChunkSpan), uint32(aggNumChunks)})
}
metrics = NewAggMetrics(uint32(*chunkSpan), uint32(*numChunks), uint32(*chunkMaxStale), uint32(*metricMaxStale), finalSettings)
handler := NewHandler(metrics)
consumer.AddConcurrentHandlers(handler, *concurrency)
nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
nsqdAdds = []string{}
}
err = consumer.ConnectToNSQDs(nsqdAdds)
if err != nil {
log.Fatal(4, "failed to connect to NSQDs. %s", err)
}
log.Info("connected to nsqd")
lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
lookupdAdds = []string{}
}
err = consumer.ConnectToNSQLookupds(lookupdAdds)
if err != nil {
log.Fatal(4, "failed to connect to NSQLookupds. %s", err)
}
go func() {
m := &runtime.MemStats{}
for range time.Tick(time.Duration(1) * time.Second) {
runtime.ReadMemStats(m)
alloc.Value(int64(m.Alloc))
totalAlloc.Value(int64(m.TotalAlloc))
sysBytes.Value(int64(m.Sys))
}
}()
go func() {
http.HandleFunc("/get", Get)
log.Info("starting listener for metrics and http/debug on %s", *listenAddr)
log.Info("%s", http.ListenAndServe(*listenAddr, nil))
}()
for {
select {
case <-consumer.StopChan:
err := metrics.Persist()
if err != nil {
log.Error(3, "failed to persist aggmetrics. %s", err)
}
log.Info("closing cassandra session.")
cSession.Close()
log.Info("terminating.")
log.Close()
return
case <-sigChan:
log.Info("Shutting down")
consumer.Stop()
}
}
}
