// NewGrafanaNet creates a special route that writes to a grafana.net datastore
// We will automatically run the route and the destination
// ignores spool for now
func NewGrafanaNet(key, prefix, sub, regex, addr, apiKey, schemasFile string, spool, sslVerify bool, bufSize, flushMaxNum, flushMaxWait, timeout int) (Route, error) {
m, err := matcher.New(prefix, sub, regex)
if err != nil {
return nil, err
}
schemas, err := persister.ReadWhisperSchemas(schemasFile)
if err != nil {
return nil, err
}
var defaultFound bool
for _, schema := range schemas {
if schema.Pattern.String() == ".*" {
defaultFound = true
}
if len(schema.Retentions) == 0 {
return nil, fmt.Errorf("retention setting cannot be empty")
}
}
if !defaultFound {
// good graphite health (not sure what graphite does if there's no .*
// but we definitely need to always be able to determine which interval to use
return nil, fmt.Errorf("storage-conf does not have a default '.*' pattern")
}
cleanAddr := util.AddrToPath(addr)
r := &GrafanaNet{
baseRoute: baseRoute{sync.Mutex{}, atomic.Value{}, key},
addr: addr,
apiKey: apiKey,
buf: make(chan []byte, bufSize), // takes about 228MB on 64bit
schemas: schemas,
bufSize: bufSize,
flushMaxNum: flushMaxNum,
flushMaxWait: time.Duration(flushMaxWait) * time.Millisecond,
timeout: time.Duration(timeout) * time.Millisecond,
sslVerify: sslVerify,
numErrFlush: stats.Counter("dest=" + cleanAddr + ".unit=Err.type=flush"),
numOut: stats.Counter("dest=" + cleanAddr + ".unit=Metric.direction=out"),
durationTickFlush: stats.Timer("dest=" + cleanAddr + ".what=durationFlush.type=ticker"),
durationManuFlush: stats.Timer("dest=" + cleanAddr + ".what=durationFlush.type=manual"),
tickFlushSize: stats.Histogram("dest=" + cleanAddr + ".unit=B.what=FlushSize.type=ticker"),
manuFlushSize: stats.Histogram("dest=" + cleanAddr + ".unit=B.what=FlushSize.type=manual"),
numBuffered: stats.Gauge("dest=" + cleanAddr + ".unit=Metric.what=numBuffered"),
}
r.config.Store(baseConfig{*m, make([]*dest.Destination, 0)})
go r.run()
return r, nil
}
// ParseConfig loads config from config file, schemas.conf, aggregation.conf
func (app *App) ParseConfig() error {
var err error
var newSchemas *persister.WhisperSchemas
var newAggregation *persister.WhisperAggregation
cfg := NewConfig()
if err := ParseConfig(app.ConfigFilename, cfg); err != nil {
return err
}
// carbon-cache prefix
if hostname, err := os.Hostname(); err == nil {
hostname = strings.Replace(hostname, ".", "_", -1)
cfg.Common.GraphPrefix = strings.Replace(cfg.Common.GraphPrefix, "{host}", hostname, -1)
} else {
cfg.Common.GraphPrefix = strings.Replace(cfg.Common.GraphPrefix, "{host}", "localhost", -1)
}
if cfg.Whisper.Enabled {
newSchemas, err = persister.ReadWhisperSchemas(cfg.Whisper.Schemas)
if err != nil {
return err
}
if cfg.Whisper.Aggregation != "" {
newAggregation, err = persister.ReadWhisperAggregation(cfg.Whisper.Aggregation)
if err != nil {
return err
}
} else {
newAggregation = persister.NewWhisperAggregation()
}
}
app.Config = cfg
app.Schemas = newSchemas
app.Aggregation = newAggregation
return nil
}
func main() {
var err error
/* CONFIG start */
configFile := flag.String("config", "", "Filename of config")
printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
checkConfig := flag.Bool("check-config", false, "Check config and exit")
printVersion := flag.Bool("version", false, "Print version")
isDaemon := flag.Bool("daemon", false, "Run in background")
pidfile := flag.String("pidfile", "", "Pidfile path (only for daemon)")
flag.Parse()
if *printVersion {
fmt.Print(Version)
return
}
cfg := newConfig()
if *printDefaultConfig {
if err = PrintConfig(cfg); err != nil {
log.Fatal(err)
}
return
}
if err = ParseConfig(*configFile, cfg); err != nil {
log.Fatal(err)
}
var runAsUser *user.User
if cfg.Common.User != "" {
runAsUser, err = user.Lookup(cfg.Common.User)
if err != nil {
log.Fatal(err)
}
}
var whisperSchemas *persister.WhisperSchemas
var whisperAggregation *persister.WhisperAggregation
if cfg.Whisper.Enabled {
whisperSchemas, err = persister.ReadWhisperSchemas(cfg.Whisper.Schemas)
if err != nil {
log.Fatal(err)
}
if cfg.Whisper.Aggregation != "" {
whisperAggregation, err = persister.ReadWhisperAggregation(cfg.Whisper.Aggregation)
if err != nil {
log.Fatal(err)
}
} else {
whisperAggregation = persister.NewWhisperAggregation()
}
}
if err := logging.SetLevel(cfg.Common.LogLevel); err != nil {
log.Fatal(err)
}
if *checkConfig {
return
}
if err := logging.PrepareFile(cfg.Common.Logfile, runAsUser); err != nil {
logrus.Fatal(err)
}
if err := logging.SetFile(cfg.Common.Logfile); err != nil {
logrus.Fatal(err)
}
if *isDaemon {
runtime.LockOSThread()
context := new(daemon.Context)
if *pidfile != "" {
context.PidFileName = *pidfile
context.PidFilePerm = 0644
}
if runAsUser != nil {
uid, err := strconv.ParseInt(runAsUser.Uid, 10, 0)
if err != nil {
log.Fatal(err)
}
gid, err := strconv.ParseInt(runAsUser.Gid, 10, 0)
if err != nil {
log.Fatal(err)
}
context.Credential = &syscall.Credential{
Uid: uint32(uid),
Gid: uint32(gid),
}
}
child, _ := context.Reborn()
if child != nil {
return
}
defer context.Release()
runtime.UnlockOSThread()
}
runtime.GOMAXPROCS(cfg.Common.MaxCPU)
/* CONFIG end */
// pprof
if cfg.Pprof.Enabled {
go func() {
logrus.Fatal(http.ListenAndServe(cfg.Pprof.Listen, nil))
}()
}
// carbon-cache prefix
if hostname, err := os.Hostname(); err == nil {
hostname = strings.Replace(hostname, ".", "_", -1)
cfg.Common.GraphPrefix = strings.Replace(cfg.Common.GraphPrefix, "{host}", hostname, -1)
} else {
cfg.Common.GraphPrefix = strings.Replace(cfg.Common.GraphPrefix, "{host}", "localhost", -1)
}
core := cache.New()
core.SetGraphPrefix(cfg.Common.GraphPrefix)
core.SetMaxSize(cfg.Cache.MaxSize)
core.SetInputCapacity(cfg.Cache.InputBuffer)
core.Start()
defer core.Stop()
/* UDP start */
udpCfg := cfg.Udp
if udpCfg.Enabled {
udpAddr, err := net.ResolveUDPAddr("udp", udpCfg.Listen)
if err != nil {
log.Fatal(err)
}
udpListener := receiver.NewUDP(core.In())
udpListener.SetGraphPrefix(cfg.Common.GraphPrefix)
if udpCfg.LogIncomplete {
udpListener.SetLogIncomplete(true)
}
defer udpListener.Stop()
if err = udpListener.Listen(udpAddr); err != nil {
log.Fatal(err)
}
}
/* UDP end */
/* TCP start */
tcpCfg := cfg.Tcp
if tcpCfg.Enabled {
tcpAddr, err := net.ResolveTCPAddr("tcp", tcpCfg.Listen)
if err != nil {
log.Fatal(err)
}
tcpListener := receiver.NewTCP(core.In())
tcpListener.SetGraphPrefix(cfg.Common.GraphPrefix)
defer tcpListener.Stop()
if err = tcpListener.Listen(tcpAddr); err != nil {
log.Fatal(err)
}
}
/* TCP end */
/* PICKLE start */
pickleCfg := cfg.Pickle
if pickleCfg.Enabled {
pickleAddr, err := net.ResolveTCPAddr("tcp", pickleCfg.Listen)
if err != nil {
log.Fatal(err)
}
pickleListener := receiver.NewPickle(core.In())
pickleListener.SetGraphPrefix(cfg.Common.GraphPrefix)
defer pickleListener.Stop()
if err = pickleListener.Listen(pickleAddr); err != nil {
log.Fatal(err)
}
}
/* PICKLE end */
/* WHISPER start */
if cfg.Whisper.Enabled {
whisperPersister := persister.NewWhisper(cfg.Whisper.DataDir, whisperSchemas, whisperAggregation, core.Out())
whisperPersister.SetGraphPrefix(cfg.Common.GraphPrefix)
whisperPersister.SetMaxUpdatesPerSecond(cfg.Whisper.MaxUpdatesPerSecond)
whisperPersister.SetWorkers(cfg.Whisper.Workers)
whisperPersister.Start()
defer whisperPersister.Stop()
}
/* WHISPER end */
/* CARBONLINK start */
if cfg.Carbonlink.Enabled {
linkAddr, err := net.ResolveTCPAddr("tcp", cfg.Carbonlink.Listen)
if err != nil {
log.Fatal(err)
}
carbonlink := cache.NewCarbonlinkListener(core.Query())
carbonlink.SetReadTimeout(cfg.Carbonlink.ReadTimeout.Value())
carbonlink.SetQueryTimeout(cfg.Carbonlink.QueryTimeout.Value())
defer carbonlink.Stop()
if err = carbonlink.Listen(linkAddr); err != nil {
log.Fatal(err)
}
}
/* CARBONLINK end */
logrus.Info("started")
select {}
}