func initLogger() {
once.Do(func() {
var (
level syslog.Priority
hook *logrus_syslog.SyslogHook
)
proto, err := config.Instance().String("syslog", "proto")
cli.CheckFatalError(err)
addr, err := config.Instance().String("syslog", "addr")
cli.CheckFatalError(err)
levelName, err := config.Instance().String("syslog", "level")
cli.CheckFatalError(err)
switch levelName {
case "debug":
level = syslog.LOG_DEBUG
case "info":
level = syslog.LOG_INFO
case "notice":
level = syslog.LOG_NOTICE
case "warning":
level = syslog.LOG_WARNING
case "err":
level = syslog.LOG_ERR
case "crit":
level = syslog.LOG_CRIT
case "alert":
level = syslog.LOG_ALERT
case "emerg":
level = syslog.LOG_EMERG
default:
level = syslog.LOG_WARNING
}
if len(proto) == 0 || len(addr) == 0 {
writer, err := syslog.New(level, "recause")
if err != nil {
cli.CheckFatalError(err)
}
hook = &logrus_syslog.SyslogHook{
Writer: writer,
}
} else {
hook, err = logrus_syslog.NewSyslogHook(proto, addr, level, "recause")
cli.CheckFatalError(err)
}
logger = log.New()
logger.Hooks.Add(hook)
log.SetOutput(logger.Writer())
})
}
// Returns HTTP server object
func NewWorkerHttp(storage storage.Storage) *WorkerHttp {
addr, err := config.Instance().String("http", "addr")
cli.CheckError(err)
maxPerPage, err := config.Instance().Int("http", "max_per_page")
if err != nil || maxPerPage <= 0 {
maxPerPage = 100
}
maxResults, err := config.Instance().Int("http", "max_results")
if err != nil || maxPerPage <= 0 {
maxResults = 1000
}
return &WorkerHttp{
addr: addr,
maxPerPage: maxPerPage,
maxResults: maxResults,
storage: storage,
}
}
// Returns packet receiver object
func NewWorkerReceiver(storage storage.Storage) *WorkerReceiver {
addr, err := config.Instance().String("receiver", "addr")
cli.CheckError(err)
reader, err := gelf.NewReader(addr)
cli.CheckError(err)
return &WorkerReceiver{
storage: storage,
reader: reader,
}
}
func actionRun(c *cc.Context) error {
_ = config.Instance(c.String("config"))
var (
wg *sync.WaitGroup = &sync.WaitGroup{}
die chan bool = make(chan bool)
storage storage.Storage
workersList []workers.Worker
)
// Listen for SIGINT
ch := make(chan os.Signal, 1)
signal.Notify(ch, os.Interrupt)
go func() {
for range ch {
logger.Instance().Info("Caught interrupt signal")
// Close all workers.
close(die)
}
}()
storage = elastic.NewElasticStorage()
go storage.PeriodicFlush(die)
workersList = append(workersList, workers.NewWorkerHttp(storage))
workersList = append(workersList, workers.NewWorkerReceiver(storage))
wg.Add(len(workersList))
for _, w := range workersList {
go w.Run(wg, die)
}
wg.Wait()
return nil
}
// Returns object to work with bleve
func NewBleveStorage() *Bleve {
datapath, err := config.Instance().String("bleve", "datapath")
if err != nil {
logger.Instance().
WithError(err).
Error("Path to bleve index is not provided")
os.Exit(1)
}
dirname := path.Dir(datapath)
if !cli.FileExists(dirname) {
logger.Instance().
WithError(err).
WithField("directory", dirname).
Error("Directory with bleve index doesn't exist")
os.Exit(1)
}
batchSize, err := config.Instance().Int("bleve", "batch_size")
if err != nil {
batchSize = 10
}
var (
defaultIntervalSecond string = "1s"
defaultIntervalMonth string = "720h"
index bv.Index
)
intervalCleanupStr, err := config.Instance().String("bleve", "interval_cleanup")
if err != nil {
intervalCleanupStr = defaultIntervalMonth
}
intervalCleanup, err := time.ParseDuration(intervalCleanupStr)
if err != nil {
intervalCleanup, _ = time.ParseDuration(defaultIntervalMonth)
}
intervalFlushStr, err := config.Instance().String("bleve", "interval_flush")
if err != nil {
intervalFlushStr = defaultIntervalSecond
}
intervalFlush, err := time.ParseDuration(intervalFlushStr)
if err != nil {
intervalFlush, _ = time.ParseDuration(defaultIntervalSecond)
}
if !cli.FileExists(datapath) {
index, err = bv.New(datapath, getIndexMapping())
if err != nil {
logger.Instance().
WithError(err).
Error("Unable to create bleve index")
os.Exit(1)
} else {
logger.Instance().
Debug("New bleve index created")
}
} else {
index, err = bv.Open(datapath)
if err != nil {
logger.Instance().
WithError(err).
Error("Unable to open bleve index")
os.Exit(1)
} else {
logger.Instance().
Debug("Bleve index successfully opened")
}
}
return &Bleve{
batchSize: batchSize,
index: index,
messages: []*storage.Message{},
mutexHandleMessage: &sync.RWMutex{},
mutexFlushMessages: &sync.RWMutex{},
intervalCleanup: -intervalCleanup,
intervalFlush: intervalFlush,
lastFlush: time.Now(),
}
}
// Returns object to work with bleve
func NewElasticStorage() *Elastic {
url, err := config.Instance().String("elastic", "url")
if err != nil {
logger.Instance().
WithError(err).
Error("Elastic url is not provided")
os.Exit(1)
}
client, err := es.NewClient(
es.SetURL(url),
es.SetSniff(false),
es.SetHealthcheck(false),
es.SetMaxRetries(0),
)
if err != nil {
logger.Instance().
WithError(err).
Error("Unable to create client to elastic")
os.Exit(1)
}
indexName, err := config.Instance().String("elastic", "index")
if err != nil {
logger.Instance().
WithError(err).
Error("Index name is not provided")
os.Exit(1)
}
typeName, err := config.Instance().String("elastic", "type")
if err != nil {
logger.Instance().
WithError(err).
Error("Type name is not provided")
os.Exit(1)
}
batchSize, err := config.Instance().Int("elastic", "batch_size")
if err != nil {
batchSize = 10
}
var (
defaultIntervalSecond string = "1s"
defaultIntervalMonth string = "720h"
)
intervalCleanupStr, err := config.Instance().String("elastic", "interval_cleanup")
if err != nil {
intervalCleanupStr = defaultIntervalMonth
}
intervalCleanup, err := time.ParseDuration(intervalCleanupStr)
if err != nil {
intervalCleanup, _ = time.ParseDuration(defaultIntervalMonth)
}
intervalFlushStr, err := config.Instance().String("elastic", "interval_flush")
if err != nil {
intervalFlushStr = defaultIntervalSecond
}
intervalFlush, err := time.ParseDuration(intervalFlushStr)
if err != nil {
intervalFlush, _ = time.ParseDuration(defaultIntervalSecond)
}
return &Elastic{
batchSize: batchSize,
indexName: indexName,
typeName: typeName,
client: client,
messages: []*storage.Message{},
mutexHandleMessage: &sync.RWMutex{},
mutexFlushMessages: &sync.RWMutex{},
ttl: int64(intervalCleanup.Seconds() * 1000), // TTL is in milliseconds
intervalFlush: intervalFlush,
lastFlush: time.Now(),
}
}