Example #1
0
func initLogger() {
	once.Do(func() {
		var (
			level syslog.Priority
			hook  *logrus_syslog.SyslogHook
		)

		proto, err := config.Instance().String("syslog", "proto")
		cli.CheckFatalError(err)

		addr, err := config.Instance().String("syslog", "addr")
		cli.CheckFatalError(err)

		levelName, err := config.Instance().String("syslog", "level")
		cli.CheckFatalError(err)

		switch levelName {
		case "debug":
			level = syslog.LOG_DEBUG
		case "info":
			level = syslog.LOG_INFO
		case "notice":
			level = syslog.LOG_NOTICE
		case "warning":
			level = syslog.LOG_WARNING
		case "err":
			level = syslog.LOG_ERR
		case "crit":
			level = syslog.LOG_CRIT
		case "alert":
			level = syslog.LOG_ALERT
		case "emerg":
			level = syslog.LOG_EMERG
		default:
			level = syslog.LOG_WARNING
		}

		if len(proto) == 0 || len(addr) == 0 {
			writer, err := syslog.New(level, "recause")
			if err != nil {
				cli.CheckFatalError(err)
			}

			hook = &logrus_syslog.SyslogHook{
				Writer: writer,
			}
		} else {
			hook, err = logrus_syslog.NewSyslogHook(proto, addr, level, "recause")
			cli.CheckFatalError(err)
		}

		logger = log.New()
		logger.Hooks.Add(hook)

		log.SetOutput(logger.Writer())
	})
}
Example #2
0
// Returns HTTP server object
func NewWorkerHttp(storage storage.Storage) *WorkerHttp {
	addr, err := config.Instance().String("http", "addr")
	cli.CheckError(err)

	maxPerPage, err := config.Instance().Int("http", "max_per_page")
	if err != nil || maxPerPage <= 0 {
		maxPerPage = 100
	}

	maxResults, err := config.Instance().Int("http", "max_results")
	if err != nil || maxPerPage <= 0 {
		maxResults = 1000
	}

	return &WorkerHttp{
		addr:       addr,
		maxPerPage: maxPerPage,
		maxResults: maxResults,
		storage:    storage,
	}
}
Example #3
0
// Returns packet receiver object
func NewWorkerReceiver(storage storage.Storage) *WorkerReceiver {
	addr, err := config.Instance().String("receiver", "addr")
	cli.CheckError(err)

	reader, err := gelf.NewReader(addr)
	cli.CheckError(err)

	return &WorkerReceiver{
		storage: storage,
		reader:  reader,
	}
}
Example #4
0
func actionRun(c *cc.Context) error {
	_ = config.Instance(c.String("config"))

	var (
		wg          *sync.WaitGroup = &sync.WaitGroup{}
		die         chan bool       = make(chan bool)
		storage     storage.Storage
		workersList []workers.Worker
	)

	// Listen for SIGINT
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	go func() {
		for range ch {
			logger.Instance().Info("Caught interrupt signal")

			// Close all workers.
			close(die)
		}
	}()

	storage = elastic.NewElasticStorage()
	go storage.PeriodicFlush(die)

	workersList = append(workersList, workers.NewWorkerHttp(storage))
	workersList = append(workersList, workers.NewWorkerReceiver(storage))

	wg.Add(len(workersList))

	for _, w := range workersList {
		go w.Run(wg, die)
	}

	wg.Wait()

	return nil
}
Example #5
0
// Returns object to work with bleve
func NewBleveStorage() *Bleve {
	datapath, err := config.Instance().String("bleve", "datapath")
	if err != nil {
		logger.Instance().
			WithError(err).
			Error("Path to bleve index is not provided")

		os.Exit(1)
	}

	dirname := path.Dir(datapath)
	if !cli.FileExists(dirname) {
		logger.Instance().
			WithError(err).
			WithField("directory", dirname).
			Error("Directory with bleve index doesn't exist")

		os.Exit(1)
	}

	batchSize, err := config.Instance().Int("bleve", "batch_size")
	if err != nil {
		batchSize = 10
	}

	var (
		defaultIntervalSecond string = "1s"
		defaultIntervalMonth  string = "720h"
		index                 bv.Index
	)

	intervalCleanupStr, err := config.Instance().String("bleve", "interval_cleanup")
	if err != nil {
		intervalCleanupStr = defaultIntervalMonth
	}

	intervalCleanup, err := time.ParseDuration(intervalCleanupStr)
	if err != nil {
		intervalCleanup, _ = time.ParseDuration(defaultIntervalMonth)
	}

	intervalFlushStr, err := config.Instance().String("bleve", "interval_flush")
	if err != nil {
		intervalFlushStr = defaultIntervalSecond
	}

	intervalFlush, err := time.ParseDuration(intervalFlushStr)
	if err != nil {
		intervalFlush, _ = time.ParseDuration(defaultIntervalSecond)
	}

	if !cli.FileExists(datapath) {
		index, err = bv.New(datapath, getIndexMapping())

		if err != nil {
			logger.Instance().
				WithError(err).
				Error("Unable to create bleve index")

			os.Exit(1)
		} else {
			logger.Instance().
				Debug("New bleve index created")
		}
	} else {
		index, err = bv.Open(datapath)

		if err != nil {
			logger.Instance().
				WithError(err).
				Error("Unable to open bleve index")

			os.Exit(1)
		} else {
			logger.Instance().
				Debug("Bleve index successfully opened")
		}
	}

	return &Bleve{
		batchSize:          batchSize,
		index:              index,
		messages:           []*storage.Message{},
		mutexHandleMessage: &sync.RWMutex{},
		mutexFlushMessages: &sync.RWMutex{},
		intervalCleanup:    -intervalCleanup,
		intervalFlush:      intervalFlush,
		lastFlush:          time.Now(),
	}
}
Example #6
0
// Returns object to work with bleve
func NewElasticStorage() *Elastic {
	url, err := config.Instance().String("elastic", "url")
	if err != nil {
		logger.Instance().
			WithError(err).
			Error("Elastic url is not provided")

		os.Exit(1)
	}

	client, err := es.NewClient(
		es.SetURL(url),
		es.SetSniff(false),
		es.SetHealthcheck(false),
		es.SetMaxRetries(0),
	)
	if err != nil {
		logger.Instance().
			WithError(err).
			Error("Unable to create client to elastic")

		os.Exit(1)
	}

	indexName, err := config.Instance().String("elastic", "index")
	if err != nil {
		logger.Instance().
			WithError(err).
			Error("Index name is not provided")

		os.Exit(1)
	}

	typeName, err := config.Instance().String("elastic", "type")
	if err != nil {
		logger.Instance().
			WithError(err).
			Error("Type name is not provided")

		os.Exit(1)
	}

	batchSize, err := config.Instance().Int("elastic", "batch_size")
	if err != nil {
		batchSize = 10
	}

	var (
		defaultIntervalSecond string = "1s"
		defaultIntervalMonth  string = "720h"
	)

	intervalCleanupStr, err := config.Instance().String("elastic", "interval_cleanup")
	if err != nil {
		intervalCleanupStr = defaultIntervalMonth
	}

	intervalCleanup, err := time.ParseDuration(intervalCleanupStr)
	if err != nil {
		intervalCleanup, _ = time.ParseDuration(defaultIntervalMonth)
	}

	intervalFlushStr, err := config.Instance().String("elastic", "interval_flush")
	if err != nil {
		intervalFlushStr = defaultIntervalSecond
	}

	intervalFlush, err := time.ParseDuration(intervalFlushStr)
	if err != nil {
		intervalFlush, _ = time.ParseDuration(defaultIntervalSecond)
	}

	return &Elastic{
		batchSize:          batchSize,
		indexName:          indexName,
		typeName:           typeName,
		client:             client,
		messages:           []*storage.Message{},
		mutexHandleMessage: &sync.RWMutex{},
		mutexFlushMessages: &sync.RWMutex{},
		ttl:                int64(intervalCleanup.Seconds() * 1000), // TTL is in milliseconds
		intervalFlush:      intervalFlush,
		lastFlush:          time.Now(),
	}
}