// Configure initializes this consumer with values from a plugin config.
func (cons *Http) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
cons.address = conf.GetString("Address", ":80")
cons.readTimeoutSec = time.Duration(conf.GetInt("ReadTimeoutSec", 3)) * time.Second
cons.withHeaders = conf.GetBool("WithHeaders", true)
certificateFile := conf.GetString("Certificate", "")
keyFile := conf.GetString("PrivateKey", "")
if certificateFile != "" || keyFile != "" {
if certificateFile == "" || keyFile == "" {
return fmt.Errorf("There must always be a certificate and a private key or none of both")
}
cons.certificate = new(tls.Config)
cons.certificate.NextProtos = []string{"http/1.1"}
keypair, err := tls.LoadX509KeyPair(certificateFile, keyFile)
if err != nil {
return err
}
cons.certificate.Certificates = []tls.Certificate{keypair}
}
return err
}
// Configure initializes this producer with values from a plugin config.
func (prod *Redis) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
fieldFormat, err := core.NewPluginWithType(conf.GetString("FieldFormatter", "format.Identifier"), conf)
if err != nil {
return err // ### return, plugin load error ###
}
prod.fieldFormat = fieldFormat.(core.Formatter)
prod.password = conf.GetString("Password", "")
prod.database = int64(conf.GetInt("Database", 0))
prod.key = conf.GetString("Key", "default")
prod.fieldFromParsed = conf.GetBool("FieldAfterFormat", false)
prod.address, prod.protocol = shared.ParseAddress(conf.GetString("Address", ":6379"))
switch strings.ToLower(conf.GetString("Storage", "hash")) {
case "hash":
prod.store = prod.storeHash
case "list":
prod.store = prod.storeList
case "set":
prod.store = prod.storeSet
case "sortedset":
prod.store = prod.storeSortedSet
default:
fallthrough
case "string":
prod.store = prod.storeString
}
return nil
}
// Configure sets the database connection values
func (writer *influxDBWriter10) configure(conf core.PluginConfig, prod *InfluxDB) error {
writer.host = conf.GetString("Host", "localhost:8086")
writer.username = conf.GetString("User", "")
writer.password = conf.GetString("Password", "")
writer.databaseTemplate = conf.GetString("Database", "default")
writer.buffer = shared.NewByteStream(4096)
writer.connectionUp = false
writer.timeBasedDBName = conf.GetBool("TimeBasedName", true)
writer.Control = prod.Control
writer.writeURL = fmt.Sprintf("http://%s/write", writer.host)
writer.queryURL = fmt.Sprintf("http://%s/query", writer.host)
writer.pingURL = fmt.Sprintf("http://%s/ping", writer.host)
writer.separator = '?'
if writer.username != "" {
credentials := fmt.Sprintf("?u=%s&p=%s", url.QueryEscape(writer.username), url.QueryEscape(writer.password))
writer.writeURL += credentials
writer.queryURL += credentials
writer.separator = '&'
}
writer.writeURL = fmt.Sprintf("%s%cprecision=ms", writer.writeURL, writer.separator)
prod.SetCheckFuseCallback(writer.isConnectionUp)
return nil
}
// Configure sets the database connection values
func (writer *influxDBWriter09) configure(conf core.PluginConfig, prod *InfluxDB) error {
writer.host = conf.GetString("Host", "localhost:8086")
writer.username = conf.GetString("User", "")
writer.password = conf.GetString("Password", "")
writer.databaseTemplate = conf.GetString("Database", "default")
writer.buffer = shared.NewByteStream(4096)
writer.connectionUp = false
writer.Control = prod.Control
writer.writeURL = fmt.Sprintf("http://%s/write", writer.host)
writer.queryURL = fmt.Sprintf("http://%s/query", writer.host)
writer.pingURL = fmt.Sprintf("http://%s/ping", writer.host)
writer.separator = '?'
writer.timeBasedDBName = conf.GetBool("TimeBasedName", true)
if writer.username != "" {
credentials := fmt.Sprintf("?u=%s&p=%s", url.QueryEscape(writer.username), url.QueryEscape(writer.password))
writer.writeURL += credentials
writer.queryURL += credentials
writer.separator = '&'
}
if retentionPolicy := conf.GetString("RetentionPolicy", ""); retentionPolicy != "" {
writer.messageHeader = fmt.Sprintf("{\"database\":\"%%s\",\"retentionPolicy\":\"%s\",\"points\":[", retentionPolicy)
} else {
writer.messageHeader = "{\"database\":\"%s\",\"points\":["
}
prod.SetCheckFuseCallback(writer.isConnectionUp)
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *Serialize) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("SerializeFormatter", "format.Forward"), conf)
if err != nil {
return err
}
format.encode = conf.GetBool("SerializeStringEncode", true)
format.base = plugin.(core.Formatter)
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *StreamName) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("StreamNameFormatter", "format.Forward"), conf)
if err != nil {
return err
}
format.separator = conf.GetString("StreamNameSeparator", " ")
format.usePrevious = conf.GetBool("StreamNameHistory", false)
format.base = plugin.(core.Formatter)
return nil
}
// Configure initializes this consumer with values from a plugin config.
func (cons *Http) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
cons.address = conf.GetString("Address", ":80")
cons.readTimeoutSec = time.Duration(conf.GetInt("ReadTimeoutSec", 3)) * time.Second
cons.withHeaders = conf.GetBool("WithHeaders", true)
return err
}
// Configure initializes this producer with values from a plugin config.
func (prod *ElasticSearch) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.SetStopCallback(prod.close)
defaultServer := []string{"localhost"}
numConnections := conf.GetInt("Connections", 6)
retrySec := conf.GetInt("RetrySec", 5)
prod.conn = elastigo.NewConn()
prod.conn.Hosts = conf.GetStringArray("Servers", defaultServer)
prod.conn.Domain = conf.GetString("Domain", prod.conn.Hosts[0])
prod.conn.ClusterDomains = prod.conn.Hosts
prod.conn.Port = strconv.Itoa(conf.GetInt("Port", 9200))
prod.conn.Username = conf.GetString("User", "")
prod.conn.Password = conf.GetString("Password", "")
prod.indexer = prod.conn.NewBulkIndexerErrors(numConnections, retrySec)
prod.indexer.BufferDelayMax = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.indexer.BulkMaxBuffer = conf.GetInt("BatchSizeByte", 32768)
prod.indexer.BulkMaxDocs = conf.GetInt("BatchMaxCount", 128)
prod.indexer.Sender = func(buf *bytes.Buffer) error {
_, err := prod.conn.DoCommand("POST", "/_bulk", nil, buf)
if err != nil {
Log.Error.Print("ElasticSearch response error - ", err)
}
return err
}
prod.index = conf.GetStreamMap("Index", "")
prod.msgType = conf.GetStreamMap("Type", "log")
prod.msgTTL = conf.GetString("TTL", "")
prod.dayBasedIndex = conf.GetBool("DayBasedIndex", false)
prod.counters = make(map[string]*int64)
prod.lastMetricUpdate = time.Now()
for _, index := range prod.index {
shared.Metric.New(elasticMetricMessages + index)
shared.Metric.New(elasticMetricMessagesSec + index)
prod.counters[index] = new(int64)
}
prod.SetCheckFuseCallback(prod.isClusterUp)
return nil
}
// Configure initializes this consumer with values from a plugin config.
func (cons *PcapHTTPConsumer) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
cons.netInterface = conf.GetString("Interface", "eth0")
cons.promiscuous = conf.GetBool("Promiscuous", true)
cons.filter = conf.GetString("Filter", "dst port 80 and dst host 127.0.0.1")
cons.sessions = make(pcapSessionMap)
cons.sessionTimeout = time.Duration(conf.GetInt("TimeoutMs", 3000)) * time.Millisecond
cons.sessionGuard = new(sync.Mutex)
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *File) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.filesByStream = make(map[core.MessageStreamID]*fileState)
prod.files = make(map[uint32]*fileState)
prod.bufferSizeMax = conf.GetInt("BatchSizeMaxKB", 8<<10) << 10 // 8 MB
prod.batchSize = conf.GetInt("BatchSizeByte", 8192)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
logFile := conf.GetString("File", "/var/prod/gollum.log")
prod.wildcardPath = strings.IndexByte(logFile, '*') != -1
prod.fileDir = filepath.Dir(logFile)
prod.fileExt = filepath.Ext(logFile)
prod.fileName = filepath.Base(logFile)
prod.fileName = prod.fileName[:len(prod.fileName)-len(prod.fileExt)]
prod.timestamp = conf.GetString("RotateTimestamp", "2006-01-02_15")
prod.flushTimeout = time.Duration(conf.GetInt("FlushTimeoutSec", 5)) * time.Second
prod.rotate.enabled = conf.GetBool("Rotate", false)
prod.rotate.timeout = time.Duration(conf.GetInt("RotateTimeoutMin", 1440)) * time.Minute
prod.rotate.sizeByte = int64(conf.GetInt("RotateSizeMB", 1024)) << 20
prod.rotate.atHour = -1
prod.rotate.atMinute = -1
prod.rotate.compress = conf.GetBool("Compress", false)
rotateAt := conf.GetString("RotateAt", "")
if rotateAt != "" {
parts := strings.Split(rotateAt, ":")
rotateAtHour, _ := strconv.ParseInt(parts[0], 10, 8)
rotateAtMin, _ := strconv.ParseInt(parts[1], 10, 8)
prod.rotate.atHour = int(rotateAtHour)
prod.rotate.atMinute = int(rotateAtMin)
}
return nil
}
// Configure sets the database connection values
func (writer *influxDBWriter08) configure(conf core.PluginConfig, prod *InfluxDB) error {
writer.host = conf.GetString("Host", "localhost:8086")
writer.username = conf.GetString("User", "")
writer.password = conf.GetString("Password", "")
writer.databaseTemplate = conf.GetString("Database", "default")
writer.buffer = shared.NewByteStream(4096)
writer.connectionUp = false
writer.timeBasedDBName = conf.GetBool("TimeBasedName", true)
writer.Control = prod.Control
writer.writeURL = fmt.Sprintf("http://%s/db/%%s/series?time_precision=ms", writer.host)
writer.testURL = fmt.Sprintf("http://%s/db", writer.host)
if writer.username != "" {
credentials := fmt.Sprintf("u=%s&p=%s", url.QueryEscape(writer.username), url.QueryEscape(writer.password))
writer.writeURL += "&" + credentials
writer.testURL += "?" + credentials
}
prod.SetCheckFuseCallback(writer.isConnectionUp)
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *ElasticSearch) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
defaultServer := []string{"localhost"}
numConnections := conf.GetInt("Connections", 6)
retrySec := conf.GetInt("RetrySec", 5)
prod.conn = elastigo.NewConn()
prod.conn.Hosts = conf.GetStringArray("Servers", defaultServer)
prod.conn.Domain = conf.GetString("Domain", prod.conn.Hosts[0])
prod.conn.ClusterDomains = prod.conn.Hosts
prod.conn.Port = strconv.Itoa(conf.GetInt("Port", 9200))
prod.conn.Username = conf.GetString("User", "")
prod.conn.Password = conf.GetString("Password", "")
prod.indexer = prod.conn.NewBulkIndexerErrors(numConnections, retrySec)
prod.indexer.BufferDelayMax = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.indexer.BulkMaxBuffer = conf.GetInt("BatchSizeByte", 32768)
prod.indexer.BulkMaxDocs = conf.GetInt("BatchMaxCount", 128)
prod.indexer.Sender = func(buf *bytes.Buffer) error {
_, err := prod.conn.DoCommand("POST", "/_bulk", nil, buf)
if err != nil {
Log.Error.Print("ElasticSearch response error - ", err)
}
return err
}
prod.index = conf.GetStreamMap("Index", "")
prod.msgType = conf.GetStreamMap("Type", "log")
prod.msgTTL = conf.GetString("TTL", "")
prod.dayBasedIndex = conf.GetBool("DayBasedIndex", false)
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *HTTPRequest) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.SetStopCallback(prod.close)
address := conf.GetString("Address", "localhost:80")
prod.protocol, prod.host, prod.port, err = shared.SplitAddress(address, "http")
if err != nil {
return err
}
if prod.host == "" {
prod.host = "localhost"
}
prod.address = fmt.Sprintf("%s://%s:%s", prod.protocol, prod.host, prod.port)
prod.encoding = conf.GetString("Encoding", "text/plain; charset=utf-8")
prod.rawPackets = conf.GetBool("RawData", true)
prod.SetCheckFuseCallback(prod.isHostUp)
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *InfluxDB) Configure(conf core.PluginConfig) error {
if err := prod.ProducerBase.Configure(conf); err != nil {
return err
}
prod.SetStopCallback(prod.close)
version := conf.GetInt("Version", 100)
if conf.GetBool("UseVersion08", false) {
version = 80
}
switch {
case version < 90:
Log.Debug.Print("Using InfluxDB 0.8.x format")
prod.writer = new(influxDBWriter08)
case version == 90:
Log.Debug.Print("Using InfluxDB 0.9.0 format")
prod.writer = new(influxDBWriter09)
default:
Log.Debug.Print("Using InfluxDB 0.9.1+ format")
prod.writer = new(influxDBWriter10)
}
if err := prod.writer.configure(conf, prod); err != nil {
return err
}
prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.batch = core.NewMessageBatch(prod.batchMaxCount)
prod.assembly = core.NewWriterAssembly(prod.writer, prod.Drop, prod.GetFormatter())
return nil
}
// Configure initializes this consumer with values from a plugin config.
func (cons *Socket) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
flags, err := strconv.ParseInt(conf.GetString("Permissions", "0770"), 8, 32)
cons.fileFlags = os.FileMode(flags)
if err != nil {
return err
}
cons.clients = list.New()
cons.clientLock = new(sync.Mutex)
cons.acknowledge = shared.Unescape(conf.GetString("Acknowledge", ""))
cons.address, cons.protocol = shared.ParseAddress(conf.GetString("Address", ":5880"))
cons.reconnectTime = time.Duration(conf.GetInt("ReconnectAfterSec", 2)) * time.Second
cons.ackTimeout = time.Duration(conf.GetInt("AckTimoutSec", 2)) * time.Second
cons.readTimeout = time.Duration(conf.GetInt("ReadTimoutSec", 5)) * time.Second
cons.clearSocket = conf.GetBool("RemoveOldSocket", true)
if cons.protocol != "unix" {
if cons.acknowledge != "" {
cons.protocol = "tcp"
} else {
cons.protocol = "udp"
}
}
cons.delimiter = shared.Unescape(conf.GetString("Delimiter", "\n"))
cons.offset = conf.GetInt("Offset", 0)
cons.flags = 0
partitioner := strings.ToLower(conf.GetString("Partitioner", "delimiter"))
switch partitioner {
case "binary_be":
cons.flags |= shared.BufferedReaderFlagBigEndian
fallthrough
case "binary", "binary_le":
cons.flags |= shared.BufferedReaderFlagEverything
switch conf.GetInt("Size", 4) {
case 1:
cons.flags |= shared.BufferedReaderFlagMLE8
case 2:
cons.flags |= shared.BufferedReaderFlagMLE16
case 4:
cons.flags |= shared.BufferedReaderFlagMLE32
case 8:
cons.flags |= shared.BufferedReaderFlagMLE64
default:
return fmt.Errorf("Size only supports the value 1,2,4 and 8")
}
case "fixed":
cons.flags |= shared.BufferedReaderFlagMLEFixed
cons.offset = conf.GetInt("Size", 1)
case "ascii":
cons.flags |= shared.BufferedReaderFlagMLE
case "delimiter":
// Nothing to add
default:
return fmt.Errorf("Unknown partitioner: %s", partitioner)
}
return err
}
// Configure initializes this producer with values from a plugin config.
func (prod *File) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.SetRollCallback(prod.rotateLog)
prod.SetStopCallback(prod.close)
prod.filesByStream = make(map[core.MessageStreamID]*fileState)
prod.files = make(map[string]*fileState)
prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.overwriteFile = conf.GetBool("FileOverwrite", false)
flags, err := strconv.ParseInt(conf.GetString("Permissions", "0664"), 8, 32)
prod.permissions = os.FileMode(flags)
if err != nil {
return err
}
logFile := conf.GetString("File", "/var/log/gollum.log")
prod.wildcardPath = strings.IndexByte(logFile, '*') != -1
prod.fileDir = filepath.Dir(logFile)
prod.fileExt = filepath.Ext(logFile)
prod.fileName = filepath.Base(logFile)
prod.fileName = prod.fileName[:len(prod.fileName)-len(prod.fileExt)]
prod.timestamp = conf.GetString("RotateTimestamp", "2006-01-02_15")
prod.flushTimeout = time.Duration(conf.GetInt("FlushTimeoutSec", 5)) * time.Second
prod.rotate.enabled = conf.GetBool("Rotate", false)
prod.rotate.timeout = time.Duration(conf.GetInt("RotateTimeoutMin", 1440)) * time.Minute
prod.rotate.sizeByte = int64(conf.GetInt("RotateSizeMB", 1024)) << 20
prod.rotate.atHour = -1
prod.rotate.atMinute = -1
prod.rotate.compress = conf.GetBool("Compress", false)
prod.pruneCount = conf.GetInt("RotatePruneCount", 0)
prod.pruneSize = int64(conf.GetInt("RotatePruneTotalSizeMB", 0)) << 20
if prod.pruneSize > 0 && prod.rotate.sizeByte > 0 {
prod.pruneSize -= prod.rotate.sizeByte >> 20
if prod.pruneSize <= 0 {
prod.pruneCount = 1
prod.pruneSize = 0
}
}
rotateAt := conf.GetString("RotateAt", "")
if rotateAt != "" {
parts := strings.Split(rotateAt, ":")
rotateAtHour, _ := strconv.ParseInt(parts[0], 10, 8)
rotateAtMin, _ := strconv.ParseInt(parts[1], 10, 8)
prod.rotate.atHour = int(rotateAtHour)
prod.rotate.atMinute = int(rotateAtMin)
}
return nil
}
// Configure initializes this consumer with values from a plugin config.
func (cons *Console) Configure(conf core.PluginConfig) error {
cons.autoexit = conf.GetBool("ExitOnEOF", false)
return cons.ConsumerBase.Configure(conf)
}