// Configure sets the database connection values
func (writer *influxDBWriter09) configure(conf core.PluginConfig, prod *InfluxDB) error {
writer.host = conf.GetString("Host", "localhost:8086")
writer.username = conf.GetString("User", "")
writer.password = conf.GetString("Password", "")
writer.databaseTemplate = conf.GetString("Database", "default")
writer.buffer = shared.NewByteStream(4096)
writer.connectionUp = false
writer.Control = prod.Control
writer.writeURL = fmt.Sprintf("http://%s/write", writer.host)
writer.queryURL = fmt.Sprintf("http://%s/query", writer.host)
writer.pingURL = fmt.Sprintf("http://%s/ping", writer.host)
writer.separator = '?'
writer.timeBasedDBName = conf.GetBool("TimeBasedName", true)
if writer.username != "" {
credentials := fmt.Sprintf("?u=%s&p=%s", url.QueryEscape(writer.username), url.QueryEscape(writer.password))
writer.writeURL += credentials
writer.queryURL += credentials
writer.separator = '&'
}
if retentionPolicy := conf.GetString("RetentionPolicy", ""); retentionPolicy != "" {
writer.messageHeader = fmt.Sprintf("{\"database\":\"%%s\",\"retentionPolicy\":\"%s\",\"points\":[", retentionPolicy)
} else {
writer.messageHeader = "{\"database\":\"%s\",\"points\":["
}
prod.SetCheckFuseCallback(writer.isConnectionUp)
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *Scribe) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
host := conf.GetString("Address", "localhost:1463")
bufferSizeMax := conf.GetInt("BatchSizeMaxKB", 8<<10) << 1 // 8 MB
prod.category = make(map[core.MessageStreamID]string, 0)
prod.batchSize = conf.GetInt("BatchSizeByte", 8192)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.batch = createScribeMessageBatch(bufferSizeMax)
prod.bufferSizeKB = conf.GetInt("ConnectionBufferSizeKB", 1<<10) // 1 MB
prod.category = conf.GetStreamMap("Category", "")
// Initialize scribe connection
prod.socket, err = thrift.NewTSocket(host)
if err != nil {
Log.Error.Print("Scribe socket error:", err)
return err
}
prod.transport = thrift.NewTFramedTransport(prod.socket)
binProtocol := thrift.NewTBinaryProtocol(prod.transport, false, false)
prod.scribe = scribe.NewScribeClientProtocol(prod.transport, binProtocol, binProtocol)
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *Redis) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
fieldFormat, err := core.NewPluginWithType(conf.GetString("FieldFormatter", "format.Identifier"), conf)
if err != nil {
return err // ### return, plugin load error ###
}
prod.fieldFormat = fieldFormat.(core.Formatter)
prod.password = conf.GetString("Password", "")
prod.database = int64(conf.GetInt("Database", 0))
prod.key = conf.GetString("Key", "default")
prod.fieldFromParsed = conf.GetBool("FieldAfterFormat", false)
prod.address, prod.protocol = shared.ParseAddress(conf.GetString("Address", ":6379"))
switch strings.ToLower(conf.GetString("Storage", "hash")) {
case "hash":
prod.store = prod.storeHash
case "list":
prod.store = prod.storeList
case "set":
prod.store = prod.storeSet
case "sortedset":
prod.store = prod.storeSortedSet
default:
fallthrough
case "string":
prod.store = prod.storeString
}
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *Socket) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.SetStopCallback(prod.close)
prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.bufferSizeByte = conf.GetInt("ConnectionBufferSizeKB", 1<<10) << 10 // 1 MB
prod.acknowledge = shared.Unescape(conf.GetString("Acknowledge", ""))
prod.ackTimeout = time.Duration(conf.GetInt("AckTimeoutMs", 2000)) * time.Millisecond
prod.address, prod.protocol = shared.ParseAddress(conf.GetString("Address", ":5880"))
if prod.protocol != "unix" {
if prod.acknowledge != "" {
prod.protocol = "tcp"
} else {
prod.protocol = "udp"
}
}
prod.batch = core.NewMessageBatch(prod.batchMaxCount)
prod.assembly = core.NewWriterAssembly(nil, prod.Drop, prod.GetFormatter())
prod.assembly.SetValidator(prod.validate)
prod.assembly.SetErrorHandler(prod.onWriteError)
prod.SetCheckFuseCallback(prod.tryConnect)
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *Socket) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
bufferSizeMax := conf.GetInt("BatchSizeMaxKB", 8<<10) << 10
prod.batchSize = conf.GetInt("BatchSizeByte", 8192)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.bufferSizeKB = conf.GetInt("ConnectionBufferSizeKB", 1<<10) // 1 MB
prod.acknowledge = shared.Unescape(conf.GetString("Acknowledge", ""))
prod.address, prod.protocol = shared.ParseAddress(conf.GetString("Address", ":5880"))
if prod.protocol != "unix" {
if prod.acknowledge != "" {
prod.protocol = "tcp"
} else {
prod.protocol = "udp"
}
}
prod.batch = core.NewMessageBatch(bufferSizeMax, prod.ProducerBase.GetFormatter())
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *Spooling) Configure(conf core.PluginConfig) error {
conf.Override("Formatter", "format.Serialize")
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.SetStopCallback(prod.close)
prod.path = conf.GetString("Path", "/var/run/gollum/spooling")
prod.maxFileSize = int64(conf.GetInt("MaxFileSizeMB", 512)) << 20
prod.maxFileAge = time.Duration(conf.GetInt("MaxFileAgeMin", 1)) * time.Minute
prod.batchMaxCount = conf.GetInt("BatchMaxCount", 100)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.outfile = make(map[core.MessageStreamID]*spoolFile)
prod.rotation = fileRotateConfig{
timeout: prod.maxFileAge,
sizeByte: prod.maxFileSize,
atHour: -1,
atMinute: -1,
enabled: true,
compress: false,
}
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *CollectdToInflux08) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("CollectdToInfluxFormatter", "format.Forward"), conf)
if err != nil {
return err
}
format.base = plugin.(core.Formatter)
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *Runlength) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("RunlengthFormatter", "format.Forward"), conf)
if err != nil {
return err
}
format.base = plugin.(core.Formatter)
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *Hostname) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("HostnameFormatter", "format.Forward"), conf)
if err != nil {
return err
}
format.separator = conf.GetString("HostnameSeparator", " ")
format.base = plugin.(core.Formatter)
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *Serialize) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("SerializeFormatter", "format.Forward"), conf)
if err != nil {
return err
}
format.encode = conf.GetBool("SerializeStringEncode", true)
format.base = plugin.(core.Formatter)
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *CollectdToInflux10) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("CollectdToInflux1009", "format.Forward"), conf)
if err != nil {
return err
}
format.base = plugin.(core.Formatter)
format.tagString = strings.NewReplacer(",", "\\,", " ", "\\ ")
format.stringString = strings.NewReplacer("\"", "\\\"")
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *StreamName) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("StreamNameFormatter", "format.Forward"), conf)
if err != nil {
return err
}
format.separator = conf.GetString("StreamNameSeparator", " ")
format.usePrevious = conf.GetBool("StreamNameHistory", false)
format.base = plugin.(core.Formatter)
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *Timestamp) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("TimestampFormatter", "format.Forward"), conf)
if err != nil {
return err
}
format.base = plugin.(core.Formatter)
format.timestampFormat = conf.GetString("Timestamp", "2006-01-02 15:04:05 MST | ")
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *StreamRoute) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("StreamRouteFormatter", "format.Forward"), conf)
if err != nil {
return err
}
format.delimiter = []byte(conf.GetString("StreamRouteDelimiter", ":"))
format.base = plugin.(core.Formatter)
return nil
}
// Configure initializes this consumer with values from a plugin config.
func (cons *Http) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
cons.address = conf.GetString("Address", ":80")
cons.readTimeoutSec = time.Duration(conf.GetInt("ReadTimeoutSec", 3)) * time.Second
cons.withHeaders = conf.GetBool("WithHeaders", true)
return err
}
// Configure initializes this formatter with values from a plugin config.
func (format *Base64Decode) Configure(conf core.PluginConfig) error {
dict := conf.GetString("Dictionary", "")
if dict == "" {
format.dictionary = base64.StdEncoding
} else {
if len(dict) != 64 {
return fmt.Errorf("Base64 dictionary must contain 64 characters.")
}
format.dictionary = base64.NewEncoding(dict)
}
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *Websocket) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.SetStopCallback(prod.close)
prod.address = conf.GetString("Address", ":81")
prod.path = conf.GetString("Path", "/")
prod.readTimeoutSec = time.Duration(conf.GetInt("ReadTimeoutSec", 3)) * time.Second
return nil
}
// Configure initializes this consumer with values from a plugin config.
func (cons *File) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
if !conf.HasValue("File") {
return core.NewConsumerError("No file configured for consumer.File")
}
cons.file = nil
cons.fileName = conf.GetString("File", "")
cons.offsetFileName = conf.GetString("OffsetFile", "")
cons.delimiter = shared.Unescape(conf.GetString("Delimiter", "\n"))
switch strings.ToLower(conf.GetString("DefaultOffset", fileOffsetEnd)) {
default:
fallthrough
case fileOffsetEnd:
cons.seek = 2
cons.seekOffset = 0
case fileOffsetStart:
cons.seek = 1
cons.seekOffset = 0
}
return nil
}
// Configure sets the database connection values
func (writer *influxDBWriter10) configure(conf core.PluginConfig, prod *InfluxDB) error {
writer.host = conf.GetString("Host", "localhost:8086")
writer.username = conf.GetString("User", "")
writer.password = conf.GetString("Password", "")
writer.databaseTemplate = conf.GetString("Database", "default")
writer.buffer = shared.NewByteStream(4096)
writer.connectionUp = false
writer.timeBasedDBName = conf.GetBool("TimeBasedName", true)
writer.Control = prod.Control
writer.writeURL = fmt.Sprintf("http://%s/write", writer.host)
writer.queryURL = fmt.Sprintf("http://%s/query", writer.host)
writer.pingURL = fmt.Sprintf("http://%s/ping", writer.host)
writer.separator = '?'
if writer.username != "" {
credentials := fmt.Sprintf("?u=%s&p=%s", url.QueryEscape(writer.username), url.QueryEscape(writer.password))
writer.writeURL += credentials
writer.queryURL += credentials
writer.separator = '&'
}
writer.writeURL = fmt.Sprintf("%s%cprecision=ms", writer.writeURL, writer.separator)
prod.SetCheckFuseCallback(writer.isConnectionUp)
return nil
}
// Configure initializes this consumer with values from a plugin config.
func (cons *File) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
cons.SetRollCallback(cons.onRoll)
cons.file = nil
cons.fileName = conf.GetString("File", "/var/run/system.log")
cons.offsetFileName = conf.GetString("OffsetFile", "")
cons.delimiter = shared.Unescape(conf.GetString("Delimiter", "\n"))
switch strings.ToLower(conf.GetString("DefaultOffset", fileOffsetEnd)) {
default:
fallthrough
case fileOffsetEnd:
cons.seek = 2
cons.seekOffset = 0
case fileOffsetStart:
cons.seek = 1
cons.seekOffset = 0
}
return nil
}
// Configure initializes this consumer with values from a plugin config.
func (cons *Socket) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
cons.acknowledge = shared.Unescape(conf.GetString("Acknowledge", ""))
cons.address, cons.protocol = shared.ParseAddress(conf.GetString("Address", ":5880"))
if cons.protocol != "unix" {
if cons.acknowledge != "" {
cons.protocol = "tcp"
} else {
cons.protocol = "udp"
}
}
cons.delimiter = shared.Unescape(conf.GetString("Delimiter", "\n"))
cons.offset = conf.GetInt("Offset", 0)
cons.flags = 0
partitioner := strings.ToLower(conf.GetString("Partitioner", "delimiter"))
switch partitioner {
case "binary_be":
cons.flags |= shared.BufferedReaderFlagBigEndian
fallthrough
case "binary", "binary_le":
cons.flags |= shared.BufferedReaderFlagEverything
switch conf.GetInt("Size", 4) {
case 1:
cons.flags |= shared.BufferedReaderFlagMLE8
case 2:
cons.flags |= shared.BufferedReaderFlagMLE16
case 4:
cons.flags |= shared.BufferedReaderFlagMLE32
case 8:
cons.flags |= shared.BufferedReaderFlagMLE64
default:
return fmt.Errorf("Size only supports the value 1,2,4 and 8")
}
case "fixed":
cons.flags |= shared.BufferedReaderFlagMLEFixed
cons.offset = conf.GetInt("Size", 1)
case "ascii":
cons.flags |= shared.BufferedReaderFlagMLE
case "delimiter":
// Nothing to add
default:
return fmt.Errorf("Unknown partitioner: %s", partitioner)
}
cons.quit = false
return err
}
// Configure initializes this consumer with values from a plugin config.
func (cons *PcapHTTPConsumer) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
cons.netInterface = conf.GetString("Interface", "eth0")
cons.promiscuous = conf.GetBool("Promiscuous", true)
cons.filter = conf.GetString("Filter", "dst port 80 and dst host 127.0.0.1")
cons.sessions = make(pcapSessionMap)
cons.sessionTimeout = time.Duration(conf.GetInt("TimeoutMs", 3000)) * time.Millisecond
cons.sessionGuard = new(sync.Mutex)
return nil
}
// Configure initializes this consumer with values from a plugin config.
func (cons *Profiler) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
numTemplates := conf.GetInt("TemplateCount", 10)
cons.profileRuns = conf.GetInt("Runs", 10000)
cons.batches = conf.GetInt("Batches", 10)
cons.chars = conf.GetString("Characters", profilerDefaultCharacters)
cons.message = conf.GetString("Message", "%# %256s")
cons.templates = make([][]byte, numTemplates)
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *Base64Decode) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("Base64Formatter", "format.Forward"), conf)
if err != nil {
return err
}
format.base = plugin.(core.Formatter)
dict := conf.GetString("Base64Dictionary", "")
if dict == "" {
format.dictionary = base64.StdEncoding
} else {
if len(dict) != 64 {
return fmt.Errorf("Base64 dictionary must contain 64 characters.")
}
format.dictionary = base64.NewEncoding(dict)
}
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *Scribe) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.SetStopCallback(prod.close)
host := conf.GetString("Address", "localhost:1463")
prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
prod.windowSize = prod.batchMaxCount
prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.batch = core.NewMessageBatch(prod.batchMaxCount)
prod.bufferSizeByte = conf.GetInt("ConnectionBufferSizeKB", 1<<10) << 10 // 1 MB
prod.category = conf.GetStreamMap("Category", "")
// Initialize scribe connection
prod.socket, err = thrift.NewTSocket(host)
if err != nil {
Log.Error.Print("Scribe socket error:", err)
return err
}
prod.transport = thrift.NewTFramedTransport(prod.socket)
binProtocol := thrift.NewTBinaryProtocol(prod.transport, false, false)
prod.scribe = scribe.NewScribeClientProtocol(prod.transport, binProtocol, binProtocol)
prod.lastMetricUpdate = time.Now()
prod.counters = make(map[string]*int64)
shared.Metric.New(scribeMetricWindowSize)
shared.Metric.SetI(scribeMetricWindowSize, prod.windowSize)
for _, category := range prod.category {
shared.Metric.New(scribeMetricMessages + category)
shared.Metric.New(scribeMetricMessagesSec + category)
prod.counters[category] = new(int64)
}
prod.SetCheckFuseCallback(prod.tryOpenConnection)
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *Console) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
console := conf.GetString("Console", "stdout")
switch strings.ToLower(console) {
default:
fallthrough
case "stdout":
prod.console = os.Stdout
case "stderr":
prod.console = os.Stderr
}
return nil
}
// Configure initializes this consumer with values from a plugin config.
func (cons *Syslogd) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
cons.address, cons.protocol = shared.ParseAddress(conf.GetString("Address", "udp://0.0.0.0:514"))
format := conf.GetString("Format", "RFC6587")
switch cons.protocol {
case "udp", "tcp", "unix":
default:
return fmt.Errorf("Syslog: unknown protocol type %s", cons.protocol) // ### return, unknown protocol ###
}
switch format {
// http://www.ietf.org/rfc/rfc3164.txt
case "RFC3164":
cons.format = syslog.RFC3164
if cons.protocol == "tcp" {
Log.Warning.Print("Syslog: RFC3164 demands UDP")
cons.protocol = "udp"
}
// https://tools.ietf.org/html/rfc5424
case "RFC5424":
cons.format = syslog.RFC5424
if cons.protocol == "tcp" {
Log.Warning.Print("Syslog: RFC5424 demands UDP")
cons.protocol = "udp"
}
// https://tools.ietf.org/html/rfc6587
case "RFC6587":
cons.format = syslog.RFC6587
default:
err = fmt.Errorf("Syslog: Format %s is not supported", format)
}
cons.sequence = new(uint64)
return err
}
// Configure initializes this filter with values from a plugin config.
func (filter *RegExp) Configure(conf core.PluginConfig) error {
var err error
exp := conf.GetString("FilterExpression", "")
if exp != "" {
filter.exp, err = regexp.Compile(exp)
if err != nil {
return err // ### return, regex parser error ###
}
}
exp = conf.GetString("FilterExpressionNot", "")
if exp != "" {
filter.expNot, err = regexp.Compile(exp)
if err != nil {
return err // ### return, regex parser error ###
}
}
return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *Identifier) Configure(conf core.PluginConfig) error {
plugin, err := core.NewPluginWithType(conf.GetString("IdentifierDataFormatter", "format.Forward"), conf)
if err != nil {
return err
}
format.base = plugin.(core.Formatter)
switch strings.ToLower(conf.GetString("IdentifierType", "time")) {
case "hash":
format.hash = format.idHash
case "seq":
format.hash = format.idSeq
case "seqhex":
format.hash = format.idSeqHex
default:
fallthrough
case "time":
format.hash = format.idTime
}
return nil
}
// Configure initializes this consumer with values from a plugin config.
func (cons *Http) Configure(conf core.PluginConfig) error {
err := cons.ConsumerBase.Configure(conf)
if err != nil {
return err
}
cons.address = conf.GetString("Address", ":80")
cons.readTimeoutSec = time.Duration(conf.GetInt("ReadTimeoutSec", 3)) * time.Second
cons.withHeaders = conf.GetBool("WithHeaders", true)
certificateFile := conf.GetString("Certificate", "")
keyFile := conf.GetString("PrivateKey", "")
if certificateFile != "" || keyFile != "" {
if certificateFile == "" || keyFile == "" {
return fmt.Errorf("There must always be a certificate and a private key or none of both")
}
cons.certificate = new(tls.Config)
cons.certificate.NextProtos = []string{"http/1.1"}
keypair, err := tls.LoadX509KeyPair(certificateFile, keyFile)
if err != nil {
return err
}
cons.certificate.Certificates = []tls.Certificate{keypair}
}
return err
}