Beispiel #1
0
// Configure initializes this filter with values from a plugin config.
func (filter *JSON) Configure(conf core.PluginConfig) error {
	rejectValues := conf.GetStringMap("FilterReject", make(map[string]string))
	acceptValues := conf.GetStringMap("FilterAccept", make(map[string]string))

	// Compile regexp from map[string]string to map[string]*regexp.Regexp
	filter.rejectValues = make(map[string]*regexp.Regexp)
	filter.acceptValues = make(map[string]*regexp.Regexp)

	for key, val := range rejectValues {
		exp, err := regexp.Compile(val)
		if err != nil {
			return err
		}
		filter.rejectValues[key] = exp
	}

	for key, val := range acceptValues {
		exp, err := regexp.Compile(val)
		if err != nil {
			return err
		}
		filter.acceptValues[key] = exp
	}

	return nil
}
Beispiel #2
0
// Configure initializes this consumer with values from a plugin config.
func (cons *File) Configure(conf core.PluginConfig) error {
	err := cons.ConsumerBase.Configure(conf)
	if err != nil {
		return err
	}

	if !conf.HasValue("File") {
		return core.NewConsumerError("No file configured for consumer.File")
	}

	cons.file = nil
	cons.fileName = conf.GetString("File", "")
	cons.offsetFileName = conf.GetString("OffsetFile", "")
	cons.delimiter = shared.Unescape(conf.GetString("Delimiter", "\n"))

	switch strings.ToLower(conf.GetString("DefaultOffset", fileOffsetEnd)) {
	default:
		fallthrough
	case fileOffsetEnd:
		cons.seek = 2
		cons.seekOffset = 0

	case fileOffsetStart:
		cons.seek = 1
		cons.seekOffset = 0
	}

	return nil
}
// Configure sets the database connection values
func (writer *influxDBWriter10) configure(conf core.PluginConfig, prod *InfluxDB) error {
	writer.host = conf.GetString("Host", "localhost:8086")
	writer.username = conf.GetString("User", "")
	writer.password = conf.GetString("Password", "")
	writer.databaseTemplate = conf.GetString("Database", "default")
	writer.buffer = shared.NewByteStream(4096)
	writer.connectionUp = false
	writer.timeBasedDBName = conf.GetBool("TimeBasedName", true)
	writer.Control = prod.Control

	writer.writeURL = fmt.Sprintf("http://%s/write", writer.host)
	writer.queryURL = fmt.Sprintf("http://%s/query", writer.host)
	writer.pingURL = fmt.Sprintf("http://%s/ping", writer.host)
	writer.separator = '?'

	if writer.username != "" {
		credentials := fmt.Sprintf("?u=%s&p=%s", url.QueryEscape(writer.username), url.QueryEscape(writer.password))
		writer.writeURL += credentials
		writer.queryURL += credentials
		writer.separator = '&'
	}

	writer.writeURL = fmt.Sprintf("%s%cprecision=ms", writer.writeURL, writer.separator)
	prod.SetCheckFuseCallback(writer.isConnectionUp)
	return nil
}
Beispiel #4
0
// Configure initializes this consumer with values from a plugin config.
func (cons *Http) Configure(conf core.PluginConfig) error {
	err := cons.ConsumerBase.Configure(conf)
	if err != nil {
		return err
	}

	cons.address = conf.GetString("Address", ":80")
	cons.readTimeoutSec = time.Duration(conf.GetInt("ReadTimeoutSec", 3)) * time.Second
	cons.withHeaders = conf.GetBool("WithHeaders", true)

	certificateFile := conf.GetString("Certificate", "")
	keyFile := conf.GetString("PrivateKey", "")

	if certificateFile != "" || keyFile != "" {
		if certificateFile == "" || keyFile == "" {
			return fmt.Errorf("There must always be a certificate and a private key or none of both")
		}

		cons.certificate = new(tls.Config)
		cons.certificate.NextProtos = []string{"http/1.1"}

		keypair, err := tls.LoadX509KeyPair(certificateFile, keyFile)
		if err != nil {
			return err
		}

		cons.certificate.Certificates = []tls.Certificate{keypair}
	}

	return err
}
// Configure initializes this formatter with values from a plugin config.
func (format *CollectdToInflux08) Configure(conf core.PluginConfig) error {
	plugin, err := core.NewPluginWithType(conf.GetString("CollectdToInfluxFormatter", "format.Forward"), conf)
	if err != nil {
		return err
	}
	format.base = plugin.(core.Formatter)
	return nil
}
Beispiel #6
0
// Configure initializes this formatter with values from a plugin config.
func (format *Runlength) Configure(conf core.PluginConfig) error {
	plugin, err := core.NewPluginWithType(conf.GetString("RunlengthFormatter", "format.Forward"), conf)
	if err != nil {
		return err
	}

	format.base = plugin.(core.Formatter)
	return nil
}
Beispiel #7
0
// Configure initializes this formatter with values from a plugin config.
func (format *Serialize) Configure(conf core.PluginConfig) error {
	plugin, err := core.NewPluginWithType(conf.GetString("SerializeFormatter", "format.Forward"), conf)
	if err != nil {
		return err
	}

	format.encode = conf.GetBool("SerializeStringEncode", true)
	format.base = plugin.(core.Formatter)
	return nil
}
// Configure initializes this formatter with values from a plugin config.
func (format *CollectdToInflux10) Configure(conf core.PluginConfig) error {
	plugin, err := core.NewPluginWithType(conf.GetString("CollectdToInflux1009", "format.Forward"), conf)
	if err != nil {
		return err
	}
	format.base = plugin.(core.Formatter)
	format.tagString = strings.NewReplacer(",", "\\,", " ", "\\ ")
	format.stringString = strings.NewReplacer("\"", "\\\"")
	return nil
}
Beispiel #9
0
// Configure initializes this formatter with values from a plugin config.
func (format *Hostname) Configure(conf core.PluginConfig) error {
	plugin, err := core.NewPluginWithType(conf.GetString("HostnameFormatter", "format.Forward"), conf)
	if err != nil {
		return err
	}

	format.separator = conf.GetString("HostnameSeparator", " ")
	format.base = plugin.(core.Formatter)
	return nil
}
Beispiel #10
0
// Configure initializes this formatter with values from a plugin config.
func (format *Timestamp) Configure(conf core.PluginConfig) error {
	plugin, err := core.NewPluginWithType(conf.GetString("TimestampFormatter", "format.Forward"), conf)
	if err != nil {
		return err
	}

	format.base = plugin.(core.Formatter)
	format.timestampFormat = conf.GetString("Timestamp", "2006-01-02 15:04:05 MST | ")

	return nil
}
Beispiel #11
0
// Configure initializes this formatter with values from a plugin config.
func (format *StreamRoute) Configure(conf core.PluginConfig) error {
	plugin, err := core.NewPluginWithType(conf.GetString("StreamRouteFormatter", "format.Forward"), conf)
	if err != nil {
		return err
	}

	format.delimiter = []byte(conf.GetString("StreamRouteDelimiter", ":"))
	format.base = plugin.(core.Formatter)

	return nil
}
Beispiel #12
0
// Configure initializes this formatter with values from a plugin config.
func (format *Base64Decode) Configure(conf core.PluginConfig) error {
	dict := conf.GetString("Dictionary", "")
	if dict == "" {
		format.dictionary = base64.StdEncoding
	} else {
		if len(dict) != 64 {
			return fmt.Errorf("Base64 dictionary must contain 64 characters.")
		}
		format.dictionary = base64.NewEncoding(dict)
	}
	return nil
}
Beispiel #13
0
// Configure initializes this distributor with values from a plugin config.
func (stream *Route) Configure(conf core.PluginConfig) error {
	if err := stream.StreamBase.ConfigureStream(conf, stream.Broadcast); err != nil {
		return err // ### return, base stream error ###
	}

	routes := conf.GetStringArray("Routes", []string{})
	for _, streamName := range routes {
		targetStream := newStreamWithID(streamName)
		stream.routes = append(stream.routes, targetStream)
	}

	return nil
}
Beispiel #14
0
// Configure initializes this consumer with values from a plugin config.
func (cons *File) Configure(conf core.PluginConfig) error {
	err := cons.ConsumerBase.Configure(conf)
	if err != nil {
		return err
	}

	cons.SetRollCallback(cons.onRoll)

	cons.file = nil
	cons.fileName = conf.GetString("File", "/var/run/system.log")
	cons.offsetFileName = conf.GetString("OffsetFile", "")
	cons.delimiter = shared.Unescape(conf.GetString("Delimiter", "\n"))

	switch strings.ToLower(conf.GetString("DefaultOffset", fileOffsetEnd)) {
	default:
		fallthrough
	case fileOffsetEnd:
		cons.seek = 2
		cons.seekOffset = 0

	case fileOffsetStart:
		cons.seek = 1
		cons.seekOffset = 0
	}

	return nil
}
Beispiel #15
0
// Configure initializes this formatter with values from a plugin config.
func (format *JSON) Configure(conf core.PluginConfig) error {
	format.parser = shared.NewTransitionParser()
	format.state = jsonReadObject
	format.initState = conf.GetString("JSONStartState", "")
	format.timeRead = conf.GetString("JSONTimestampRead", "20060102150405")
	format.timeWrite = conf.GetString("JSONTimestampWrite", "2006-01-02 15:04:05 MST")
	format.parseLock = new(sync.Mutex)

	if !conf.HasValue("JSONDirectives") {
		Log.Warning.Print("JSON formatter has no JSONDirectives setting")
		return nil // ### return, no directives ###
	}

	directiveStrings := conf.GetStringArray("JSONDirectives", []string{})
	if len(directiveStrings) == 0 {
		Log.Warning.Print("JSON formatter has no directives")
		return nil // ### return, no directives ###
	}

	// Parse directives

	parserFunctions := make(map[string]shared.ParsedFunc)
	parserFunctions["key"] = format.readKey
	parserFunctions["val"] = format.readValue
	parserFunctions["esc"] = format.readEscaped
	parserFunctions["dat"] = format.readDate
	parserFunctions["arr"] = format.readArray
	parserFunctions["obj"] = format.readObject
	parserFunctions["end"] = format.readEnd
	parserFunctions["arr+val"] = format.readArrayValue
	parserFunctions["arr+esc"] = format.readArrayEscaped
	parserFunctions["val+end"] = format.readValueEnd
	parserFunctions["esc+end"] = format.readEscapedEnd
	parserFunctions["dat+end"] = format.readDateEnd

	directives := []shared.TransitionDirective{}
	for _, dirString := range directiveStrings {
		directive, err := shared.ParseTransitionDirective(dirString, parserFunctions)
		if err != nil {
			return fmt.Errorf("%s: %s", err.Error(), dirString) // ### return, malformed directive ###
		}
		if format.initState == "" {
			format.initState = directive.State
		}
		directives = append(directives, directive)
	}

	format.parser.AddDirectives(directives)
	return nil
}
Beispiel #16
0
// Configure initializes this consumer with values from a plugin config.
func (cons *Profiler) Configure(conf core.PluginConfig) error {
	err := cons.ConsumerBase.Configure(conf)
	if err != nil {
		return err
	}
	numTemplates := conf.GetInt("TemplateCount", 10)

	cons.profileRuns = conf.GetInt("Runs", 10000)
	cons.batches = conf.GetInt("Batches", 10)
	cons.chars = conf.GetString("Characters", profilerDefaultCharacters)
	cons.message = conf.GetString("Message", "%# %256s")
	cons.templates = make([][]byte, numTemplates)

	return nil
}
Beispiel #17
0
// Configure initializes this consumer with values from a plugin config.
func (cons *LoopBack) Configure(conf core.PluginConfig) error {
	err := cons.ConsumerBase.Configure(conf)
	if err != nil {
		return err
	}

	routes := conf.GetStreamRoutes("Routes")
	cons.routes = make(map[core.MessageStreamID]map[core.MessageStreamID]core.Stream)

	for sourceID, targetIDs := range routes {
		for _, targetID := range targetIDs {
			cons.addRoute(sourceID, targetID)
		}
	}

	core.EnableRetryQueue(conf.GetInt("Channel", 8192))
	return nil
}
Beispiel #18
0
// Configure initializes this formatter with values from a plugin config.
func (format *Base64Decode) Configure(conf core.PluginConfig) error {
	plugin, err := core.NewPluginWithType(conf.GetString("Base64Formatter", "format.Forward"), conf)
	if err != nil {
		return err
	}
	format.base = plugin.(core.Formatter)

	dict := conf.GetString("Base64Dictionary", "")
	if dict == "" {
		format.dictionary = base64.StdEncoding
	} else {
		if len(dict) != 64 {
			return fmt.Errorf("Base64 dictionary must contain 64 characters.")
		}
		format.dictionary = base64.NewEncoding(dict)
	}
	return nil
}
Beispiel #19
0
// Configure initializes this producer with values from a plugin config.
func (prod *Console) Configure(conf core.PluginConfig) error {
	err := prod.ProducerBase.Configure(conf)
	if err != nil {
		return err
	}

	console := conf.GetString("Console", "stdout")

	switch strings.ToLower(console) {
	default:
		fallthrough
	case "stdout":
		prod.console = os.Stdout
	case "stderr":
		prod.console = os.Stderr
	}

	return nil
}
Beispiel #20
0
// Configure initializes this consumer with values from a plugin config.
func (cons *Syslogd) Configure(conf core.PluginConfig) error {
	err := cons.ConsumerBase.Configure(conf)
	if err != nil {
		return err
	}

	cons.address, cons.protocol = shared.ParseAddress(conf.GetString("Address", "udp://0.0.0.0:514"))
	format := conf.GetString("Format", "RFC6587")

	switch cons.protocol {
	case "udp", "tcp", "unix":
	default:
		return fmt.Errorf("Syslog: unknown protocol type %s", cons.protocol) // ### return, unknown protocol ###
	}

	switch format {
	// http://www.ietf.org/rfc/rfc3164.txt
	case "RFC3164":
		cons.format = syslog.RFC3164
		if cons.protocol == "tcp" {
			Log.Warning.Print("Syslog: RFC3164 demands UDP")
			cons.protocol = "udp"
		}

	// https://tools.ietf.org/html/rfc5424
	case "RFC5424":
		cons.format = syslog.RFC5424
		if cons.protocol == "tcp" {
			Log.Warning.Print("Syslog: RFC5424 demands UDP")
			cons.protocol = "udp"
		}

	// https://tools.ietf.org/html/rfc6587
	case "RFC6587":
		cons.format = syslog.RFC6587

	default:
		err = fmt.Errorf("Syslog: Format %s is not supported", format)
	}

	cons.sequence = new(uint64)
	return err
}
Beispiel #21
0
// Configure initializes this filter with values from a plugin config.
func (filter *RegExp) Configure(conf core.PluginConfig) error {
	var err error

	exp := conf.GetString("FilterExpression", "")
	if exp != "" {
		filter.exp, err = regexp.Compile(exp)
		if err != nil {
			return err // ### return, regex parser error ###
		}
	}

	exp = conf.GetString("FilterExpressionNot", "")
	if exp != "" {
		filter.expNot, err = regexp.Compile(exp)
		if err != nil {
			return err // ### return, regex parser error ###
		}
	}

	return nil
}
Beispiel #22
0
// Configure initializes this formatter with values from a plugin config.
func (format *Identifier) Configure(conf core.PluginConfig) error {
	plugin, err := core.NewPluginWithType(conf.GetString("IdentifierDataFormatter", "format.Forward"), conf)
	if err != nil {
		return err
	}
	format.base = plugin.(core.Formatter)

	switch strings.ToLower(conf.GetString("IdentifierType", "time")) {
	case "hash":
		format.hash = format.idHash
	case "seq":
		format.hash = format.idSeq
	case "seqhex":
		format.hash = format.idSeqHex
	default:
		fallthrough
	case "time":
		format.hash = format.idTime
	}
	return nil
}
// Configure sets the database connection values
func (writer *influxDBWriter08) configure(conf core.PluginConfig, prod *InfluxDB) error {
	writer.host = conf.GetString("Host", "localhost:8086")
	writer.username = conf.GetString("User", "")
	writer.password = conf.GetString("Password", "")
	writer.databaseTemplate = conf.GetString("Database", "default")
	writer.buffer = shared.NewByteStream(4096)
	writer.connectionUp = false
	writer.timeBasedDBName = conf.GetBool("TimeBasedName", true)
	writer.Control = prod.Control

	writer.writeURL = fmt.Sprintf("http://%s/db/%%s/series?time_precision=ms", writer.host)
	writer.testURL = fmt.Sprintf("http://%s/db", writer.host)

	if writer.username != "" {
		credentials := fmt.Sprintf("u=%s&p=%s", url.QueryEscape(writer.username), url.QueryEscape(writer.password))
		writer.writeURL += "&" + credentials
		writer.testURL += "?" + credentials
	}

	prod.SetCheckFuseCallback(writer.isConnectionUp)
	return nil
}
Beispiel #24
0
// Configure initializes this consumer with values from a plugin config.
func (cons *PcapHTTPConsumer) Configure(conf core.PluginConfig) error {
	err := cons.ConsumerBase.Configure(conf)
	if err != nil {
		return err
	}

	cons.netInterface = conf.GetString("Interface", "eth0")
	cons.promiscuous = conf.GetBool("Promiscuous", true)
	cons.filter = conf.GetString("Filter", "dst port 80 and dst host 127.0.0.1")
	cons.sessions = make(pcapSessionMap)
	cons.sessionTimeout = time.Duration(conf.GetInt("TimeoutMs", 3000)) * time.Millisecond
	cons.sessionGuard = new(sync.Mutex)

	return nil
}
Beispiel #25
0
// Configure initializes this producer with values from a plugin config.
func (prod *InfluxDB) Configure(conf core.PluginConfig) error {
	if err := prod.ProducerBase.Configure(conf); err != nil {
		return err
	}
	prod.SetStopCallback(prod.close)

	version := conf.GetInt("Version", 100)
	if conf.GetBool("UseVersion08", false) {
		version = 80
	}

	switch {
	case version < 90:
		Log.Debug.Print("Using InfluxDB 0.8.x format")
		prod.writer = new(influxDBWriter08)
	case version == 90:
		Log.Debug.Print("Using InfluxDB 0.9.0 format")
		prod.writer = new(influxDBWriter09)
	default:
		Log.Debug.Print("Using InfluxDB 0.9.1+ format")
		prod.writer = new(influxDBWriter10)
	}

	if err := prod.writer.configure(conf, prod); err != nil {
		return err
	}

	prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
	prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
	prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
	prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second

	prod.batch = core.NewMessageBatch(prod.batchMaxCount)
	prod.assembly = core.NewWriterAssembly(prod.writer, prod.Drop, prod.GetFormatter())
	return nil
}
Beispiel #26
0
// Configure initializes this formatter with values from a plugin config.
func (format *StreamName) Configure(conf core.PluginConfig) error {
	plugin, err := core.NewPluginWithType(conf.GetString("StreamNameFormatter", "format.Forward"), conf)
	if err != nil {
		return err
	}
	format.separator = conf.GetString("StreamNameSeparator", " ")
	format.usePrevious = conf.GetBool("StreamNameHistory", false)
	format.base = plugin.(core.Formatter)
	return nil
}
Beispiel #27
0
// Configure initializes this consumer with values from a plugin config.
func (cons *Http) Configure(conf core.PluginConfig) error {
	err := cons.ConsumerBase.Configure(conf)
	if err != nil {
		return err
	}

	cons.address = conf.GetString("Address", ":80")
	cons.readTimeoutSec = time.Duration(conf.GetInt("ReadTimeoutSec", 3)) * time.Second
	cons.withHeaders = conf.GetBool("WithHeaders", true)
	return err
}
Beispiel #28
0
// Configure initializes this formatter with values from a plugin config.
func (format *Envelope) Configure(conf core.PluginConfig) error {
	plugin, err := core.NewPluginWithType(conf.GetString("EnvelopeFormatter", "format.Forward"), conf)
	if err != nil {
		return err
	}

	format.base = plugin.(core.Formatter)
	format.prefix = shared.Unescape(conf.GetString("EnvelopePrefix", ""))
	format.postfix = shared.Unescape(conf.GetString("EnvelopePostfix", "\n"))

	return nil
}
Beispiel #29
0
// Configure initializes this producer with values from a plugin config.
func (prod *Websocket) Configure(conf core.PluginConfig) error {
	err := prod.ProducerBase.Configure(conf)
	if err != nil {
		return err
	}
	prod.SetStopCallback(prod.close)

	prod.address = conf.GetString("Address", ":81")
	prod.path = conf.GetString("Path", "/")
	prod.readTimeoutSec = time.Duration(conf.GetInt("ReadTimeoutSec", 3)) * time.Second

	return nil
}
Beispiel #30
0
// Configure initializes this producer with values from a plugin config.
func (prod *HTTPRequest) Configure(conf core.PluginConfig) error {
	err := prod.ProducerBase.Configure(conf)
	if err != nil {
		return err
	}
	prod.SetStopCallback(prod.close)

	address := conf.GetString("Address", "localhost:80")
	prod.protocol, prod.host, prod.port, err = shared.SplitAddress(address, "http")
	if err != nil {
		return err
	}

	if prod.host == "" {
		prod.host = "localhost"
	}

	prod.address = fmt.Sprintf("%s://%s:%s", prod.protocol, prod.host, prod.port)
	prod.encoding = conf.GetString("Encoding", "text/plain; charset=utf-8")
	prod.rawPackets = conf.GetBool("RawData", true)
	prod.SetCheckFuseCallback(prod.isHostUp)
	return nil
}