예제 #1
0
func (l *RedisInput) Init(config inputs.MothershipConfig) error {

	l.Config = config

	if config.Host == "" {
		return errors.New("No Input Host specified")
	}
	l.Host = config.Host

	if config.Port == 0 {
		return errors.New("No Input Port specified")
	}
	l.Port = config.Port

	l.DB = config.DB

	if config.Key == "" {
		return errors.New("No Input Key specified")
	}
	l.Key = strings.TrimSpace(config.Key)

	if config.Type == "" {
		return errors.New("No Event Type specified")
	}
	l.Type = strings.TrimSpace(config.Type)

	logp.Debug("redisinput", "Using Host %s", l.Host)
	logp.Debug("redisinput", "Using Port %d", l.Port)
	logp.Debug("redisinput", "Using Database %d", l.DB)
	logp.Debug("redisinput", "Using Key %s", l.Key)
	logp.Debug("redisinput", "Adding Event Type %s", l.Type)

	return nil
}
예제 #2
0
func (l *RedisInput) averageSortedEvents(sorted_events map[string][]common.MapStr) ([]common.MapStr, error) {
	var output_events []common.MapStr
	var merged_event common.MapStr
	var metric_value_string string
	//var metric_value_bytes []byte
	metric_value := 0.0
	for _, events := range sorted_events {
		metric_value = 0.0
		merged_event = common.MapStr{}
		for _, event := range events {
			merged_event.Update(event)
			logp.Debug("groupstuff", "metric value: %v", event["metric_value"])
			metric_value_string = event["metric_value"].(string)
			//			metric_value_bytes = []byte(metric_value_string)
			//			metric_value += float64(common.Bytes_Ntohll(metric_value_bytes))
			metric_value_float, err := strconv.ParseFloat(metric_value_string, 65)
			if err != nil {
				logp.Err("Error parsing metric_value: %v", err)
			}
			metric_value += metric_value_float
		}
		logp.Debug("groupstuff", "the summed values is %v", metric_value)
		logp.Debug("groupstuff", "the length is %v", float64(len(events)))
		metric_value = metric_value / float64(len(events))
		logp.Debug("groupstuff", "the avg value is %v", metric_value)
		merged_event["metric_value"] = metric_value
		output_events = append(output_events, merged_event)
	}
	return output_events, nil
}
예제 #3
0
func FiltersRun(config common.MapStr, plugins map[Filter]FilterPlugin,
	next chan common.MapStr, stopCb func()) (input chan common.MapStr, err error) {

	logp.Debug("filters", "Initializing filters plugins")

	for filter, plugin := range plugins {
		Filters.Register(filter, plugin)
	}
	filters_plugins, err :=
		LoadConfiguredFilters(config)
	if err != nil {
		return nil, fmt.Errorf("Error loading filters plugins: %v", err)
	}
	logp.Debug("filters", "Filters plugins order: %v", filters_plugins)

	if len(filters_plugins) > 0 {
		runner := NewFilterRunner(next, filters_plugins)
		go func() {
			err := runner.Run()
			if err != nil {
				logp.Critical("Filters runner failed: %v", err)
				// shutting down
				stopCb()
			}
		}()
		input = runner.FiltersQueue
	} else {
		input = next
	}

	return input, nil
}
예제 #4
0
//TODO: Check for Errors Here
func (jsonexpander *JSONExpander) Filter(event common.MapStr) (common.MapStr, error) {
	text := event["message"]
	text_string := text.(*string)
	logp.Debug("jsonexpander", "Attempting to expand: %v", event)

	if isJSONString(*text_string) {
		data := []byte(*text_string)
		err := json.Unmarshal(data, &event)
		if err != nil {
			logp.Err("jsonexpander", "Could not expand json data")
			return event, nil
		}
	} else {
		logp.Debug("jsonexpander", "Message does not appear to be JSON data: %s", text_string)
	}

	now := func() time.Time {
		t := time.Now()
		return t
	}

	event.EnsureTimestampField(now)

	logp.Debug("jsonexpander", "Final Event: %v", event)
	return event, nil
}
예제 #5
0
// Each shipper publishes a list of IPs together with its name to Elasticsearch
func (out *ElasticsearchOutput) PublishIPs(name string, localAddrs []string) error {
	if !out.ttlEnabled {
		logp.Debug("output_elasticsearch", "Not publishing IPs because TTL was not yet confirmed to be enabled")
		return nil
	}

	logp.Debug("output_elasticsearch", "Publish IPs %s with expiration time %d", localAddrs, out.TopologyExpire)
	params := map[string]string{
		"ttl":     fmt.Sprintf("%dms", out.TopologyExpire),
		"refresh": "true",
	}
	_, err := out.Conn.Index(
		".packetbeat-topology", /*index*/
		"server-ip",            /*type*/
		name,                   /* id */
		params,                 /* parameters */
		PublishedTopology{name, strings.Join(localAddrs, ",")} /* body */)

	if err != nil {
		logp.Err("Fail to publish IP addresses: %s", err)
		return err
	}

	out.UpdateLocalTopologyMap()

	return nil
}
예제 #6
0
func (out *KafkaOutput) SendMessagesGoroutine() {

	for {
		select {

		case queueMsg := <-out.sendingQueue:

			if !out.connected {
				logp.Debug("output_kafka", "Droping pkt ...")
				continue
			}
			logp.Debug("output_kafka", "Send event to kafka")

			out.Producer.Input() <- &sarama.ProducerMessage{
				Topic: out.Topic,
				Key:   nil,
				Value: &queueMsg,
			}

		case err := <-out.Producer.Errors():
			logp.Err("Failed to publish event to kafka: %s", err)
			out.connected = false
			out.Close()
			go out.Reconnect()
			return
		}
	}
}
예제 #7
0
// Create an HTTP request and send it to Elasticsearch. The request is retransmitted max_retries
// before returning an error.
func (es *Elasticsearch) Request(method string, path string,
	params map[string]string, body interface{}) ([]byte, error) {

	var errors []error

	for attempt := 0; attempt < es.MaxRetries; attempt++ {

		conn := es.connectionPool.GetConnection()
		logp.Debug("elasticsearch", "Use connection %s", conn.Url)

		url := conn.Url + path
		if len(params) > 0 {
			url = url + "?" + UrlEncode(params)
		}

		logp.Debug("elasticsearch", "%s %s %s", method, url, body)

		var obj []byte
		var err error
		if body != nil {
			obj, err = json.Marshal(body)
			if err != nil {
				return nil, fmt.Errorf("Fail to JSON encode the body: %s", err)
			}
		} else {
			obj = nil
		}
		req, err := http.NewRequest(method, url, bytes.NewReader(obj))
		if err != nil {
			return nil, fmt.Errorf("NewRequest fails: %s", err)
		}

		resp, retry, err := es.PerformRequest(conn, req)
		if retry == true {
			// retry
			if err != nil {
				errors = append(errors, err)
			}
			continue
		}
		if err != nil {
			return nil, err
		}
		return resp, nil

	}

	logp.Warn("Request fails to be send after %d retries", es.MaxRetries)

	return nil, fmt.Errorf("Request fails after %d retries. Errors: %v", es.MaxRetries, errors)
}
예제 #8
0
func (l *RedisInput) Run(output chan common.MapStr) error {
	logp.Debug("redisinput", "Running Redis Input")
	var keysScript = redis.NewScript(1, `return redis.call('KEYS', KEYS[1])`)

	go func() {
		redisURL := fmt.Sprintf("redis://%s:%d/%d", l.Host, l.Port, l.DB)
		dialConnectTimeout := redis.DialConnectTimeout(3 * time.Second)
		dialReadTimeout := redis.DialReadTimeout(10 * time.Second)
		var backOffCount = 0
		var backOffDuration time.Duration = 5 * time.Second
		for {
			logp.Debug("redisinput", "Connecting to: %s", redisURL)
			server, err := redis.DialURL(redisURL, dialConnectTimeout, dialReadTimeout)
			if err != nil {
				logp.Err("couldn't start listening: " + err.Error())
				return
			}
			logp.Debug("redisinput", "Connected to Redis Server")

			reply, err := keysScript.Do(server, "*")
			if err != nil {
				logp.Err("An error occured while executing KEYS command: %s\n", err)
				return
			}

			keys, err := redis.Strings(reply, err)
			if err != nil {
				logp.Err("An error occured while converting reply to String: %s\n", err)
				return
			}

			for _, key := range keys {
				logp.Debug("redisinput", "key is %s", key)
				lineCount, err := l.handleConn(server, output, key)
				if err == nil {
					logp.Debug("redisinput", "Read %v events", lineCount)
					backOffCount = 0
					backOffDuration = time.Duration(backOffCount) * time.Second
					time.Sleep(backOffDuration)
				} else {
					backOffCount++
					backOffDuration = time.Duration(backOffCount) * time.Second
					time.Sleep(backOffDuration)
				}
			}
			defer server.Close()
		}
	}()
	return nil
}
예제 #9
0
// Create a HTTP request containing a bunch of operations and send them to Elasticsearch.
// The request is retransmitted up to max_retries before returning an error.
func (es *Elasticsearch) BulkRequest(method string, path string,
	params map[string]string, body chan interface{}) ([]byte, error) {

	var buf bytes.Buffer
	enc := json.NewEncoder(&buf)
	for obj := range body {
		enc.Encode(obj)
	}

	if buf.Len() == 0 {
		logp.Debug("elasticsearch", "Empty channel. Wait for more data.")
		return nil, nil
	}

	var errors []error

	for attempt := 0; attempt < es.MaxRetries; attempt++ {

		conn := es.connectionPool.GetConnection()
		logp.Debug("elasticsearch", "Use connection %s", conn.Url)

		url := conn.Url + path
		if len(params) > 0 {
			url = url + "?" + UrlEncode(params)
		}
		logp.Debug("elasticsearch", "Sending bulk request to %s", url)

		req, err := http.NewRequest(method, url, &buf)
		if err != nil {
			return nil, fmt.Errorf("NewRequest fails: %s", err)
		}

		resp, retry, err := es.PerformRequest(conn, req)
		if retry == true {
			// retry
			if err != nil {
				errors = append(errors, err)
			}
			continue
		}
		if err != nil {
			return nil, fmt.Errorf("PerformRequest fails: %s", err)
		}
		return resp, nil
	}

	logp.Warn("Request fails to be send after %d retries", es.MaxRetries)

	return nil, fmt.Errorf("Request fails after %d retries. Errors: %v", es.MaxRetries, errors)
}
예제 #10
0
func (l *StdinInput) doStuff(output chan common.MapStr) {
	reader := bufio.NewReader(os.Stdin)
	buffer := new(bytes.Buffer)

	var source string = fmt.Sprintf("%s:%s", os.Getenv("REMOTE_HOST"), os.Getenv("REMOTE_PORT"))
	var ssl_client_dn string = os.Getenv("SSL_CLIENT_DN")
	var offset int64 = 0
	var line uint64 = 0
	var read_timeout = 10 * time.Second

	logp.Debug("stdinput", "Handling New Connection from %s", source)

	now := func() time.Time {
		t := time.Now()
		return t
	}

	for {
		text, bytesread, err := l.readline(reader, buffer, read_timeout)

		if err != nil {
			logp.Info("Unexpected state reading from %v; error: %s\n", os.Getenv("SSL_CLIENT_DN"), err)
			return
		}

		logp.Debug("stdinputlines", "New Line: %s", &text)

		line++

		event := common.MapStr{}
		event["ssl_client_dn"] = &ssl_client_dn
		event["source"] = &source
		event["offset"] = offset
		event["line"] = line
		event["message"] = text
		event["type"] = l.Type

		event.EnsureTimestampField(now)
		event.EnsureCountField()

		offset += int64(bytesread)

		logp.Debug("stdinput", "InputEvent: %v", event)
		output <- event // ship the new event downstream
		os.Stdout.Write([]byte("OK"))
	}
	logp.Debug("stdinput", "Closed Connection from %s", source)
}
예제 #11
0
// If a connection fails, it will be marked as dead and put on timeout.
// timeout = default_timeout * 2 ** (fail_count - 1)
// When the timeout is over, the connection will be resurrected and
// returned to the live pool
func (pool *ConnectionPool) MarkDead(conn *Connection) error {

	if !conn.dead {
		logp.Debug("elasticsearch", "Mark dead %s", conn.Url)
		conn.dead = true
		conn.dead_count = conn.dead_count + 1
		timeout := pool.Dead_timeout * time.Duration(math.Pow(2, float64(conn.dead_count)-1))
		conn.timer = time.AfterFunc(timeout*time.Second, func() {
			// timeout expires
			conn.dead = false
			logp.Debug("elasticsearch", "Timeout expired. Mark it as alive: %s", conn.Url)
		})
	}

	return nil
}
예제 #12
0
// Handles OS signals that ask the service/daemon to stop.
// The stopFunction should break the loop in the Beat so that
// the service shut downs gracefully.
func HandleSignals(stopFunction func()) {
	// On ^C or SIGTERM, gracefully stop the sniffer
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		<-sigc
		logp.Debug("service", "Received sigterm/sigint, stopping")
		stopFunction()
	}()

	// Handle the Windows service events
	go ProcessWindowsControlEvents(func() {
		logp.Debug("service", "Received svc stop/shutdown request")
		stopFunction()
	})
}
예제 #13
0
func (l *ProcfsInput) Run(output chan common.MapStr) error {
	logp.Debug("[procfsinput]", "Starting up Procfs Input")

	go inputs.PeriodicTaskRunner(l, output, runTick, runMinor, runMajor)

	return nil
}
예제 #14
0
// Update the local topology map
func (out *ElasticsearchOutput) UpdateLocalTopologyMap() {

	// get all shippers IPs from Elasticsearch
	TopologyMapTmp := make(map[string]string)

	res, err := out.Conn.SearchUri(".packetbeat-topology", "server-ip", nil)
	if err == nil {
		for _, obj := range res.Hits.Hits {
			var result QueryResult
			err = json.Unmarshal(obj, &result)
			if err != nil {
				return
			}

			var pub PublishedTopology
			err = json.Unmarshal(result.Source, &pub)
			if err != nil {
				logp.Err("json.Unmarshal fails with: %s", err)
			}
			// add mapping
			ipaddrs := strings.Split(pub.IPs, ",")
			for _, addr := range ipaddrs {
				TopologyMapTmp[addr] = pub.Name
			}
		}
	} else {
		logp.Err("Getting topology map fails with: %s", err)
	}

	// update topology map
	out.TopologyMap = TopologyMapTmp

	logp.Debug("output_elasticsearch", "Topology map %s", out.TopologyMap)
}
예제 #15
0
// Publish an event by adding it to the queue of events.
func (out *ElasticsearchOutput) PublishEvent(ts time.Time, event common.MapStr) error {

	out.sendingQueue <- EventMsg{Ts: ts, Event: event}

	logp.Debug("output_elasticsearch", "Publish event: %s", event)
	return nil
}
예제 #16
0
파일: tail.go 프로젝트: inst-tech/turnbeat
func (l *TailInput) doStuff(output chan common.MapStr) {
	now := func() time.Time {
		t := time.Now()
		return t
	}

	var line uint64 = 0
	var read_timeout = 30 * time.Second

	// open file
	// basic error handling, if we hit an error, log and return
	// this ends the currently running thread without impacting other threads
	f, err := os.Open(l.FileName)
	if err != nil {
		logp.Err("Error opening file " + err.Error())
		return
	}
	l.FileP = f

	// seek to end
	// for offset, we use the actual file offset
	// we initialize it to the end of the file at time of open
	l.offset, err = l.FileP.Seek(0, 2)
	if err != nil {
		logp.Err("Error seeking in file " + err.Error())
		return
	}
	l.LastOpen = time.Now()

	buffer := new(bytes.Buffer)
	reader := bufio.NewReader(l.FileP)

	for {
		l.CheckReopen()
		text, bytesread, err := readline(reader, buffer, read_timeout)
		if err != nil && err != io.EOF {
			// EOF errors are expected, since we are tailing the file
			logp.Err("Error reading file " + err.Error())
			return
		}

		if bytesread > 0 {
			l.offset += int64(bytesread)
			line++

			event := common.MapStr{}
			event["filename"] = l.FileName
			event["line"] = line
			event["message"] = text
			event["offset"] = l.offset
			event["type"] = l.Type

			event.EnsureTimestampField(now)
			event.EnsureCountField()

			logp.Debug("tailinput", "InputEvent: %v", event)
			output <- event // ship the new event downstream
		}
	}
}
예제 #17
0
파일: tcp.go 프로젝트: inst-tech/turnbeat
func (l *TcpInput) handleConn(client net.Conn, output chan common.MapStr) {
	reader := bufio.NewReader(client)
	buffer := new(bytes.Buffer)

	var source string = client.RemoteAddr().String()
	var offset int64 = 0
	var line uint64 = 0
	var read_timeout = 10 * time.Second

	logp.Debug("tcpinput", "Handling New Connection from %s", source)

	now := func() time.Time {
		t := time.Now()
		return t
	}

	for {
		text, bytesread, err := l.readline(reader, buffer, read_timeout)

		if err != nil {
			logp.Info("Unexpected state reading from %v; error: %s\n", client.RemoteAddr().String, err)
			return
		}

		logp.Debug("tcpinputlines", "New Line: %s", &text)

		line++

		event := common.MapStr{}
		event["source"] = &source
		event["offset"] = offset
		event["line"] = line
		event["message"] = text
		event["type"] = l.Type

		event.EnsureTimestampField(now)
		event.EnsureCountField()

		offset += int64(bytesread)

		logp.Debug("tcpinput", "InputEvent: %v", event)
		output <- event // ship the new event downstream
		client.Write([]byte("OK"))
	}
	logp.Debug("tcpinput", "Closed Connection from %s", source)
}
예제 #18
0
파일: null.go 프로젝트: inst-tech/turnbeat
// If you had a periodic type input, use the below as the "Run" method instead of the above "Run"
func (l *NullInput) RunPeriodic(output chan common.MapStr) error {
	logp.Debug("[nullinput]", "Starting up Null Input")

	// use the runTick for tick interval, empty functions for minor and major
	go inputs.PeriodicTaskRunner(l, output, l.doStuff, inputs.EmptyFunc, inputs.EmptyFunc)

	return nil
}
예제 #19
0
func PrintPublishEvent(event common.MapStr) {
	json, err := json.MarshalIndent(event, "", "  ")
	if err != nil {
		logp.Err("json.Marshal: %s", err)
	} else {
		logp.Debug("publish", "Publish: %s", string(json))
	}
}
예제 #20
0
func (reader *ReaderType) PrintReaderEvent(event common.MapStr) {
	json, err := json.MarshalIndent(event, "", "  ")
	if err != nil {
		logp.Err("json.Marshal: %s", err)
	} else {
		logp.Debug("reader", "Reader: %s", string(json))
	}
}
예제 #21
0
func (l *PackagesInput) Run(output chan common.MapStr) error {
	logp.Debug("[PackagesInput]", "Running Packages Input")

	// dispatch thread here
	go inputs.PeriodicTaskRunner(l, output, l.doStuff, inputs.EmptyFunc, inputs.EmptyFunc)

	return nil
}
예제 #22
0
// LoadConfiguredFilters interprets the [filters] configuration, loads the configured
// plugins and returns the order in which they need to be executed.
func LoadConfiguredFilters(config map[string]interface{}) ([]FilterPlugin, error) {
	var err error
	plugins := []FilterPlugin{}

	filters_list, exists := config["filters"]
	if !exists {
		return plugins, nil
	}
	filters_iface, ok := filters_list.([]interface{})
	if !ok {
		return nil, fmt.Errorf("Expected the filters to be an array of strings")
	}

	for _, filter_iface := range filters_iface {
		filter, ok := filter_iface.(string)
		if !ok {
			return nil, fmt.Errorf("Expected the filters array to only contain strings")
		}
		cfg, exists := config[filter]
		var plugin_type Filter
		var plugin_config map[string]interface{}
		if !exists {
			// Maybe default configuration by name
			plugin_type, err = FilterFromName(filter)
			if err != nil {
				return nil, fmt.Errorf("No such filter type and no corresponding configuration: %s", filter)
			}
		} else {
			logp.Debug("filters", "%v", cfg)
			plugin_config, ok := cfg.(map[interface{}]interface{})
			if !ok {
				return nil, fmt.Errorf("Invalid configuration for: %s", filter)
			}
			type_str, ok := plugin_config["type"].(string)
			if !ok {
				return nil, fmt.Errorf("Couldn't get type for filter: %s", filter)
			}
			plugin_type, err = FilterFromName(type_str)
			if err != nil {
				return nil, fmt.Errorf("No such filter type: %s", type_str)
			}
		}

		filter_plugin := Filters.Get(plugin_type)
		if filter_plugin == nil {
			return nil, fmt.Errorf("No plugin loaded for %s", plugin_type)
		}
		plugin, err := filter_plugin.New(filter, plugin_config)
		if err != nil {
			return nil, fmt.Errorf("Initializing filter plugin %s failed: %v",
				plugin_type, err)
		}
		plugins = append(plugins, plugin)

	}

	return plugins, nil
}
예제 #23
0
파일: tcp.go 프로젝트: inst-tech/turnbeat
func (l *TcpInput) Init(config inputs.MothershipConfig) error {

	l.Config = config
	if config.Port == 0 {
		return errors.New("No Input Port specified")
	}
	l.Port = config.Port

	if config.Type == "" {
		return errors.New("No Event Type specified")
	}
	l.Type = config.Type

	logp.Debug("tcpinput", "Using Port %d", l.Port)
	logp.Debug("tcpinput", "Adding Event Type %s", l.Type)

	return nil
}
예제 #24
0
// A connection that has been previously marked as dead and succeeds will be marked
// as live and the dead_count is set to zero
func (pool *ConnectionPool) MarkLive(conn *Connection) error {
	if conn.dead {
		logp.Debug("elasticsearch", "Mark live %s", conn.Url)
		conn.dead = false
		conn.dead_count = 0
		conn.timer.Stop()
	}
	return nil
}
예제 #25
0
func (r *GraphiteMetricExp) FindStringSubmatchMap(s string) (map[string]string, error) {
	captures := make(map[string]string)

	match := r.FindStringSubmatch(s)

	if match == nil {
		return captures, errors.New("Line did not match regex")
	}

	logp.Debug("filter_graphite", "Regex Matches: %v", match)

	for i, name := range r.SubexpNames() {
		//the first is the original string, skip it
		if i == 0 {
			continue
		}
		captures[name] = strings.TrimSpace(match[i])
	}
	logp.Debug("filter_graphite", "Completed Captures Array: %v", captures)

	if _, present := captures["metric_tags"]; present == false {
		captures["metric_tags"] = ""
	}

	if _, present := captures["metric_value"]; present == false {
		captures["metric_value"] = "+1"
		captures["metric_verb"] = "increment"
	} else if captures["metric_value"] == "" {
		captures["metric_value"] = "+1"
		captures["metric_verb"] = "increment"
	} else {
		captures["metric_verb"] = "put"
	}

	var tags string = ""
	tags = buildTagString(tags, "datacenter", captures)
	tags = buildTagString(tags, "host", captures)
	tags = buildTagString(tags, "service_type", captures)
	tags = buildTagString(tags, "service_id", captures)
	tags = buildTagString(tags, "component", captures)

	captures["metric_tags"] += fmt.Sprintf(" %s", tags)
	return captures, nil
}
예제 #26
0
파일: tail.go 프로젝트: inst-tech/turnbeat
func (l *TailInput) Run(output chan common.MapStr) error {
	logp.Debug("[TailInput]", "Running File Input")

	// dispatch thread here
	go func(output chan common.MapStr) {
		l.doStuff(output)
	}(output)

	return nil
}
예제 #27
0
파일: tail.go 프로젝트: inst-tech/turnbeat
func readline(reader *bufio.Reader, buffer *bytes.Buffer, eof_timeout time.Duration) (*string, int, error) {
	var is_partial bool = true
	var newline_length int = 1
	start_time := time.Now()

	logp.Debug("tcpinputlines", "Readline Called")

	for {
		segment, err := reader.ReadBytes('\n')

		if segment != nil && len(segment) > 0 {
			if segment[len(segment)-1] == '\n' {
				// Found a complete line
				is_partial = false

				// Check if also a CR present
				if len(segment) > 1 && segment[len(segment)-2] == '\r' {
					newline_length++
				}
			}

			// TODO(sissel): if buffer exceeds a certain length, maybe report an error condition? chop it?
			buffer.Write(segment)
		}

		if err != nil {
			if err == io.EOF && is_partial {
				time.Sleep(1 * time.Second) // TODO(sissel): Implement backoff

				// Give up waiting for data after a certain amount of time.
				// If we time out, return the error (eof)
				if time.Since(start_time) > eof_timeout {
					return nil, 0, err
				}
				continue
			} else {
				//emit("error: Harvester.readLine: %s", err.Error())
				return nil, 0, err // TODO(sissel): don't do this?
			}
		}

		// If we got a full line, return the whole line without the EOL chars (CRLF or LF)
		if !is_partial {
			// Get the str length with the EOL chars (LF or CRLF)
			bufferSize := buffer.Len()
			str := new(string)
			*str = buffer.String()[:bufferSize-newline_length]
			// Reset the buffer for the next line
			buffer.Reset()
			return str, bufferSize, nil
		}
	} /* forever read chunks */

	return nil, 0, nil
}
예제 #28
0
func (out *KafkaOutput) PublishEvent(ts time.Time, event common.MapStr) error {

	json_event, err := json.Marshal(event)
	if err != nil {
		logp.Err("Failed to convert the event to JSON: %s", err)
		return err
	}

	out.sendingQueue <- KafkaQueueMsg{msg: json_event}

	logp.Debug("output_kafka", "Publish event")
	return nil
}
예제 #29
0
파일: udp.go 프로젝트: inst-tech/turnbeat
func (l *UdpInput) handlePacket(buffer []byte, rlen int, count int, source *net.UDPAddr, output chan common.MapStr) {
	now := func() time.Time {
		t := time.Now()
		return t
	}

	text := string(buffer[0:rlen])

	logp.Debug("udpinputlines", "New Line: %s", &text)

	event := common.MapStr{}
	event["source"] = &source
	event["offset"] = rlen
	event["line"] = count
	event["message"] = &text
	event["type"] = l.Type

	event.EnsureTimestampField(now)
	event.EnsureCountField()

	logp.Debug("udpinput", "InputEvent: %v", event)
	output <- event // ship the new event downstream
}
예제 #30
0
func (reader *ReaderType) Init(inputMap map[string]inputs.MothershipConfig) error {
	logp.Info("reader input config", inputMap)

	var globalConf inputs.MothershipConfig

	for inputId, config := range inputMap {
		// default instance 0
		inputName, instance := inputId, "0"
		if strings.Contains(inputId, "_") {
			// otherwise grok tcp_2 as inputName = tcp, instance = 2
			sv := strings.Split(inputId, "_")
			inputName, instance = sv[0], sv[1]
		}
		logp.Info(fmt.Sprintf("input type: %s instance: %s\n", inputName, instance))
		logp.Debug("reader", "instance config: %s", config)

		// handling for "global" config section
		if inputName == "global" {
			logp.Info("global input configuration read")
			globalConf = config
		}

		plugin := newInputInstance(inputName)
		if plugin != nil && config.Enabled {
			config.Normalize(globalConf)
			err := plugin.Init(config)
			if err != nil {
				logp.Err("Fail to initialize %s plugin as input: %s", inputName, err)
				return err
			} else {
				logp.Info("Initialized %s plugin as input", inputName)
			}
			reader.Input = append(reader.Input, plugin)
		}
	}

	if len(reader.Input) == 0 {
		logp.Info("No inputs are defined. Please define one under the input section.")
		return errors.New("No input are defined. Please define one under the input section.")
	} else {
		logp.Info("%d inputs defined", len(reader.Input))
	}

	return nil
}