func (l *TcpInput) Init(config inputs.MothershipConfig) error { if config.Port == 0 { return errors.New("No Input Port specified") } l.Port = config.Port if config.Type == "" { return errors.New("No Event Type specified") } l.Type = config.Type logp.Debug("tcpinput", "Using Port %d", l.Port) logp.Debug("tcpinput", "Adding Event Type %s", l.Type) return nil }
func (l *TcpInput) handleConn(client net.Conn, output chan common.MapStr) { reader := bufio.NewReader(client) buffer := new(bytes.Buffer) var source string = client.RemoteAddr().String() var offset int64 = 0 var line uint64 = 0 var read_timeout = 10 * time.Second logp.Debug("tcpinput", "Handling New Connection from %s", source) now := func() time.Time { t := time.Now() return t } for { text, bytesread, err := l.readline(reader, buffer, read_timeout) if err != nil { logp.Info("Unexpected state reading from %v; error: %s\n", client.RemoteAddr().String, err) return } logp.Debug("tcpinputlines", "New Line: %s", &text) line++ event := common.MapStr{} event["source"] = &source event["offset"] = offset event["line"] = line event["message"] = text event["type"] = l.Type event.EnsureTimestampField(now) event.EnsureCountField() offset += int64(bytesread) logp.Debug("tcpinput", "InputEvent: %v", event) output <- event // ship the new event downstream client.Write([]byte("OK")) } logp.Debug("tcpinput", "Closed Connection from %s", source) }
func (reader *ReaderType) PrintReaderEvent(event common.MapStr) { json, err := json.MarshalIndent(event, "", " ") if err != nil { logp.Err("json.Marshal: %s", err) } else { logp.Debug("reader", "Reader: %s", string(json)) } }
func (l *NullInput) Run(output chan common.MapStr) error { logp.Debug("[NullInput]", "Running Null Input") // dispatch thread here go func(output chan common.MapStr) { l.doStuff(output) }(output) return nil }
func (l *TcpInput) readline(reader *bufio.Reader, buffer *bytes.Buffer, eof_timeout time.Duration) (*string, int, error) { var is_partial bool = true var newline_length int = 1 start_time := time.Now() logp.Debug("tcpinputlines", "Readline Called") for { segment, err := reader.ReadBytes('\n') if segment != nil && len(segment) > 0 { if segment[len(segment)-1] == '\n' { // Found a complete line is_partial = false // Check if also a CR present if len(segment) > 1 && segment[len(segment)-2] == '\r' { newline_length++ } } // TODO(sissel): if buffer exceeds a certain length, maybe report an error condition? chop it? buffer.Write(segment) } if err != nil { if err == io.EOF && is_partial { time.Sleep(1 * time.Second) // TODO(sissel): Implement backoff // Give up waiting for data after a certain amount of time. // If we time out, return the error (eof) if time.Since(start_time) > eof_timeout { return nil, 0, err } continue } else { //emit("error: Harvester.readLine: %s", err.Error()) return nil, 0, err // TODO(sissel): don't do this? } } // If we got a full line, return the whole line without the EOL chars (CRLF or LF) if !is_partial { // Get the str length with the EOL chars (LF or CRLF) bufferSize := buffer.Len() str := new(string) *str = buffer.String()[:bufferSize-newline_length] // Reset the buffer for the next line buffer.Reset() return str, bufferSize, nil } } /* forever read chunks */ return nil, 0, nil }
func (l *ProcfsInput) Run(output chan common.MapStr) error { logp.Debug("[procfsinput]", "Starting up Procfs Input") go func(output chan common.MapStr) { for { l.periodic(output) time.Sleep(time.Duration(l.Sleep) * time.Second) } }(output) return nil }
func (reader *ReaderType) Init(inputs map[string]inputs.MothershipConfig) error { logp.Info("reader input config", inputs) for inputId, config := range inputs { // default instance 0 inputName, instance := inputId, "0" if strings.Contains(inputId, "_") { // otherwise grok tcp_2 as inputName = tcp, instance = 2 sv := strings.Split(inputId, "_") inputName, instance = sv[0], sv[1] } logp.Info(fmt.Sprintf("input type: %s instance: %s\n", inputName, instance)) logp.Debug("reader", "instance config: %s", config) plugin := newInputInstance(inputName) if plugin != nil && config.Enabled { err := plugin.Init(config) if err != nil { logp.Err("Fail to initialize %s plugin as input: %s", inputName, err) return err } else { logp.Info("Initialized %s plugin as input", inputName) } reader.Input = append(reader.Input, plugin) } } if len(reader.Input) == 0 { logp.Info("No inputs are defined. Please define one under the input section.") return errors.New("No input are defined. Please define one under the input section.") } else { logp.Info("%d inputs defined", len(reader.Input)) } return nil }
func (l *ProcfsInput) periodic(output chan common.MapStr) { logp.Debug("[procfsinput]", "Running..") scanProcs(output) }
func (l *SyslogInput) Run(output chan common.MapStr) error { logp.Debug("sysloginput", "Running Syslog Input") logp.Debug("sysloginput", "Listening on %d", l.Port) listen := fmt.Sprintf("0.0.0.0:%d", l.Port) channel := make(syslog.LogPartsChannel) handler := syslog.NewChannelHandler(channel) server := syslog.NewServer() server.SetFormat(syslog.Automatic) server.SetHandler(handler) err := server.ListenUDP(listen) if err != nil { logp.Err("couldn't start ListenUDP: " + err.Error()) } err = server.ListenTCP(listen) if err != nil { logp.Err("couldn't start ListenTCP: " + err.Error()) } err = server.Boot() if err != nil { logp.Err("couldn't start server.Boot(): " + err.Error()) } go func(channel syslog.LogPartsChannel, output chan common.MapStr) { var line uint64 = 0 now := func() time.Time { t := time.Now() return t } for logParts := range channel { logp.Debug("sysloginput", "InputEvent: %v", logParts) line++ event := common.MapStr{} event["line"] = line event["type"] = l.Type for k, v := range logParts { event[k] = v } event["source"] = event["client"].(string) if event["message"] != nil { message := event["message"].(string) event["message"] = &message } else if event["content"] != nil { message := event["content"].(string) event["message"] = &message } // This syslog parser uses the standard name "tag" // which is usually the program that wrote it. // The logstash syslog_pri puts "program" for this field. if event["tag"] != nil { program := event["tag"].(string) event["program"] = &program } event.EnsureTimestampField(now) event.EnsureCountField() logp.Debug("sysloginput", "Output Event: %v", event) output <- event // ship the new event downstream } }(channel, output) return nil }