func NewHarvester( cfg *config.HarvesterConfig, path string, stat *FileStat, spooler chan *input.FileEvent, ) (*Harvester, error) { var err error encoding, ok := encoding.FindEncoding(cfg.Encoding) if !ok || encoding == nil { return nil, fmt.Errorf("unknown encoding('%v')", cfg.Encoding) } h := &Harvester{ Path: path, Config: cfg, Stat: stat, SpoolerChan: spooler, encoding: encoding, } h.ExcludeLinesRegexp, err = InitRegexps(cfg.ExcludeLines) if err != nil { return h, err } h.IncludeLinesRegexp, err = InitRegexps(cfg.IncludeLines) if err != nil { return h, err } return h, nil }
func NewHarvester( cfg *config.HarvesterConfig, path string, state input.FileState, spooler chan *input.FileEvent, offset int64, done chan struct{}, ) (*Harvester, error) { encoding, ok := encoding.FindEncoding(cfg.Encoding) if !ok || encoding == nil { return nil, fmt.Errorf("unknown encoding('%v')", cfg.Encoding) } h := &Harvester{ Path: path, Config: cfg, State: state, SpoolerChan: spooler, encoding: encoding, offset: offset, done: done, } h.ExcludeLinesRegexp = cfg.ExcludeLines h.IncludeLinesRegexp = cfg.IncludeLines return h, nil }
func NewHarvester( cfg *common.Config, state file.State, prospectorChan chan *input.Event, done chan struct{}, ) (*Harvester, error) { h := &Harvester{ config: defaultConfig, state: state, prospectorChan: prospectorChan, done: done, } if err := cfg.Unpack(&h.config); err != nil { return nil, err } encodingFactory, ok := encoding.FindEncoding(h.config.Encoding) if !ok || encodingFactory == nil { return nil, fmt.Errorf("unknown encoding('%v')", h.config.Encoding) } h.encodingFactory = encodingFactory return h, nil }
func NewHarvester( prospectorCfg config.ProspectorConfig, cfg *config.HarvesterConfig, path string, stat *FileStat, spooler chan *input.FileEvent, ) (*Harvester, error) { encoding, ok := encoding.FindEncoding(cfg.Encoding) if !ok || encoding == nil { return nil, fmt.Errorf("unknown encoding('%v')", cfg.Encoding) } h := &Harvester{ Path: path, ProspectorConfig: prospectorCfg, Config: cfg, Stat: stat, SpoolerChan: spooler, encoding: encoding, backoff: prospectorCfg.Harvester.BackoffDuration, } return h, nil }
func createMultilineTestReader(t *testing.T, in *bytes.Buffer, cfg MultilineConfig) LineProcessor { encFactory, ok := encoding.FindEncoding("plain") if !ok { t.Fatalf("unable to find 'plain' encoding") } enc, err := encFactory(in) if err != nil { t.Fatalf("failed to initialize encoding: %v", err) } var reader LineProcessor reader, err = NewLineEncoder(in, enc, 4096) if err != nil { t.Fatalf("Failed to initialize line reader: %v", err) } reader, err = NewMultiline(NewStripNewline(reader), "\n", 1<<20, &cfg) if err != nil { t.Fatalf("failed to initializ reader: %v", err) } return reader }
func TestReadLine(t *testing.T) { absPath, err := filepath.Abs("../tests/files/logs/") // All files starting with tmp are ignored logFile := absPath + "/tmp" + strconv.Itoa(rand.Int()) + ".log" assert.NotNil(t, absPath) assert.Nil(t, err) if err != nil { t.Fatalf("Error creating the absolute path: %s", absPath) } file, err := os.Create(logFile) defer file.Close() defer os.Remove(logFile) assert.Nil(t, err) assert.NotNil(t, file) firstLineString := "9Characte\n" secondLineString := "This is line 2\n" length, err := file.WriteString(firstLineString) assert.Nil(t, err) assert.NotNil(t, length) length, err = file.WriteString(secondLineString) assert.Nil(t, err) assert.NotNil(t, length) file.Sync() // Open file for reading readFile, err := os.Open(logFile) defer readFile.Close() assert.Nil(t, err) f := source.File{readFile} h := Harvester{ config: harvesterConfig{ CloseInactive: 500 * time.Millisecond, Backoff: 100 * time.Millisecond, MaxBackoff: 1 * time.Second, BackoffFactor: 2, BufferSize: 100, MaxBytes: 1000, }, file: f, } assert.NotNil(t, h) var ok bool h.encodingFactory, ok = encoding.FindEncoding(h.config.Encoding) assert.True(t, ok) h.encoding, err = h.encodingFactory(readFile) assert.NoError(t, err) r, err := h.newLogFileReader() assert.NoError(t, err) // Read third line _, text, bytesread, _, err := readLine(r) fmt.Printf("received line: '%s'\n", text) assert.Nil(t, err) assert.Equal(t, text, firstLineString[0:len(firstLineString)-1]) assert.Equal(t, bytesread, len(firstLineString)) // read second line _, text, bytesread, _, err = readLine(r) fmt.Printf("received line: '%s'\n", text) assert.Equal(t, text, secondLineString[0:len(secondLineString)-1]) assert.Equal(t, bytesread, len(secondLineString)) assert.Nil(t, err) // Read third line, which doesn't exist _, text, bytesread, _, err = readLine(r) fmt.Printf("received line: '%s'\n", text) assert.Equal(t, "", text) assert.Equal(t, bytesread, 0) assert.Equal(t, err, ErrInactive) }
func TestReaderEncodings(t *testing.T) { for _, test := range tests { t.Logf("test codec: %v", test.encoding) codecFactory, ok := encoding.FindEncoding(test.encoding) if !ok { t.Errorf("can not find encoding '%v'", test.encoding) continue } buffer := bytes.NewBuffer(nil) codec, _ := codecFactory(buffer) // write with encoding to buffer writer := transform.NewWriter(buffer, codec.NewEncoder()) var expectedCount []int for _, line := range test.strings { writer.Write([]byte(line)) writer.Write([]byte{'\n'}) expectedCount = append(expectedCount, buffer.Len()) } // create line reader reader, err := NewLine(buffer, codec, 1024) if err != nil { t.Errorf("failed to initialize reader: %v", err) continue } // read decodec lines from buffer var readLines []string var byteCounts []int current := 0 for { bytes, sz, err := reader.Next() if sz > 0 { readLines = append(readLines, string(bytes[:len(bytes)-1])) } if err != nil { break } current += sz byteCounts = append(byteCounts, current) } // validate lines and byte offsets if len(test.strings) != len(readLines) { t.Errorf("number of lines mismatch (expected=%v actual=%v)", len(test.strings), len(readLines)) continue } for i := range test.strings { expected := test.strings[i] actual := readLines[i] assert.Equal(t, expected, actual) assert.Equal(t, expectedCount[i], byteCounts[i]) } } }