func NewHarvester(
cfg *config.HarvesterConfig,
path string,
stat *FileStat,
spooler chan *input.FileEvent,
) (*Harvester, error) {
var err error
encoding, ok := encoding.FindEncoding(cfg.Encoding)
if !ok || encoding == nil {
return nil, fmt.Errorf("unknown encoding('%v')", cfg.Encoding)
}
h := &Harvester{
Path: path,
Config: cfg,
Stat: stat,
SpoolerChan: spooler,
encoding: encoding,
}
h.ExcludeLinesRegexp, err = InitRegexps(cfg.ExcludeLines)
if err != nil {
return h, err
}
h.IncludeLinesRegexp, err = InitRegexps(cfg.IncludeLines)
if err != nil {
return h, err
}
return h, nil
}
func NewHarvester(
cfg *config.HarvesterConfig,
path string,
state input.FileState,
spooler chan *input.FileEvent,
offset int64,
done chan struct{},
) (*Harvester, error) {
encoding, ok := encoding.FindEncoding(cfg.Encoding)
if !ok || encoding == nil {
return nil, fmt.Errorf("unknown encoding('%v')", cfg.Encoding)
}
h := &Harvester{
Path: path,
Config: cfg,
State: state,
SpoolerChan: spooler,
encoding: encoding,
offset: offset,
done: done,
}
h.ExcludeLinesRegexp = cfg.ExcludeLines
h.IncludeLinesRegexp = cfg.IncludeLines
return h, nil
}
func NewHarvester(
cfg *common.Config,
state file.State,
prospectorChan chan *input.Event,
done chan struct{},
) (*Harvester, error) {
h := &Harvester{
config: defaultConfig,
state: state,
prospectorChan: prospectorChan,
done: done,
}
if err := cfg.Unpack(&h.config); err != nil {
return nil, err
}
encodingFactory, ok := encoding.FindEncoding(h.config.Encoding)
if !ok || encodingFactory == nil {
return nil, fmt.Errorf("unknown encoding('%v')", h.config.Encoding)
}
h.encodingFactory = encodingFactory
return h, nil
}
func NewHarvester(
prospectorCfg config.ProspectorConfig,
cfg *config.HarvesterConfig,
path string,
stat *FileStat,
spooler chan *input.FileEvent,
) (*Harvester, error) {
encoding, ok := encoding.FindEncoding(cfg.Encoding)
if !ok || encoding == nil {
return nil, fmt.Errorf("unknown encoding('%v')", cfg.Encoding)
}
h := &Harvester{
Path: path,
ProspectorConfig: prospectorCfg,
Config: cfg,
Stat: stat,
SpoolerChan: spooler,
encoding: encoding,
backoff: prospectorCfg.Harvester.BackoffDuration,
}
return h, nil
}
func createMultilineTestReader(t *testing.T, in *bytes.Buffer, cfg MultilineConfig) LineProcessor {
encFactory, ok := encoding.FindEncoding("plain")
if !ok {
t.Fatalf("unable to find 'plain' encoding")
}
enc, err := encFactory(in)
if err != nil {
t.Fatalf("failed to initialize encoding: %v", err)
}
var reader LineProcessor
reader, err = NewLineEncoder(in, enc, 4096)
if err != nil {
t.Fatalf("Failed to initialize line reader: %v", err)
}
reader, err = NewMultiline(NewStripNewline(reader), "\n", 1<<20, &cfg)
if err != nil {
t.Fatalf("failed to initializ reader: %v", err)
}
return reader
}
func TestReadLine(t *testing.T) {
absPath, err := filepath.Abs("../tests/files/logs/")
// All files starting with tmp are ignored
logFile := absPath + "/tmp" + strconv.Itoa(rand.Int()) + ".log"
assert.NotNil(t, absPath)
assert.Nil(t, err)
if err != nil {
t.Fatalf("Error creating the absolute path: %s", absPath)
}
file, err := os.Create(logFile)
defer file.Close()
defer os.Remove(logFile)
assert.Nil(t, err)
assert.NotNil(t, file)
firstLineString := "9Characte\n"
secondLineString := "This is line 2\n"
length, err := file.WriteString(firstLineString)
assert.Nil(t, err)
assert.NotNil(t, length)
length, err = file.WriteString(secondLineString)
assert.Nil(t, err)
assert.NotNil(t, length)
file.Sync()
// Open file for reading
readFile, err := os.Open(logFile)
defer readFile.Close()
assert.Nil(t, err)
f := source.File{readFile}
h := Harvester{
config: harvesterConfig{
CloseInactive: 500 * time.Millisecond,
Backoff: 100 * time.Millisecond,
MaxBackoff: 1 * time.Second,
BackoffFactor: 2,
BufferSize: 100,
MaxBytes: 1000,
},
file: f,
}
assert.NotNil(t, h)
var ok bool
h.encodingFactory, ok = encoding.FindEncoding(h.config.Encoding)
assert.True(t, ok)
h.encoding, err = h.encodingFactory(readFile)
assert.NoError(t, err)
r, err := h.newLogFileReader()
assert.NoError(t, err)
// Read third line
_, text, bytesread, _, err := readLine(r)
fmt.Printf("received line: '%s'\n", text)
assert.Nil(t, err)
assert.Equal(t, text, firstLineString[0:len(firstLineString)-1])
assert.Equal(t, bytesread, len(firstLineString))
// read second line
_, text, bytesread, _, err = readLine(r)
fmt.Printf("received line: '%s'\n", text)
assert.Equal(t, text, secondLineString[0:len(secondLineString)-1])
assert.Equal(t, bytesread, len(secondLineString))
assert.Nil(t, err)
// Read third line, which doesn't exist
_, text, bytesread, _, err = readLine(r)
fmt.Printf("received line: '%s'\n", text)
assert.Equal(t, "", text)
assert.Equal(t, bytesread, 0)
assert.Equal(t, err, ErrInactive)
}
func TestReaderEncodings(t *testing.T) {
for _, test := range tests {
t.Logf("test codec: %v", test.encoding)
codecFactory, ok := encoding.FindEncoding(test.encoding)
if !ok {
t.Errorf("can not find encoding '%v'", test.encoding)
continue
}
buffer := bytes.NewBuffer(nil)
codec, _ := codecFactory(buffer)
// write with encoding to buffer
writer := transform.NewWriter(buffer, codec.NewEncoder())
var expectedCount []int
for _, line := range test.strings {
writer.Write([]byte(line))
writer.Write([]byte{'\n'})
expectedCount = append(expectedCount, buffer.Len())
}
// create line reader
reader, err := NewLine(buffer, codec, 1024)
if err != nil {
t.Errorf("failed to initialize reader: %v", err)
continue
}
// read decodec lines from buffer
var readLines []string
var byteCounts []int
current := 0
for {
bytes, sz, err := reader.Next()
if sz > 0 {
readLines = append(readLines, string(bytes[:len(bytes)-1]))
}
if err != nil {
break
}
current += sz
byteCounts = append(byteCounts, current)
}
// validate lines and byte offsets
if len(test.strings) != len(readLines) {
t.Errorf("number of lines mismatch (expected=%v actual=%v)",
len(test.strings), len(readLines))
continue
}
for i := range test.strings {
expected := test.strings[i]
actual := readLines[i]
assert.Equal(t, expected, actual)
assert.Equal(t, expectedCount[i], byteCounts[i])
}
}
}