func TestReaderPartialWithEncodings(t *testing.T) {
for _, test := range tests {
t.Logf("test codec: %v", test.encoding)
codecFactory, ok := encoding.FindEncoding(test.encoding)
if !ok {
t.Errorf("can not find encoding '%v'", test.encoding)
continue
}
buffer := bytes.NewBuffer(nil)
codec, _ := codecFactory(buffer)
writer := transform.NewWriter(buffer, codec.NewEncoder())
reader, err := newLineReader(buffer, codec, 1024)
if err != nil {
t.Errorf("failed to initialize reader: %v", err)
continue
}
var expected []string
var partials []string
lastString := ""
for _, str := range test.strings {
writer.Write([]byte(str))
lastString += str
expected = append(expected, lastString)
line, sz, err := reader.next()
assert.NotNil(t, err)
assert.Equal(t, 0, sz)
assert.Nil(t, line)
partial, _, err := reader.partial()
partials = append(partials, string(partial))
t.Logf("partials: %v", partials)
}
// finish line:
writer.Write([]byte{'\n'})
// finally read line
line, _, err := reader.next()
assert.Nil(t, err)
t.Logf("line: '%v'", line)
// validate partial lines
if len(test.strings) != len(expected) {
t.Errorf("number of lines mismatch (expected=%v actual=%v)",
len(test.strings), len(partials))
continue
}
for i := range expected {
assert.Equal(t, expected[i], partials[i])
}
assert.Equal(t, lastString+"\n", string(line))
}
}
func NewHarvester(
prospectorCfg config.ProspectorConfig,
cfg *config.HarvesterConfig,
path string,
stat *FileStat,
spooler chan *input.FileEvent,
) (*Harvester, error) {
encoding, ok := encoding.FindEncoding(cfg.Encoding)
if !ok || encoding == nil {
return nil, fmt.Errorf("unknown encoding('%v')", cfg.Encoding)
}
h := &Harvester{
Path: path,
ProspectorConfig: prospectorCfg,
Config: cfg,
Stat: stat,
SpoolerChan: spooler,
encoding: encoding,
backoff: prospectorCfg.Harvester.BackoffDuration,
}
return h, nil
}
func TestReaderEncodings(t *testing.T) {
for _, test := range tests {
t.Logf("test codec: %v", test.encoding)
codecFactory, ok := encoding.FindEncoding(test.encoding)
if !ok {
t.Errorf("can not find encoding '%v'", test.encoding)
continue
}
buffer := bytes.NewBuffer(nil)
codec, _ := codecFactory(buffer)
// write with encoding to buffer
writer := transform.NewWriter(buffer, codec.NewEncoder())
var expectedCount []int
for _, line := range test.strings {
writer.Write([]byte(line))
writer.Write([]byte{'\n'})
expectedCount = append(expectedCount, buffer.Len())
}
// create line reader
reader, err := newLineReader(buffer, codec, 1024)
if err != nil {
t.Errorf("failed to initialize reader: %v", err)
continue
}
// read decodec lines from buffer
var readLines []string
var byteCounts []int
current := 0
for {
bytes, sz, err := reader.next()
if sz > 0 {
readLines = append(readLines, string(bytes[:len(bytes)-1]))
}
if err != nil {
break
}
current += sz
byteCounts = append(byteCounts, current)
}
// validate lines and byte offsets
if len(test.strings) != len(readLines) {
t.Errorf("number of lines mismatch (expected=%v actual=%v)",
len(test.strings), len(readLines))
continue
}
for i := range test.strings {
expected := test.strings[i]
actual := readLines[i]
assert.Equal(t, expected, actual)
assert.Equal(t, expectedCount[i], byteCounts[i])
}
}
}