func TestReaderPartialWithEncodings(t *testing.T) { for _, test := range tests { t.Logf("test codec: %v", test.encoding) codecFactory, ok := encoding.FindEncoding(test.encoding) if !ok { t.Errorf("can not find encoding '%v'", test.encoding) continue } buffer := bytes.NewBuffer(nil) codec, _ := codecFactory(buffer) writer := transform.NewWriter(buffer, codec.NewEncoder()) reader, err := newLineReader(buffer, codec, 1024) if err != nil { t.Errorf("failed to initialize reader: %v", err) continue } var expected []string var partials []string lastString := "" for _, str := range test.strings { writer.Write([]byte(str)) lastString += str expected = append(expected, lastString) line, sz, err := reader.next() assert.NotNil(t, err) assert.Equal(t, 0, sz) assert.Nil(t, line) partial, _, err := reader.partial() partials = append(partials, string(partial)) t.Logf("partials: %v", partials) } // finish line: writer.Write([]byte{'\n'}) // finally read line line, _, err := reader.next() assert.Nil(t, err) t.Logf("line: '%v'", line) // validate partial lines if len(test.strings) != len(expected) { t.Errorf("number of lines mismatch (expected=%v actual=%v)", len(test.strings), len(partials)) continue } for i := range expected { assert.Equal(t, expected[i], partials[i]) } assert.Equal(t, lastString+"\n", string(line)) } }
func NewHarvester( prospectorCfg config.ProspectorConfig, cfg *config.HarvesterConfig, path string, stat *FileStat, spooler chan *input.FileEvent, ) (*Harvester, error) { encoding, ok := encoding.FindEncoding(cfg.Encoding) if !ok || encoding == nil { return nil, fmt.Errorf("unknown encoding('%v')", cfg.Encoding) } h := &Harvester{ Path: path, ProspectorConfig: prospectorCfg, Config: cfg, Stat: stat, SpoolerChan: spooler, encoding: encoding, backoff: prospectorCfg.Harvester.BackoffDuration, } return h, nil }
func TestReaderEncodings(t *testing.T) { for _, test := range tests { t.Logf("test codec: %v", test.encoding) codecFactory, ok := encoding.FindEncoding(test.encoding) if !ok { t.Errorf("can not find encoding '%v'", test.encoding) continue } buffer := bytes.NewBuffer(nil) codec, _ := codecFactory(buffer) // write with encoding to buffer writer := transform.NewWriter(buffer, codec.NewEncoder()) var expectedCount []int for _, line := range test.strings { writer.Write([]byte(line)) writer.Write([]byte{'\n'}) expectedCount = append(expectedCount, buffer.Len()) } // create line reader reader, err := newLineReader(buffer, codec, 1024) if err != nil { t.Errorf("failed to initialize reader: %v", err) continue } // read decodec lines from buffer var readLines []string var byteCounts []int current := 0 for { bytes, sz, err := reader.next() if sz > 0 { readLines = append(readLines, string(bytes[:len(bytes)-1])) } if err != nil { break } current += sz byteCounts = append(byteCounts, current) } // validate lines and byte offsets if len(test.strings) != len(readLines) { t.Errorf("number of lines mismatch (expected=%v actual=%v)", len(test.strings), len(readLines)) continue } for i := range test.strings { expected := test.strings[i] actual := readLines[i] assert.Equal(t, expected, actual) assert.Equal(t, expectedCount[i], byteCounts[i]) } } }