Example #1
0
// NewUniqReader returns a UniqReader over the given Reader.  maxSize is the
// maximum byte size of the cache of known record hashes.
func NewUniqReader(r Reader, maxSize int) (*UniqReader, error) {
	d, err := dedup.New(maxSize)
	if err != nil {
		return nil, fmt.Errorf("error creating Deduper: %v", err)
	}
	return &UniqReader{r, d}, nil
}
Example #2
0
// NewReader returns a reader that consumes records from r, using a cache of up
// to maxSize bytes for known record hashes.
func NewReader(r io.Reader, maxSize int) (*Reader, error) {
	d, err := dedup.New(maxSize)
	if err != nil {
		return nil, err
	}
	return &Reader{delimited.NewReader(r), d}, nil
}