func TestReader(t *testing.T) {
var readers = map[string]func([]byte) io.Reader{
"io.Reader": func(b []byte) io.Reader {
return struct{ io.Reader }{bytes.NewReader(b)}
},
"bytes.Buffer": func(b []byte) io.Reader {
return bytes.NewBuffer(b)
},
"bytes.Reader": func(b []byte) io.Reader {
return bytes.NewReader(b)
},
"string.Reader": func(b []byte) io.Reader {
return strings.NewReader(string(b))
},
"compress.ByteReader": func(b []byte) io.Reader {
return struct{ compress.ByteReader }{bytes.NewReader(b)}
},
"compress.BufferedReader": func(b []byte) io.Reader {
return struct{ compress.BufferedReader }{bufio.NewReader(bytes.NewReader(b))}
},
}
var endians = map[string]bool{"littleEndian": false, "bigEndian": true}
var i int
for ne, endian := range endians {
for nr, newReader := range readers {
var br Reader
buf := make([]byte, len(testVector))
copy(buf, testVector)
if endian {
for i, c := range buf {
buf[i] = internal.ReverseLUT[c]
}
}
rd := newReader(buf)
br.Init(rd, endian)
var pd Decoder
pd.Init(testCodes)
r := testutil.NewRand(0)
loop:
for j := 0; ; j++ {
// Stop if we read enough bits.
offset := 8*br.Offset - int64(br.numBits)
if br.bufRd != nil {
discardBits := br.discardBits + int(br.fedBits-br.numBits)
offset = 8*br.Offset + int64(discardBits)
}
if offset > 8*testSize {
break
}
switch j % 4 {
case 0:
// Test unaligned Read.
if br.numBits%8 != 0 {
cnt, err := br.Read([]byte{0})
if cnt != 0 {
t.Errorf("test %d, %s %s, write count mismatch: got %d, want 0", i, ne, nr, cnt)
break loop
}
if err == nil {
t.Errorf("test %d, %s %s, unexpected write success", i, ne, nr)
break loop
}
}
pads := br.ReadPads()
if pads != 0 {
t.Errorf("test %d, %s %s, bit padding mismatch: got %d, want 0", i, ne, nr, pads)
break loop
}
want := r.Bytes(r.Intn(16))
if endian {
for i, c := range want {
want[i] = internal.ReverseLUT[c]
}
}
got := make([]byte, len(want))
cnt, err := io.ReadFull(&br, got)
if cnt != len(want) {
t.Errorf("test %d, %s %s, read count mismatch: got %d, want %d", i, ne, nr, cnt, len(want))
break loop
}
if err != nil {
t.Errorf("test %d, %s %s, unexpected read error: got %v", i, ne, nr, err)
break loop
}
if bytes.Compare(want, got) != 0 {
t.Errorf("test %d, %s %s, read bytes mismatch:\ngot %x\nwant %x", i, ne, nr, got, want)
break loop
}
case 1:
n := int(testRanges.End() - testRanges.Base())
want := uint(testRanges.Base() + uint32(r.Intn(n)))
got := br.ReadOffset(&pd, testRanges)
if got != want {
t.Errorf("test %d, %s %s, read offset mismatch: got %d, want %d", i, ne, nr, got, want)
break loop
}
case 2:
nb := uint(r.Intn(24))
want := uint(r.Int() & (1<<nb - 1))
got, ok := br.TryReadBits(nb)
if !ok {
got = br.ReadBits(nb)
}
if got != want {
t.Errorf("test %d, %s %s, read bits mismatch: got %d, want %d", i, ne, nr, got, want)
break loop
}
case 3:
want := uint(testCodes[r.Intn(len(testCodes))].Sym)
got, ok := br.TryReadSymbol(&pd)
if !ok {
got = br.ReadSymbol(&pd)
}
if got != want {
t.Errorf("test %d, %s %s, read symbol mismatch: got %d, want %d", i, ne, nr, got, want)
break loop
}
}
}
pads := br.ReadPads()
if pads != 0 {
t.Errorf("test %d, %s %s, bit padding mismatch: got %d, want 0", i, ne, nr, pads)
}
ofs, err := br.Flush()
if br.numBits != 0 {
t.Errorf("test %d, %s, bit buffer not drained: got %d, want < 8", i, ne, br.numBits)
}
if ofs != int64(len(testVector)) {
t.Errorf("test %d, %s, offset mismatch: got %d, want %d", i, ne, ofs, len(testVector))
}
if err != nil {
t.Errorf("test %d, %s, unexpected flush error: got %v", i, ne, err)
}
i++
}
}
}
func TestRange(t *testing.T) {
var vectors = []struct {
input RangeCodes
valid bool
}{{
input: RangeCodes{},
valid: false,
}, {
input: RangeCodes{{5, 2}, {10, 5}}, // Gap in-between
valid: false,
}, {
input: RangeCodes{{5, 20}, {7, 5}}, // All-encompassing overlap
valid: false,
}, {
input: RangeCodes{{7, 5}, {5, 2}}, // Out-of-order
valid: false,
}, {
input: RangeCodes{{5, 10}, {6, 11}}, // Forward-overlap is okay
valid: true,
}, {
input: testRanges,
valid: true,
}, {
input: MakeRangeCodes(0, []uint{
0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24,
}),
valid: true,
}, {
input: MakeRangeCodes(2, []uint{
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24,
}),
valid: true,
}, {
input: MakeRangeCodes(1, []uint{
2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 24,
}),
valid: true,
}, {
input: MakeRangeCodes(2, []uint{
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
}),
valid: true,
}, {
input: append(MakeRangeCodes(3, []uint{
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5,
}), RangeCode{Base: 258, Len: 0}),
valid: true,
}, {
input: MakeRangeCodes(1, []uint{
0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
}),
valid: true,
}}
r := testutil.NewRand(0)
for i, v := range vectors {
if valid := v.input.checkValid(); valid != v.valid {
t.Errorf("test %d, validity mismatch: got %v, want %v", i, valid, v.valid)
}
if !v.valid {
continue // No point further testing invalid ranges
}
var re RangeEncoder
re.Init(v.input)
for _, rc := range v.input {
offset := rc.Base + uint32(r.Intn(int(rc.End()-rc.Base)))
sym := re.Encode(uint(offset))
if int(sym) >= len(v.input) {
t.Errorf("test %d, invalid symbol: re.Encode(%d) = %d", i, offset, sym)
}
rc := v.input[sym]
if offset < rc.Base || offset >= rc.End() {
t.Errorf("test %d, symbol not in range: %d not in %d..%d", i, offset, rc.Base, rc.End()-1)
}
}
}
}
func TestGenerate(t *testing.T) {
r := testutil.NewRand(0)
var makeCodes = func(freqs []uint) PrefixCodes {
codes := make(PrefixCodes, len(freqs))
for i, j := range r.Perm(len(freqs)) {
codes[i] = PrefixCode{Sym: uint32(i), Cnt: uint32(freqs[j])}
}
codes.SortByCount()
return codes
}
var vectors = []struct {
maxBits uint // Maximum prefix bit-length (0 to skip GenerateLengths)
input PrefixCodes
valid bool
}{{
maxBits: 15,
input: makeCodes([]uint{}),
valid: true,
}, {
maxBits: 15,
input: makeCodes([]uint{0}),
valid: true,
}, {
maxBits: 15,
input: makeCodes([]uint{5}),
valid: true,
}, {
maxBits: 15,
input: makeCodes([]uint{0, 0}),
valid: true,
}, {
maxBits: 15,
input: makeCodes([]uint{5, 15}),
valid: true,
}, {
maxBits: 15,
input: makeCodes([]uint{1, 1, 2, 4}),
valid: true,
}, {
maxBits: 2,
input: makeCodes([]uint{1, 1, 2, 4}),
valid: true,
}, {
maxBits: 7,
input: makeCodes([]uint{100, 101, 102, 103}),
valid: true,
}, {
maxBits: 10,
input: makeCodes([]uint{2, 2, 2, 2, 5, 5, 5}),
valid: true,
}, {
maxBits: 15,
input: makeCodes([]uint{1, 2, 3, 4, 5, 6, 7, 8, 9}),
valid: true,
}, {
maxBits: 15,
input: makeCodes([]uint{0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}),
valid: true,
}, {
maxBits: 7,
input: makeCodes([]uint{0, 0, 2, 3, 4, 4, 4, 5, 5, 6, 6, 7, 7, 9, 10, 11, 13, 15}),
valid: true,
}, {
maxBits: 20,
input: makeCodes([]uint{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}),
valid: true,
}, {
maxBits: 12,
input: makeCodes([]uint{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}),
valid: true,
}, {
maxBits: 15,
input: makeCodes([]uint{
1, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 6, 6, 7, 7, 8, 8, 9, 9, 11, 11,
11, 11, 14, 15, 15, 17, 17, 18, 19, 19, 19, 20, 20, 21, 24, 26, 26, 31,
32, 34, 35, 38, 40, 43, 47, 48, 50, 59, 62, 63, 75, 78, 79, 85, 86, 97,
100, 100, 102, 114, 119, 128, 128, 139, 153, 166, 170, 174, 182, 184,
185, 186, 205, 325, 536, 948, 1610, 2555, 2628, 3741,
}),
valid: true,
}, {
// Input counts are not sorted in ascending order.
maxBits: 15,
input: []PrefixCode{
{Sym: 0, Cnt: 3},
{Sym: 1, Cnt: 2},
{Sym: 2, Cnt: 1},
},
valid: false,
}, {
// Input symbols are not sorted in ascending order.
maxBits: 0,
input: []PrefixCode{
{Sym: 2, Len: 1},
{Sym: 1, Len: 2},
{Sym: 0, Len: 2},
},
valid: false,
}, {
// Input symbols are not unique.
maxBits: 0,
input: []PrefixCode{
{Sym: 5, Len: 1},
{Sym: 5, Len: 1},
},
valid: false,
}, {
// Invalid small tree.
maxBits: 0,
input: []PrefixCode{
{Sym: 0, Len: 500},
},
valid: false,
}, {
// Some bit-length is too short.
maxBits: 0,
input: []PrefixCode{
{Sym: 0, Len: 1},
{Sym: 1, Len: 2},
{Sym: 2, Len: 0},
},
valid: false,
}, {
// Under-subscribed tree.
maxBits: 0,
input: []PrefixCode{
{Sym: 0, Len: 3},
{Sym: 1, Len: 4},
{Sym: 2, Len: 3},
},
valid: false,
}, {
// Over-subscribed tree.
maxBits: 0,
input: []PrefixCode{
{Sym: 0, Len: 1},
{Sym: 1, Len: 3},
{Sym: 2, Len: 4},
{Sym: 3, Len: 3},
{Sym: 4, Len: 2},
},
valid: false,
}}
for i, v := range vectors {
var sum uint32
var maxLen uint
var lens []int
var symBits [valueBits + 1]uint
codes := v.input
if v.maxBits == 0 {
goto genPrefixes
}
if err := GenerateLengths(codes, v.maxBits); err != nil {
if v.valid {
t.Errorf("test %d, unexpected failure", i)
}
continue
}
for _, c := range codes {
if maxLen < uint(c.Len) {
maxLen = uint(c.Len)
}
symBits[c.Len]++
lens = append(lens, int(c.Len))
sum += c.Cnt
}
if !codes.checkLengths() {
t.Errorf("test %d, incomplete tree generated", i)
}
if !sort.IsSorted(sort.Reverse(sort.IntSlice(lens))) {
t.Errorf("test %d, bit-lengths are not sorted:\ngot %v", i, lens)
}
if maxLen > v.maxBits {
t.Errorf("test %d, max bit-length exceeded: %d not in 1..%d", i, maxLen, v.maxBits)
}
// The whole point of prefix encoding is that the resulting bit-lengths
// produce an encoding with close to ideal entropy. Thus, compute the
// best-case entropy and check that we're not too far from it.
if len(codes) >= 4 && sum > 0 {
var worst, got, best float64
worst = math.Log2(float64(len(codes)))
got = float64(codes.Length()) / float64(sum)
for _, c := range codes {
if c.Cnt > 0 {
p := float64(c.Cnt) / float64(sum)
best += -(p * math.Log2(p))
}
}
if got > worst {
t.Errorf("test %d, actual entropy worst than worst-case: %0.3f > %0.3f", i, got, worst)
}
if got < best {
t.Errorf("test %d, actual entropy better than best-case: %0.3f < %0.3f", i, got, best)
}
if got > 1.15*best {
t.Errorf("test %d, actual entropy too high: %0.3f > %0.3f", i, got, 1.15*best)
}
}
codes.SortBySymbol()
genPrefixes:
if err := GeneratePrefixes(codes); err != nil {
if v.valid {
t.Errorf("test %d, unexpected failure", i)
}
continue
}
if !codes.checkPrefixes() {
t.Errorf("test %d, tree with non-unique prefixes generated", i)
}
if !codes.checkCanonical() {
t.Errorf("test %d, tree with non-canonical prefixes generated", i)
}
if !v.valid {
t.Errorf("test %d, unexpected success", i)
}
}
}
func TestPrefix(t *testing.T) {
var makeCodes = func(freqs []uint) PrefixCodes {
codes := make(PrefixCodes, len(freqs))
for i, n := range freqs {
codes[i] = PrefixCode{Sym: uint32(i), Cnt: uint32(n)}
}
codes.SortByCount()
if err := GenerateLengths(codes, 15); err != nil {
t.Fatalf("unexpected error: %v", err)
}
codes.SortBySymbol()
if err := GeneratePrefixes(codes); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return codes
}
var vectors = []struct {
codes PrefixCodes
}{{
codes: makeCodes([]uint{}),
}, {
codes: makeCodes([]uint{0}),
}, {
codes: makeCodes([]uint{2, 4, 3, 2, 2, 4}),
}, {
codes: makeCodes([]uint{2, 2, 2, 2, 5, 5, 5}),
}, {
codes: makeCodes([]uint{100, 101, 102, 103}),
}, {
codes: makeCodes([]uint{
1, 1, 1, 1, 1, 2, 2, 2, 3, 4, 5, 6, 6, 7, 8, 9, 9, 10, 11, 11, 12, 12,
14, 15, 15, 16, 18, 18, 19, 19, 20, 20, 20, 25, 25, 27, 29, 31, 32, 35,
39, 44, 47, 52, 60, 62, 71, 73, 74, 82, 86, 97, 98, 103, 108, 110, 112,
125, 130, 142, 154, 155, 160, 185, 198, 204, 204, 219, 222, 259, 262,
292, 296, 302, 334, 434, 450, 679, 697, 1032, 1441, 1888, 1892, 2188,
}),
}, {
codes: testCodes,
}, {
// Sparsely allocated symbols.
codes: []PrefixCode{
{Sym: 16, Val: 0, Len: 1},
{Sym: 32, Val: 1, Len: 2},
{Sym: 64, Val: 3, Len: 3},
{Sym: 128, Val: 7, Len: 3},
},
}, {
// Large number of symbols.
codes: func() PrefixCodes {
freqs := make([]uint, 4096)
for i := range freqs {
freqs[i] = uint(i)
}
return makeCodes(freqs)
}(),
}, {
// Max RLE codes from Brotli.
codes: func() (codes PrefixCodes) {
codes = PrefixCodes{{Sym: 0, Val: 0, Len: 1}}
for i := uint32(0); i < 16; i++ {
var code = PrefixCode{Sym: i + 1, Val: i<<1 | 1, Len: 5}
codes = append(codes, code)
}
return codes
}(),
}, {
// Window bits codes from Brotli.
codes: func() (codes PrefixCodes) {
for i := uint32(9); i <= 24; i++ {
var code PrefixCode
switch {
case i == 16:
code = PrefixCode{Sym: i, Val: (i-16)<<0 | 0, Len: 1} // Symbols: 16
case i > 17:
code = PrefixCode{Sym: i, Val: (i-17)<<1 | 1, Len: 4} // Symbols: 18..24
case i < 17:
code = PrefixCode{Sym: i, Val: (i-8)<<4 | 1, Len: 7} // Symbols: 9..15
default:
code = PrefixCode{Sym: i, Val: (i-17)<<4 | 1, Len: 7} // Symbols: 17
}
codes = append(codes, code)
}
codes[0].Sym = 0
return codes
}(),
}, {
// Count codes from Brotli.
codes: func() (codes PrefixCodes) {
codes = PrefixCodes{{Sym: 1, Val: 0, Len: 1}}
c := codes[len(codes)-1]
for i := uint32(0); i < 8; i++ {
for j := uint32(0); j < 1<<i; j++ {
c.Sym = c.Sym + 1
c.Val = j<<4 | i<<1 | 1
c.Len = uint32(i + 4)
codes = append(codes, c)
}
}
return codes
}(),
}, {
// Fixed literal codes from DEFLATE.
codes: func() (codes PrefixCodes) {
for i := 0; i < 144; i++ {
codes = append(codes, PrefixCode{Sym: uint32(i), Len: 8})
}
for i := 144; i < 256; i++ {
codes = append(codes, PrefixCode{Sym: uint32(i), Len: 9})
}
for i := 256; i < 280; i++ {
codes = append(codes, PrefixCode{Sym: uint32(i), Len: 7})
}
for i := 280; i < 288; i++ {
codes = append(codes, PrefixCode{Sym: uint32(i), Len: 8})
}
if err := GeneratePrefixes(codes); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return codes
}(),
}, {
// Fixed distance codes from DEFLATE.
codes: func() (codes PrefixCodes) {
for i := 0; i < 32; i++ {
codes = append(codes, PrefixCode{Sym: uint32(i), Len: 5})
}
if err := GeneratePrefixes(codes); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return codes
}(),
}}
for i, v := range vectors {
// Generate an arbitrary prefix Decoder and Encoder.
var pd Decoder
var pe Encoder
pd.Init(v.codes)
pe.Init(v.codes)
if len(v.codes) == 0 {
continue
}
// Create an arbitrary list of symbols to encode.
r := testutil.NewRand(0)
syms := make([]uint, 1000)
for i := range syms {
syms[i] = uint(v.codes[r.Intn(len(v.codes))].Sym)
}
// Setup a Reader and Writer.
var buf bytes.Buffer
var rd Reader
var wr Writer
rdwr := struct {
io.Reader
io.ByteReader
io.Writer
}{&buf, &buf, &buf}
rd.Init(rdwr, false)
wr.Init(rdwr, false)
// Write some symbols.
for _, sym := range syms {
ok := wr.TryWriteSymbol(sym, &pe)
if !ok {
wr.WriteSymbol(sym, &pe)
}
}
wr.WritePads(0)
if _, err := wr.Flush(); err != nil {
t.Errorf("test %d, unexpected Writer error: %v", i, err)
}
// Verify some Writer statistics.
if wr.Offset != int64(buf.Len()) {
t.Errorf("test %d, offset mismatch: got %d, want %d", i, wr.Offset, buf.Len())
}
if wr.numBits != 0 {
t.Errorf("test %d, residual bits remaining: got %d, want 0", i, wr.numBits)
}
if wr.cntBuf != 0 {
t.Errorf("test %d, residual bytes remaining: got %d, want 0", i, wr.cntBuf)
}
// Read some symbols.
for i := range syms {
sym, ok := rd.TryReadSymbol(&pd)
if !ok {
sym = rd.ReadSymbol(&pd)
}
if sym != syms[i] {
t.Errorf("test %d, read back wrong symbol: got %d, want %d", i, sym, syms[i])
}
if rd.numBits >= 8 {
t.Errorf("test %d, residual bits remaining: got %d, want < 8", i, rd.numBits)
}
}
pads := rd.ReadPads()
if _, err := rd.Flush(); err != nil {
t.Errorf("test %d, unexpected Reader error: %v", i, err)
}
// Verify some Reader statistics.
if pads != 0 {
t.Errorf("test %d, unexpected padding bits: got %d, want 0", i, pads)
}
if rd.numBits != 0 {
t.Errorf("test %d, residual bits remaining: got %d, want 0", i, rd.numBits)
}
if rd.Offset != wr.Offset {
t.Errorf("test %d, offset mismatch: got %d, want %d", i, rd.Offset, wr.Offset)
}
}
}
func TestWriter(t *testing.T) {
var endians = map[string]bool{"littleEndian": false, "bigEndian": true}
var i int
for ne, endian := range endians {
var bw Writer
wr := bytes.NewBuffer(nil)
bw.Init(wr, endian)
var pe Encoder
pe.Init(testCodes)
var re RangeEncoder
re.Init(testRanges)
r := testutil.NewRand(0)
loop:
for j := 0; 8*bw.Offset+int64(8*bw.cntBuf)+int64(bw.numBits) < 8*testSize; j++ {
switch j % 4 {
case 0:
// Test unaligned Write.
if bw.numBits%8 != 0 {
cnt, err := bw.Write([]byte{0})
if cnt != 0 {
t.Errorf("test %d, %s, write count mismatch: got %d, want 0", i, ne, cnt)
break loop
}
if err == nil {
t.Errorf("test %d, %s, unexpected write success", i, ne)
break loop
}
}
bw.WritePads(0)
b := r.Bytes(r.Intn(16))
if endian {
for i, c := range b {
b[i] = internal.ReverseLUT[c]
}
}
cnt, err := bw.Write(b)
if cnt != len(b) {
t.Errorf("test %d, %s, write count mismatch: got %d, want %d", i, ne, cnt, len(b))
break loop
}
if err != nil {
t.Errorf("test %d, %s, unexpected write error: got %v", i, ne, err)
break loop
}
case 1:
n := int(testRanges.End() - testRanges.Base())
ofs := uint(testRanges.Base() + uint32(r.Intn(n)))
bw.WriteOffset(ofs, &pe, &re)
case 2:
nb := uint(r.Intn(24))
val := uint(r.Int() & (1<<nb - 1))
ok := bw.TryWriteBits(val, nb)
if !ok {
bw.WriteBits(val, nb)
}
case 3:
sym := uint(testCodes[r.Intn(len(testCodes))].Sym)
ok := bw.TryWriteSymbol(sym, &pe)
if !ok {
bw.WriteSymbol(sym, &pe)
}
}
}
// Flush the Writer.
bw.WritePads(0)
ofs, err := bw.Flush()
if bw.numBits != 0 {
t.Errorf("test %d, %s, bit buffer not drained: got %d, want 0", i, ne, bw.numBits)
}
if bw.cntBuf != 0 {
t.Errorf("test %d, %s, byte buffer not drained: got %d, want 0", i, ne, bw.cntBuf)
}
if ofs != int64(wr.Len()) {
t.Errorf("test %d, %s, offset mismatch: got %d, want %d", i, ne, ofs, wr.Len())
}
if err != nil {
t.Errorf("test %d, %s, unexpected flush error: got %v", i, ne, err)
}
// Check that output matches expected.
buf := wr.Bytes()
if endian {
for i, c := range buf {
buf[i] = internal.ReverseLUT[c]
}
}
if bytes.Compare(buf, testVector) != 0 {
t.Errorf("test %d, %s, output string mismatch:\ngot %x\nwant %x", i, ne, buf, testVector)
}
i++
}
}
