// finishBlock finishes the current block and returns its block handle, which is
// its offset and length in the table.
func (w *Writer) finishBlock() (blockHandle, error) {
// Write the restart points to the buffer.
if w.nEntries == 0 {
// Every block must have at least one restart point.
w.restarts = w.restarts[:1]
w.restarts[0] = 0
}
tmp4 := w.tmp[:4]
for _, x := range w.restarts {
binary.LittleEndian.PutUint32(tmp4, x)
w.buf.Write(tmp4)
}
binary.LittleEndian.PutUint32(tmp4, uint32(len(w.restarts)))
w.buf.Write(tmp4)
// Compress the buffer, discarding the result if the improvement
// isn't at least 12.5%.
b := w.buf.Bytes()
w.tmp[0] = noCompressionBlockType
if w.compression == db.SnappyCompression {
compressed, err := snappy.Encode(w.compressedBuf, b)
if err != nil {
return blockHandle{}, err
}
w.compressedBuf = compressed[:cap(compressed)]
if len(compressed) < len(b)-len(b)/8 {
w.tmp[0] = snappyCompressionBlockType
b = compressed
}
}
// Calculate the checksum.
checksum := crc.New(b).Update(w.tmp[:1]).Value()
binary.LittleEndian.PutUint32(w.tmp[1:5], checksum)
// Write the bytes to the file.
if _, err := w.writer.Write(b); err != nil {
return blockHandle{}, err
}
if _, err := w.writer.Write(w.tmp[:5]); err != nil {
return blockHandle{}, err
}
bh := blockHandle{w.offset, uint64(len(b))}
w.offset += uint64(len(b)) + blockTrailerLen
// Reset the per-block state.
w.buf.Reset()
w.nEntries = 0
w.restarts = w.restarts[:0]
return bh, nil
}
func TestCmp(t *testing.T) {
var ts, tz, to int
for i := 0; i <= 17; i++ {
filename := filepath.Join("testdata", testFiles[i].filename)
if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 {
if !*download {
t.Fatal("test data not found; skipping test without the -download flag")
}
// Download the official snappy C++ implementation reference test data
// files for benchmarking.
if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) {
t.Fatalf("failed to create testdata: %s", err)
}
for _, tf := range testFiles {
if err := downloadTestdata(tf.filename); err != nil {
t.Fatalf("failed to download testdata: %s", err)
}
}
}
data := readFile2(t, filename)
orig := len(data)
to += orig
senc, err := snappy.Encode(nil, data)
if err != nil {
t.Fatal(err)
}
ns := len(senc)
zenc, err := Encode(nil, data)
if err != nil {
t.Fatal(err)
}
nz := len(zenc)
t.Logf("%35s: snappy %7d, zappy %7d, %.3f, orig %7d", filename, ns, nz, float64(nz)/float64(ns), orig)
ts += ns
tz += nz
}
t.Logf("%35s: snappy %7d, zappy %7d, %.3f, orig %7d", "TOTAL", ts, tz, float64(tz)/float64(ts), to)
}
func TestBitIndex(t *testing.T) {
rng := rand.New(rand.NewSource(42))
for n := 16; n <= 1<<16; n <<= 1 {
data := make([]byte, n)
for i := 0; i < n/1000+1; i++ {
data[rng.Int()%n] = 1
}
senc, err := snappy.Encode(nil, data)
if err != nil {
t.Fatal(err)
}
ns := len(senc)
zenc, err := Encode(nil, data)
if err != nil {
t.Fatal(err)
}
nz := len(zenc)
t.Logf("Sparse bit index %7d B: snappy %7d, zappy %7d, %.3f", n, ns, nz, float64(nz)/float64(ns))
}
}