コード例 #1
0
ファイル: writer.go プロジェクト: rfistman/camlistore
// finishBlock finishes the current block and returns its block handle, which is
// its offset and length in the table.
func (w *Writer) finishBlock() (blockHandle, error) {
	// Write the restart points to the buffer.
	if w.nEntries == 0 {
		// Every block must have at least one restart point.
		w.restarts = w.restarts[:1]
		w.restarts[0] = 0
	}
	tmp4 := w.tmp[:4]
	for _, x := range w.restarts {
		binary.LittleEndian.PutUint32(tmp4, x)
		w.buf.Write(tmp4)
	}
	binary.LittleEndian.PutUint32(tmp4, uint32(len(w.restarts)))
	w.buf.Write(tmp4)

	// Compress the buffer, discarding the result if the improvement
	// isn't at least 12.5%.
	b := w.buf.Bytes()
	w.tmp[0] = noCompressionBlockType
	if w.compression == db.SnappyCompression {
		compressed, err := snappy.Encode(w.compressedBuf, b)
		if err != nil {
			return blockHandle{}, err
		}
		w.compressedBuf = compressed[:cap(compressed)]
		if len(compressed) < len(b)-len(b)/8 {
			w.tmp[0] = snappyCompressionBlockType
			b = compressed
		}
	}

	// Calculate the checksum.
	checksum := crc.New(b).Update(w.tmp[:1]).Value()
	binary.LittleEndian.PutUint32(w.tmp[1:5], checksum)

	// Write the bytes to the file.
	if _, err := w.writer.Write(b); err != nil {
		return blockHandle{}, err
	}
	if _, err := w.writer.Write(w.tmp[:5]); err != nil {
		return blockHandle{}, err
	}
	bh := blockHandle{w.offset, uint64(len(b))}
	w.offset += uint64(len(b)) + blockTrailerLen

	// Reset the per-block state.
	w.buf.Reset()
	w.nEntries = 0
	w.restarts = w.restarts[:0]
	return bh, nil
}
コード例 #2
0
func TestCmp(t *testing.T) {
	var ts, tz, to int
	for i := 0; i <= 17; i++ {
		filename := filepath.Join("testdata", testFiles[i].filename)
		if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 {
			if !*download {
				t.Fatal("test data not found; skipping test without the -download flag")
			}
			// Download the official snappy C++ implementation reference test data
			// files for benchmarking.
			if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) {
				t.Fatalf("failed to create testdata: %s", err)
			}
			for _, tf := range testFiles {
				if err := downloadTestdata(tf.filename); err != nil {
					t.Fatalf("failed to download testdata: %s", err)
				}
			}
		}
		data := readFile2(t, filename)
		orig := len(data)
		to += orig
		senc, err := snappy.Encode(nil, data)
		if err != nil {
			t.Fatal(err)
		}

		ns := len(senc)
		zenc, err := Encode(nil, data)
		if err != nil {
			t.Fatal(err)
		}

		nz := len(zenc)
		t.Logf("%35s: snappy %7d, zappy %7d, %.3f, orig %7d", filename, ns, nz, float64(nz)/float64(ns), orig)
		ts += ns
		tz += nz
	}
	t.Logf("%35s: snappy %7d, zappy %7d, %.3f, orig %7d", "TOTAL", ts, tz, float64(tz)/float64(ts), to)
}
コード例 #3
0
func TestBitIndex(t *testing.T) {
	rng := rand.New(rand.NewSource(42))
	for n := 16; n <= 1<<16; n <<= 1 {
		data := make([]byte, n)
		for i := 0; i < n/1000+1; i++ {
			data[rng.Int()%n] = 1
		}

		senc, err := snappy.Encode(nil, data)
		if err != nil {
			t.Fatal(err)
		}

		ns := len(senc)
		zenc, err := Encode(nil, data)
		if err != nil {
			t.Fatal(err)
		}

		nz := len(zenc)
		t.Logf("Sparse bit index %7d B: snappy %7d, zappy %7d, %.3f", n, ns, nz, float64(nz)/float64(ns))
	}
}