func snappify(b []byte) ([]byte, error) {
b[4] |= (2 << 4) // set the document type to '2' (incr snappy)
optHeaderLength, optHeaderSize := varintdecode(b[5:])
optHeaderLength += optHeaderSize
// XXX this could be more efficient! I'm creating a new buffer to
// store the compressed document, which isn't necessary. You
// could probably write directly to the slice after the header
// and after the varint holding the length
compressed, err := snappy.Encode(nil, b[5+optHeaderLength:])
if err != nil {
return nil, err
}
// XXX I'm sure that this could be using a slice of b rather than nil
// so we don't need to copy, but my Go-fu is too low to do it.
compressedLength := varint(nil, uint(len(compressed)))
copy(b[5+optHeaderLength:], compressedLength)
bytesCopied := copy(b[5+optHeaderLength+len(compressedLength):], compressed)
// XXX should we verify that bytesCopied == len(compressed)?
return b[0:bytesCopied], nil
}
func (cn *connection) _send(opcode byte, body []byte, compression bool) error {
if cn.c == nil {
return driver.ErrBadConn
}
var flags byte = 0x00
if compression {
var err error
body, err = snappy.Encode(nil, body)
if err != nil {
return err
}
flags = flagCompressed
}
frame := make([]byte, len(body)+8)
frame[0] = protoRequest
frame[1] = flags
frame[2] = 0
frame[3] = opcode
binary.BigEndian.PutUint32(frame[4:8], uint32(len(body)))
copy(frame[8:], body)
if _, err := cn.c.Write(frame); err != nil {
return err
}
return nil
}
func (this *SnappyCodec) Forward(src, dst []byte) (uint, uint, error) {
if src == nil {
return uint(0), uint(0), errors.New("Invalid null source buffer")
}
if dst == nil {
return uint(0), uint(0), errors.New("Invalid null destination buffer")
}
if kanzi.SameByteSlices(src, dst, false) {
return 0, 0, errors.New("Input and output buffers cannot be equal")
}
count := this.size
if this.size == 0 {
count = uint(len(src))
}
if n := snappy.MaxEncodedLen(int(count)); len(dst) < n {
return 0, 0, fmt.Errorf("Output buffer is too small - size: %d, required %d", len(dst), n)
}
res, err := snappy.Encode(dst, src[0:count])
if err != nil {
return 0, 0, fmt.Errorf("Encoding error: %v", err)
}
return count, uint(len(res)), nil
}
func TestSnappyCompressor(t *testing.T) {
c := SnappyCompressor{}
if c.Name() != "snappy" {
t.Fatalf("expected name to be 'snappy', got %v", c.Name())
}
str := "My Test String"
//Test Encoding
if expected, err := snappy.Encode(nil, []byte(str)); err != nil {
t.Fatalf("failed to encode '%v' with error %v", str, err)
} else if res, err := c.Encode([]byte(str)); err != nil {
t.Fatalf("failed to encode '%v' with error %v", str, err)
} else if bytes.Compare(expected, res) != 0 {
t.Fatal("failed to match the expected encoded value with the result encoded value.")
}
val, err := c.Encode([]byte(str))
if err != nil {
t.Fatalf("failed to encode '%v' with error '%v'", str, err)
}
//Test Decoding
if expected, err := snappy.Decode(nil, val); err != nil {
t.Fatalf("failed to decode '%v' with error %v", val, err)
} else if res, err := c.Decode(val); err != nil {
t.Fatalf("failed to decode '%v' with error %v", val, err)
} else if bytes.Compare(expected, res) != 0 {
t.Fatal("failed to match the expected decoded value with the result decoded value.")
}
}
// compressedChunkGreaterN like compressedChunk produces a single, compressed,
// snappy-framed block. The returned block will have decoded length at most n
// and encoded length greater than n.
func compressedChunkGreaterN(t *testing.T, n int) []byte {
decoded := make([]byte, n)
var numTries int
var encoded []byte
for len(encoded) <= n && numTries < 3 {
numTries++
nrd, err := io.ReadFull(rand.Reader, decoded)
if err != nil {
t.Errorf("crypto/rand: %v", err)
return nil
}
if nrd != n {
t.Errorf("crypto/rand: bad read (%d bytes)", nrd)
return nil
}
encoded, err = snappy.Encode(encoded[:cap(encoded)], decoded)
if err != nil {
t.Errorf("snappy: %v", err)
return nil
}
}
if len(encoded) <= n {
t.Error(errNotEnoughEntropy)
return nil
}
return compressedChunk(t, decoded)
}
//*********COMMPRESSION*************************
func Compress(src []byte) ([]byte, bool) {
dst, err := snappy.Encode(nil, src)
if err != nil {
return nil, false
}
return dst, true
}
func compressor(fw *Writer, toCompress <-chan *writerBlock, toWrite chan<- *writerBlock) {
switch fw.CompressionCodec {
case CompressionDeflate:
bb := new(bytes.Buffer)
comp, _ := flate.NewWriter(bb, flate.DefaultCompression)
for block := range toCompress {
_, block.err = comp.Write(block.encoded.Bytes())
block.err = comp.Close()
if block.err == nil {
block.compressed = bb.Bytes()
toWrite <- block
}
bb = new(bytes.Buffer)
comp.Reset(bb)
}
case CompressionNull:
for block := range toCompress {
block.compressed = block.encoded.Bytes()
toWrite <- block
}
case CompressionSnappy:
for block := range toCompress {
block.compressed, block.err = snappy.Encode(block.compressed, block.encoded.Bytes())
if block.err != nil {
block.err = fmt.Errorf("cannot compress: %v", block.err)
}
toWrite <- block
}
}
close(toWrite)
}
func store(b []byte) string {
encoded := make([]byte, len(b))
encoded, err := snappy.Encode(encoded, b)
if err != nil {
log.Fatal(err)
}
return fmt.Sprintf("%x", sha1.Sum(encoded))
}
func main() {
f := file{Name: "", Dir: false}
f.Name = opt.Args[0]
if f.Name != "" {
//Determine if fname is a file or directory
fi, err := os.Stat(f.Name)
if err != nil {
fmt.Printf("%s does not exit!\n", f.Name)
os.Exit(2)
}
mode := fi.Mode()
if mode&os.ModeSymlink == os.ModeSymlink {
fmt.Printf("%s is a symbolic link\n", f.Name)
os.Exit(2)
}
if fi.IsDir() {
f.Dir = true
}
oldfile, err := os.Open(f.Name)
if err != nil {
fmt.Printf("Error: %s\n", err.Error())
os.Exit(2)
}
buffer := make([]byte, 100)
//for n, e := oldfile.Read(buffer); e == nil; n, e = oldfile.Read(buffer) {
// if n > 0 {
// os.Stdout.Write(buffer[0:n])
// }
//}
_, err = oldfile.Read(buffer)
if err != nil {
fmt.Printf("Error reading file %s: %s", f.Name, err)
os.Exit(2)
}
newfile := make([]byte, 100)
_, err = snappy.Encode(buffer, newfile)
if err != nil {
fmt.Printf("Errer compressing file: %s", f.Name)
os.Exit(2)
}
f.EncName = f.Name + ".sz"
export, err := os.OpenFile(f.EncName, os.O_WRONLY|os.O_CREATE, 0644) //Should keep
//original perms
export.Write(newfile)
export.Close()
os.Exit(0)
} else {
//Should never get here but just in-case
fmt.Println("No arguments given!")
os.Exit(2)
}
}
func (d *DFS) GetSnappyBlock(blockSize int, reply *DataChunk) error {
err := d.GetBlock(blockSize, reply)
handleError(err)
//
reply.Chunk, err = snappy.Encode(reply.Chunk, reply.Chunk)
handleError(err)
//
return nil
}
func (w *Writer) saveBlock() error {
// Calculate hash of uncompressed data for ref.
ref := calculateRef(w.h, w.buf[:w.n])
//TODO check if this block exists on disk.
if blockExistsOnDisk(ref) {
// Append ref to list.
w.refs = append(w.refs, ref)
w.n = 0
w.blockCount++
return nil
}
// Compress.
compressedData, err := snappy.Encode(w.cdata[headerSize:], w.buf[:w.n])
if err != nil {
return err
}
dataLen := headerSize + len(compressedData)
// Pad with zeroes so that the encrypted box is multiple of PadSize.
var paddedLen int
if dataLen == 0 {
paddedLen = PadSize - nonceSize - secretbox.Overhead
} else {
paddedLen = (((dataLen + nonceSize + secretbox.Overhead) + (PadSize - 1)) / PadSize) * PadSize
paddedLen -= nonceSize + secretbox.Overhead
}
for i := dataLen; i < paddedLen; i++ {
w.cdata[i] = 0
}
plainBlock := w.cdata[:paddedLen]
// Set block kind.
plainBlock[0] = w.kind
// Store compressed length.
binary.BigEndian.PutUint32(plainBlock[1:], uint32(len(compressedData)))
// Encrypt.
var nonce [24]byte
if err := generateNonce(&nonce); err != nil {
return err
}
//TODO avoid allocation
fullBox := make([]byte, len(nonce)+len(plainBlock)+secretbox.Overhead)
copy(fullBox, nonce[:])
secretbox.Seal(fullBox[len(nonce):len(nonce)], plainBlock, &nonce, &config.Keys.BlockEnc)
// Save to disk.
if err := writeBlockToDisk(ref, fullBox); err != nil {
return err
}
// Append ref to list.
w.refs = append(w.refs, ref)
w.n = 0
w.blockCount++
return nil
}
func writeResponse(w io.Writer, id uint64, serr string, response proto.Message) (err error) {
// clean response if error
if serr != "" {
response = nil
}
// marshal response
pbResponse := []byte{}
if response != nil {
pbResponse, err = proto.Marshal(response)
if err != nil {
return err
}
}
// compress serialized proto data
compressedPbResponse, err := snappy.Encode(nil, pbResponse)
if err != nil {
return err
}
// generate header
header := &wire.ResponseHeader{
Id: id,
Error: serr,
RawResponseLen: uint32(len(pbResponse)),
SnappyCompressedResponseLen: uint32(len(compressedPbResponse)),
Checksum: crc32.ChecksumIEEE(compressedPbResponse),
}
// check header size
pbHeader, err := proto.Marshal(header)
if err != err {
return
}
if uint32(len(pbHeader)) > wire.Default_Const_MaxHeaderLen {
return fmt.Errorf("protorpc.writeResponse: header larger than max_header_len: %d.",
len(pbHeader),
)
}
// send header (more)
if err = sendFrame(w, pbHeader); err != nil {
return
}
// send body (end)
if err = sendFrame(w, compressedPbResponse); err != nil {
return
}
return nil
}
func NewSnappyEncodeOp() stream.Operator {
generator := func() interface{} {
fn := func(in []byte) [][]byte {
compressed, err := snappy.Encode(nil, in)
if err != nil {
log.Printf("Error in snappy compression %v", err)
}
return [][]byte{compressed}
}
return fn
}
return mapper.NewOpFactory(generator, "NewSnappyEncodeOp")
}
// finishBlock finishes the current block and returns its block handle, which is
// its offset and length in the table.
func (w *Writer) finishBlock() (blockHandle, error) {
// Write the restart points to the buffer.
if w.nEntries == 0 {
// Every block must have at least one restart point.
w.restarts = w.restarts[:1]
w.restarts[0] = 0
}
tmp4 := w.tmp[:4]
for _, x := range w.restarts {
binary.LittleEndian.PutUint32(tmp4, x)
w.buf.Write(tmp4)
}
binary.LittleEndian.PutUint32(tmp4, uint32(len(w.restarts)))
w.buf.Write(tmp4)
// Compress the buffer, discarding the result if the improvement
// isn't at least 12.5%.
b := w.buf.Bytes()
w.tmp[0] = noCompressionBlockType
if w.compression == db.SnappyCompression {
compressed, err := snappy.Encode(w.compressedBuf, b)
if err != nil {
return blockHandle{}, err
}
w.compressedBuf = compressed[:cap(compressed)]
if len(compressed) < len(b)-len(b)/8 {
w.tmp[0] = snappyCompressionBlockType
b = compressed
}
}
// Calculate the checksum.
checksum := crc.New(b).Update(w.tmp[:1]).Value()
binary.LittleEndian.PutUint32(w.tmp[1:5], checksum)
// Write the bytes to the file.
if _, err := w.writer.Write(b); err != nil {
return blockHandle{}, err
}
if _, err := w.writer.Write(w.tmp[:5]); err != nil {
return blockHandle{}, err
}
bh := blockHandle{w.offset, uint64(len(b))}
w.offset += uint64(len(b)) + blockTrailerLen
// Reset the per-block state.
w.buf.Reset()
w.nEntries = 0
w.restarts = w.restarts[:0]
return bh, nil
}
// write attempts to encode p as a block and write it to the underlying writer.
// The returned int may not equal p's length if compression below
// MaxBlockSize-4 could not be achieved.
func (w *writer) write(p []byte) (int, error) {
var err error
if len(p) > MaxBlockSize {
return 0, errors.New(fmt.Sprintf("block too large %d > %d", len(p), MaxBlockSize))
}
w.dst = w.dst[:cap(w.dst)] // Encode does dumb resize w/o context. reslice avoids alloc.
w.dst, err = snappy.Encode(w.dst, p)
if err != nil {
return 0, err
}
block := w.dst
n := len(p)
compressed := true
// check for data which is better left uncompressed. this is determined if
// the encoded content is longer than the source.
if len(w.dst) >= len(p) {
compressed = false
block = p[:n]
}
if !w.sentStreamID {
_, err := w.writer.Write(streamID)
if err != nil {
return 0, err
}
w.sentStreamID = true
}
// set the block type
if compressed {
writeHeader(w.hdr, blockCompressed, block, p[:n])
} else {
writeHeader(w.hdr, blockUncompressed, block, p[:n])
}
_, err = w.writer.Write(w.hdr)
if err != nil {
return 0, err
}
_, err = w.writer.Write(block)
if err != nil {
return 0, err
}
return n, nil
}
func (w *writer) write(p []byte) (int, error) {
var err error
if len(p) > MaxBlockSize {
return 0, errors.New(fmt.Sprintf("block too large %d > %d", len(p), MaxBlockSize))
}
w.dst, err = snappy.Encode(w.dst, p)
if err != nil {
return 0, err
}
if !w.sentStreamID {
_, err := w.writer.Write(streamID)
if err != nil {
return 0, err
}
w.sentStreamID = true
}
length := uint32(len(w.dst)) + 4 // +4 for checksum
w.hdr[0] = 0x00 // compressed frame ID
// 3 byte little endian length
w.hdr[1] = byte(length)
w.hdr[2] = byte(length >> 8)
w.hdr[3] = byte(length >> 16)
// 4 byte little endian CRC32 checksum
checksum := maskChecksum(crc32.Checksum(p, crcTable))
w.hdr[4] = byte(checksum)
w.hdr[5] = byte(checksum >> 8)
w.hdr[6] = byte(checksum >> 16)
w.hdr[7] = byte(checksum >> 24)
_, err = w.writer.Write(w.hdr)
if err != nil {
return 0, err
}
_, err = w.writer.Write(w.dst)
if err != nil {
return 0, err
}
return len(p), nil
}
func (m *Message) encode(pe packetEncoder) error {
pe.push(&crc32Field{})
pe.putInt8(messageFormat)
attributes := int8(m.Codec) & compressionCodecMask
pe.putInt8(attributes)
err := pe.putBytes(m.Key)
if err != nil {
return err
}
var payload []byte
if m.compressedCache != nil {
payload = m.compressedCache
m.compressedCache = nil
} else {
switch m.Codec {
case CompressionNone:
payload = m.Value
case CompressionGZIP:
var buf bytes.Buffer
writer := gzip.NewWriter(&buf)
writer.Write(m.Value)
writer.Close()
m.compressedCache = buf.Bytes()
payload = m.compressedCache
case CompressionSnappy:
tmp, err := snappy.Encode(nil, m.Value)
if err != nil {
return err
}
m.compressedCache = tmp
payload = m.compressedCache
default:
return EncodingError
}
}
err = pe.putBytes(payload)
if err != nil {
return err
}
return pe.pop()
}
// Put adds the package documentation to the database.
func (db *Database) Put(pdoc *doc.Package, nextCrawl time.Time) error {
c := db.Pool.Get()
defer c.Close()
score := documentScore(pdoc)
terms := documentTerms(pdoc, score)
var gobBuf bytes.Buffer
if err := gob.NewEncoder(&gobBuf).Encode(pdoc); err != nil {
return err
}
// Truncate large documents.
if gobBuf.Len() > 700000 {
pdocNew := *pdoc
pdoc = &pdocNew
pdoc.Truncated = true
pdoc.Vars = nil
pdoc.Funcs = nil
pdoc.Types = nil
pdoc.Consts = nil
pdoc.Examples = nil
gobBuf.Reset()
if err := gob.NewEncoder(&gobBuf).Encode(pdoc); err != nil {
return err
}
}
gobBytes, err := snappy.Encode(nil, gobBuf.Bytes())
if err != nil {
return err
}
kind := "p"
switch {
case pdoc.Name == "":
kind = "d"
case pdoc.IsCmd:
kind = "c"
}
t := int64(0)
if !nextCrawl.IsZero() {
t = nextCrawl.Unix()
}
_, err = putScript.Do(c, pdoc.ImportPath, pdoc.Synopsis, score, gobBytes, strings.Join(terms, " "), pdoc.Etag, kind, t)
return err
}
func writeRequest(w io.Writer, id uint64, method string, request proto.Message) error {
// marshal request
pbRequest := []byte{}
if request != nil {
var err error
pbRequest, err = proto.Marshal(request)
if err != nil {
return err
}
}
// compress serialized proto data
compressedPbRequest, err := snappy.Encode(nil, pbRequest)
if err != nil {
return err
}
// generate header
header := &wire.RequestHeader{
Id: id,
Method: method,
RawRequestLen: uint32(len(pbRequest)),
SnappyCompressedRequestLen: uint32(len(compressedPbRequest)),
Checksum: crc32.ChecksumIEEE(compressedPbRequest),
}
// check header size
pbHeader, err := proto.Marshal(header)
if err != err {
return err
}
if uint32(len(pbHeader)) > wire.Default_Const_MaxHeaderLen {
return fmt.Errorf("protorpc.writeRequest: header larger than max_header_len: %d.", len(pbHeader))
}
// send header (more)
if err := sendFrame(w, pbHeader); err != nil {
return err
}
// send body (end)
if err := sendFrame(w, compressedPbRequest); err != nil {
return err
}
return nil
}
func KVWrite(db *rpcplus.Client, table, uid string, value interface{}) (result bool, err error) {
//ts("KVWrite", table, uid)
//defer te("KVWrite", table, uid)
var buf []byte
switch value.(type) {
case gp.Message:
buf, err = gp.Marshal(value.(gp.Message))
if err != nil {
logger.Error("KVWrite Error On Marshal %s : %s (%s)", table, uid, err.Error())
return
}
case []byte:
buf = value.([]byte)
default:
logger.Error("KVWrite args type error %v", value)
return
}
dst, err := snappy.Encode(nil, buf)
if err != nil {
logger.Error("KVWrite Error On snappy.Encode %s : %s (%s)", table, uid, err.Error())
return
}
var reply protobuf.DBWriteResult
err = db.Call("DBServer.Write", protobuf.DBWrite{table, uid, dst}, &reply)
if err != nil {
logger.Error("KVWrite Error On Create %s: %s (%s)", table, uid, err.Error())
return
}
if reply.Code != protobuf.Ok {
logger.Error("KVWrite Error On Create %s: %s code (%d)", table, uid, reply.Code)
return
}
result = true
return
}
// compressedChunk encodes b returning a single, compressed, snappy-framed
// block. compressedChunk can encode source data larger than allowed in the
// specification.
func compressedChunk(t *testing.T, src []byte) []byte {
encoded, err := snappy.Encode(nil, src)
if err != nil {
t.Errorf("snappy: %v", err)
return nil
}
if len(encoded) > (1<<24)-5 { // account for the 4-byte checksum
t.Errorf("block data too large %d", len(src))
return nil
}
chunk := make([]byte, len(encoded)+8)
writeHeader(chunk[:8], blockCompressed, encoded, src)
copy(chunk[8:], encoded)
return chunk
}
func (ev *CompressEventV2) Encode(buffer *bytes.Buffer) {
if ev.CompressType != COMPRESSOR_NONE && ev.CompressType != COMPRESSOR_SNAPPY {
ev.CompressType = COMPRESSOR_NONE
}
EncodeUInt64Value(buffer, uint64(ev.CompressType))
var buf bytes.Buffer
EncodeEvent(&buf, ev.Ev)
switch ev.CompressType {
case COMPRESSOR_NONE:
EncodeUInt64Value(buffer, uint64(buf.Len()))
buffer.Write(buf.Bytes())
case COMPRESSOR_SNAPPY:
evbuf := make([]byte, 0)
newbuf, _ := snappy.Encode(evbuf, buf.Bytes())
EncodeUInt64Value(buffer, uint64(len(newbuf)))
buffer.Write(newbuf)
}
buf.Reset()
}
func RunTestSnappy(data []byte) {
log.Printf("encoding/RunTestSnappy: Testing comprssion Snappy\n")
now := time.Now()
e, err := snappy.Encode(nil, data)
if err != nil {
log.Fatalf("encoding/RunTestSnappy: encoding error: %v\n", err)
}
log.Printf("encoding/RunTestSnappy: Compressed from %d bytes to %d bytes in %d ns\n", len(data), len(e), time.Since(now).Nanoseconds())
d, err := snappy.Decode(nil, e)
if err != nil {
log.Fatalf("encoding/RunTestSnappy: decoding error: %v\n", err)
}
log.Printf("encoding/RunTestSnappy: Uncompressed from %d bytes to %d bytes in %d ns\n", len(e), len(d), time.Since(now).Nanoseconds())
if !bytes.Equal(data, d) {
log.Fatalf("encoding/RunTestSnappy: roundtrip mismatch\n")
}
}
func (ev *CompressEvent) Encode(buffer *bytes.Buffer) {
if ev.CompressType != COMPRESSOR_NONE && ev.CompressType != COMPRESSOR_SNAPPY {
ev.CompressType = COMPRESSOR_NONE
}
EncodeUInt64Value(buffer, uint64(ev.CompressType))
var buf bytes.Buffer
EncodeEvent(&buf, ev.Ev)
switch ev.CompressType {
case COMPRESSOR_NONE:
buffer.Write(buf.Bytes())
case COMPRESSOR_SNAPPY:
evbuf := make([]byte, 0)
newbuf, _ := snappy.Encode(evbuf, buf.Bytes())
buffer.Write(newbuf)
// case COMPRESSOR_LZ4:
// newbuf, _ := ioutil.ReadAll(lz4.NewWriter(&buf))
// buffer.Write(newbuf)
}
buf.Reset()
}
// Put adds the package documentation to the database.
func (db *Database) Put(pdoc *doc.Package) error {
c := db.Pool.Get()
defer c.Close()
projectRoot := pdoc.ProjectRoot
if projectRoot == "" {
projectRoot = "go"
}
rank := documentRank(pdoc)
terms := documentTerms(pdoc, rank)
var gobBuf bytes.Buffer
if err := gob.NewEncoder(&gobBuf).Encode(pdoc); err != nil {
return err
}
// Truncate large documents.
if gobBuf.Len() > 700000 {
pdocNew := *pdoc
pdoc = &pdocNew
pdoc.Truncated = true
pdoc.Vars = nil
pdoc.Funcs = nil
pdoc.Types = nil
pdoc.Consts = nil
pdoc.Examples = nil
gobBuf.Reset()
if err := gob.NewEncoder(&gobBuf).Encode(pdoc); err != nil {
return err
}
}
gobBytes, err := snappy.Encode(nil, gobBuf.Bytes())
if err != nil {
return err
}
_, err = putScript.Do(c, pdoc.ImportPath, pdoc.Synopsis, rank, gobBytes, strings.Join(terms, " "), pdoc.Updated.Unix())
return err
}
func (db *BlockDB) BlockAdd(height uint32, bl *Block) (e error) {
var pos int64
var flagz [4]byte
pos, e = db.blockdata.Seek(0, os.SEEK_END)
if e != nil {
panic(e.Error())
}
flagz[0] |= BLOCK_COMPRSD | BLOCK_SNAPPED // gzip compression is deprecated
cbts, _ := snappy.Encode(nil, bl.Raw)
blksize := uint32(len(cbts))
_, e = db.blockdata.Write(cbts)
if e != nil {
panic(e.Error())
}
ipos, _ := db.blockindx.Seek(0, os.SEEK_CUR) // at this point the file shall always be at its end
if bl.Trusted {
flagz[0] |= BLOCK_TRUSTED
}
db.blockindx.Write(flagz[:])
db.blockindx.Write(bl.Hash.Hash[0:32])
db.blockindx.Write(bl.Raw[4:36])
binary.Write(db.blockindx, binary.LittleEndian, uint32(height))
binary.Write(db.blockindx, binary.LittleEndian, uint32(bl.BlockTime))
binary.Write(db.blockindx, binary.LittleEndian, uint32(bl.Bits))
binary.Write(db.blockindx, binary.LittleEndian, uint64(pos))
binary.Write(db.blockindx, binary.LittleEndian, blksize)
db.mutex.Lock()
db.blockIndex[bl.Hash.BIdx()] = &oneBl{fpos: uint64(pos),
blen: blksize, ipos: ipos, trusted: bl.Trusted, compressed: true, snappied: true}
db.addToCache(bl.Hash, bl.Raw)
db.mutex.Unlock()
return
}
func TestCmp(t *testing.T) {
var ts, tz, to int
for i := 0; i <= 17; i++ {
filename := filepath.Join("testdata", testFiles[i].filename)
if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 {
if !*download {
t.Fatal("test data not found; skipping test without the -download flag")
}
// Download the official snappy C++ implementation reference test data
// files for benchmarking.
if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) {
t.Fatalf("failed to create testdata: %s", err)
}
for _, tf := range testFiles {
if err := downloadTestdata(tf.filename); err != nil {
t.Fatalf("failed to download testdata: %s", err)
}
}
}
data := readFile2(t, filename)
orig := len(data)
to += orig
senc, err := snappy.Encode(nil, data)
if err != nil {
t.Fatal(err)
}
ns := len(senc)
zenc, err := Encode(nil, data)
if err != nil {
t.Fatal(err)
}
nz := len(zenc)
t.Logf("%35s: snappy %7d, zappy %7d, %.3f, orig %7d", filename, ns, nz, float64(nz)/float64(ns), orig)
ts += ns
tz += nz
}
t.Logf("%35s: snappy %7d, zappy %7d, %.3f, orig %7d", "TOTAL", ts, tz, float64(tz)/float64(ts), to)
}
func (t *Writer) write(buf []byte, bi *bInfo, raw bool) (err error) {
compression := opt.NoCompression
if !raw {
compression = t.o.GetCompressionType()
}
switch compression {
case opt.SnappyCompression:
buf, err = snappy.Encode(nil, buf)
if err != nil {
return
}
}
if bi != nil {
bi.offset = uint64(t.off)
bi.size = uint64(len(buf))
}
_, err = t.w.Write(buf)
if err != nil {
return
}
compbit := []byte{byte(compression)}
_, err = t.w.Write(compbit)
if err != nil {
return
}
crc := hash.NewCRC32C()
crc.Write(buf)
crc.Write(compbit)
err = binary.Write(t.w, binary.LittleEndian, hash.MaskCRC32(crc.Sum32()))
if err != nil {
return
}
t.off += len(buf) + 5
return
}
func keyTweet(tw *tweetItem) (rout *tweetItem) {
et, err := snappy.Encode(nil, *tw)
if err != nil {
panic(err)
}
var tweet struct {
Created_At string
}
if err := json.Unmarshal(*tw, &tweet); err == nil {
if t, err := time.Parse(time.RubyDate, tweet.Created_At); err == nil {
l := len(et)
out := make(tweetItem, 12+l)
key := t.UnixNano()
out[0] = byte(key)
out[1] = byte(key >> 8)
out[2] = byte(key >> 16)
out[3] = byte(key >> 24)
out[4] = byte(key >> 32)
out[5] = byte(key >> 40)
out[6] = byte(key >> 48)
out[7] = byte(key >> 56)
out[8] = byte(l)
out[9] = byte(l >> 8)
out[10] = byte(l >> 16)
out[11] = byte(l >> 24)
copy(out[12:12+l], et)
rout = &out
} else {
log.Fatal("Could not parse time:", err)
}
} else {
log.Fatal("Could not unmarshal:", string(*tw))
}
return
}
func snappify(b []byte) ([]byte, error) {
b[4] |= (2 << 4) // set the document type to '2' (incr snappy)
optHeaderLength, optHeaderSize := varintdecode(b[5:])
optHeaderLength += optHeaderSize
// XXX this could be more efficient! I'm creating a new buffer to
// store the compressed document, which isn't necessary. You
// could probably write directly to the slice after the header
// and after the varint holding the length
compressed, err := snappy.Encode(nil, b[5+optHeaderLength:])
if err != nil {
return nil, err
}
// resize b to just the original header
b = b[:5+optHeaderLength]
b = varint(b, uint(len(compressed)))
b = append(b, compressed...)
return b, nil
}