func main() { verbose := flag.Bool("v", false, "verbose") flag.Parse() scanner := bufio.NewScanner(os.Stdin) w := bufio.NewWriter(os.Stdout) defer w.Flush() for scanner.Scan() { fname := scanner.Text() b, err := ioutil.ReadFile(fname) if err != nil { log.Println(err) continue } sip := siphash.Hash(0, 0, b) data, err := lz4.Encode(nil, b) if err != nil { log.Println("error packing: ", err) continue } const headerSize = 4 + 8 + 2 + 4 // + len(fname) h := struct { PacketSize uint32 SipHash uint64 Fname []byte `binpack:"lenprefix=uint16"` UncompressedSize uint32 }{ PacketSize: uint32(headerSize + len(fname) + len(data)), SipHash: sip, Fname: []byte(fname), UncompressedSize: uint32(len(b)), } if *verbose { log.Printf("%s: %d -> %d\n", fname, len(b), len(data)) } err = binpack.Write(w, binary.LittleEndian, h) if err != nil { log.Printf("error writing header %+v\n", err) break } _, err = w.Write(data) if err != nil { log.Printf("error writing %+v\n", err) break } } }
func (w *lz4Writer) Write(bs []byte) (int, error) { w.mut.Lock() defer w.mut.Unlock() var err error w.buf, err = lz4.Encode(w.buf[:cap(w.buf)], bs) if err != nil { return 0, err } var hdr [8]byte binary.BigEndian.PutUint32(hdr[0:], lz4Magic) binary.BigEndian.PutUint32(hdr[4:], uint32(len(w.buf))) _, err = w.wr.Write(hdr[:]) if err != nil { return 0, err } _, err = w.wr.Write(w.buf) if err != nil { return 0, err } if debug { l.Debugf("lz4 write; %d / %d bytes", len(bs), 8+len(w.buf)) } return len(bs), nil }
func (c *rawConnection) lz4Compress(src []byte) ([]byte, error) { var err error buf := buffers.get(len(src)) buf, err = lz4.Encode(buf, src) if err != nil { return nil, err } binary.BigEndian.PutUint32(buf, binary.LittleEndian.Uint32(buf)) return buf, nil }
func main() { var optCPUProfile = flag.String("cpuprofile", "", "profile") flag.Parse() if *optCPUProfile != "" { f, err := os.Create(*optCPUProfile) if err != nil { log.Fatal(err) } pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } args := flag.Args() var data []byte if len(args) < 2 { fmt.Print("Usage: lz4 [-d] <input> <output>\n") os.Exit(1) } input, err := os.OpenFile(args[0], os.O_RDONLY, 0644) if err != nil { fmt.Printf("Failed to open input file %s\n", args[0]) os.Exit(1) } defer input.Close() if *decompress { data, _ = ioutil.ReadAll(input) data, err = lz4.Decode(nil, data) if err != nil { fmt.Println("Failed to decode:", err) return } } else { data, _ = ioutil.ReadAll(input) data, err = lz4.Encode(nil, data) if err != nil { fmt.Println("Failed to encode:", err) return } } err = ioutil.WriteFile(args[1], data, 0644) if err != nil { fmt.Printf("Failed to open output file %s\n", args[1]) os.Exit(1) } }
func (c *rawConnection) writerLoop() { var msgBuf = make([]byte, 8) // buffer for wire format message, kept and reused var uncBuf []byte // buffer for uncompressed message, kept and reused for { var tempBuf []byte var err error select { case hm := <-c.outbox: if hm.msg != nil { // Uncompressed message in uncBuf uncBuf, err = hm.msg.AppendXDR(uncBuf[:0]) if err != nil { c.close(err) return } compress := false switch c.compression { case CompressAlways: compress = true case CompressMetadata: compress = hm.hdr.msgType != messageTypeResponse } if compress && len(uncBuf) >= compressionThreshold { // Use compression for large messages hm.hdr.compression = true // Make sure we have enough space for the compressed message plus header in msgBug msgBuf = msgBuf[:cap(msgBuf)] if maxLen := lz4.CompressBound(len(uncBuf)) + 8; maxLen > len(msgBuf) { msgBuf = make([]byte, maxLen) } // Compressed is written to msgBuf, we keep tb for the length only tempBuf, err = lz4.Encode(msgBuf[8:], uncBuf) binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(tempBuf))) msgBuf = msgBuf[0 : len(tempBuf)+8] if debug { l.Debugf("write compressed message; %v (len=%d)", hm.hdr, len(tempBuf)) } } else { // No point in compressing very short messages hm.hdr.compression = false msgBuf = msgBuf[:cap(msgBuf)] if l := len(uncBuf) + 8; l > len(msgBuf) { msgBuf = make([]byte, l) } binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(uncBuf))) msgBuf = msgBuf[0 : len(uncBuf)+8] copy(msgBuf[8:], uncBuf) if debug { l.Debugf("write uncompressed message; %v (len=%d)", hm.hdr, len(uncBuf)) } } } else { if debug { l.Debugf("write empty message; %v", hm.hdr) } binary.BigEndian.PutUint32(msgBuf[4:8], 0) msgBuf = msgBuf[:8] } binary.BigEndian.PutUint32(msgBuf[0:4], encodeHeader(hm.hdr)) if err == nil { var n int n, err = c.cw.Write(msgBuf) if debug { l.Debugf("wrote %d bytes on the wire", n) } } if err != nil { c.close(err) return } case <-c.closed: return } } }