func main() {
verbose := flag.Bool("v", false, "verbose")
flag.Parse()
scanner := bufio.NewScanner(os.Stdin)
w := bufio.NewWriter(os.Stdout)
defer w.Flush()
for scanner.Scan() {
fname := scanner.Text()
b, err := ioutil.ReadFile(fname)
if err != nil {
log.Println(err)
continue
}
sip := siphash.Hash(0, 0, b)
data, err := lz4.Encode(nil, b)
if err != nil {
log.Println("error packing: ", err)
continue
}
const headerSize = 4 + 8 + 2 + 4 // + len(fname)
h := struct {
PacketSize uint32
SipHash uint64
Fname []byte `binpack:"lenprefix=uint16"`
UncompressedSize uint32
}{
PacketSize: uint32(headerSize + len(fname) + len(data)),
SipHash: sip,
Fname: []byte(fname),
UncompressedSize: uint32(len(b)),
}
if *verbose {
log.Printf("%s: %d -> %d\n", fname, len(b), len(data))
}
err = binpack.Write(w, binary.LittleEndian, h)
if err != nil {
log.Printf("error writing header %+v\n", err)
break
}
_, err = w.Write(data)
if err != nil {
log.Printf("error writing %+v\n", err)
break
}
}
}
func (w *lz4Writer) Write(bs []byte) (int, error) {
w.mut.Lock()
defer w.mut.Unlock()
var err error
w.buf, err = lz4.Encode(w.buf[:cap(w.buf)], bs)
if err != nil {
return 0, err
}
var hdr [8]byte
binary.BigEndian.PutUint32(hdr[0:], lz4Magic)
binary.BigEndian.PutUint32(hdr[4:], uint32(len(w.buf)))
_, err = w.wr.Write(hdr[:])
if err != nil {
return 0, err
}
_, err = w.wr.Write(w.buf)
if err != nil {
return 0, err
}
if debug {
l.Debugf("lz4 write; %d / %d bytes", len(bs), 8+len(w.buf))
}
return len(bs), nil
}
func (c *rawConnection) lz4Compress(src []byte) ([]byte, error) {
var err error
buf := buffers.get(len(src))
buf, err = lz4.Encode(buf, src)
if err != nil {
return nil, err
}
binary.BigEndian.PutUint32(buf, binary.LittleEndian.Uint32(buf))
return buf, nil
}
func main() {
var optCPUProfile = flag.String("cpuprofile", "", "profile")
flag.Parse()
if *optCPUProfile != "" {
f, err := os.Create(*optCPUProfile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
args := flag.Args()
var data []byte
if len(args) < 2 {
fmt.Print("Usage: lz4 [-d] <input> <output>\n")
os.Exit(1)
}
input, err := os.OpenFile(args[0], os.O_RDONLY, 0644)
if err != nil {
fmt.Printf("Failed to open input file %s\n", args[0])
os.Exit(1)
}
defer input.Close()
if *decompress {
data, _ = ioutil.ReadAll(input)
data, err = lz4.Decode(nil, data)
if err != nil {
fmt.Println("Failed to decode:", err)
return
}
} else {
data, _ = ioutil.ReadAll(input)
data, err = lz4.Encode(nil, data)
if err != nil {
fmt.Println("Failed to encode:", err)
return
}
}
err = ioutil.WriteFile(args[1], data, 0644)
if err != nil {
fmt.Printf("Failed to open output file %s\n", args[1])
os.Exit(1)
}
}
func (c *rawConnection) writerLoop() {
var msgBuf = make([]byte, 8) // buffer for wire format message, kept and reused
var uncBuf []byte // buffer for uncompressed message, kept and reused
for {
var tempBuf []byte
var err error
select {
case hm := <-c.outbox:
if hm.msg != nil {
// Uncompressed message in uncBuf
uncBuf, err = hm.msg.AppendXDR(uncBuf[:0])
if err != nil {
c.close(err)
return
}
compress := false
switch c.compression {
case CompressAlways:
compress = true
case CompressMetadata:
compress = hm.hdr.msgType != messageTypeResponse
}
if compress && len(uncBuf) >= compressionThreshold {
// Use compression for large messages
hm.hdr.compression = true
// Make sure we have enough space for the compressed message plus header in msgBug
msgBuf = msgBuf[:cap(msgBuf)]
if maxLen := lz4.CompressBound(len(uncBuf)) + 8; maxLen > len(msgBuf) {
msgBuf = make([]byte, maxLen)
}
// Compressed is written to msgBuf, we keep tb for the length only
tempBuf, err = lz4.Encode(msgBuf[8:], uncBuf)
binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(tempBuf)))
msgBuf = msgBuf[0 : len(tempBuf)+8]
if debug {
l.Debugf("write compressed message; %v (len=%d)", hm.hdr, len(tempBuf))
}
} else {
// No point in compressing very short messages
hm.hdr.compression = false
msgBuf = msgBuf[:cap(msgBuf)]
if l := len(uncBuf) + 8; l > len(msgBuf) {
msgBuf = make([]byte, l)
}
binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(uncBuf)))
msgBuf = msgBuf[0 : len(uncBuf)+8]
copy(msgBuf[8:], uncBuf)
if debug {
l.Debugf("write uncompressed message; %v (len=%d)", hm.hdr, len(uncBuf))
}
}
} else {
if debug {
l.Debugf("write empty message; %v", hm.hdr)
}
binary.BigEndian.PutUint32(msgBuf[4:8], 0)
msgBuf = msgBuf[:8]
}
binary.BigEndian.PutUint32(msgBuf[0:4], encodeHeader(hm.hdr))
if err == nil {
var n int
n, err = c.cw.Write(msgBuf)
if debug {
l.Debugf("wrote %d bytes on the wire", n)
}
}
if err != nil {
c.close(err)
return
}
case <-c.closed:
return
}
}
}