// NewReaderDict is like NewReader but uses a preset dictionary.
// NewReaderDict ignores the dictionary if the compressed data does not refer to it.
func NewReaderDict(r io.Reader, dict []byte) (io.ReadCloser, os.Error) {
z := new(reader)
if fr, ok := r.(flate.Reader); ok {
z.r = fr
} else {
z.r = bufio.NewReader(r)
}
_, err := io.ReadFull(z.r, z.scratch[0:2])
if err != nil {
return nil, err
}
h := uint(z.scratch[0])<<8 | uint(z.scratch[1])
if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) {
return nil, HeaderError
}
if z.scratch[1]&0x20 != 0 {
_, err = io.ReadFull(z.r, z.scratch[0:4])
if err != nil {
return nil, err
}
checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3])
if checksum != adler32.Checksum(dict) {
return nil, DictionaryError
}
z.decompressor = flate.NewReaderDict(z.r, dict)
} else {
z.decompressor = flate.NewReader(z.r)
}
z.digest = adler32.New()
return z, nil
}
func (z *reader) Reset(r io.Reader, dict []byte) error {
*z = reader{decompressor: z.decompressor}
if fr, ok := r.(flate.Reader); ok {
z.r = fr
} else {
z.r = bufio.NewReader(r)
}
// Read the header (RFC 1950 section 2.2.).
_, z.err = io.ReadFull(z.r, z.scratch[0:2])
if z.err != nil {
if z.err == io.EOF {
z.err = io.ErrUnexpectedEOF
}
return z.err
}
h := uint(z.scratch[0])<<8 | uint(z.scratch[1])
if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) {
z.err = ErrHeader
return z.err
}
haveDict := z.scratch[1]&0x20 != 0
if haveDict {
_, z.err = io.ReadFull(z.r, z.scratch[0:4])
if z.err != nil {
if z.err == io.EOF {
z.err = io.ErrUnexpectedEOF
}
return z.err
}
checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3])
if checksum != adler32.Checksum(dict) {
z.err = ErrDictionary
return z.err
}
}
if z.decompressor == nil {
if haveDict {
z.decompressor = flate.NewReaderDict(z.r, dict)
} else {
z.decompressor = flate.NewReader(z.r)
}
} else {
z.decompressor.(flate.Resetter).Reset(z.r, dict)
}
z.digest = adler32.New()
return nil
}
func (self *FlateCompressor) DecompressFromReader(src io.Reader) ([]byte, error) {
ddest := bytes.NewBuffer(nil)
var decompressor io.ReadCloser
if self.dict == nil {
decompressor = flate.NewReader(src)
} else {
decompressor = flate.NewReaderDict(src, self.dict)
}
err := decompressor.Close()
if err != nil {
fmt.Println("DecompressFromReader err:%s", err.Error())
} else {
_, err = io.Copy(ddest, decompressor)
}
return ddest.Bytes(), err
}
func unmarshalPacket(data []byte, modes map[int]*PacketMode) (node *Node, err error) {
const modeIdLength = 1
const digestLength = 20
messageLength := len(data) - digestLength
compressedLength := messageLength - modeIdLength
if compressedLength < 1 {
err = fmt.Errorf("packet is too short: %d bytes", len(data))
return
}
modeId := int(data[0])
mode := modes[modeId]
if mode == nil {
err = fmt.Errorf("packet has unknown mode: %d", modeId)
return
}
message := data[:messageLength]
digest := data[messageLength:]
mac := hmac.New(sha1.New, mode.Secret)
mac.Write(message)
if !hmac.Equal(mac.Sum(nil), digest) {
err = fmt.Errorf("packet is inauthentic (mode %d)", modeId)
return
}
compressed := data[modeIdLength:messageLength]
deflater := flate.NewReaderDict(bytes.NewBuffer(compressed), PacketCompressionDict)
defer deflater.Close()
node = new(Node)
err = json.NewDecoder(deflater).Decode(node)
return
}
func (z *reader) Reset(r io.Reader, dict []byte) error {
if fr, ok := r.(flate.Reader); ok {
z.r = fr
} else {
z.r = bufio.NewReader(r)
}
_, err := io.ReadFull(z.r, z.scratch[0:2])
if err != nil {
return err
}
h := uint(z.scratch[0])<<8 | uint(z.scratch[1])
if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) {
return ErrHeader
}
haveDict := z.scratch[1]&0x20 != 0
if haveDict {
_, err = io.ReadFull(z.r, z.scratch[0:4])
if err != nil {
return err
}
checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3])
if checksum != adler32.Checksum(dict) {
return ErrDictionary
}
}
if z.decompressor == nil {
if haveDict {
z.decompressor = flate.NewReaderDict(z.r, dict)
} else {
z.decompressor = flate.NewReader(z.r)
}
} else {
z.decompressor.(flate.Resetter).Reset(z.r, dict)
}
z.digest = adler32.New()
return nil
}
func decompressdict(src io.Reader, dest io.Writer, dict []byte) {
decompressor := flate.NewReaderDict(src, dict)
io.Copy(dest, decompressor)
decompressor.Close()
}
// A preset dictionary can be used to improve the compression ratio.
// The downside to using a dictionary is that the compressor and decompressor
// must agree in advance what dictionary to use.
func Example_dictionary() {
// The dictionary is a string of bytes. When compressing some input data,
// the compressor will attempt to substitute substrings with matches found
// in the dictionary. As such, the dictionary should only contain substrings
// that are expected to be found in the actual data stream.
const dict = `<?xml version="1.0"?>` + `<book>` + `<data>` + `<meta name="` + `" content="`
// The data to compress should (but is not required to) contain frequent
// substrings that match those in the dictionary.
const data = `<?xml version="1.0"?>
<book>
<meta name="title" content="The Go Programming Language"/>
<meta name="authors" content="Alan Donovan and Brian Kernighan"/>
<meta name="published" content="2015-10-26"/>
<meta name="isbn" content="978-0134190440"/>
<data>...</data>
</book>
`
var b bytes.Buffer
// Compress the data using the specially crafted dictionary.
zw, err := flate.NewWriterDict(&b, flate.DefaultCompression, []byte(dict))
if err != nil {
log.Fatal(err)
}
if _, err := io.Copy(zw, strings.NewReader(data)); err != nil {
log.Fatal(err)
}
if err := zw.Close(); err != nil {
log.Fatal(err)
}
// The decompressor must use the same dictionary as the compressor.
// Otherwise, the input may appear as corrupted.
fmt.Println("Decompressed output using the dictionary:")
zr := flate.NewReaderDict(bytes.NewReader(b.Bytes()), []byte(dict))
if _, err := io.Copy(os.Stdout, zr); err != nil {
log.Fatal(err)
}
if err := zr.Close(); err != nil {
log.Fatal(err)
}
fmt.Println()
// Substitute all of the bytes in the dictionary with a '#' to visually
// demonstrate the approximate effectiveness of using a preset dictionary.
fmt.Println("Substrings matched by the dictionary are marked with #:")
hashDict := []byte(dict)
for i := range hashDict {
hashDict[i] = '#'
}
zr = flate.NewReaderDict(&b, hashDict)
if _, err := io.Copy(os.Stdout, zr); err != nil {
log.Fatal(err)
}
if err := zr.Close(); err != nil {
log.Fatal(err)
}
// Output:
// Decompressed output using the dictionary:
// <?xml version="1.0"?>
// <book>
// <meta name="title" content="The Go Programming Language"/>
// <meta name="authors" content="Alan Donovan and Brian Kernighan"/>
// <meta name="published" content="2015-10-26"/>
// <meta name="isbn" content="978-0134190440"/>
// <data>...</data>
// </book>
//
// Substrings matched by the dictionary are marked with #:
// #####################
// ######
// ############title###########The Go Programming Language"/#
// ############authors###########Alan Donovan and Brian Kernighan"/#
// ############published###########2015-10-26"/#
// ############isbn###########978-0134190440"/#
// ######...</#####
// </#####
}