// Deflate utility that compresses a string using the flate algo
func Deflate(inflated string) []byte {
var b bytes.Buffer
w, _ := flate.NewWriter(&b, -1)
w.Write([]byte(inflated))
w.Close()
return b.Bytes()
}
func (output *BeegoOutput) Body(content []byte) {
output_writer := output.res.(io.Writer)
if output.EnableGzip == true && output.Context.Input.Header("Accept-Encoding") != "" {
splitted := strings.SplitN(output.Context.Input.Header("Accept-Encoding"), ",", -1)
encodings := make([]string, len(splitted))
for i, val := range splitted {
encodings[i] = strings.TrimSpace(val)
}
for _, val := range encodings {
if val == "gzip" {
output.Header("Content-Encoding", "gzip")
output_writer, _ = gzip.NewWriterLevel(output.res, gzip.BestSpeed)
break
} else if val == "deflate" {
output.Header("Content-Encoding", "deflate")
output_writer, _ = flate.NewWriter(output.res, flate.BestSpeed)
break
}
}
} else {
output.Header("Content-Length", strconv.Itoa(len(content)))
}
output_writer.Write(content)
switch output_writer.(type) {
case *gzip.Writer:
output_writer.(*gzip.Writer).Close()
case *flate.Writer:
output_writer.(*flate.Writer).Close()
}
}
func ExampleWriter_RegisterCompressor() {
// Override the default Deflate compressor with a higher compression
// level.
// Create a buffer to write our archive to.
buf := new(bytes.Buffer)
// Create a new zip archive.
w := zip.NewWriter(buf)
var fw *flate.Writer
// Register the deflator.
w.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
var err error
if fw == nil {
// Creating a flate compressor for every file is
// expensive, create one and reuse it.
fw, err = flate.NewWriter(out, flate.BestCompression)
} else {
fw.Reset(out)
}
return fw, err
})
// Proceed to add files to w.
}
func compressor(fw *Writer, toCompress <-chan *writerBlock, toWrite chan<- *writerBlock) {
switch fw.CompressionCodec {
case CompressionDeflate:
bb := new(bytes.Buffer)
comp, _ := flate.NewWriter(bb, flate.DefaultCompression)
for block := range toCompress {
_, block.err = comp.Write(block.encoded.Bytes())
block.err = comp.Close()
if block.err == nil {
block.compressed = bb.Bytes()
toWrite <- block
}
bb = new(bytes.Buffer)
comp.Reset(bb)
}
case CompressionNull:
for block := range toCompress {
block.compressed = block.encoded.Bytes()
toWrite <- block
}
case CompressionSnappy:
for block := range toCompress {
block.compressed = snappy.Encode(block.compressed, block.encoded.Bytes())
toWrite <- block
}
}
close(toWrite)
}
// Save persists a Siegfried struct to disk (path)
func (s *Siegfried) Save(path string) error {
ls := persist.NewLoadSaver(nil)
ls.SaveTime(s.C)
s.em.Save(ls)
s.cm.Save(ls)
s.bm.Save(ls)
s.tm.Save(ls)
ls.SaveTinyUInt(len(s.ids))
for _, i := range s.ids {
i.Save(ls)
}
if ls.Err != nil {
return ls.Err
}
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
_, err = f.Write(append(config.Magic(), byte(config.Version()[0]), byte(config.Version()[1])))
if err != nil {
return err
}
z, err := flate.NewWriter(f, 1)
if err != nil {
return err
}
_, err = z.Write(ls.Bytes())
z.Close()
return err
}
func (ent *RawEntry) writeTo(out io.Writer) error {
var buf bytes.Buffer
z, err := flate.NewWriter(&buf, 8)
if err != nil {
return err
}
if err := binary.Write(z, binary.LittleEndian, uint32(len(ent.LeafInput))); err != nil {
return err
}
if _, err := z.Write(ent.LeafInput); err != nil {
return err
}
if err := binary.Write(z, binary.LittleEndian, uint32(len(ent.ExtraData))); err != nil {
return err
}
if _, err := z.Write(ent.ExtraData); err != nil {
return err
}
if err := z.Close(); err != nil {
return err
}
bytes := buf.Bytes()
if err := binary.Write(out, binary.LittleEndian, uint32(len(bytes))); err != nil {
return err
}
if _, err := out.Write(bytes); err != nil {
return err
}
return nil
}
func CompressString(in string) string {
buf := new(bytes.Buffer)
compressor, _ := flate.NewWriter(buf, 9)
compressor.Write([]byte(in))
compressor.Close()
return buf.String()
}
// conditionally compress the HTTP response. this function must be executed
// after all response headers have been set by the underlying handler (because
// it needs to inspect them) but before they have been written to the client
// (because it needs to change the headers and the data writer).
func compressResponse(w *ResponseWriter, req *http.Request) {
// rudimentary "can this be compressed" check
if !compressable(w.Header().Get("Content-Type")) {
return
}
// do not re-encode
if w.Header().Get("Content-Encoding") != "" {
return
}
ae := req.Header.Get("Accept-Encoding")
// no q for u
switch {
case strings.Contains(ae, "gzip"):
w.WrapBodyWriter(func(w io.Writer) io.Writer {
return gzip.NewWriter(w)
})
w.Header().Set("Content-Encoding", "gzip")
w.Header().Del("content-length")
break
case strings.Contains(ae, "deflate"):
w.WrapBodyWriter(func(w io.Writer) io.Writer {
def, _ := flate.NewWriter(w, flate.DefaultCompression)
return def
})
w.Header().Set("Content-Encoding", "deflate")
w.Header().Del("content-length")
break
}
}
func (data *Data) Compress() error {
if data.IsCompressed() || len(data.Content) < 32 {
// already compressed, or not worth compressing
// because data is nil or too short
return nil
}
buf := bytes.Buffer{}
w, err := flate.NewWriter(&buf, flate.BestCompression)
if err != nil {
return err
}
n, err := w.Write(data.Content)
if err != nil {
return err
}
if n != len(data.Content) {
return errs.ServerError("cannot compress")
}
err = w.Close()
if err != nil {
return err
}
compressedBytes := buf.Bytes()
if len(compressedBytes) < len(data.Content) {
data.UncompressedLength = len(data.Content)
data.Content = compressedBytes
data.ContentEncoding = ceDeflate
}
return nil
}
//NewLogger creates a new Logger instance
func NewLogger(directory string) (*Logger, error) {
var err error
if directory == "" {
if directory, err = ioutil.TempDir("", "dlog"); err != nil {
return nil, err
}
}
os.MkdirAll(directory, 0755)
filePath := filepath.Join(directory, "dlog.bin")
wFile, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
log.Fatal(err)
}
w, _ := flate.NewWriter(wFile, flate.BestCompression)
l := &Logger{
wChan: make(chan model.LogEntry, 1000),
wFile: wFile,
}
go l.writeRoutine(w)
return l, nil
}
func handlerError(rw http.ResponseWriter, html string, code int) {
var b bytes.Buffer
w, err := flate.NewWriter(&b, flate.BestCompression)
if err != nil {
rw.Header().Set("Content-Type", "text/plain")
rw.WriteHeader(http.StatusBadGateway)
io.WriteString(rw, err.Error())
}
fmt.Fprintf(w, "HTTP/1.1 %d\r\n", code)
fmt.Fprintf(w, "Content-Type: text/html; charset=utf-8\r\n")
fmt.Fprintf(w, "Content-Length: %d\r\n", len(html))
io.WriteString(w, "\r\n")
io.WriteString(w, html)
w.Close()
b0 := make([]byte, 2)
binary.BigEndian.PutUint16(b0, uint16(b.Len()))
rw.Header().Set("Content-Type", "image/gif")
rw.Header().Set("Content-Length", strconv.Itoa(len(b0)+b.Len()))
rw.WriteHeader(http.StatusOK)
rw.Write(b0)
rw.Write(b.Bytes())
}
func BenchmarkWriteWithBestDeflateCompression(b *testing.B) {
file := benchFile()
w, _ := flate.NewWriter(file, flate.BestSpeed)
benchmarkWrite(b, w)
fi, _ := file.Stat()
b.SetBytes(int64(int(fi.Size()) / 1024 / 1024))
}
func Zip(unBytes []byte, contentEncoding string) (zipBytes []byte, err error) {
switch contentEncoding {
case "":
zipBytes = unBytes
break
case "gzip":
var bf bytes.Buffer
gw := gzip.NewWriter(&bf)
gw.Write(unBytes)
gw.Close()
zipBytes = bf.Bytes()
case "deflate":
var bf bytes.Buffer
var gw *flate.Writer
gw, err = flate.NewWriter(&bf, -1)
if err != nil {
return
}
gw.Write(unBytes)
gw.Close()
zipBytes = bf.Bytes()
default:
err = fmt.Errorf("Zip Unkown Content-Encoding: %v", contentEncoding)
}
return
}
func Flate(r io.Writer) (c Compressor) {
var err error
if c, err = flate.NewWriter(r, -1); err != nil {
panic(err)
}
return
}
func deflate(inp []byte) ([]byte, error) {
out := new(bytes.Buffer)
writer, _ := flate.NewWriter(out, 1)
io.Copy(writer, bytes.NewBuffer(inp))
err := writer.Close()
return out.Bytes(), err
}
// NewWriterLevel creates a new io.WriteCloser that satisfies writes by compressing data written to w.
// It is the caller's responsibility to call Close on the WriteCloser when done.
// level is the compression level, which can be DefaultCompression, NoCompression,
// or any integer value between BestSpeed and BestCompression (inclusive).
func NewWriterLevel(w io.Writer, level int) (io.WriteCloser, os.Error) {
z := new(writer)
// ZLIB has a two-byte header (as documented in RFC 1950).
// The first four bits is the CINFO (compression info), which is 7 for the default deflate window size.
// The next four bits is the CM (compression method), which is 8 for deflate.
z.scratch[0] = 0x78
// The next two bits is the FLEVEL (compression level). The four values are:
// 0=fastest, 1=fast, 2=default, 3=best.
// The next bit, FDICT, is unused, in this implementation.
// The final five FCHECK bits form a mod-31 checksum.
switch level {
case 0, 1:
z.scratch[1] = 0x01
case 2, 3, 4, 5:
z.scratch[1] = 0x5e
case 6, -1:
z.scratch[1] = 0x9c
case 7, 8, 9:
z.scratch[1] = 0xda
default:
return nil, os.NewError("level out of range")
}
_, err := w.Write(z.scratch[0:2])
if err != nil {
return nil, err
}
z.w = w
z.compressor = flate.NewWriter(w, level)
z.digest = adler32.New()
return z, nil
}
func Encrypt(key string, privatekey PrivateKey, win io.Writer, length, sequence int64) (w io.Writer, e os.Error) {
c, e := simpleCipher(key)
if e != nil {
return
}
priv, e := readRSAKey(privatekey)
if e != nil {
return
}
iv := make([]byte, c.BlockSize())
_, e = rand.Read(iv)
if e != nil {
return
}
_, e = win.Write(iv) // pass the iv across first
if e != nil {
return
}
wraw := cipher.StreamWriter{cipher.NewCFBEncrypter(c, iv), win, nil}
wenc := flate.NewWriter(wraw, flate.BestCompression)
e = binary.Write(wenc, binary.LittleEndian, length)
if e != nil {
return
}
e = binary.Write(wenc, binary.LittleEndian, sequence)
if e != nil {
return
}
return &hashWriter{wenc, wraw, sha256.New(), priv, length}, nil
}
func allocFlateWriter() interface{} {
// flate.NewWriter (as of this writing) only returns an error
// if the second argument is invalid. As we are using a standard
// compression level here, there is no way this can err
w, _ := flate.NewWriter(emptyBuffer, flate.DefaultCompression)
return w
}
func Compress(in []byte) []byte {
buf := new(bytes.Buffer)
compressor, _ := flate.NewWriter(buf, 9)
compressor.Write(in)
compressor.Close()
return buf.Bytes()
}
// Save persists a Siegfried struct to disk (path)
func (s *Siegfried) Save(path string) error {
ls := persist.NewLoadSaver(nil)
ls.SaveTime(s.C)
namematcher.Save(s.nm, ls)
mimematcher.Save(s.mm, ls)
containermatcher.Save(s.cm, ls)
xmlmatcher.Save(s.xm, ls)
riffmatcher.Save(s.rm, ls)
bytematcher.Save(s.bm, ls)
textmatcher.Save(s.tm, ls)
ls.SaveTinyUInt(len(s.ids))
for _, i := range s.ids {
i.Save(ls)
}
if ls.Err != nil {
return ls.Err
}
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
_, err = f.Write(append(config.Magic(), byte(config.Version()[0]), byte(config.Version()[1])))
if err != nil {
return err
}
z, err := flate.NewWriter(f, 1)
if err != nil {
return err
}
_, err = z.Write(ls.Bytes())
z.Close()
return err
}
func (c *Controller) writeToWriter(rb []byte) {
output_writer := c.Ctx.ResponseWriter.(io.Writer)
if EnableGzip == true && c.Ctx.Request.Header.Get("Accept-Encoding") != "" {
splitted := strings.SplitN(c.Ctx.Request.Header.Get("Accept-Encoding"), ",", -1)
encodings := make([]string, len(splitted))
for i, val := range splitted {
encodings[i] = strings.TrimSpace(val)
}
for _, val := range encodings {
if val == "gzip" {
c.Ctx.ResponseWriter.Header().Set("Content-Encoding", "gzip")
output_writer, _ = gzip.NewWriterLevel(c.Ctx.ResponseWriter, gzip.BestSpeed)
break
} else if val == "deflate" {
c.Ctx.ResponseWriter.Header().Set("Content-Encoding", "deflate")
output_writer, _ = flate.NewWriter(c.Ctx.ResponseWriter, flate.BestSpeed)
break
}
}
} else {
c.Ctx.SetHeader("Content-Length", strconv.Itoa(len(rb)), true)
}
output_writer.Write(rb)
switch output_writer.(type) {
case *gzip.Writer:
output_writer.(*gzip.Writer).Close()
case *flate.Writer:
output_writer.(*flate.Writer).Close()
case io.WriteCloser:
output_writer.(io.WriteCloser).Close()
}
}
func CompressHandlerFunc(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
acceptEncoding := r.Header.Get("Accept-Encoding")
switch {
case strings.Contains(acceptEncoding, "deflate"):
w.Header().Set("Content-Encoding", "deflate")
gzw, err := flate.NewWriter(w, -1)
if err != nil {
panic(err)
}
defer gzw.Close()
gzr := ResponseWriterWraper{Writer: gzw, ResponseWriter: w}
f(gzr, r)
return
case strings.Contains(acceptEncoding, "gzip"):
w.Header().Set("Content-Encoding", "gzip")
gzw := gzip.NewWriter(w)
defer gzw.Close()
gzr := ResponseWriterWraper{Writer: gzw, ResponseWriter: w}
f(gzr, r)
return
default:
f(w, r)
}
}
}
func (r *ResponseWriter) compressor(w io.Writer, l int) io.WriteCloser {
// If headers sent then we're too late for compression.
if r.headersSent {
return nil
}
if r.compMinLength == 0 || l < r.compMinLength {
return nil
}
e := strings.Split(r.acceptedEncoding, ",")
for _, ae := range e {
switch strings.TrimSpace(ae) {
case "gzip":
c := gzip.NewWriter(w)
r.rw.Header().Set("Content-Encoding", "gzip")
return c
case "deflate":
c, err := flate.NewWriter(w, flate.DefaultCompression)
if err != nil {
return nil
}
r.rw.Header().Set("Content-Encoding", "deflate")
return c
}
}
return nil
}
func CompressHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
L:
for _, enc := range strings.Split(r.Header.Get("Accept-Encoding"), ",") {
switch strings.TrimSpace(enc) {
case "gzip":
w.Header().Set("Content-Encoding", "gzip")
w.Header().Add("Vary", "Accept-Encoding")
gw := gzip.NewWriter(w)
defer gw.Close()
w = &compressResponseWriter{
Writer: gw,
ResponseWriter: w,
}
break L
case "deflate":
w.Header().Set("Content-Encoding", "deflate")
w.Header().Add("Vary", "Accept-Encoding")
fw, _ := flate.NewWriter(w, flate.DefaultCompression)
defer fw.Close()
w = &compressResponseWriter{
Writer: fw,
ResponseWriter: w,
}
break L
}
}
h.ServeHTTP(w, r)
})
}
func NewCompressedReadWriteCloser(rwc io.ReadWriteCloser) io.ReadWriteCloser {
w, _ := flate.NewWriter(rwc, 1)
return &compressedRWC{
flate.NewReader(rwc),
w,
}
}
// Init Compression Buffer (Call Before Writing to Client)
func (w *Web) InitCompression() {
if w.Env.Get("Connection") == "Upgrade" {
return
}
if w.Env.Get("Accept-Encoding") == "" {
return
}
for _, encoding := range strings.Split(w.Env.Get("Accept-Encoding"), ",") {
encoding = strings.TrimSpace(strings.ToLower(encoding))
switch encoding {
case "gzip":
w.pri.reswrite = gzip.NewWriter(w.webInterface)
w.Header().Set("Content-Encoding", encoding)
return
case "deflate":
w.pri.reswrite, _ = flate.NewWriter(w.webInterface, flate.DefaultCompression)
w.Header().Set("Content-Encoding", encoding)
return
}
}
}
// Write writes a compressed form of p to the underlying io.Writer. The
// compressed bytes are not necessarily flushed until the Writer is closed.
func (z *Writer) Write(p []byte) (int, error) {
if z.err != nil {
return 0, z.err
}
var n int
// Write the GZIP header lazily.
if !z.wroteHeader {
z.wroteHeader = true
z.buf = [10]byte{0: gzipID1, 1: gzipID2, 2: gzipDeflate}
if z.Extra != nil {
z.buf[3] |= 0x04
}
if z.Name != "" {
z.buf[3] |= 0x08
}
if z.Comment != "" {
z.buf[3] |= 0x10
}
if z.ModTime.After(time.Unix(0, 0)) {
// Section 2.3.1, the zero value for MTIME means that the
// modified time is not set.
le.PutUint32(z.buf[4:8], uint32(z.ModTime.Unix()))
}
if z.level == BestCompression {
z.buf[8] = 2
} else if z.level == BestSpeed {
z.buf[8] = 4
}
z.buf[9] = z.OS
n, z.err = z.w.Write(z.buf[:10])
if z.err != nil {
return n, z.err
}
if z.Extra != nil {
z.err = z.writeBytes(z.Extra)
if z.err != nil {
return n, z.err
}
}
if z.Name != "" {
z.err = z.writeString(z.Name)
if z.err != nil {
return n, z.err
}
}
if z.Comment != "" {
z.err = z.writeString(z.Comment)
if z.err != nil {
return n, z.err
}
}
if z.compressor == nil {
z.compressor, _ = flate.NewWriter(z.w, z.level)
}
}
z.size += uint32(len(p))
z.digest = crc32.Update(z.digest, crc32.IEEETable, p)
n, z.err = z.compressor.Write(p)
return n, z.err
}
func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver Model) Connection {
cr := &countingReader{Reader: reader}
cw := &countingWriter{Writer: writer}
flrd := flate.NewReader(cr)
flwr, err := flate.NewWriter(cw, flate.BestSpeed)
if err != nil {
panic(err)
}
wb := bufio.NewWriter(flwr)
c := rawConnection{
id: nodeID,
receiver: nativeModel{receiver},
reader: flrd,
cr: cr,
xr: xdr.NewReader(flrd),
writer: flwr,
cw: cw,
wb: wb,
xw: xdr.NewWriter(wb),
awaiting: make([]chan asyncResult, 0x1000),
indexSent: make(map[string]map[string][2]int64),
outbox: make(chan []encodable),
nextID: make(chan int),
closed: make(chan struct{}),
}
go c.readerLoop()
go c.writerLoop()
go c.pingerLoop()
go c.idGenerator()
return wireFormatConnection{&c}
}
// SerializeCompressed serializes a compressed data packet to w and
// returns a WriteCloser to which the literal data packets themselves
// can be written and which MUST be closed on completion. If cc is
// nil, sensible defaults will be used to configure the compression
// algorithm.
func SerializeCompressed(w io.WriteCloser, algo CompressionAlgo, cc *CompressionConfig) (literaldata io.WriteCloser, err error) {
compressed, err := serializeStreamHeader(w, packetTypeCompressed)
if err != nil {
return
}
_, err = compressed.Write([]byte{uint8(algo)})
if err != nil {
return
}
level := DefaultCompression
if cc != nil {
level = cc.Level
}
var compressor io.WriteCloser
switch algo {
case CompressionZIP:
compressor, err = flate.NewWriter(compressed, level)
case CompressionZLIB:
compressor, err = zlib.NewWriterLevel(compressed, level)
default:
s := strconv.Itoa(int(algo))
err = errors.UnsupportedError("Unsupported compression algorithm: " + s)
}
if err != nil {
return
}
literaldata = compressedWriteCloser{compressed, compressor}
return
}
func (ctx *Context) write_gzip(content []byte) {
output_writer := ctx.W.(io.Writer)
splitted := strings.SplitN(ctx.GetHeader("Accept-Encoding"), ",", -1)
encodings := make([]string, len(splitted))
for i, val := range splitted {
encodings[i] = strings.TrimSpace(val)
}
for _, val := range encodings {
if val == "gzip" {
ctx.SetHeader("Content-Encoding", "gzip")
output_writer, _ = gzip.NewWriterLevel(ctx.W, gzip.BestSpeed)
break
} else if val == "deflate" {
ctx.SetHeader("Content-Encoding", "deflate")
output_writer, _ = flate.NewWriter(ctx.W, flate.BestSpeed)
break
}
}
output_writer.Write(content)
switch output_writer.(type) {
case *gzip.Writer:
output_writer.(*gzip.Writer).Close()
case *flate.Writer:
output_writer.(*flate.Writer).Close()
/* case io.WriteCloser:
output_writer.(io.WriteCloser).Close()*/
}
}