Example #1
0
// Deflate utility that compresses a string using the flate algo
func Deflate(inflated string) []byte {
	var b bytes.Buffer
	w, _ := flate.NewWriter(&b, -1)
	w.Write([]byte(inflated))
	w.Close()
	return b.Bytes()
}
Example #2
0
func (output *BeegoOutput) Body(content []byte) {
	output_writer := output.res.(io.Writer)
	if output.EnableGzip == true && output.Context.Input.Header("Accept-Encoding") != "" {
		splitted := strings.SplitN(output.Context.Input.Header("Accept-Encoding"), ",", -1)
		encodings := make([]string, len(splitted))

		for i, val := range splitted {
			encodings[i] = strings.TrimSpace(val)
		}
		for _, val := range encodings {
			if val == "gzip" {
				output.Header("Content-Encoding", "gzip")
				output_writer, _ = gzip.NewWriterLevel(output.res, gzip.BestSpeed)

				break
			} else if val == "deflate" {
				output.Header("Content-Encoding", "deflate")
				output_writer, _ = flate.NewWriter(output.res, flate.BestSpeed)
				break
			}
		}
	} else {
		output.Header("Content-Length", strconv.Itoa(len(content)))
	}
	output_writer.Write(content)
	switch output_writer.(type) {
	case *gzip.Writer:
		output_writer.(*gzip.Writer).Close()
	case *flate.Writer:
		output_writer.(*flate.Writer).Close()
	}
}
Example #3
0
func ExampleWriter_RegisterCompressor() {
	// Override the default Deflate compressor with a higher compression
	// level.

	// Create a buffer to write our archive to.
	buf := new(bytes.Buffer)

	// Create a new zip archive.
	w := zip.NewWriter(buf)

	var fw *flate.Writer

	// Register the deflator.
	w.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
		var err error
		if fw == nil {
			// Creating a flate compressor for every file is
			// expensive, create one and reuse it.
			fw, err = flate.NewWriter(out, flate.BestCompression)
		} else {
			fw.Reset(out)
		}
		return fw, err
	})

	// Proceed to add files to w.
}
Example #4
0
func compressor(fw *Writer, toCompress <-chan *writerBlock, toWrite chan<- *writerBlock) {
	switch fw.CompressionCodec {
	case CompressionDeflate:
		bb := new(bytes.Buffer)
		comp, _ := flate.NewWriter(bb, flate.DefaultCompression)
		for block := range toCompress {
			_, block.err = comp.Write(block.encoded.Bytes())
			block.err = comp.Close()
			if block.err == nil {
				block.compressed = bb.Bytes()
				toWrite <- block
			}
			bb = new(bytes.Buffer)
			comp.Reset(bb)
		}
	case CompressionNull:
		for block := range toCompress {
			block.compressed = block.encoded.Bytes()
			toWrite <- block
		}
	case CompressionSnappy:
		for block := range toCompress {
			block.compressed = snappy.Encode(block.compressed, block.encoded.Bytes())
			toWrite <- block
		}
	}
	close(toWrite)
}
Example #5
0
// Save persists a Siegfried struct to disk (path)
func (s *Siegfried) Save(path string) error {
	ls := persist.NewLoadSaver(nil)
	ls.SaveTime(s.C)
	s.em.Save(ls)
	s.cm.Save(ls)
	s.bm.Save(ls)
	s.tm.Save(ls)
	ls.SaveTinyUInt(len(s.ids))
	for _, i := range s.ids {
		i.Save(ls)
	}
	if ls.Err != nil {
		return ls.Err
	}
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = f.Write(append(config.Magic(), byte(config.Version()[0]), byte(config.Version()[1])))
	if err != nil {
		return err
	}
	z, err := flate.NewWriter(f, 1)
	if err != nil {
		return err
	}
	_, err = z.Write(ls.Bytes())
	z.Close()
	return err
}
Example #6
0
func (ent *RawEntry) writeTo(out io.Writer) error {
	var buf bytes.Buffer
	z, err := flate.NewWriter(&buf, 8)
	if err != nil {
		return err
	}
	if err := binary.Write(z, binary.LittleEndian, uint32(len(ent.LeafInput))); err != nil {
		return err
	}
	if _, err := z.Write(ent.LeafInput); err != nil {
		return err
	}
	if err := binary.Write(z, binary.LittleEndian, uint32(len(ent.ExtraData))); err != nil {
		return err
	}
	if _, err := z.Write(ent.ExtraData); err != nil {
		return err
	}
	if err := z.Close(); err != nil {
		return err
	}

	bytes := buf.Bytes()
	if err := binary.Write(out, binary.LittleEndian, uint32(len(bytes))); err != nil {
		return err
	}
	if _, err := out.Write(bytes); err != nil {
		return err
	}

	return nil
}
Example #7
0
func CompressString(in string) string {
	buf := new(bytes.Buffer)
	compressor, _ := flate.NewWriter(buf, 9)
	compressor.Write([]byte(in))
	compressor.Close()
	return buf.String()
}
Example #8
0
// conditionally compress the HTTP response. this function must be executed
// after all response headers have been set by the underlying handler (because
// it needs to inspect them) but before they have been written to the client
// (because it needs to change the headers and the data writer).
func compressResponse(w *ResponseWriter, req *http.Request) {
	// rudimentary "can this be compressed" check
	if !compressable(w.Header().Get("Content-Type")) {
		return
	}
	// do not re-encode
	if w.Header().Get("Content-Encoding") != "" {
		return
	}
	ae := req.Header.Get("Accept-Encoding")
	// no q for u
	switch {
	case strings.Contains(ae, "gzip"):
		w.WrapBodyWriter(func(w io.Writer) io.Writer {
			return gzip.NewWriter(w)
		})
		w.Header().Set("Content-Encoding", "gzip")
		w.Header().Del("content-length")
		break
	case strings.Contains(ae, "deflate"):
		w.WrapBodyWriter(func(w io.Writer) io.Writer {
			def, _ := flate.NewWriter(w, flate.DefaultCompression)
			return def
		})
		w.Header().Set("Content-Encoding", "deflate")
		w.Header().Del("content-length")
		break
	}
}
Example #9
0
func (data *Data) Compress() error {
	if data.IsCompressed() || len(data.Content) < 32 {
		// already compressed, or not worth compressing
		// because data is nil or too short
		return nil
	}

	buf := bytes.Buffer{}
	w, err := flate.NewWriter(&buf, flate.BestCompression)
	if err != nil {
		return err
	}
	n, err := w.Write(data.Content)
	if err != nil {
		return err
	}
	if n != len(data.Content) {
		return errs.ServerError("cannot compress")
	}
	err = w.Close()
	if err != nil {
		return err
	}
	compressedBytes := buf.Bytes()

	if len(compressedBytes) < len(data.Content) {
		data.UncompressedLength = len(data.Content)
		data.Content = compressedBytes
		data.ContentEncoding = ceDeflate
	}

	return nil
}
Example #10
0
//NewLogger creates a new Logger instance
func NewLogger(directory string) (*Logger, error) {
	var err error
	if directory == "" {
		if directory, err = ioutil.TempDir("", "dlog"); err != nil {
			return nil, err
		}
	}
	os.MkdirAll(directory, 0755)
	filePath := filepath.Join(directory, "dlog.bin")

	wFile, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}
	w, _ := flate.NewWriter(wFile, flate.BestCompression)

	l := &Logger{
		wChan: make(chan model.LogEntry, 1000),
		wFile: wFile,
	}

	go l.writeRoutine(w)

	return l, nil
}
Example #11
0
func handlerError(rw http.ResponseWriter, html string, code int) {
	var b bytes.Buffer
	w, err := flate.NewWriter(&b, flate.BestCompression)
	if err != nil {
		rw.Header().Set("Content-Type", "text/plain")
		rw.WriteHeader(http.StatusBadGateway)
		io.WriteString(rw, err.Error())
	}

	fmt.Fprintf(w, "HTTP/1.1 %d\r\n", code)
	fmt.Fprintf(w, "Content-Type: text/html; charset=utf-8\r\n")
	fmt.Fprintf(w, "Content-Length: %d\r\n", len(html))
	io.WriteString(w, "\r\n")
	io.WriteString(w, html)
	w.Close()

	b0 := make([]byte, 2)
	binary.BigEndian.PutUint16(b0, uint16(b.Len()))

	rw.Header().Set("Content-Type", "image/gif")
	rw.Header().Set("Content-Length", strconv.Itoa(len(b0)+b.Len()))
	rw.WriteHeader(http.StatusOK)
	rw.Write(b0)
	rw.Write(b.Bytes())
}
Example #12
0
func BenchmarkWriteWithBestDeflateCompression(b *testing.B) {
	file := benchFile()
	w, _ := flate.NewWriter(file, flate.BestSpeed)
	benchmarkWrite(b, w)
	fi, _ := file.Stat()
	b.SetBytes(int64(int(fi.Size()) / 1024 / 1024))
}
Example #13
0
func Zip(unBytes []byte, contentEncoding string) (zipBytes []byte, err error) {

	switch contentEncoding {
	case "":
		zipBytes = unBytes
		break

	case "gzip":
		var bf bytes.Buffer
		gw := gzip.NewWriter(&bf)
		gw.Write(unBytes)
		gw.Close()

		zipBytes = bf.Bytes()

	case "deflate":
		var bf bytes.Buffer
		var gw *flate.Writer
		gw, err = flate.NewWriter(&bf, -1)
		if err != nil {
			return
		}
		gw.Write(unBytes)
		gw.Close()

		zipBytes = bf.Bytes()

	default:
		err = fmt.Errorf("Zip Unkown Content-Encoding: %v", contentEncoding)
	}

	return
}
Example #14
0
func Flate(r io.Writer) (c Compressor) {
	var err error
	if c, err = flate.NewWriter(r, -1); err != nil {
		panic(err)
	}
	return
}
Example #15
0
func deflate(inp []byte) ([]byte, error) {
	out := new(bytes.Buffer)
	writer, _ := flate.NewWriter(out, 1)
	io.Copy(writer, bytes.NewBuffer(inp))
	err := writer.Close()
	return out.Bytes(), err
}
// NewWriterLevel creates a new io.WriteCloser that satisfies writes by compressing data written to w.
// It is the caller's responsibility to call Close on the WriteCloser when done.
// level is the compression level, which can be DefaultCompression, NoCompression,
// or any integer value between BestSpeed and BestCompression (inclusive).
func NewWriterLevel(w io.Writer, level int) (io.WriteCloser, os.Error) {
	z := new(writer)
	// ZLIB has a two-byte header (as documented in RFC 1950).
	// The first four bits is the CINFO (compression info), which is 7 for the default deflate window size.
	// The next four bits is the CM (compression method), which is 8 for deflate.
	z.scratch[0] = 0x78
	// The next two bits is the FLEVEL (compression level). The four values are:
	// 0=fastest, 1=fast, 2=default, 3=best.
	// The next bit, FDICT, is unused, in this implementation.
	// The final five FCHECK bits form a mod-31 checksum.
	switch level {
	case 0, 1:
		z.scratch[1] = 0x01
	case 2, 3, 4, 5:
		z.scratch[1] = 0x5e
	case 6, -1:
		z.scratch[1] = 0x9c
	case 7, 8, 9:
		z.scratch[1] = 0xda
	default:
		return nil, os.NewError("level out of range")
	}
	_, err := w.Write(z.scratch[0:2])
	if err != nil {
		return nil, err
	}
	z.w = w
	z.compressor = flate.NewWriter(w, level)
	z.digest = adler32.New()
	return z, nil
}
Example #17
0
func Encrypt(key string, privatekey PrivateKey, win io.Writer, length, sequence int64) (w io.Writer, e os.Error) {
	c, e := simpleCipher(key)
	if e != nil {
		return
	}
	priv, e := readRSAKey(privatekey)
	if e != nil {
		return
	}
	iv := make([]byte, c.BlockSize())
	_, e = rand.Read(iv)
	if e != nil {
		return
	}
	_, e = win.Write(iv) // pass the iv across first
	if e != nil {
		return
	}
	wraw := cipher.StreamWriter{cipher.NewCFBEncrypter(c, iv), win, nil}
	wenc := flate.NewWriter(wraw, flate.BestCompression)
	e = binary.Write(wenc, binary.LittleEndian, length)
	if e != nil {
		return
	}
	e = binary.Write(wenc, binary.LittleEndian, sequence)
	if e != nil {
		return
	}
	return &hashWriter{wenc, wraw, sha256.New(), priv, length}, nil
}
Example #18
0
func allocFlateWriter() interface{} {
	// flate.NewWriter (as of this writing) only returns an error
	// if the second argument is invalid. As we are using a standard
	// compression level here, there is no way this can err
	w, _ := flate.NewWriter(emptyBuffer, flate.DefaultCompression)
	return w
}
Example #19
0
func Compress(in []byte) []byte {
	buf := new(bytes.Buffer)
	compressor, _ := flate.NewWriter(buf, 9)
	compressor.Write(in)
	compressor.Close()
	return buf.Bytes()
}
Example #20
0
// Save persists a Siegfried struct to disk (path)
func (s *Siegfried) Save(path string) error {
	ls := persist.NewLoadSaver(nil)
	ls.SaveTime(s.C)
	namematcher.Save(s.nm, ls)
	mimematcher.Save(s.mm, ls)
	containermatcher.Save(s.cm, ls)
	xmlmatcher.Save(s.xm, ls)
	riffmatcher.Save(s.rm, ls)
	bytematcher.Save(s.bm, ls)
	textmatcher.Save(s.tm, ls)
	ls.SaveTinyUInt(len(s.ids))
	for _, i := range s.ids {
		i.Save(ls)
	}
	if ls.Err != nil {
		return ls.Err
	}
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = f.Write(append(config.Magic(), byte(config.Version()[0]), byte(config.Version()[1])))
	if err != nil {
		return err
	}
	z, err := flate.NewWriter(f, 1)
	if err != nil {
		return err
	}
	_, err = z.Write(ls.Bytes())
	z.Close()
	return err
}
Example #21
0
func (c *Controller) writeToWriter(rb []byte) {
	output_writer := c.Ctx.ResponseWriter.(io.Writer)
	if EnableGzip == true && c.Ctx.Request.Header.Get("Accept-Encoding") != "" {
		splitted := strings.SplitN(c.Ctx.Request.Header.Get("Accept-Encoding"), ",", -1)
		encodings := make([]string, len(splitted))

		for i, val := range splitted {
			encodings[i] = strings.TrimSpace(val)
		}
		for _, val := range encodings {
			if val == "gzip" {
				c.Ctx.ResponseWriter.Header().Set("Content-Encoding", "gzip")
				output_writer, _ = gzip.NewWriterLevel(c.Ctx.ResponseWriter, gzip.BestSpeed)

				break
			} else if val == "deflate" {
				c.Ctx.ResponseWriter.Header().Set("Content-Encoding", "deflate")
				output_writer, _ = flate.NewWriter(c.Ctx.ResponseWriter, flate.BestSpeed)
				break
			}
		}
	} else {
		c.Ctx.SetHeader("Content-Length", strconv.Itoa(len(rb)), true)
	}
	output_writer.Write(rb)
	switch output_writer.(type) {
	case *gzip.Writer:
		output_writer.(*gzip.Writer).Close()
	case *flate.Writer:
		output_writer.(*flate.Writer).Close()
	case io.WriteCloser:
		output_writer.(io.WriteCloser).Close()
	}
}
Example #22
0
func CompressHandlerFunc(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		acceptEncoding := r.Header.Get("Accept-Encoding")
		switch {
		case strings.Contains(acceptEncoding, "deflate"):
			w.Header().Set("Content-Encoding", "deflate")
			gzw, err := flate.NewWriter(w, -1)
			if err != nil {
				panic(err)
			}
			defer gzw.Close()
			gzr := ResponseWriterWraper{Writer: gzw, ResponseWriter: w}
			f(gzr, r)
			return
		case strings.Contains(acceptEncoding, "gzip"):
			w.Header().Set("Content-Encoding", "gzip")
			gzw := gzip.NewWriter(w)
			defer gzw.Close()
			gzr := ResponseWriterWraper{Writer: gzw, ResponseWriter: w}
			f(gzr, r)
			return
		default:
			f(w, r)
		}
	}
}
Example #23
0
func (r *ResponseWriter) compressor(w io.Writer, l int) io.WriteCloser {
	// If headers sent then we're too late for compression.
	if r.headersSent {
		return nil
	}
	if r.compMinLength == 0 || l < r.compMinLength {
		return nil
	}
	e := strings.Split(r.acceptedEncoding, ",")
	for _, ae := range e {
		switch strings.TrimSpace(ae) {
		case "gzip":
			c := gzip.NewWriter(w)
			r.rw.Header().Set("Content-Encoding", "gzip")
			return c
		case "deflate":
			c, err := flate.NewWriter(w, flate.DefaultCompression)
			if err != nil {
				return nil
			}
			r.rw.Header().Set("Content-Encoding", "deflate")
			return c
		}
	}
	return nil
}
Example #24
0
func CompressHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	L:
		for _, enc := range strings.Split(r.Header.Get("Accept-Encoding"), ",") {
			switch strings.TrimSpace(enc) {
			case "gzip":
				w.Header().Set("Content-Encoding", "gzip")
				w.Header().Add("Vary", "Accept-Encoding")

				gw := gzip.NewWriter(w)
				defer gw.Close()

				w = &compressResponseWriter{
					Writer:         gw,
					ResponseWriter: w,
				}
				break L
			case "deflate":
				w.Header().Set("Content-Encoding", "deflate")
				w.Header().Add("Vary", "Accept-Encoding")

				fw, _ := flate.NewWriter(w, flate.DefaultCompression)
				defer fw.Close()

				w = &compressResponseWriter{
					Writer:         fw,
					ResponseWriter: w,
				}
				break L
			}
		}

		h.ServeHTTP(w, r)
	})
}
Example #25
0
func NewCompressedReadWriteCloser(rwc io.ReadWriteCloser) io.ReadWriteCloser {
	w, _ := flate.NewWriter(rwc, 1)
	return &compressedRWC{
		flate.NewReader(rwc),
		w,
	}
}
Example #26
0
// Init Compression Buffer (Call Before Writing to Client)
func (w *Web) InitCompression() {
	if w.Env.Get("Connection") == "Upgrade" {
		return
	}

	if w.Env.Get("Accept-Encoding") == "" {
		return
	}

	for _, encoding := range strings.Split(w.Env.Get("Accept-Encoding"), ",") {
		encoding = strings.TrimSpace(strings.ToLower(encoding))

		switch encoding {
		case "gzip":
			w.pri.reswrite = gzip.NewWriter(w.webInterface)
			w.Header().Set("Content-Encoding", encoding)
			return
		case "deflate":
			w.pri.reswrite, _ = flate.NewWriter(w.webInterface, flate.DefaultCompression)
			w.Header().Set("Content-Encoding", encoding)
			return
		}

	}
}
Example #27
0
// Write writes a compressed form of p to the underlying io.Writer. The
// compressed bytes are not necessarily flushed until the Writer is closed.
func (z *Writer) Write(p []byte) (int, error) {
	if z.err != nil {
		return 0, z.err
	}
	var n int
	// Write the GZIP header lazily.
	if !z.wroteHeader {
		z.wroteHeader = true
		z.buf = [10]byte{0: gzipID1, 1: gzipID2, 2: gzipDeflate}
		if z.Extra != nil {
			z.buf[3] |= 0x04
		}
		if z.Name != "" {
			z.buf[3] |= 0x08
		}
		if z.Comment != "" {
			z.buf[3] |= 0x10
		}
		if z.ModTime.After(time.Unix(0, 0)) {
			// Section 2.3.1, the zero value for MTIME means that the
			// modified time is not set.
			le.PutUint32(z.buf[4:8], uint32(z.ModTime.Unix()))
		}
		if z.level == BestCompression {
			z.buf[8] = 2
		} else if z.level == BestSpeed {
			z.buf[8] = 4
		}
		z.buf[9] = z.OS
		n, z.err = z.w.Write(z.buf[:10])
		if z.err != nil {
			return n, z.err
		}
		if z.Extra != nil {
			z.err = z.writeBytes(z.Extra)
			if z.err != nil {
				return n, z.err
			}
		}
		if z.Name != "" {
			z.err = z.writeString(z.Name)
			if z.err != nil {
				return n, z.err
			}
		}
		if z.Comment != "" {
			z.err = z.writeString(z.Comment)
			if z.err != nil {
				return n, z.err
			}
		}
		if z.compressor == nil {
			z.compressor, _ = flate.NewWriter(z.w, z.level)
		}
	}
	z.size += uint32(len(p))
	z.digest = crc32.Update(z.digest, crc32.IEEETable, p)
	n, z.err = z.compressor.Write(p)
	return n, z.err
}
Example #28
0
func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver Model) Connection {
	cr := &countingReader{Reader: reader}
	cw := &countingWriter{Writer: writer}

	flrd := flate.NewReader(cr)
	flwr, err := flate.NewWriter(cw, flate.BestSpeed)
	if err != nil {
		panic(err)
	}
	wb := bufio.NewWriter(flwr)

	c := rawConnection{
		id:        nodeID,
		receiver:  nativeModel{receiver},
		reader:    flrd,
		cr:        cr,
		xr:        xdr.NewReader(flrd),
		writer:    flwr,
		cw:        cw,
		wb:        wb,
		xw:        xdr.NewWriter(wb),
		awaiting:  make([]chan asyncResult, 0x1000),
		indexSent: make(map[string]map[string][2]int64),
		outbox:    make(chan []encodable),
		nextID:    make(chan int),
		closed:    make(chan struct{}),
	}

	go c.readerLoop()
	go c.writerLoop()
	go c.pingerLoop()
	go c.idGenerator()

	return wireFormatConnection{&c}
}
Example #29
0
// SerializeCompressed serializes a compressed data packet to w and
// returns a WriteCloser to which the literal data packets themselves
// can be written and which MUST be closed on completion. If cc is
// nil, sensible defaults will be used to configure the compression
// algorithm.
func SerializeCompressed(w io.WriteCloser, algo CompressionAlgo, cc *CompressionConfig) (literaldata io.WriteCloser, err error) {
	compressed, err := serializeStreamHeader(w, packetTypeCompressed)
	if err != nil {
		return
	}

	_, err = compressed.Write([]byte{uint8(algo)})
	if err != nil {
		return
	}

	level := DefaultCompression
	if cc != nil {
		level = cc.Level
	}

	var compressor io.WriteCloser
	switch algo {
	case CompressionZIP:
		compressor, err = flate.NewWriter(compressed, level)
	case CompressionZLIB:
		compressor, err = zlib.NewWriterLevel(compressed, level)
	default:
		s := strconv.Itoa(int(algo))
		err = errors.UnsupportedError("Unsupported compression algorithm: " + s)
	}
	if err != nil {
		return
	}

	literaldata = compressedWriteCloser{compressed, compressor}

	return
}
Example #30
0
func (ctx *Context) write_gzip(content []byte) {
	output_writer := ctx.W.(io.Writer)
	splitted := strings.SplitN(ctx.GetHeader("Accept-Encoding"), ",", -1)
	encodings := make([]string, len(splitted))

	for i, val := range splitted {
		encodings[i] = strings.TrimSpace(val)
	}
	for _, val := range encodings {
		if val == "gzip" {
			ctx.SetHeader("Content-Encoding", "gzip")
			output_writer, _ = gzip.NewWriterLevel(ctx.W, gzip.BestSpeed)
			break
		} else if val == "deflate" {
			ctx.SetHeader("Content-Encoding", "deflate")
			output_writer, _ = flate.NewWriter(ctx.W, flate.BestSpeed)
			break
		}
	}
	output_writer.Write(content)
	switch output_writer.(type) {
	case *gzip.Writer:
		output_writer.(*gzip.Writer).Close()
	case *flate.Writer:
		output_writer.(*flate.Writer).Close()
		/*	case io.WriteCloser:
			output_writer.(io.WriteCloser).Close()*/
	}
}