// GetPage is an HTTP client that automatically decodes gzip when necessary. func GetPage(url string) ([]byte, error) { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client := &http.Client{Transport: tr} req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } req.Header.Add("Accept-Encoding", "gzip, deflate") resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() var body []byte if resp.Header.Get("Content-Encoding") == "gzip" { var gz *gzip.Reader gz, err = gzip.NewReader(resp.Body) if err != nil { return nil, err } defer gz.Close() body, err = ioutil.ReadAll(gz) } else { body, err = ioutil.ReadAll(resp.Body) } if err != nil { return nil, err } return body, nil }
func NewDscin(waiter *sync.WaitGroup, filename string) *Dscin { d := new(Dscin) d.Operation.Waiter = waiter d.Operation.Waiter.Add(1) file, err := os.Open(filename) if err != nil { return nil } d.closer = func() { file.Close() } var reader io.ReadCloser = file var uncompressor *gzip.Reader if strings.HasSuffix(filename, ".gz") { uncompressor, err = gzip.NewReader(file) d.closer = func() { uncompressor.Close(); file.Close() } reader = uncompressor } uncompressed_name := strings.TrimRight(filename, ".gz") switch { case strings.HasSuffix(uncompressed_name, ".gob"): d.marshaler = new(formats.GobMarshaler) case strings.HasSuffix(uncompressed_name, ".xml"): d.marshaler = new(formats.XmlMarshaler) } if d.marshaler != nil { d.marshaler.ValidateFile(reader) } return d }
func handleAssets(w http.ResponseWriter, r *http.Request) { assets := auto.Assets() path := r.URL.Path[1:] if path == "" { path = "index.html" } bs, ok := assets[path] if !ok { w.WriteHeader(http.StatusNotFound) return } mtype := mimeTypeForFile(path) if len(mtype) != 0 { w.Header().Set("Content-Type", mtype) } if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { w.Header().Set("Content-Encoding", "gzip") } else { // ungzip if browser not send gzip accepted header var gr *gzip.Reader gr, _ = gzip.NewReader(bytes.NewReader(bs)) bs, _ = ioutil.ReadAll(gr) gr.Close() } w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs))) w.Write(bs) }
// readBytes reads an encrypted/compressed steam from an io.Reader // and returns a decoded byte slice func readBytes(in io.Reader, key string) ([]byte, error) { var gzReader *gzip.Reader // compressed reader var iv [aes.BlockSize]byte // initialization vector var cb cipher.Block // cipher block interface var outBytes *bytes.Buffer // output buffer var err error // general error handler // init cipher block if cb, err = aes.NewCipher(hashKey(key)); err != nil { return nil, err } // init encrypted reader encReader := &cipher.StreamReader{ S: cipher.NewOFB(cb, iv[:]), R: in, } // wrap encrypted reader if gzReader, err = gzip.NewReader(encReader); err != nil { return nil, err } // close when done defer gzReader.Close() // init output outBytes = new(bytes.Buffer) // read data into output buffer decompressing and decrypting along the way _, err = io.Copy(outBytes, gzReader) // return bytes and last error state return outBytes.Bytes(), err }
// Decode returns the decoded data for the bundle entry. Returns a // slice of bytes with the decoded, decompressed (if required), ready // to use entry data, and an error indication which is not-nil if the // data cannot be decoded. If argument "flag" is NODC, and the entry // data are compressed (Entry.Gzip == true), Decode will not // decompress the data it returns (it will only decode them). In most // cases it is preferable to use the Reader interface instead of // calling Decode. func (e *Entry) Decode(flag int) ([]byte, error) { var rs *strings.Reader var r64 io.Reader var rz *gzip.Reader var buf *bytes.Buffer var err error rs = strings.NewReader(e.Data) r64 = base64.NewDecoder(base64.StdEncoding, rs) if e.Gzip && (flag&NODC == 0) { rz, err = gzip.NewReader(r64) if err != nil { return nil, err } defer rz.Close() } else { rz = nil } buf = new(bytes.Buffer) if rz != nil { _, err = io.Copy(buf, rz) } else { _, err = io.Copy(buf, r64) } if err != nil { return nil, err } return buf.Bytes(), nil }
// Reads transfer-encoding: chunked payloads from the connection reader. func (c *Connection) readChunkedData() error { var err error var line []byte var size uint64 var start time.Time start = time.Now() writer := &nonEmptyWriter{os.Stdout} var buffer *bytes.Buffer var decompressor *gzip.Reader var zipReader *bufio.Reader var data []byte if c.conf.GZip == true { buffer = bytes.NewBufferString("") } for err == nil { line, _, err = c.reader.ReadLine() if err != nil { return err } size, err = decodeHexString(line) if err != nil { str := fmt.Sprintf("Expected hex, got %v", string(line)) return errors.New(str) } if c.conf.GZip == false { _, err = io.CopyN(writer, c.reader, int64(size)) } else { _, err = io.CopyN(buffer, c.reader, int64(size)) if err != nil { return err } if decompressor == nil { decompressor, err = gzip.NewReader(buffer) defer decompressor.Close() if err != nil { return err } zipReader = bufio.NewReader(decompressor) } data = make([]byte, 512, 512) _, err = zipReader.Read(data) if err != nil { return err } strBuffer := bytes.NewBuffer(data) io.CopyN(writer, strBuffer, int64(len(data))) } if c.conf.TTL > 0 { if time.Now().Sub(start).Nanoseconds() > c.conf.TTL { return nil } } } return err }
func (s embeddedStatic) ServeHTTP(w http.ResponseWriter, r *http.Request) { file := r.URL.Path if file[0] == '/' { file = file[1:] } if len(file) == 0 { file = "index.html" } if s.assetDir != "" { p := filepath.Join(s.assetDir, filepath.FromSlash(file)) _, err := os.Stat(p) if err == nil { http.ServeFile(w, r, p) return } } s.mut.RLock() theme := s.theme modified := s.lastModified s.mut.RUnlock() bs, ok := s.assets[theme+"/"+file] if !ok { bs, ok = s.assets[config.DefaultTheme+"/"+file] if !ok { http.NotFound(w, r) return } } if modifiedSince, err := time.Parse(r.Header.Get("If-Modified-Since"), http.TimeFormat); err == nil && modified.Before(modifiedSince) { w.WriteHeader(http.StatusNotModified) return } mtype := s.mimeTypeForFile(file) if len(mtype) != 0 { w.Header().Set("Content-Type", mtype) } if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { w.Header().Set("Content-Encoding", "gzip") } else { // ungzip if browser not send gzip accepted header var gr *gzip.Reader gr, _ = gzip.NewReader(bytes.NewReader(bs)) bs, _ = ioutil.ReadAll(gr) gr.Close() } w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs))) w.Header().Set("Last-Modified", modified.Format(http.TimeFormat)) w.Header().Set("Cache-Control", "public") w.Write(bs) }
// extract 'control' file from 'reader'. the contents of a 'control' file // is a set of key-value pairs as described in // https://www.debian.org/doc/debian-policy/ch-controlfields.html func extractControlFromIpk(reader io.Reader) (string, error) { var ( arReader *ar.Reader tarReader *tar.Reader gzReader *gzip.Reader ) arReader = ar.NewReader(reader) for { header, err := arReader.Next() if err != nil && err != io.EOF { return "", fmt.Errorf("extracting contents: %v", err) } else if header == nil { break } // NOTE: strangeley the name of the files end with a "/" ... content error? if header.Name == "control.tar.gz/" || header.Name == "control.tar.gz" { gzReader, err = gzip.NewReader(arReader) if err != nil { return "", fmt.Errorf("analyzing control.tar.gz: %v", err) } break } } if gzReader == nil { return "", fmt.Errorf("missing control.tar.gz entry") } defer gzReader.Close() buffer := bytes.NewBuffer(nil) tarReader = tar.NewReader(gzReader) for { header, err := tarReader.Next() if err != nil && err != io.EOF { return "", fmt.Errorf("extracting control.tar.gz: %v", err) } else if header == nil { break } if header.Name != "./control" { continue } io.Copy(buffer, tarReader) break } if buffer.Len() == 0 { return "", fmt.Errorf("missing or empty 'control' file inside 'control.tar.gz'") } return buffer.String(), nil }
func (s embeddedStatic) ServeHTTP(w http.ResponseWriter, r *http.Request) { file := r.URL.Path if file[0] == '/' { file = file[1:] } if len(file) == 0 { file = "index.html" } if s.assetDir != "" { p := filepath.Join(s.assetDir, filepath.FromSlash(file)) _, err := os.Stat(p) if err == nil { http.ServeFile(w, r, p) return } } bs, ok := s.assets[file] if !ok { http.NotFound(w, r) return } if r.Header.Get("If-Modified-Since") == auto.AssetsBuildDate { w.WriteHeader(http.StatusNotModified) return } mtype := s.mimeTypeForFile(file) if len(mtype) != 0 { w.Header().Set("Content-Type", mtype) } if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { w.Header().Set("Content-Encoding", "gzip") } else { // ungzip if browser not send gzip accepted header var gr *gzip.Reader gr, _ = gzip.NewReader(bytes.NewReader(bs)) bs, _ = ioutil.ReadAll(gr) gr.Close() } w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs))) w.Header().Set("Last-Modified", auto.AssetsBuildDate) w.Header().Set("Cache-Control", "public") w.Write(bs) }
func readTheThings() { file_r_handle, _ := os.OpenFile("testing/testfile.tar.gz", os.O_RDWR, os.ModePerm) defer file_r_handle.Close() var zip_r_handle *gzip.Reader var err error if zip_r_handle, err = gzip.NewReader(file_r_handle); err != nil { return } defer zip_r_handle.Close() tar_r_handle := tar.NewReader(zip_r_handle) tar_r_handle.Next() /* for header, err := tar_r_handle.Next(); err == nil; tar_r_handle.Next() { fmt.Printf("File handle: %s\n", header.Name) } */ }
func openInvoiceFile(filename string) (io.ReadCloser, func(), error) { file, err := os.Open(filename) if err != nil { return nil, nil, err } closer := func() { file.Close() } var reader io.ReadCloser = file var decompressor *gzip.Reader if strings.HasSuffix(filename, ".gz") { if decompressor, err = gzip.NewReader(file); err != nil { return file, closer, err } closer = func() { decompressor.Close(); file.Close() } reader = decompressor } return reader, closer, nil }
func NewDscin(filename string) *Dscin { d := new(Dscin) file, err := os.Open(filename) if err != nil { return nil } d.closer = func() { fmt.Printf("closing file %s\n", filename) file.Close() } var reader io.ReadCloser = file var decompressor *gzip.Reader if strings.HasSuffix(filename, ".gz") { decompressor, _ = gzip.NewReader(file) d.closer = func() { decompressor.Close(); file.Close() } reader = decompressor } d.marshaler = formats.GobMarshaler{} d.marshaler.ValidateFile(reader) return d }
func main() { var file *os.File var err error var reader *gzip.Reader if file, err = os.Open("output/sample.tar.gz"); err != nil { log.Fatalln(err) } defer file.Close() if reader, err = gzip.NewReader(file); err != nil { log.Fatalln(err) } defer reader.Close() tr := tar.NewReader(reader) var header *tar.Header for { header, err = tr.Next() if err == io.EOF { // ファイルの最後 break } if err != nil { log.Fatalln(err) } buf := new(bytes.Buffer) if _, err = io.Copy(buf, tr); err != nil { log.Fatalln(err) } if err = ioutil.WriteFile("output/"+header.Name, buf.Bytes(), 0755); err != nil { log.Fatal(err) } } }
// deal with .tar.gz func openTarGz(za *ZipAssets) (err error) { var ( f *os.File tr *tar.Reader gr *gzip.Reader ) if f, err = os.Open(za.path); err != nil { return } defer f.Close() if gr, err = gzip.NewReader(f); err != nil { return } defer gr.Close() tr = tar.NewReader(gr) err = openTar(za, tr) return }
// RingOrBuilder attempts to determine whether a file is a Ring or Builder file // and then loads it accordingly. func RingOrBuilder(fileName string) (Ring, *Builder, error) { var f *os.File var r Ring var b *Builder var err error if f, err = os.Open(fileName); err != nil { return r, b, err } var gf *gzip.Reader if gf, err = gzip.NewReader(f); err != nil { return r, b, err } header := make([]byte, 16) if _, err = io.ReadFull(gf, header); err != nil { return r, b, err } if string(header[:5]) == "RINGv" { if string(header[:16]) != RINGVERSION { return r, b, fmt.Errorf("Ring Version missmatch, expected %s found %s", RINGVERSION, header[:16]) } gf.Close() if _, err = f.Seek(0, 0); err != nil { return r, b, err } r, err = LoadRing(f) } else if string(header[:12]) == "RINGBUILDERv" { if string(header[:16]) != BUILDERVERSION { return r, b, fmt.Errorf("Builder Version missmatch, expected %s found %s", BUILDERVERSION, header[:16]) } gf.Close() if _, err = f.Seek(0, 0); err != nil { return r, b, err } b, err = LoadBuilder(f) } return r, b, err }
func UnpackTar(filename string, path string) (err error) { var file *os.File if file, err = os.Open(filename); err != nil { return err } defer file.Close() var fileReader io.Reader = file var decompressor *gzip.Reader if strings.HasSuffix(filename, ".gz") { if decompressor, err = gzip.NewReader(file); err != nil { return err } defer decompressor.Close() } else if strings.HasSuffix(filename, ".bz2") { fileReader = bzip2.NewReader(file) } var reader *tar.Reader if decompressor != nil { reader = tar.NewReader(decompressor) } else { reader = tar.NewReader(fileReader) } return unpackTarFiles(reader, path) }
// Closes a gzip reader and returns it to the pool: func ReturnGZipReader(gz *gzip.Reader) { gz.Close() gzipReaderCache.Put(gz) }
func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) { file := r.URL.Path if file[0] == '/' { file = file[1:] } if len(file) == 0 { file = "index.html" } s.mut.RLock() theme := s.theme s.mut.RUnlock() // Check for an override for the current theme. if s.assetDir != "" { p := filepath.Join(s.assetDir, theme, filepath.FromSlash(file)) if _, err := os.Stat(p); err == nil { http.ServeFile(w, r, p) return } } // Check for a compiled in asset for the current theme. bs, ok := s.assets[theme+"/"+file] if !ok { // Check for an overridden default asset. if s.assetDir != "" { p := filepath.Join(s.assetDir, config.DefaultTheme, filepath.FromSlash(file)) if _, err := os.Stat(p); err == nil { http.ServeFile(w, r, p) return } } // Check for a compiled in default asset. bs, ok = s.assets[config.DefaultTheme+"/"+file] if !ok { http.NotFound(w, r) return } } mtype := s.mimeTypeForFile(file) if len(mtype) != 0 { w.Header().Set("Content-Type", mtype) } if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { w.Header().Set("Content-Encoding", "gzip") } else { // ungzip if browser not send gzip accepted header var gr *gzip.Reader gr, _ = gzip.NewReader(bytes.NewReader(bs)) bs, _ = ioutil.ReadAll(gr) gr.Close() } w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs))) w.Write(bs) }
func xsavez2(q *Context) { var fpz, fpgz *os.File var z *zip.Writer var gz *gzip.Reader var dact *dbxml.Db var docs *dbxml.Docs var dirname, fulldirname string var okall bool defer func() { if z != nil { z.Close() } if fpz != nil { fpz.Close() } if gz != nil { gz.Close() } if fpgz != nil { fpgz.Close() } if docs != nil { docs.Close() } if dact != nil { dact.Close() } if !okall { os.RemoveAll(fulldirname) q.db.Exec(fmt.Sprintf("DELETE FROM `%s_info` WHERE `id` = %q", Cfg.Prefix, dirname)) } }() protected := 0 if !q.auth { http.Error(q.w, "Je bent niet ingelogd", http.StatusUnauthorized) return } methode := firstf(q.form, "mt") if methode != "dx" { methode = "std" } corpora := make([]string, 0, len(q.form.Value["db"])) for _, c := range q.form.Value["db"] { if s := strings.TrimSpace(c); s != "" { corpora = append(corpora, s) } } for _, corpus := range corpora { if !q.prefixes[corpus] { http.Error(q.w, "Geen toegang tot corpus", http.StatusUnauthorized) return } if q.protected[corpus] || !q.myprefixes[corpus] { protected = 1 } } if len(corpora) == 0 { writeHtml(q, "Fout", "Geen corpora gekozen") return } xpath := firstf(q.form, "xpath") if xpath == "" { writeHtml(q, "Fout", "Zoekterm ontbreekt") return } title := maxtitlelen(firstf(q.form, "title")) if title == "" { writeHtml(q, "Fout", "Titel ontbreekt") return } maxdup, _ := strconv.Atoi(firstf(q.form, "maxdup")) if maxdup < 1 || maxdup > Cfg.Maxdup { maxdup = Cfg.Maxdup } dirname, fulldirname, ok := beginNewCorpus(q, q.db, title, hErr) if !ok { return } fpz, err := os.Create(fulldirname + "/data") if hErr(q, err) { fpz = nil return } z = zip.NewWriter(fpz) linecount := 0 for _, prefix := range corpora { if linecount == maxdup && maxdup > 0 { break } global, ok := isGlobal(q, prefix) if !ok { return } pathlen, ok := getPathLen(q, prefix, global, true) if !ok { return } dactfiles := make([]string, 0) if !global { dactfiles = append(dactfiles, fmt.Sprintf("%s/data/%s/data.dact", paqudir, prefix)) } else { rows, err := q.db.Query(fmt.Sprintf("SELECT `arch` FROM `%s_c_%s_arch` ORDER BY `id`", Cfg.Prefix, prefix)) if hErr(q, err) { return } for rows.Next() { var s string if hErr(q, rows.Scan(&s)) { rows.Close() return } if strings.HasSuffix(s, ".dact") { dactfiles = append(dactfiles, s) } } if hErr(q, rows.Err()) { return } } fullquery := xpath if strings.Contains(xpath, "%") { rules := getMacrosRules(q) fullquery = macroKY.ReplaceAllStringFunc(xpath, func(s string) string { return rules[s[1:len(s)-1]] }) } queryparts := strings.Split(fullquery, "+|+") for _, dactfile := range dactfiles { if linecount == maxdup && maxdup > 0 { break } if Cfg.Dactx && methode == "dx" { dactfile += "x" } var data []byte dact, err = dbxml.Open(dactfile) if hErr(q, err) { dact = nil return } qu, err := dact.Prepare(queryparts[0]) if hErr(q, err) { return } docs, err = qu.Run() if hErr(q, err) { docs = nil return } seen := make(map[string]bool) NEXTDOC: for docs.Next() { if linecount == maxdup && maxdup > 0 { break } filename := docs.Name() if seen[filename] { continue } seen[filename] = true found := false if len(queryparts) == 1 { found = true data = []byte(docs.Content()) } else { doctxt := fmt.Sprintf("[dbxml:metadata('dbxml:name')=%q]", filename) for i := 1; i < len(queryparts)-1; i++ { docs2, err := dact.Query(doctxt + queryparts[i]) if hErr(q, err) { return } if !docs2.Next() { continue NEXTDOC } docs2.Close() } docs2, err := dact.Query(doctxt + queryparts[len(queryparts)-1]) if hErr(q, err) { return } found = false if docs2.Next() { found = true data = []byte(docs2.Content()) docs2.Close() } } if !found { continue } newfile := filename if global { newfile = dactfile[pathlen:len(dactfile)-5] + "::" + filename } if len(corpora) > 1 { newfile = prefix + "/" + newfile data = xmlSetSource(data, prefix) } f, err := z.Create(newfile) if hErr(q, err) { return } if methode == "dx" { data, err = unexpandDact(data) if hErr(q, err) { return } } _, err = f.Write(data) if hErr(q, err) { return } linecount++ } // for docs.Next() err = docs.Error() docs = nil if hErr(q, err) { return } dact.Close() dact = nil } // for range dactfiles } // for range corpora err = z.Close() z = nil if hErr(q, err) { return } fpz.Close() fpz = nil s := "xmlzip-d" if protected != 0 { s = "xmlzip-p" } newCorpus(q, q.db, dirname, title, s, protected, hErr, true) okall = true }
func TestGzippedReplication(t *testing.T) { if testing.Short() { t.Skip() } cluster := createCluster() member1 := createMember(6106) repServer1, _ := New(&Config{ Registrator: cluster.Registrator(member1), Membership: cluster.Membership()}) defer repServer1.Stop() <-repServer1.Sync(1) member2 := createMember(6206) repServer2, _ := New(&Config{ Registrator: cluster.Registrator(member2), Membership: cluster.Membership()}) defer repServer2.Stop() <-repServer2.Sync(1) var resp *http.Response replicator2, _ := repServer2.GetReplicator(auth.NamespaceFrom("ns1")) client := http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 1}} var requestData bytes.Buffer for i := 0; i < 100; i++ { requestData.Write([]byte("9999999999999999999999999999999999")) } // Iterate over both sync and replication HTTP endpoints for _, serviceType := range []string{syncContext, repContext} { // Iterate over both content-encoding options- gzip and no zip (identity) for _, encoding := range []string{"gzip", "identity"} { var dec decoder var unZipper *gzip.Reader var respBuff bytes.Buffer // prepare event that triggers a replication or a sync if serviceType == repContext { time.AfterFunc(time.Duration(2)*time.Second, func() { replicator2.Broadcast(requestData.Bytes()) }) } else { // grab the channel that was created for us // insert an entity inside it and close the channel to signal sync finish outMsg := &outMessage{Data: requestData.Bytes()} buff, _ := json.Marshal(outMsg) time.AfterFunc(time.Duration(2)*time.Second, func() { ch := <-repServer2.SyncRequest() ch <- buff close(ch) }) } url := fmt.Sprintf("http://%s:%d/%s/%s", network.GetPrivateIP(), 6206, version, serviceType) req, _ := http.NewRequest("GET", url, nil) req.Header.Set("Accept-Encoding", encoding) req.Header.Set(headerMemberID, "fake member id") resp, _ = client.Do(req) // wait 4 sec to read enough data, since ioutil.ReadAll blocks until EOF // and we use an indefinite connection in case of replication service endpoint if serviceType == repContext { time.AfterFunc(time.Duration(4)*time.Second, func() { resp.Body.Close() }) } responseData, _ := ioutil.ReadAll(resp.Body) respBuff.Write(responseData) // if we've sent a gzipped entity, decode it before comparing // but also ensure that the returned payload was compressed // by comparing the size of the payload to the size sent if encoding == "gzip" { assert.True(t, len(requestData.Bytes()) > len(responseData), "Seems like the gzip hasn't shrunk the payload size enough or at all") unZipper, _ = gzip.NewReader(&respBuff) dec.Reader = bufio.NewReader(unZipper) } else { dec.Reader = bufio.NewReader(&respBuff) } event, _ := dec.Decode() if encoding == "gzip" { unZipper.Close() } var entityFromServer outMessage json.Unmarshal([]byte(event.Data()), &entityFromServer) assert.Equal(t, string(requestData.Bytes()), string(entityFromServer.Data), "Didn't receive the same content we sent in the case where encoding is %s", encoding) } } }
func savez2(q *Context) { var fpz, fpgz *os.File var z *zip.Writer var gz *gzip.Reader var dact interface{} var err error var dirname, fulldirname string var okall bool defer func() { if z != nil { z.Close() } if fpz != nil { fpz.Close() } if gz != nil { gz.Close() } if fpgz != nil { fpgz.Close() } saveCloseDact(dact) if !okall { os.RemoveAll(fulldirname) q.db.Exec(fmt.Sprintf("DELETE FROM `%s_info` WHERE `id` = %q", Cfg.Prefix, dirname)) } }() protected := 0 if !q.auth { http.Error(q.w, "Je bent niet ingelogd", http.StatusUnauthorized) return } corpora := make([]string, 0, len(q.form.Value["db"])) for _, c := range q.form.Value["db"] { if s := strings.TrimSpace(c); s != "" { corpora = append(corpora, s) } } for _, corpus := range corpora { if !q.prefixes[corpus] { http.Error(q.w, "Geen toegang tot corpus", http.StatusUnauthorized) return } if q.protected[corpus] || !q.myprefixes[corpus] { protected = 1 } } if len(corpora) == 0 { writeHtml(q, "Fout", "Geen corpora gekozen") return } word := firstf(q.form, "word") rel := firstf(q.form, "rel") hword := firstf(q.form, "hword") postag := firstf(q.form, "postag") hpostag := firstf(q.form, "hpostag") meta := firstf(q.form, "meta") if word == "" && hword == "" && rel == "" && postag == "" && hpostag == "" && meta == "" { writeHtml(q, "Fout", "Zoektermen ontbreken") return } title := maxtitlelen(firstf(q.form, "title")) if title == "" { writeHtml(q, "Fout", "Titel ontbreekt") return } maxdup, _ := strconv.Atoi(firstf(q.form, "maxdup")) if maxdup < 1 || maxdup > Cfg.Maxdup { maxdup = Cfg.Maxdup } dirname, fulldirname, ok := beginNewCorpus(q, q.db, title, hErr) if !ok { return } fpz, err = os.Create(fulldirname + "/data") if hErr(q, err) { fpz = nil return } z = zip.NewWriter(fpz) linecount := 0 chClose := make(<-chan bool) for _, prefix := range corpora { if linecount == maxdup && maxdup > 0 { break } global, ok := isGlobal(q, prefix) if !ok { return } pathlen, ok := getPathLen(q, prefix, global, false) if !ok { return } query, joins, usererr, syserr := makeQueryF(q, prefix, "c", chClose) if hErr(q, syserr) { return } if uhErr(q, usererr) { return } query = fmt.Sprintf( "SELECT DISTINCT `f`.`file`, `c`.`arch` FROM `%s_c_%s_deprel` `c` "+ "JOIN `%s_c_%s_file` `f` ON (`f`.`id`=`c`.`file`) %s WHERE %s", Cfg.Prefix, prefix, Cfg.Prefix, prefix, joins, query) rows, err := q.db.Query(query) if hErr(q, err) { return } currentarch := -1 dact = nil var arch int var filename, dactname string for rows.Next() { if linecount == maxdup && maxdup > 0 { rows.Close() break } err = rows.Scan(&filename, &arch) if hErr(q, err) { rows.Close() return } var data []byte if arch < 0 { fpgz, err = os.Open(filename + ".gz") if err == nil { gz, err = gzip.NewReader(fpgz) if hErr(q, err) { gz = nil rows.Close() return } data, err = ioutil.ReadAll(gz) if hErr(q, err) { rows.Close() return } gz.Close() gz = nil fpgz.Close() fpgz = nil } else { fpgz, err = os.Open(filename) if hErr(q, err) { fpgz = nil rows.Close() return } data, err = ioutil.ReadAll(fpgz) if hErr(q, err) { rows.Close() return } fpgz.Close() fpgz = nil } } else { if arch != currentarch { currentarch = arch saveCloseDact(dact) dact, dactname = saveOpenDact(q, prefix, arch) } data = saveGetDact(q, dact, filename) } var newfile string if arch < 0 { newfile = filename[pathlen:] if !global { if strings.Contains(q.params[prefix], "-lbl") || strings.HasPrefix(q.params[prefix], "folia") || strings.HasPrefix(q.params[prefix], "tei") { newfile = decode_filename(newfile[10:]) } else if strings.HasPrefix(q.params[prefix], "xmlzip") || q.params[prefix] == "dact" { newfile = decode_filename(newfile[5:]) } } } else { newfile = dactname[pathlen:] + "::" + filename } if len(corpora) > 1 { newfile = prefix + "/" + newfile data = xmlSetSource(data, prefix) } f, err := z.Create(newfile) if hErr(q, err) { rows.Close() return } _, err = f.Write(data) if hErr(q, err) { rows.Close() return } linecount++ } // for rows.Next() err = rows.Err() if hErr(q, err) { return } saveCloseDact(dact) dact = nil } err = z.Close() z = nil if hErr(q, err) { return } fpz.Close() fpz = nil s := "xmlzip-d" if protected != 0 { s = "xmlzip-p" } newCorpus(q, q.db, dirname, title, s, protected, hErr, true) okall = true }
func (hec *HTTPEventCollectorMock) ServeHTTP(writer http.ResponseWriter, request *http.Request) { var err error hec.numOfRequests++ if hec.simulateServerError { if request.Body != nil { defer request.Body.Close() } writer.WriteHeader(http.StatusInternalServerError) return } switch request.Method { case http.MethodOptions: // Verify that options method is getting called only once if hec.connectionVerified { hec.test.Errorf("Connection should not be verified more than once. Got second request with %s method.", request.Method) } hec.connectionVerified = true writer.WriteHeader(http.StatusOK) case http.MethodPost: // Always verify that Driver is using correct path to HEC if request.URL.String() != "/services/collector/event/1.0" { hec.test.Errorf("Unexpected path %v", request.URL) } defer request.Body.Close() if authorization, ok := request.Header["Authorization"]; !ok || authorization[0] != ("Splunk "+hec.token) { hec.test.Error("Authorization header is invalid.") } gzipEnabled := false if contentEncoding, ok := request.Header["Content-Encoding"]; ok && contentEncoding[0] == "gzip" { gzipEnabled = true } if hec.gzipEnabled == nil { hec.gzipEnabled = &gzipEnabled } else if gzipEnabled != *hec.gzipEnabled { // Nothing wrong with that, but we just know that Splunk Logging Driver does not do that hec.test.Error("Driver should not change Content Encoding.") } var gzipReader *gzip.Reader var reader io.Reader if gzipEnabled { gzipReader, err = gzip.NewReader(request.Body) if err != nil { hec.test.Fatal(err) } reader = gzipReader } else { reader = request.Body } // Read body var body []byte body, err = ioutil.ReadAll(reader) if err != nil { hec.test.Fatal(err) } // Parse message messageStart := 0 for i := 0; i < len(body); i++ { if i == len(body)-1 || (body[i] == '}' && body[i+1] == '{') { var message splunkMessage err = json.Unmarshal(body[messageStart:i+1], &message) if err != nil { hec.test.Log(string(body[messageStart : i+1])) hec.test.Fatal(err) } hec.messages = append(hec.messages, &message) messageStart = i + 1 } } if gzipEnabled { gzipReader.Close() } writer.WriteHeader(http.StatusOK) default: hec.test.Errorf("Unexpected HTTP method %s", http.MethodOptions) writer.WriteHeader(http.StatusBadRequest) } }
func releaseGzipReader(zr *gzip.Reader) { zr.Close() gzipReaderPool.Put(zr) }
// PutReader closes and returns a gzip.Reader to the pool // so that it can be reused via GetReader. func (pool *GzipPool) PutReader(reader *gzip.Reader) { reader.Close() pool.readers.Put(reader) }
func TestCompressed(t *testing.T) { var entries []string var e *bundle.Entry var br *bundle.Reader var gr *gzip.Reader var data, rdata, fdata, ddata []byte var i int var err error entries, err = mkentries(data_dir) if err != nil { t.Fatalf("mkentries failed (%s): %s", data_dir, err) } for i = 0; i < len(entries); i++ { // Read data from file fdata, err = ioutil.ReadFile(data_dir + entries[i]) if err != nil { t.Fatalf("ReadFile error: %s", err) } // Get bundle entry e = _bundleIdx.Entry(entries[i]) if e == nil { t.Fatalf("Not found: %s", entries[i]) } // Get data from bundle using Decode data, err = e.Decode(bundle.NODC) if err != nil { t.Fatalf("bundle.Decode(): %s", err) } gr, err = gzip.NewReader(bytes.NewReader(data)) if err != nil { t.Fatalf("gzip.NewReader: %s", err) } ddata, err = ioutil.ReadAll(gr) if err != nil { t.Fatalf("ReadAll(gr): %s") } err = gr.Close() if err != nil { t.Fatalf("gr.Close(): %s") } if len(ddata) != e.Size { t.Fatalf("len(ddata) %d != e.Size %d", len(ddata), e.Size) } if len(ddata) != len(fdata) { t.Fatalf("Bad ddata sz for: %s", entries[i]) } if bytes.Compare(ddata, fdata) != 0 { t.Fatalf("Bad ddata for: %s", entries[i]) } // Get data from bundle using bundle.Reader br, err = e.Open(bundle.NODC) if err != nil { t.Fatalf("e.Open(): %s", err) } gr, err = gzip.NewReader(br) if err != nil { t.Fatalf("gzip.NewReader(br): %s") } ddata, err = ioutil.ReadAll(gr) if err != nil { t.Fatalf("ReadAll(gr): %s") } err = gr.Close() if err != nil { t.Fatalf("gr.Close(): %s") } if len(ddata) != e.Size { t.Fatalf("len(ddata) %d != e.Size %d", len(rdata), e.Size) } if len(ddata) != len(fdata) { t.Fatalf("Bad ddata sz for: %s", entries[i]) } if bytes.Compare(ddata, fdata) != 0 { t.Fatalf("Bad ddata for: %s", entries[i]) } t.Logf("Entry: %s, Size: %d, Gzip: %v", e.Name, e.Size, e.Gzip) } }
func ringOrBuilder(fileName string) (r ring.Ring, b *ring.Builder, err error) { var f *os.File if f, err = os.Open(fileName); err != nil { return } var gf *gzip.Reader if gf, err = gzip.NewReader(f); err != nil { return } header := make([]byte, 16) if _, err = io.ReadFull(gf, header); err != nil { return } if string(header[:5]) == "RINGv" { gf.Close() if _, err = f.Seek(0, 0); err != nil { return } r, err = ring.LoadRing(f) } else if string(header[:12]) == "RINGBUILDERv" { gf.Close() if _, err = f.Seek(0, 0); err != nil { return } b, err = ring.LoadBuilder(f) } return }