func HashesFromGZHeader(inpath string, md5crcBuffer []byte) (*Hashes, int64, error) {
romGZ, err := os.Open(inpath)
if err != nil {
return nil, 0, err
}
defer romGZ.Close()
gzr, err := gzip.NewReader(romGZ)
if err != nil {
return nil, 0, err
}
defer gzr.Close()
md5crcBuffer = gzr.Header.Extra
var hh *Hashes
var size int64
if len(md5crcBuffer) == md5.Size+crc32.Size+8 {
hh = new(Hashes)
hh.Md5 = make([]byte, md5.Size)
copy(hh.Md5, md5crcBuffer[:md5.Size])
hh.Crc = make([]byte, crc32.Size)
copy(hh.Crc, md5crcBuffer[md5.Size:md5.Size+crc32.Size])
size = util.BytesToInt64(md5crcBuffer[md5.Size+crc32.Size:])
}
return hh, size, nil
}
func TestWithCompressorGZIPConcrete(t *testing.T) {
testWithCompressorConcrete(t, httputil.CompressGZIP, func(r io.Reader) string {
zr, err := gzip.NewReader(r)
assert.NoError(t, err)
defer zr.Close()
var un bytes.Buffer
zr.WriteTo(&un)
return un.String()
})
}
func acquireGzipReader(r io.Reader) (*gzip.Reader, error) {
v := gzipReaderPool.Get()
if v == nil {
return gzip.NewReader(r)
}
zr := v.(*gzip.Reader)
if err := zr.Reset(r); err != nil {
return nil, err
}
return zr, nil
}
func NewReader(r io.ReadSeeker) (*Reader, error) {
or := new(Reader)
or.r = r
gz, err := gzip.NewReader(or.createUnderlyingReader())
if err != nil {
return nil, err
}
gz.Multistream(false)
or.gz = gz
return or, nil
}
// Return a buffered reader from an io.Reader
// If f == "-", then it will attempt to read from os.Stdin.
// If the file is gzipped, it will be read as such.
func Buf(r io.Reader) *Reader {
b := bufio.NewReaderSize(r, getSize())
var rdr *gzip.Reader
if is, err := IsGzip(b); err != nil && err != io.EOF {
log.Fatal(err)
} else if is {
rdr, err = gzip.NewReader(b)
if err != nil {
log.Fatal(err)
}
b = bufio.NewReaderSize(rdr, getSize())
}
return &Reader{b, r, rdr}
}
func HashesForGZFile(inpath string) (*Hashes, error) {
file, err := os.Open(inpath)
if err != nil {
return nil, err
}
defer file.Close()
gzipReader, err := gzip.NewReader(file)
if err != nil {
return nil, err
}
defer gzipReader.Close()
return hashesForReader(gzipReader)
}
func (depot *Depot) SHA1InDepot(sha1Hex string) (bool, *Hashes, string, int64, error) {
for _, root := range depot.roots {
rompath := pathFromSha1HexEncoding(root, sha1Hex, gzipSuffix)
exists, err := PathExists(rompath)
if err != nil {
return false, nil, "", 0, err
}
var size int64
if exists {
hh := new(Hashes)
sha1Bytes, err := hex.DecodeString(sha1Hex)
if err != nil {
return false, nil, "", 0, err
}
hh.Sha1 = sha1Bytes
romGZ, err := os.Open(rompath)
if err != nil {
return false, nil, "", 0, err
}
defer romGZ.Close()
gzr, err := gzip.NewReader(romGZ)
if err != nil {
return false, nil, "", 0, err
}
defer gzr.Close()
md5crcBuffer := gzr.Header.Extra
if len(md5crcBuffer) == md5.Size+crc32.Size+8 {
hh.Md5 = make([]byte, md5.Size)
copy(hh.Md5, md5crcBuffer[:md5.Size])
hh.Crc = make([]byte, crc32.Size)
copy(hh.Crc, md5crcBuffer[md5.Size:md5.Size+crc32.Size])
size = util.BytesToInt64(md5crcBuffer[md5.Size+crc32.Size:])
} else {
glog.Warningf("rom %s has missing gzip md5 or crc header", rompath)
}
return true, hh, rompath, size, nil
}
}
return false, nil, "", 0, nil
}
func openGzipReadCloser(inpath string) (io.ReadCloser, error) {
f, err := os.Open(inpath)
if err != nil {
return nil, err
}
_, err = f.Stat()
if err != nil {
f.Close()
return nil, err
}
zr, err := gzip.NewReader(f)
if err != nil {
f.Close()
return nil, err
}
return &gzipReadCloser{
file: f,
zr: zr,
}, nil
}
func TestGzip(t *testing.T) {
req, _ := http.NewRequest("GET", "/", nil)
req.Header.Add("Accept-Encoding", "gzip")
w := httptest.NewRecorder()
r := newServer()
r.ServeHTTP(w, req)
assert.Equal(t, w.Code, 200)
assert.Equal(t, w.Header().Get("Content-Encoding"), "gzip")
assert.Equal(t, w.Header().Get("Vary"), "Accept-Encoding")
assert.Equal(t, w.Header().Get("Content-Length"), "")
assert.NotEqual(t, w.Body.Len(), 19)
gr, err := gzip.NewReader(w.Body)
assert.NoError(t, err)
defer gr.Close()
body, _ := ioutil.ReadAll(gr)
assert.Equal(t, string(body), testResponse)
}
// Convert a whole gzip file into a multi-gzip file. mode can be used to
// select between using a normal writer, or the rsync-friendly writer.
func Convert(w io.Writer, r io.Reader, mode ConvertMode) error {
// We want to match the same algorithm originally used, to preserve
// the rsyncable effect. The gzip library doesn't expose this data in the
// headers, so we parse it. We don't do additional checks here, as if the
// header is broken, gzip.NewReader will error out just afterwards.
var gzhead [10]byte
if _, err := io.ReadFull(r, gzhead[:]); err != nil {
return err
}
comprlevel := gzip.DefaultCompression
if gzhead[8] == 0x2 {
comprlevel = gzip.BestCompression
} else if gzhead[8] == 0x4 {
comprlevel = gzip.BestSpeed
}
var oz io.WriteCloser
switch mode {
case ConvertNormal:
oz, _ = NewWriterLevel(w, comprlevel, DefaultBlockSize)
case ConvertRsyncable:
oz, _ = NewWriterLevelRsyncable(w, comprlevel)
default:
return errInvalidConvertMode
}
defer oz.Close()
fz, err := gzip.NewReader(io.MultiReader(bytes.NewReader(gzhead[:]), r))
if err != nil {
return nil
}
defer fz.Close()
if _, err = io.Copy(oz, fz); err != nil {
return err
}
return nil
}
func (or *Reader) Seek(o Offset) error {
cur := or.Offset()
if cur.Block == o.Block && cur.Off < o.Off {
_, err := io.CopyN(ioutil.Discard, or, o.Off-cur.Off)
if err != nil {
return err
}
return nil
}
or.r.Seek(o.Block, 0)
or.cnt = o.Block
if or.gz == nil {
gz, err := gzip.NewReader(or.createUnderlyingReader())
if err != nil {
return err
}
or.gz = gz
} else {
or.gz.Close()
if or.gz.Reset(or.createUnderlyingReader()) == io.EOF {
or.gz = nil
return errWrongOffset
}
}
or.gz.Multistream(false)
or.block = o.Block
or.noff = 0
_, err := io.CopyN(ioutil.Discard, or, o.Off)
if err != nil {
return err
}
return nil
}
func Test_ServeHTTP_Compressed(t *testing.T) {
gzipHandler := Gzip(DefaultCompression)
w := httptest.NewRecorder()
req, err := http.NewRequest("GET", "http://localhost/foobar", nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set(headerAcceptEncoding, encodingGzip)
gzipHandler.ServeHTTP(w, req, testHTTPContent)
gr, err := gzip.NewReader(w.Body)
if err != nil {
t.Fatal(err)
}
defer gr.Close()
body, _ := ioutil.ReadAll(gr)
if string(body) != gzipTestString {
t.Fail()
}
}
func (depot *Depot) buildGame(game *types.Game, gamePath string,
unzipGame bool, deduper dedup.Deduper) (*types.Game, bool, error) {
var gameTorrent *torrentzip.Writer
var err error
glog.V(4).Infof("building game %s with path %s", game.Name, gamePath)
if unzipGame {
err := os.Mkdir(gamePath, 0777)
if err != nil {
glog.Errorf("error mkdir %s: %v", gamePath, err)
return nil, false, err
}
} else {
gameDir := filepath.Dir(game.Name)
if gameDir != "." {
// name has dirs in it
err := os.MkdirAll(filepath.Dir(gamePath), 0777)
if err != nil {
glog.Errorf("error mkdir %s: %v", filepath.Dir(gamePath), err)
return nil, false, err
}
}
gameFile, err := os.Create(gamePath + zipSuffix)
if err != nil {
glog.Errorf("error creating zip file %s: %v", gamePath+zipSuffix, err)
return nil, false, err
}
defer gameFile.Close()
gameTorrent, err = torrentzip.NewWriterWithTemp(gameFile, config.GlobalConfig.General.TmpDir)
if err != nil {
glog.Errorf("error writing to torrentzip file %s: %v", gamePath+zipSuffix, err)
return nil, false, err
}
defer gameTorrent.Close()
}
var fixGame *types.Game
foundRom := false
for _, rom := range game.Roms {
err = depot.romDB.CompleteRom(rom)
if err != nil {
glog.Errorf("error completing rom %s: %v", rom.Name, err)
return nil, false, err
}
if rom.Sha1 == nil && rom.Size > 0 {
if fixGame == nil {
fixGame = new(types.Game)
fixGame.Name = game.Name
fixGame.Description = game.Description
}
fixGame.Roms = append(fixGame.Roms, rom)
continue
}
romGZ, err := depot.OpenRomGZ(rom)
if err != nil {
glog.Errorf("error opening rom %s from depot: %v", rom.Name, err)
return nil, false, err
}
if romGZ == nil {
if glog.V(2) {
glog.Warningf("game %s has missing rom %s (sha1 %s)", game.Name, rom.Name, hex.EncodeToString(rom.Sha1))
}
seenRom, err := deduper.Seen(rom)
if err != nil {
return nil, false, err
}
if !seenRom {
err = deduper.Declare(rom)
if err != nil {
glog.Errorf("error deduping rom %s: %v", rom.Name, err)
return nil, false, err
}
if fixGame == nil {
fixGame = new(types.Game)
fixGame.Name = game.Name
fixGame.Description = game.Description
}
fixGame.Roms = append(fixGame.Roms, rom)
}
continue
}
foundRom = true
src, err := gzip.NewReader(romGZ)
if err != nil {
glog.Errorf("error opening rom gz file %s: %v", rom.Name, err)
return nil, false, err
}
var dstWriter io.WriteCloser
if unzipGame {
romPath := filepath.Join(gamePath, rom.Name)
if strings.ContainsRune(rom.Name, filepath.Separator) {
err := os.MkdirAll(filepath.Dir(romPath), 0777)
if err != nil {
glog.Errorf("error mkdir %s: %v", filepath.Dir(romPath), err)
return nil, false, err
}
}
dst, err := os.Create(romPath)
if err != nil {
glog.Errorf("error creating destination rom file %s: %v", dst, err)
return nil, false, err
}
dstWriter = dst
} else {
dst, err := gameTorrent.Create(rom.Name)
if err != nil {
glog.Errorf("error creating torrentzip rom entry %s: %v", rom.Name, err)
return nil, false, err
}
dstWriter = nopWriterCloser{dst}
}
_, err = io.Copy(dstWriter, src)
if err != nil {
glog.Errorf("error copying rom %s: %v", rom.Name, err)
return nil, false, err
}
src.Close()
dstWriter.Close()
romGZ.Close()
}
return fixGame, foundRom, nil
}