Ejemplo n.º 1
0
func HashesFromGZHeader(inpath string, md5crcBuffer []byte) (*Hashes, int64, error) {
	romGZ, err := os.Open(inpath)
	if err != nil {
		return nil, 0, err
	}
	defer romGZ.Close()

	gzr, err := gzip.NewReader(romGZ)
	if err != nil {
		return nil, 0, err
	}
	defer gzr.Close()

	md5crcBuffer = gzr.Header.Extra

	var hh *Hashes
	var size int64

	if len(md5crcBuffer) == md5.Size+crc32.Size+8 {
		hh = new(Hashes)
		hh.Md5 = make([]byte, md5.Size)
		copy(hh.Md5, md5crcBuffer[:md5.Size])
		hh.Crc = make([]byte, crc32.Size)
		copy(hh.Crc, md5crcBuffer[md5.Size:md5.Size+crc32.Size])
		size = util.BytesToInt64(md5crcBuffer[md5.Size+crc32.Size:])
	}
	return hh, size, nil
}
Ejemplo n.º 2
0
func TestWithCompressorGZIPConcrete(t *testing.T) {
	testWithCompressorConcrete(t, httputil.CompressGZIP, func(r io.Reader) string {
		zr, err := gzip.NewReader(r)
		assert.NoError(t, err)
		defer zr.Close()
		var un bytes.Buffer
		zr.WriteTo(&un)
		return un.String()
	})
}
Ejemplo n.º 3
0
func acquireGzipReader(r io.Reader) (*gzip.Reader, error) {
	v := gzipReaderPool.Get()
	if v == nil {
		return gzip.NewReader(r)
	}
	zr := v.(*gzip.Reader)
	if err := zr.Reset(r); err != nil {
		return nil, err
	}
	return zr, nil
}
Ejemplo n.º 4
0
func NewReader(r io.ReadSeeker) (*Reader, error) {
	or := new(Reader)
	or.r = r
	gz, err := gzip.NewReader(or.createUnderlyingReader())
	if err != nil {
		return nil, err
	}
	gz.Multistream(false)
	or.gz = gz
	return or, nil
}
Ejemplo n.º 5
0
// Return a buffered reader from an io.Reader
// If f == "-", then it will attempt to read from os.Stdin.
// If the file is gzipped, it will be read as such.
func Buf(r io.Reader) *Reader {
	b := bufio.NewReaderSize(r, getSize())
	var rdr *gzip.Reader
	if is, err := IsGzip(b); err != nil && err != io.EOF {
		log.Fatal(err)
	} else if is {
		rdr, err = gzip.NewReader(b)
		if err != nil {
			log.Fatal(err)
		}
		b = bufio.NewReaderSize(rdr, getSize())
	}
	return &Reader{b, r, rdr}
}
Ejemplo n.º 6
0
func HashesForGZFile(inpath string) (*Hashes, error) {
	file, err := os.Open(inpath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	gzipReader, err := gzip.NewReader(file)
	if err != nil {
		return nil, err
	}
	defer gzipReader.Close()

	return hashesForReader(gzipReader)
}
Ejemplo n.º 7
0
func (depot *Depot) SHA1InDepot(sha1Hex string) (bool, *Hashes, string, int64, error) {
	for _, root := range depot.roots {
		rompath := pathFromSha1HexEncoding(root, sha1Hex, gzipSuffix)
		exists, err := PathExists(rompath)
		if err != nil {
			return false, nil, "", 0, err
		}

		var size int64

		if exists {
			hh := new(Hashes)
			sha1Bytes, err := hex.DecodeString(sha1Hex)
			if err != nil {
				return false, nil, "", 0, err
			}
			hh.Sha1 = sha1Bytes

			romGZ, err := os.Open(rompath)
			if err != nil {
				return false, nil, "", 0, err
			}
			defer romGZ.Close()

			gzr, err := gzip.NewReader(romGZ)
			if err != nil {
				return false, nil, "", 0, err
			}
			defer gzr.Close()

			md5crcBuffer := gzr.Header.Extra

			if len(md5crcBuffer) == md5.Size+crc32.Size+8 {
				hh.Md5 = make([]byte, md5.Size)
				copy(hh.Md5, md5crcBuffer[:md5.Size])
				hh.Crc = make([]byte, crc32.Size)
				copy(hh.Crc, md5crcBuffer[md5.Size:md5.Size+crc32.Size])
				size = util.BytesToInt64(md5crcBuffer[md5.Size+crc32.Size:])
			} else {
				glog.Warningf("rom %s has missing gzip md5 or crc header", rompath)
			}

			return true, hh, rompath, size, nil
		}
	}
	return false, nil, "", 0, nil
}
Ejemplo n.º 8
0
func openGzipReadCloser(inpath string) (io.ReadCloser, error) {
	f, err := os.Open(inpath)
	if err != nil {
		return nil, err
	}
	_, err = f.Stat()
	if err != nil {
		f.Close()
		return nil, err
	}
	zr, err := gzip.NewReader(f)
	if err != nil {
		f.Close()
		return nil, err
	}

	return &gzipReadCloser{
		file: f,
		zr:   zr,
	}, nil
}
Ejemplo n.º 9
0
func TestGzip(t *testing.T) {
	req, _ := http.NewRequest("GET", "/", nil)
	req.Header.Add("Accept-Encoding", "gzip")

	w := httptest.NewRecorder()
	r := newServer()
	r.ServeHTTP(w, req)

	assert.Equal(t, w.Code, 200)
	assert.Equal(t, w.Header().Get("Content-Encoding"), "gzip")
	assert.Equal(t, w.Header().Get("Vary"), "Accept-Encoding")
	assert.Equal(t, w.Header().Get("Content-Length"), "")
	assert.NotEqual(t, w.Body.Len(), 19)

	gr, err := gzip.NewReader(w.Body)
	assert.NoError(t, err)
	defer gr.Close()

	body, _ := ioutil.ReadAll(gr)
	assert.Equal(t, string(body), testResponse)
}
Ejemplo n.º 10
0
// Convert a whole gzip file into a multi-gzip file. mode can be used to
// select between using a normal writer, or the rsync-friendly writer.
func Convert(w io.Writer, r io.Reader, mode ConvertMode) error {

	// We want to match the same algorithm originally used, to preserve
	// the rsyncable effect. The gzip library doesn't expose this data in the
	// headers, so we parse it. We don't do additional checks here, as if the
	// header is broken, gzip.NewReader will error out just afterwards.
	var gzhead [10]byte
	if _, err := io.ReadFull(r, gzhead[:]); err != nil {
		return err
	}
	comprlevel := gzip.DefaultCompression
	if gzhead[8] == 0x2 {
		comprlevel = gzip.BestCompression
	} else if gzhead[8] == 0x4 {
		comprlevel = gzip.BestSpeed
	}

	var oz io.WriteCloser
	switch mode {
	case ConvertNormal:
		oz, _ = NewWriterLevel(w, comprlevel, DefaultBlockSize)
	case ConvertRsyncable:
		oz, _ = NewWriterLevelRsyncable(w, comprlevel)
	default:
		return errInvalidConvertMode
	}
	defer oz.Close()

	fz, err := gzip.NewReader(io.MultiReader(bytes.NewReader(gzhead[:]), r))
	if err != nil {
		return nil
	}
	defer fz.Close()

	if _, err = io.Copy(oz, fz); err != nil {
		return err
	}

	return nil
}
Ejemplo n.º 11
0
func (or *Reader) Seek(o Offset) error {
	cur := or.Offset()
	if cur.Block == o.Block && cur.Off < o.Off {
		_, err := io.CopyN(ioutil.Discard, or, o.Off-cur.Off)
		if err != nil {
			return err
		}
		return nil
	}

	or.r.Seek(o.Block, 0)
	or.cnt = o.Block

	if or.gz == nil {
		gz, err := gzip.NewReader(or.createUnderlyingReader())
		if err != nil {
			return err
		}
		or.gz = gz
	} else {
		or.gz.Close()
		if or.gz.Reset(or.createUnderlyingReader()) == io.EOF {
			or.gz = nil
			return errWrongOffset
		}
	}

	or.gz.Multistream(false)
	or.block = o.Block
	or.noff = 0

	_, err := io.CopyN(ioutil.Discard, or, o.Off)
	if err != nil {
		return err
	}

	return nil
}
Ejemplo n.º 12
0
func Test_ServeHTTP_Compressed(t *testing.T) {
	gzipHandler := Gzip(DefaultCompression)
	w := httptest.NewRecorder()

	req, err := http.NewRequest("GET", "http://localhost/foobar", nil)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set(headerAcceptEncoding, encodingGzip)

	gzipHandler.ServeHTTP(w, req, testHTTPContent)

	gr, err := gzip.NewReader(w.Body)
	if err != nil {
		t.Fatal(err)
	}
	defer gr.Close()

	body, _ := ioutil.ReadAll(gr)

	if string(body) != gzipTestString {
		t.Fail()
	}
}
Ejemplo n.º 13
0
func (depot *Depot) buildGame(game *types.Game, gamePath string,
	unzipGame bool, deduper dedup.Deduper) (*types.Game, bool, error) {

	var gameTorrent *torrentzip.Writer
	var err error

	glog.V(4).Infof("building game %s with path %s", game.Name, gamePath)

	if unzipGame {
		err := os.Mkdir(gamePath, 0777)
		if err != nil {
			glog.Errorf("error mkdir %s: %v", gamePath, err)
			return nil, false, err
		}
	} else {
		gameDir := filepath.Dir(game.Name)
		if gameDir != "." {
			// name has dirs in it
			err := os.MkdirAll(filepath.Dir(gamePath), 0777)
			if err != nil {
				glog.Errorf("error mkdir %s: %v", filepath.Dir(gamePath), err)
				return nil, false, err
			}
		}

		gameFile, err := os.Create(gamePath + zipSuffix)
		if err != nil {
			glog.Errorf("error creating zip file %s: %v", gamePath+zipSuffix, err)
			return nil, false, err
		}
		defer gameFile.Close()

		gameTorrent, err = torrentzip.NewWriterWithTemp(gameFile, config.GlobalConfig.General.TmpDir)
		if err != nil {
			glog.Errorf("error writing to torrentzip file %s: %v", gamePath+zipSuffix, err)
			return nil, false, err
		}
		defer gameTorrent.Close()
	}

	var fixGame *types.Game

	foundRom := false

	for _, rom := range game.Roms {
		err = depot.romDB.CompleteRom(rom)
		if err != nil {
			glog.Errorf("error completing rom %s: %v", rom.Name, err)
			return nil, false, err
		}

		if rom.Sha1 == nil && rom.Size > 0 {
			if fixGame == nil {
				fixGame = new(types.Game)
				fixGame.Name = game.Name
				fixGame.Description = game.Description
			}

			fixGame.Roms = append(fixGame.Roms, rom)
			continue
		}

		romGZ, err := depot.OpenRomGZ(rom)
		if err != nil {
			glog.Errorf("error opening rom %s from depot: %v", rom.Name, err)
			return nil, false, err
		}

		if romGZ == nil {
			if glog.V(2) {
				glog.Warningf("game %s has missing rom %s (sha1 %s)", game.Name, rom.Name, hex.EncodeToString(rom.Sha1))
			}

			seenRom, err := deduper.Seen(rom)
			if err != nil {
				return nil, false, err
			}

			if !seenRom {
				err = deduper.Declare(rom)
				if err != nil {
					glog.Errorf("error deduping rom %s: %v", rom.Name, err)
					return nil, false, err
				}

				if fixGame == nil {
					fixGame = new(types.Game)
					fixGame.Name = game.Name
					fixGame.Description = game.Description
				}

				fixGame.Roms = append(fixGame.Roms, rom)
			}
			continue
		}

		foundRom = true

		src, err := gzip.NewReader(romGZ)
		if err != nil {
			glog.Errorf("error opening rom gz file %s: %v", rom.Name, err)
			return nil, false, err
		}

		var dstWriter io.WriteCloser

		if unzipGame {
			romPath := filepath.Join(gamePath, rom.Name)
			if strings.ContainsRune(rom.Name, filepath.Separator) {
				err := os.MkdirAll(filepath.Dir(romPath), 0777)
				if err != nil {
					glog.Errorf("error mkdir %s: %v", filepath.Dir(romPath), err)
					return nil, false, err
				}
			}
			dst, err := os.Create(romPath)
			if err != nil {
				glog.Errorf("error creating destination rom file %s: %v", dst, err)
				return nil, false, err
			}
			dstWriter = dst
		} else {
			dst, err := gameTorrent.Create(rom.Name)
			if err != nil {
				glog.Errorf("error creating torrentzip rom entry %s: %v", rom.Name, err)
				return nil, false, err
			}
			dstWriter = nopWriterCloser{dst}
		}
		_, err = io.Copy(dstWriter, src)
		if err != nil {
			glog.Errorf("error copying rom %s: %v", rom.Name, err)
			return nil, false, err
		}

		src.Close()
		dstWriter.Close()

		romGZ.Close()
	}
	return fixGame, foundRom, nil
}