func extractBz2(body []byte, location string) (string, error) {
bodyCopy := make([]byte, len(body))
copy(bodyCopy, body)
tarFile := bzip2.NewReader(bytes.NewReader(body))
tarReader := tar.NewReader(tarFile)
var dirList []string
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
dirList = append(dirList, header.Name)
}
basedir := findBaseDir(dirList)
tarFile = bzip2.NewReader(bytes.NewReader(bodyCopy))
tarReader = tar.NewReader(tarFile)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
//return location, err
}
path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1))
info := header.FileInfo()
if info.IsDir() {
if err = os.MkdirAll(path, info.Mode()); err != nil {
return location, err
}
continue
}
if header.Typeflag == tar.TypeSymlink {
err = os.Symlink(header.Linkname, path)
continue
}
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
if err == nil {
defer file.Close()
}
_, err = io.Copy(file, tarReader)
if err != nil {
//return location, err
}
}
return location, nil
}
func tarFilesCount(sourcefile string) int {
flreader, _ := os.Open(sourcefile)
defer flreader.Close()
var fltarReader *tar.Reader
var flReader io.ReadCloser = flreader
if strings.HasSuffix(sourcefile, ".gz") ||
strings.HasSuffix(sourcefile, ".tgz") {
flgzipReader, err := gzip.NewReader(flreader)
checkErr("In tarFilesCounter - NewReader", err)
fltarReader = tar.NewReader(flgzipReader)
defer flReader.Close()
} else if strings.HasSuffix(sourcefile, ".bz2") {
flbz2Reader := bzip2.NewReader(flreader)
fltarReader = tar.NewReader(flbz2Reader)
} else {
fltarReader = tar.NewReader(flreader)
}
trfl := fltarReader
counter := 0
for {
_, err := trfl.Next()
if err != nil {
if err == io.EOF {
break
}
checkErr("Extract error::ReadTarArchive", err)
}
counter++
}
fmt.Println("Files in archive -", counter)
return counter
}
func TestBzip2(t *testing.T) {
var compressed, uncompressed bytes.Buffer
w := bzip.NewWriter(&compressed)
// Write a repetitive message in a million pieces,
// compressing one copy but not the other.
tee := io.MultiWriter(w, &uncompressed)
for i := 0; i < 1000000; i++ {
io.WriteString(tee, "hello")
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
// Check the size of the compressed stream.
if got, want := compressed.Len(), 255; got != want {
t.Errorf("1 million hellos compressed to %d bytes, want %d", got, want)
}
// Decompress and compare with original.
var decompressed bytes.Buffer
io.Copy(&decompressed, bzip2.NewReader(&compressed))
if !bytes.Equal(uncompressed.Bytes(), decompressed.Bytes()) {
t.Error("decompression yielded a different message")
}
}
func skipToArMember(arReader *ar.Reader, memberPrefix string) (io.Reader, error) {
var err error
// find the right ar member
var header *ar.Header
for {
header, err = arReader.Next()
if err != nil {
return nil, err
}
if strings.HasPrefix(header.Name, memberPrefix) {
break
}
}
// figure out what compression to use
var dataReader io.Reader
switch {
case strings.HasSuffix(header.Name, ".gz"):
dataReader, err = gzip.NewReader(arReader)
if err != nil {
return nil, err
}
case strings.HasSuffix(header.Name, ".bz2"):
dataReader = bzip2.NewReader(arReader)
case strings.HasSuffix(header.Name, ".xz"):
dataReader = xzPipeReader(arReader)
default:
return nil, fmt.Errorf("Can not handle %s", header.Name)
}
return dataReader, nil
}
func OpenDecompress(fn string) (r io.ReadCloser, err error) {
f, err := os.Open(fn)
if err != nil {
return nil, err
}
ext := filepath.Ext(fn)
switch ext {
case ".log", ".txt":
return f, err
case ".gz":
gzr, err := gzip.NewReader(f)
return &WrappedDecompressor{
ReadCloser: gzr,
wrapped: f,
}, err
case ".bz2":
bzr := bzip2.NewReader(f)
return &WrappedDecompressor{
ReadCloser: ioutil.NopCloser(bzr),
wrapped: f,
}, nil
default:
return f, err
}
}
func maybeDecompress(rs io.ReadSeeker) (io.Reader, error) {
// TODO(jonboulle): this is a bit redundant with detectValType
typ, err := aci.DetectFileType(rs)
if err != nil {
return nil, err
}
if _, err := rs.Seek(0, 0); err != nil {
return nil, err
}
var r io.Reader
switch typ {
case aci.TypeGzip:
r, err = gzip.NewReader(rs)
if err != nil {
return nil, fmt.Errorf("error reading gzip: %v", err)
}
case aci.TypeBzip2:
r = bzip2.NewReader(rs)
case aci.TypeXz:
r = aci.XzReader(rs)
case aci.TypeTar:
r = rs
case aci.TypeUnknown:
return nil, errors.New("unknown filetype")
default:
// should never happen
panic(fmt.Sprintf("bad type returned from DetectFileType: %v", typ))
}
return r, nil
}
func getSampleLines(path string) []string {
file, err := os.Open(path)
if err != nil {
panic("Could not open sample file: " + err.Error())
}
defer file.Close()
bz := bzip2.NewReader(file)
rv := make([]string, 0, 250000)
bio := bufio.NewReader(bz)
bytesread := int64(0)
done := false
for !done {
line, err := bio.ReadString('\n')
switch err {
case nil:
rv = append(rv, line)
bytesread += int64(len(line))
case io.EOF:
done = true
default:
panic("Could not load samples: " + err.Error())
}
}
return rv
}
func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
buf := make([]byte, 10)
totalN := 0
for totalN < 10 {
n, err := archive.Read(buf[totalN:])
if err != nil {
if err == io.EOF {
return nil, fmt.Errorf("Tarball too short")
}
return nil, err
}
totalN += n
utils.Debugf("[tar autodetect] n: %d", n)
}
compression := DetectCompression(buf)
wrap := io.MultiReader(bytes.NewReader(buf), archive)
switch compression {
case Uncompressed:
return ioutil.NopCloser(wrap), nil
case Gzip:
return gzip.NewReader(wrap)
case Bzip2:
return ioutil.NopCloser(bzip2.NewReader(wrap)), nil
case Xz:
return xzDecompress(wrap)
default:
return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
}
}
func ls(path string) (map[string]bool, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
files := make(map[string]bool)
bz2R := bzip2.NewReader(f)
r := tar.NewReader(bz2R)
for {
h, err := r.Next()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
files[h.Name] = true
}
// don't exclude the metadata
delete(files, "./")
delete(files, "repositories")
return files, nil
}
func loadBlocks(t *testing.T, file string) (blocks []*dcrutil.Block, err error) {
fi, err := os.Open(file)
if err != nil {
t.Errorf("failed to open file %v, err %v", file, err)
return nil, err
}
bcStream := bzip2.NewReader(fi)
defer fi.Close()
// Create a buffer of the read file
bcBuf := new(bytes.Buffer)
bcBuf.ReadFrom(bcStream)
// Create decoder from the buffer and a map to store the data
bcDecoder := gob.NewDecoder(bcBuf)
blockchain := make(map[int64][]byte)
// Decode the blockchain into the map
if err := bcDecoder.Decode(&blockchain); err != nil {
t.Errorf("error decoding test blockchain")
}
blocks = make([]*dcrutil.Block, 0, len(blockchain))
for height := int64(1); height < int64(len(blockchain)); height++ {
block, err := dcrutil.NewBlockFromBytes(blockchain[height])
if err != nil {
t.Errorf("failed to parse block %v", height)
return nil, err
}
block.SetHeight(height - 1)
blocks = append(blocks, block)
}
return
}
// unpackRootfs unpacks a stage1 rootfs (compressed file, pointed to by rfs)
// into dir, returning any error encountered
func unpackRootfs(rfs string, dir string) error {
fh, err := os.Open(rfs)
if err != nil {
return fmt.Errorf("error opening stage1 rootfs: %v", err)
}
typ, err := aci.DetectFileType(fh)
if err != nil {
return fmt.Errorf("error detecting image type: %v", err)
}
if _, err := fh.Seek(0, 0); err != nil {
return fmt.Errorf("error seeking image: %v", err)
}
var r io.Reader
switch typ {
case aci.TypeGzip:
r, err = gzip.NewReader(fh)
if err != nil {
return fmt.Errorf("error reading gzip: %v", err)
}
case aci.TypeBzip2:
r = bzip2.NewReader(fh)
case aci.TypeXz:
r = aci.XzReader(fh)
case aci.TypeTar:
r = fh
case aci.TypeUnknown:
return fmt.Errorf("error: unknown image filetype")
default:
// should never happen
panic("no type returned from DetectFileType?")
}
return untarRootfs(r, dir)
}
func (b *benchmarkCase) Open() (io.ReadCloser, error) {
buf := bytes.NewBufferString(b.casefile)
return &benchmarkFile{
Reader: bzip2.NewReader(base64.NewDecoder(base64.StdEncoding, buf)),
buffer: buf,
}, nil
}
// Attempts to extract a compressed project into the given destination.
func unpackExampleProject(root string) (err error) {
var stat os.FileInfo
// Validating destination.
if stat, err = os.Stat(root); err != nil {
return err
}
if stat.IsDir() == false {
return fmt.Errorf("Expecting a directory.")
}
// Creating a tarbz2 reader.
tbz := tar.NewReader(bzip2.NewReader(bytes.NewBuffer(compressedProject)))
// Extracting tarred files.
for {
hdr, err := tbz.Next()
if err != nil {
if err == io.EOF {
break
}
panic(err.Error())
}
// See http://en.wikipedia.org/wiki/Tar_(computing)
filePath := root + pathSeparator + hdr.Name
switch hdr.Typeflag {
case '0':
// Normal file
fp, err := os.Create(filePath)
if err != nil {
return err
}
io.Copy(fp, tbz)
err = os.Chmod(filePath, os.FileMode(hdr.Mode))
if err != nil {
return err
}
fp.Close()
case '5':
// Directory
os.MkdirAll(filePath, os.FileMode(hdr.Mode))
default:
// fmt.Printf("--> %s, %d, %c\n", hdr.Name, hdr.Mode, hdr.Typeflag)
panic(fmt.Sprintf("Unhandled tar type: %c in file: %s", hdr.Typeflag, hdr.Name))
}
}
return nil
}
// Test creating a writer, writing a significant amount of data to it, and
// closing it again.
func TestWriter_Write12MiB(t *testing.T) {
// Generate a sequence of data.
uncompressedStrideIn := make([]byte, 3*1024*1024)
for i := 0; i < len(uncompressedStrideIn); i++ {
uncompressedStrideIn[i] = byte(i % 256)
}
// Test with varying block sizes.
for blockSize100k := 1; blockSize100k <= 9; blockSize100k++ {
// Compress the data.
compressed := AssertWriterWrites(t, blockSize100k, [][]byte{
uncompressedStrideIn,
uncompressedStrideIn,
uncompressedStrideIn,
uncompressedStrideIn,
})
// Decompress the data and verify equality.
uncompressedOut, err := ioutil.ReadAll(bzip2.NewReader(bytes.NewBuffer(compressed)))
if err != nil {
t.Fatalf("Failed to decompress compressed bzip2 data: %v", err)
}
if len(uncompressedOut) != 4*len(uncompressedStrideIn) {
t.Fatalf("Expected decompressed data to be %d B but it is %d B", 4*len(uncompressedStrideIn), len(uncompressedOut))
}
for i := 0; i < 4; i++ {
if !bytes.Equal(uncompressedOut[i*len(uncompressedStrideIn):(i+1)*len(uncompressedStrideIn)], uncompressedStrideIn) {
t.Fatalf("Decompressed data does not match original data")
}
}
}
}
func main() {
flag.Parse()
if *file == "" {
log.Fatal("You must supply a bz2 dump file.")
}
session, err := mgo.Dial(*dburl)
if err != nil {
panic(err)
}
f, err := os.Open(*file)
if err != nil {
log.Fatalf("Error opening file: %v", err)
}
defer f.Close()
z := bzip2.NewReader(f)
p, err := wikiparse.NewParser(z)
if err != nil {
log.Fatalf("Error setting up new page parser: %v", err)
}
err = session.DB(*dbname).C(*collection).EnsureIndex(titleIndex)
if err != nil {
log.Fatal("Error creating title index", err)
}
processDump(p, session.DB(*dbname))
}
// SetupTarTestFixture extracts the tarFile to outputDir.
func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) {
input, err := os.Open(tarFile)
defer input.Close()
OK(t, err)
var rd io.Reader
switch filepath.Ext(tarFile) {
case ".gz":
r, err := gzip.NewReader(input)
OK(t, err)
defer r.Close()
rd = r
case ".bzip2":
rd = bzip2.NewReader(input)
default:
rd = input
}
cmd := exec.Command("tar", "xf", "-")
cmd.Dir = outputDir
cmd.Stdin = rd
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
OK(t, cmd.Run())
}
func read(name string, wg *sync.WaitGroup, reg *regexp.Regexp) {
defer wg.Done()
f, err := os.Open(name)
if err != nil {
errlog.Println(err)
return
}
defer f.Close()
var r io.Reader
ext := filepath.Ext(name)
switch ext {
case ".gz":
r, err = gzip.NewReader(f)
if err != nil {
log.Println(err)
return
}
case ".bz":
r = bzip2.NewReader(f)
default:
errlog.Println("Unknown extension:", ext)
return
}
scanner := bufio.NewScanner(r)
for scanner.Scan() {
b := scanner.Bytes()
if reg.Match(b) {
log.Printf("%s: %s", name, b)
}
}
if err := scanner.Err(); err != nil {
errlog.Printf("Error while reading %s: %s", name, err)
}
}
func multiStreamWorker(src IndexedParseSource, wg *sync.WaitGroup,
p *multiStreamParser) {
defer wg.Done()
r, err := src.OpenData()
if err != nil {
log.Fatalf("Error opening data: %v", err)
}
defer r.Close()
for idxChunk := range p.workerch {
_, err := r.Seek(idxChunk.offset, 0)
if err != nil {
log.Fatalf("Error seeking to specified offset: %v", err)
}
bz := bzip2.NewReader(r)
d := xml.NewDecoder(bz)
for i := 0; i < idxChunk.count && err != io.EOF; i++ {
newpage := &Page{}
err = d.Decode(newpage)
if err == nil {
p.entries <- newpage
}
}
}
}
func (mpq *Mpq) File(filename string) (file *File, err error) {
// First see if the file is already in the map
file, found := mpq.files[filename]
if !found {
// If it's not, try to load it
fileHash, hashErr := mpq.getHashEntry(filename)
if hashErr != nil {
err = fmt.Errorf("Unable to find file: %v",
filename)
return
}
fileBlock := mpq.BlockEntries[fileHash.BlockIndex]
file = newFile(filename, fileHash, fileBlock)
mpq.files[filename] = file
}
_, err = mpq.reader.Seek(int64(mpq.ArchiveOffset+
file.block.FilePosition), 0)
if err != nil {
return
}
compType := make([]byte, 1)
_, err = mpq.reader.Read(compType)
if err != nil {
return
}
file.compressionType = compType[0]
switch file.compressionType {
case CompressBzip2:
mpq.fileReader = bzip2.NewReader(mpq.reader)
break
case CompressNone:
// Explicitly found that this doesn't have
// compression, so don't go back to the
// original file position as in the default.
mpq.fileReader = mpq.reader
break
default:
// Don't know this compression type, so just
// reset back one byte and read from there as
// if this file doesn't have a header byte. This
// can happen if the file flags say the file is
// compressed but the header byte doesn't exist.
_, err = mpq.reader.Seek(int64(mpq.ArchiveOffset+
file.block.FilePosition), 0)
if err != nil {
return
}
mpq.fileReader = mpq.reader
break
}
mpq.file = file
// Reset the number of bytes read from the file
mpq.fileBytesRead = 0
return
}
func Fuzz(data []byte) int {
_, err := ioutil.ReadAll(bzip2.NewReader(bytes.NewReader(data)))
if err != nil {
return 0
}
return 1
}
func readFile(fn string, ch chan<- string) {
f, err := os.Open(fn)
maybeFatal(err)
defer f.Close()
br := bufio.NewReader(bzip2.NewReader(f))
for i := 0; ; i++ {
b, err := br.ReadBytes('\n')
switch err {
case io.EOF:
log.Printf("Processed %s lines total",
humanize.Comma(int64(i)))
return
case nil:
ch <- string(b[:len(b)-1])
default:
log.Fatalf("Error reading input: %v", err)
}
if i%100000 == 0 {
log.Printf("Processed %s lines",
humanize.Comma(int64(i)))
}
}
}
// archiveVersion retrieves the GUI version from the juju-gui-* directory
// included in the given tar.bz2 archive reader.
func archiveVersion(r io.Reader) (version.Number, error) {
var vers version.Number
prefix := "jujugui-"
tr := tar.NewReader(bzip2.NewReader(r))
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return vers, errors.New("cannot read Juju GUI archive")
}
info := hdr.FileInfo()
if !info.IsDir() || !strings.HasPrefix(hdr.Name, prefix) {
continue
}
n := filepath.Dir(hdr.Name)[len(prefix):]
vers, err = version.Parse(n)
if err != nil {
return vers, errors.Errorf("invalid version %q in archive", n)
}
return vers, nil
}
return vers, errors.New("cannot find Juju GUI version in archive")
}
// guiVersion retrieves the GUI version from the juju-gui-* directory included
// in the bz2 archive at the given path.
func guiVersion(path string) (version.Number, error) {
var number version.Number
f, err := os.Open(path)
if err != nil {
return number, errors.Annotate(err, "cannot open Juju GUI archive")
}
defer f.Close()
prefix := "jujugui-"
r := tar.NewReader(bzip2.NewReader(f))
for {
hdr, err := r.Next()
if err == io.EOF {
break
}
if err != nil {
return number, errors.New("cannot read Juju GUI archive")
}
info := hdr.FileInfo()
if !info.IsDir() || !strings.HasPrefix(hdr.Name, prefix) {
continue
}
n := info.Name()[len(prefix):]
number, err = version.Parse(n)
if err != nil {
return number, errors.Errorf("cannot parse version %q", n)
}
return number, nil
}
return number, errors.New("cannot find Juju GUI version")
}
func (s *CompressSuite) TestCompress(c *C) {
err := CompressFile(s.tempfile, false)
c.Assert(err, IsNil)
file, err := os.Open(s.tempfile.Name() + ".gz")
c.Assert(err, IsNil)
gzReader, err := gzip.NewReader(file)
c.Assert(err, IsNil)
buf, err := ioutil.ReadAll(gzReader)
c.Assert(err, IsNil)
gzReader.Close()
file.Close()
c.Check(string(buf), Equals, testString)
file, err = os.Open(s.tempfile.Name() + ".bz2")
c.Assert(err, IsNil)
bzReader := bzip2.NewReader(file)
_, err = bzReader.Read(buf)
c.Assert(err, IsNil)
file.Close()
c.Check(string(buf), Equals, testString)
}
func openFile(name string) (*os.File, io.Reader, error) {
f, err := os.Open(name)
if err != nil {
return nil, nil, err
}
var r io.Reader
// Detect compression.
switch filepath.Ext(name) {
case ".gzip", ".gz":
r, err = gzip.NewReader(f)
if err != nil {
return nil, nil, err
}
case ".bzip2", ".bz2":
r = bzip2.NewReader(f)
default:
r = f
}
return f, r, nil
}
func TestIncompleteBlock(t *testing.T) {
var buf bytes.Buffer
var out bytes.Buffer
expected := []byte("banana")
writer := NewWriter(&buf)
_, err := writer.Write(expected)
if err == nil {
err = writer.Close()
}
if err != nil {
t.Fatal(err)
}
reader := bzip2.NewReader(&buf)
_, err = io.Copy(&out, reader)
if err != nil {
t.Fatal(err)
}
if out.String() != string(expected) {
t.Error("Output is incorrect. Got", out.String(), "wanted",
string(expected))
}
}
func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
p := pools.BufioReader32KPool
buf := p.Get(archive)
bs, err := buf.Peek(10)
if err != nil {
return nil, err
}
compression := DetectCompression(bs)
switch compression {
case Uncompressed:
readBufWrapper := p.NewReadCloserWrapper(buf, buf)
return readBufWrapper, nil
case Gzip:
gzReader, err := gzip.NewReader(buf)
if err != nil {
return nil, err
}
readBufWrapper := p.NewReadCloserWrapper(buf, gzReader)
return readBufWrapper, nil
case Bzip2:
bz2Reader := bzip2.NewReader(buf)
readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader)
return readBufWrapper, nil
case Xz:
xzReader, err := xzDecompress(buf)
if err != nil {
return nil, err
}
readBufWrapper := p.NewReadCloserWrapper(buf, xzReader)
return readBufWrapper, nil
default:
return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
}
}
func TestStreamingReaderTimeout(t *testing.T) {
assert := assert.New(t)
// create a http client with a timeout too low to get the entire replay
client := http.Client{
Timeout: 2 * time.Second,
}
// request the replay, the timeout should be enough for it to begin successfully
resp, err := client.Get("http://replay111.valve.net/570/2526759078_1821332888.dem.bz2")
if err != nil {
t.Fatalf("unable to get remote replay: %s", err)
}
defer resp.Body.Close()
// create a new parser with the streaming response
parser, err := NewStreamParser(bzip2.NewReader(resp.Body))
if err != nil {
t.Fatalf("unable to create parser: %s", err)
}
// expect the parser to fail after a bit under 2 seconds with a timeout error
err = parser.Start()
netErr, ok := err.(net.Error)
assert.True(ok)
assert.True(netErr.Timeout())
}
func main() {
var file *os.File
var err error
if file, err = os.Open("files/sample.tar.bz2"); err != nil {
log.Fatalln(err)
}
defer file.Close()
reader := tar.NewReader(bzip2.NewReader(file))
var header *tar.Header
for {
header, err = reader.Next()
if err == io.EOF {
// ファイルの最後
break
}
if err != nil {
log.Fatalln(err)
}
buf := new(bytes.Buffer)
if _, err = io.Copy(buf, reader); err != nil {
log.Fatalln(err)
}
if err = ioutil.WriteFile("output/"+header.Name, buf.Bytes(), 0755); err != nil {
log.Fatal(err)
}
}
}
// FIXME: mostly copy/paste from github.com/appc/spec/actool/validate.go
func DecompressingReader(rd io.Reader) (io.Reader, error) {
brd := bufio.NewReaderSize(rd, 1024)
header, err := brd.Peek(768)
if err != nil {
return nil, errors.Trace(err)
}
typ, err := aci.DetectFileType(bytes.NewReader(header))
if err != nil {
return nil, errors.Trace(err)
}
var r io.Reader
switch typ {
case aci.TypeGzip:
r, err = gzip.NewReader(brd)
if err != nil {
return nil, errors.Trace(err)
}
case aci.TypeBzip2:
r = bzip2.NewReader(brd)
case aci.TypeXz:
r = aci.XzReader(brd)
case aci.TypeTar:
r = brd
case aci.TypeUnknown:
return nil, errors.New("unknown filetype")
default:
panic(fmt.Sprintf("bad type returned from DetectFileType: %v", typ))
}
return r, nil
}
