// NewInputTarStream wraps the Reader stream of a tar archive and provides a
// Reader stream of the same.
//
// In the middle it will pack the segments and file metadata to storage.Packer
// `p`.
//
// The the storage.FilePutter is where payload of files in the stream are
// stashed. If this stashing is not needed, you can provide a nil
// storage.FilePutter. Since the checksumming is still needed, then a default
// of NewDiscardFilePutter will be used internally
func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io.Reader, error) {
// What to do here... folks will want their own access to the Reader that is
// their tar archive stream, but we'll need that same stream to use our
// forked 'archive/tar'.
// Perhaps do an io.TeeReader that hands back an io.Reader for them to read
// from, and we'll MITM the stream to store metadata.
// We'll need a storage.FilePutter too ...
// Another concern, whether to do any storage.FilePutter operations, such that we
// don't extract any amount of the archive. But then again, we're not making
// files/directories, hardlinks, etc. Just writing the io to the storage.FilePutter.
// Perhaps we have a DiscardFilePutter that is a bit bucket.
// we'll return the pipe reader, since TeeReader does not buffer and will
// only read what the outputRdr Read's. Since Tar archives have padding on
// the end, we want to be the one reading the padding, even if the user's
// `archive/tar` doesn't care.
pR, pW := io.Pipe()
outputRdr := io.TeeReader(r, pW)
// we need a putter that will generate the crc64 sums of file payloads
if fp == nil {
fp = storage.NewDiscardFilePutter()
}
go func() {
tr := tar.NewReader(outputRdr)
tr.RawAccounting = true
for {
hdr, err := tr.Next()
if err != nil {
if err != io.EOF {
pW.CloseWithError(err)
return
}
// even when an EOF is reached, there is often 1024 null bytes on
// the end of an archive. Collect them too.
_, err := p.AddEntry(storage.Entry{
Type: storage.SegmentType,
Payload: tr.RawBytes(),
})
if err != nil {
pW.CloseWithError(err)
}
break // not return. We need the end of the reader.
}
if hdr == nil {
break // not return. We need the end of the reader.
}
if _, err := p.AddEntry(storage.Entry{
Type: storage.SegmentType,
Payload: tr.RawBytes(),
}); err != nil {
pW.CloseWithError(err)
}
var csum []byte
if hdr.Size > 0 {
var err error
_, csum, err = fp.Put(hdr.Name, tr)
if err != nil {
pW.CloseWithError(err)
}
}
// File entries added, regardless of size
_, err = p.AddEntry(storage.Entry{
Type: storage.FileType,
Name: hdr.Name,
Size: hdr.Size,
Payload: csum,
})
if err != nil {
pW.CloseWithError(err)
}
if b := tr.RawBytes(); len(b) > 0 {
_, err = p.AddEntry(storage.Entry{
Type: storage.SegmentType,
Payload: b,
})
if err != nil {
pW.CloseWithError(err)
}
}
}
// it is allowable, and not uncommon that there is further padding on the
// end of an archive, apart from the expected 1024 null bytes.
remainder, err := ioutil.ReadAll(outputRdr)
if err != nil && err != io.EOF {
pW.CloseWithError(err)
}
_, err = p.AddEntry(storage.Entry{
Type: storage.SegmentType,
Payload: remainder,
})
if err != nil {
pW.CloseWithError(err)
} else {
pW.Close()
}
}()
return pR, nil
}
func main() {
flag.Parse()
log.SetOutput(os.Stderr)
for _, arg := range flag.Args() {
func() {
// Open the tar archive
fh, err := os.Open(arg)
if err != nil {
log.Fatal(err, arg)
}
defer fh.Close()
output, err := os.Create(fmt.Sprintf("%s.out", arg))
if err != nil {
log.Fatal(err)
}
defer output.Close()
log.Printf("writing %q to %q", fh.Name(), output.Name())
fi, err := fh.Stat()
if err != nil {
log.Fatal(err, fh.Name())
}
size := fi.Size()
var sum int64
tr := tar.NewReader(fh)
tr.RawAccounting = true
for {
hdr, err := tr.Next()
if err != nil {
if err != io.EOF {
log.Println(err)
}
// even when an EOF is reached, there is often 1024 null bytes on
// the end of an archive. Collect them too.
post := tr.RawBytes()
output.Write(post)
sum += int64(len(post))
fmt.Printf("EOF padding: %d\n", len(post))
break
}
pre := tr.RawBytes()
output.Write(pre)
sum += int64(len(pre))
var i int64
if i, err = io.Copy(output, tr); err != nil {
log.Println(err)
break
}
sum += i
fmt.Println(hdr.Name, "pre:", len(pre), "read:", i)
}
// it is allowable, and not uncommon that there is further padding on the
// end of an archive, apart from the expected 1024 null bytes
remainder, err := ioutil.ReadAll(fh)
if err != nil && err != io.EOF {
log.Fatal(err, fh.Name())
}
output.Write(remainder)
sum += int64(len(remainder))
fmt.Printf("Remainder: %d\n", len(remainder))
if size != sum {
fmt.Printf("Size: %d; Sum: %d; Diff: %d\n", size, sum, size-sum)
fmt.Printf("Compare like `cmp -bl %s %s | less`\n", fh.Name(), output.Name())
} else {
fmt.Printf("Size: %d; Sum: %d\n", size, sum)
}
}()
}
}