// NewInputTarStream wraps the Reader stream of a tar archive and provides a // Reader stream of the same. // // In the middle it will pack the segments and file metadata to storage.Packer // `p`. // // The the storage.FilePutter is where payload of files in the stream are // stashed. If this stashing is not needed, you can provide a nil // storage.FilePutter. Since the checksumming is still needed, then a default // of NewDiscardFilePutter will be used internally func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io.Reader, error) { // What to do here... folks will want their own access to the Reader that is // their tar archive stream, but we'll need that same stream to use our // forked 'archive/tar'. // Perhaps do an io.TeeReader that hands back an io.Reader for them to read // from, and we'll MITM the stream to store metadata. // We'll need a storage.FilePutter too ... // Another concern, whether to do any storage.FilePutter operations, such that we // don't extract any amount of the archive. But then again, we're not making // files/directories, hardlinks, etc. Just writing the io to the storage.FilePutter. // Perhaps we have a DiscardFilePutter that is a bit bucket. // we'll return the pipe reader, since TeeReader does not buffer and will // only read what the outputRdr Read's. Since Tar archives have padding on // the end, we want to be the one reading the padding, even if the user's // `archive/tar` doesn't care. pR, pW := io.Pipe() outputRdr := io.TeeReader(r, pW) // we need a putter that will generate the crc64 sums of file payloads if fp == nil { fp = storage.NewDiscardFilePutter() } go func() { tr := tar.NewReader(outputRdr) tr.RawAccounting = true for { hdr, err := tr.Next() if err != nil { if err != io.EOF { pW.CloseWithError(err) return } // even when an EOF is reached, there is often 1024 null bytes on // the end of an archive. Collect them too. _, err := p.AddEntry(storage.Entry{ Type: storage.SegmentType, Payload: tr.RawBytes(), }) if err != nil { pW.CloseWithError(err) } break // not return. We need the end of the reader. } if hdr == nil { break // not return. We need the end of the reader. } if _, err := p.AddEntry(storage.Entry{ Type: storage.SegmentType, Payload: tr.RawBytes(), }); err != nil { pW.CloseWithError(err) } var csum []byte if hdr.Size > 0 { var err error _, csum, err = fp.Put(hdr.Name, tr) if err != nil { pW.CloseWithError(err) } } // File entries added, regardless of size _, err = p.AddEntry(storage.Entry{ Type: storage.FileType, Name: hdr.Name, Size: hdr.Size, Payload: csum, }) if err != nil { pW.CloseWithError(err) } if b := tr.RawBytes(); len(b) > 0 { _, err = p.AddEntry(storage.Entry{ Type: storage.SegmentType, Payload: b, }) if err != nil { pW.CloseWithError(err) } } } // it is allowable, and not uncommon that there is further padding on the // end of an archive, apart from the expected 1024 null bytes. remainder, err := ioutil.ReadAll(outputRdr) if err != nil && err != io.EOF { pW.CloseWithError(err) } _, err = p.AddEntry(storage.Entry{ Type: storage.SegmentType, Payload: remainder, }) if err != nil { pW.CloseWithError(err) } else { pW.Close() } }() return pR, nil }
func main() { flag.Parse() log.SetOutput(os.Stderr) for _, arg := range flag.Args() { func() { // Open the tar archive fh, err := os.Open(arg) if err != nil { log.Fatal(err, arg) } defer fh.Close() output, err := os.Create(fmt.Sprintf("%s.out", arg)) if err != nil { log.Fatal(err) } defer output.Close() log.Printf("writing %q to %q", fh.Name(), output.Name()) fi, err := fh.Stat() if err != nil { log.Fatal(err, fh.Name()) } size := fi.Size() var sum int64 tr := tar.NewReader(fh) tr.RawAccounting = true for { hdr, err := tr.Next() if err != nil { if err != io.EOF { log.Println(err) } // even when an EOF is reached, there is often 1024 null bytes on // the end of an archive. Collect them too. post := tr.RawBytes() output.Write(post) sum += int64(len(post)) fmt.Printf("EOF padding: %d\n", len(post)) break } pre := tr.RawBytes() output.Write(pre) sum += int64(len(pre)) var i int64 if i, err = io.Copy(output, tr); err != nil { log.Println(err) break } sum += i fmt.Println(hdr.Name, "pre:", len(pre), "read:", i) } // it is allowable, and not uncommon that there is further padding on the // end of an archive, apart from the expected 1024 null bytes remainder, err := ioutil.ReadAll(fh) if err != nil && err != io.EOF { log.Fatal(err, fh.Name()) } output.Write(remainder) sum += int64(len(remainder)) fmt.Printf("Remainder: %d\n", len(remainder)) if size != sum { fmt.Printf("Size: %d; Sum: %d; Diff: %d\n", size, sum, size-sum) fmt.Printf("Compare like `cmp -bl %s %s | less`\n", fh.Name(), output.Name()) } else { fmt.Printf("Size: %d; Sum: %d\n", size, sum) } }() } }