func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent string, layer *roLayer) error {
digester := digest.Canonical.New()
tr := io.TeeReader(ts, digester.Hash())
tsw, err := tx.TarSplitWriter(true)
if err != nil {
return err
}
metaPacker := storage.NewJSONPacker(tsw)
defer tsw.Close()
// we're passing nil here for the file putter, because the ApplyDiff will
// handle the extraction of the archive
rdr, err := asm.NewInputTarStream(tr, metaPacker, nil)
if err != nil {
return err
}
applySize, err := ls.driver.ApplyDiff(layer.cacheID, parent, archive.Reader(rdr))
if err != nil {
return err
}
// Discard trailing data but ensure metadata is picked up to reconstruct stream
io.Copy(ioutil.Discard, rdr) // ignore error as reader may be closed
layer.size = applySize
layer.diffID = DiffID(digester.Digest())
logrus.Debugf("Applied tar %s to %s, size: %d", layer.diffID, layer.cacheID, applySize)
return nil
}
func (c *checksums) checksumForGraphIDNoTarsplit(id, parent, newTarDataPath string) (diffID layer.DiffID, size int64, err error) {
rawarchive, err := c.driver.TarStream(id, parent)
if err != nil {
return
}
defer rawarchive.Close()
f, err := os.Create(newTarDataPath)
if err != nil {
return
}
defer f.Close()
mfz := gzip.NewWriter(f)
defer mfz.Close()
metaPacker := storage.NewJSONPacker(mfz)
packerCounter := &packSizeCounter{metaPacker, &size}
archive, err := asm.NewInputTarStream(rawarchive, packerCounter, nil)
if err != nil {
return
}
dgst, err := digest.FromReader(archive)
if err != nil {
return
}
diffID = layer.DiffID(dgst)
return
}
func (graph *Graph) disassembleAndApplyTarLayer(id, parent string, layerData archive.ArchiveReader, root string) (size int64, err error) {
// this is saving the tar-split metadata
mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
if err != nil {
return 0, err
}
mfz := gzip.NewWriter(mf)
metaPacker := storage.NewJSONPacker(mfz)
defer mf.Close()
defer mfz.Close()
inflatedLayerData, err := archive.DecompressStream(layerData)
if err != nil {
return 0, err
}
// we're passing nil here for the file putter, because the ApplyDiff will
// handle the extraction of the archive
rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil)
if err != nil {
return 0, err
}
if size, err = graph.driver.ApplyDiff(id, parent, archive.ArchiveReader(rdr)); err != nil {
return 0, err
}
return
}
func CommandDisasm(c *cli.Context) {
if len(c.Args()) != 1 {
logrus.Fatalf("please specify tar to be disabled <NAME|->")
}
if len(c.String("output")) == 0 {
logrus.Fatalf("--output filename must be set")
}
// Set up the tar input stream
var inputStream io.Reader
if c.Args()[0] == "-" {
inputStream = os.Stdin
} else {
fh, err := os.Open(c.Args()[0])
if err != nil {
logrus.Fatal(err)
}
defer fh.Close()
inputStream = fh
}
// Set up the metadata storage
mf, err := os.OpenFile(c.String("output"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
if err != nil {
logrus.Fatal(err)
}
defer mf.Close()
mfz := gzip.NewWriter(mf)
defer mfz.Close()
metaPacker := storage.NewJSONPacker(mfz)
// we're passing nil here for the file putter, because the ApplyDiff will
// handle the extraction of the archive
its, err := asm.NewInputTarStream(inputStream, metaPacker, nil)
if err != nil {
logrus.Fatal(err)
}
i, err := io.Copy(os.Stdout, its)
if err != nil {
logrus.Fatal(err)
}
logrus.Infof("created %s from %s (read %d bytes)", c.String("output"), c.Args()[0], i)
}
func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData io.Reader, root string) (err error) {
var ar io.Reader
if graph.tarSplitDisabled {
ar = layerData
} else {
// this is saving the tar-split metadata
mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
if err != nil {
return err
}
mfz := gzip.NewWriter(mf)
metaPacker := storage.NewJSONPacker(mfz)
defer mf.Close()
defer mfz.Close()
inflatedLayerData, err := archive.DecompressStream(layerData)
if err != nil {
return err
}
// we're passing nil here for the file putter, because the ApplyDiff will
// handle the extraction of the archive
rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil)
if err != nil {
return err
}
ar = archive.Reader(rdr)
}
if img.Size, err = graph.driver.ApplyDiff(img.ID, img.Parent, ar); err != nil {
return err
}
return nil
}
func writeTarSplitFile(name string, tarContent []byte) error {
f, err := os.OpenFile(name, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer f.Close()
fz := gzip.NewWriter(f)
metaPacker := storage.NewJSONPacker(fz)
defer fz.Close()
rdr, err := asm.NewInputTarStream(bytes.NewReader(tarContent), metaPacker, nil)
if err != nil {
return err
}
if _, err := io.Copy(ioutil.Discard, rdr); err != nil {
return err
}
return nil
}
func (ls *layerStore) migrateLayer(tx MetadataTransaction, tarDataFile string, layer *roLayer) error {
var ar io.Reader
var tdf *os.File
var err error
if tarDataFile != "" {
tdf, err = os.Open(tarDataFile)
if err != nil {
if !os.IsNotExist(err) {
return err
}
tdf = nil
}
defer tdf.Close()
}
if tdf != nil {
tsw, err := tx.TarSplitWriter()
if err != nil {
return err
}
defer tsw.Close()
uncompressed, err := gzip.NewReader(tdf)
if err != nil {
return err
}
defer uncompressed.Close()
tr := io.TeeReader(uncompressed, tsw)
trc := ioutils.NewReadCloserWrapper(tr, uncompressed.Close)
ar, err = ls.assembleTar(layer.cacheID, trc, &layer.size)
if err != nil {
return err
}
} else {
var graphParent string
if layer.parent != nil {
graphParent = layer.parent.cacheID
}
archiver, err := ls.driver.Diff(layer.cacheID, graphParent)
if err != nil {
return err
}
defer archiver.Close()
tsw, err := tx.TarSplitWriter()
if err != nil {
return err
}
metaPacker := storage.NewJSONPacker(tsw)
packerCounter := &packSizeCounter{metaPacker, &layer.size}
defer tsw.Close()
ar, err = asm.NewInputTarStream(archiver, packerCounter, nil)
if err != nil {
return err
}
}
digester := digest.Canonical.New()
_, err = io.Copy(digester.Hash(), ar)
if err != nil {
return err
}
layer.diffID = DiffID(digester.Digest())
return nil
}
func main() {
flag.Parse()
for _, arg := range flag.Args() {
fh, err := os.Open(arg)
if err != nil {
log.Fatal(err)
}
defer fh.Close()
fi, err := fh.Stat()
if err != nil {
log.Fatal(err)
}
fmt.Printf("inspecting %q (size %dk)\n", fh.Name(), fi.Size()/1024)
packFh, err := ioutil.TempFile("", "packed.")
if err != nil {
log.Fatal(err)
}
defer packFh.Close()
if *flCleanup {
defer os.Remove(packFh.Name())
}
sp := storage.NewJSONPacker(packFh)
fp := storage.NewDiscardFilePutter()
dissam, err := asm.NewInputTarStream(fh, sp, fp)
if err != nil {
log.Fatal(err)
}
var num int
tr := tar.NewReader(dissam)
for {
_, err = tr.Next()
if err != nil {
if err == io.EOF {
break
}
log.Fatal(err)
}
num++
if _, err := io.Copy(ioutil.Discard, tr); err != nil {
log.Fatal(err)
}
}
fmt.Printf(" -- number of files: %d\n", num)
if err := packFh.Sync(); err != nil {
log.Fatal(err)
}
fi, err = packFh.Stat()
if err != nil {
log.Fatal(err)
}
fmt.Printf(" -- size of metadata uncompressed: %dk\n", fi.Size()/1024)
gzPackFh, err := ioutil.TempFile("", "packed.gz.")
if err != nil {
log.Fatal(err)
}
defer gzPackFh.Close()
if *flCleanup {
defer os.Remove(gzPackFh.Name())
}
gzWrtr := gzip.NewWriter(gzPackFh)
if _, err := packFh.Seek(0, 0); err != nil {
log.Fatal(err)
}
if _, err := io.Copy(gzWrtr, packFh); err != nil {
log.Fatal(err)
}
gzWrtr.Close()
if err := gzPackFh.Sync(); err != nil {
log.Fatal(err)
}
fi, err = gzPackFh.Stat()
if err != nil {
log.Fatal(err)
}
fmt.Printf(" -- size of gzip compressed metadata: %dk\n", fi.Size()/1024)
}
}
func CommandChecksize(c *cli.Context) {
if len(c.Args()) == 0 {
logrus.Fatalf("please specify tar archives to check ('-' will check stdin)")
}
for _, arg := range c.Args() {
fh, err := os.Open(arg)
if err != nil {
log.Fatal(err)
}
defer fh.Close()
fi, err := fh.Stat()
if err != nil {
log.Fatal(err)
}
fmt.Printf("inspecting %q (size %dk)\n", fh.Name(), fi.Size()/1024)
packFh, err := ioutil.TempFile("", "packed.")
if err != nil {
log.Fatal(err)
}
defer packFh.Close()
if !c.Bool("work") {
defer os.Remove(packFh.Name())
} else {
fmt.Printf(" -- working file preserved: %s\n", packFh.Name())
}
sp := storage.NewJSONPacker(packFh)
fp := storage.NewDiscardFilePutter()
dissam, err := asm.NewInputTarStream(fh, sp, fp)
if err != nil {
log.Fatal(err)
}
var num int
tr := tar.NewReader(dissam)
for {
_, err = tr.Next()
if err != nil {
if err == io.EOF {
break
}
log.Fatal(err)
}
num++
if _, err := io.Copy(ioutil.Discard, tr); err != nil {
log.Fatal(err)
}
}
fmt.Printf(" -- number of files: %d\n", num)
if err := packFh.Sync(); err != nil {
log.Fatal(err)
}
fi, err = packFh.Stat()
if err != nil {
log.Fatal(err)
}
fmt.Printf(" -- size of metadata uncompressed: %dk\n", fi.Size()/1024)
gzPackFh, err := ioutil.TempFile("", "packed.gz.")
if err != nil {
log.Fatal(err)
}
defer gzPackFh.Close()
if !c.Bool("work") {
defer os.Remove(gzPackFh.Name())
}
gzWrtr := gzip.NewWriter(gzPackFh)
if _, err := packFh.Seek(0, 0); err != nil {
log.Fatal(err)
}
if _, err := io.Copy(gzWrtr, packFh); err != nil {
log.Fatal(err)
}
gzWrtr.Close()
if err := gzPackFh.Sync(); err != nil {
log.Fatal(err)
}
fi, err = gzPackFh.Stat()
if err != nil {
log.Fatal(err)
}
fmt.Printf(" -- size of gzip compressed metadata: %dk\n", fi.Size()/1024)
}
}
