func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent string, layer *roLayer) error { digester := digest.Canonical.New() tr := io.TeeReader(ts, digester.Hash()) tsw, err := tx.TarSplitWriter(true) if err != nil { return err } metaPacker := storage.NewJSONPacker(tsw) defer tsw.Close() // we're passing nil here for the file putter, because the ApplyDiff will // handle the extraction of the archive rdr, err := asm.NewInputTarStream(tr, metaPacker, nil) if err != nil { return err } applySize, err := ls.driver.ApplyDiff(layer.cacheID, parent, archive.Reader(rdr)) if err != nil { return err } // Discard trailing data but ensure metadata is picked up to reconstruct stream io.Copy(ioutil.Discard, rdr) // ignore error as reader may be closed layer.size = applySize layer.diffID = DiffID(digester.Digest()) logrus.Debugf("Applied tar %s to %s, size: %d", layer.diffID, layer.cacheID, applySize) return nil }
func (c *checksums) checksumForGraphIDNoTarsplit(id, parent, newTarDataPath string) (diffID layer.DiffID, size int64, err error) { rawarchive, err := c.driver.TarStream(id, parent) if err != nil { return } defer rawarchive.Close() f, err := os.Create(newTarDataPath) if err != nil { return } defer f.Close() mfz := gzip.NewWriter(f) defer mfz.Close() metaPacker := storage.NewJSONPacker(mfz) packerCounter := &packSizeCounter{metaPacker, &size} archive, err := asm.NewInputTarStream(rawarchive, packerCounter, nil) if err != nil { return } dgst, err := digest.FromReader(archive) if err != nil { return } diffID = layer.DiffID(dgst) return }
func (graph *Graph) disassembleAndApplyTarLayer(id, parent string, layerData archive.ArchiveReader, root string) (size int64, err error) { // this is saving the tar-split metadata mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) if err != nil { return 0, err } mfz := gzip.NewWriter(mf) metaPacker := storage.NewJSONPacker(mfz) defer mf.Close() defer mfz.Close() inflatedLayerData, err := archive.DecompressStream(layerData) if err != nil { return 0, err } // we're passing nil here for the file putter, because the ApplyDiff will // handle the extraction of the archive rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil) if err != nil { return 0, err } if size, err = graph.driver.ApplyDiff(id, parent, archive.ArchiveReader(rdr)); err != nil { return 0, err } return }
func CommandDisasm(c *cli.Context) { if len(c.Args()) != 1 { logrus.Fatalf("please specify tar to be disabled <NAME|->") } if len(c.String("output")) == 0 { logrus.Fatalf("--output filename must be set") } // Set up the tar input stream var inputStream io.Reader if c.Args()[0] == "-" { inputStream = os.Stdin } else { fh, err := os.Open(c.Args()[0]) if err != nil { logrus.Fatal(err) } defer fh.Close() inputStream = fh } // Set up the metadata storage mf, err := os.OpenFile(c.String("output"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) if err != nil { logrus.Fatal(err) } defer mf.Close() mfz := gzip.NewWriter(mf) defer mfz.Close() metaPacker := storage.NewJSONPacker(mfz) // we're passing nil here for the file putter, because the ApplyDiff will // handle the extraction of the archive its, err := asm.NewInputTarStream(inputStream, metaPacker, nil) if err != nil { logrus.Fatal(err) } i, err := io.Copy(os.Stdout, its) if err != nil { logrus.Fatal(err) } logrus.Infof("created %s from %s (read %d bytes)", c.String("output"), c.Args()[0], i) }
func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData io.Reader, root string) (err error) { var ar io.Reader if graph.tarSplitDisabled { ar = layerData } else { // this is saving the tar-split metadata mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) if err != nil { return err } mfz := gzip.NewWriter(mf) metaPacker := storage.NewJSONPacker(mfz) defer mf.Close() defer mfz.Close() inflatedLayerData, err := archive.DecompressStream(layerData) if err != nil { return err } // we're passing nil here for the file putter, because the ApplyDiff will // handle the extraction of the archive rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil) if err != nil { return err } ar = archive.Reader(rdr) } if img.Size, err = graph.driver.ApplyDiff(img.ID, img.Parent, ar); err != nil { return err } return nil }
func writeTarSplitFile(name string, tarContent []byte) error { f, err := os.OpenFile(name, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return err } defer f.Close() fz := gzip.NewWriter(f) metaPacker := storage.NewJSONPacker(fz) defer fz.Close() rdr, err := asm.NewInputTarStream(bytes.NewReader(tarContent), metaPacker, nil) if err != nil { return err } if _, err := io.Copy(ioutil.Discard, rdr); err != nil { return err } return nil }
func (ls *layerStore) migrateLayer(tx MetadataTransaction, tarDataFile string, layer *roLayer) error { var ar io.Reader var tdf *os.File var err error if tarDataFile != "" { tdf, err = os.Open(tarDataFile) if err != nil { if !os.IsNotExist(err) { return err } tdf = nil } defer tdf.Close() } if tdf != nil { tsw, err := tx.TarSplitWriter() if err != nil { return err } defer tsw.Close() uncompressed, err := gzip.NewReader(tdf) if err != nil { return err } defer uncompressed.Close() tr := io.TeeReader(uncompressed, tsw) trc := ioutils.NewReadCloserWrapper(tr, uncompressed.Close) ar, err = ls.assembleTar(layer.cacheID, trc, &layer.size) if err != nil { return err } } else { var graphParent string if layer.parent != nil { graphParent = layer.parent.cacheID } archiver, err := ls.driver.Diff(layer.cacheID, graphParent) if err != nil { return err } defer archiver.Close() tsw, err := tx.TarSplitWriter() if err != nil { return err } metaPacker := storage.NewJSONPacker(tsw) packerCounter := &packSizeCounter{metaPacker, &layer.size} defer tsw.Close() ar, err = asm.NewInputTarStream(archiver, packerCounter, nil) if err != nil { return err } } digester := digest.Canonical.New() _, err = io.Copy(digester.Hash(), ar) if err != nil { return err } layer.diffID = DiffID(digester.Digest()) return nil }
func main() { flag.Parse() for _, arg := range flag.Args() { fh, err := os.Open(arg) if err != nil { log.Fatal(err) } defer fh.Close() fi, err := fh.Stat() if err != nil { log.Fatal(err) } fmt.Printf("inspecting %q (size %dk)\n", fh.Name(), fi.Size()/1024) packFh, err := ioutil.TempFile("", "packed.") if err != nil { log.Fatal(err) } defer packFh.Close() if *flCleanup { defer os.Remove(packFh.Name()) } sp := storage.NewJSONPacker(packFh) fp := storage.NewDiscardFilePutter() dissam, err := asm.NewInputTarStream(fh, sp, fp) if err != nil { log.Fatal(err) } var num int tr := tar.NewReader(dissam) for { _, err = tr.Next() if err != nil { if err == io.EOF { break } log.Fatal(err) } num++ if _, err := io.Copy(ioutil.Discard, tr); err != nil { log.Fatal(err) } } fmt.Printf(" -- number of files: %d\n", num) if err := packFh.Sync(); err != nil { log.Fatal(err) } fi, err = packFh.Stat() if err != nil { log.Fatal(err) } fmt.Printf(" -- size of metadata uncompressed: %dk\n", fi.Size()/1024) gzPackFh, err := ioutil.TempFile("", "packed.gz.") if err != nil { log.Fatal(err) } defer gzPackFh.Close() if *flCleanup { defer os.Remove(gzPackFh.Name()) } gzWrtr := gzip.NewWriter(gzPackFh) if _, err := packFh.Seek(0, 0); err != nil { log.Fatal(err) } if _, err := io.Copy(gzWrtr, packFh); err != nil { log.Fatal(err) } gzWrtr.Close() if err := gzPackFh.Sync(); err != nil { log.Fatal(err) } fi, err = gzPackFh.Stat() if err != nil { log.Fatal(err) } fmt.Printf(" -- size of gzip compressed metadata: %dk\n", fi.Size()/1024) } }
func CommandChecksize(c *cli.Context) { if len(c.Args()) == 0 { logrus.Fatalf("please specify tar archives to check ('-' will check stdin)") } for _, arg := range c.Args() { fh, err := os.Open(arg) if err != nil { log.Fatal(err) } defer fh.Close() fi, err := fh.Stat() if err != nil { log.Fatal(err) } fmt.Printf("inspecting %q (size %dk)\n", fh.Name(), fi.Size()/1024) packFh, err := ioutil.TempFile("", "packed.") if err != nil { log.Fatal(err) } defer packFh.Close() if !c.Bool("work") { defer os.Remove(packFh.Name()) } else { fmt.Printf(" -- working file preserved: %s\n", packFh.Name()) } sp := storage.NewJSONPacker(packFh) fp := storage.NewDiscardFilePutter() dissam, err := asm.NewInputTarStream(fh, sp, fp) if err != nil { log.Fatal(err) } var num int tr := tar.NewReader(dissam) for { _, err = tr.Next() if err != nil { if err == io.EOF { break } log.Fatal(err) } num++ if _, err := io.Copy(ioutil.Discard, tr); err != nil { log.Fatal(err) } } fmt.Printf(" -- number of files: %d\n", num) if err := packFh.Sync(); err != nil { log.Fatal(err) } fi, err = packFh.Stat() if err != nil { log.Fatal(err) } fmt.Printf(" -- size of metadata uncompressed: %dk\n", fi.Size()/1024) gzPackFh, err := ioutil.TempFile("", "packed.gz.") if err != nil { log.Fatal(err) } defer gzPackFh.Close() if !c.Bool("work") { defer os.Remove(gzPackFh.Name()) } gzWrtr := gzip.NewWriter(gzPackFh) if _, err := packFh.Seek(0, 0); err != nil { log.Fatal(err) } if _, err := io.Copy(gzWrtr, packFh); err != nil { log.Fatal(err) } gzWrtr.Close() if err := gzPackFh.Sync(); err != nil { log.Fatal(err) } fi, err = gzPackFh.Stat() if err != nil { log.Fatal(err) } fmt.Printf(" -- size of gzip compressed metadata: %dk\n", fi.Size()/1024) } }