func diff(id, parent string) (diff archive.Archive, err error) { // create pod // start or replace pod glog.Infof("Diff between %s and %s", id, parent) layerFs := "/tmp/test1" if parent == "" { archive, err := archive.Tar(layerFs, archive.Uncompressed) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() return err }), nil } parentFs := "/tmp/test2" changes, err := archive.ChangesDirs(layerFs, parentFs) if err != nil { return nil, err } archive, err := archive.ExportChanges(layerFs, changes) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() return err }), nil }
func (c *devicemapper) TarStream(id, parent string) (io.ReadCloser, error) { mainPath, releaseMain, err := c.Mount(id) if err != nil { return nil, err } if parent == "" { tar, err := archive.Tar(mainPath, archive.Uncompressed) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(tar, func() error { releaseMain() return tar.Close() }), nil } parentPath, releaseParent, err := c.Mount(parent) if err != nil { releaseMain() return nil, err } tar, err := Diff(mainPath, parentPath) if err != nil { releaseParent() releaseMain() return nil, err } return ioutils.NewReadCloserWrapper(tar, func() error { releaseParent() releaseMain() return tar.Close() }), nil }
// Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch archive.Archive, err error) { startTime := time.Now() driver := gdw.ProtoDriver layerFs, err := driver.Get(id, "") if err != nil { return nil, err } defer func() { if err != nil { driver.Put(id) } }() if parent == "" { archive, err := archive.Tar(layerFs, archive.Uncompressed) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() driver.Put(id) return err }), nil } parentFs, err := driver.Get(parent, "") if err != nil { return nil, err } defer driver.Put(parent) changes, err := archive.ChangesDirs(layerFs, parentFs) if err != nil { return nil, err } archive, err := archive.ExportChanges(layerFs, changes, gdw.uidMaps, gdw.gidMaps) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() driver.Put(id) // NaiveDiffDriver compares file metadata with parent layers. Parent layers // are extracted from tar's with full second precision on modified time. // We need this hack here to make sure calls within same second receive // correct result. time.Sleep(startTime.Truncate(time.Second).Add(time.Second).Sub(time.Now())) return err }), nil }
// TarLayer returns a tar archive of the image's filesystem layer. func (img *Image) TarLayer() (arch archive.Archive, err error) { if img.graph == nil { return nil, fmt.Errorf("Can't load storage driver for unregistered image %s", img.ID) } driver := img.graph.Driver() if differ, ok := driver.(graphdriver.Differ); ok { return differ.Diff(img.ID) } imgFs, err := driver.Get(img.ID, "") if err != nil { return nil, err } defer func() { if err != nil { driver.Put(img.ID) } }() if img.Parent == "" { archive, err := archive.Tar(imgFs, archive.Uncompressed) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() driver.Put(img.ID) return err }), nil } parentFs, err := driver.Get(img.Parent, "") if err != nil { return nil, err } defer driver.Put(img.Parent) changes, err := archive.ChangesDirs(imgFs, parentFs) if err != nil { return nil, err } archive, err := archive.ExportChanges(imgFs, changes) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() driver.Put(img.ID) return err }), nil }
// Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". func (gdw *naiveDiffDriver) Diff(id, parent string) (arch archive.Archive, err error) { driver := gdw.ProtoDriver layerFs, err := driver.Get(id, "") if err != nil { return nil, err } defer func() { if err != nil { driver.Put(id) } }() if parent == "" { archive, err := archive.Tar(layerFs, archive.Uncompressed) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() driver.Put(id) return err }), nil } parentFs, err := driver.Get(parent, "") if err != nil { return nil, err } defer driver.Put(parent) changes, err := archive.ChangesDirs(layerFs, parentFs) if err != nil { return nil, err } archive, err := archive.ExportChanges(layerFs, changes) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() driver.Put(id) return err }), nil }
func sockRequestRaw(method, endpoint string, data io.Reader, ct string) (*http.Response, io.ReadCloser, error) { c, err := sockConn(time.Duration(10 * time.Second)) if err != nil { return nil, nil, fmt.Errorf("could not dial docker daemon: %v", err) } client := httputil.NewClientConn(c, nil) req, err := http.NewRequest(method, endpoint, data) if err != nil { client.Close() return nil, nil, fmt.Errorf("could not create new request: %v", err) } if ct != "" { req.Header.Set("Content-Type", ct) } resp, err := client.Do(req) if err != nil { client.Close() return nil, nil, fmt.Errorf("could not perform request: %v", err) } body := ioutils.NewReadCloserWrapper(resp.Body, func() error { defer resp.Body.Close() return client.Close() }) return resp, body, nil }
func (container *Container) Copy(resource string) (io.ReadCloser, error) { container.Lock() defer container.Unlock() var err error if err := container.Mount(); err != nil { return nil, err } defer func() { if err != nil { // unmount any volumes container.UnmountVolumes(true) // unmount the container's rootfs container.Unmount() } }() mounts, err := container.setupMounts() if err != nil { return nil, err } for _, m := range mounts { dest, err := container.GetResourcePath(m.Destination) if err != nil { return nil, err } if err := mount.Mount(m.Source, dest, "bind", "rbind,ro"); err != nil { return nil, err } } basePath, err := container.GetResourcePath(resource) if err != nil { return nil, err } stat, err := os.Stat(basePath) if err != nil { return nil, err } var filter []string if !stat.IsDir() { d, f := filepath.Split(basePath) basePath = d filter = []string{f} } else { filter = []string{filepath.Base(basePath)} basePath = filepath.Dir(basePath) } archive, err := archive.TarWithOptions(basePath, &archive.TarOptions{ Compression: archive.Uncompressed, IncludeFiles: filter, }) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() container.UnmountVolumes(true) container.Unmount() return err }), nil }
func makeTestLayer(paths []string) (rc io.ReadCloser, err error) { tmpDir, err := ioutil.TempDir("", "graphdriver-test-mklayer") if err != nil { return } defer func() { if err != nil { os.RemoveAll(tmpDir) } }() for _, p := range paths { if p[len(p)-1] == filepath.Separator { if err = os.MkdirAll(filepath.Join(tmpDir, p), 0700); err != nil { return } } else { if err = ioutil.WriteFile(filepath.Join(tmpDir, p), nil, 0600); err != nil { return } } } archive, err := Tar(tmpDir, Uncompressed) if err != nil { return } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() os.RemoveAll(tmpDir) return err }), nil }
// Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". // The layer should be mounted when calling this function func (d *Driver) Diff(id, parent string) (_ archive.Archive, err error) { rID, err := d.resolveID(id) if err != nil { return } layerChain, err := d.getLayerChain(rID) if err != nil { return } // this is assuming that the layer is unmounted if err := hcsshim.UnprepareLayer(d.info, rID); err != nil { return nil, err } prepare := func() { if err := hcsshim.PrepareLayer(d.info, rID, layerChain); err != nil { logrus.Warnf("Failed to Deactivate %s: %s", rID, err) } } arch, err := d.exportLayer(rID, layerChain) if err != nil { prepare() return } return ioutils.NewReadCloserWrapper(arch, func() error { err := arch.Close() prepare() return err }), nil }
func (daemon *Daemon) Diff(container *Container) (archive.Archive, error) { if differ, ok := daemon.driver.(graphdriver.Differ); ok { return differ.Diff(container.ID) } changes, err := daemon.Changes(container) if err != nil { return nil, err } cDir, err := daemon.driver.Get(container.ID, "") if err != nil { return nil, fmt.Errorf("Error getting container rootfs %s from driver %s: %s", container.ID, container.daemon.driver, err) } archive, err := archive.ExportChanges(cDir, changes) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() daemon.driver.Put(container.ID) return err }), nil }
func (container *Container) ExportRw(options *archive.ChangeOptions) (archive.Archive, error) { if err := container.Mount(); err != nil { return nil, err } if container.daemon == nil { return nil, fmt.Errorf("Can't load storage driver for unregistered container %s", container.ID) } var ( commitID = fmt.Sprintf("%s-commit", container.ID) driver = container.daemon.driver ) // create tmp rootfs for commit if err := container.daemon.createTmpRootfs(container); err != nil { return nil, err } defer driver.Remove(commitID) archive, err := driver.Diff(container.ID, commitID, options) if err != nil { container.Unmount() return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() container.Unmount() return err }), nil }
// getContextFromReader will read the contents of the given reader as either a // Dockerfile or tar archive. Returns a tar archive used as a context and a // path to the Dockerfile inside the tar. func getContextFromReader(r io.ReadCloser, dockerfileName string) (out io.ReadCloser, relDockerfile string, err error) { buf := bufio.NewReader(r) magic, err := buf.Peek(archive.HeaderSize) if err != nil && err != io.EOF { return nil, "", fmt.Errorf("failed to peek context header from STDIN: %v", err) } if archive.IsArchive(magic) { return ioutils.NewReadCloserWrapper(buf, func() error { return r.Close() }), dockerfileName, nil } // Input should be read as a Dockerfile. tmpDir, err := ioutil.TempDir("", "docker-build-context-") if err != nil { return nil, "", fmt.Errorf("unbale to create temporary context directory: %v", err) } f, err := os.Create(filepath.Join(tmpDir, api.DefaultDockerfileName)) if err != nil { return nil, "", err } _, err = io.Copy(f, buf) if err != nil { f.Close() return nil, "", err } if err := f.Close(); err != nil { return nil, "", err } if err := r.Close(); err != nil { return nil, "", err } tar, err := archive.Tar(tmpDir, archive.Uncompressed) if err != nil { return nil, "", err } return ioutils.NewReadCloserWrapper(tar, func() error { err := tar.Close() os.RemoveAll(tmpDir) return err }), api.DefaultDockerfileName, nil }
func (container *Container) Copy(resource string) (io.ReadCloser, error) { if err := container.Mount(); err != nil { return nil, err } basePath, err := container.getResourcePath(resource) if err != nil { container.Unmount() return nil, err } // Check if this is actually in a volume for _, mnt := range container.VolumeMounts() { if len(mnt.MountToPath) > 0 && strings.HasPrefix(resource, mnt.MountToPath[1:]) { return mnt.Export(resource) } } // Check if this is a special one (resolv.conf, hostname, ..) if resource == "etc/resolv.conf" { basePath = container.ResolvConfPath } if resource == "etc/hostname" { basePath = container.HostnamePath } if resource == "etc/hosts" { basePath = container.HostsPath } stat, err := os.Stat(basePath) if err != nil { container.Unmount() return nil, err } var filter []string if !stat.IsDir() { d, f := path.Split(basePath) basePath = d filter = []string{f} } else { filter = []string{path.Base(basePath)} basePath = path.Dir(basePath) } archive, err := archive.TarWithOptions(basePath, &archive.TarOptions{ Compression: archive.Uncompressed, IncludeFiles: filter, }) if err != nil { container.Unmount() return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() container.Unmount() return err }), nil }
func (bufPool *BufioReaderPool) NewReadCloserWrapper(buf *bufio.Reader, r io.Reader) io.ReadCloser { return ioutils.NewReadCloserWrapper(r, func() error { if readCloser, ok := r.(io.ReadCloser); ok { return readCloser.Close() } return nil }) }
func (s *containerRouter) postContainersAttach(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { err := httputils.ParseForm(r) if err != nil { return err } containerName := vars["name"] _, upgrade := r.Header["Upgrade"] keys := []byte{} detachKeys := r.FormValue("detachKeys") if detachKeys != "" { keys, err = term.ToBytes(detachKeys) if err != nil { logrus.Warnf("Invalid escape keys provided (%s) using default : ctrl-p ctrl-q", detachKeys) } } hijacker, ok := w.(http.Hijacker) if !ok { return derr.ErrorCodeNoHijackConnection.WithArgs(containerName) } setupStreams := func() (io.ReadCloser, io.Writer, io.Writer, error) { conn, _, err := hijacker.Hijack() if err != nil { return nil, nil, nil, err } // set raw mode conn.Write([]byte{}) if upgrade { fmt.Fprintf(conn, "HTTP/1.1 101 UPGRADED\r\nContent-Type: application/vnd.docker.raw-stream\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\n") } else { fmt.Fprintf(conn, "HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\n") } closer := func() error { httputils.CloseStreams(conn) return nil } return ioutils.NewReadCloserWrapper(conn, closer), conn, conn, nil } attachConfig := &backend.ContainerAttachConfig{ GetStreams: setupStreams, UseStdin: httputils.BoolValue(r, "stdin"), UseStdout: httputils.BoolValue(r, "stdout"), UseStderr: httputils.BoolValue(r, "stderr"), Logs: httputils.BoolValue(r, "logs"), Stream: httputils.BoolValue(r, "stream"), DetachKeys: keys, MuxStreams: true, } return s.backend.ContainerAttach(containerName, attachConfig) }
func (container *Container) Copy(resource string) (io.ReadCloser, error) { container.Lock() defer container.Unlock() var err error if err := container.Mount(); err != nil { return nil, err } defer func() { if err != nil { container.Unmount() } }() if err = container.mountVolumes(); err != nil { container.unmountVolumes() return nil, err } defer func() { if err != nil { container.unmountVolumes() } }() basePath, err := container.GetResourcePath(resource) if err != nil { return nil, err } stat, err := os.Stat(basePath) if err != nil { return nil, err } var filter []string if !stat.IsDir() { d, f := path.Split(basePath) basePath = d filter = []string{f} } else { filter = []string{path.Base(basePath)} basePath = path.Dir(basePath) } archive, err := archive.TarWithOptions(basePath, &archive.TarOptions{ Compression: archive.Uncompressed, IncludeFiles: filter, }) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() container.unmountVolumes() container.Unmount() return err }), nil }
func (daemon *Daemon) exportContainerRw(container *Container) (archive.Archive, error) { archive, err := daemon.diff(container) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() return err }), nil }
// Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". func (d *Driver) Diff(id, parent string) (_ archive.Archive, err error) { rID, err := d.resolveID(id) if err != nil { return } // Getting the layer paths must be done outside of the lock. layerChain, err := d.getLayerChain(rID) if err != nil { return } var undo func() d.Lock() // To support export, a layer must be activated but not prepared. if d.info.Flavour == filterDriver { if d.active[rID] == 0 { if err = hcsshim.ActivateLayer(d.info, rID); err != nil { d.Unlock() return } undo = func() { if err := hcsshim.DeactivateLayer(d.info, rID); err != nil { logrus.Warnf("Failed to Deactivate %s: %s", rID, err) } } } else { if err = hcsshim.UnprepareLayer(d.info, rID); err != nil { d.Unlock() return } undo = func() { if err := hcsshim.PrepareLayer(d.info, rID, layerChain); err != nil { logrus.Warnf("Failed to re-PrepareLayer %s: %s", rID, err) } } } } d.Unlock() arch, err := d.exportLayer(rID, layerChain) if err != nil { undo() return } return ioutils.NewReadCloserWrapper(arch, func() error { defer undo() return arch.Close() }), nil }
// getContextFromURL uses a remote URL as context for a `docker build`. The // remote resource is downloaded as either a Dockerfile or a tar archive. // Returns the tar archive used for the context and a path of the // dockerfile inside the tar. func getContextFromURL(out io.Writer, remoteURL, dockerfileName string) (io.ReadCloser, string, error) { response, err := httputils.Download(remoteURL) if err != nil { return nil, "", fmt.Errorf("unable to download remote context %s: %v", remoteURL, err) } progressOutput := streamformatter.NewStreamFormatter().NewProgressOutput(out, true) // Pass the response body through a progress reader. progReader := progress.NewProgressReader(response.Body, progressOutput, response.ContentLength, "", fmt.Sprintf("Downloading build context from remote url: %s", remoteURL)) return getContextFromReader(ioutils.NewReadCloserWrapper(progReader, func() error { return response.Body.Close() }), dockerfileName) }
func (container *Container) exportContainerRw() (archive.Archive, error) { if container.daemon == nil { return nil, fmt.Errorf("Can't load storage driver for unregistered container %s", container.ID) } archive, err := container.daemon.diff(container) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() return err }), nil }
func (ld *v1LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) { progress.Update(progressOutput, ld.ID(), "Pulling fs layer") layerReader, err := ld.session.GetRemoteImageLayer(ld.v1LayerID, ld.endpoint, ld.layerSize) if err != nil { progress.Update(progressOutput, ld.ID(), "Error pulling dependent layers") if uerr, ok := err.(*url.Error); ok { err = uerr.Err } if terr, ok := err.(net.Error); ok && terr.Timeout() { return nil, 0, err } return nil, 0, xfer.DoNotRetry{Err: err} } *ld.layersDownloaded = true ld.tmpFile, err = ioutil.TempFile("", "GetImageBlob") if err != nil { layerReader.Close() return nil, 0, err } reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerReader), progressOutput, ld.layerSize, ld.ID(), "Downloading") defer reader.Close() _, err = io.Copy(ld.tmpFile, reader) if err != nil { ld.Close() return nil, 0, err } progress.Update(progressOutput, ld.ID(), "Download complete") logrus.Debugf("Downloaded %s to tempfile %s", ld.ID(), ld.tmpFile.Name()) ld.tmpFile.Seek(0, 0) // hand off the temporary file to the download manager, so it will only // be closed once tmpFile := ld.tmpFile ld.tmpFile = nil return ioutils.NewReadCloserWrapper(tmpFile, func() error { tmpFile.Close() err := os.RemoveAll(tmpFile.Name()) if err != nil { logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name()) } return err }), ld.layerSize, nil }
func (daemon *Daemon) exportContainerRw(container *Container) (archive.Archive, error) { if err := daemon.Mount(container); err != nil { return nil, err } archive, err := container.rwlayer.TarStream() if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { return daemon.layerStore.Unmount(container.ID) }), nil }
func (container *Container) exportContainerRw() (archive.Archive, error) { if container.daemon == nil { return nil, derr.ErrorCodeUnregisteredContainer.WithArgs(container.ID) } archive, err := container.daemon.diff(container) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() return err }), nil }
func (fms *fileMetadataStore) TarSplitReader(layer ChainID) (io.ReadCloser, error) { fz, err := os.Open(fms.getLayerFilename(layer, "tar-split.json.gz")) if err != nil { return nil, err } f, err := gzip.NewReader(fz) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(f, func() error { f.Close() return fz.Close() }), nil }
func (daemon *Daemon) exportContainerRw(container *container.Container) (archive.Archive, error) { if err := daemon.Mount(container); err != nil { return nil, err } archive, err := container.RWLayer.TarStream() if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { archive.Close() return container.RWLayer.Unmount() }), nil }
func (d *Daemon) queryRootDir() (string, error) { // update daemon root by asking /info endpoint (to support user // namespaced daemon with root remapped uid.gid directory) var ( conn net.Conn err error ) if d.useDefaultHost { conn, err = net.Dial("unix", "/var/run/docker.sock") } else { conn, err = net.Dial("unix", filepath.Join(d.folder, "docker.sock")) } if err != nil { return "", err } client := httputil.NewClientConn(conn, nil) req, err := http.NewRequest("GET", "/info", nil) if err != nil { client.Close() return "", err } req.Header.Set("Content-Type", "application/json") resp, err := client.Do(req) if err != nil { client.Close() return "", err } body := ioutils.NewReadCloserWrapper(resp.Body, func() error { defer client.Close() return resp.Body.Close() }) type Info struct { DockerRootDir string } var b []byte var i Info b, err = readBody(body) if err == nil && resp.StatusCode == 200 { // read the docker root dir if err = json.Unmarshal(b, &i); err == nil { return i.DockerRootDir, nil } } return "", err }
// drainBody dump the body, it reads the body data into memory and // see go sources /go/src/net/http/httputil/dump.go func drainBody(body io.ReadCloser) ([]byte, io.ReadCloser, error) { bufReader := bufio.NewReaderSize(body, maxBodySize) newBody := ioutils.NewReadCloserWrapper(bufReader, func() error { return body.Close() }) data, err := bufReader.Peek(maxBodySize) if err != io.EOF { // This means the request is larger than our max if err == bufio.ErrBufferFull { return nil, newBody, nil } // This means we had an error reading return nil, nil, err } return data, newBody, nil }
func (daemon *Daemon) exportContainerRw(container *container.Container) (io.ReadCloser, error) { if err := daemon.Mount(container); err != nil { return nil, err } archive, err := container.RWLayer.TarStream() if err != nil { daemon.Unmount(container) // logging is already handled in the `Unmount` function return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { archive.Close() return container.RWLayer.Unmount() }), nil }
// exportLayer generates an archive from a layer based on the given ID. func (d *Driver) exportLayer(id string, parentLayerPaths []string) (archive.Archive, error) { if hcsshim.IsTP4() { // Export in TP4 format to maintain compatibility with existing images and // because ExportLayer is somewhat broken on TP4 and can't work with the new // scheme. tempFolder, err := ioutil.TempDir("", "hcs") if err != nil { return nil, err } defer func() { if err != nil { os.RemoveAll(tempFolder) } }() if err = hcsshim.ExportLayer(d.info, id, tempFolder, parentLayerPaths); err != nil { return nil, err } archive, err := archive.Tar(tempFolder, archive.Uncompressed) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() os.RemoveAll(tempFolder) return err }), nil } var r hcsshim.LayerReader r, err := hcsshim.NewLayerReader(d.info, id, parentLayerPaths) if err != nil { return nil, err } archive, w := io.Pipe() go func() { err := writeTarFromLayer(r, w) cerr := r.Close() if err == nil { err = cerr } w.CloseWithError(err) }() return archive, nil }
func (container *Container) Export() (archive.Archive, error) { if err := container.Mount(); err != nil { return nil, err } archive, err := archive.Tar(container.basefs, archive.Uncompressed) if err != nil { container.Unmount() return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() container.Unmount() return err }), nil }