func (devices *DeviceSet) deactivateDevice(hash string) error { utils.Debugf("[devmapper] deactivateDevice(%s)", hash) defer utils.Debugf("[devmapper] deactivateDevice END") var devname string // FIXME: shouldn't we just register the pool into devices? devname, err := devices.byHash(hash) if err != nil { return err } devinfo, err := getInfo(devname) if err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } if devinfo.Exists != 0 { if err := removeDevice(devname); err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } if err := devices.waitRemove(hash); err != nil { return err } } return nil }
func (devices *DeviceSet) AddDevice(hash, baseHash string) error { devices.Lock() defer devices.Unlock() if devices.Devices[hash] != nil { return fmt.Errorf("hash %s already exists", hash) } baseInfo := devices.Devices[baseHash] if baseInfo == nil { return fmt.Errorf("Error adding device for '%s': can't find device for parent '%s'", hash, baseHash) } deviceId := devices.allocateDeviceId() if err := devices.createSnapDevice(devices.getPoolDevName(), deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil { utils.Debugf("Error creating snap device: %s\n", err) return err } if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size); err != nil { deleteDevice(devices.getPoolDevName(), deviceId) utils.Debugf("Error registering device: %s\n", err) return err } return nil }
func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr, clientKey *connTrackKey) { defer func() { proxy.connTrackLock.Lock() delete(proxy.connTrackTable, *clientKey) proxy.connTrackLock.Unlock() utils.Debugf("Done proxying between udp/%v and udp/%v", clientAddr.String(), proxy.backendAddr.String()) proxyConn.Close() }() readBuf := make([]byte, UDPBufSize) for { proxyConn.SetReadDeadline(time.Now().Add(UDPConnTrackTimeout)) again: read, err := proxyConn.Read(readBuf) if err != nil { if err, ok := err.(*net.OpError); ok && err.Err == syscall.ECONNREFUSED { // This will happen if the last write failed // (e.g: nothing is actually listening on the // proxied port on the container), ignore it // and continue until UDPConnTrackTimeout // expires: goto again } return } for i := 0; i != read; { written, err := proxy.listener.WriteToUDP(readBuf[i:read], clientAddr) if err != nil { return } i += written utils.Debugf("Forwarded %v/%v bytes to udp/%v", i, read, clientAddr.String()) } } }
func spawnGlobalDaemon() { if globalRuntime != nil { utils.Debugf("Global runtime already exists. Skipping.") return } t := log.New(os.Stderr, "", 0) eng := NewTestEngine(t) globalEngine = eng globalRuntime = mkRuntimeFromEngine(eng, t) // Spawn a Daemon go func() { utils.Debugf("Spawning global daemon for integration tests") listenURL := &url.URL{ Scheme: testDaemonProto, Host: testDaemonAddr, } job := eng.Job("serveapi", listenURL.String()) job.SetenvBool("Logging", true) if err := job.Run(); err != nil { log.Fatalf("Unable to spawn the test daemon: %s", err) } }() // Give some time to ListenAndServer to actually start // FIXME: use inmem transports instead of tcp time.Sleep(time.Second) }
func (container *Container) setupPty() error { ptyMaster, ptySlave, err := pty.Open() if err != nil { return err } container.ptyMaster = ptyMaster container.command.Stdout = ptySlave container.command.Stderr = ptySlave // Copy the PTYs to our broadcasters go func() { defer container.stdout.CloseWriters() utils.Debugf("startPty: begin of stdout pipe") io.Copy(container.stdout, ptyMaster) utils.Debugf("startPty: end of stdout pipe") }() // stdin if container.Config.OpenStdin { container.command.Stdin = ptySlave container.command.SysProcAttr.Setctty = true go func() { defer container.stdin.Close() utils.Debugf("startPty: begin of stdin pipe") io.Copy(ptyMaster, container.stdin) utils.Debugf("startPty: end of stdin pipe") }() } return nil }
// waitRemove blocks until either: // a) the device registered at <device_set_prefix>-<hash> is removed, // or b) the 1 second timeout expires. func (devices *DeviceSet) waitRemove(hash string) error { utils.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, hash) defer utils.Debugf("[deviceset %s] waitRemove(%) END", devices.devicePrefix, hash) devname, err := devices.byHash(hash) if err != nil { return err } i := 0 for ; i < 1000; i += 1 { devinfo, err := getInfo(devname) if err != nil { // If there is an error we assume the device doesn't exist. // The error might actually be something else, but we can't differentiate. return nil } if i%100 == 0 { utils.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists) } if devinfo.Exists == 0 { break } time.Sleep(1 * time.Millisecond) } if i == 1000 { return fmt.Errorf("Timeout while waiting for device %s to be removed", devname) } return nil }
func makeHttpHandler(eng *engine.Engine, logging bool, localMethod string, localRoute string, handlerFunc HttpApiFunc, enableCors bool, dockerVersion string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // log the request utils.Debugf("Calling %s %s", localMethod, localRoute) if logging { log.Println(r.Method, r.RequestURI) } if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") { userAgent := strings.Split(r.Header.Get("User-Agent"), "/") if len(userAgent) == 2 && userAgent[1] != dockerVersion { utils.Debugf("Warning: client and server don't have the same version (client: %s, server: %s)", userAgent[1], dockerVersion) } } version, err := strconv.ParseFloat(mux.Vars(r)["version"], 64) if err != nil { version = APIVERSION } if enableCors { writeCorsHeaders(w, r) } if version == 0 || version > APIVERSION { http.Error(w, fmt.Errorf("client and server don't have same version (client : %g, server: %g)", version, APIVERSION).Error(), http.StatusNotFound) return } if err := handlerFunc(eng, version, w, r, mux.Vars(r)); err != nil { utils.Errorf("Error: %s", err) httpError(w, err) } } }
func (devices *DeviceSet) UnmountDevice(hash, path string, deactivate bool) error { utils.Debugf("[devmapper] UnmountDevice(hash=%s path=%s)", hash, path) defer utils.Debugf("[devmapper] UnmountDevice END") devices.Lock() defer devices.Unlock() utils.Debugf("[devmapper] Unmount(%s)", path) if err := sysUnmount(path, 0); err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } utils.Debugf("[devmapper] Unmount done") // Wait for the unmount to be effective, // by watching the value of Info.OpenCount for the device if err := devices.waitClose(hash); err != nil { return err } if count := devices.activeMounts[path]; count > 1 { devices.activeMounts[path] = count - 1 } else { delete(devices.activeMounts, path) } if deactivate { devices.deactivateDevice(hash) } return nil }
func removeDevice(name string) error { utils.Debugf("[devmapper] removeDevice START") defer utils.Debugf("[devmapper] removeDevice END") task, err := createTask(DeviceRemove, name) if task == nil { return err } if err = task.Run(); err != nil { return fmt.Errorf("Error running removeDevice") } return nil }
func (devices *DeviceSet) setupBaseImage() error { oldInfo := devices.Devices[""] if oldInfo != nil && oldInfo.Initialized { return nil } if oldInfo != nil && !oldInfo.Initialized { utils.Debugf("Removing uninitialized base image") if err := devices.removeDevice(""); err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } } utils.Debugf("Initializing base device-manager snapshot") id := devices.allocateDeviceId() // Create initial device if err := createDevice(devices.getPoolDevName(), id); err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } utils.Debugf("Registering base device (id %v) with FS size %v", id, DefaultBaseFsSize) info, err := devices.registerDevice(id, "", DefaultBaseFsSize) if err != nil { _ = deleteDevice(devices.getPoolDevName(), id) utils.Debugf("\n--->Err: %s\n", err) return err } utils.Debugf("Creating filesystem on base device-manager snapshot") if err = devices.activateDeviceIfNeeded(""); err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } if err := devices.createFilesystem(info); err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } info.Initialized = true if err = devices.saveMetadata(); err != nil { info.Initialized = false utils.Debugf("\n--->Err: %s\n", err) return err } return nil }
func (proxy *TCPProxy) Run() { quit := make(chan bool) defer close(quit) utils.Debugf("Starting proxy on tcp/%v for tcp/%v", proxy.frontendAddr, proxy.backendAddr) for { client, err := proxy.listener.Accept() if err != nil { utils.Debugf("Stopping proxy on tcp/%v for tcp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) return } go proxy.clientLoop(client.(*net.TCPConn), quit) } }
// Tar creates an archive from the directory at `path`, only including files whose relative // paths are included in `filter`. If `filter` is nil, then all files are included. func TarFilter(srcPath string, options *TarOptions) (io.Reader, error) { pipeReader, pipeWriter := io.Pipe() compressWriter, err := CompressStream(pipeWriter, options.Compression) if err != nil { return nil, err } tw := tar.NewWriter(compressWriter) go func() { // In general we log errors here but ignore them because // during e.g. a diff operation the container can continue // mutating the filesystem and we can see transient errors // from this if options.Includes == nil { options.Includes = []string{"."} } for _, include := range options.Includes { filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error { if err != nil { utils.Debugf("Tar: Can't stat file %s to tar: %s\n", srcPath, err) return nil } relFilePath, err := filepath.Rel(srcPath, filePath) if err != nil { return nil } if err := addTarFile(filePath, relFilePath, tw); err != nil { utils.Debugf("Can't add file %s to tar: %s\n", srcPath, err) } return nil }) } // Make sure to check the error on Close. if err := tw.Close(); err != nil { utils.Debugf("Can't close tar writer: %s\n", err) } if err := compressWriter.Close(); err != nil { utils.Debugf("Can't close compress writer: %s\n", err) } }() return pipeReader, nil }
// probeCache checks to see if image-caching is enabled (`b.utilizeCache`) // and if so attempts to look up the current `b.image` and `b.config` pair // in the current server `b.srv`. If an image is found, probeCache returns // `(true, nil)`. If no image is found, it returns `(false, nil)`. If there // is any error, it returns `(false, err)`. func (b *buildFile) probeCache() (bool, error) { if b.utilizeCache { if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { return false, err } else if cache != nil { fmt.Fprintf(b.outStream, " ---> Using cache\n") utils.Debugf("[BUILDER] Use cached version") b.image = cache.ID return true, nil } else { utils.Debugf("[BUILDER] Cache miss") } } return false, nil }
func (b *buildFile) addContext(container *Container, orig, dest string) error { var ( origPath = path.Join(b.contextPath, orig) destPath = path.Join(container.BasefsPath(), dest) ) // Preserve the trailing '/' if strings.HasSuffix(dest, "/") { destPath = destPath + "/" } fi, err := os.Stat(origPath) if err != nil { if os.IsNotExist(err) { return fmt.Errorf("%s: no such file or directory", orig) } return err } if fi.IsDir() { if err := archive.CopyWithTar(origPath, destPath); err != nil { return err } // First try to unpack the source as an archive } else if err := archive.UntarPath(origPath, destPath); err != nil { utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err) // If that fails, just copy it as a regular file if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil { return err } if err := archive.CopyWithTar(origPath, destPath); err != nil { return err } } return nil }
// Destroy unregisters a container from the runtime and cleanly removes its contents from the filesystem. func (runtime *Runtime) Destroy(container *Container) error { if container == nil { return fmt.Errorf("The given container is <nil>") } element := runtime.getContainerElement(container.ID) if element == nil { return fmt.Errorf("Container %v not found - maybe it was already destroyed?", container.ID) } if err := container.Stop(3); err != nil { return err } if err := runtime.driver.Remove(container.ID); err != nil { return fmt.Errorf("Driver %s failed to remove root filesystem %s: %s", runtime.driver, container.ID, err) } initID := fmt.Sprintf("%s-init", container.ID) if err := runtime.driver.Remove(initID); err != nil { return fmt.Errorf("Driver %s failed to remove init filesystem %s: %s", runtime.driver, initID, err) } if _, err := runtime.containerGraph.Purge(container.ID); err != nil { utils.Debugf("Unable to remove container from link graph: %s", err) } // Deregister the container before removing its directory, to avoid race conditions runtime.idIndex.Delete(container.ID) runtime.containers.Remove(element) if err := os.RemoveAll(container.root); err != nil { return fmt.Errorf("Unable to remove filesystem for %v: %v", container.ID, err) } return nil }
func UdevWait(cookie uint) error { if res := DmUdevWait(cookie); res != 1 { utils.Debugf("Failed to wait on udev cookie %d", cookie) return ErrUdevWait } return nil }
func New(root string) (driver Driver, err error) { for _, name := range []string{os.Getenv("DOCKER_DRIVER"), DefaultDriver} { if name != "" { return GetDriver(name, root) } } // Check for priority drivers first for _, name := range priority { if driver, err = GetDriver(name, root); err != nil { utils.Debugf("Error loading driver %s: %s", name, err) continue } return driver, nil } // Check all registered drivers if no priority driver is found for _, initFunc := range drivers { if driver, err = initFunc(root); err != nil { continue } return driver, nil } return nil, err }
func (devices *DeviceSet) log(level int, file string, line int, dmError int, message string) { if level >= 7 { return // Ignore _LOG_DEBUG } utils.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) }
func SetDevDir(dir string) error { if res := DmSetDevDir(dir); res != 1 { utils.Debugf("Error dm_set_dev_dir") return ErrSetDevDir } return nil }
// Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) if err != nil { return nil, err } setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err } defer res.Body.Close() if res.StatusCode != 200 { if res.StatusCode == 401 { return nil, ErrLoginRequired } return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } jsonString, err := ioutil.ReadAll(res.Body) if err != nil { return nil, fmt.Errorf("Error while reading the http response: %s", err) } utils.Debugf("Ancestry: %s", jsonString) history := new([]string) if err := json.Unmarshal(jsonString, history); err != nil { return nil, err } return *history, nil }
// ensureImage creates a sparse file of <size> bytes at the path // <root>/devicemapper/<name>. // If the file already exists, it does nothing. // Either way it returns the full path. func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) { dirname := devices.loopbackDir() filename := path.Join(dirname, name) if err := osMkdirAll(dirname, 0700); err != nil && !osIsExist(err) { return "", err } if _, err := osStat(filename); err != nil { if !osIsNotExist(err) { return "", err } utils.Debugf("Creating loopback file %s for device-manage use", filename) file, err := osOpenFile(filename, osORdWr|osOCreate, 0600) if err != nil { return "", err } defer file.Close() if err = file.Truncate(size); err != nil { return "", err } } return filename, nil }
func (b *buildFile) CmdRun(args string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to run") } config, _, _, err := ParseRun(append([]string{b.image}, b.buildCmdFromJson(args)...), nil) if err != nil { return err } cmd := b.config.Cmd b.config.Cmd = nil MergeConfig(b.config, config) defer func(cmd []string) { b.config.Cmd = cmd }(cmd) utils.Debugf("Command to be executed: %v", b.config.Cmd) hit, err := b.probeCache() if err != nil { return err } if hit { return nil } cid, err := b.run() if err != nil { return err } if err := b.commit(cid, cmd, "run"); err != nil { return err } return nil }
func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { utils.Debugf("Index server: %s", r.indexEndpoint) u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err } if r.authConfig != nil && len(r.authConfig.Username) > 0 { req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") res, err := r.client.Do(req) if err != nil { return nil, err } defer res.Body.Close() if res.StatusCode != 200 { return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) } rawData, err := ioutil.ReadAll(res.Body) if err != nil { return nil, err } result := new(SearchResults) err = json.Unmarshal(rawData, result) return result, err }
// Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { errBody = []byte(err.Error()) } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) } return nil }
func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) if err != nil { return err } setTokenAuth(req, token) req.Header.Set("X-Docker-Checksum", imgData.Checksum) res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() if len(res.Cookies()) > 0 { r.client.Jar.SetCookies(req.URL, res.Cookies()) } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { errBody = []byte(err.Error()) } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) } return nil }
func DetectCompression(source []byte) Compression { sourceLen := len(source) for compression, m := range map[Compression][]byte{ Bzip2: {0x42, 0x5A, 0x68}, Gzip: {0x1F, 0x8B, 0x08}, Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, } { fail := false if len(m) > sourceLen { utils.Debugf("Len too short") continue } i := 0 for _, b := range m { if b != source[i] { fail = true break } i++ } if !fail { return compression } } return Uncompressed }
func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, err error) { utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") tarsumLayer := &utils.TarSum{Reader: layer} req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { return "", err } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) } if rc, ok := layer.(io.Closer); ok { if err := rc.Close(); err != nil { return "", err } } defer res.Body.Close() if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { return "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } return "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } return tarsumLayer.Sum(jsonRaw), nil }
// CopyWithTar creates a tar archive of filesystem path `src`, and // unpacks it at filesystem path `dst`. // The archive is streamed directly with fixed buffering and no // intermediary disk IO. // func CopyWithTar(src, dst string) error { srcSt, err := os.Stat(src) if err != nil { return err } if !srcSt.IsDir() { return CopyFileWithTar(src, dst) } // Create dst, copy src's content into it utils.Debugf("Creating dest directory: %s", dst) if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) { return err } utils.Debugf("Calling TarUntar(%s, %s)", src, dst) return TarUntar(src, dst) }
func NewRegistry(authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, } r = &Registry{ authConfig: authConfig, client: &http.Client{ Transport: httpTransport, }, indexEndpoint: indexEndpoint, } r.client.Jar, err = cookiejar.New(nil) if err != nil { return nil, err } // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. if indexEndpoint != auth.IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { standalone, err := pingRegistryEndpoint(indexEndpoint) if err != nil { return nil, err } if standalone { utils.Debugf("Endpoint %s is eligible for private registry auth. Enabling decorator.", indexEndpoint) dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) } } r.reqFactory = factory return r, nil }
func DecompressStream(archive io.Reader) (io.Reader, error) { buf := make([]byte, 10) totalN := 0 for totalN < 10 { n, err := archive.Read(buf[totalN:]) if err != nil { if err == io.EOF { return nil, fmt.Errorf("Tarball too short") } return nil, err } totalN += n utils.Debugf("[tar autodetect] n: %d", n) } compression := DetectCompression(buf) wrap := io.MultiReader(bytes.NewReader(buf), archive) switch compression { case Uncompressed: return wrap, nil case Gzip: return gzip.NewReader(wrap) case Bzip2: return bzip2.NewReader(wrap), nil case Xz: return xzDecompress(wrap) default: return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) } }