func spawnGlobalDaemon() { if globalDaemon != nil { log.Debugf("Global daemon already exists. Skipping.") return } t := std_log.New(os.Stderr, "", 0) eng := NewTestEngine(t) globalEngine = eng globalDaemon = mkDaemonFromEngine(eng, t) // Spawn a Daemon go func() { log.Debugf("Spawning global daemon for integration tests") listenURL := &url.URL{ Scheme: testDaemonProto, Host: testDaemonAddr, } job := eng.Job("serveapi", listenURL.String()) job.SetenvBool("Logging", true) if err := job.Run(); err != nil { log.Fatalf("Unable to spawn the test daemon: %s", err) } }() // Give some time to ListenAndServer to actually start // FIXME: use inmem transports instead of tcp time.Sleep(time.Second) if err := eng.Job("acceptconnections").Run(); err != nil { log.Fatalf("Unable to accept connections for test api: %s", err) } }
func (s *TagStore) recursiveLoad(eng *engine.Engine, address, tmpImageDir string) error { if err := eng.Job("image_get", address).Run(); err != nil { log.Debugf("Loading %s", address) imageJson, err := ioutil.ReadFile(path.Join(tmpImageDir, "repo", address, "json")) if err != nil { log.Debugf("Error reading json", err) return err } layer, err := os.Open(path.Join(tmpImageDir, "repo", address, "layer.tar")) if err != nil { log.Debugf("Error reading embedded tar", err) return err } img, err := image.NewImgJSON(imageJson) if err != nil { log.Debugf("Error unmarshalling json", err) return err } if img.Parent != "" { if !s.graph.Exists(img.Parent) { if err := s.recursiveLoad(eng, img.Parent, tmpImageDir); err != nil { return err } } } if err := s.graph.Register(imageJson, layer, img); err != nil { return err } } log.Debugf("Completed processing %s", address) return nil }
func (devices *DeviceSet) AddDevice(hash, baseHash string) error { baseInfo, err := devices.lookupDevice(baseHash) if err != nil { return err } baseInfo.lock.Lock() defer baseInfo.lock.Unlock() devices.Lock() defer devices.Unlock() if info, _ := devices.lookupDevice(hash); info != nil { return fmt.Errorf("device %s already exists", hash) } deviceId := devices.nextDeviceId if err := createSnapDevice(devices.getPoolDevName(), &deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil { log.Debugf("Error creating snap device: %s", err) return err } // Ids are 24bit, so wrap around devices.nextDeviceId = (deviceId + 1) & 0xffffff if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size); err != nil { deleteDevice(devices.getPoolDevName(), deviceId) log.Debugf("Error registering device: %s", err) return err } return nil }
// waitRemove blocks until either: // a) the device registered at <device_set_prefix>-<hash> is removed, // or b) the 10 second timeout expires. func (devices *DeviceSet) waitRemove(devname string) error { log.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, devname) defer log.Debugf("[deviceset %s] waitRemove(%s) END", devices.devicePrefix, devname) i := 0 for ; i < 1000; i += 1 { devinfo, err := getInfo(devname) if err != nil { // If there is an error we assume the device doesn't exist. // The error might actually be something else, but we can't differentiate. return nil } if i%100 == 0 { log.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists) } if devinfo.Exists == 0 { break } devices.Unlock() time.Sleep(10 * time.Millisecond) devices.Lock() } if i == 1000 { return fmt.Errorf("Timeout while waiting for device %s to be removed", devname) } return nil }
func makeHttpHandler(eng *engine.Engine, logging bool, localMethod string, localRoute string, handlerFunc HttpApiFunc, enableCors bool, dockerVersion version.Version) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // log the request log.Debugf("Calling %s %s", localMethod, localRoute) if logging { log.Infof("%s %s", r.Method, r.RequestURI) } if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") { userAgent := strings.Split(r.Header.Get("User-Agent"), "/") if len(userAgent) == 2 && !dockerVersion.Equal(version.Version(userAgent[1])) { log.Debugf("Warning: client and server don't have the same version (client: %s, server: %s)", userAgent[1], dockerVersion) } } version := version.Version(mux.Vars(r)["version"]) if version == "" { version = api.APIVERSION } if enableCors { writeCorsHeaders(w, r) } if version.GreaterThan(api.APIVERSION) { http.Error(w, fmt.Errorf("client and server don't have same version (client : %s, server: %s)", version, api.APIVERSION).Error(), http.StatusNotFound) return } if err := handlerFunc(eng, version, w, r, mux.Vars(r)); err != nil { log.Errorf("Handler for %s %s returned error: %s", localMethod, localRoute, err) httpError(w, err) } } }
func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { var ( retries = 5 client *http.Client res *http.Response imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) req, err := r.reqFactory.NewRequest("GET", imageURL, nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } setTokenAuth(req, token) for i := 1; i <= retries; i++ { res, client, err = r.doRequest(req) if err != nil { res.Body.Close() if i == retries { return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", res.StatusCode, imgID) } time.Sleep(time.Duration(i) * 5 * time.Second) continue } break } if res.StatusCode != 200 { res.Body.Close() return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", res.StatusCode, imgID) } // Check if encrypted. if res.Header.Get("X-Docker-Encrypt") == "True" { key, err := utils.GetDefaultKey() if err != nil { return nil, fmt.Errorf("Server error: Status %d while getting default key(%s): %v", res.StatusCode, imgID, err) } res.Body, err = utils.AesReadCloserWrapper(key, res.Body) if err != nil { return nil, fmt.Errorf("Server error: Status %d while creating AES read closer(%s): %v", res.StatusCode, imgID, err) } } if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { log.Debugf("server supports resume") return httputils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil } log.Debugf("server doesn't support resume") return res.Body, nil }
func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { if endpoint == IndexServerAddress() { // Skip the check, we now this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil } req, err := http.NewRequest("GET", endpoint+"_ping", nil) if err != nil { return RegistryInfo{Standalone: false}, err } resp, _, err := doRequest(req, nil, ConnectTimeout) if err != nil { return RegistryInfo{Standalone: false}, err } defer resp.Body.Close() jsonString, err := ioutil.ReadAll(resp.Body) if err != nil { return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) } // If the header is absent, we assume true for compatibility with earlier // versions of the registry. default to true info := RegistryInfo{ Standalone: true, } if err := json.Unmarshal(jsonString, &info); err != nil { log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) // don't stop here. Just assume sane defaults } if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { log.Debugf("Registry version header: '%s'", hdr) info.Version = hdr } log.Debugf("RegistryInfo.Version: %q", info.Version) standalone := resp.Header.Get("X-Docker-Registry-Standalone") log.Debugf("Registry standalone header: '%s'", standalone) // Accepted values are "true" (case-insensitive) and "1". if strings.EqualFold(standalone, "true") || standalone == "1" { info.Standalone = true } else if len(standalone) > 0 { // there is a header set, and it is not "true" or "1", so assume fails info.Standalone = false } log.Debugf("RegistryInfo.Standalone: %q", info.Standalone) return info, nil }
func (devices *DeviceSet) setupBaseImage() error { oldInfo, _ := devices.lookupDevice("") if oldInfo != nil && oldInfo.Initialized { return nil } if oldInfo != nil && !oldInfo.Initialized { log.Debugf("Removing uninitialized base image") if err := devices.deleteDevice(oldInfo); err != nil { return err } } log.Debugf("Initializing base device-manager snapshot") id := devices.nextDeviceId // Create initial device if err := createDevice(devices.getPoolDevName(), &id); err != nil { return err } // Ids are 24bit, so wrap around devices.nextDeviceId = (id + 1) & 0xffffff log.Debugf("Registering base device (id %v) with FS size %v", id, devices.baseFsSize) info, err := devices.registerDevice(id, "", devices.baseFsSize) if err != nil { _ = deleteDevice(devices.getPoolDevName(), id) return err } log.Debugf("Creating filesystem on base device-manager snapshot") if err = devices.activateDeviceIfNeeded(info); err != nil { return err } if err := devices.createFilesystem(info); err != nil { return err } info.Initialized = true if err = devices.saveMetadata(info); err != nil { info.Initialized = false return err } return nil }
func (devices *DeviceSet) deactivatePool() error { log.Debugf("[devmapper] deactivatePool()") defer log.Debugf("[devmapper] deactivatePool END") devname := devices.getPoolDevName() devinfo, err := getInfo(devname) if err != nil { return err } if devinfo.Exists != 0 { return removeDevice(devname) } return nil }
// probeCache checks to see if image-caching is enabled (`b.utilizeCache`) // and if so attempts to look up the current `b.image` and `b.config` pair // in the current server `b.daemon`. If an image is found, probeCache returns // `(true, nil)`. If no image is found, it returns `(false, nil)`. If there // is any error, it returns `(false, err)`. func (b *buildFile) probeCache() (bool, error) { if b.utilizeCache { if cache, err := b.daemon.ImageGetCached(b.image, b.config); err != nil { return false, err } else if cache != nil { fmt.Fprintf(b.outStream, " ---> Using cache\n") log.Debugf("[BUILDER] Use cached version") b.image = cache.ID return true, nil } else { log.Debugf("[BUILDER] Cache miss") } } return false, nil }
func createDevice(poolName string, deviceId *int) error { log.Debugf("[devmapper] createDevice(poolName=%v, deviceId=%v)", poolName, *deviceId) for { task, err := createTask(DeviceTargetMsg, poolName) if task == nil { return err } if err := task.SetSector(0); err != nil { return fmt.Errorf("Can't set sector %s", err) } if err := task.SetMessage(fmt.Sprintf("create_thin %d", *deviceId)); err != nil { return fmt.Errorf("Can't set message %s", err) } dmSawExist = false if err := task.Run(); err != nil { if dmSawExist { // Already exists, try next id *deviceId++ continue } return fmt.Errorf("Error running createDevice %s", err) } break } return nil }
// NewRequest() creates a new *http.Request, // applies all decorators in the HTTPRequestFactory on the request, // then applies decorators provided by d on the request. func (h *HTTPRequestFactory) NewRequest(method, urlStr string, body io.Reader, d ...HTTPRequestDecorator) (*http.Request, error) { req, err := http.NewRequest(method, urlStr, body) if err != nil { return nil, err } // By default, a nil factory should work. if h == nil { return req, nil } for _, dec := range h.decorators { req, err = dec.ChangeRequest(req) if err != nil { return nil, err } } for _, dec := range d { req, err = dec.ChangeRequest(req) if err != nil { return nil, err } } log.Debugf("%v -- HEADERS: %v", req.URL, req.Header) return req, err }
func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) if err != nil { return err } setTokenAuth(req, token) req.Header.Set("X-Docker-Checksum", imgData.Checksum) req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) res, _, err := r.doRequest(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() if len(res.Cookies()) > 0 { r.jar.SetCookies(req.URL, res.Cookies()) } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { errBody = []byte(err.Error()) } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) } return nil }
func (devices *DeviceSet) log(level int, file string, line int, dmError int, message string) { if level >= 7 { return // Ignore _LOG_DEBUG } log.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) }
// ensureImage creates a sparse file of <size> bytes at the path // <root>/devicemapper/<name>. // If the file already exists, it does nothing. // Either way it returns the full path. func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) { dirname := devices.loopbackDir() filename := path.Join(dirname, name) if err := os.MkdirAll(dirname, 0700); err != nil && !os.IsExist(err) { return "", err } if _, err := os.Stat(filename); err != nil { if !os.IsNotExist(err) { return "", err } log.Debugf("Creating loopback file %s for device-manage use", filename) file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) if err != nil { return "", err } defer file.Close() if err = file.Truncate(size); err != nil { return "", err } } return filename, nil }
func (devices *DeviceSet) registerDevice(id int, hash string, size uint64) (*DevInfo, error) { log.Debugf("registerDevice(%v, %v)", id, hash) info := &DevInfo{ Hash: hash, DeviceId: id, Size: size, TransactionId: devices.allocateTransactionId(), Initialized: false, devices: devices, } devices.devicesLock.Lock() devices.Devices[hash] = info devices.devicesLock.Unlock() if err := devices.saveMetadata(info); err != nil { // Try to remove unused device devices.devicesLock.Lock() delete(devices.Devices, hash) devices.devicesLock.Unlock() return nil, err } return info, nil }
// CopyWithTar creates a tar archive of filesystem path `src`, and // unpacks it at filesystem path `dst`. // The archive is streamed directly with fixed buffering and no // intermediary disk IO. // func CopyWithTar(src, dst string) error { srcSt, err := os.Stat(src) if err != nil { return err } if !srcSt.IsDir() { return CopyFileWithTar(src, dst) } // Create dst, copy src's content into it log.Debugf("Creating dest directory: %s", dst) if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) { return err } log.Debugf("Calling TarUntar(%s, %s)", src, dst) return TarUntar(src, dst) }
func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Session, err error) { r = &Session{ authConfig: authConfig, indexEndpoint: indexEndpoint, } if timeout { r.timeout = ReceiveTimeout } r.jar, err = cookiejar.New(nil) if err != nil { return nil, err } // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { info, err := pingRegistryEndpoint(indexEndpoint) if err != nil { return nil, err } if info.Standalone { log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) } } r.reqFactory = factory return r, nil }
func UdevWait(cookie uint) error { if res := DmUdevWait(cookie); res != 1 { log.Debugf("Failed to wait on udev cookie %d", cookie) return ErrUdevWait } return nil }
func SetDevDir(dir string) error { if res := DmSetDevDir(dir); res != 1 { log.Debugf("Error dm_set_dev_dir") return ErrSetDevDir } return nil }
func handlerAccessLog(handler http.Handler) http.Handler { logHandler := func(w http.ResponseWriter, r *http.Request) { log.Debugf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) handler.ServeHTTP(w, r) } return http.HandlerFunc(logHandler) }
func removeDevice(name string) error { log.Debugf("[devmapper] removeDevice START") defer log.Debugf("[devmapper] removeDevice END") task, err := createTask(DeviceRemove, name) if task == nil { return err } dmSawBusy = false if err = task.Run(); err != nil { if dmSawBusy { return ErrBusy } return fmt.Errorf("Error running removeDevice %s", err) } return nil }
// Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Session) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) if err != nil { return nil, err } setTokenAuth(req, token) res, _, err := r.doRequest(req) if err != nil { return nil, err } defer res.Body.Close() if res.StatusCode != 200 { if res.StatusCode == 401 { return nil, errLoginRequired } return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } jsonString, err := ioutil.ReadAll(res.Body) if err != nil { return nil, fmt.Errorf("Error while reading the http response: %s", err) } log.Debugf("Ancestry: %s", jsonString) history := new([]string) if err := json.Unmarshal(jsonString, history); err != nil { return nil, err } return *history, nil }
func (r *Session) SearchRepositories(term string) (*SearchResults, error) { log.Debugf("Index server: %s", r.indexEndpoint) u := r.indexEndpoint + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err } if r.authConfig != nil && len(r.authConfig.Username) > 0 { req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") res, _, err := r.doRequest(req) if err != nil { return nil, err } defer res.Body.Close() if res.StatusCode != 200 { return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) } rawData, err := ioutil.ReadAll(res.Body) if err != nil { return nil, err } result := new(SearchResults) err = json.Unmarshal(rawData, result) return result, err }
// Push a local image to the registry func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) res, _, err := r.doRequest(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() if res.StatusCode == 401 && strings.HasPrefix(registry, "http://") { return utils.NewHTTPRequestError("HTTP code 401, Docker will not send auth headers over HTTP.", res) } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { errBody = []byte(err.Error()) } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) } return nil }
func (container *Container) KillSig(sig int) error { log.Debugf("Sending %d to %s", sig, container.ID) container.Lock() defer container.Unlock() // We could unpause the container for them rather than returning this error if container.State.IsPaused() { return fmt.Errorf("Container %s is paused. Unpause the container before stopping", container.ID) } if !container.State.IsRunning() { return nil } // signal to the monitor that it should not restart the container // after we send the kill signal container.monitor.ExitOnNext() // if the container is currently restarting we do not need to send the signal // to the process. Telling the monitor that it should exit on it's next event // loop is enough if container.State.IsRestarting() { return nil } return container.daemon.Kill(container, sig) }
func (b *buildFile) buildCmdFromJson(args string) []string { var cmd []string if err := json.Unmarshal([]byte(args), &cmd); err != nil { log.Debugf("Error unmarshalling: %s, setting to /bin/sh -c", err) cmd = []string{"/bin/sh", "-c", args} } return cmd }
func createBridgeIface(name string) error { kv, err := kernel.GetKernelVersion() // only set the bridge's mac address if the kernel version is > 3.3 // before that it was not supported setBridgeMacAddr := err == nil && (kv.Kernel >= 3 && kv.Major >= 3) log.Debugf("setting bridge mac address = %v", setBridgeMacAddr) return netlink.CreateBridge(name, setBridgeMacAddr) }
// Retrieve the all the images to be uploaded in the correct order func (s *TagStore) getImageList(localRepo map[string]string, requestedTag string) ([]string, map[string][]string, error) { var ( imageList []string imagesSeen map[string]bool = make(map[string]bool) tagsByImage map[string][]string = make(map[string][]string) ) for tag, id := range localRepo { if requestedTag != "" && requestedTag != tag { continue } var imageListForThisTag []string tagsByImage[id] = append(tagsByImage[id], tag) for img, err := s.graph.Get(id); img != nil; img, err = img.GetParent() { if err != nil { return nil, nil, err } if imagesSeen[img.ID] { // This image is already on the list, we can ignore it and all its parents break } imagesSeen[img.ID] = true imageListForThisTag = append(imageListForThisTag, img.ID) } // reverse the image list for this tag (so the "most"-parent image is first) for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 { imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i] } // append to main image list imageList = append(imageList, imageListForThisTag...) } if len(imageList) == 0 { return nil, nil, fmt.Errorf("No images found for the requested repository / tag") } log.Debugf("Image list: %v", imageList) log.Debugf("Tags by image: %v", tagsByImage) return imageList, tagsByImage, nil }
// TarUntar is a convenience function which calls Tar and Untar, with // the output of one piped into the other. If either Tar or Untar fails, // TarUntar aborts and returns the error. func TarUntar(src string, dst string) error { log.Debugf("TarUntar(%s %s)", src, dst) archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) if err != nil { return err } defer archive.Close() return Untar(archive, dst, nil) }