func (devices *DeviceSet) AddDevice(hash, baseHash string) error {
baseInfo, err := devices.lookupDevice(baseHash)
if err != nil {
return err
}
baseInfo.lock.Lock()
defer baseInfo.lock.Unlock()
devices.Lock()
defer devices.Unlock()
if info, _ := devices.lookupDevice(hash); info != nil {
return fmt.Errorf("device %s already exists", hash)
}
deviceId := devices.nextDeviceId
if err := createSnapDevice(devices.getPoolDevName(), &deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil {
utils.Debugf("Error creating snap device: %s\n", err)
return err
}
// Ids are 24bit, so wrap around
devices.nextDeviceId = (deviceId + 1) & 0xffffff
if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size); err != nil {
deleteDevice(devices.getPoolDevName(), deviceId)
utils.Debugf("Error registering device: %s\n", err)
return err
}
return nil
}
func spawnGlobalDaemon() {
if globalDaemon != nil {
utils.Debugf("Global daemon already exists. Skipping.")
return
}
t := log.New(os.Stderr, "", 0)
eng := NewTestEngine(t)
globalEngine = eng
globalDaemon = mkDaemonFromEngine(eng, t)
// Spawn a Daemon
go func() {
utils.Debugf("Spawning global daemon for integration tests")
listenURL := &url.URL{
Scheme: testDaemonProto,
Host: testDaemonAddr,
}
job := eng.Job("serveapi", listenURL.String())
job.SetenvBool("Logging", true)
if err := job.Run(); err != nil {
log.Fatalf("Unable to spawn the test daemon: %s", err)
}
}()
// Give some time to ListenAndServer to actually start
// FIXME: use inmem transports instead of tcp
time.Sleep(time.Second)
if err := eng.Job("acceptconnections").Run(); err != nil {
log.Fatalf("Unable to accept connections for test api: %s", err)
}
}
func makeHttpHandler(eng *engine.Engine, logging bool, localMethod string, localRoute string, handlerFunc HttpApiFunc, enableCors bool, dockerVersion version.Version) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
// log the request
utils.Debugf("Calling %s %s", localMethod, localRoute)
if logging {
log.Println(r.Method, r.RequestURI)
}
if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") {
userAgent := strings.Split(r.Header.Get("User-Agent"), "/")
if len(userAgent) == 2 && !dockerVersion.Equal(version.Version(userAgent[1])) {
utils.Debugf("Warning: client and server don't have the same version (client: %s, server: %s)", userAgent[1], dockerVersion)
}
}
version := version.Version(mux.Vars(r)["version"])
if version == "" {
version = api.APIVERSION
}
if enableCors {
writeCorsHeaders(w, r)
}
if version.GreaterThan(api.APIVERSION) {
http.Error(w, fmt.Errorf("client and server don't have same version (client : %s, server: %s)", version, api.APIVERSION).Error(), http.StatusNotFound)
return
}
if err := handlerFunc(eng, version, w, r, mux.Vars(r)); err != nil {
utils.Errorf("Handler for %s %s returned error: %s", localMethod, localRoute, err)
httpError(w, err)
}
}
}
func (s *TagStore) recursiveLoad(eng *engine.Engine, address, tmpImageDir string) error {
if err := eng.Job("image_get", address).Run(); err != nil {
utils.Debugf("Loading %s", address)
imageJson, err := ioutil.ReadFile(path.Join(tmpImageDir, "repo", address, "json"))
if err != nil {
utils.Debugf("Error reading json", err)
return err
}
layer, err := os.Open(path.Join(tmpImageDir, "repo", address, "layer.tar"))
if err != nil {
utils.Debugf("Error reading embedded tar", err)
return err
}
img, err := image.NewImgJSON(imageJson)
if err != nil {
utils.Debugf("Error unmarshalling json", err)
return err
}
if img.Parent != "" {
if !s.graph.Exists(img.Parent) {
if err := s.recursiveLoad(eng, img.Parent, tmpImageDir); err != nil {
return err
}
}
}
if err := s.graph.Register(imageJson, layer, img); err != nil {
return err
}
}
utils.Debugf("Completed processing %s", address)
return nil
}
// waitRemove blocks until either:
// a) the device registered at <device_set_prefix>-<hash> is removed,
// or b) the 10 second timeout expires.
func (devices *DeviceSet) waitRemove(devname string) error {
utils.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, devname)
defer utils.Debugf("[deviceset %s] waitRemove(%s) END", devices.devicePrefix, devname)
i := 0
for ; i < 1000; i += 1 {
devinfo, err := getInfo(devname)
if err != nil {
// If there is an error we assume the device doesn't exist.
// The error might actually be something else, but we can't differentiate.
return nil
}
if i%100 == 0 {
utils.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists)
}
if devinfo.Exists == 0 {
break
}
devices.Unlock()
time.Sleep(10 * time.Millisecond)
devices.Lock()
}
if i == 1000 {
return fmt.Errorf("Timeout while waiting for device %s to be removed", devname)
}
return nil
}
func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) {
if endpoint == IndexServerAddress() {
// Skip the check, we now this one is valid
// (and we never want to fallback to http in case of error)
return RegistryInfo{Standalone: false}, nil
}
req, err := http.NewRequest("GET", endpoint+"_ping", nil)
if err != nil {
return RegistryInfo{Standalone: false}, err
}
resp, _, err := doRequest(req, nil, ConnectTimeout)
if err != nil {
return RegistryInfo{Standalone: false}, err
}
defer resp.Body.Close()
jsonString, err := ioutil.ReadAll(resp.Body)
if err != nil {
return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err)
}
// If the header is absent, we assume true for compatibility with earlier
// versions of the registry. default to true
info := RegistryInfo{
Standalone: true,
}
if err := json.Unmarshal(jsonString, &info); err != nil {
utils.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err)
// don't stop here. Just assume sane defaults
}
if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" {
utils.Debugf("Registry version header: '%s'", hdr)
info.Version = hdr
}
utils.Debugf("RegistryInfo.Version: %q", info.Version)
standalone := resp.Header.Get("X-Docker-Registry-Standalone")
utils.Debugf("Registry standalone header: '%s'", standalone)
// Accepted values are "true" (case-insensitive) and "1".
if strings.EqualFold(standalone, "true") || standalone == "1" {
info.Standalone = true
} else if len(standalone) > 0 {
// there is a header set, and it is not "true" or "1", so assume fails
info.Standalone = false
}
utils.Debugf("RegistryInfo.Standalone: %q", info.Standalone)
return info, nil
}
func (devices *DeviceSet) setupBaseImage() error {
oldInfo, _ := devices.lookupDevice("")
if oldInfo != nil && oldInfo.Initialized {
return nil
}
if oldInfo != nil && !oldInfo.Initialized {
utils.Debugf("Removing uninitialized base image")
if err := devices.deleteDevice(oldInfo); err != nil {
return err
}
}
utils.Debugf("Initializing base device-manager snapshot")
id := devices.nextDeviceId
// Create initial device
if err := createDevice(devices.getPoolDevName(), &id); err != nil {
return err
}
// Ids are 24bit, so wrap around
devices.nextDeviceId = (id + 1) & 0xffffff
utils.Debugf("Registering base device (id %v) with FS size %v", id, devices.baseFsSize)
info, err := devices.registerDevice(id, "", devices.baseFsSize)
if err != nil {
_ = deleteDevice(devices.getPoolDevName(), id)
return err
}
utils.Debugf("Creating filesystem on base device-manager snapshot")
if err = devices.activateDeviceIfNeeded(info); err != nil {
return err
}
if err := devices.createFilesystem(info); err != nil {
return err
}
info.Initialized = true
if err = devices.saveMetadata(info); err != nil {
info.Initialized = false
return err
}
return nil
}
func (devices *DeviceSet) deactivatePool() error {
utils.Debugf("[devmapper] deactivatePool()")
defer utils.Debugf("[devmapper] deactivatePool END")
devname := devices.getPoolDevName()
devinfo, err := getInfo(devname)
if err != nil {
return err
}
if devinfo.Exists != 0 {
return removeDevice(devname)
}
return nil
}
// probeCache checks to see if image-caching is enabled (`b.utilizeCache`)
// and if so attempts to look up the current `b.image` and `b.config` pair
// in the current server `b.daemon`. If an image is found, probeCache returns
// `(true, nil)`. If no image is found, it returns `(false, nil)`. If there
// is any error, it returns `(false, err)`.
func (b *buildFile) probeCache() (bool, error) {
if b.utilizeCache {
if cache, err := b.daemon.ImageGetCached(b.image, b.config); err != nil {
return false, err
} else if cache != nil {
fmt.Fprintf(b.outStream, " ---> Using cache\n")
utils.Debugf("[BUILDER] Use cached version")
b.image = cache.ID
return true, nil
} else {
utils.Debugf("[BUILDER] Cache miss")
}
}
return false, nil
}
func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Registry, err error) {
r = &Registry{
authConfig: authConfig,
indexEndpoint: indexEndpoint,
}
if timeout {
r.timeout = ReceiveTimeout
}
r.jar, err = cookiejar.New(nil)
if err != nil {
return nil, err
}
// If we're working with a standalone private registry over HTTPS, send Basic Auth headers
// alongside our requests.
if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") {
info, err := pingRegistryEndpoint(indexEndpoint)
if err != nil {
return nil, err
}
if info.Standalone {
utils.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint)
dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password)
factory.AddDecorator(dec)
}
}
r.reqFactory = factory
return r, nil
}
func handlerAccessLog(handler http.Handler) http.Handler {
logHandler := func(w http.ResponseWriter, r *http.Request) {
utils.Debugf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL)
handler.ServeHTTP(w, r)
}
return http.HandlerFunc(logHandler)
}
func (devices *DeviceSet) registerDevice(id int, hash string, size uint64) (*DevInfo, error) {
utils.Debugf("registerDevice(%v, %v)", id, hash)
info := &DevInfo{
Hash: hash,
DeviceId: id,
Size: size,
TransactionId: devices.allocateTransactionId(),
Initialized: false,
devices: devices,
}
devices.devicesLock.Lock()
devices.Devices[hash] = info
devices.devicesLock.Unlock()
if err := devices.saveMetadata(info); err != nil {
// Try to remove unused device
devices.devicesLock.Lock()
delete(devices.Devices, hash)
devices.devicesLock.Unlock()
return nil, err
}
return info, nil
}
// ensureImage creates a sparse file of <size> bytes at the path
// <root>/devicemapper/<name>.
// If the file already exists, it does nothing.
// Either way it returns the full path.
func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
dirname := devices.loopbackDir()
filename := path.Join(dirname, name)
if err := os.MkdirAll(dirname, 0700); err != nil && !os.IsExist(err) {
return "", err
}
if _, err := os.Stat(filename); err != nil {
if !os.IsNotExist(err) {
return "", err
}
utils.Debugf("Creating loopback file %s for device-manage use", filename)
file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
if err != nil {
return "", err
}
defer file.Close()
if err = file.Truncate(size); err != nil {
return "", err
}
}
return filename, nil
}
func removeDevice(name string) error {
utils.Debugf("[devmapper] removeDevice START")
defer utils.Debugf("[devmapper] removeDevice END")
task, err := createTask(DeviceRemove, name)
if task == nil {
return err
}
dmSawBusy = false
if err = task.Run(); err != nil {
if dmSawBusy {
return ErrBusy
}
return fmt.Errorf("Error running removeDevice %s", err)
}
return nil
}
func createDevice(poolName string, deviceId *int) error {
utils.Debugf("[devmapper] createDevice(poolName=%v, deviceId=%v)", poolName, *deviceId)
for {
task, err := createTask(DeviceTargetMsg, poolName)
if task == nil {
return err
}
if err := task.SetSector(0); err != nil {
return fmt.Errorf("Can't set sector %s", err)
}
if err := task.SetMessage(fmt.Sprintf("create_thin %d", *deviceId)); err != nil {
return fmt.Errorf("Can't set message %s", err)
}
dmSawExist = false
if err := task.Run(); err != nil {
if dmSawExist {
// Already exists, try next id
*deviceId++
continue
}
return fmt.Errorf("Error running createDevice %s", err)
}
break
}
return nil
}
func (container *Container) waitForStart() error {
callback := func(command *execdriver.Command) {
if command.Tty {
// The callback is called after the process Start()
// so we are in the parent process. In TTY mode, stdin/out/err is the PtySlace
// which we close here.
if c, ok := command.Stdout.(io.Closer); ok {
c.Close()
}
}
container.State.SetRunning(command.Pid())
if err := container.ToDisk(); err != nil {
utils.Debugf("%s", err)
}
}
// We use a callback here instead of a goroutine and an chan for
// syncronization purposes
cErr := utils.Go(func() error { return container.monitor(callback) })
waitStart := make(chan struct{})
go func() {
container.State.WaitRunning(-1 * time.Second)
close(waitStart)
}()
// Start should not return until the process is actually running
select {
case <-waitStart:
case err := <-cErr:
return err
}
return nil
}
func UdevWait(cookie uint) error {
if res := DmUdevWait(cookie); res != 1 {
utils.Debugf("Failed to wait on udev cookie %d", cookie)
return ErrUdevWait
}
return nil
}
// Retrieve the history of a given image from the Registry.
// Return a list of the parent's json (requested image included)
func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) {
req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil)
if err != nil {
return nil, err
}
setTokenAuth(req, token)
res, _, err := r.doRequest(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != 200 {
if res.StatusCode == 401 {
return nil, errLoginRequired
}
return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res)
}
jsonString, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, fmt.Errorf("Error while reading the http response: %s", err)
}
utils.Debugf("Ancestry: %s", jsonString)
history := new([]string)
if err := json.Unmarshal(jsonString, history); err != nil {
return nil, err
}
return *history, nil
}
func (devices *DeviceSet) log(level int, file string, line int, dmError int, message string) {
if level >= 7 {
return // Ignore _LOG_DEBUG
}
utils.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
}
func (r *Registry) SearchRepositories(term string) (*SearchResults, error) {
utils.Debugf("Index server: %s", r.indexEndpoint)
u := r.indexEndpoint + "search?q=" + url.QueryEscape(term)
req, err := r.reqFactory.NewRequest("GET", u, nil)
if err != nil {
return nil, err
}
if r.authConfig != nil && len(r.authConfig.Username) > 0 {
req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password)
}
req.Header.Set("X-Docker-Token", "true")
res, _, err := r.doRequest(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res)
}
rawData, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
result := new(SearchResults)
err = json.Unmarshal(rawData, result)
return result, err
}
// Push a local image to the registry
func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error {
utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json")
req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw))
if err != nil {
return err
}
req.Header.Add("Content-type", "application/json")
setTokenAuth(req, token)
res, _, err := r.doRequest(req)
if err != nil {
return fmt.Errorf("Failed to upload metadata: %s", err)
}
defer res.Body.Close()
if res.StatusCode == 401 && strings.HasPrefix(registry, "http://") {
return utils.NewHTTPRequestError("HTTP code 401, Docker will not send auth headers over HTTP.", res)
}
if res.StatusCode != 200 {
errBody, err := ioutil.ReadAll(res.Body)
if err != nil {
return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res)
}
var jsonBody map[string]string
if err := json.Unmarshal(errBody, &jsonBody); err != nil {
errBody = []byte(err.Error())
} else if jsonBody["error"] == "Image already exists" {
return ErrAlreadyExists
}
return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res)
}
return nil
}
func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error {
utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum")
req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil)
if err != nil {
return err
}
setTokenAuth(req, token)
req.Header.Set("X-Docker-Checksum", imgData.Checksum)
req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload)
res, _, err := r.doRequest(req)
if err != nil {
return fmt.Errorf("Failed to upload metadata: %s", err)
}
defer res.Body.Close()
if len(res.Cookies()) > 0 {
r.jar.SetCookies(req.URL, res.Cookies())
}
if res.StatusCode != 200 {
errBody, err := ioutil.ReadAll(res.Body)
if err != nil {
return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err)
}
var jsonBody map[string]string
if err := json.Unmarshal(errBody, &jsonBody); err != nil {
errBody = []byte(err.Error())
} else if jsonBody["error"] == "Image already exists" {
return ErrAlreadyExists
}
return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody)
}
return nil
}
// CopyWithTar creates a tar archive of filesystem path `src`, and
// unpacks it at filesystem path `dst`.
// The archive is streamed directly with fixed buffering and no
// intermediary disk IO.
//
func CopyWithTar(src, dst string) error {
srcSt, err := os.Stat(src)
if err != nil {
return err
}
if !srcSt.IsDir() {
return CopyFileWithTar(src, dst)
}
// Create dst, copy src's content into it
utils.Debugf("Creating dest directory: %s", dst)
if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) {
return err
}
utils.Debugf("Calling TarUntar(%s, %s)", src, dst)
return TarUntar(src, dst)
}
func SetDevDir(dir string) error {
if res := DmSetDevDir(dir); res != 1 {
utils.Debugf("Error dm_set_dev_dir")
return ErrSetDevDir
}
return nil
}
func (b *buildFile) buildCmdFromJson(args string) []string {
var cmd []string
if err := json.Unmarshal([]byte(args), &cmd); err != nil {
utils.Debugf("Error unmarshalling: %s, setting to /bin/sh -c", err)
cmd = []string{"/bin/sh", "-c", args}
}
return cmd
}
func createBridgeIface(name string) error {
kv, err := kernel.GetKernelVersion()
// only set the bridge's mac address if the kernel version is > 3.3
// before that it was not supported
setBridgeMacAddr := err == nil && (kv.Kernel >= 3 && kv.Major >= 3)
utils.Debugf("setting bridge mac address = %v", setBridgeMacAddr)
return netlink.CreateBridge(name, setBridgeMacAddr)
}
// Retrieve the all the images to be uploaded in the correct order
func (s *TagStore) getImageList(localRepo map[string]string, requestedTag string) ([]string, map[string][]string, error) {
var (
imageList []string
imagesSeen map[string]bool = make(map[string]bool)
tagsByImage map[string][]string = make(map[string][]string)
)
for tag, id := range localRepo {
if requestedTag != "" && requestedTag != tag {
continue
}
var imageListForThisTag []string
tagsByImage[id] = append(tagsByImage[id], tag)
for img, err := s.graph.Get(id); img != nil; img, err = img.GetParent() {
if err != nil {
return nil, nil, err
}
if imagesSeen[img.ID] {
// This image is already on the list, we can ignore it and all its parents
break
}
imagesSeen[img.ID] = true
imageListForThisTag = append(imageListForThisTag, img.ID)
}
// reverse the image list for this tag (so the "most"-parent image is first)
for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 {
imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i]
}
// append to main image list
imageList = append(imageList, imageListForThisTag...)
}
if len(imageList) == 0 {
return nil, nil, fmt.Errorf("No images found for the requested repository / tag")
}
utils.Debugf("Image list: %v", imageList)
utils.Debugf("Tags by image: %v", tagsByImage)
return imageList, tagsByImage, nil
}
func (container *Container) DisableLink(name string) {
if container.activeLinks != nil {
if link, exists := container.activeLinks[name]; exists {
link.Disable()
} else {
utils.Debugf("Could not find active link for %s", name)
}
}
}
func changeGroup(addr string, nameOrGid string) error {
gid, err := lookupGidByName(nameOrGid)
if err != nil {
return err
}
utils.Debugf("%s group found. gid: %d", nameOrGid, gid)
return os.Chown(addr, 0, gid)
}
func (devices *DeviceSet) activateDeviceIfNeeded(info *DevInfo) error {
utils.Debugf("activateDeviceIfNeeded(%v)", info.Hash)
if devinfo, _ := getInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
return nil
}
return activateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size)
}