// FIXME: Do more extensive tests (ex: create multiple, delete, recreate;
// create multiple, check the amount of images and paths, etc..)
func TestGraphCreate(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
archive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
img, err := graph.Create(archive, "", "", "Testing", "", nil, nil)
if err != nil {
t.Fatal(err)
}
if err := image.ValidateID(img.ID); err != nil {
t.Fatal(err)
}
if img.Comment != "Testing" {
t.Fatalf("Wrong comment: should be '%s', not '%s'", "Testing", img.Comment)
}
if img.DockerVersion != dockerversion.VERSION {
t.Fatalf("Wrong docker_version: should be '%s', not '%s'", dockerversion.VERSION, img.DockerVersion)
}
images := graph.Map()
if l := len(images); l != 1 {
t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l)
}
if images[img.ID] == nil {
t.Fatalf("Could not find image with id %s", img.ID)
}
}
func validateRemoteName(remoteName string) error {
var (
namespace string
name string
)
nameParts := strings.SplitN(remoteName, "/", 2)
if len(nameParts) < 2 {
namespace = "library"
name = nameParts[0]
// the repository name must not be a valid image ID
if err := image.ValidateID(name); err == nil {
return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name)
}
} else {
namespace = nameParts[0]
name = nameParts[1]
}
if !validNamespaceChars.MatchString(namespace) {
return fmt.Errorf("Invalid namespace name (%s). Only [a-z0-9-_] are allowed.", namespace)
}
if len(namespace) < 2 || len(namespace) > 255 {
return fmt.Errorf("Invalid namespace name (%s). Cannot be fewer than 2 or more than 255 characters.", namespace)
}
if strings.HasPrefix(namespace, "-") || strings.HasSuffix(namespace, "-") {
return fmt.Errorf("Invalid namespace name (%s). Cannot begin or end with a hyphen.", namespace)
}
if strings.Contains(namespace, "--") {
return fmt.Errorf("Invalid namespace name (%s). Cannot contain consecutive hyphens.", namespace)
}
if !validRepo.MatchString(name) {
return fmt.Errorf("Invalid repository name (%s), only [a-z0-9-_.] are allowed", name)
}
return nil
}
// Register imports a pre-existing image into the graph.
func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) (err error) {
if err := image.ValidateID(img.ID); err != nil {
return err
}
// We need this entire operation to be atomic within the engine. Note that
// this doesn't mean Register is fully safe yet.
graph.imageMutex.Lock(img.ID)
defer graph.imageMutex.Unlock(img.ID)
defer func() {
// If any error occurs, remove the new dir from the driver.
// Don't check for errors since the dir might not have been created.
// FIXME: this leaves a possible race condition.
if err != nil {
graph.driver.Remove(img.ID)
}
}()
// (This is a convenience to save time. Race conditions are taken care of by os.Rename)
if graph.Exists(img.ID) {
return fmt.Errorf("Image %s already exists", img.ID)
}
// Ensure that the image root does not exist on the filesystem
// when it is not registered in the graph.
// This is common when you switch from one graph driver to another
if err := os.RemoveAll(graph.ImageRoot(img.ID)); err != nil && !os.IsNotExist(err) {
return err
}
// If the driver has this ID but the graph doesn't, remove it from the driver to start fresh.
// (the graph is the source of truth).
// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
// (FIXME: make that mandatory for drivers).
graph.driver.Remove(img.ID)
tmp, err := graph.Mktemp("")
defer os.RemoveAll(tmp)
if err != nil {
return fmt.Errorf("Mktemp failed: %s", err)
}
// Create root filesystem in the driver
if err := graph.driver.Create(img.ID, img.Parent); err != nil {
return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err)
}
// Apply the diff/layer
img.SetGraph(graph)
if err := image.StoreImage(img, layerData, tmp); err != nil {
return err
}
// Commit
if err := os.Rename(tmp, graph.ImageRoot(img.ID)); err != nil {
return err
}
graph.idIndex.Add(img.ID)
return nil
}
// loadImage fetches the image with the given id from the graph.
func (graph *Graph) loadImage(id string) (*image.Image, error) {
root := graph.imageRoot(id)
// Open the JSON file to decode by streaming
jsonSource, err := os.Open(jsonPath(root))
if err != nil {
return nil, err
}
defer jsonSource.Close()
img := &image.Image{}
dec := json.NewDecoder(jsonSource)
// Decode the JSON data
if err := dec.Decode(img); err != nil {
return nil, err
}
if img.ID == "" {
img.ID = id
}
if img.Parent == "" && img.ParentID != "" && img.ParentID.Validate() == nil {
img.Parent = img.ParentID.Hex()
}
// compatibilityID for parent
parent, err := ioutil.ReadFile(filepath.Join(root, parentFileName))
if err == nil && len(parent) > 0 {
img.Parent = string(parent)
}
if err := image.ValidateID(img.ID); err != nil {
return nil, err
}
if buf, err := ioutil.ReadFile(filepath.Join(root, layersizeFileName)); err != nil {
if !os.IsNotExist(err) {
return nil, err
}
// If the layersize file does not exist then set the size to a negative number
// because a layer size of 0 (zero) is valid
img.Size = -1
} else {
// Using Atoi here instead would temporarily convert the size to a machine
// dependent integer type, which causes images larger than 2^31 bytes to
// display negative sizes on 32-bit machines:
size, err := strconv.ParseInt(string(buf), 10, 64)
if err != nil {
return nil, err
}
img.Size = int64(size)
}
return img, nil
}
func validateRemoteName(remoteName string) error {
if !strings.Contains(remoteName, "/") {
// the repository name must not be a valid image ID
if err := image.ValidateID(remoteName); err == nil {
return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", remoteName)
}
}
return v2.ValidateRepositoryName(remoteName)
}
func (s *TagStore) recursiveLoad(address, tmpImageDir string) error {
if _, err := s.LookupImage(address); err != nil {
logrus.Debugf("Loading %s", address)
imageJson, err := ioutil.ReadFile(filepath.Join(tmpImageDir, "repo", address, "json"))
if err != nil {
logrus.Debugf("Error reading json: %v", err)
return err
}
layer, err := os.Open(filepath.Join(tmpImageDir, "repo", address, "layer.tar"))
if err != nil {
logrus.Debugf("Error reading embedded tar: %v", err)
return err
}
img, err := image.NewImgJSON(imageJson)
if err != nil {
logrus.Debugf("Error unmarshalling json: %v", err)
return err
}
if err := image.ValidateID(img.ID); err != nil {
logrus.Debugf("Error validating ID: %v", err)
return err
}
// ensure no two downloads of the same layer happen at the same time
if c, err := s.poolAdd("pull", "layer:"+img.ID); err != nil {
if c != nil {
logrus.Debugf("Image (id: %s) load is already running, waiting: %v", img.ID, err)
<-c
return nil
}
return err
}
defer s.poolRemove("pull", "layer:"+img.ID)
if img.Parent != "" {
if !s.graph.Exists(img.Parent) {
if err := s.recursiveLoad(img.Parent, tmpImageDir); err != nil {
return err
}
}
}
if err := s.graph.Register(img, layer); err != nil {
return err
}
}
logrus.Debugf("Completed processing %s", address)
return nil
}
// Register imports a pre-existing image into the graph.
// Returns nil if the image is already registered.
func (graph *Graph) Register(im image.ImageDescriptor, layerData archive.ArchiveReader) (err error) {
imgID := im.ID()
if err := image.ValidateID(imgID); err != nil {
return err
}
// We need this entire operation to be atomic within the engine. Note that
// this doesn't mean Register is fully safe yet.
graph.imageMutex.Lock(imgID)
defer graph.imageMutex.Unlock(imgID)
return graph.register(im, layerData)
}
// Register imports a pre-existing image into the graph.
// Returns nil if the image is already registered.
func (graph *Graph) Register(im image.Descriptor, layerData io.Reader) (err error) {
imgID := im.ID()
if err := image.ValidateID(imgID); err != nil {
return err
}
// this is needed cause pull_v2 attemptIDReuse could deadlock
graph.imagesMutex.Lock()
defer graph.imagesMutex.Unlock()
// We need this entire operation to be atomic within the engine. Note that
// this doesn't mean Register is fully safe yet.
graph.imageMutex.Lock(imgID)
defer graph.imageMutex.Unlock(imgID)
return graph.register(im, layerData)
}
// fixManifestLayers removes repeated layers from the manifest and checks the
// correctness of the parent chain.
func fixManifestLayers(m *schema1.Manifest) error {
images := make([]*image.Image, len(m.FSLayers))
for i := range m.FSLayers {
img, err := image.NewImgJSON([]byte(m.History[i].V1Compatibility))
if err != nil {
return err
}
images[i] = img
if err := image.ValidateID(img.ID); err != nil {
return err
}
}
if images[len(images)-1].Parent != "" && !allowBaseParentImage {
// Windows base layer can point to a base layer parent that is not in manifest.
return errors.New("Invalid parent ID in the base layer of the image.")
}
// check general duplicates to error instead of a deadlock
idmap := make(map[string]struct{})
var lastID string
for _, img := range images {
// skip IDs that appear after each other, we handle those later
if _, exists := idmap[img.ID]; img.ID != lastID && exists {
return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
}
lastID = img.ID
idmap[lastID] = struct{}{}
}
// backwards loop so that we keep the remaining indexes after removing items
for i := len(images) - 2; i >= 0; i-- {
if images[i].ID == images[i+1].ID { // repeated ID. remove and continue
m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
m.History = append(m.History[:i], m.History[i+1:]...)
} else if images[i].Parent != images[i+1].ID {
return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", images[i+1].ID, images[i].Parent)
}
}
return nil
}
// Register imports a pre-existing image into the graph.
// Returns nil if the image is already registered.
func (graph *Graph) Register(img *image.Image, layerData io.Reader) (err error) {
if err := image.ValidateID(img.ID); err != nil {
return err
}
// We need this entire operation to be atomic within the engine. Note that
// this doesn't mean Register is fully safe yet.
graph.imageMutex.Lock(img.ID)
defer graph.imageMutex.Unlock(img.ID)
// Skip register if image is already registered
if graph.Exists(img.ID) {
return nil
}
// The returned `error` must be named in this function's signature so that
// `err` is not shadowed in this deferred cleanup.
defer func() {
// If any error occurs, remove the new dir from the driver.
// Don't check for errors since the dir might not have been created.
if err != nil {
graph.driver.Remove(img.ID)
}
}()
// Ensure that the image root does not exist on the filesystem
// when it is not registered in the graph.
// This is common when you switch from one graph driver to another
if err := os.RemoveAll(graph.imageRoot(img.ID)); err != nil && !os.IsNotExist(err) {
return err
}
// If the driver has this ID but the graph doesn't, remove it from the driver to start fresh.
// (the graph is the source of truth).
// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
// (FIXME: make that mandatory for drivers).
graph.driver.Remove(img.ID)
tmp, err := graph.mktemp("")
defer os.RemoveAll(tmp)
if err != nil {
return fmt.Errorf("mktemp failed: %s", err)
}
// Create root filesystem in the driver
if err := createRootFilesystemInDriver(graph, img, layerData); err != nil {
return err
}
// Apply the diff/layer
if err := graph.storeImage(img, layerData, tmp); err != nil {
return err
}
// Commit
if err := os.Rename(tmp, graph.imageRoot(img.ID)); err != nil {
return err
}
graph.idIndex.Add(img.ID)
return nil
}
func (p *v1Puller) pullRepository(askedTag string) error {
out := p.config.OutStream
out.Write(p.sf.FormatStatus("", "Pulling repository %s", p.repoInfo.CanonicalName))
repoData, err := p.session.GetRepositoryData(p.repoInfo.RemoteName)
if err != nil {
if strings.Contains(err.Error(), "HTTP code: 404") {
return fmt.Errorf("Error: image %s not found", utils.ImageReference(p.repoInfo.RemoteName, askedTag))
}
// Unexpected HTTP error
return err
}
logrus.Debugf("Retrieving the tag list")
tagsList := make(map[string]string)
if askedTag == "" {
tagsList, err = p.session.GetRemoteTags(repoData.Endpoints, p.repoInfo.RemoteName)
} else {
var tagID string
tagID, err = p.session.GetRemoteTag(repoData.Endpoints, p.repoInfo.RemoteName, askedTag)
tagsList[askedTag] = tagID
}
if err != nil {
if err == registry.ErrRepoNotFound && askedTag != "" {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName)
}
logrus.Errorf("unable to get remote tags: %s", err)
return err
}
for tag, id := range tagsList {
repoData.ImgList[id] = ®istry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
logrus.Debugf("Registering tags")
// If no tag has been specified, pull them all
if askedTag == "" {
for tag, id := range tagsList {
repoData.ImgList[id].Tag = tag
}
} else {
// Otherwise, check that the tag exists and use only that one
id, exists := tagsList[askedTag]
if !exists {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName)
}
repoData.ImgList[id].Tag = askedTag
}
errors := make(chan error)
layersDownloaded := false
imgIDs := []string{}
sessionID := p.session.ID()
defer func() {
p.graph.Release(sessionID, imgIDs...)
}()
for _, imgData := range repoData.ImgList {
downloadImage := func(img *registry.ImgData) {
if askedTag != "" && img.Tag != askedTag {
errors <- nil
return
}
if img.Tag == "" {
logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
errors <- nil
return
}
if err := image.ValidateID(img.ID); err != nil {
errors <- err
return
}
// ensure no two downloads of the same image happen at the same time
poolKey := "img:" + img.ID
broadcaster, found := p.poolAdd("pull", poolKey)
broadcaster.Add(out)
if found {
errors <- broadcaster.Wait()
return
}
defer p.poolRemove("pull", poolKey)
// we need to retain it until tagging
p.graph.Retain(sessionID, img.ID)
imgIDs = append(imgIDs, img.ID)
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, p.repoInfo.CanonicalName), nil))
success := false
var lastErr, err error
var isDownloaded bool
for _, ep := range p.repoInfo.Index.Mirrors {
ep += "v1/"
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil))
if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil {
// Don't report errors when pulling from mirrors.
logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err)
continue
}
layersDownloaded = layersDownloaded || isDownloaded
success = true
break
}
if !success {
for _, ep := range repoData.Endpoints {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil))
if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil {
// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
// As the error is also given to the output stream the user will see the error.
lastErr = err
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err), nil))
continue
}
layersDownloaded = layersDownloaded || isDownloaded
success = true
break
}
}
if !success {
err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.CanonicalName, lastErr)
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil))
errors <- err
broadcaster.CloseWithError(err)
return
}
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
errors <- nil
}
go downloadImage(imgData)
}
var lastError error
for i := 0; i < len(repoData.ImgList); i++ {
if err := <-errors; err != nil {
lastError = err
}
}
if lastError != nil {
return lastError
}
for tag, id := range tagsList {
if askedTag != "" && tag != askedTag {
continue
}
if err := p.Tag(p.repoInfo.LocalName, tag, id, true); err != nil {
return err
}
}
requestedTag := p.repoInfo.LocalName
if len(askedTag) > 0 {
requestedTag = utils.ImageReference(p.repoInfo.LocalName, askedTag)
}
writeStatus(requestedTag, out, p.sf, layersDownloaded)
return nil
}
func MakePod(podName, image, workdir, src, vol, shellDir string, cmds, entrys []string) (string, error) {
if image == "" {
return "", fmt.Errorf("image can not be null")
}
if err := dockerimage.ValidateID(image); err == nil {
image = image[:12]
}
var (
env = []pod.UserEnvironmentVar{}
containerList = []pod.UserContainer{}
volList = []pod.UserVolume{}
cVols = []pod.UserVolumeReference{}
)
if src != "" {
myVol1 := pod.UserVolumeReference{
Path: "/tmp/src/",
Volume: "source",
ReadOnly: false,
}
myVol2 := pod.UserVolumeReference{
Path: "/tmp/shell/",
Volume: "shell",
ReadOnly: false,
}
cVols = append(cVols, myVol1)
cVols = append(cVols, myVol2)
vol1 := pod.UserVolume{
Name: "source",
Source: src,
Driver: "vfs",
}
vol2 := pod.UserVolume{
Name: "shell",
Source: shellDir,
Driver: "vfs",
}
volList = append(volList, vol1)
volList = append(volList, vol2)
}
var container = pod.UserContainer{
Name: "image-builder",
Image: image,
Command: cmds,
Workdir: workdir,
Entrypoint: entrys,
Ports: []pod.UserContainerPort{},
Envs: env,
Volumes: cVols,
Files: []pod.UserFileReference{},
RestartPolicy: "never",
}
containerList = append(containerList, container)
var userPod = &pod.UserPod{
Name: podName,
Containers: containerList,
Resource: pod.UserResource{Vcpu: 1, Memory: 512},
Files: []pod.UserFile{},
Volumes: volList,
Tty: false,
}
jsonString, err := utils.JSONMarshal(userPod, true)
if err != nil {
return "", err
}
return string(jsonString), nil
}
