func (s *TagStore) recursiveLoad(eng *engine.Engine, address, tmpImageDir string) error {
if err := eng.Job("image_get", address).Run(); err != nil {
log.Debugf("Loading %s", address)
imageJson, err := ioutil.ReadFile(path.Join(tmpImageDir, "repo", address, "json"))
if err != nil {
log.Debugf("Error reading json", err)
return err
}
layer, err := os.Open(path.Join(tmpImageDir, "repo", address, "layer.tar"))
if err != nil {
log.Debugf("Error reading embedded tar", err)
return err
}
img, err := image.NewImgJSON(imageJson)
if err != nil {
log.Debugf("Error unmarshalling json", err)
return err
}
if img.Parent != "" {
if !s.graph.Exists(img.Parent) {
if err := s.recursiveLoad(eng, img.Parent, tmpImageDir); err != nil {
return err
}
}
}
if err := s.graph.Register(imageJson, layer, img); err != nil {
return err
}
}
log.Debugf("Completed processing %s", address)
return nil
}
// CmdSet stores a new image in the graph.
// Images are stored in the graph using 4 elements:
// - A user-defined ID
// - A collection of metadata describing the image
// - A directory tree stored as a tar archive (also called the "layer")
// - A reference to a "parent" ID on top of which the layer should be applied
//
// NOTE: even though the parent ID is only useful in relation to the layer and how
// to apply it (ie you could represent the full directory tree as 'parent_layer + layer',
// it is treated as a top-level property of the image. This is an artifact of early
// design and should probably be cleaned up in the future to simplify the design.
//
// Syntax: image_set ID
// Input:
// - Layer content must be streamed in tar format on stdin. An empty input is
// valid and represents a nil layer.
//
// - Image metadata must be passed in the command environment.
// 'json': a json-encoded object with all image metadata.
// It will be stored as-is, without any encoding/decoding artifacts.
// That is a requirement of the current registry client implementation,
// because a re-encoded json might invalidate the image checksum at
// the next upload, even with functionaly identical content.
func (s *TagStore) CmdSet(job *engine.Job) engine.Status {
if len(job.Args) != 1 {
return job.Errorf("usage: %s NAME", job.Name)
}
var (
imgJSON = []byte(job.Getenv("json"))
layer = job.Stdin
)
if len(imgJSON) == 0 {
return job.Errorf("mandatory key 'json' is not set")
}
// We have to pass an *image.Image object, even though it will be completely
// ignored in favor of the redundant json data.
// FIXME: the current prototype of Graph.Register is stupid and redundant.
img, err := image.NewImgJSON(imgJSON)
if err != nil {
return job.Error(err)
}
if err := s.graph.Register(imgJSON, layer, img); err != nil {
return job.Error(err)
}
return engine.StatusOK
}
func (s *TagStore) pullImage(r *registry.Session, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) error {
history, err := r.GetRemoteHistory(imgID, endpoint, token)
if err != nil {
return err
}
out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling dependent layers", nil))
// FIXME: Try to stream the images?
// FIXME: Launch the getRemoteImage() in goroutines
for i := len(history) - 1; i >= 0; i-- {
id := history[i]
// ensure no two downloads of the same layer happen at the same time
if c, err := s.poolAdd("pull", "layer:"+id); err != nil {
log.Debugf("Image (id: %s) pull is already running, skipping: %v", id, err)
<-c
}
defer s.poolRemove("pull", "layer:"+id)
if !s.graph.Exists(id) {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling metadata", nil))
var (
imgJSON []byte
imgSize int
err error
img *image.Image
)
retries := 5
for j := 1; j <= retries; j++ {
imgJSON, imgSize, err = r.GetRemoteImageJSON(id, endpoint, token)
if err != nil && j == retries {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
return err
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
}
img, err = image.NewImgJSON(imgJSON)
if err != nil && j == retries {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
return fmt.Errorf("Failed to parse json: %s", err)
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else {
break
}
}
for j := 1; j <= retries; j++ {
// Get the layer
status := "Pulling fs layer"
if j > 1 {
status = fmt.Sprintf("Pulling fs layer [retries: %d]", j)
}
out.Write(sf.FormatProgress(utils.TruncateID(id), status, nil))
layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token, int64(imgSize))
if uerr, ok := err.(*url.Error); ok {
err = uerr.Err
}
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
return err
}
defer layer.Close()
err = s.graph.Register(imgJSON,
utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading"),
img)
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error downloading dependent layers", nil))
return err
} else {
break
}
}
}
out.Write(sf.FormatProgress(utils.TruncateID(id), "Download complete", nil))
}
return nil
}