// Set associates an image with a V1 ID.
func (idserv *V1IDService) Set(v1ID, registry string, id layer.DiffID) error {
if idserv.store == nil {
return nil
}
if err := v1.ValidateID(v1ID); err != nil {
return err
}
return idserv.store.Set(idserv.namespace(), registry+","+v1ID, []byte(id))
}
// ParseIDOrReference parses string for an image ID or a reference. ID can be
// without a default prefix.
func ParseIDOrReference(idOrRef string) (digest.Digest, Named, error) {
if err := v1.ValidateID(idOrRef); err == nil {
idOrRef = "sha256:" + idOrRef
}
if dgst, err := digest.ParseDigest(idOrRef); err == nil {
return dgst, nil, nil
}
ref, err := ParseNamed(idOrRef)
return "", ref, err
}
func validateRemoteName(remoteName reference.Named) error {
remoteNameStr := remoteName.Name()
if !strings.Contains(remoteNameStr, "/") {
// the repository name must not be a valid image ID
if err := v1.ValidateID(remoteNameStr); err == nil {
return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", remoteName)
}
}
return nil
}
// Get finds a layer by its V1 ID.
func (idserv *V1IDService) Get(v1ID, registry string) (layer.ChainID, error) {
if err := v1.ValidateID(v1ID); err != nil {
return layer.ChainID(""), err
}
idBytes, err := idserv.store.Get(idserv.namespace(), registry+","+v1ID)
if err != nil {
return layer.ChainID(""), err
}
return layer.ChainID(idBytes), nil
}
func (p *v1Puller) downloadImage(ctx context.Context, repoData *registry.RepositoryData, img *registry.ImgData, layersDownloaded *bool) error {
if img.Tag == "" {
logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
return nil
}
localNameRef, err := reference.WithTag(p.repoInfo.LocalName, img.Tag)
if err != nil {
retErr := fmt.Errorf("Image (id: %s) has invalid tag: %s", img.ID, img.Tag)
logrus.Debug(retErr.Error())
return retErr
}
if err := v1.ValidateID(img.ID); err != nil {
return err
}
progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Pulling image (%s) from %s", img.Tag, p.repoInfo.CanonicalName.Name())
success := false
var lastErr error
for _, ep := range p.repoInfo.Index.Mirrors {
ep += "v1/"
progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep))
if err = p.pullImage(ctx, img.ID, ep, localNameRef, layersDownloaded); err != nil {
// Don't report errors when pulling from mirrors.
logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep, err)
continue
}
success = true
break
}
if !success {
for _, ep := range repoData.Endpoints {
progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep)
if err = p.pullImage(ctx, img.ID, ep, localNameRef, layersDownloaded); err != nil {
// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
// As the error is also given to the output stream the user will see the error.
lastErr = err
progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep, err)
continue
}
success = true
break
}
}
if !success {
err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.CanonicalName.Name(), lastErr)
progress.Update(p.config.ProgressOutput, stringid.TruncateID(img.ID), err.Error())
return err
}
progress.Update(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Download complete")
return nil
}
func migrateImages(root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) error {
graphDir := filepath.Join(root, graphDirName)
if _, err := os.Lstat(graphDir); err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
mfile := filepath.Join(root, migrationFileName)
f, err := os.Open(mfile)
if err != nil && !os.IsNotExist(err) {
return err
} else if err == nil {
err := json.NewDecoder(f).Decode(&mappings)
if err != nil {
f.Close()
return err
}
f.Close()
}
dir, err := ioutil.ReadDir(graphDir)
if err != nil {
return err
}
for _, v := range dir {
v1ID := v.Name()
if err := imagev1.ValidateID(v1ID); err != nil {
continue
}
if _, exists := mappings[v1ID]; exists {
continue
}
if err := migrateImage(v1ID, root, ls, is, ms, mappings); err != nil {
continue
}
}
f, err = os.OpenFile(mfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer f.Close()
if err := json.NewEncoder(f).Encode(mappings); err != nil {
return err
}
return nil
}
// Get finds a layer by its V1 ID.
func (idserv *V1IDService) Get(v1ID, registry string) (layer.DiffID, error) {
if idserv.store == nil {
return "", errors.New("no v1IDService storage")
}
if err := v1.ValidateID(v1ID); err != nil {
return layer.DiffID(""), err
}
idBytes, err := idserv.store.Get(idserv.namespace(), registry+","+v1ID)
if err != nil {
return layer.DiffID(""), err
}
return layer.DiffID(idBytes), nil
}
// fixManifestLayers removes repeated layers from the manifest and checks the
// correctness of the parent chain.
func fixManifestLayers(m *schema1.Manifest) error {
imgs := make([]*image.V1Image, len(m.FSLayers))
for i := range m.FSLayers {
img := &image.V1Image{}
if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), img); err != nil {
return err
}
imgs[i] = img
if err := v1.ValidateID(img.ID); err != nil {
return err
}
}
if imgs[len(imgs)-1].Parent != "" && runtime.GOOS != "windows" {
// Windows base layer can point to a base layer parent that is not in manifest.
return errors.New("Invalid parent ID in the base layer of the image.")
}
// check general duplicates to error instead of a deadlock
idmap := make(map[string]struct{})
var lastID string
for _, img := range imgs {
// skip IDs that appear after each other, we handle those later
if _, exists := idmap[img.ID]; img.ID != lastID && exists {
return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
}
lastID = img.ID
idmap[lastID] = struct{}{}
}
// backwards loop so that we keep the remaining indexes after removing items
for i := len(imgs) - 2; i >= 0; i-- {
if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue
m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
m.History = append(m.History[:i], m.History[i+1:]...)
} else if imgs[i].Parent != imgs[i+1].ID {
return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent)
}
}
return nil
}
// CalculateLayerChecksums walks an old graph directory and calculates checksums
// for each layer. These checksums are later used for migration.
func CalculateLayerChecksums(root string, ls checksumCalculator, mappings map[string]image.ID) {
graphDir := filepath.Join(root, graphDirName)
// spawn some extra workers also for maximum performance because the process is bounded by both cpu and io
workers := runtime.NumCPU() * 3
workQueue := make(chan string, workers)
wg := sync.WaitGroup{}
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
for id := range workQueue {
start := time.Now()
if err := calculateLayerChecksum(graphDir, id, ls); err != nil {
logrus.Errorf("could not calculate checksum for %q, %q", id, err)
}
elapsed := time.Since(start)
logrus.Debugf("layer %s took %.2f seconds", id, elapsed.Seconds())
}
wg.Done()
}()
}
dir, err := ioutil.ReadDir(graphDir)
if err != nil {
logrus.Errorf("could not read directory %q", graphDir)
return
}
for _, v := range dir {
v1ID := v.Name()
if err := imagev1.ValidateID(v1ID); err != nil {
continue
}
if _, ok := mappings[v1ID]; ok { // support old migrations without helper files
continue
}
workQueue <- v1ID
}
close(workQueue)
wg.Wait()
}
func migrateImages(root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) error {
graphDir := filepath.Join(root, graphDirName)
dir, err := ioutil.ReadDir(graphDir)
if err != nil {
return err
}
for _, v := range dir {
v1ID := v.Name()
if err := imagev1.ValidateID(v1ID); err != nil {
continue
}
if _, exists := mappings[v1ID]; exists {
continue
}
if err := migrateImage(v1ID, root, ls, is, ms, mappings); err != nil {
continue
}
}
return nil
}
func validateName(name string) error {
if err := v1.ValidateID(name); err == nil {
return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name)
}
return nil
}
func (p *v1Puller) downloadImage(out io.Writer, repoData *registry.RepositoryData, img *registry.ImgData, layerDownloaded chan struct{}, errors chan error) {
if img.Tag == "" {
logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
return
}
localNameRef, err := reference.WithTag(p.repoInfo.LocalName, img.Tag)
if err != nil {
retErr := fmt.Errorf("Image (id: %s) has invalid tag: %s", img.ID, img.Tag)
logrus.Debug(retErr.Error())
errors <- retErr
}
if err := v1.ValidateID(img.ID); err != nil {
errors <- err
return
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, p.repoInfo.CanonicalName.Name()), nil))
success := false
var lastErr error
var isDownloaded bool
for _, ep := range p.repoInfo.Index.Mirrors {
ep += "v1/"
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep), nil))
if isDownloaded, err = p.pullImage(out, img.ID, ep, localNameRef); err != nil {
// Don't report errors when pulling from mirrors.
logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep, err)
continue
}
if isDownloaded {
layerDownloaded <- struct{}{}
}
success = true
break
}
if !success {
for _, ep := range repoData.Endpoints {
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep), nil))
if isDownloaded, err = p.pullImage(out, img.ID, ep, localNameRef); err != nil {
// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
// As the error is also given to the output stream the user will see the error.
lastErr = err
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep, err), nil))
continue
}
if isDownloaded {
layerDownloaded <- struct{}{}
}
success = true
break
}
}
if !success {
err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.CanonicalName.Name(), lastErr)
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil))
errors <- err
return
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
}
