func (v *ImageStore) ListImages(ctx context.Context, store *url.URL, IDs []string) ([]*portlayer.Image, error) {
storeName, err := util.ImageStoreName(store)
if err != nil {
return nil, err
}
res, err := v.ds.Ls(ctx, v.imageStorePath(storeName))
if err != nil {
return nil, err
}
images := []*portlayer.Image{}
for _, f := range res.File {
file, ok := f.(*types.FileInfo)
if !ok {
continue
}
ID := file.Path
img, err := v.GetImage(ctx, store, ID)
if err != nil {
return nil, err
}
images = append(images, img)
}
return images, nil
}
func (c *MockDataStore) WriteImage(op trace.Operation, parent *Image, ID string, meta map[string][]byte, sum string, r io.Reader) (*Image, error) {
storeName, err := util.ImageStoreName(parent.Store)
if err != nil {
return nil, err
}
selflink, err := util.ImageURL(storeName, ID)
if err != nil {
return nil, err
}
var parentLink *url.URL
if parent.ID != "" {
parentLink, err = util.ImageURL(storeName, parent.ID)
if err != nil {
return nil, err
}
}
i := &Image{
ID: ID,
Store: parent.Store,
ParentLink: parentLink,
SelfLink: selflink,
Metadata: meta,
}
c.db[*parent.Store][ID] = i
return i, nil
}
// GetImage gets the specified image from the given store by retreiving it from the cache.
func (c *NameLookupCache) GetImage(ctx context.Context, store *url.URL, ID string) (*Image, error) {
storeName, err := util.ImageStoreName(store)
if err != nil {
return nil, err
}
// Check the store exists
if _, err = c.GetImageStore(ctx, storeName); err != nil {
return nil, err
}
c.storeCacheLock.Lock()
defer c.storeCacheLock.Unlock()
var ok bool
i := &Image{}
*i, ok = c.storeCache[*store][ID]
if !ok {
log.Infof("Image %s not in cache, retreiving from datastore", ID)
// Not in the cache. Try to load it.
i, err = c.DataStore.GetImage(ctx, store, ID)
if err != nil {
return nil, err
}
c.storeCache[*store][ID] = *i
}
return i, nil
}
func (v *ImageStore) ListImages(op trace.Operation, store *url.URL, IDs []string) ([]*portlayer.Image, error) {
storeName, err := util.ImageStoreName(store)
if err != nil {
return nil, err
}
res, err := v.ds.Ls(op, v.imageStorePath(storeName))
if err != nil {
return nil, err
}
images := []*portlayer.Image{}
for _, f := range res.File {
file, ok := f.(*types.FileInfo)
if !ok {
continue
}
ID := file.Path
// GetImage verifies the image is good by calling verifyImage.
img, err := v.GetImage(op, store, ID)
if err != nil {
return nil, err
}
images = append(images, img)
}
return images, nil
}
func (v *ImageStore) GetImage(ctx context.Context, store *url.URL, ID string) (*portlayer.Image, error) {
defer trace.End(trace.Begin(store.String()))
storeName, err := util.ImageStoreName(store)
if err != nil {
return nil, err
}
imageURL, err := util.ImageURL(storeName, ID)
if err != nil {
return nil, err
}
p := v.imageDirPath(storeName, ID)
info, err := v.ds.Stat(ctx, p)
if err != nil {
return nil, err
}
_, ok := info.(*types.FolderFileInfo)
if !ok {
return nil, fmt.Errorf("Stat error: image doesn't exist (%s)", p)
}
// get the metadata
metaDataDir := v.imageMetadataDirPath(storeName, ID)
meta, err := getMetadata(ctx, v.ds, metaDataDir)
if err != nil {
return nil, err
}
var s = *store
var parentURL *url.URL
parentID := v.parents.Get(ID)
if parentID != "" {
parentURL, _ = util.ImageURL(storeName, parentID)
}
newImage := &portlayer.Image{
ID: ID,
SelfLink: imageURL,
// We're relying on the parent map for this since we don't currently have a
// way to get the disk's spec. See VIC #482 for details. Parent:
// parent.SelfLink,
Store: &s,
Parent: parentURL,
Metadata: meta,
}
log.Debugf("Returning image from location %s with parent url %s", newImage.SelfLink, newImage.Parent)
return newImage, nil
}
// DeleteImage deletes an image from the image store. If the image is in
// use either by way of inheritance or because it's attached to a
// container, this will return an error.
func (v *ImageStore) DeleteImage(op trace.Operation, image *portlayer.Image) error {
// check if the image is in use.
if err := imagesInUse(op, image.ID); err != nil {
op.Errorf("ImageStore: delete image error: %s", err.Error())
return err
}
storeName, err := util.ImageStoreName(image.Store)
if err != nil {
return err
}
return v.deleteImage(op, storeName, image.ID)
}
// WriteImage creates a new image layer from the given parent.
// Eg parentImage + newLayer = new Image built from parent
//
// parent - The parent image to create the new image from.
// ID - textual ID for the image to be written
// meta - metadata associated with the image
// Tag - the tag of the image to be written
func (v *ImageStore) WriteImage(op trace.Operation, parent *portlayer.Image, ID string, meta map[string][]byte, sum string,
r io.Reader) (*portlayer.Image, error) {
storeName, err := util.ImageStoreName(parent.Store)
if err != nil {
return nil, err
}
imageURL, err := util.ImageURL(storeName, ID)
if err != nil {
return nil, err
}
// If this is scratch, then it's the root of the image store. All images
// will be descended from this created and prepared fs.
if ID == portlayer.Scratch.ID {
// Create the scratch layer
if err := v.scratch(op, storeName); err != nil {
return nil, err
}
} else {
if parent.ID == "" {
return nil, fmt.Errorf("parent ID is empty")
}
// persist the relationship
v.parents.Add(ID, parent.ID)
if err := v.parents.Save(op); err != nil {
return nil, err
}
if err := v.writeImage(op, storeName, parent.ID, ID, meta, sum, r); err != nil {
return nil, err
}
}
newImage := &portlayer.Image{
ID: ID,
SelfLink: imageURL,
ParentLink: parent.SelfLink,
Store: parent.Store,
Metadata: meta,
}
return newImage, nil
}
func (c *NameLookupCache) WriteImage(ctx context.Context, parent *Image, ID string, meta map[string][]byte, sum string, r io.Reader) (*Image, error) {
// Check the parent exists (at least in the cache).
p, err := c.GetImage(ctx, parent.Store, parent.ID)
if err != nil {
return nil, fmt.Errorf("parent (%s) doesn't exist in %s: %s", parent.ID, parent.Store.String(), err)
}
// Check the image doesn't already exist in the cache. A miss in this will trigger a datastore lookup.
i, err := c.GetImage(ctx, p.Store, ID)
if err == nil && i != nil {
// TODO(FA) check sums to make sure this is the right image
// if meta supplied write it
if meta != nil && len(meta) != 0 {
// get the storename from the url
storeName, err := util.ImageStoreName(p.Store)
if err != nil {
return nil, err
}
c.DataStore.WriteMetadata(ctx, storeName, i.ID, meta)
i.Metadata = meta
c.AddImageToStore(*p.Store, i.ID, *i)
}
return i, nil
}
// Definitely not in cache or image store, create image.
h := sha256.New()
t := io.TeeReader(r, h)
i, err = c.DataStore.WriteImage(ctx, p, ID, meta, t)
if err != nil {
return nil, err
}
actualSum := fmt.Sprintf("sha256:%x", h.Sum(nil))
if actualSum != sum {
// TODO(jzt): cleanup?
return nil, fmt.Errorf("Failed to validate image checksum. Expected %s, got %s", sum, actualSum)
}
// Add the new image to the cache
c.AddImageToStore(*p.Store, i.ID, *i)
return i, nil
}
// GetImageStore checks to see if a named image store exists and returls the
// URL to it if so or error.
func (c *NameLookupCache) GetImageStore(ctx context.Context, storeName string) (*url.URL, error) {
store, err := util.ImageStoreNameToURL(storeName)
if err != nil {
return nil, err
}
c.storeCacheLock.Lock()
defer c.storeCacheLock.Unlock()
// check the cache
_, ok := c.storeCache[*store]
if !ok {
log.Info("Refreshing image cache from datastore.")
// Store isn't in the cache. Look it up in the datastore.
storeName, err := util.ImageStoreName(store)
if err != nil {
return nil, err
}
// If the store doesn't exist, we'll fall out here.
_, err = c.DataStore.GetImageStore(ctx, storeName)
if err != nil {
return nil, err
}
c.storeCache[*store] = make(map[string]Image)
// Fall out here if there are no images. We should at least have a scratch.
images, err := c.DataStore.ListImages(ctx, store, nil)
if err != nil {
return nil, err
}
// add the images we retrieved to the cache.
for _, v := range images {
log.Infof("Imagestore: Found image %s on datastore.", v.ID)
c.storeCache[*store][v.ID] = *v
}
// Assert there's a scratch
if _, ok = c.storeCache[*store][Scratch.ID]; !ok {
return nil, fmt.Errorf("Scratch does not exist. Imagestore is corrrupt.")
}
}
return store, nil
}
// GetImage gets the specified image from the given store by retreiving it from the cache.
func (c *NameLookupCache) GetImage(op trace.Operation, store *url.URL, ID string) (*Image, error) {
op.Debugf("Getting image %s from %s", ID, store.String())
storeName, err := util.ImageStoreName(store)
if err != nil {
return nil, err
}
// Check the store exists
if _, err = c.GetImageStore(op, storeName); err != nil {
return nil, err
}
c.storeCacheLock.Lock()
indx := c.storeCache[*store]
c.storeCacheLock.Unlock()
imgURL, err := util.ImageURL(storeName, ID)
if err != nil {
return nil, err
}
node, err := c.storeCache[*store].Get(imgURL.String())
var img *Image
if err != nil {
if err == index.ErrNodeNotFound {
debugf("Image %s not in cache, retreiving from datastore", ID)
// Not in the cache. Try to load it.
img, err = c.DataStore.GetImage(op, store, ID)
if err != nil {
return nil, err
}
if err = indx.Insert(img); err != nil {
return nil, err
}
} else {
return nil, err
}
} else {
img, _ = node.(*Image)
}
return img, nil
}
// ListImages returns a list of Images for a list of IDs, or all if no IDs are passed
func (c *NameLookupCache) ListImages(op trace.Operation, store *url.URL, IDs []string) ([]*Image, error) {
// Filter the results
imageList := make([]*Image, 0, len(IDs))
if len(IDs) > 0 {
for _, id := range IDs {
i, err := c.GetImage(op, store, id)
if err == nil {
imageList = append(imageList, i)
}
}
} else {
storeName, err := util.ImageStoreName(store)
if err != nil {
return nil, err
}
// Check the store exists before we start iterating it. This will populate the cache if it's empty.
if _, err := c.GetImageStore(op, storeName); err != nil {
return nil, err
}
// get the relevant cache
c.storeCacheLock.Lock()
indx := c.storeCache[*store]
c.storeCacheLock.Unlock()
images, err := indx.List()
if err != nil {
return nil, err
}
for _, v := range images {
img, _ := v.(*Image)
// filter out scratch
if img.ID == Scratch.ID {
continue
}
imageList = append(imageList, img)
}
}
return imageList, nil
}
func (v *ImageStore) GetImage(op trace.Operation, store *url.URL, ID string) (*portlayer.Image, error) {
defer trace.End(trace.Begin(store.String()))
storeName, err := util.ImageStoreName(store)
if err != nil {
return nil, err
}
imageURL, err := util.ImageURL(storeName, ID)
if err != nil {
return nil, err
}
if err = v.verifyImage(op, storeName, ID); err != nil {
return nil, err
}
// get the metadata
metaDataDir := v.imageMetadataDirPath(storeName, ID)
meta, err := getMetadata(op, v.ds, metaDataDir)
if err != nil {
return nil, err
}
var s = *store
var parentURL *url.URL
parentID := v.parents.Get(ID)
if parentID != "" {
parentURL, _ = util.ImageURL(storeName, parentID)
}
newImage := &portlayer.Image{
ID: ID,
SelfLink: imageURL,
// We're relying on the parent map for this since we don't currently have a
// way to get the disk's spec. See VIC #482 for details. Parent:
// parent.SelfLink,
Store: &s,
ParentLink: parentURL,
Metadata: meta,
}
op.Debugf("Returning image from location %s with parent url %s", newImage.SelfLink, newImage.Parent())
return newImage, nil
}
func (c *MockDataStore) WriteImage(op trace.Operation, parent *spl.Image, ID string, meta map[string][]byte, sum string, r io.Reader) (*spl.Image, error) {
storeName, err := util.ImageStoreName(parent.Store)
if err != nil {
return nil, err
}
selflink, err := util.ImageURL(storeName, ID)
if err != nil {
return nil, err
}
i := spl.Image{
ID: ID,
Store: parent.Store,
ParentLink: parent.SelfLink,
SelfLink: selflink,
Metadata: meta,
}
return &i, nil
}
// Find any image directories without the manifest file and remove them.
func (v *ImageStore) cleanup(ctx context.Context, store *url.URL) error {
log.Infof("Checking for inconsistent images on %s", store.String())
storeName, err := util.ImageStoreName(store)
if err != nil {
return err
}
res, err := v.ds.Ls(ctx, v.imageStorePath(storeName))
if err != nil {
return err
}
for _, f := range res.File {
file, ok := f.(*types.FileInfo)
if !ok {
continue
}
ID := file.Path
if ID == portlayer.Scratch.ID {
continue
}
imageDir := v.imageDirPath(storeName, ID)
// check for the manifest file.
_, err = v.ds.Stat(ctx, path.Join(imageDir, manifest))
if err != nil {
log.Infof("Removing inconsistent image (%s) %s", ID, imageDir)
// Eat the error so we can continue cleaning up. The tasks package will log the error if there is one.
_ = v.ds.Rm(ctx, imageDir)
}
}
return nil
}
// Find any image directories without the manifest file and remove them.
func (v *ImageStore) cleanup(op trace.Operation, store *url.URL) error {
op.Infof("Checking for inconsistent images on %s", store.String())
storeName, err := util.ImageStoreName(store)
if err != nil {
return err
}
res, err := v.ds.Ls(op, v.imageStorePath(storeName))
if err != nil {
return err
}
// We could call v.ListImages here but that results in calling GetImage,
// which pulls and unmarshalls the metadata. We don't need that.
for _, f := range res.File {
file, ok := f.(*types.FileInfo)
if !ok {
continue
}
ID := file.Path
if ID == portlayer.Scratch.ID {
continue
}
if err := v.verifyImage(op, storeName, ID); err != nil {
if err = v.deleteImage(op, storeName, ID); err != nil {
// deleteImage logs the error in the event there is one.
return err
}
}
}
return nil
}
// ListImages resturns a list of Images for a list of IDs, or all if no IDs are passed
func (c *NameLookupCache) ListImages(ctx context.Context, store *url.URL, IDs []string) ([]*Image, error) {
storeName, err := util.ImageStoreName(store)
if err != nil {
return nil, err
}
// Check the store exists before we start iterating it. This will populate the cache if it's empty.
if _, err := c.GetImageStore(ctx, storeName); err != nil {
return nil, err
}
c.storeCacheLock.Lock()
defer c.storeCacheLock.Unlock()
// Filter the results
var imageList []*Image
if len(IDs) > 0 {
for _, id := range IDs {
if i, ok := c.storeCache[*store][id]; ok {
newImage := i
imageList = append(imageList, &newImage)
}
}
} else {
for _, v := range c.storeCache[*store] {
// filter out scratch
if v.ID == Scratch.ID {
continue
}
newImage := v
imageList = append(imageList, &newImage)
}
}
return imageList, nil
}
// GetImageStore checks to see if a named image store exists and returls the
// URL to it if so or error.
func (c *NameLookupCache) GetImageStore(op trace.Operation, storeName string) (*url.URL, error) {
store, err := util.ImageStoreNameToURL(storeName)
if err != nil {
return nil, err
}
c.storeCacheLock.Lock()
defer c.storeCacheLock.Unlock()
// check the cache
_, ok := c.storeCache[*store]
if !ok {
infof("Refreshing image cache from datastore.")
// Store isn't in the cache. Look it up in the datastore.
storeName, err := util.ImageStoreName(store)
if err != nil {
return nil, err
}
// If the store doesn't exist, we'll fall out here.
_, err = c.DataStore.GetImageStore(op, storeName)
if err != nil {
return nil, err
}
idx := index.NewIndex()
c.storeCache[*store] = idx
// Add Scratch
scratch, err := c.DataStore.GetImage(op, store, Scratch.ID)
if err != nil {
log.Errorf("ImageCache Error: looking up scratch on %s: %s", store.String(), err)
return nil, ErrCorruptImageStore
}
if err = idx.Insert(scratch); err != nil {
return nil, err
}
// XXX after creating the indx and populating the map, we can put the rest in a go routine
// Fall out here if there are no images. We should at least have a scratch.
images, err := c.DataStore.ListImages(op, store, nil)
if err != nil {
return nil, err
}
debugf("Found %d images", len(images))
// Build image map to simplify tree traversal.
imageMap := make(map[string]*Image, len(images))
for _, img := range images {
if img.ID == Scratch.ID {
continue
}
imageMap[img.Self()] = img
}
for k := range imageMap {
parentTree(k, idx, imageMap)
}
}
return store, nil
}