// loadImage loads an os image from the blobstore, // downloading and caching it if necessary. func (h *imagesDownloadHandler) loadImage(st *state.State, envuuid, kind, series, arch string) ( *imagestorage.Metadata, io.ReadCloser, error, ) { // We want to ensure that if an image needs to be downloaded and cached, // this only happens once. imageIdent := fmt.Sprintf("image-%s-%s-%s-%s", envuuid, kind, series, arch) lockDir := filepath.Join(h.dataDir, "locks") lock, err := fslock.NewLock(lockDir, imageIdent) if err != nil { return nil, nil, errors.Trace(err) } lock.Lock("fetch and cache image " + imageIdent) defer lock.Unlock() storage := st.ImageStorage() metadata, imageReader, err := storage.Image(kind, series, arch) // Not in storage, so go fetch it. if errors.IsNotFound(err) { err = h.fetchAndCacheLxcImage(storage, envuuid, series, arch) if err != nil { return nil, nil, errors.Annotate(err, "error fetching and caching image") } err = utils.NetworkOperationWitDefaultRetries(func() error { metadata, imageReader, err = storage.Image(string(instance.LXC), series, arch) return err }, "streaming os image from blobstore")() } if err != nil { return nil, nil, errors.Trace(err) } return metadata, imageReader, nil }
func (s *networkSuite) TestOpSuccess(c *gc.C) { isCalled := false f := func() error { isCalled = true return nil } err := utils.NetworkOperationWitDefaultRetries(f, "do it")() c.Assert(err, jc.ErrorIsNil) c.Assert(isCalled, jc.IsTrue) }
func (s *networkSuite) TestOpNestedFailureRetries(c *gc.C) { s.PatchValue(&utils.DefaultNetworkOperationRetryDelay, 1*time.Millisecond) netErr := &netError{true} callCount := 0 f := func() error { callCount++ return errors.Annotate(errors.Trace(netErr), "create a wrapped error") } err := utils.NetworkOperationWitDefaultRetries(f, "do it")() c.Assert(errors.Cause(err), gc.Equals, netErr) c.Assert(callCount, gc.Equals, 10) }
func (s *networkSuite) TestOpFailureNoRetry(c *gc.C) { s.PatchValue(&utils.DefaultNetworkOperationRetryDelay, 1*time.Millisecond) netErr := &netError{false} callCount := 0 f := func() error { callCount++ return netErr } err := utils.NetworkOperationWitDefaultRetries(f, "do it")() c.Assert(errors.Cause(err), gc.Equals, netErr) c.Assert(callCount, gc.Equals, 1) }
func (s *networkSuite) TestOpSucceedsAfterRetries(c *gc.C) { s.PatchValue(&utils.DefaultNetworkOperationRetryDelay, 1*time.Millisecond) netErr := &netError{true} callCount := 0 f := func() error { callCount++ if callCount == 5 { return nil } return netErr } err := utils.NetworkOperationWitDefaultRetries(f, "do it")() c.Assert(err, jc.ErrorIsNil) c.Assert(callCount, gc.Equals, 5) }
// fetchAndCacheLxcImage fetches an lxc image tarball from http://cloud-images.ubuntu.com // and caches it in the state blobstore. func (h *imagesDownloadHandler) fetchAndCacheLxcImage(storage imagestorage.Storage, envuuid, series, arch string) error { imageURL, err := container.ImageDownloadURL(instance.LXC, series, arch) if err != nil { return errors.Annotatef(err, "cannot determine LXC image URL: %v", err) } // Fetch the image checksum. imageFilename := path.Base(imageURL) shafile := strings.Replace(imageURL, imageFilename, "SHA256SUMS", -1) shaResp, err := http.Get(shafile) if err != nil { return errors.Annotatef(err, "cannot get sha256 data from %v", shafile) } defer shaResp.Body.Close() shaInfo, err := ioutil.ReadAll(shaResp.Body) if err != nil { return errors.Annotatef(err, "cannot read sha256 data from %v", shafile) } // The sha file has lines like: // "<checksum> *<imageFilename>" checksum := "" for _, line := range strings.Split(string(shaInfo), "\n") { parts := strings.Split(line, "*") if len(parts) != 2 { continue } if parts[1] == imageFilename { checksum = strings.TrimSpace(parts[0]) break } } if checksum == "" { return errors.Errorf("cannot find sha256 checksum for %v", imageFilename) } // Fetch the image. logger.Debugf("fetching LXC image from: %v", imageURL) resp, err := http.Get(imageURL) if err != nil { return errors.Annotatef(err, "cannot get image from %v", imageURL) } logger.Debugf("lxc image has size: %v bytes", resp.ContentLength) defer resp.Body.Close() hash := sha256.New() // Set up a chain of readers to pull in the data and calculate the checksum. rdr := io.TeeReader(resp.Body, hash) metadata := &imagestorage.Metadata{ EnvUUID: envuuid, Kind: string(instance.LXC), Series: series, Arch: arch, Size: resp.ContentLength, SHA256: checksum, SourceURL: imageURL, } // Stream the image to storage. err = utils.NetworkOperationWitDefaultRetries(func() error { return storage.AddImage(rdr, metadata) }, "add os image to blobstore")() if err != nil { return errors.Trace(err) } // Better check the downloaded image checksum. downloadChecksum := fmt.Sprintf("%x", hash.Sum(nil)) if downloadChecksum != checksum { err = utils.NetworkOperationWitDefaultRetries(func() error { return storage.DeleteImage(metadata) }, "delete os image from blobstore")() if err != nil { logger.Errorf("checksum mismatch, failed to delete image from storage: %v", err) } return errors.Errorf("download checksum mismatch %s != %s", downloadChecksum, checksum) } return nil }