Esempio n. 1
0
// loadImage loads an os image from the blobstore,
// downloading and caching it if necessary.
func (h *imagesDownloadHandler) loadImage(st *state.State, envuuid, kind, series, arch string) (
	*imagestorage.Metadata, io.ReadCloser, error,
) {
	// We want to ensure that if an image needs to be downloaded and cached,
	// this only happens once.
	imageIdent := fmt.Sprintf("image-%s-%s-%s-%s", envuuid, kind, series, arch)
	lockDir := filepath.Join(h.dataDir, "locks")
	lock, err := fslock.NewLock(lockDir, imageIdent)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	lock.Lock("fetch and cache image " + imageIdent)
	defer lock.Unlock()
	storage := st.ImageStorage()
	metadata, imageReader, err := storage.Image(kind, series, arch)
	// Not in storage, so go fetch it.
	if errors.IsNotFound(err) {
		err = h.fetchAndCacheLxcImage(storage, envuuid, series, arch)
		if err != nil {
			return nil, nil, errors.Annotate(err, "error fetching and caching image")
		}
		err = utils.NetworkOperationWitDefaultRetries(func() error {
			metadata, imageReader, err = storage.Image(string(instance.LXC), series, arch)
			return err
		}, "streaming os image from blobstore")()
	}
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	return metadata, imageReader, nil
}
Esempio n. 2
0
func (s *networkSuite) TestOpSuccess(c *gc.C) {
	isCalled := false
	f := func() error {
		isCalled = true
		return nil
	}
	err := utils.NetworkOperationWitDefaultRetries(f, "do it")()
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(isCalled, jc.IsTrue)
}
Esempio n. 3
0
func (s *networkSuite) TestOpNestedFailureRetries(c *gc.C) {
	s.PatchValue(&utils.DefaultNetworkOperationRetryDelay, 1*time.Millisecond)
	netErr := &netError{true}
	callCount := 0
	f := func() error {
		callCount++
		return errors.Annotate(errors.Trace(netErr), "create a wrapped error")
	}
	err := utils.NetworkOperationWitDefaultRetries(f, "do it")()
	c.Assert(errors.Cause(err), gc.Equals, netErr)
	c.Assert(callCount, gc.Equals, 10)
}
Esempio n. 4
0
func (s *networkSuite) TestOpFailureNoRetry(c *gc.C) {
	s.PatchValue(&utils.DefaultNetworkOperationRetryDelay, 1*time.Millisecond)
	netErr := &netError{false}
	callCount := 0
	f := func() error {
		callCount++
		return netErr
	}
	err := utils.NetworkOperationWitDefaultRetries(f, "do it")()
	c.Assert(errors.Cause(err), gc.Equals, netErr)
	c.Assert(callCount, gc.Equals, 1)
}
Esempio n. 5
0
func (s *networkSuite) TestOpSucceedsAfterRetries(c *gc.C) {
	s.PatchValue(&utils.DefaultNetworkOperationRetryDelay, 1*time.Millisecond)
	netErr := &netError{true}
	callCount := 0
	f := func() error {
		callCount++
		if callCount == 5 {
			return nil
		}
		return netErr
	}
	err := utils.NetworkOperationWitDefaultRetries(f, "do it")()
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(callCount, gc.Equals, 5)
}
Esempio n. 6
0
// fetchAndCacheLxcImage fetches an lxc image tarball from http://cloud-images.ubuntu.com
// and caches it in the state blobstore.
func (h *imagesDownloadHandler) fetchAndCacheLxcImage(storage imagestorage.Storage, envuuid, series, arch string) error {
	imageURL, err := container.ImageDownloadURL(instance.LXC, series, arch)
	if err != nil {
		return errors.Annotatef(err, "cannot determine LXC image URL: %v", err)
	}

	// Fetch the image checksum.
	imageFilename := path.Base(imageURL)
	shafile := strings.Replace(imageURL, imageFilename, "SHA256SUMS", -1)
	shaResp, err := http.Get(shafile)
	if err != nil {
		return errors.Annotatef(err, "cannot get sha256 data from %v", shafile)
	}
	defer shaResp.Body.Close()
	shaInfo, err := ioutil.ReadAll(shaResp.Body)
	if err != nil {
		return errors.Annotatef(err, "cannot read sha256 data from %v", shafile)
	}

	// The sha file has lines like:
	// "<checksum> *<imageFilename>"
	checksum := ""
	for _, line := range strings.Split(string(shaInfo), "\n") {
		parts := strings.Split(line, "*")
		if len(parts) != 2 {
			continue
		}
		if parts[1] == imageFilename {
			checksum = strings.TrimSpace(parts[0])
			break
		}
	}
	if checksum == "" {
		return errors.Errorf("cannot find sha256 checksum for %v", imageFilename)
	}

	// Fetch the image.
	logger.Debugf("fetching LXC image from: %v", imageURL)
	resp, err := http.Get(imageURL)
	if err != nil {
		return errors.Annotatef(err, "cannot get image from %v", imageURL)
	}
	logger.Debugf("lxc image has size: %v bytes", resp.ContentLength)
	defer resp.Body.Close()

	hash := sha256.New()
	// Set up a chain of readers to pull in the data and calculate the checksum.
	rdr := io.TeeReader(resp.Body, hash)

	metadata := &imagestorage.Metadata{
		EnvUUID:   envuuid,
		Kind:      string(instance.LXC),
		Series:    series,
		Arch:      arch,
		Size:      resp.ContentLength,
		SHA256:    checksum,
		SourceURL: imageURL,
	}

	// Stream the image to storage.
	err = utils.NetworkOperationWitDefaultRetries(func() error {
		return storage.AddImage(rdr, metadata)
	}, "add os image to blobstore")()
	if err != nil {
		return errors.Trace(err)
	}
	// Better check the downloaded image checksum.
	downloadChecksum := fmt.Sprintf("%x", hash.Sum(nil))
	if downloadChecksum != checksum {
		err = utils.NetworkOperationWitDefaultRetries(func() error {
			return storage.DeleteImage(metadata)
		}, "delete os image from blobstore")()
		if err != nil {
			logger.Errorf("checksum mismatch, failed to delete image from storage: %v", err)
		}
		return errors.Errorf("download checksum mismatch %s != %s", downloadChecksum, checksum)
	}
	return nil
}