Example #1
0
func (sto *condStorage) EnumerateBlobs(dest chan<- blobref.SizedBlobRef, after string, limit int, wait time.Duration) error {
	if sto.read != nil {
		rsto := blobserver.MaybeWrapContext(sto.read, sto.ctx)
		return rsto.EnumerateBlobs(dest, after, limit, wait)
	}
	return errors.New("cond: Read not configured")
}
Example #2
0
func (sto *replicaStorage) RemoveBlobs(blobs []*blobref.BlobRef) error {
	errch := make(chan error, buffered)
	removeFrom := func(s blobserver.Storage) {
		s = blobserver.MaybeWrapContext(s, sto.ctx)
		errch <- s.RemoveBlobs(blobs)
	}
	for _, replica := range sto.replicas {
		go removeFrom(replica)
	}
	var reterr error
	nSuccess := 0
	for _ = range errch {
		if err := <-errch; err != nil {
			reterr = err
		} else {
			nSuccess++
		}
	}
	if nSuccess > 0 {
		// TODO: decide on the return value. for now this is best
		// effort and we return nil if any of the blobservers said
		// success.  maybe a bit weird, though.
		return nil
	}
	return reterr
}
Example #3
0
func (sto *condStorage) StatBlobs(dest chan<- blobref.SizedBlobRef, blobs []*blobref.BlobRef, wait time.Duration) error {
	if sto.read != nil {
		rsto := blobserver.MaybeWrapContext(sto.read, sto.ctx)
		return rsto.StatBlobs(dest, blobs, wait)
	}
	return errors.New("cond: Read not configured")
}
Example #4
0
func (sto *condStorage) RemoveBlobs(blobs []*blobref.BlobRef) error {
	if sto.remove != nil {
		rsto := blobserver.MaybeWrapContext(sto.remove, sto.ctx)
		return rsto.RemoveBlobs(blobs)
	}
	return errors.New("cond: Remove not configured")
}
Example #5
0
func (sto *condStorage) FetchStreaming(b *blobref.BlobRef) (file io.ReadCloser, size int64, err error) {
	if sto.read != nil {
		rsto := blobserver.MaybeWrapContext(sto.read, sto.ctx)
		return rsto.FetchStreaming(b)
	}
	err = errors.New("cond: Read not configured")
	return
}
Example #6
0
func (sto *replicaStorage) wrappedReplicas() []blobserver.Storage {
	if sto.ctx == nil {
		return sto.replicas
	}
	w := make([]blobserver.Storage, len(sto.replicas))
	for i, r := range sto.replicas {
		w[i] = blobserver.MaybeWrapContext(r, sto.ctx)
	}
	return w
}
Example #7
0
// TODO-GO: s/xxgo/_/ once Go issue 1802 is fixd
func (sto *replicaStorage) ReceiveBlob(b *blobref.BlobRef, source io.Reader) (xxgo blobref.SizedBlobRef, err error) {
	nReplicas := len(sto.replicas)
	rpipe, wpipe, writer := make([]*io.PipeReader, nReplicas), make([]*io.PipeWriter, nReplicas), make([]io.Writer, nReplicas)
	for idx := range sto.replicas {
		rpipe[idx], wpipe[idx] = io.Pipe()
		writer[idx] = wpipe[idx]
		// TODO: deal with slow/hung clients. this scheme of pipes +
		// multiwriter (even with a bufio.Writer thrown in) isn't
		// sufficient to guarantee forward progress. perhaps something
		// like &MoveOrDieWriter{Writer: wpipe[idx], HeartbeatSec: 10}
	}
	upResult := make(chan sizedBlobAndError, nReplicas)
	uploadToReplica := func(source io.Reader, s blobserver.Storage) {
		s = blobserver.MaybeWrapContext(s, sto.ctx)
		sb, err := s.ReceiveBlob(b, source)
		if err != nil {
			io.Copy(ioutil.Discard, source)
		}
		upResult <- sizedBlobAndError{sb, err}
	}
	for idx, replica := range sto.wrappedReplicas() {
		go uploadToReplica(rpipe[idx], replica)
	}
	size, err := io.Copy(io.MultiWriter(writer...), source)
	if err != nil {
		return
	}
	for idx := range sto.replicas {
		wpipe[idx].Close()
	}
	nSuccess, nFailures := 0, 0
	for _ = range sto.replicas {
		res := <-upResult
		switch {
		case res.err == nil && res.sb.Size == size:
			nSuccess++
			if nSuccess == sto.minWritesForSuccess {
				sto.GetBlobHub().NotifyBlobReceived(b)
				return res.sb, nil
			}
		case res.err == nil:
			nFailures++
			err = fmt.Errorf("replica: upload shard reported size %d, expected %d", res.sb.Size, size)
		default:
			nFailures++
			err = res.err
		}
	}
	if nFailures > 0 {
		log.Printf("replica: receiving blob, %d successes, %d failures; last error = %v",
			nSuccess, nFailures, err)
	}
	return
}
Example #8
0
func (sto *condStorage) ReceiveBlob(b *blobref.BlobRef, source io.Reader) (sb blobref.SizedBlobRef, err error) {
	destSto, overRead, err := sto.storageForReceive(source)
	if err != nil {
		return
	}
	if len(overRead) > 0 {
		source = io.MultiReader(bytes.NewBuffer(overRead), source)
	}
	destSto = blobserver.MaybeWrapContext(destSto, sto.ctx)
	return destSto.ReceiveBlob(b, source)
}