Example #1
0
// CreateSignature calculate the signature of target.
func (ctx *Context) CreateSignature(fileIndex int64, fileReader io.Reader, writeHash SignatureWriter) error {
	s := bufio.NewScanner(fileReader)
	s.Buffer(make([]byte, ctx.blockSize), 0)
	s.Split(splitfunc.New(ctx.blockSize))

	blockIndex := int64(0)

	hashBlock := func(block []byte) error {
		weakHash, _, _ := βhash(block)
		strongHash := ctx.uniqueHash(block)

		blockHash := BlockHash{
			FileIndex:  fileIndex,
			BlockIndex: blockIndex,
			WeakHash:   weakHash,
			StrongHash: strongHash,
		}

		if len(block) < ctx.blockSize {
			blockHash.ShortSize = int32(len(block))
		}

		err := writeHash(blockHash)
		if err != nil {
			return errors.Wrap(err, 1)
		}
		blockIndex++
		return nil
	}

	for s.Scan() {
		err := hashBlock(s.Bytes())
		if err != nil {
			return errors.Wrap(err, 1)
		}
	}

	err := s.Err()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	// let empty files have a 0-length shortblock
	if blockIndex == 0 {
		err := hashBlock([]byte{})
		if err != nil {
			return errors.Wrap(err, 1)
		}
	}

	return nil
}
Example #2
0
func (vs *ValidatingSink) Store(loc BlockLocation, data []byte) error {
	vs.log("validating %+v (%d bytes)", loc, len(data))

	if vs.hashGroups == nil {
		vs.log("making hash groups")
		err := vs.makeHashGroups()
		if err != nil {
			return errors.Wrap(err, 1)
		}

		vs.blockBuf = make([]byte, pwr.BlockSize)
		vs.split = splitfunc.New(int(pwr.BlockSize))
		vs.sctx = wsync.NewContext(int(pwr.BlockSize))
	}

	hashGroup := vs.hashGroups[loc]

	// see also wsync.CreateSignature
	s := bufio.NewScanner(bytes.NewReader(data))
	s.Buffer(vs.blockBuf, 0)
	s.Split(vs.split)

	hashIndex := 0

	for ; s.Scan(); hashIndex++ {
		smallBlock := s.Bytes()
		vs.log("validating sub #%d of %+v (%d bytes)", hashIndex, loc, len(smallBlock))

		weakHash, strongHash := vs.sctx.HashBlock(smallBlock)
		bh := hashGroup[hashIndex]

		if bh.WeakHash != weakHash {
			err := fmt.Errorf("at %+v, expected weak hash %x, got %x", loc, bh.WeakHash, weakHash)
			return errors.Wrap(err, 1)
		}

		if !bytes.Equal(bh.StrongHash, strongHash) {
			err := fmt.Errorf("at %+v, expected strong hash %x, got %x", loc, bh.StrongHash, strongHash)
			return errors.Wrap(err, 1)
		}
	}

	return vs.Sink.Store(loc, data)
}
Example #3
0
func (ru *ResumableUpload) uploadChunks(reader io.Reader, done chan bool, errs chan error) {
	var offset int64 = 0

	var maxSendBuf = ru.MaxChunkGroup * gcsChunkSize // 16MB
	sendBuf := make([]byte, 0, maxSendBuf)
	reqBlocks := make(chan blockItem, ru.MaxChunkGroup)

	// when closed, all subtasks should abort
	canceller := make(chan bool)

	sendBytes := func(buf []byte, isLast bool) error {
		retryCtx := ru.newRetryContext()

		for retryCtx.ShouldTry() {
			err := ru.trySendBytes(buf, offset, isLast)
			if err != nil {
				if ne, ok := err.(*netError); ok {
					retryCtx.Retry(ne.Error())
					continue
				} else if re, ok := err.(*retryError); ok {
					offset += re.committedBytes
					buf = buf[re.committedBytes:]
					retryCtx.Retry("Having troubles uploading some blocks")
					continue
				} else {
					return errors.Wrap(err, 1)
				}
			} else {
				offset += int64(len(buf))
				return nil
			}
		}

		return fmt.Errorf("Too many errors, giving up.")
	}

	subDone := make(chan bool)
	subErrs := make(chan error)

	ru.Debugf("sender: starting up, upload URL: %s", ru.uploadURL)

	go func() {
		isLast := false

		// last block needs special treatment (different headers, etc.)
		for !isLast {
			sendBuf = sendBuf[:0]

			// fill send buffer with as many blocks as are
			// available. if none are available, wait for one.
			for len(sendBuf) < maxSendBuf && !isLast {
				var item blockItem
				if len(sendBuf) == 0 {
					ru.VerboseDebugf("sender: doing blocking receive")
					select {
					case item = <-reqBlocks:
						// done waiting, got one, can resume upload now
					case <-canceller:
						ru.Debugf("sender: cancelled (from blocking receive)")
						return
					}
				} else {
					ru.VerboseDebugf("sender: doing non-blocking receive")
					select {
					case item = <-reqBlocks:
						// cool
					case <-canceller:
						ru.Debugf("sender: cancelled (from non-blocking receive)")
						return
					default:
						ru.VerboseDebugf("sent faster than scanned, uploading smaller chunk")
						break
					}
				}

				// if the last item is in sendBuf, sendBuf is the last upload we'll
				// do (save for retries)
				if item.isLast {
					isLast = true
				}

				sendBuf = append(sendBuf, item.buf...)
			}

			if len(sendBuf) > 0 {
				err := sendBytes(sendBuf, isLast)
				if err != nil {
					ru.Debugf("sender: send error, bailing out")
					subErrs <- errors.Wrap(err, 1)
					return
				}
			}
		}

		subDone <- true
		ru.Debugf("sender: all done")
	}()

	// use a bufio.Scanner to break input into blocks of gcsChunkSize
	// at most. last block might be smaller. see splitfunc.go
	s := bufio.NewScanner(reader)
	s.Buffer(make([]byte, gcsChunkSize), 0)
	s.Split(splitfunc.New(gcsChunkSize))

	scannedBufs := make(chan []byte)
	usedBufs := make(chan bool)

	go func() {
		for s.Scan() {
			select {
			case scannedBufs <- s.Bytes():
				// woo
			case <-canceller:
				ru.Debugf("scan cancelled (1)")
				break
			}
			select {
			case <-usedBufs:
				// woo
			case <-canceller:
				ru.Debugf("scan cancelled (2)")
				break
			}
		}
		close(scannedBufs)
	}()

	// using two buffers lets us detect EOF even when the last block
	// is an exact multiple of gcsChunkSize - the `for := range` will
	// end and we'll have the last block left in buf1
	buf1 := make([]byte, 0, gcsChunkSize)
	buf2 := make([]byte, 0, gcsChunkSize)

	go func() {
		for scannedBuf := range scannedBufs {
			buf2 = append(buf2[:0], buf1...)
			buf1 = append(buf1[:0], scannedBuf...)
			usedBufs <- true

			// on first iteration, buf2 is still empty.
			if len(buf2) > 0 {
				select {
				case reqBlocks <- blockItem{buf: append([]byte{}, buf2...), isLast: false}:
					// sender received the block, we can keep going
				case <-canceller:
					ru.Debugf("scan cancelled (3)")
					return
				}
			}
		}

		err := s.Err()
		if err != nil {
			ru.Debugf("scanner error :(")
			subErrs <- errors.Wrap(err, 1)
			return
		}

		select {
		case reqBlocks <- blockItem{buf: append([]byte{}, buf1...), isLast: true}:
		case <-canceller:
			ru.Debugf("scan cancelled (right near the finish line)")
			return
		}

		subDone <- true
		ru.Debugf("scanner done")
	}()

	for i := 0; i < 2; i++ {
		select {
		case <-subDone:
			// woo!
		case err := <-subErrs:
			ru.Debugf("got sub error: %s, bailing", err.Error())
			// any error that travels this far up cancels the whole upload
			close(canceller)
			errs <- errors.Wrap(err, 1)
			return
		}
	}

	done <- true
	ru.Debugf("done sent!")
}