// CreateSignature calculate the signature of target.
func (ctx *Context) CreateSignature(fileIndex int64, fileReader io.Reader, writeHash SignatureWriter) error {
s := bufio.NewScanner(fileReader)
s.Buffer(make([]byte, ctx.blockSize), 0)
s.Split(splitfunc.New(ctx.blockSize))
blockIndex := int64(0)
hashBlock := func(block []byte) error {
weakHash, _, _ := βhash(block)
strongHash := ctx.uniqueHash(block)
blockHash := BlockHash{
FileIndex: fileIndex,
BlockIndex: blockIndex,
WeakHash: weakHash,
StrongHash: strongHash,
}
if len(block) < ctx.blockSize {
blockHash.ShortSize = int32(len(block))
}
err := writeHash(blockHash)
if err != nil {
return errors.Wrap(err, 1)
}
blockIndex++
return nil
}
for s.Scan() {
err := hashBlock(s.Bytes())
if err != nil {
return errors.Wrap(err, 1)
}
}
err := s.Err()
if err != nil {
return errors.Wrap(err, 1)
}
// let empty files have a 0-length shortblock
if blockIndex == 0 {
err := hashBlock([]byte{})
if err != nil {
return errors.Wrap(err, 1)
}
}
return nil
}
func (vs *ValidatingSink) Store(loc BlockLocation, data []byte) error {
vs.log("validating %+v (%d bytes)", loc, len(data))
if vs.hashGroups == nil {
vs.log("making hash groups")
err := vs.makeHashGroups()
if err != nil {
return errors.Wrap(err, 1)
}
vs.blockBuf = make([]byte, pwr.BlockSize)
vs.split = splitfunc.New(int(pwr.BlockSize))
vs.sctx = wsync.NewContext(int(pwr.BlockSize))
}
hashGroup := vs.hashGroups[loc]
// see also wsync.CreateSignature
s := bufio.NewScanner(bytes.NewReader(data))
s.Buffer(vs.blockBuf, 0)
s.Split(vs.split)
hashIndex := 0
for ; s.Scan(); hashIndex++ {
smallBlock := s.Bytes()
vs.log("validating sub #%d of %+v (%d bytes)", hashIndex, loc, len(smallBlock))
weakHash, strongHash := vs.sctx.HashBlock(smallBlock)
bh := hashGroup[hashIndex]
if bh.WeakHash != weakHash {
err := fmt.Errorf("at %+v, expected weak hash %x, got %x", loc, bh.WeakHash, weakHash)
return errors.Wrap(err, 1)
}
if !bytes.Equal(bh.StrongHash, strongHash) {
err := fmt.Errorf("at %+v, expected strong hash %x, got %x", loc, bh.StrongHash, strongHash)
return errors.Wrap(err, 1)
}
}
return vs.Sink.Store(loc, data)
}
func (ru *ResumableUpload) uploadChunks(reader io.Reader, done chan bool, errs chan error) {
var offset int64 = 0
var maxSendBuf = ru.MaxChunkGroup * gcsChunkSize // 16MB
sendBuf := make([]byte, 0, maxSendBuf)
reqBlocks := make(chan blockItem, ru.MaxChunkGroup)
// when closed, all subtasks should abort
canceller := make(chan bool)
sendBytes := func(buf []byte, isLast bool) error {
retryCtx := ru.newRetryContext()
for retryCtx.ShouldTry() {
err := ru.trySendBytes(buf, offset, isLast)
if err != nil {
if ne, ok := err.(*netError); ok {
retryCtx.Retry(ne.Error())
continue
} else if re, ok := err.(*retryError); ok {
offset += re.committedBytes
buf = buf[re.committedBytes:]
retryCtx.Retry("Having troubles uploading some blocks")
continue
} else {
return errors.Wrap(err, 1)
}
} else {
offset += int64(len(buf))
return nil
}
}
return fmt.Errorf("Too many errors, giving up.")
}
subDone := make(chan bool)
subErrs := make(chan error)
ru.Debugf("sender: starting up, upload URL: %s", ru.uploadURL)
go func() {
isLast := false
// last block needs special treatment (different headers, etc.)
for !isLast {
sendBuf = sendBuf[:0]
// fill send buffer with as many blocks as are
// available. if none are available, wait for one.
for len(sendBuf) < maxSendBuf && !isLast {
var item blockItem
if len(sendBuf) == 0 {
ru.VerboseDebugf("sender: doing blocking receive")
select {
case item = <-reqBlocks:
// done waiting, got one, can resume upload now
case <-canceller:
ru.Debugf("sender: cancelled (from blocking receive)")
return
}
} else {
ru.VerboseDebugf("sender: doing non-blocking receive")
select {
case item = <-reqBlocks:
// cool
case <-canceller:
ru.Debugf("sender: cancelled (from non-blocking receive)")
return
default:
ru.VerboseDebugf("sent faster than scanned, uploading smaller chunk")
break
}
}
// if the last item is in sendBuf, sendBuf is the last upload we'll
// do (save for retries)
if item.isLast {
isLast = true
}
sendBuf = append(sendBuf, item.buf...)
}
if len(sendBuf) > 0 {
err := sendBytes(sendBuf, isLast)
if err != nil {
ru.Debugf("sender: send error, bailing out")
subErrs <- errors.Wrap(err, 1)
return
}
}
}
subDone <- true
ru.Debugf("sender: all done")
}()
// use a bufio.Scanner to break input into blocks of gcsChunkSize
// at most. last block might be smaller. see splitfunc.go
s := bufio.NewScanner(reader)
s.Buffer(make([]byte, gcsChunkSize), 0)
s.Split(splitfunc.New(gcsChunkSize))
scannedBufs := make(chan []byte)
usedBufs := make(chan bool)
go func() {
for s.Scan() {
select {
case scannedBufs <- s.Bytes():
// woo
case <-canceller:
ru.Debugf("scan cancelled (1)")
break
}
select {
case <-usedBufs:
// woo
case <-canceller:
ru.Debugf("scan cancelled (2)")
break
}
}
close(scannedBufs)
}()
// using two buffers lets us detect EOF even when the last block
// is an exact multiple of gcsChunkSize - the `for := range` will
// end and we'll have the last block left in buf1
buf1 := make([]byte, 0, gcsChunkSize)
buf2 := make([]byte, 0, gcsChunkSize)
go func() {
for scannedBuf := range scannedBufs {
buf2 = append(buf2[:0], buf1...)
buf1 = append(buf1[:0], scannedBuf...)
usedBufs <- true
// on first iteration, buf2 is still empty.
if len(buf2) > 0 {
select {
case reqBlocks <- blockItem{buf: append([]byte{}, buf2...), isLast: false}:
// sender received the block, we can keep going
case <-canceller:
ru.Debugf("scan cancelled (3)")
return
}
}
}
err := s.Err()
if err != nil {
ru.Debugf("scanner error :(")
subErrs <- errors.Wrap(err, 1)
return
}
select {
case reqBlocks <- blockItem{buf: append([]byte{}, buf1...), isLast: true}:
case <-canceller:
ru.Debugf("scan cancelled (right near the finish line)")
return
}
subDone <- true
ru.Debugf("scanner done")
}()
for i := 0; i < 2; i++ {
select {
case <-subDone:
// woo!
case err := <-subErrs:
ru.Debugf("got sub error: %s, bailing", err.Error())
// any error that travels this far up cancels the whole upload
close(canceller)
errs <- errors.Wrap(err, 1)
return
}
}
done <- true
ru.Debugf("done sent!")
}