// use rollsum do weak sum
func weakSum(p []byte) (s uint32) {
var rs rollsum.Rollsum
rs.Init()
rs.Update(p)
return rs.Digest()
}
func (session *BackupSession) storeFile(path string, entry *FileEntry) (err error) {
defer func() {
// Panic error handling
if r := recover(); r != nil {
// we need this because some obscure files on OSX does open but then generates "bad file descriptor" on read
if e, ok := r.(*os.PathError); ok && e.Err == syscall.EBADF {
err = e.Err
} else {
panic(r) // Any other error is not normal and should panic
}
}
}()
var links []core.Byte128
chain := FileChainBlock{}
var file *os.File
if file, err = os.Open(path); err != nil {
return err
}
defer file.Close()
var maxSum rollsum.Rollsum
maxSum.Init()
var fileData bytearray.ByteArray
defer fileData.Release()
for offset := int64(0); offset < int64(entry.FileSize); {
Debug("storeFile(%s) offset %d", path, offset)
session.PrintStoreProgress(PROGRESS_INTERVAL_SECS)
var left int64 = int64(entry.FileSize) - offset
var maxBlockSize int = MAX_BLOCK_SIZE
if left < int64(maxBlockSize) {
maxBlockSize = int(left)
}
var blockData bytearray.ByteArray
// Fill the fileData buffer
core.CopyNOrPanic(&fileData, file, maxBlockSize-fileData.Len())
fileData.ReadSeek(0, os.SEEK_CUR) // TODO: figure out why this line is here because I do not remember
var splitPosition int = fileData.Len()
if fileData.Len() > MIN_BLOCK_SIZE*2 { // Candidate for rolling sum split
rollIn, rollOut := fileData, fileData // Shallow copy the file data
rollInBase, rollOutBase := 0, 0
rollInPos, rollOutPos := 0, 0
rollInSlice, _ := rollIn.ReadSlice()
rollOutSlice, _ := rollOut.ReadSlice()
partSum := maxSum
var maxd = uint32(0)
for rollInPos < fileData.Len() {
if rollInPos-rollInBase >= len(rollInSlice) { // Next slice please
rollInBase, _ = rollIn.ReadSeek(len(rollInSlice), os.SEEK_CUR)
rollInSlice, _ = rollIn.ReadSlice()
}
if rollOutPos-rollOutBase >= len(rollOutSlice) { // Next slice please
rollOutBase, _ = rollOut.ReadSeek(len(rollOutSlice), os.SEEK_CUR)
rollOutSlice, _ = rollOut.ReadSlice()
}
if rollInPos >= MIN_BLOCK_SIZE {
partSum.Rollout(rollOutSlice[rollOutPos-rollOutBase])
rollOutPos++
}
partSum.Rollin(rollInSlice[rollInPos-rollInBase])
rollInPos++
if rollInPos >= MIN_BLOCK_SIZE {
d := partSum.Digest()
if d >= maxd {
maxd = d
splitPosition = rollInPos
maxSum = partSum // Keep the sum so we can continue from here
}
}
}
}
// Split an swap
right := fileData.Split(splitPosition)
blockData = fileData
fileData = right
offset += int64(blockData.Len())
session.ReadData += int64(blockData.Len())
// TODO: add encryption and custom compression here
var datakey core.Byte128
id := session.Client.StoreData(core.BlockDataTypeZlib, blockData, nil)
links = append(links, id)
chain.ChainBlocks = append(chain.ChainBlocks, id)
chain.DecryptKeys = append(chain.DecryptKeys, datakey)
}
if len(chain.ChainBlocks) > 1 {
id := session.Client.StoreData(core.BlockDataTypeZlib, SerializeToByteArray(chain), links)
entry.ContentType = ContentTypeFileChain
entry.ContentBlockID = id
} else {
entry.ContentType = ContentTypeFileData
entry.ContentBlockID = chain.ChainBlocks[0]
entry.DecryptKey = chain.DecryptKeys[0]
}
return nil
}
func (d *delta) genDelta(src io.ReadSeeker, srcLen int64) (err error) {
var (
c byte
p []byte
rs rollsum.Rollsum
rb *rotateBuffer
srcPos int64
matchAt int64
blockLen int
)
if d.debug {
d.dumpSign()
}
blockLen = int(d.sig.block_len)
rb = NewRotateBuffer(srcLen, d.sig.block_len, src)
p, srcPos, err = rb.rollFirst()
if err == nil {
// 计算初始weaksum
rs.Init()
rs.Update(p)
for err == nil {
// srcPos是当前读取src文件的绝对位置，matchAt对应于dstSig和dst文件的位置
matchAt = d.findMatch(p, srcPos, rs.Digest())
if matchAt < 0 {
p, c, srcPos, err = rb.rollByte()
if err != nil {
break
}
rs.Rotate(c, p[blockLen-1])
} else {
p, srcPos, err = rb.rollBlock()
rs.Init()
if err != nil {
break
}
rs.Update(p)
}
}
} else if err == noBytesLeft {
// reader没有内容
if d.debug {
fmt.Println("reader has no content:", srcLen)
}
err = nil
return
}
if err != noBytesLeft && err != notEnoughBytes {
// 出错
return
}
if d.debug {
fmt.Printf("rotate buffer left no more than a block: block=%d start=%d end=%d absHead=%d absTail=%d eof=%v\n",
blockLen, rb.start, rb.end, rb.absHead, rb.absTail, rb.eof)
}
if p, c, srcPos, err = rb.rollLeft(); err == nil {
rs.Init()
rs.Update(p)
for err == nil {
matchAt = d.findMatch(p, srcPos, rs.Digest())
if matchAt >= 0 {
// 剩余的内容已经匹配到，不需要继续处理
break
} else {
p, c, srcPos, err = rb.rollLeft()
if err != nil {
break
}
rs.Rollout(c)
}
}
} else {
if d.debug {
fmt.Println(string(p), c, srcPos, err)
}
}
if err == noBytesLeft || err == nil {
if d.debug {
fmt.Println("last match stat:", d.ms.match, d.ms.pos, d.ms.length)
}
d.mss = append(d.mss, d.ms)
err = nil
}
return
}
