Esempio n. 1
0
File: sum.go Progetto: smtc/rsync
// use rollsum do weak sum
func weakSum(p []byte) (s uint32) {
	var rs rollsum.Rollsum

	rs.Init()
	rs.Update(p)
	return rs.Digest()
}
Esempio n. 2
0
func (session *BackupSession) storeFile(path string, entry *FileEntry) (err error) {
	defer func() {
		// Panic error handling
		if r := recover(); r != nil {
			// we need this because some obscure files on OSX does open but then generates "bad file descriptor" on read
			if e, ok := r.(*os.PathError); ok && e.Err == syscall.EBADF {
				err = e.Err
			} else {
				panic(r) // Any other error is not normal and should panic
			}
		}
	}()

	var links []core.Byte128

	chain := FileChainBlock{}

	var file *os.File
	if file, err = os.Open(path); err != nil {
		return err
	}
	defer file.Close()

	var maxSum rollsum.Rollsum
	maxSum.Init()

	var fileData bytearray.ByteArray
	defer fileData.Release()

	for offset := int64(0); offset < int64(entry.FileSize); {
		Debug("storeFile(%s) offset %d", path, offset)

		session.PrintStoreProgress(PROGRESS_INTERVAL_SECS)

		var left int64 = int64(entry.FileSize) - offset
		var maxBlockSize int = MAX_BLOCK_SIZE
		if left < int64(maxBlockSize) {
			maxBlockSize = int(left)
		}

		var blockData bytearray.ByteArray

		// Fill the fileData buffer
		core.CopyNOrPanic(&fileData, file, maxBlockSize-fileData.Len())
		fileData.ReadSeek(0, os.SEEK_CUR) // TODO: figure out why this line is here because I do not remember

		var splitPosition int = fileData.Len()
		if fileData.Len() > MIN_BLOCK_SIZE*2 { // Candidate for rolling sum split
			rollIn, rollOut := fileData, fileData // Shallow copy the file data
			rollInBase, rollOutBase := 0, 0
			rollInPos, rollOutPos := 0, 0
			rollInSlice, _ := rollIn.ReadSlice()
			rollOutSlice, _ := rollOut.ReadSlice()

			partSum := maxSum
			var maxd = uint32(0)
			for rollInPos < fileData.Len() {
				if rollInPos-rollInBase >= len(rollInSlice) { // Next slice please
					rollInBase, _ = rollIn.ReadSeek(len(rollInSlice), os.SEEK_CUR)
					rollInSlice, _ = rollIn.ReadSlice()
				}
				if rollOutPos-rollOutBase >= len(rollOutSlice) { // Next slice please
					rollOutBase, _ = rollOut.ReadSeek(len(rollOutSlice), os.SEEK_CUR)
					rollOutSlice, _ = rollOut.ReadSlice()
				}

				if rollInPos >= MIN_BLOCK_SIZE {
					partSum.Rollout(rollOutSlice[rollOutPos-rollOutBase])
					rollOutPos++
				}
				partSum.Rollin(rollInSlice[rollInPos-rollInBase])
				rollInPos++

				if rollInPos >= MIN_BLOCK_SIZE {
					d := partSum.Digest()
					if d >= maxd {
						maxd = d
						splitPosition = rollInPos
						maxSum = partSum // Keep the sum so we can continue from here
					}
				}
			}
		}

		// Split an swap
		right := fileData.Split(splitPosition)
		blockData = fileData
		fileData = right

		offset += int64(blockData.Len())
		session.ReadData += int64(blockData.Len())

		// TODO: add encryption and custom compression here
		var datakey core.Byte128

		id := session.Client.StoreData(core.BlockDataTypeZlib, blockData, nil)
		links = append(links, id)
		chain.ChainBlocks = append(chain.ChainBlocks, id)
		chain.DecryptKeys = append(chain.DecryptKeys, datakey)
	}

	if len(chain.ChainBlocks) > 1 {
		id := session.Client.StoreData(core.BlockDataTypeZlib, SerializeToByteArray(chain), links)
		entry.ContentType = ContentTypeFileChain
		entry.ContentBlockID = id

	} else {
		entry.ContentType = ContentTypeFileData
		entry.ContentBlockID = chain.ChainBlocks[0]
		entry.DecryptKey = chain.DecryptKeys[0]
	}

	return nil
}
Esempio n. 3
0
File: delta.go Progetto: smtc/rsync
func (d *delta) genDelta(src io.ReadSeeker, srcLen int64) (err error) {
	var (
		c        byte
		p        []byte
		rs       rollsum.Rollsum
		rb       *rotateBuffer
		srcPos   int64
		matchAt  int64
		blockLen int
	)

	if d.debug {
		d.dumpSign()
	}

	blockLen = int(d.sig.block_len)

	rb = NewRotateBuffer(srcLen, d.sig.block_len, src)
	p, srcPos, err = rb.rollFirst()
	if err == nil {
		// 计算初始weaksum
		rs.Init()
		rs.Update(p)
		for err == nil {
			// srcPos是当前读取src文件的绝对位置,matchAt对应于dstSig和dst文件的位置
			matchAt = d.findMatch(p, srcPos, rs.Digest())
			if matchAt < 0 {
				p, c, srcPos, err = rb.rollByte()
				if err != nil {
					break
				}
				rs.Rotate(c, p[blockLen-1])
			} else {
				p, srcPos, err = rb.rollBlock()
				rs.Init()
				if err != nil {
					break
				}
				rs.Update(p)
			}
		}
	} else if err == noBytesLeft {
		// reader没有内容
		if d.debug {
			fmt.Println("reader has no content:", srcLen)
		}
		err = nil
		return
	}

	if err != noBytesLeft && err != notEnoughBytes {
		// 出错
		return
	}

	if d.debug {
		fmt.Printf("rotate buffer left no more than a block: block=%d start=%d end=%d absHead=%d absTail=%d eof=%v\n",
			blockLen, rb.start, rb.end, rb.absHead, rb.absTail, rb.eof)
	}

	if p, c, srcPos, err = rb.rollLeft(); err == nil {
		rs.Init()
		rs.Update(p)

		for err == nil {
			matchAt = d.findMatch(p, srcPos, rs.Digest())

			if matchAt >= 0 {
				// 剩余的内容已经匹配到,不需要继续处理
				break
			} else {
				p, c, srcPos, err = rb.rollLeft()
				if err != nil {
					break
				}
				rs.Rollout(c)
			}
		}
	} else {
		if d.debug {
			fmt.Println(string(p), c, srcPos, err)
		}
	}

	if err == noBytesLeft || err == nil {
		if d.debug {
			fmt.Println("last match stat:", d.ms.match, d.ms.pos, d.ms.length)
		}
		d.mss = append(d.mss, d.ms)
		err = nil
	}

	return
}