Exemple #1
0
func ZlibCompress(src bytearray.ByteArray) (dst bytearray.ByteArray) {
	src.ReadSeek(0, bytearray.SEEK_SET)
	zw := zpool.GetWriter(&dst)
	CopyOrPanic(zw, &src)
	zw.Close()
	zpool.PutWriter(zw)
	return dst
}
Exemple #2
0
func ZlibUncompress(src bytearray.ByteArray) (dst bytearray.ByteArray) {
	src.ReadSeek(0, bytearray.SEEK_SET)
	zr, err := zlib.NewReader(&src)
	if err != nil {
		panic(err)
	}
	defer zr.Close()
	CopyOrPanic(&dst, zr)
	return dst
}
Exemple #3
0
func (session *BackupSession) storeFile(path string, entry *FileEntry) (err error) {
	defer func() {
		// Panic error handling
		if r := recover(); r != nil {
			// we need this because some obscure files on OSX does open but then generates "bad file descriptor" on read
			if e, ok := r.(*os.PathError); ok && e.Err == syscall.EBADF {
				err = e.Err
			} else {
				panic(r) // Any other error is not normal and should panic
			}
		}
	}()

	var links []core.Byte128

	chain := FileChainBlock{}

	var file *os.File
	if file, err = os.Open(path); err != nil {
		return err
	}
	defer file.Close()

	var maxSum rollsum.Rollsum
	maxSum.Init()

	var fileData bytearray.ByteArray
	defer fileData.Release()

	for offset := int64(0); offset < int64(entry.FileSize); {
		Debug("storeFile(%s) offset %d", path, offset)

		session.PrintStoreProgress(PROGRESS_INTERVAL_SECS)

		var left int64 = int64(entry.FileSize) - offset
		var maxBlockSize int = MAX_BLOCK_SIZE
		if left < int64(maxBlockSize) {
			maxBlockSize = int(left)
		}

		var blockData bytearray.ByteArray

		// Fill the fileData buffer
		core.CopyNOrPanic(&fileData, file, maxBlockSize-fileData.Len())
		fileData.ReadSeek(0, os.SEEK_CUR) // TODO: figure out why this line is here because I do not remember

		var splitPosition int = fileData.Len()
		if fileData.Len() > MIN_BLOCK_SIZE*2 { // Candidate for rolling sum split
			rollIn, rollOut := fileData, fileData // Shallow copy the file data
			rollInBase, rollOutBase := 0, 0
			rollInPos, rollOutPos := 0, 0
			rollInSlice, _ := rollIn.ReadSlice()
			rollOutSlice, _ := rollOut.ReadSlice()

			partSum := maxSum
			var maxd = uint32(0)
			for rollInPos < fileData.Len() {
				if rollInPos-rollInBase >= len(rollInSlice) { // Next slice please
					rollInBase, _ = rollIn.ReadSeek(len(rollInSlice), os.SEEK_CUR)
					rollInSlice, _ = rollIn.ReadSlice()
				}
				if rollOutPos-rollOutBase >= len(rollOutSlice) { // Next slice please
					rollOutBase, _ = rollOut.ReadSeek(len(rollOutSlice), os.SEEK_CUR)
					rollOutSlice, _ = rollOut.ReadSlice()
				}

				if rollInPos >= MIN_BLOCK_SIZE {
					partSum.Rollout(rollOutSlice[rollOutPos-rollOutBase])
					rollOutPos++
				}
				partSum.Rollin(rollInSlice[rollInPos-rollInBase])
				rollInPos++

				if rollInPos >= MIN_BLOCK_SIZE {
					d := partSum.Digest()
					if d >= maxd {
						maxd = d
						splitPosition = rollInPos
						maxSum = partSum // Keep the sum so we can continue from here
					}
				}
			}
		}

		// Split an swap
		right := fileData.Split(splitPosition)
		blockData = fileData
		fileData = right

		offset += int64(blockData.Len())
		session.ReadData += int64(blockData.Len())

		// TODO: add encryption and custom compression here
		var datakey core.Byte128

		id := session.Client.StoreData(core.BlockDataTypeZlib, blockData, nil)
		links = append(links, id)
		chain.ChainBlocks = append(chain.ChainBlocks, id)
		chain.DecryptKeys = append(chain.DecryptKeys, datakey)
	}

	if len(chain.ChainBlocks) > 1 {
		id := session.Client.StoreData(core.BlockDataTypeZlib, SerializeToByteArray(chain), links)
		entry.ContentType = ContentTypeFileChain
		entry.ContentBlockID = id

	} else {
		entry.ContentType = ContentTypeFileData
		entry.ContentBlockID = chain.ChainBlocks[0]
		entry.DecryptKey = chain.DecryptKeys[0]
	}

	return nil
}