func (self *CatFile) appendToLastBlock(b []byte, offset int64) (int, error) {
blockOff := self.blockNumber() - 1 // index of the last block
err := self.getBlock(blockOff)
if err == ErrNoBlocks {
return 0, nil
}
if err != nil {
return 0, err
}
if len(self.curBlockContent) == int(config.BlockSize()) {
// if the last block is full
// nothing to write
return 0, nil
}
if offset > int64(len(self.curBlockContent)) {
offset = int64(len(self.curBlockContent))
}
blockRemain := int(config.BlockSize() - offset)
self.curChanged = true
if blockRemain >= len(b) {
self.curBlockContent = append(self.curBlockContent[:offset], b...)
return len(b), nil
}
self.curBlockContent = append(self.curBlockContent[:offset], b[:blockRemain]...)
return blockRemain, nil
}
func (self *CatFile) getBlock(blockOff int64) error {
// if current block offset is the one we want to get
if self.curBlockOff == blockOff {
return nil
}
err := self.Sync()
if err != nil {
return err
}
master := self.pool.MasterServer()
blockquery := &proc.BlockQueryParam{
Path: self.path,
Offset: config.BlockSize() * blockOff,
Length: config.BlockSize(),
Lease: self.lease,
}
// get block meta data
var resp proc.GetBlocksLocationResponse
err = master.GetBlockLocation(blockquery, &resp)
if err != nil {
return err
}
if len(resp.Blocks) == 0 {
self.curBlock = nil
return ErrNoBlocks
}
// contact data server
location := resp.Blocks[0].Locations[0]
dataServer := self.pool.DataServer(location)
var lease proc.CatLease
param := &proc.GetBlockParam{
Block: resp.Blocks[0],
}
err = dataServer.GetBlock(param, &lease)
if err != nil {
return err
}
// get data
blockClient := self.pool.NewBlockClient(location)
ch := make(chan []byte)
go blockClient.GetBlock(ch, lease.ID)
var blockContent []byte
for data := range ch {
blockContent = append(blockContent, data...)
}
// set EOF and curBlock info
self.isEOF = resp.EOF
self.curBlock = resp.Blocks[0]
self.curBlockContent = blockContent
self.curBlockOff = blockOff
return nil
}
func (self *CatDFSFile) Status() *proc.CatFileStatus {
status := &proc.CatFileStatus{
Filename: self.filename,
Length: int64(len(self.blocks)) * config.BlockSize(),
IsDir: false,
}
return status
}
// WriteAt writes len(b) bytes to the File starting at byte offset off. It
// returns the number of bytes written and an error, if any. WriteAt returns a
// non-nil error when n != len(b).
func (self *CatFile) WriteAt(b []byte, off int64) (int, error) {
self.lock.Lock()
defer self.lock.Unlock()
dataWrite := 0
// blockOffset of off
blockOff := off / config.BlockSize()
// ceiling of length / blocksize
fileBlockNumber := self.blockNumber()
offset := off % config.BlockSize()
for {
// if it is the last block or more
if blockOff >= fileBlockNumber-1 {
offset += (blockOff - fileBlockNumber + 1) * config.BlockSize()
n, err := self.appendToLastBlock(b, offset)
dataWrite += n
// read enough or there is an err
if err != nil || dataWrite == len(b) {
return dataWrite, err
}
n, err = self.appendBlock(b[dataWrite:])
return n + dataWrite, err
}
// get the block of blockOff
err := self.getBlock(blockOff)
if err != nil {
return dataWrite, err
}
// this should work, because the size of curBlockContent should be the block size
n := copy(self.curBlockContent[offset:], b[dataWrite:])
dataWrite += n
// set current has changed
self.curChanged = true
if dataWrite == len(b) {
offset = int64(n)
break
}
offset = 0
blockOff++
self.Sync()
self.setFileOffset(blockOff, offset)
}
self.setFileOffset(blockOff, offset)
return dataWrite, nil
}
func (self *CatDFSFile) QueryBlocks(offset int64, length int64) (BlockList, bool) {
blockSize := config.BlockSize()
startBlockOff := offset / blockSize
endBlockOff := (offset+length-1)/blockSize + 1 // not include
eof := false
if endBlockOff >= int64(len(self.blocks)) {
endBlockOff = int64(len(self.blocks))
eof = true
}
return self.blocks[startBlockOff:endBlockOff], eof
}
// ReadAt reads len(b) bytes from the File starting at byte offset off. It
// returns the number of bytes read and the error, if any. ReadAt always returns
// a non-nil error when n < len(b). At end of file, that error is io.EOF.
func (self *CatFile) ReadAt(b []byte, off int64) (int, error) {
self.lock.Lock()
defer self.lock.Unlock()
// blockOffset of off
blockOff := off / config.BlockSize()
err := self.getBlock(blockOff)
if err != nil {
return 0, err
}
// offset of off in a block
offset := off % config.BlockSize()
dataRead := 0
for {
n := copy(b[dataRead:], self.curBlockContent[self.offset():])
dataRead += n
// if read enough data
if dataRead == len(b) {
offset = int64(n)
break
}
// if it is the end of file
if self.isEOF {
self.setFileOffset(blockOff, int64(len(self.curBlockContent)))
return dataRead, io.EOF
}
// rest offset and blockOff
offset = 0
blockOff++
// get next block
err := self.getBlock(blockOff)
if err != nil {
return 0, err
}
}
// set offset of the file
self.setFileOffset(blockOff, offset)
return dataRead, nil
}
func (self *CatFile) appendBlock(b []byte) (int, error) {
dataWrite := 0
n := 0
blockOff := self.blockNumber() - 1 // index of the last block
master := self.pool.MasterServer()
for dataWrite < len(b) {
err := self.Sync()
if err != nil && dataWrite == 0 {
return dataWrite, err
}
if err != nil {
// && dataWrite != 0
// write new block failed
// TODO abandom block ?
return dataWrite, err
}
blockContent := make([]byte, config.BlockSize())
n = copy(blockContent, b[dataWrite:])
dataWrite += n
blockContent = blockContent[:n]
blockOff++ // add block offset by 1
param := &proc.AddBlockParam{
Path: self.path,
Lease: self.lease,
}
var block proc.CatBlock
err = master.AddBlock(param, &block)
if err != nil {
return dataWrite, err
}
self.curBlock = &block
self.curBlockContent = blockContent
self.curChanged = true
}
self.setFileOffset(blockOff, int64(len(self.curBlockContent)))
return n, nil
}
func (self *CatFile) blockNumber() int64 {
return (self.filestatus.Length-1)/config.BlockSize() + 1
}
func (self *CatFile) setFileOffset(blockOff, offset int64) {
self.fileOffset = blockOff*config.BlockSize() + offset
}
func (self *CatFile) blockOffset() int64 {
return self.fileOffset / config.BlockSize()
}