func (self *CatFile) appendToLastBlock(b []byte, offset int64) (int, error) { blockOff := self.blockNumber() - 1 // index of the last block err := self.getBlock(blockOff) if err == ErrNoBlocks { return 0, nil } if err != nil { return 0, err } if len(self.curBlockContent) == int(config.BlockSize()) { // if the last block is full // nothing to write return 0, nil } if offset > int64(len(self.curBlockContent)) { offset = int64(len(self.curBlockContent)) } blockRemain := int(config.BlockSize() - offset) self.curChanged = true if blockRemain >= len(b) { self.curBlockContent = append(self.curBlockContent[:offset], b...) return len(b), nil } self.curBlockContent = append(self.curBlockContent[:offset], b[:blockRemain]...) return blockRemain, nil }
func (self *CatFile) getBlock(blockOff int64) error { // if current block offset is the one we want to get if self.curBlockOff == blockOff { return nil } err := self.Sync() if err != nil { return err } master := self.pool.MasterServer() blockquery := &proc.BlockQueryParam{ Path: self.path, Offset: config.BlockSize() * blockOff, Length: config.BlockSize(), Lease: self.lease, } // get block meta data var resp proc.GetBlocksLocationResponse err = master.GetBlockLocation(blockquery, &resp) if err != nil { return err } if len(resp.Blocks) == 0 { self.curBlock = nil return ErrNoBlocks } // contact data server location := resp.Blocks[0].Locations[0] dataServer := self.pool.DataServer(location) var lease proc.CatLease param := &proc.GetBlockParam{ Block: resp.Blocks[0], } err = dataServer.GetBlock(param, &lease) if err != nil { return err } // get data blockClient := self.pool.NewBlockClient(location) ch := make(chan []byte) go blockClient.GetBlock(ch, lease.ID) var blockContent []byte for data := range ch { blockContent = append(blockContent, data...) } // set EOF and curBlock info self.isEOF = resp.EOF self.curBlock = resp.Blocks[0] self.curBlockContent = blockContent self.curBlockOff = blockOff return nil }
func (self *CatDFSFile) Status() *proc.CatFileStatus { status := &proc.CatFileStatus{ Filename: self.filename, Length: int64(len(self.blocks)) * config.BlockSize(), IsDir: false, } return status }
// WriteAt writes len(b) bytes to the File starting at byte offset off. It // returns the number of bytes written and an error, if any. WriteAt returns a // non-nil error when n != len(b). func (self *CatFile) WriteAt(b []byte, off int64) (int, error) { self.lock.Lock() defer self.lock.Unlock() dataWrite := 0 // blockOffset of off blockOff := off / config.BlockSize() // ceiling of length / blocksize fileBlockNumber := self.blockNumber() offset := off % config.BlockSize() for { // if it is the last block or more if blockOff >= fileBlockNumber-1 { offset += (blockOff - fileBlockNumber + 1) * config.BlockSize() n, err := self.appendToLastBlock(b, offset) dataWrite += n // read enough or there is an err if err != nil || dataWrite == len(b) { return dataWrite, err } n, err = self.appendBlock(b[dataWrite:]) return n + dataWrite, err } // get the block of blockOff err := self.getBlock(blockOff) if err != nil { return dataWrite, err } // this should work, because the size of curBlockContent should be the block size n := copy(self.curBlockContent[offset:], b[dataWrite:]) dataWrite += n // set current has changed self.curChanged = true if dataWrite == len(b) { offset = int64(n) break } offset = 0 blockOff++ self.Sync() self.setFileOffset(blockOff, offset) } self.setFileOffset(blockOff, offset) return dataWrite, nil }
func (self *CatDFSFile) QueryBlocks(offset int64, length int64) (BlockList, bool) { blockSize := config.BlockSize() startBlockOff := offset / blockSize endBlockOff := (offset+length-1)/blockSize + 1 // not include eof := false if endBlockOff >= int64(len(self.blocks)) { endBlockOff = int64(len(self.blocks)) eof = true } return self.blocks[startBlockOff:endBlockOff], eof }
// ReadAt reads len(b) bytes from the File starting at byte offset off. It // returns the number of bytes read and the error, if any. ReadAt always returns // a non-nil error when n < len(b). At end of file, that error is io.EOF. func (self *CatFile) ReadAt(b []byte, off int64) (int, error) { self.lock.Lock() defer self.lock.Unlock() // blockOffset of off blockOff := off / config.BlockSize() err := self.getBlock(blockOff) if err != nil { return 0, err } // offset of off in a block offset := off % config.BlockSize() dataRead := 0 for { n := copy(b[dataRead:], self.curBlockContent[self.offset():]) dataRead += n // if read enough data if dataRead == len(b) { offset = int64(n) break } // if it is the end of file if self.isEOF { self.setFileOffset(blockOff, int64(len(self.curBlockContent))) return dataRead, io.EOF } // rest offset and blockOff offset = 0 blockOff++ // get next block err := self.getBlock(blockOff) if err != nil { return 0, err } } // set offset of the file self.setFileOffset(blockOff, offset) return dataRead, nil }
func (self *CatFile) appendBlock(b []byte) (int, error) { dataWrite := 0 n := 0 blockOff := self.blockNumber() - 1 // index of the last block master := self.pool.MasterServer() for dataWrite < len(b) { err := self.Sync() if err != nil && dataWrite == 0 { return dataWrite, err } if err != nil { // && dataWrite != 0 // write new block failed // TODO abandom block ? return dataWrite, err } blockContent := make([]byte, config.BlockSize()) n = copy(blockContent, b[dataWrite:]) dataWrite += n blockContent = blockContent[:n] blockOff++ // add block offset by 1 param := &proc.AddBlockParam{ Path: self.path, Lease: self.lease, } var block proc.CatBlock err = master.AddBlock(param, &block) if err != nil { return dataWrite, err } self.curBlock = &block self.curBlockContent = blockContent self.curChanged = true } self.setFileOffset(blockOff, int64(len(self.curBlockContent))) return n, nil }
func (self *CatFile) blockNumber() int64 { return (self.filestatus.Length-1)/config.BlockSize() + 1 }
func (self *CatFile) setFileOffset(blockOff, offset int64) { self.fileOffset = blockOff*config.BlockSize() + offset }
func (self *CatFile) blockOffset() int64 { return self.fileOffset / config.BlockSize() }