Ejemplo n.º 1
0
// flush flush writer buffer.
func (b *SuperBlock) flush(force bool) (err error) {
	var (
		fd     uintptr
		offset int64
		size   int64
	)
	if b.write++; !force && b.write < b.conf.Block.SyncWrite {
		return
	}
	b.write = 0
	offset = needle.BlockOffset(b.syncOffset)
	size = needle.BlockOffset(b.Offset - b.syncOffset)
	fd = b.w.Fd()
	if b.conf.Block.Syncfilerange {
		if err = myos.Syncfilerange(fd, offset, size, myos.SYNC_FILE_RANGE_WRITE); err != nil {
			log.Errorf("block: %s Syncfilerange() error(%v)", b.File, err)
			b.LastErr = err
			return
		}
	} else {
		if err = myos.Fdatasync(fd); err != nil {
			log.Errorf("block: %s Fdatasync() error(%v)", b.File, err)
			b.LastErr = err
			return
		}
	}
	if err = myos.Fadvise(fd, offset, size, myos.POSIX_FADV_DONTNEED); err == nil {
		b.syncOffset = b.Offset
	} else {
		log.Errorf("block: %s Fadvise() error(%v)", b.File, err)
		b.LastErr = err
	}
	return
}
Ejemplo n.º 2
0
// init recovery super block from index or super block.
func (v *Volume) init() (err error) {
	var (
		size       int64
		offset     uint32
		lastOffset uint32
	)
	// recovery from index
	if err = v.Indexer.Recovery(func(ix *index.Index) error {
		// must no less than last offset
		if ix.Offset < lastOffset {
			log.Error("recovery index: %s lastoffset: %d error(%v)", ix, lastOffset, errors.ErrIndexOffset)
			return errors.ErrIndexOffset
		}
		// WARN if index's offset more than the block, discard it.
		if size = int64(ix.Size) + needle.BlockOffset(ix.Offset); size > v.Block.Size {
			log.Error("recovery index: %s EOF", ix)
			return errors.ErrIndexEOF
		}
		v.needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
		offset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
		lastOffset = ix.Offset
		return nil
	}); err != nil && err != errors.ErrIndexEOF {
		return
	}
	// recovery from super block
	if err = v.Block.Recovery(offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
		if n.Flag == needle.FlagOK {
			if err1 = v.Indexer.Write(n.Key, so, n.TotalSize); err1 != nil {
				return
			}
		} else {
			so = needle.CacheDelOffset
		}
		v.needles[n.Key] = needle.NewCache(so, n.TotalSize)
		return
	}); err != nil {
		return
	}
	// recheck offset, keep size and offset consistency
	if v.Block.Size != needle.BlockOffset(v.Block.Offset) {
		log.Error("block: %s size: %d, offset: %d (%d size) not consistency", v.Block.File, v.Block.Size, v.Block.Offset, needle.BlockOffset(v.Block.Offset))
		return errors.ErrSuperBlockOffset
	}
	// flush index
	err = v.Indexer.Flush()
	return
}
Ejemplo n.º 3
0
// Recovery recovery needles map from super block.
func (b *SuperBlock) Recovery(offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	// WARN block may be no left data, must update block offset first
	if offset == 0 {
		offset = needle.NeedleOffset(_headerOffset)
	}
	b.Offset = offset
	if err = b.Scan(b.r, offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
		if err1 = fn(n, so, eo); err1 == nil {
			b.Offset = eo
		}
		return
	}); err != nil {
		return
	}
	// advise random read
	// POSIX_FADV_RANDOM disables file readahead entirely.
	// These changes affect the entire file, not just the specified region
	// (but other open file handles to the same file are unaffected).
	if err = myos.Fadvise(b.r.Fd(), 0, 0, myos.POSIX_FADV_RANDOM); err != nil {
		log.Errorf("block: %s Fadvise() error(%v)", b.File)
		return
	}
	// reset b.w offset, discard left space which can't parse to a needle
	if _, err = b.w.Seek(needle.BlockOffset(b.Offset), os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File, err)
	}
	return
}
Ejemplo n.º 4
0
// WriteAt write a needle by specified offset;
func (b *SuperBlock) WriteAt(offset uint32, n *needle.Needle) (err error) {
	if b.LastErr != nil {
		return b.LastErr
	}
	if _, err = b.w.WriteAt(n.Buffer(), needle.BlockOffset(offset)); err != nil {
		b.LastErr = err
	}
	return
}
Ejemplo n.º 5
0
// Delete logical del a needls, only update the flag to it.
func (b *SuperBlock) Delete(offset uint32) (err error) {
	if b.LastErr != nil {
		return b.LastErr
	}
	// WriteAt won't update the file offset.
	_, err = b.w.WriteAt(needle.FlagDelBytes, needle.BlockOffset(offset)+needle.FlagOffset)
	b.LastErr = err
	return
}
Ejemplo n.º 6
0
// ReadAt read a needle by specified offset, before call it, must set needle
// TotalSize.
func (b *SuperBlock) ReadAt(n *needle.Needle) (err error) {
	if b.LastErr != nil {
		return b.LastErr
	}
	if _, err = b.r.ReadAt(n.Buffer(), needle.BlockOffset(n.Offset)); err == nil {
		err = n.Parse()
	} else {
		b.LastErr = err
	}
	return
}
Ejemplo n.º 7
0
func compareTestOffset(b *SuperBlock, n *needle.Needle, offset uint32) (err error) {
	var v int64
	if b.Offset != offset+needle.NeedleOffset(int64(n.TotalSize)) {
		err = fmt.Errorf("b.Offset: %d not match %d", b.Offset, offset)
		return
	}
	if v, err = b.w.Seek(0, os.SEEK_CUR); err != nil {
		err = fmt.Errorf("b.Seek() error(%v)", err)
		return
	} else {
		if v != needle.BlockOffset(b.Offset) {
			err = fmt.Errorf("offset: %d not match", v)
			return
		}
	}
	return
}
Ejemplo n.º 8
0
// Recovery recovery needles map from super block.
func (b *SuperBlock) Recovery(offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	var rsize int64
	// WARN block may be no left data, must update block offset first
	if offset == 0 {
		offset = needle.NeedleOffset(_headerOffset)
	}
	b.Offset = offset
	if err = b.Scan(b.r, offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
		if err1 = fn(n, so, eo); err1 == nil {
			b.Offset = eo
		}
		return
	}); err != nil {
		return
	}
	// advise random read
	// POSIX_FADV_RANDOM disables file readahead entirely.
	// These changes affect the entire file, not just the specified region
	// (but other open file handles to the same file are unaffected).
	if err = myos.Fadvise(b.r.Fd(), 0, 0, myos.POSIX_FADV_RANDOM); err != nil {
		log.Errorf("block: %s Fadvise() error(%v)", b.File)
		return
	}
	rsize = needle.BlockOffset(b.Offset)
	// reset b.w offset, discard left space which can't parse to a needle
	if _, err = b.w.Seek(rsize, os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File, err)
		return
	}
	// recheck offset, keep size and offset consistency
	if b.Size != rsize {
		log.Warningf("block: %s [real size: %d, offset: %d] but [size: %d, offset: %d] not consistency, truncate file for force recovery, this may lost data",
			b.File, b.Size, needle.NeedleOffset(b.Size),
			rsize, b.Offset)
		// truncate file
		if err = b.w.Truncate(rsize); err != nil {
			log.Errorf("block: %s Truncate() error(%v)", b.File, err)
		}
	}
	return
}
Ejemplo n.º 9
0
// Scan scan a block file.
func (b *SuperBlock) Scan(r *os.File, offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	var (
		so, eo uint32
		bso    int64
		fi     os.FileInfo
		fd     = r.Fd()
		n      = new(needle.Needle)
		rd     = bufio.NewReaderSize(r, b.conf.Block.BufferSize)
	)
	if offset == 0 {
		offset = needle.NeedleOffset(_headerOffset)
	}
	so, eo = offset, offset
	bso = needle.BlockOffset(so)
	// advise sequential read
	if fi, err = r.Stat(); err != nil {
		log.Errorf("block: %s Stat() error(%v)", b.File)
		return
	}
	if err = myos.Fadvise(fd, bso, fi.Size(), myos.POSIX_FADV_SEQUENTIAL); err != nil {
		log.Errorf("block: %s Fadvise() error(%v)", b.File)
		return
	}
	log.Infof("scan block: %s from offset: %d", b.File, offset)
	if _, err = r.Seek(bso, os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File)
		return
	}
	for {
		if err = n.ParseFrom(rd); err != nil {
			if err != io.EOF {
				log.Errorf("block: parse needle from offset: %d:%d error(%v)", so, eo, err)
			}
			break
		}
		if n.TotalSize > int32(b.conf.BlockMaxSize) {
			log.Errorf("scan block: %s error(%v)", n, errors.ErrNeedleSize)
			err = errors.ErrNeedleSize
			break
		}
		if log.V(1) {
			log.Info(n.String())
		}
		eo += n.IncrOffset
		if err = fn(n, so, eo); err != nil {
			log.Errorf("block: callback from offset: %d:%d error(%v)", so, eo, err)
			break
		}
		so = eo
	}
	if err == io.EOF {
		// advise no need page cache
		if err = myos.Fadvise(fd, bso, needle.BlockOffset(eo-so), myos.POSIX_FADV_DONTNEED); err != nil {
			log.Errorf("block: %s Fadvise() error(%v)", b.File)
			return
		}
		log.Infof("scan block: %s to offset: %d [ok]", b.File, eo)
		err = nil
	} else {
		log.Infof("scan block: %s to offset: %d error(%v) [failed]", b.File, eo, err)
	}
	return
}