Пример #1
0
// Scan scan a indexer file.
func (i *Indexer) Scan(r *os.File, fn func(*Index) error) (err error) {
	var (
		data []byte
		fi   os.FileInfo
		fd   = r.Fd()
		ix   = &Index{}
		rd   = bufio.NewReaderSize(r, i.conf.Index.BufferSize)
	)
	log.Infof("scan index: %s", i.File)
	// advise sequential read
	if fi, err = r.Stat(); err != nil {
		log.Errorf("index: %s Stat() error(%v)", i.File)
		return
	}
	if err = myos.Fadvise(fd, 0, fi.Size(), myos.POSIX_FADV_SEQUENTIAL); err != nil {
		log.Errorf("index: %s Fadvise() error(%v)", i.File)
		return
	}
	if _, err = r.Seek(0, os.SEEK_SET); err != nil {
		log.Errorf("index: %s Seek() error(%v)", i.File, err)
		return
	}
	for {
		if data, err = rd.Peek(_indexSize); err != nil {
			break
		}
		if err = ix.parse(data); err != nil {
			break
		}
		if ix.Size > int32(i.conf.BlockMaxSize) {
			log.Errorf("scan index: %s error(%v)", ix, errors.ErrIndexSize)
			err = errors.ErrIndexSize
			break
		}
		if _, err = rd.Discard(_indexSize); err != nil {
			break
		}
		if log.V(1) {
			log.Info(ix.String())
		}
		if err = fn(ix); err != nil {
			break
		}
	}
	if err == io.EOF {
		// advise no need page cache
		if err = myos.Fadvise(fd, 0, fi.Size(), myos.POSIX_FADV_DONTNEED); err == nil {
			err = nil
			log.Infof("scan index: %s [ok]", i.File)
			return
		} else {
			log.Errorf("index: %s Fadvise() error(%v)", i.File)
		}
	}
	log.Infof("scan index: %s error(%v) [failed]", i.File, err)
	return
}
Пример #2
0
// flush flush writer buffer.
func (b *SuperBlock) flush(force bool) (err error) {
	var (
		fd     uintptr
		offset int64
		size   int64
	)
	if b.write++; !force && b.write < b.conf.Block.SyncWrite {
		return
	}
	b.write = 0
	offset = needle.BlockOffset(b.syncOffset)
	size = needle.BlockOffset(b.Offset - b.syncOffset)
	fd = b.w.Fd()
	if b.conf.Block.Syncfilerange {
		if err = myos.Syncfilerange(fd, offset, size, myos.SYNC_FILE_RANGE_WRITE); err != nil {
			log.Errorf("block: %s Syncfilerange() error(%v)", b.File, err)
			b.LastErr = err
			return
		}
	} else {
		if err = myos.Fdatasync(fd); err != nil {
			log.Errorf("block: %s Fdatasync() error(%v)", b.File, err)
			b.LastErr = err
			return
		}
	}
	if err = myos.Fadvise(fd, offset, size, myos.POSIX_FADV_DONTNEED); err == nil {
		b.syncOffset = b.Offset
	} else {
		log.Errorf("block: %s Fadvise() error(%v)", b.File, err)
		b.LastErr = err
	}
	return
}
Пример #3
0
// Recovery recovery needles map from super block.
func (b *SuperBlock) Recovery(offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	// WARN block may be no left data, must update block offset first
	if offset == 0 {
		offset = needle.NeedleOffset(_headerOffset)
	}
	b.Offset = offset
	if err = b.Scan(b.r, offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
		if err1 = fn(n, so, eo); err1 == nil {
			b.Offset = eo
		}
		return
	}); err != nil {
		return
	}
	// advise random read
	// POSIX_FADV_RANDOM disables file readahead entirely.
	// These changes affect the entire file, not just the specified region
	// (but other open file handles to the same file are unaffected).
	if err = myos.Fadvise(b.r.Fd(), 0, 0, myos.POSIX_FADV_RANDOM); err != nil {
		log.Errorf("block: %s Fadvise() error(%v)", b.File)
		return
	}
	// reset b.w offset, discard left space which can't parse to a needle
	if _, err = b.w.Seek(needle.BlockOffset(b.Offset), os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File, err)
	}
	return
}
Пример #4
0
// Recovery recovery needles map from super block.
func (b *SuperBlock) Recovery(offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	var rsize int64
	// WARN block may be no left data, must update block offset first
	if offset == 0 {
		offset = needle.NeedleOffset(_headerOffset)
	}
	b.Offset = offset
	if err = b.Scan(b.r, offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
		if err1 = fn(n, so, eo); err1 == nil {
			b.Offset = eo
		}
		return
	}); err != nil {
		return
	}
	// advise random read
	// POSIX_FADV_RANDOM disables file readahead entirely.
	// These changes affect the entire file, not just the specified region
	// (but other open file handles to the same file are unaffected).
	if err = myos.Fadvise(b.r.Fd(), 0, 0, myos.POSIX_FADV_RANDOM); err != nil {
		log.Errorf("block: %s Fadvise() error(%v)", b.File)
		return
	}
	rsize = needle.BlockOffset(b.Offset)
	// reset b.w offset, discard left space which can't parse to a needle
	if _, err = b.w.Seek(rsize, os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File, err)
		return
	}
	// recheck offset, keep size and offset consistency
	if b.Size != rsize {
		log.Warningf("block: %s [real size: %d, offset: %d] but [size: %d, offset: %d] not consistency, truncate file for force recovery, this may lost data",
			b.File, b.Size, needle.NeedleOffset(b.Size),
			rsize, b.Offset)
		// truncate file
		if err = b.w.Truncate(rsize); err != nil {
			log.Errorf("block: %s Truncate() error(%v)", b.File, err)
		}
	}
	return
}
Пример #5
0
// flush the in-memory data flush to disk.
func (i *Indexer) flush(force bool) (err error) {
	var (
		fd     uintptr
		offset int64
		size   int64
	)
	if i.write++; !force && i.write < i.conf.Index.SyncWrite {
		return
	}
	if _, err = i.f.Write(i.buf[:i.bn]); err != nil {
		i.LastErr = err
		log.Errorf("index: %s Write() error(%v)", i.File, err)
		return
	}
	i.Offset += int64(i.bn)
	i.bn = 0
	i.write = 0
	offset = i.syncOffset
	size = i.Offset - i.syncOffset
	fd = i.f.Fd()
	if i.conf.Index.Syncfilerange {
		if err = myos.Syncfilerange(fd, offset, size, myos.SYNC_FILE_RANGE_WRITE); err != nil {
			i.LastErr = err
			log.Errorf("index: %s Syncfilerange() error(%v)", i.File, err)
			return
		}
	} else {
		if err = myos.Fdatasync(fd); err != nil {
			i.LastErr = err
			log.Errorf("index: %s Fdatasync() error(%v)", i.File, err)
			return
		}
	}
	if err = myos.Fadvise(fd, offset, size, myos.POSIX_FADV_DONTNEED); err == nil {
		i.syncOffset = i.Offset
	} else {
		log.Errorf("index: %s Fadvise() error(%v)", i.File, err)
		i.LastErr = err
	}
	return
}
Пример #6
0
// Scan scan a block file.
func (b *SuperBlock) Scan(r *os.File, offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	var (
		so, eo uint32
		bso    int64
		fi     os.FileInfo
		fd     = r.Fd()
		n      = new(needle.Needle)
		rd     = bufio.NewReaderSize(r, b.conf.Block.BufferSize)
	)
	if offset == 0 {
		offset = needle.NeedleOffset(_headerOffset)
	}
	so, eo = offset, offset
	bso = needle.BlockOffset(so)
	// advise sequential read
	if fi, err = r.Stat(); err != nil {
		log.Errorf("block: %s Stat() error(%v)", b.File)
		return
	}
	if err = myos.Fadvise(fd, bso, fi.Size(), myos.POSIX_FADV_SEQUENTIAL); err != nil {
		log.Errorf("block: %s Fadvise() error(%v)", b.File)
		return
	}
	log.Infof("scan block: %s from offset: %d", b.File, offset)
	if _, err = r.Seek(bso, os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File)
		return
	}
	for {
		if err = n.ParseFrom(rd); err != nil {
			if err != io.EOF {
				log.Errorf("block: parse needle from offset: %d:%d error(%v)", so, eo, err)
			}
			break
		}
		if n.TotalSize > int32(b.conf.BlockMaxSize) {
			log.Errorf("scan block: %s error(%v)", n, errors.ErrNeedleSize)
			err = errors.ErrNeedleSize
			break
		}
		if log.V(1) {
			log.Info(n.String())
		}
		eo += n.IncrOffset
		if err = fn(n, so, eo); err != nil {
			log.Errorf("block: callback from offset: %d:%d error(%v)", so, eo, err)
			break
		}
		so = eo
	}
	if err == io.EOF {
		// advise no need page cache
		if err = myos.Fadvise(fd, bso, needle.BlockOffset(eo-so), myos.POSIX_FADV_DONTNEED); err != nil {
			log.Errorf("block: %s Fadvise() error(%v)", b.File)
			return
		}
		log.Infof("scan block: %s to offset: %d [ok]", b.File, eo)
		err = nil
	} else {
		log.Infof("scan block: %s to offset: %d error(%v) [failed]", b.File, eo, err)
	}
	return
}