Example #1
0
// Needle get a needle from sync.Pool.
func (s *Store) Needle() *needle.Needle {
	var n interface{}
	if n = s.np.Get(); n != nil {
		return n.(*needle.Needle)
	}
	return needle.NewNeedle(s.conf.NeedleMaxSize)
}
Example #2
0
// Scan scan a block file.
func (b *SuperBlock) Scan(r *os.File, offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	var (
		so, eo uint32
		bso    int64
		fi     os.FileInfo
		fd     = r.Fd()
		n      = needle.NewNeedle(b.Options.NeedleMaxSize)
		rd     = bufio.NewReaderSize(r, b.Options.BufferSize)
	)
	if offset == 0 {
		offset = needle.NeedleOffset(_headerOffset)
	}
	so, eo = offset, offset
	bso = needle.BlockOffset(so)
	// advise sequential read
	if fi, err = r.Stat(); err != nil {
		log.Errorf("block: %s Stat() error(%v)", b.File)
		return
	}
	if err = myos.Fadvise(fd, bso, fi.Size(), myos.POSIX_FADV_SEQUENTIAL); err != nil {
		log.Errorf("block: %s Fadvise() error(%v)", b.File)
		return
	}
	log.Infof("scan block: %s from offset: %d", b.File, offset)
	if _, err = r.Seek(bso, os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File)
		return
	}
	for {
		if err = n.ParseFrom(rd); err != nil {
			if err != io.EOF {
				log.Errorf("block: parse needle from offset: %d:%d error(%v)", so, eo, err)
			}
			break
		}
		if n.Size > int32(b.Options.NeedleMaxSize) {
			log.Error("scan block: %s error(%v)", n, errors.ErrNeedleSize)
			err = errors.ErrNeedleSize
			break
		}
		if log.V(1) {
			log.Info(n.String())
		}
		eo += needle.NeedleOffset(int64(n.TotalSize))
		if err = fn(n, so, eo); err != nil {
			log.Errorf("block: callback from offset: %d:%d error(%v)", so, eo, err)
			break
		}
		so = eo
	}
	if err == io.EOF {
		// advise no need page cache
		if err = myos.Fadvise(fd, bso, needle.BlockOffset(eo-so), myos.POSIX_FADV_DONTNEED); err != nil {
			log.Errorf("block: %s Fadvise() error(%v)", b.File)
			return
		}
		log.Infof("scan block: %s to offset: %d [ok]", b.File, eo)
		err = nil
	} else {
		log.Infof("scan block: %s to offset: %d error(%v) [failed]", b.File, eo, err)
	}
	return
}