// Scan scan a indexer file.
func (i *Indexer) Scan(r *os.File, fn func(*Index) error) (err error) {
var (
data []byte
fi os.FileInfo
fd = r.Fd()
ix = &Index{}
rd = bufio.NewReaderSize(r, i.Options.BufferSize)
)
log.Infof("scan index: %s", i.File)
// advise sequential read
if fi, err = r.Stat(); err != nil {
log.Errorf("index: %s Stat() error(%v)", i.File)
return
}
if err = myos.Fadvise(fd, 0, fi.Size(), myos.POSIX_FADV_SEQUENTIAL); err != nil {
log.Errorf("index: %s Fadvise() error(%v)", i.File)
return
}
if _, err = r.Seek(0, os.SEEK_SET); err != nil {
log.Errorf("index: %s Seek() error(%v)", i.File, err)
return
}
for {
if data, err = rd.Peek(_indexSize); err != nil {
break
}
if err = ix.parse(data); err != nil {
break
}
if ix.Size > int32(i.Options.NeedleMaxSize) {
log.Error("scan index: %s error(%v)", ix, errors.ErrIndexSize)
err = errors.ErrIndexSize
break
}
if _, err = rd.Discard(_indexSize); err != nil {
break
}
if log.V(1) {
log.Info(ix.String())
}
if err = fn(ix); err != nil {
break
}
}
if err == io.EOF {
// advise no need page cache
if err = myos.Fadvise(fd, 0, fi.Size(), myos.POSIX_FADV_DONTNEED); err == nil {
err = nil
log.Infof("scan index: %s [ok]", i.File)
return
} else {
log.Errorf("index: %s Fadvise() error(%v)", i.File)
}
}
log.Infof("scan index: %s error(%v) [failed]", i.File, err)
return
}
// Recovery recovery needles map from super block.
func (b *SuperBlock) Recovery(offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
// WARN block may be no left data, must update block offset first
if offset == 0 {
offset = needle.NeedleOffset(_headerOffset)
}
b.Offset = offset
if err = b.Scan(b.r, offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
if err1 = fn(n, so, eo); err1 == nil {
b.Offset = eo
}
return
}); err != nil {
return
}
// advise random read
// POSIX_FADV_RANDOM disables file readahead entirely.
// These changes affect the entire file, not just the specified region
// (but other open file handles to the same file are unaffected).
if err = myos.Fadvise(b.r.Fd(), 0, 0, myos.POSIX_FADV_RANDOM); err != nil {
log.Errorf("block: %s Fadvise() error(%v)", b.File)
return
}
// reset b.w offset, discard left space which can't parse to a needle
if _, err = b.w.Seek(needle.BlockOffset(b.Offset), os.SEEK_SET); err != nil {
log.Errorf("block: %s Seek() error(%v)", b.File, err)
}
return
}
// flush flush writer buffer.
func (b *SuperBlock) flush(force bool) (err error) {
var (
fd uintptr
offset int64
size int64
)
if b.write++; !force && b.write < b.Options.SyncAtWrite {
return
}
b.write = 0
offset = needle.BlockOffset(b.syncOffset)
size = needle.BlockOffset(b.Offset - b.syncOffset)
fd = b.w.Fd()
if b.Options.Syncfilerange {
if err = myos.Syncfilerange(fd, offset, size, myos.SYNC_FILE_RANGE_WRITE); err != nil {
log.Errorf("block: %s Syncfilerange() error(%v)", b.File, err)
b.LastErr = err
return
}
} else {
if err = myos.Fdatasync(fd); err != nil {
log.Errorf("block: %s Fdatasync() error(%v)", b.File, err)
b.LastErr = err
return
}
}
if err = myos.Fadvise(fd, offset, size, myos.POSIX_FADV_DONTNEED); err == nil {
b.syncOffset = b.Offset
} else {
log.Errorf("block: %s Fadvise() error(%v)", b.File, err)
b.LastErr = err
}
return
}
// flush the in-memory data flush to disk.
func (i *Indexer) flush(force bool) (err error) {
var (
fd uintptr
offset int64
size int64
)
if i.write++; !force && i.write < i.Options.SyncAtWrite {
return
}
if _, err = i.f.Write(i.buf[:i.bn]); err != nil {
i.LastErr = err
log.Errorf("index: %s Write() error(%v)", i.File, err)
return
}
i.Offset += int64(i.bn)
i.bn = 0
i.write = 0
offset = i.syncOffset
size = i.Offset - i.syncOffset
fd = i.f.Fd()
if i.Options.Syncfilerange {
if err = myos.Syncfilerange(fd, offset, size, myos.SYNC_FILE_RANGE_WRITE); err != nil {
i.LastErr = err
log.Errorf("index: %s Syncfilerange() error(%v)", i.File, err)
return
}
} else {
if err = myos.Fdatasync(fd); err != nil {
i.LastErr = err
log.Errorf("index: %s Fdatasync() error(%v)", i.File, err)
return
}
}
if err = myos.Fadvise(fd, offset, size, myos.POSIX_FADV_DONTNEED); err == nil {
i.syncOffset = i.Offset
} else {
log.Errorf("index: %s Fadvise() error(%v)", i.File, err)
i.LastErr = err
}
return
}
// sync sync the in-memory data flush to disk.
func (i *Indexer) sync() (err error) {
var (
fd uintptr
offset int64
size int64
)
// append N times call flush then clean the os page cache
// page cache no used here...
// after upload a photo, we cache in our own cache server.
offset = i.syncOffset
size = i.Offset - i.syncOffset
if i.write < i.Options.MergeAtWrite {
return
}
i.write = 0
fd = i.f.Fd()
if i.Options.Syncfilerange {
if err = myos.Syncfilerange(fd, offset, size, myos.SYNC_FILE_RANGE_WRITE); err != nil {
i.LastErr = err
log.Errorf("index: %s Syncfilerange() error(%v)", i.File, err)
return
}
} else {
if err = myos.Fdatasync(fd); err != nil {
i.LastErr = err
log.Errorf("index: %s Fdatasync() error(%v)", i.File, err)
return
}
}
if err = myos.Fadvise(fd, offset, size, myos.POSIX_FADV_DONTNEED); err == nil {
i.syncOffset = i.Offset
} else {
log.Errorf("index: %s Fadvise() error(%v)", i.File, err)
i.LastErr = err
}
return
}
// sync sync the in-memory data flush to disk.
func (b *SuperBlock) sync() (err error) {
var (
fd uintptr
offset int64
size int64
)
// append N times call flush then clean the os page cache
// page cache no used here...
// after upload a photo, we cache in our own cache server.
offset = needle.BlockOffset(b.syncOffset)
size = needle.BlockOffset(b.Offset - b.syncOffset)
if b.write < b.Options.SyncAtWrite {
return
}
b.write = 0
fd = b.w.Fd()
if b.Options.Syncfilerange {
if err = myos.Syncfilerange(fd, offset, size, myos.SYNC_FILE_RANGE_WRITE); err != nil {
b.LastErr = err
log.Errorf("block: %s Syncfilerange() error(%v)", b.File, err)
return
}
} else {
if err = myos.Fdatasync(fd); err != nil {
b.LastErr = err
log.Errorf("block: %s Fdatasync() error(%v)", b.File, err)
return
}
}
if err = myos.Fadvise(fd, offset, size, myos.POSIX_FADV_DONTNEED); err == nil {
b.syncOffset = b.Offset
} else {
log.Errorf("block: %s Fadvise() error(%v)", b.File, err)
b.LastErr = err
}
return
}
// Scan scan a block file.
func (b *SuperBlock) Scan(r *os.File, offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
var (
data []byte
so, eo uint32
bso int64
fi os.FileInfo
fd = r.Fd()
n = &needle.Needle{}
rd = bufio.NewReaderSize(r, b.Options.BufferSize)
)
if offset == 0 {
offset = needle.NeedleOffset(_headerOffset)
}
so, eo = offset, offset
bso = needle.BlockOffset(so)
// advise sequential read
if fi, err = r.Stat(); err != nil {
log.Errorf("block: %s Stat() error(%v)", b.File)
return
}
if err = myos.Fadvise(fd, bso, fi.Size(), myos.POSIX_FADV_SEQUENTIAL); err != nil {
log.Errorf("block: %s Fadvise() error(%v)", b.File)
return
}
log.Infof("scan block: %s from offset: %d", b.File, offset)
if _, err = r.Seek(bso, os.SEEK_SET); err != nil {
log.Errorf("block: %s Seek() error(%v)", b.File)
return
}
for {
// header
if data, err = rd.Peek(needle.HeaderSize); err != nil {
break
}
if err = n.ParseHeader(data); err != nil {
break
}
if _, err = rd.Discard(needle.HeaderSize); err != nil {
break
}
// data
if data, err = rd.Peek(int(n.Size)); err != nil {
break
}
if err = n.ParseData(data); err != nil {
break
}
if _, err = rd.Discard(int(n.Size)); err != nil {
break
}
// footer
if data, err = rd.Peek(int(n.FooterSize)); err != nil {
break
}
if err = n.ParseFooter(data); err != nil {
break
}
if _, err = rd.Discard(int(n.FooterSize)); err != nil {
break
}
if log.V(1) {
log.Info(n.String())
}
eo += needle.NeedleOffset(int64(n.TotalSize))
if err = fn(n, so, eo); err != nil {
break
}
so = eo
}
if err == io.EOF {
// advise no need page cache
if err = myos.Fadvise(fd, bso, needle.BlockOffset(eo-so), myos.POSIX_FADV_DONTNEED); err != nil {
log.Errorf("block: %s Fadvise() error(%v)", b.File)
return
}
log.Infof("scan block: %s to offset: %d [ok]", b.File, eo)
err = nil
} else {
log.Infof("scan block: %s to offset: %d error(%v) [failed]", b.File, eo, err)
}
return
}
// Scan scan a block file.
func (b *SuperBlock) Scan(r *os.File, offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
var (
so, eo uint32
bso int64
fi os.FileInfo
fd = r.Fd()
n = needle.NewNeedle(b.Options.NeedleMaxSize)
rd = bufio.NewReaderSize(r, b.Options.BufferSize)
)
if offset == 0 {
offset = needle.NeedleOffset(_headerOffset)
}
so, eo = offset, offset
bso = needle.BlockOffset(so)
// advise sequential read
if fi, err = r.Stat(); err != nil {
log.Errorf("block: %s Stat() error(%v)", b.File)
return
}
if err = myos.Fadvise(fd, bso, fi.Size(), myos.POSIX_FADV_SEQUENTIAL); err != nil {
log.Errorf("block: %s Fadvise() error(%v)", b.File)
return
}
log.Infof("scan block: %s from offset: %d", b.File, offset)
if _, err = r.Seek(bso, os.SEEK_SET); err != nil {
log.Errorf("block: %s Seek() error(%v)", b.File)
return
}
for {
if err = n.ParseFrom(rd); err != nil {
if err != io.EOF {
log.Errorf("block: parse needle from offset: %d:%d error(%v)", so, eo, err)
}
break
}
if n.Size > int32(b.Options.NeedleMaxSize) {
log.Error("scan block: %s error(%v)", n, errors.ErrNeedleSize)
err = errors.ErrNeedleSize
break
}
if log.V(1) {
log.Info(n.String())
}
eo += needle.NeedleOffset(int64(n.TotalSize))
if err = fn(n, so, eo); err != nil {
log.Errorf("block: callback from offset: %d:%d error(%v)", so, eo, err)
break
}
so = eo
}
if err == io.EOF {
// advise no need page cache
if err = myos.Fadvise(fd, bso, needle.BlockOffset(eo-so), myos.POSIX_FADV_DONTNEED); err != nil {
log.Errorf("block: %s Fadvise() error(%v)", b.File)
return
}
log.Infof("scan block: %s to offset: %d [ok]", b.File, eo)
err = nil
} else {
log.Infof("scan block: %s to offset: %d error(%v) [failed]", b.File, eo, err)
}
return
}