// Recovery recovery needles map from super block.
func (b *SuperBlock) Recovery(offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
// WARN block may be no left data, must update block offset first
if offset == 0 {
offset = needle.NeedleOffset(_headerOffset)
}
b.Offset = offset
if err = b.Scan(b.r, offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
if err1 = fn(n, so, eo); err1 == nil {
b.Offset = eo
}
return
}); err != nil {
return
}
// advise random read
// POSIX_FADV_RANDOM disables file readahead entirely.
// These changes affect the entire file, not just the specified region
// (but other open file handles to the same file are unaffected).
if err = myos.Fadvise(b.r.Fd(), 0, 0, myos.POSIX_FADV_RANDOM); err != nil {
log.Errorf("block: %s Fadvise() error(%v)", b.File)
return
}
// reset b.w offset, discard left space which can't parse to a needle
if _, err = b.w.Seek(needle.BlockOffset(b.Offset), os.SEEK_SET); err != nil {
log.Errorf("block: %s Seek() error(%v)", b.File, err)
}
return
}
// init init block file, add/parse meta info.
func (b *SuperBlock) init() (err error) {
var stat os.FileInfo
if stat, err = b.r.Stat(); err != nil {
log.Errorf("block: %s Stat() error(%v)", b.File, err)
return
}
if b.Size = stat.Size(); b.Size == 0 {
if err = myos.Fallocate(b.w.Fd(), myos.FALLOC_FL_KEEP_SIZE, 0, _maxSize); err != nil {
log.Errorf("block: %s Fallocate() error(%s)", b.File, err)
return
}
if err = b.writeMeta(); err != nil {
log.Errorf("block: %s writeMeta() error(%v)", b.File, err)
return
}
b.Size = _headerSize
} else {
if err = b.parseMeta(); err != nil {
log.Errorf("block: %s parseMeta() error(%v)", b.File, err)
return
}
if _, err = b.w.Seek(_headerOffset, os.SEEK_SET); err != nil {
log.Errorf("block: %s Seek() error(%v)", b.File, err)
return
}
}
b.Offset = needle.NeedleOffset(_headerOffset)
return
}
// Recovery recovery needles map from super block.
func (b *SuperBlock) Recovery(offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
var rsize int64
// WARN block may be no left data, must update block offset first
if offset == 0 {
offset = needle.NeedleOffset(_headerOffset)
}
b.Offset = offset
if err = b.Scan(b.r, offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
if err1 = fn(n, so, eo); err1 == nil {
b.Offset = eo
}
return
}); err != nil {
return
}
// advise random read
// POSIX_FADV_RANDOM disables file readahead entirely.
// These changes affect the entire file, not just the specified region
// (but other open file handles to the same file are unaffected).
if err = myos.Fadvise(b.r.Fd(), 0, 0, myos.POSIX_FADV_RANDOM); err != nil {
log.Errorf("block: %s Fadvise() error(%v)", b.File)
return
}
rsize = needle.BlockOffset(b.Offset)
// reset b.w offset, discard left space which can't parse to a needle
if _, err = b.w.Seek(rsize, os.SEEK_SET); err != nil {
log.Errorf("block: %s Seek() error(%v)", b.File, err)
return
}
// recheck offset, keep size and offset consistency
if b.Size != rsize {
log.Warningf("block: %s [real size: %d, offset: %d] but [size: %d, offset: %d] not consistency, truncate file for force recovery, this may lost data",
b.File, b.Size, needle.NeedleOffset(b.Size),
rsize, b.Offset)
// truncate file
if err = b.w.Truncate(rsize); err != nil {
log.Errorf("block: %s Truncate() error(%v)", b.File, err)
}
}
return
}
// init recovery super block from index or super block.
func (v *Volume) init() (err error) {
var (
size int64
offset uint32
lastOffset uint32
)
// recovery from index
if err = v.Indexer.Recovery(func(ix *index.Index) error {
// must no less than last offset
if ix.Offset < lastOffset {
log.Error("recovery index: %s lastoffset: %d error(%v)", ix, lastOffset, errors.ErrIndexOffset)
return errors.ErrIndexOffset
}
// WARN if index's offset more than the block, discard it.
if size = int64(ix.Size) + needle.BlockOffset(ix.Offset); size > v.Block.Size {
log.Error("recovery index: %s EOF", ix)
return errors.ErrIndexEOF
}
v.needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
offset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
lastOffset = ix.Offset
return nil
}); err != nil && err != errors.ErrIndexEOF {
return
}
// recovery from super block
if err = v.Block.Recovery(offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
if n.Flag == needle.FlagOK {
if err1 = v.Indexer.Write(n.Key, so, n.TotalSize); err1 != nil {
return
}
} else {
so = needle.CacheDelOffset
}
v.needles[n.Key] = needle.NewCache(so, n.TotalSize)
return
}); err != nil {
return
}
// recheck offset, keep size and offset consistency
if v.Block.Size != needle.BlockOffset(v.Block.Offset) {
log.Error("block: %s size: %d, offset: %d (%d size) not consistency", v.Block.File, v.Block.Size, v.Block.Offset, needle.BlockOffset(v.Block.Offset))
return errors.ErrSuperBlockOffset
}
// flush index
err = v.Indexer.Flush()
return
}
func compareTestOffset(b *SuperBlock, n *needle.Needle, offset uint32) (err error) {
var v int64
if b.Offset != offset+needle.NeedleOffset(int64(n.TotalSize)) {
err = fmt.Errorf("b.Offset: %d not match %d", b.Offset, offset)
return
}
if v, err = b.w.Seek(0, os.SEEK_CUR); err != nil {
err = fmt.Errorf("b.Seek() error(%v)", err)
return
} else {
if v != needle.BlockOffset(b.Offset) {
err = fmt.Errorf("offset: %d not match", v)
return
}
}
return
}
func TestIndex(t *testing.T) {
var (
i *Indexer
err error
noffset uint32
file = "../test/test.idx"
needles = make(map[int64]int64)
)
os.Remove(file)
defer os.Remove(file)
if i, err = NewIndexer(file, testConf); err != nil {
t.Errorf("NewIndexer() error(%v)", err)
t.FailNow()
}
i.Close()
// test closed
if err = i.Add(1, 1, 8); err != errors.ErrIndexClosed {
t.Errorf("Add() error(%v)", err)
t.FailNow()
}
// test open
if err = i.Open(); err != nil {
t.Errorf("Open() error(%v)", err)
t.FailNow()
}
defer i.Close()
// test add
if err = i.Add(1, 1, 8); err != nil {
t.Errorf("Add() error(%v)", err)
t.FailNow()
}
if err = i.Add(2, 2, 8); err != nil {
t.Errorf("Add() error(%v)", err)
t.FailNow()
}
if err = i.Add(5, 3, 8); err != nil {
t.Errorf("Add() error(%v)", err)
t.FailNow()
}
if err = i.Add(6, 4, 8); err != nil {
t.Errorf("Add() error(%v)", err)
t.FailNow()
}
i.Signal()
time.Sleep(1 * time.Second)
i.Flush()
// test recovery
if err = i.Recovery(func(ix *Index) error {
needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
noffset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
return nil
}); err != nil {
t.Errorf("Recovery() error(%v)", err)
t.FailNow()
}
// add 4 index, start with 5
if noffset != 5 {
t.Errorf("noffset: %d not match", noffset)
t.FailNow()
}
if o, s := needle.Cache(needles[1]); o != 1 && s != 8 {
t.Error("needle cache not match")
t.FailNow()
}
if o, s := needle.Cache(needles[2]); o != 2 && s != 8 {
t.Error("needle cache not match")
t.FailNow()
}
if o, s := needle.Cache(needles[5]); o != 3 && s != 8 {
t.Error("needle cache not match")
t.FailNow()
}
if o, s := needle.Cache(needles[6]); o != 4 && s != 8 {
t.Error("needle cache not match")
t.FailNow()
}
// test write
if err = i.Write(10, 5, 8); err != nil {
t.Error("Write() error(%v)", err)
t.FailNow()
}
if err = i.Flush(); err != nil {
t.Error("Flush() error(%v)", err)
t.FailNow()
}
// test recovery
noffset = 0
if err = i.Recovery(func(ix *Index) error {
needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
noffset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
return nil
}); err != nil {
t.Errorf("Recovery() error(%v)", err)
t.FailNow()
}
// add 5 index, start with 6
if noffset != 6 {
t.Errorf("noffset: %d not match", noffset)
t.FailNow()
}
if o, s := needle.Cache(needles[10]); o != 5 && s != 8 {
t.Error("needle.Value(1) not match")
t.FailNow()
}
}
// Scan scan a block file.
func (b *SuperBlock) Scan(r *os.File, offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
var (
so, eo uint32
bso int64
fi os.FileInfo
fd = r.Fd()
n = new(needle.Needle)
rd = bufio.NewReaderSize(r, b.conf.Block.BufferSize)
)
if offset == 0 {
offset = needle.NeedleOffset(_headerOffset)
}
so, eo = offset, offset
bso = needle.BlockOffset(so)
// advise sequential read
if fi, err = r.Stat(); err != nil {
log.Errorf("block: %s Stat() error(%v)", b.File)
return
}
if err = myos.Fadvise(fd, bso, fi.Size(), myos.POSIX_FADV_SEQUENTIAL); err != nil {
log.Errorf("block: %s Fadvise() error(%v)", b.File)
return
}
log.Infof("scan block: %s from offset: %d", b.File, offset)
if _, err = r.Seek(bso, os.SEEK_SET); err != nil {
log.Errorf("block: %s Seek() error(%v)", b.File)
return
}
for {
if err = n.ParseFrom(rd); err != nil {
if err != io.EOF {
log.Errorf("block: parse needle from offset: %d:%d error(%v)", so, eo, err)
}
break
}
if n.TotalSize > int32(b.conf.BlockMaxSize) {
log.Errorf("scan block: %s error(%v)", n, errors.ErrNeedleSize)
err = errors.ErrNeedleSize
break
}
if log.V(1) {
log.Info(n.String())
}
eo += n.IncrOffset
if err = fn(n, so, eo); err != nil {
log.Errorf("block: callback from offset: %d:%d error(%v)", so, eo, err)
break
}
so = eo
}
if err == io.EOF {
// advise no need page cache
if err = myos.Fadvise(fd, bso, needle.BlockOffset(eo-so), myos.POSIX_FADV_DONTNEED); err != nil {
log.Errorf("block: %s Fadvise() error(%v)", b.File)
return
}
log.Infof("scan block: %s to offset: %d [ok]", b.File, eo)
err = nil
} else {
log.Infof("scan block: %s to offset: %d error(%v) [failed]", b.File, eo, err)
}
return
}
func TestSuperBlock(t *testing.T) {
var (
b *SuperBlock
offset, v2, v3, v4 uint32
err error
buf = &bytes.Buffer{}
needles = make(map[int64]int64)
data = []byte("test")
n = needle.NewBufferNeedle(4)
file = "../test/test.block"
ifile = "../test/test.idx"
//indexer *Indexer
)
os.Remove(file)
os.Remove(ifile)
defer os.Remove(file)
defer os.Remove(ifile)
// test new block file
if b, err = NewSuperBlock(file, testConf); err != nil {
t.Errorf("NewSuperBlock(\"%s\") error(%v)", file, err)
t.FailNow()
}
b.Close()
// test parse block file
if b, err = NewSuperBlock(file, testConf); err != nil {
t.Errorf("NewSuperBlock(\"%s\") error(%v)", file, err)
t.FailNow()
}
b.Close()
// test open
if err = b.Open(); err != nil {
t.Errorf("Open() error(%v)", err)
t.FailNow()
}
defer b.Close()
// test write
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(1, 1, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = b.Write(n); err != nil {
t.Errorf("b.Write() error(%v)", err)
t.FailNow()
}
if err = compareTestOffset(b, n, needle.NeedleOffset(int64(_headerSize))); err != nil {
t.Errorf("compareTestOffset() error(%v)", err)
t.FailNow()
}
offset = b.Offset
v2 = b.Offset
// test get
if err = b.ReadAt(1, n); err != nil {
t.Errorf("b.ReadAt() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 1, 1, needle.FlagOK, n, data); err != nil {
t.Errorf("compareTestNeedle() error(%v)", err)
t.FailNow()
}
// test write
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(2, 2, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = b.Write(n); err != nil {
t.Errorf("b.Write() error(%v)", err)
t.FailNow()
}
if err = compareTestOffset(b, n, offset); err != nil {
t.Errorf("compareTestOffset() error(%v)", err)
t.FailNow()
}
offset = b.Offset
v3 = b.Offset
if err = b.ReadAt(6, n); err != nil {
t.Errorf("b.ReadAt() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 2, 2, needle.FlagOK, n, data); err != nil {
t.Error("compareTestNeedle(2)")
t.FailNow()
}
// test write
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(3, 3, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = b.Write(n); err != nil {
t.Errorf("b.Write() error(%v)", err)
t.FailNow()
}
offset = b.Offset
v4 = b.Offset
// test write
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(4, 4, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = b.Write(n); err != nil {
t.Errorf("b.Write() error(%v)", err)
t.FailNow()
}
if err = b.flush(true); err != nil {
t.Errorf("Flush() error(%v)", err)
t.FailNow()
}
if err = compareTestOffset(b, n, offset); err != nil {
t.Errorf("compareTestOffset() error(%v)", err)
t.FailNow()
}
if err = b.ReadAt(11, n); err != nil {
t.Errorf("Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 3, 3, needle.FlagOK, n, data); err != nil {
t.Error("compareTestNeedle(3)")
t.FailNow()
}
if err = b.ReadAt(16, n); err != nil {
t.Errorf("Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 4, 4, needle.FlagOK, n, data); err != nil {
t.Error("compareTestNeedle(r)")
t.FailNow()
}
// test del, del first needles
if err = b.Delete(1); err != nil {
t.Errorf("Del() error(%v)", err)
t.FailNow()
}
// test get
if err = b.ReadAt(1, n); err != nil {
t.Errorf("Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 1, 1, needle.FlagDel, n, data); err != nil {
t.FailNow()
}
if err = b.ReadAt(11, n); err != nil {
t.Errorf("Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 3, 3, needle.FlagOK, n, data); err != nil {
t.FailNow()
}
if err = b.ReadAt(16, n); err != nil {
t.Errorf("b.Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 4, 4, needle.FlagOK, n, data); err != nil {
t.FailNow()
}
// test recovery
offset = b.Offset
if err = b.Recovery(0, func(rn *needle.Needle, so, eo uint32) (err1 error) {
if rn.Flag != needle.FlagOK {
so = needle.CacheDelOffset
}
needles[rn.Key] = needle.NewCache(so, rn.TotalSize)
return
}); err != nil {
t.Errorf("Recovery() error(%v)", err)
t.FailNow()
}
if b.Offset != offset {
err = fmt.Errorf("b.Offset not match %d", b.Offset)
t.Error(err)
t.FailNow()
}
if o, s := needle.Cache(needles[1]); o != needle.CacheDelOffset && s != 40 {
t.Error("needle.Cache() not match")
t.FailNow()
}
if o, s := needle.Cache(needles[2]); o != v2 && s != 40 {
t.Error("needle.Cache() not match")
t.FailNow()
}
if o, s := needle.Cache(needles[3]); o != v3 && s != 40 {
t.Error("needle.Cache() not match")
t.FailNow()
}
if o, s := needle.Cache(needles[4]); o != v4 && s != 40 {
t.Error("needle.Cache() not match")
t.FailNow()
}
needles = make(map[int64]int64)
if err = b.Recovery(v2, func(rn *needle.Needle, so, eo uint32) (err1 error) {
if rn.Flag != needle.FlagOK {
so = needle.CacheDelOffset
}
needles[rn.Key] = needle.NewCache(so, rn.TotalSize)
return
}); err != nil {
t.Errorf("b.Recovery() error(%v)", err)
t.FailNow()
}
// skip first needle, so key:1 must not exist
if o, s := needle.Cache(needles[1]); o != 0 && s != 0 {
t.Error("needle.Value(1) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[2]); o != v2 && s != 40 {
t.Error("needle.Value(2) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[3]); o != v3 && s != 40 {
t.Error("needle.Value(3) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[4]); o != v4 && s != 40 {
t.Error("needle.Value(4) not match")
t.FailNow()
}
// test repair
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(3, 3, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = b.WriteAt(v3, n); err != nil {
t.Errorf("b.Repair(3) error(%v)", err)
t.FailNow()
}
if err = b.ReadAt(v3, n); err != nil {
t.Errorf("b.Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 3, 3, needle.FlagOK, n, data); err != nil {
t.Error("compareTestNeedle(3)")
t.FailNow()
}
// test compress
if err = b.Compact(0, func(rn *needle.Needle, so, eo uint32) (err1 error) {
if rn.Flag != needle.FlagOK {
return
}
needles[rn.Key] = needle.NewCache(so, rn.TotalSize)
return
}); err != nil {
t.Errorf("b.Compress() error(%v)", err)
t.FailNow()
}
// skip first needle, so key:1 must not exist
if o, s := needle.Cache(needles[1]); o != 0 && s != 0 {
t.Error("needle.Value(1) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[2]); o != v2 && s != 40 {
t.Error("needle.Value(2) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[3]); o != v3 && s != 40 {
t.Error("needle.Value(3) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[4]); o != v4 && s != 40 {
t.Error("needle.Value(4) not match")
t.FailNow()
}
}