// Opens table. It returns a cache handle, which should
// be released after use.
func (t *tOps) open(f *tFile) (ch *cache.Handle, err error) {
ch = t.cache.Get(0, uint64(f.fd.Num), func() (size int, value cache.Value) {
var r storage.Reader
r, err = t.s.stor.Open(f.fd)
if err != nil {
return 0, nil
}
var bcache *cache.NamespaceGetter
if t.bcache != nil {
bcache = &cache.NamespaceGetter{Cache: t.bcache, NS: uint64(f.fd.Num)}
}
var tr *table.Reader
tr, err = table.NewReader(r, f.size, f.fd, bcache, t.bpool, t.s.o.Options)
if err != nil {
r.Close()
return 0, nil
}
return 1, tr
})
if ch == nil && err == nil {
err = ErrClosed
}
return
}
func recoverTable(s *session, o *opt.Options) error {
o = dupOptions(o)
// Mask StrictReader, lets StrictRecovery doing its job.
o.Strict &= ^opt.StrictReader
// Get all tables and sort it by file number.
fds, err := s.stor.List(storage.TypeTable)
if err != nil {
return err
}
sortFds(fds)
var (
maxSeq uint64
recoveredKey, goodKey, corruptedKey, corruptedBlock, droppedTable int
// We will drop corrupted table.
strict = o.GetStrict(opt.StrictRecovery)
noSync = o.GetNoSync()
rec = &sessionRecord{}
bpool = util.NewBufferPool(o.GetBlockSize() + 5)
)
buildTable := func(iter iterator.Iterator) (tmpFd storage.FileDesc, size int64, err error) {
tmpFd = s.newTemp()
writer, err := s.stor.Create(tmpFd)
if err != nil {
return
}
defer func() {
writer.Close()
if err != nil {
s.stor.Remove(tmpFd)
tmpFd = storage.FileDesc{}
}
}()
// Copy entries.
tw := table.NewWriter(writer, o)
for iter.Next() {
key := iter.Key()
if validInternalKey(key) {
err = tw.Append(key, iter.Value())
if err != nil {
return
}
}
}
err = iter.Error()
if err != nil {
return
}
err = tw.Close()
if err != nil {
return
}
if !noSync {
err = writer.Sync()
if err != nil {
return
}
}
size = int64(tw.BytesLen())
return
}
recoverTable := func(fd storage.FileDesc) error {
s.logf("table@recovery recovering @%d", fd.Num)
reader, err := s.stor.Open(fd)
if err != nil {
return err
}
var closed bool
defer func() {
if !closed {
reader.Close()
}
}()
// Get file size.
size, err := reader.Seek(0, 2)
if err != nil {
return err
}
var (
tSeq uint64
tgoodKey, tcorruptedKey, tcorruptedBlock int
imin, imax []byte
)
tr, err := table.NewReader(reader, size, fd, nil, bpool, o)
if err != nil {
return err
}
iter := tr.NewIterator(nil, nil)
if itererr, ok := iter.(iterator.ErrorCallbackSetter); ok {
itererr.SetErrorCallback(func(err error) {
if errors.IsCorrupted(err) {
s.logf("table@recovery block corruption @%d %q", fd.Num, err)
tcorruptedBlock++
}
})
}
// Scan the table.
for iter.Next() {
key := iter.Key()
_, seq, _, kerr := parseInternalKey(key)
if kerr != nil {
tcorruptedKey++
continue
}
tgoodKey++
if seq > tSeq {
tSeq = seq
}
if imin == nil {
imin = append([]byte{}, key...)
}
imax = append(imax[:0], key...)
}
if err := iter.Error(); err != nil {
iter.Release()
return err
}
iter.Release()
goodKey += tgoodKey
corruptedKey += tcorruptedKey
corruptedBlock += tcorruptedBlock
if strict && (tcorruptedKey > 0 || tcorruptedBlock > 0) {
droppedTable++
s.logf("table@recovery dropped @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", fd.Num, tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
return nil
}
if tgoodKey > 0 {
if tcorruptedKey > 0 || tcorruptedBlock > 0 {
// Rebuild the table.
s.logf("table@recovery rebuilding @%d", fd.Num)
iter := tr.NewIterator(nil, nil)
tmpFd, newSize, err := buildTable(iter)
iter.Release()
if err != nil {
return err
}
closed = true
reader.Close()
if err := s.stor.Rename(tmpFd, fd); err != nil {
return err
}
size = newSize
}
if tSeq > maxSeq {
maxSeq = tSeq
}
recoveredKey += tgoodKey
// Add table to level 0.
rec.addTable(0, fd.Num, size, imin, imax)
s.logf("table@recovery recovered @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", fd.Num, tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
} else {
droppedTable++
s.logf("table@recovery unrecoverable @%d Ck·%d Cb·%d S·%d", fd.Num, tcorruptedKey, tcorruptedBlock, size)
}
return nil
}
// Recover all tables.
if len(fds) > 0 {
s.logf("table@recovery F·%d", len(fds))
// Mark file number as used.
s.markFileNum(fds[len(fds)-1].Num)
for _, fd := range fds {
if err := recoverTable(fd); err != nil {
return err
}
}
s.logf("table@recovery recovered F·%d N·%d Gk·%d Ck·%d Q·%d", len(fds), recoveredKey, goodKey, corruptedKey, maxSeq)
}
// Set sequence number.
rec.setSeqNum(maxSeq)
// Create new manifest.
if err := s.create(); err != nil {
return err
}
// Commit.
return s.commit(rec)
}
func (ts *testingStorage) scanTable(fd storage.FileDesc, checksum bool) (corrupted bool) {
r, err := ts.Open(fd)
if err != nil {
log.Fatal(err)
}
defer r.Close()
size, err := r.Seek(0, os.SEEK_END)
if err != nil {
log.Fatal(err)
}
o := &opt.Options{
DisableLargeBatchTransaction: true,
Strict: opt.NoStrict,
}
if checksum {
o.Strict = opt.StrictBlockChecksum | opt.StrictReader
}
tr, err := table.NewReader(r, size, fd, nil, bpool, o)
if err != nil {
log.Fatal(err)
}
defer tr.Release()
checkData := func(i int, t string, data []byte) bool {
if len(data) == 0 {
panic(fmt.Sprintf("[%v] nil data: i=%d t=%s", fd, i, t))
}
checksum0, checksum1 := dataChecksum(data)
if checksum0 != checksum1 {
atomic.StoreUint32(&fail, 1)
atomic.StoreUint32(&done, 1)
corrupted = true
data0, data1 := dataSplit(data)
data0c0, data0c1 := dataChecksum(data0)
data1c0, data1c1 := dataChecksum(data1)
log.Printf("FATAL: [%v] Corrupted data i=%d t=%s (%#x != %#x): %x(%v) vs %x(%v)",
fd, i, t, checksum0, checksum1, data0, data0c0 == data0c1, data1, data1c0 == data1c1)
return true
}
return false
}
iter := tr.NewIterator(nil, nil)
defer iter.Release()
for i := 0; iter.Next(); i++ {
ukey, _, kt, kerr := parseIkey(iter.Key())
if kerr != nil {
atomic.StoreUint32(&fail, 1)
atomic.StoreUint32(&done, 1)
corrupted = true
log.Printf("FATAL: [%v] Corrupted ikey i=%d: %v", fd, i, kerr)
return
}
if checkData(i, "key", ukey) {
return
}
if kt == ktVal && checkData(i, "value", iter.Value()) {
return
}
}
if err := iter.Error(); err != nil {
if errors.IsCorrupted(err) {
atomic.StoreUint32(&fail, 1)
atomic.StoreUint32(&done, 1)
corrupted = true
log.Printf("FATAL: [%v] Corruption detected: %v", fd, err)
} else {
log.Fatal(err)
}
}
return
}