// Verify attempts to find any structural errors in DB wrt the organization of
// it as defined by lldb.Allocator. Any problems found are reported to 'log'
// except non verify related errors like disk read fails etc. If 'log' returns
// false or the error doesn't allow to (reliably) continue, the verification
// process is stopped and an error is returned from the Verify function.
// Passing a nil log works like providing a log function always returning
// false. Any non-structural errors, like for instance Filer read errors, are
// NOT reported to 'log', but returned as the Verify's return value, because
// Verify cannot proceed in such cases. Verify returns nil only if it fully
// completed verifying DB without detecting any error.
//
// It is recommended to limit the number reported problems by returning false
// from 'log' after reaching some limit. Huge and corrupted DB can produce an
// overwhelming error report dataset.
//
// The verifying process will scan the whole DB at least 3 times (a trade
// between processing space and time consumed). It doesn't read the content of
// free blocks above the head/tail info bytes. If the 3rd phase detects lost
// free space, then a 4th scan (a faster one) is performed to precisely report
// all of them.
//
// Statistics are returned via 'stats' if non nil. The statistics are valid
// only if Verify succeeded, ie. it didn't reported anything to log and it
// returned a nil error.
func (db *DB) Verify(log func(error) bool, stats *lldb.AllocStats) (err error) {
bitmapf, err := fileutil.TempFile(".", "verifier", ".tmp")
if err != nil {
return
}
defer func() {
tn := bitmapf.Name()
bitmapf.Close()
os.Remove(tn)
}()
bitmap := lldb.NewSimpleFileFiler(bitmapf)
if err = db.enter(); err != nil {
return
}
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("%v", e)
}
db.leave(&err)
}()
return db.alloc.Verify(bitmap, log, stats)
}
func fillseq() {
dbname := os.Args[0] + ".db"
f, err := os.OpenFile(dbname, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
if err != nil {
log.Fatal(err)
}
defer func() {
f.Close()
os.Remove(f.Name())
}()
filer := lldb.NewSimpleFileFiler(f)
a, err := lldb.NewAllocator(filer, &lldb.Options{})
if err != nil {
log.Println(err)
return
}
a.Compress = true
b, _, err := lldb.CreateBTree(a, nil)
if err != nil {
log.Println(err)
return
}
var keys [N][16]byte
for i := range keys {
binary.BigEndian.PutUint32(keys[i][:], uint32(i))
}
debug.FreeOSMemory()
t0 := time.Now()
for _, key := range keys {
if err = b.Set(key[:], value100); err != nil {
log.Println(err)
return
}
}
if err := filer.Sync(); err != nil {
log.Println(err)
return
}
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
d := time.Since(t0)
fi, err := f.Stat()
if err != nil {
log.Println(err)
return
}
secs := float64(d/time.Nanosecond) / float64(time.Second)
sz := fi.Size()
fmt.Printf("fillseq :%19v/op;%7.1f MB/s (%g secs, %d bytes)\n", d/N, float64(sz)/secs/1e6, secs, sz)
nn, bytes := bufs.GCache.Stats()
fmt.Printf("%d %d\n", nn, bytes)
fmt.Printf("%+v\n", ms)
}
// Open opens the named DB file for reading/writing. If successful, methods on
// the returned DB can be used for I/O; the associated file descriptor has mode
// os.O_RDWR. If there is an error, it will be of type *os.PathError.
//
// For the meaning of opts please see documentation of Options.
func Open(name string, opts *Options) (db *DB, err error) {
defer func() {
lock := opts.lock
if err != nil && lock != nil {
n := lock.Name()
lock.Close()
os.Remove(n)
db = nil
}
if err != nil {
if db != nil {
db.Close()
db = nil
}
}
}()
if err = opts.check(name, false, true); err != nil {
return
}
f, err := os.OpenFile(name, os.O_RDWR, 0666)
if err != nil {
return
}
filer := lldb.Filer(lldb.NewSimpleFileFiler(f))
sz, err := filer.Size()
if err != nil {
return
}
if sz%16 != 0 {
return nil, &os.PathError{Op: "dbm.Open:", Path: name, Err: fmt.Errorf("file size %d(%#x) is not 0 (mod 16)", sz, sz)}
}
var b [16]byte
if n, err := filer.ReadAt(b[:], 0); n != 16 || err != nil {
return nil, &os.PathError{Op: "dbm.Open.ReadAt", Path: name, Err: err}
}
var h header
if err = h.rd(b[:]); err != nil {
return nil, &os.PathError{Op: "dbm.Open:validate header", Path: name, Err: err}
}
db = &DB{f: f, lock: opts.lock, closed: make(chan bool)}
if filer, err = opts.acidFiler(db, filer); err != nil {
return nil, err
}
db.filer = filer
switch h.ver {
default:
return nil, &os.PathError{Op: "dbm.Open", Path: name, Err: fmt.Errorf("unknown dbm file format version %#x", h.ver)}
case 0x00:
return open00(name, db)
}
}
// Create creates the named DB file mode 0666 (before umask). The file must not
// already exist. If successful, methods on the returned DB can be used for
// I/O; the associated file descriptor has mode os.O_RDWR. If there is an
// error, it will be of type *os.PathError.
//
// For the meaning of opts please see documentation of Options.
func Create(name string, opts *Options) (db *DB, err error) {
f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
if err != nil {
return
}
return create(f, lldb.NewSimpleFileFiler(f), opts, false)
}
// CreateTemp creates a new temporary DB in the directory dir with a basename
// beginning with prefix and name ending in suffix. If dir is the empty string,
// CreateTemp uses the default directory for temporary files (see os.TempDir).
// Multiple programs calling CreateTemp simultaneously will not choose the same
// file name for the DB. The caller can use Name() to find the pathname of the
// DB file. It is the caller's responsibility to remove the file when no longer
// needed.
//
// For the meaning of opts please see documentation of Options.
func CreateTemp(dir, prefix, suffix string, opts *Options) (db *DB, err error) {
f, err := fileutil.TempFile(dir, prefix, suffix)
if err != nil {
return
}
return create(f, lldb.NewSimpleFileFiler(f), opts, false)
}
func verifyAllocator(a *lldb.Allocator) error {
bits, err := ioutil.TempFile("", "kv-verify-")
if err != nil {
return err
}
defer func() {
nm := bits.Name()
bits.Close()
os.Remove(nm)
}()
var lerr error
if err = a.Verify(
lldb.NewSimpleFileFiler(bits),
func(err error) bool {
lerr = err
return false
},
nil,
); err != nil {
return err
}
if lerr != nil {
return lerr
}
t, err := lldb.OpenBTree(a, nil, 1)
if err != nil {
return err
}
e, err := t.SeekFirst()
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
for {
_, _, err := e.Next()
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
}
}
func TestProf(t *testing.T) {
dbname := os.Args[0] + ".db"
f, err := os.OpenFile(dbname, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
if err != nil {
t.Fatal(err)
}
defer func() {
f.Close()
os.Remove(f.Name())
}()
filer := lldb.NewSimpleFileFiler(f) // file
//filer := lldb.NewMemFiler() // mem
a, err := lldb.NewAllocator(filer, &lldb.Options{})
if err != nil {
t.Error(err)
return
}
a.Compress = true
b, _, err := lldb.CreateBTree(a, nil)
if err != nil {
t.Error(err)
return
}
var key [16]byte
for i := uint32(0); int(i) < 1e6; i++ {
binary.BigEndian.PutUint32(key[:], i)
if err = b.Set(key[:], value100); err != nil {
t.Error(err)
return
}
}
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
bufsU, bufsT, bytesU, bytesT, h, m := a.CacheStats()
const p = 100.0
t.Logf(
"cache: buffers %d/%d(%.1f%%), bytes %d/%d(%.1f%%), hits %d(%.1f%%), misses %d(%.1f%%)",
bufsU, bufsT, p*float64(bufsU)/float64(bufsT),
bytesU, bytesT, p*float64(bytesU)/float64(bytesT),
h, p*float64(h)/float64(h+m),
m, p*float64(m)/float64(h+m),
)
nn, bts := bufs.GCache.Stats()
t.Logf("bufs.GCache.Stats() {%d, %d}", nn, bts)
t.Logf("%+v\n", ms)
}
func verifyDbFile(fn string) error {
f, err := os.Open(fn) // O_RDONLY
if err != nil {
return err
}
defer f.Close()
a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
if err != nil {
return err
}
return verifyAllocator(a)
}
func BenchmarkMem(b *testing.B) {
f, err := ioutil.TempFile("", "")
if err != nil {
b.Fatal(err)
}
defer func() {
f.Close()
os.Remove(f.Name())
}()
filer := lldb.NewSimpleFileFiler(f)
a, err := lldb.NewAllocator(filer, &lldb.Options{})
if err != nil {
b.Error(err)
return
}
a.Compress = true
t, _, err := lldb.CreateBTree(a, nil)
if err != nil {
b.Error(err)
return
}
b.ResetTimer()
var key [16]byte
for i := uint32(0); int(i) < b.N; i++ {
binary.BigEndian.PutUint32(key[:], i)
if err = t.Set(key[:], value100); err != nil {
b.Error(err)
return
}
}
if err := filer.Sync(); err != nil {
b.Error(err)
return
}
}
// Open opens the named DB file for reading/writing. If successful, methods on
// the returned DB can be used for I/O; the associated file descriptor has mode
// os.O_RDWR. If there is an error, it will be of type *os.PathError.
//
// Note: While a DB is opened, it is locked and cannot be simultaneously opened
// again.
//
// For the meaning of opts please see documentation of Options.
func Open(name string, opts *Options) (db *DB, err error) {
opts = opts.clone()
opts._ACID = _ACIDFull
defer func() {
lock := opts.lock
if err != nil && lock != nil {
lock.Close()
db = nil
}
if err != nil {
if db != nil {
db.Close()
db = nil
}
}
}()
if err = opts.check(name, false, true); err != nil {
return
}
f, err := os.OpenFile(name, os.O_RDWR, 0666)
if err != nil {
return
}
filer := lldb.Filer(lldb.NewSimpleFileFiler(f))
sz, err := filer.Size()
if err != nil {
return
}
if sz%16 != 0 {
return nil, &os.PathError{Op: "kv.Open:", Path: name, Err: fmt.Errorf("file size %d(%#x) is not 0 (mod 16)", sz, sz)}
}
var b [16]byte
if n, err := filer.ReadAt(b[:], 0); n != 16 || err != nil {
return nil, &os.PathError{Op: "kv.Open.ReadAt", Path: name, Err: err}
}
var h header
if err = h.rd(b[:]); err != nil {
return nil, &os.PathError{Op: "kv.Open:validate header", Path: name, Err: err}
}
db = &DB{f: f, lock: opts.lock}
if filer, err = opts.acidFiler(db, filer); err != nil {
return nil, err
}
db.filer = filer
switch h.ver {
default:
return nil, &os.PathError{Op: "kv.Open", Path: name, Err: fmt.Errorf("unknown/unsupported kv file format version %#x", h.ver)}
case 0x00:
if _, err = open00(name, db); err != nil {
return nil, err
}
}
db.root, err = lldb.OpenBTree(db.alloc, opts.Compare, 1)
db.wal = opts.wal
return
}