func (s *file) OpenIndex(unique bool, handle int64) (btreeIndex, error) {
t, err := lldb.OpenBTree(s.a, s.collate, handle)
if err != nil {
return nil, err
}
return &fileIndex{s, handle, t, unique}, nil
}
func (t *treeCache) getTree(db *DB, prefix int, name string, canCreate bool, cacheSize int) (r *lldb.BTree, err error) {
m := t.get()
r, ok := m[name]
if ok {
return
}
root, err := db.root()
if err != nil {
return
}
val, err := root.get(prefix, name)
if err != nil {
return
}
switch x := val.(type) {
case nil:
if !canCreate {
return
}
var h int64
r, h, err = lldb.CreateBTree(db.alloc, collate)
if err != nil {
return nil, err
}
if err = root.set(h, prefix, name); err != nil {
return nil, err
}
case int64:
if r, err = lldb.OpenBTree(db.alloc, collate, x); err != nil {
return nil, err
}
default:
return nil, &lldb.ErrINVAL{Src: "corrupted root directory value for", Val: fmt.Sprintf("%q, %q", prefix, name)}
}
if len(m) > cacheSize {
i, j, n := 0, cacheSize/2, mathutil.Min(cacheSize/20, 10)
loop:
for k := range m {
if i++; i >= j {
delete(m, k)
if n == 0 {
break loop
}
n--
}
}
}
m[name] = r
return
}
func verifyAllocator(a *lldb.Allocator) error {
bits, err := ioutil.TempFile("", "kv-verify-")
if err != nil {
return err
}
defer func() {
nm := bits.Name()
bits.Close()
os.Remove(nm)
}()
var lerr error
if err = a.Verify(
lldb.NewSimpleFileFiler(bits),
func(err error) bool {
lerr = err
return false
},
nil,
); err != nil {
return err
}
if lerr != nil {
return lerr
}
t, err := lldb.OpenBTree(a, nil, 1)
if err != nil {
return err
}
e, err := t.SeekFirst()
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
for {
_, _, err := e.Next()
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
}
}
func (db *DB) root() (r *Array, err error) {
if r = db._root; r != nil {
return
}
sz, err := db.filer.Size()
if err != nil {
return
}
switch {
case sz < db.emptySize:
panic(fmt.Errorf("internal error: %d", sz))
case sz == db.emptySize:
tree, h, err := lldb.CreateBTree(db.alloc, collate)
if err != nil {
return nil, err
}
if h != 1 {
panic("internal error")
}
r = &Array{db, tree, nil, "", 0}
db._root = r
return r, nil
default:
tree, err := lldb.OpenBTree(db.alloc, collate, 1)
if err != nil {
return nil, err
}
r = &Array{db, tree, nil, "", 0}
db._root = r
return r, nil
}
}
func main0(fn string, oMax int, w func(s string, a ...interface{}), oStat bool, first, last string, dump bool, oBuckets bool) error {
f, err := os.Open(fn) // O_RDONLY
if err != nil {
return err
}
defer f.Close()
bits, err := ioutil.TempFile("", "kvaudit-")
if err != nil {
return err
}
defer func() {
nm := bits.Name()
bits.Close()
os.Remove(nm)
}()
a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
if err != nil {
return err
}
cnt := 0
var stats lldb.AllocStats
err = a.Verify(lldb.NewSimpleFileFiler(bits), func(err error) bool {
cnt++
w("%d: %v\n", cnt, err)
return cnt < oMax
}, &stats)
if err != nil {
return err
}
if oStat {
w("Handles %10d // total valid handles in use\n", stats.Handles)
w("Compression %10d // number of compressed blocks\n", stats.Compression)
w("TotalAtoms %10d // total number of atoms == AllocAtoms + FreeAtoms\n", stats.TotalAtoms)
w("AllocBytes %10d // bytes allocated (after decompression, if/where used)\n", stats.AllocBytes)
w("AllocAtoms %10d // atoms allocated/used, including relocation atoms\n", stats.AllocAtoms)
w("Relocations %10d // number of relocated used blocks\n", stats.Relocations)
w("FreeAtoms %10d // atoms unused\n", stats.FreeAtoms)
}
if oBuckets {
var alloc, free [14]int64
sizes := [14]int64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 4112}
for atoms, cnt := range stats.AllocMap {
if atoms >= 4096 {
alloc[13] += cnt
continue
}
for i := range sizes {
if sizes[i+1] > atoms {
alloc[i] += cnt
break
}
}
}
for atoms, cnt := range stats.FreeMap {
if atoms > 4096 {
free[13] += cnt
continue
}
for i := range sizes {
if sizes[i+1] > atoms {
free[i] += cnt
break
}
}
}
w("Alloc blocks\n")
for i, v := range alloc {
w("%4d: %10d\n", sizes[i], v)
}
w("Free blocks\n")
for i, v := range free {
w("%4d: %10d\n", sizes[i], v)
}
}
if !(first != "" || last != "" || dump) {
return nil
}
t, err := lldb.OpenBTree(a, nil, 1)
if err != nil {
return err
}
dw := bufio.NewWriter(os.Stdout)
defer dw.Flush()
var e *lldb.BTreeEnumerator
switch {
case first != "":
e, _, err = t.Seek([]byte(first))
default:
e, err = t.SeekFirst()
}
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
blast := []byte(last)
sep := []byte("->")
for {
k, v, err := e.Next()
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
dw.WriteString(fmt.Sprintf("+%d,%d:", len(k), len(v)))
dw.Write(k)
dw.Write(sep)
dw.Write(v)
dw.WriteByte('\n')
if len(blast) != 0 && bytes.Compare(k, blast) >= 0 {
break
}
}
return nil
}
func main0(fn string, oMax int, w func(s string, a ...interface{}), oStat bool, first, last string, dump bool) error {
f, err := os.Open(fn) // O_RDONLY
if err != nil {
return err
}
defer f.Close()
bits, err := ioutil.TempFile("", "kvaudit-")
if err != nil {
return err
}
defer bits.Close()
a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
if err != nil {
return err
}
cnt := 0
var stats lldb.AllocStats
err = a.Verify(lldb.NewSimpleFileFiler(bits), func(err error) bool {
cnt++
w("%d: %v\n", cnt, err)
return cnt < oMax
}, &stats)
if oStat {
w("%#v\n", &stats)
}
if err != nil {
return err
}
if !(first != "" || last != "" || dump) {
return nil
}
t, err := lldb.OpenBTree(a, nil, 1)
if err != nil {
return err
}
dw := bufio.NewWriter(os.Stdout)
defer dw.Flush()
var e *lldb.BTreeEnumerator
switch {
case first != "":
e, _, err = t.Seek([]byte(first))
default:
e, err = t.SeekFirst()
}
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
blast := []byte(last)
sep := []byte("->")
for {
k, v, err := e.Next()
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
dw.WriteString(fmt.Sprintf("+%d,%d:", len(k), len(v)))
dw.Write(k)
dw.Write(sep)
dw.Write(v)
dw.WriteByte('\n')
if len(blast) != 0 && bytes.Compare(k, blast) >= 0 {
break
}
}
return nil
}
func (db *DB) victor(removes Array, h int64) {
atomic.AddInt32(&activeVictors, 1)
var err error
var finished bool
defer func() {
if finished {
func() {
db.enter()
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("%v", e)
}
db.leave(&err)
}()
lldb.RemoveBTree(db.alloc, h)
removes.delete(h)
db.setRemoving(h, false)
}()
}
db.wg.Done()
atomic.AddInt32(&activeVictors, -1)
}()
db.enter()
doLeave := true
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("%v", e)
}
if doLeave {
db.leave(&err)
}
}()
t, err := lldb.OpenBTree(db.alloc, collate, h)
if err != nil {
finished = true
return
}
doLeave = false
if db.leave(&err) != nil {
return
}
for {
runtime.Gosched()
select {
case _, ok := <-db.stop:
if !ok {
return
}
default:
}
db.enter()
doLeave = true
if finished, err = t.DeleteAny(); finished || err != nil {
return
}
doLeave = false
if db.leave(&err) != nil {
return
}
}
}
// Open opens the named DB file for reading/writing. If successful, methods on
// the returned DB can be used for I/O; the associated file descriptor has mode
// os.O_RDWR. If there is an error, it will be of type *os.PathError.
//
// Note: While a DB is opened, it is locked and cannot be simultaneously opened
// again.
//
// For the meaning of opts please see documentation of Options.
func Open(name string, opts *Options) (db *DB, err error) {
opts = opts.clone()
opts._ACID = _ACIDFull
defer func() {
if db != nil {
db.opts = opts
}
}()
defer func() {
lock := opts.lock
if err != nil && lock != nil {
lock.Close()
db = nil
}
if err != nil {
if db != nil {
db.Close()
db = nil
}
}
}()
if err = opts.check(name, false, true); err != nil {
return
}
f, err := os.OpenFile(name, os.O_RDWR, 0666)
if err != nil {
return
}
filer := lldb.Filer(lldb.NewSimpleFileFiler(f))
sz, err := filer.Size()
if err != nil {
return
}
if sz%16 != 0 {
return nil, &os.PathError{Op: "kv.Open:", Path: name, Err: fmt.Errorf("file size %d(%#x) is not 0 (mod 16)", sz, sz)}
}
var b [16]byte
if n, err := filer.ReadAt(b[:], 0); n != 16 || err != nil {
return nil, &os.PathError{Op: "kv.Open.ReadAt", Path: name, Err: err}
}
var h header
if err = h.rd(b[:]); err != nil {
return nil, &os.PathError{Op: "kv.Open:validate header", Path: name, Err: err}
}
db = &DB{f: f, lock: opts.lock}
if filer, err = opts.acidFiler(db, filer); err != nil {
return nil, err
}
db.filer = filer
switch h.ver {
default:
return nil, &os.PathError{Op: "kv.Open", Path: name, Err: fmt.Errorf("unknown/unsupported kv file format version %#x", h.ver)}
case 0x00:
if _, err = open00(name, db); err != nil {
return nil, err
}
}
db.root, err = lldb.OpenBTree(db.alloc, opts.Compare, 1)
db.wal = opts.wal
if opts.VerifyDbAfterOpen {
err = verifyAllocator(db.alloc)
}
return
}