Exemple #1
0
func (s *file) OpenIndex(unique bool, handle int64) (btreeIndex, error) {
	t, err := lldb.OpenBTree(s.a, s.collate, handle)
	if err != nil {
		return nil, err
	}

	return &fileIndex{s, handle, t, unique}, nil
}
Exemple #2
0
func (t *treeCache) getTree(db *DB, prefix int, name string, canCreate bool, cacheSize int) (r *lldb.BTree, err error) {
	m := t.get()
	r, ok := m[name]
	if ok {
		return
	}

	root, err := db.root()
	if err != nil {
		return
	}

	val, err := root.get(prefix, name)
	if err != nil {
		return
	}

	switch x := val.(type) {
	case nil:
		if !canCreate {
			return
		}

		var h int64
		r, h, err = lldb.CreateBTree(db.alloc, collate)
		if err != nil {
			return nil, err
		}

		if err = root.set(h, prefix, name); err != nil {
			return nil, err
		}
	case int64:
		if r, err = lldb.OpenBTree(db.alloc, collate, x); err != nil {
			return nil, err
		}
	default:
		return nil, &lldb.ErrINVAL{Src: "corrupted root directory value for", Val: fmt.Sprintf("%q, %q", prefix, name)}
	}

	if len(m) > cacheSize {
		i, j, n := 0, cacheSize/2, mathutil.Min(cacheSize/20, 10)
	loop:
		for k := range m {
			if i++; i >= j {
				delete(m, k)
				if n == 0 {
					break loop
				}

				n--
			}
		}
	}

	m[name] = r
	return
}
Exemple #3
0
func verifyAllocator(a *lldb.Allocator) error {
	bits, err := ioutil.TempFile("", "kv-verify-")
	if err != nil {
		return err
	}

	defer func() {
		nm := bits.Name()
		bits.Close()
		os.Remove(nm)
	}()

	var lerr error
	if err = a.Verify(
		lldb.NewSimpleFileFiler(bits),
		func(err error) bool {
			lerr = err
			return false
		},
		nil,
	); err != nil {
		return err
	}

	if lerr != nil {
		return lerr
	}

	t, err := lldb.OpenBTree(a, nil, 1)
	if err != nil {
		return err
	}

	e, err := t.SeekFirst()
	if err != nil {
		if err == io.EOF {
			err = nil
		}
		return err
	}

	for {
		_, _, err := e.Next()
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return err
		}
	}
}
Exemple #4
0
func (db *DB) root() (r *Array, err error) {
	if r = db._root; r != nil {
		return
	}

	sz, err := db.filer.Size()
	if err != nil {
		return
	}

	switch {
	case sz < db.emptySize:
		panic(fmt.Errorf("internal error: %d", sz))
	case sz == db.emptySize:
		tree, h, err := lldb.CreateBTree(db.alloc, collate)
		if err != nil {
			return nil, err
		}

		if h != 1 {
			panic("internal error")
		}

		r = &Array{db, tree, nil, "", 0}
		db._root = r
		return r, nil
	default:
		tree, err := lldb.OpenBTree(db.alloc, collate, 1)
		if err != nil {
			return nil, err
		}

		r = &Array{db, tree, nil, "", 0}
		db._root = r
		return r, nil
	}
}
Exemple #5
0
func main0(fn string, oMax int, w func(s string, a ...interface{}), oStat bool, first, last string, dump bool, oBuckets bool) error {
	f, err := os.Open(fn) // O_RDONLY
	if err != nil {
		return err
	}

	defer f.Close()

	bits, err := ioutil.TempFile("", "kvaudit-")
	if err != nil {
		return err
	}

	defer func() {
		nm := bits.Name()
		bits.Close()
		os.Remove(nm)
	}()

	a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
	if err != nil {
		return err
	}

	cnt := 0
	var stats lldb.AllocStats
	err = a.Verify(lldb.NewSimpleFileFiler(bits), func(err error) bool {
		cnt++
		w("%d: %v\n", cnt, err)
		return cnt < oMax
	}, &stats)
	if err != nil {
		return err
	}

	if oStat {
		w("Handles     %10d  // total valid handles in use\n", stats.Handles)
		w("Compression %10d  // number of compressed blocks\n", stats.Compression)
		w("TotalAtoms  %10d  // total number of atoms == AllocAtoms + FreeAtoms\n", stats.TotalAtoms)
		w("AllocBytes  %10d  // bytes allocated (after decompression, if/where used)\n", stats.AllocBytes)
		w("AllocAtoms  %10d  // atoms allocated/used, including relocation atoms\n", stats.AllocAtoms)
		w("Relocations %10d  // number of relocated used blocks\n", stats.Relocations)
		w("FreeAtoms   %10d  // atoms unused\n", stats.FreeAtoms)
	}
	if oBuckets {
		var alloc, free [14]int64
		sizes := [14]int64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 4112}
		for atoms, cnt := range stats.AllocMap {
			if atoms >= 4096 {
				alloc[13] += cnt
				continue
			}

			for i := range sizes {
				if sizes[i+1] > atoms {
					alloc[i] += cnt
					break
				}
			}
		}
		for atoms, cnt := range stats.FreeMap {
			if atoms > 4096 {
				free[13] += cnt
				continue
			}

			for i := range sizes {
				if sizes[i+1] > atoms {
					free[i] += cnt
					break
				}
			}
		}
		w("Alloc blocks\n")
		for i, v := range alloc {
			w("%4d: %10d\n", sizes[i], v)
		}
		w("Free blocks\n")
		for i, v := range free {
			w("%4d: %10d\n", sizes[i], v)
		}
	}

	if !(first != "" || last != "" || dump) {
		return nil
	}

	t, err := lldb.OpenBTree(a, nil, 1)
	if err != nil {
		return err
	}

	dw := bufio.NewWriter(os.Stdout)
	defer dw.Flush()

	var e *lldb.BTreeEnumerator
	switch {
	case first != "":
		e, _, err = t.Seek([]byte(first))
	default:
		e, err = t.SeekFirst()
	}
	if err != nil {
		if err == io.EOF {
			err = nil
		}
		return err
	}

	blast := []byte(last)
	sep := []byte("->")
	for {
		k, v, err := e.Next()
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return err
		}

		dw.WriteString(fmt.Sprintf("+%d,%d:", len(k), len(v)))
		dw.Write(k)
		dw.Write(sep)
		dw.Write(v)
		dw.WriteByte('\n')

		if len(blast) != 0 && bytes.Compare(k, blast) >= 0 {
			break
		}
	}

	return nil
}
Exemple #6
0
func main0(fn string, oMax int, w func(s string, a ...interface{}), oStat bool, first, last string, dump bool) error {
	f, err := os.Open(fn) // O_RDONLY
	if err != nil {
		return err
	}

	defer f.Close()

	bits, err := ioutil.TempFile("", "kvaudit-")
	if err != nil {
		return err
	}

	defer bits.Close()

	a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
	if err != nil {
		return err
	}

	cnt := 0
	var stats lldb.AllocStats
	err = a.Verify(lldb.NewSimpleFileFiler(bits), func(err error) bool {
		cnt++
		w("%d: %v\n", cnt, err)
		return cnt < oMax
	}, &stats)
	if oStat {
		w("%#v\n", &stats)
	}
	if err != nil {
		return err
	}

	if !(first != "" || last != "" || dump) {
		return nil
	}

	t, err := lldb.OpenBTree(a, nil, 1)
	if err != nil {
		return err
	}

	dw := bufio.NewWriter(os.Stdout)
	defer dw.Flush()

	var e *lldb.BTreeEnumerator
	switch {
	case first != "":
		e, _, err = t.Seek([]byte(first))
	default:
		e, err = t.SeekFirst()
	}
	if err != nil {
		if err == io.EOF {
			err = nil
		}
		return err
	}

	blast := []byte(last)
	sep := []byte("->")
	for {
		k, v, err := e.Next()
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return err
		}

		dw.WriteString(fmt.Sprintf("+%d,%d:", len(k), len(v)))
		dw.Write(k)
		dw.Write(sep)
		dw.Write(v)
		dw.WriteByte('\n')

		if len(blast) != 0 && bytes.Compare(k, blast) >= 0 {
			break
		}
	}

	return nil
}
Exemple #7
0
func (db *DB) victor(removes Array, h int64) {
	atomic.AddInt32(&activeVictors, 1)
	var err error
	var finished bool
	defer func() {
		if finished {
			func() {
				db.enter()

				defer func() {
					if e := recover(); e != nil {
						err = fmt.Errorf("%v", e)
					}
					db.leave(&err)
				}()

				lldb.RemoveBTree(db.alloc, h)
				removes.delete(h)
				db.setRemoving(h, false)
			}()
		}
		db.wg.Done()
		atomic.AddInt32(&activeVictors, -1)
	}()

	db.enter()

	doLeave := true
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("%v", e)
		}
		if doLeave {
			db.leave(&err)
		}
	}()

	t, err := lldb.OpenBTree(db.alloc, collate, h)
	if err != nil {
		finished = true
		return
	}

	doLeave = false
	if db.leave(&err) != nil {
		return
	}

	for {
		runtime.Gosched()
		select {
		case _, ok := <-db.stop:
			if !ok {
				return
			}
		default:
		}

		db.enter()
		doLeave = true
		if finished, err = t.DeleteAny(); finished || err != nil {
			return
		}

		doLeave = false
		if db.leave(&err) != nil {
			return
		}
	}
}
Exemple #8
0
// Open opens the named DB file for reading/writing. If successful, methods on
// the returned DB can be used for I/O; the associated file descriptor has mode
// os.O_RDWR. If there is an error, it will be of type *os.PathError.
//
// Note: While a DB is opened, it is locked and cannot be simultaneously opened
// again.
//
// For the meaning of opts please see documentation of Options.
func Open(name string, opts *Options) (db *DB, err error) {
	opts = opts.clone()
	opts._ACID = _ACIDFull
	defer func() {
		if db != nil {
			db.opts = opts
		}
	}()
	defer func() {
		lock := opts.lock
		if err != nil && lock != nil {
			lock.Close()
			db = nil
		}
		if err != nil {
			if db != nil {
				db.Close()
				db = nil
			}
		}
	}()

	if err = opts.check(name, false, true); err != nil {
		return
	}

	f, err := os.OpenFile(name, os.O_RDWR, 0666)
	if err != nil {
		return
	}

	filer := lldb.Filer(lldb.NewSimpleFileFiler(f))
	sz, err := filer.Size()
	if err != nil {
		return
	}

	if sz%16 != 0 {
		return nil, &os.PathError{Op: "kv.Open:", Path: name, Err: fmt.Errorf("file size %d(%#x) is not 0 (mod 16)", sz, sz)}
	}

	var b [16]byte
	if n, err := filer.ReadAt(b[:], 0); n != 16 || err != nil {
		return nil, &os.PathError{Op: "kv.Open.ReadAt", Path: name, Err: err}
	}

	var h header
	if err = h.rd(b[:]); err != nil {
		return nil, &os.PathError{Op: "kv.Open:validate header", Path: name, Err: err}
	}

	db = &DB{f: f, lock: opts.lock}
	if filer, err = opts.acidFiler(db, filer); err != nil {
		return nil, err
	}

	db.filer = filer
	switch h.ver {
	default:
		return nil, &os.PathError{Op: "kv.Open", Path: name, Err: fmt.Errorf("unknown/unsupported kv file format version %#x", h.ver)}
	case 0x00:
		if _, err = open00(name, db); err != nil {
			return nil, err
		}
	}

	db.root, err = lldb.OpenBTree(db.alloc, opts.Compare, 1)
	db.wal = opts.wal
	if opts.VerifyDbAfterOpen {
		err = verifyAllocator(db.alloc)
	}
	return
}