Пример #1
0
func create(f *os.File, filer lldb.Filer, opts *Options, isMem bool) (db *DB, err error) {
	defer func() {
		if db != nil {
			db.opts = opts
		}
	}()
	defer func() {
		lock := opts.lock
		if err != nil && lock != nil {
			lock.Close()
			db = nil
		}
	}()

	if err = opts.check(filer.Name(), true, !isMem); err != nil {
		return
	}

	b := [16]byte{byte(magic[0]), byte(magic[1]), byte(magic[2]), byte(magic[3]), 0x00} // ver 0x00
	if n, err := filer.WriteAt(b[:], 0); n != 16 {
		return nil, &os.PathError{Op: "kv.create.WriteAt", Path: filer.Name(), Err: err}
	}

	db = &DB{f: f, lock: opts.lock}

	if filer, err = opts.acidFiler(db, filer); err != nil {
		return nil, err
	}

	db.filer = filer
	if err = filer.BeginUpdate(); err != nil {
		return
	}

	defer func() {
		if e := filer.EndUpdate(); e != nil {
			if err == nil {
				err = e
			}
		}
	}()

	if db.alloc, err = lldb.NewAllocator(lldb.NewInnerFiler(filer, 16), &lldb.Options{}); err != nil {
		return nil, &os.PathError{Op: "kv.create", Path: filer.Name(), Err: err}
	}

	db.alloc.Compress = true
	db.isMem = isMem
	var h int64
	if db.root, h, err = lldb.CreateBTree(db.alloc, opts.Compare); err != nil {
		return
	}

	if h != 1 {
		panic("internal error")
	}

	db.wal = opts.wal
	return
}
Пример #2
0
func open00(name string, in *DB) (db *DB, err error) {
	db = in
	if db.alloc, err = lldb.NewAllocator(lldb.NewInnerFiler(db.filer, 16), &lldb.Options{}); err != nil {
		return nil, &os.PathError{Op: "dbm.Open", Path: name, Err: err}
	}

	db.alloc.Compress = true
	return db, db.boot()
}
Пример #3
0
func verifyDbFile(fn string) error {
	f, err := os.Open(fn) // O_RDONLY
	if err != nil {
		return err
	}

	defer f.Close()

	a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
	if err != nil {
		return err
	}

	return verifyAllocator(a)
}
Пример #4
0
Файл: dbm.go Проект: pmezard/exp
func create(f *os.File, filer lldb.Filer, opts *Options, isMem bool) (db *DB, err error) {
	defer func() {
		lock := opts.lock
		if err != nil && lock != nil {
			n := lock.Name()
			lock.Close()
			os.Remove(n)
			db = nil
		}
	}()

	if err = opts.check(filer.Name(), true, !isMem); err != nil {
		return
	}

	b := [16]byte{byte(magic[0]), byte(magic[1]), byte(magic[2]), byte(magic[3]), 0x00} // ver 0x00
	if n, err := filer.WriteAt(b[:], 0); n != 16 {
		return nil, &os.PathError{Op: "dbm.Create.WriteAt", Path: filer.Name(), Err: err}
	}

	db = &DB{emptySize: 128, f: f, lock: opts.lock, closed: make(chan bool)}

	if filer, err = opts.acidFiler(db, filer); err != nil {
		return nil, err
	}

	db.filer = filer
	if err = filer.BeginUpdate(); err != nil {
		return
	}

	defer func() {
		if e := filer.EndUpdate(); e != nil {
			if err == nil {
				err = e
			}
		}
	}()

	if db.alloc, err = lldb.NewAllocator(lldb.NewInnerFiler(filer, 16), &lldb.Options{}); err != nil {
		return nil, &os.PathError{Op: "dbm.Create", Path: filer.Name(), Err: err}
	}

	db.alloc.Compress = compress
	db.isMem = isMem
	return db, db.boot()
}
Пример #5
0
func newFileFromOSFile(f lldb.OSFile) (fi *file, err error) {
	nm := lockName(f.Name())
	lck, err := lock.Lock(nm)
	if err != nil {
		if lck != nil {
			lck.Close()
		}
		return nil, err
	}

	close := true
	defer func() {
		if close && lck != nil {
			lck.Close()
		}
	}()

	var w *os.File
	closew := false
	wn := walName(f.Name())
	w, err = os.OpenFile(wn, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
	closew = true
	defer func() {
		if closew {
			nm := w.Name()
			w.Close()
			os.Remove(nm)
			w = nil
		}
	}()

	if err != nil {
		if !os.IsExist(err) {
			return nil, err
		}

		closew = false
		w, err = os.OpenFile(wn, os.O_RDWR, 0666)
		if err != nil {
			return nil, err
		}

		closew = true
		st, err := w.Stat()
		if err != nil {
			return nil, err
		}

		if st.Size() != 0 {
			return nil, fmt.Errorf("non empty WAL file %s exists", wn)
		}
	}

	info, err := f.Stat()
	if err != nil {
		return nil, err
	}

	switch sz := info.Size(); {
	case sz == 0:
		b := make([]byte, 16)
		copy(b, []byte(magic))
		if _, err := f.Write(b); err != nil {
			return nil, err
		}

		filer := lldb.Filer(lldb.NewOSFiler(f))
		filer = lldb.NewInnerFiler(filer, 16)
		if filer, err = lldb.NewACIDFiler(filer, w); err != nil {
			return nil, err
		}

		a, err := lldb.NewAllocator(filer, &lldb.Options{})
		if err != nil {
			return nil, err
		}

		a.Compress = true
		s := &file{
			a:     a,
			codec: newGobCoder(),
			f0:    f,
			f:     filer,
			lck:   lck,
			name:  f.Name(),
			wal:   w,
		}
		if err = s.BeginTransaction(); err != nil {
			return nil, err
		}

		h, err := s.Create()
		if err != nil {
			return nil, err
		}

		if h != 1 { // root
			log.Panic("internal error")
		}

		if h, err = s.a.Alloc(make([]byte, 8)); err != nil {
			return nil, err
		}

		if h != 2 { // id
			log.Panic("internal error")
		}

		close, closew = false, false
		return s, s.Commit()
	default:
		b := make([]byte, 16)
		if _, err := f.Read(b); err != nil {
			return nil, err
		}

		if string(b[:len(magic)]) != magic {
			return nil, fmt.Errorf("unknown file format")
		}

		filer := lldb.Filer(lldb.NewOSFiler(f))
		filer = lldb.NewInnerFiler(filer, 16)
		if filer, err = lldb.NewACIDFiler(filer, w); err != nil {
			return nil, err
		}

		a, err := lldb.NewAllocator(filer, &lldb.Options{})
		if err != nil {
			return nil, err
		}

		bid, err := a.Get(nil, 2) // id
		if err != nil {
			return nil, err
		}

		if len(bid) != 8 {
			return nil, fmt.Errorf("corrupted id |% x|", bid)
		}

		id := int64(0)
		for _, v := range bid {
			id = (id << 8) | int64(v)
		}

		a.Compress = true
		s := &file{
			a:     a,
			codec: newGobCoder(),
			f0:    f,
			f:     filer,
			id:    id,
			lck:   lck,
			name:  f.Name(),
			wal:   w,
		}

		close, closew = false, false
		return s, nil
	}
}
Пример #6
0
func main0(fn string, oMax int, w func(s string, a ...interface{}), oStat bool, first, last string, dump bool, oBuckets bool) error {
	f, err := os.Open(fn) // O_RDONLY
	if err != nil {
		return err
	}

	defer f.Close()

	bits, err := ioutil.TempFile("", "kvaudit-")
	if err != nil {
		return err
	}

	defer func() {
		nm := bits.Name()
		bits.Close()
		os.Remove(nm)
	}()

	a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
	if err != nil {
		return err
	}

	cnt := 0
	var stats lldb.AllocStats
	err = a.Verify(lldb.NewSimpleFileFiler(bits), func(err error) bool {
		cnt++
		w("%d: %v\n", cnt, err)
		return cnt < oMax
	}, &stats)
	if err != nil {
		return err
	}

	if oStat {
		w("Handles     %10d  // total valid handles in use\n", stats.Handles)
		w("Compression %10d  // number of compressed blocks\n", stats.Compression)
		w("TotalAtoms  %10d  // total number of atoms == AllocAtoms + FreeAtoms\n", stats.TotalAtoms)
		w("AllocBytes  %10d  // bytes allocated (after decompression, if/where used)\n", stats.AllocBytes)
		w("AllocAtoms  %10d  // atoms allocated/used, including relocation atoms\n", stats.AllocAtoms)
		w("Relocations %10d  // number of relocated used blocks\n", stats.Relocations)
		w("FreeAtoms   %10d  // atoms unused\n", stats.FreeAtoms)
	}
	if oBuckets {
		var alloc, free [14]int64
		sizes := [14]int64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 4112}
		for atoms, cnt := range stats.AllocMap {
			if atoms >= 4096 {
				alloc[13] += cnt
				continue
			}

			for i := range sizes {
				if sizes[i+1] > atoms {
					alloc[i] += cnt
					break
				}
			}
		}
		for atoms, cnt := range stats.FreeMap {
			if atoms > 4096 {
				free[13] += cnt
				continue
			}

			for i := range sizes {
				if sizes[i+1] > atoms {
					free[i] += cnt
					break
				}
			}
		}
		w("Alloc blocks\n")
		for i, v := range alloc {
			w("%4d: %10d\n", sizes[i], v)
		}
		w("Free blocks\n")
		for i, v := range free {
			w("%4d: %10d\n", sizes[i], v)
		}
	}

	if !(first != "" || last != "" || dump) {
		return nil
	}

	t, err := lldb.OpenBTree(a, nil, 1)
	if err != nil {
		return err
	}

	dw := bufio.NewWriter(os.Stdout)
	defer dw.Flush()

	var e *lldb.BTreeEnumerator
	switch {
	case first != "":
		e, _, err = t.Seek([]byte(first))
	default:
		e, err = t.SeekFirst()
	}
	if err != nil {
		if err == io.EOF {
			err = nil
		}
		return err
	}

	blast := []byte(last)
	sep := []byte("->")
	for {
		k, v, err := e.Next()
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return err
		}

		dw.WriteString(fmt.Sprintf("+%d,%d:", len(k), len(v)))
		dw.Write(k)
		dw.Write(sep)
		dw.Write(v)
		dw.WriteByte('\n')

		if len(blast) != 0 && bytes.Compare(k, blast) >= 0 {
			break
		}
	}

	return nil
}
Пример #7
0
func main0(fn string, oMax int, w func(s string, a ...interface{}), oStat bool, first, last string, dump bool) error {
	f, err := os.Open(fn) // O_RDONLY
	if err != nil {
		return err
	}

	defer f.Close()

	bits, err := ioutil.TempFile("", "kvaudit-")
	if err != nil {
		return err
	}

	defer bits.Close()

	a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
	if err != nil {
		return err
	}

	cnt := 0
	var stats lldb.AllocStats
	err = a.Verify(lldb.NewSimpleFileFiler(bits), func(err error) bool {
		cnt++
		w("%d: %v\n", cnt, err)
		return cnt < oMax
	}, &stats)
	if oStat {
		w("%#v\n", &stats)
	}
	if err != nil {
		return err
	}

	if !(first != "" || last != "" || dump) {
		return nil
	}

	t, err := lldb.OpenBTree(a, nil, 1)
	if err != nil {
		return err
	}

	dw := bufio.NewWriter(os.Stdout)
	defer dw.Flush()

	var e *lldb.BTreeEnumerator
	switch {
	case first != "":
		e, _, err = t.Seek([]byte(first))
	default:
		e, err = t.SeekFirst()
	}
	if err != nil {
		if err == io.EOF {
			err = nil
		}
		return err
	}

	blast := []byte(last)
	sep := []byte("->")
	for {
		k, v, err := e.Next()
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return err
		}

		dw.WriteString(fmt.Sprintf("+%d,%d:", len(k), len(v)))
		dw.Write(k)
		dw.Write(sep)
		dw.Write(v)
		dw.WriteByte('\n')

		if len(blast) != 0 && bytes.Compare(k, blast) >= 0 {
			break
		}
	}

	return nil
}