func create(f *os.File, filer lldb.Filer, opts *Options, isMem bool) (db *DB, err error) {
defer func() {
if db != nil {
db.opts = opts
}
}()
defer func() {
lock := opts.lock
if err != nil && lock != nil {
lock.Close()
db = nil
}
}()
if err = opts.check(filer.Name(), true, !isMem); err != nil {
return
}
b := [16]byte{byte(magic[0]), byte(magic[1]), byte(magic[2]), byte(magic[3]), 0x00} // ver 0x00
if n, err := filer.WriteAt(b[:], 0); n != 16 {
return nil, &os.PathError{Op: "kv.create.WriteAt", Path: filer.Name(), Err: err}
}
db = &DB{f: f, lock: opts.lock}
if filer, err = opts.acidFiler(db, filer); err != nil {
return nil, err
}
db.filer = filer
if err = filer.BeginUpdate(); err != nil {
return
}
defer func() {
if e := filer.EndUpdate(); e != nil {
if err == nil {
err = e
}
}
}()
if db.alloc, err = lldb.NewAllocator(lldb.NewInnerFiler(filer, 16), &lldb.Options{}); err != nil {
return nil, &os.PathError{Op: "kv.create", Path: filer.Name(), Err: err}
}
db.alloc.Compress = true
db.isMem = isMem
var h int64
if db.root, h, err = lldb.CreateBTree(db.alloc, opts.Compare); err != nil {
return
}
if h != 1 {
panic("internal error")
}
db.wal = opts.wal
return
}
func fillseq() {
dbname := os.Args[0] + ".db"
f, err := os.OpenFile(dbname, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
if err != nil {
log.Fatal(err)
}
defer func() {
f.Close()
os.Remove(f.Name())
}()
filer := lldb.NewSimpleFileFiler(f)
a, err := lldb.NewAllocator(filer, &lldb.Options{})
if err != nil {
log.Println(err)
return
}
a.Compress = true
b, _, err := lldb.CreateBTree(a, nil)
if err != nil {
log.Println(err)
return
}
var keys [N][16]byte
for i := range keys {
binary.BigEndian.PutUint32(keys[i][:], uint32(i))
}
debug.FreeOSMemory()
t0 := time.Now()
for _, key := range keys {
if err = b.Set(key[:], value100); err != nil {
log.Println(err)
return
}
}
if err := filer.Sync(); err != nil {
log.Println(err)
return
}
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
d := time.Since(t0)
fi, err := f.Stat()
if err != nil {
log.Println(err)
return
}
secs := float64(d/time.Nanosecond) / float64(time.Second)
sz := fi.Size()
fmt.Printf("fillseq :%19v/op;%7.1f MB/s (%g secs, %d bytes)\n", d/N, float64(sz)/secs/1e6, secs, sz)
nn, bytes := bufs.GCache.Stats()
fmt.Printf("%d %d\n", nn, bytes)
fmt.Printf("%+v\n", ms)
}
func (s *file) CreateTemp(asc bool) (bt temp, err error) {
f, err := ioutil.TempFile("", "ql-tmp-")
if err != nil {
return nil, err
}
fn := f.Name()
filer := lldb.NewSimpleFileFiler(f)
a, err := lldb.NewAllocator(filer, &lldb.Options{})
if err != nil {
f.Close()
os.Remove(fn)
return nil, err
}
t, _, err := lldb.CreateBTree(a, lldbCollators[asc])
if err != nil {
f.Close()
os.Remove(fn)
return nil, err
}
return &fileTemp{
a: a,
f: f,
t: t,
}, nil
}
func (s *file) CreateTemp(asc bool) (bt temp, err error) {
f, err := s.tempFile("", "ql-tmp-")
if err != nil {
return nil, err
}
fn := f.Name()
filer := lldb.NewOSFiler(f)
a, err := lldb.NewAllocator(filer, &lldb.Options{})
if err != nil {
f.Close()
os.Remove(fn)
return nil, err
}
k := 1
if !asc {
k = -1
}
t, _, err := lldb.CreateBTree(a, func(a, b []byte) int { //TODO w/ error return
da, err := lldb.DecodeScalars(a)
if err != nil {
log.Panic(err)
}
if err = s.expandBytes(da); err != nil {
log.Panic(err)
}
db, err := lldb.DecodeScalars(b)
if err != nil {
log.Panic(err)
}
if err = s.expandBytes(db); err != nil {
log.Panic(err)
}
return k * collate(da, db)
})
if err != nil {
f.Close()
if fn != "" {
os.Remove(fn)
}
return nil, err
}
x := &fileTemp{file: &file{
a: a,
codec: newGobCoder(),
f0: f,
},
t: t}
return x, nil
}
func open00(name string, in *DB) (db *DB, err error) {
db = in
if db.alloc, err = lldb.NewAllocator(lldb.NewInnerFiler(db.filer, 16), &lldb.Options{}); err != nil {
return nil, &os.PathError{Op: "dbm.Open", Path: name, Err: err}
}
db.alloc.Compress = true
return db, db.boot()
}
func TestProf(t *testing.T) {
dbname := os.Args[0] + ".db"
f, err := os.OpenFile(dbname, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
if err != nil {
t.Fatal(err)
}
defer func() {
f.Close()
os.Remove(f.Name())
}()
filer := lldb.NewSimpleFileFiler(f) // file
//filer := lldb.NewMemFiler() // mem
a, err := lldb.NewAllocator(filer, &lldb.Options{})
if err != nil {
t.Error(err)
return
}
a.Compress = true
b, _, err := lldb.CreateBTree(a, nil)
if err != nil {
t.Error(err)
return
}
var key [16]byte
for i := uint32(0); int(i) < 1e6; i++ {
binary.BigEndian.PutUint32(key[:], i)
if err = b.Set(key[:], value100); err != nil {
t.Error(err)
return
}
}
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
bufsU, bufsT, bytesU, bytesT, h, m := a.CacheStats()
const p = 100.0
t.Logf(
"cache: buffers %d/%d(%.1f%%), bytes %d/%d(%.1f%%), hits %d(%.1f%%), misses %d(%.1f%%)",
bufsU, bufsT, p*float64(bufsU)/float64(bufsT),
bytesU, bytesT, p*float64(bytesU)/float64(bytesT),
h, p*float64(h)/float64(h+m),
m, p*float64(m)/float64(h+m),
)
nn, bts := bufs.GCache.Stats()
t.Logf("bufs.GCache.Stats() {%d, %d}", nn, bts)
t.Logf("%+v\n", ms)
}
func verifyDbFile(fn string) error {
f, err := os.Open(fn) // O_RDONLY
if err != nil {
return err
}
defer f.Close()
a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
if err != nil {
return err
}
return verifyAllocator(a)
}
func create(f *os.File, filer lldb.Filer, opts *Options, isMem bool) (db *DB, err error) {
defer func() {
lock := opts.lock
if err != nil && lock != nil {
n := lock.Name()
lock.Close()
os.Remove(n)
db = nil
}
}()
if err = opts.check(filer.Name(), true, !isMem); err != nil {
return
}
b := [16]byte{byte(magic[0]), byte(magic[1]), byte(magic[2]), byte(magic[3]), 0x00} // ver 0x00
if n, err := filer.WriteAt(b[:], 0); n != 16 {
return nil, &os.PathError{Op: "dbm.Create.WriteAt", Path: filer.Name(), Err: err}
}
db = &DB{emptySize: 128, f: f, lock: opts.lock, closed: make(chan bool)}
if filer, err = opts.acidFiler(db, filer); err != nil {
return nil, err
}
db.filer = filer
if err = filer.BeginUpdate(); err != nil {
return
}
defer func() {
if e := filer.EndUpdate(); e != nil {
if err == nil {
err = e
}
}
}()
if db.alloc, err = lldb.NewAllocator(lldb.NewInnerFiler(filer, 16), &lldb.Options{}); err != nil {
return nil, &os.PathError{Op: "dbm.Create", Path: filer.Name(), Err: err}
}
db.alloc.Compress = compress
db.isMem = isMem
return db, db.boot()
}
func BenchmarkMem(b *testing.B) {
f, err := ioutil.TempFile("", "")
if err != nil {
b.Fatal(err)
}
defer func() {
f.Close()
os.Remove(f.Name())
}()
filer := lldb.NewSimpleFileFiler(f)
a, err := lldb.NewAllocator(filer, &lldb.Options{})
if err != nil {
b.Error(err)
return
}
a.Compress = true
t, _, err := lldb.CreateBTree(a, nil)
if err != nil {
b.Error(err)
return
}
b.ResetTimer()
var key [16]byte
for i := uint32(0); int(i) < b.N; i++ {
binary.BigEndian.PutUint32(key[:], i)
if err = t.Set(key[:], value100); err != nil {
b.Error(err)
return
}
}
if err := filer.Sync(); err != nil {
b.Error(err)
return
}
}
func newFileFromOSFile(f lldb.OSFile) (fi *file, err error) {
nm := lockName(f.Name())
lck, err := lock.Lock(nm)
if err != nil {
if lck != nil {
lck.Close()
}
return nil, err
}
close := true
defer func() {
if close && lck != nil {
lck.Close()
}
}()
var w *os.File
closew := false
wn := walName(f.Name())
w, err = os.OpenFile(wn, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
closew = true
defer func() {
if closew {
nm := w.Name()
w.Close()
os.Remove(nm)
w = nil
}
}()
if err != nil {
if !os.IsExist(err) {
return nil, err
}
closew = false
w, err = os.OpenFile(wn, os.O_RDWR, 0666)
if err != nil {
return nil, err
}
closew = true
st, err := w.Stat()
if err != nil {
return nil, err
}
if st.Size() != 0 {
return nil, fmt.Errorf("non empty WAL file %s exists", wn)
}
}
info, err := f.Stat()
if err != nil {
return nil, err
}
switch sz := info.Size(); {
case sz == 0:
b := make([]byte, 16)
copy(b, []byte(magic))
if _, err := f.Write(b); err != nil {
return nil, err
}
filer := lldb.Filer(lldb.NewOSFiler(f))
filer = lldb.NewInnerFiler(filer, 16)
if filer, err = lldb.NewACIDFiler(filer, w); err != nil {
return nil, err
}
a, err := lldb.NewAllocator(filer, &lldb.Options{})
if err != nil {
return nil, err
}
a.Compress = true
s := &file{
a: a,
codec: newGobCoder(),
f0: f,
f: filer,
lck: lck,
name: f.Name(),
wal: w,
}
if err = s.BeginTransaction(); err != nil {
return nil, err
}
h, err := s.Create()
if err != nil {
return nil, err
}
if h != 1 { // root
log.Panic("internal error")
}
if h, err = s.a.Alloc(make([]byte, 8)); err != nil {
return nil, err
}
if h != 2 { // id
log.Panic("internal error")
}
close, closew = false, false
return s, s.Commit()
default:
b := make([]byte, 16)
if _, err := f.Read(b); err != nil {
return nil, err
}
if string(b[:len(magic)]) != magic {
return nil, fmt.Errorf("unknown file format")
}
filer := lldb.Filer(lldb.NewOSFiler(f))
filer = lldb.NewInnerFiler(filer, 16)
if filer, err = lldb.NewACIDFiler(filer, w); err != nil {
return nil, err
}
a, err := lldb.NewAllocator(filer, &lldb.Options{})
if err != nil {
return nil, err
}
bid, err := a.Get(nil, 2) // id
if err != nil {
return nil, err
}
if len(bid) != 8 {
return nil, fmt.Errorf("corrupted id |% x|", bid)
}
id := int64(0)
for _, v := range bid {
id = (id << 8) | int64(v)
}
a.Compress = true
s := &file{
a: a,
codec: newGobCoder(),
f0: f,
f: filer,
id: id,
lck: lck,
name: f.Name(),
wal: w,
}
close, closew = false, false
return s, nil
}
}
func main0(fn string, oMax int, w func(s string, a ...interface{}), oStat bool, first, last string, dump bool, oBuckets bool) error {
f, err := os.Open(fn) // O_RDONLY
if err != nil {
return err
}
defer f.Close()
bits, err := ioutil.TempFile("", "kvaudit-")
if err != nil {
return err
}
defer func() {
nm := bits.Name()
bits.Close()
os.Remove(nm)
}()
a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
if err != nil {
return err
}
cnt := 0
var stats lldb.AllocStats
err = a.Verify(lldb.NewSimpleFileFiler(bits), func(err error) bool {
cnt++
w("%d: %v\n", cnt, err)
return cnt < oMax
}, &stats)
if err != nil {
return err
}
if oStat {
w("Handles %10d // total valid handles in use\n", stats.Handles)
w("Compression %10d // number of compressed blocks\n", stats.Compression)
w("TotalAtoms %10d // total number of atoms == AllocAtoms + FreeAtoms\n", stats.TotalAtoms)
w("AllocBytes %10d // bytes allocated (after decompression, if/where used)\n", stats.AllocBytes)
w("AllocAtoms %10d // atoms allocated/used, including relocation atoms\n", stats.AllocAtoms)
w("Relocations %10d // number of relocated used blocks\n", stats.Relocations)
w("FreeAtoms %10d // atoms unused\n", stats.FreeAtoms)
}
if oBuckets {
var alloc, free [14]int64
sizes := [14]int64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 4112}
for atoms, cnt := range stats.AllocMap {
if atoms >= 4096 {
alloc[13] += cnt
continue
}
for i := range sizes {
if sizes[i+1] > atoms {
alloc[i] += cnt
break
}
}
}
for atoms, cnt := range stats.FreeMap {
if atoms > 4096 {
free[13] += cnt
continue
}
for i := range sizes {
if sizes[i+1] > atoms {
free[i] += cnt
break
}
}
}
w("Alloc blocks\n")
for i, v := range alloc {
w("%4d: %10d\n", sizes[i], v)
}
w("Free blocks\n")
for i, v := range free {
w("%4d: %10d\n", sizes[i], v)
}
}
if !(first != "" || last != "" || dump) {
return nil
}
t, err := lldb.OpenBTree(a, nil, 1)
if err != nil {
return err
}
dw := bufio.NewWriter(os.Stdout)
defer dw.Flush()
var e *lldb.BTreeEnumerator
switch {
case first != "":
e, _, err = t.Seek([]byte(first))
default:
e, err = t.SeekFirst()
}
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
blast := []byte(last)
sep := []byte("->")
for {
k, v, err := e.Next()
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
dw.WriteString(fmt.Sprintf("+%d,%d:", len(k), len(v)))
dw.Write(k)
dw.Write(sep)
dw.Write(v)
dw.WriteByte('\n')
if len(blast) != 0 && bytes.Compare(k, blast) >= 0 {
break
}
}
return nil
}
func main0(fn string, oMax int, w func(s string, a ...interface{}), oStat bool, first, last string, dump bool) error {
f, err := os.Open(fn) // O_RDONLY
if err != nil {
return err
}
defer f.Close()
bits, err := ioutil.TempFile("", "kvaudit-")
if err != nil {
return err
}
defer bits.Close()
a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
if err != nil {
return err
}
cnt := 0
var stats lldb.AllocStats
err = a.Verify(lldb.NewSimpleFileFiler(bits), func(err error) bool {
cnt++
w("%d: %v\n", cnt, err)
return cnt < oMax
}, &stats)
if oStat {
w("%#v\n", &stats)
}
if err != nil {
return err
}
if !(first != "" || last != "" || dump) {
return nil
}
t, err := lldb.OpenBTree(a, nil, 1)
if err != nil {
return err
}
dw := bufio.NewWriter(os.Stdout)
defer dw.Flush()
var e *lldb.BTreeEnumerator
switch {
case first != "":
e, _, err = t.Seek([]byte(first))
default:
e, err = t.SeekFirst()
}
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
blast := []byte(last)
sep := []byte("->")
for {
k, v, err := e.Next()
if err != nil {
if err == io.EOF {
err = nil
}
return err
}
dw.WriteString(fmt.Sprintf("+%d,%d:", len(k), len(v)))
dw.Write(k)
dw.Write(sep)
dw.Write(v)
dw.WriteByte('\n')
if len(blast) != 0 && bytes.Compare(k, blast) >= 0 {
break
}
}
return nil
}