// Create new memdb and froze the old one; need external synchronization.
// newMem only called synchronously by the writer.
func (db *DB) newMem(n int) (mem *memdb.DB, err error) {
num := db.s.allocFileNum()
file := db.s.getJournalFile(num)
w, err := file.Create()
if err != nil {
db.s.reuseFileNum(num)
return
}
db.memMu.Lock()
defer db.memMu.Unlock()
if db.journal == nil {
db.journal = journal.NewWriter(w)
} else {
db.journal.Reset(w)
db.journalWriter.Close()
db.frozenJournalFile = db.journalFile
}
db.journalWriter = w
db.journalFile = file
db.frozenMem = db.mem
db.mem = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n))
mem = db.mem
// The seq only incremented by the writer. And whoever called newMem
// should hold write lock, so no need additional synchronization here.
db.frozenSeq = db.seq
return
}
// Create new memdb and froze the old one; need external synchronization.
// newMem only called synchronously by the writer.
func (d *DB) newMem() (mem *memdb.DB, err error) {
s := d.s
num := s.allocFileNum()
file := s.getJournalFile(num)
w, err := file.Create()
if err != nil {
s.reuseFileNum(num)
return
}
d.memMu.Lock()
if d.journal == nil {
d.journal = journal.NewWriter(w)
} else {
d.journal.Reset(w)
d.journalWriter.Close()
d.frozenJournalFile = d.journalFile
}
d.journalWriter = w
d.journalFile = file
d.frozenMem = d.mem
d.mem = memdb.New(s.cmp, toPercent(d.s.o.GetWriteBuffer(), kWriteBufferPercent))
mem = d.mem
// The seq only incremented by the writer.
d.frozenSeq = d.seq
d.memMu.Unlock()
return
}
func (mc *stConstructor_MergedMemDB) init(t *testing.T, ho *stHarnessOpt) error {
ho.Randomize = true
mc.t = t
for i := range mc.db {
mc.db[i] = memdb.New(comparer.DefaultComparer, 0)
}
return nil
}
func TestBatch_Size(t *testing.T) {
b := new(Batch)
for i := 0; i < 2; i++ {
b.Put([]byte("key1"), []byte("value1"))
b.Put([]byte("key2"), []byte("value2"))
b.Delete([]byte("key1"))
b.Put([]byte("foo"), []byte("foovalue"))
b.Put([]byte("bar"), []byte("barvalue"))
mem := memdb.New(&iComparer{comparer.DefaultComparer}, 0)
b.memReplay(mem)
if b.size() != mem.Size() {
t.Errorf("invalid batch size calculation, want=%d got=%d", mem.Size(), b.size())
}
b.Reset()
}
}
func (d *DB) recoverJournal() error {
s := d.s
icmp := s.cmp
ff0, err := s.getFiles(storage.TypeJournal)
if err != nil {
return err
}
ff1 := files(ff0)
ff1.sort()
ff2 := make([]storage.File, 0, len(ff1))
for _, file := range ff1 {
if file.Num() >= s.stJournalNum || file.Num() == s.stPrevJournalNum {
s.markFileNum(file.Num())
ff2 = append(ff2, file)
}
}
var jr *journal.Reader
var of storage.File
var mem *memdb.DB
batch := new(Batch)
cm := newCMem(s)
buf := new(util.Buffer)
// Options.
strict := s.o.GetStrict(opt.StrictJournal)
checksum := s.o.GetStrict(opt.StrictJournalChecksum)
writeBuffer := s.o.GetWriteBuffer()
recoverJournal := func(file storage.File) error {
s.logf("journal@recovery recovering @%d", file.Num())
reader, err := file.Open()
if err != nil {
return err
}
defer reader.Close()
if jr == nil {
jr = journal.NewReader(reader, dropper{s, file}, strict, checksum)
} else {
jr.Reset(reader, dropper{s, file}, strict, checksum)
}
if of != nil {
if mem.Len() > 0 {
if err := cm.flush(mem, 0); err != nil {
return err
}
}
if err := cm.commit(file.Num(), d.seq); err != nil {
return err
}
cm.reset()
of.Remove()
of = nil
}
// Reset memdb.
mem.Reset()
for {
r, err := jr.Next()
if err != nil {
if err == io.EOF {
break
}
return err
}
buf.Reset()
if _, err := buf.ReadFrom(r); err != nil {
if strict {
return err
}
continue
}
if err := batch.decode(buf.Bytes()); err != nil {
return err
}
if err := batch.memReplay(mem); err != nil {
return err
}
d.seq = batch.seq + uint64(batch.len())
if mem.Size() >= writeBuffer {
// Large enough, flush it.
if err := cm.flush(mem, 0); err != nil {
return err
}
// Reset memdb.
mem.Reset()
}
}
of = file
return nil
}
// Recover all journals.
if len(ff2) > 0 {
s.logf("journal@recovery F·%d", len(ff2))
mem = memdb.New(icmp, toPercent(writeBuffer, kWriteBufferPercent))
for _, file := range ff2 {
if err := recoverJournal(file); err != nil {
return err
}
}
// Flush the last journal.
if mem.Len() > 0 {
if err := cm.flush(mem, 0); err != nil {
return err
}
}
}
// Create a new journal.
if _, err := d.newMem(); err != nil {
return err
}
// Commit.
if err := cm.commit(d.journalFile.Num(), d.seq); err != nil {
return err
}
// Remove the last journal.
if of != nil {
of.Remove()
}
return nil
}
func (mc *stConstructor_MemDB) init(t *testing.T, ho *stHarnessOpt) error {
ho.Randomize = true
mc.t = t
mc.db = memdb.New(comparer.DefaultComparer, 0)
return nil
}