// replayLogFile replays the edits in the named log file.
//
// d.mu must be held when calling this, but the mutex may be dropped and
// re-acquired during the course of this method.
func (d *DB) replayLogFile(ve *versionEdit, fs db.FileSystem, filename string) (maxSeqNum uint64, err error) {
file, err := fs.Open(filename)
if err != nil {
return 0, err
}
defer file.Close()
var (
mem *memdb.MemDB
batchBuf = new(bytes.Buffer)
ikey = make(internalKey, 512)
rr = record.NewReader(file)
)
for {
r, err := rr.Next()
if err == io.EOF {
break
}
if err != nil {
return 0, err
}
_, err = io.Copy(batchBuf, r)
if err != nil {
return 0, err
}
if batchBuf.Len() < batchHeaderLen {
return 0, fmt.Errorf("leveldb: corrupt log file %q", filename)
}
b := Batch{batchBuf.Bytes()}
seqNum := b.seqNum()
seqNum1 := seqNum + uint64(b.count())
if maxSeqNum < seqNum1 {
maxSeqNum = seqNum1
}
if mem == nil {
mem = memdb.New(&d.icmpOpts)
}
t := b.iter()
for ; seqNum != seqNum1; seqNum++ {
kind, ukey, value, ok := t.next()
if !ok {
return 0, fmt.Errorf("leveldb: corrupt log file %q", filename)
}
// Convert seqNum, kind and key into an internalKey, and add that ikey/value
// pair to mem.
//
// TODO: instead of copying to an intermediate buffer (ikey), is it worth
// adding a SetTwoPartKey(db.TwoPartKey{key0, key1}, value, opts) method to
// memdb.MemDB? What effect does that have on the db.Comparer interface?
//
// The C++ LevelDB code does not need an intermediate copy because its memdb
// implementation is a private implementation detail, and copies each internal
// key component from the Batch format straight to the skiplist buffer.
//
// Go's LevelDB considers the memdb functionality to be useful in its own
// right, and so leveldb/memdb is a separate package that is usable without
// having to import the top-level leveldb package. That extra abstraction
// means that we need to copy to an intermediate buffer here, to reconstruct
// the complete internal key to pass to the memdb.
ikey = makeInternalKey(ikey, ukey, kind, seqNum)
mem.Set(ikey, value, nil)
}
if len(t) != 0 {
return 0, fmt.Errorf("leveldb: corrupt log file %q", filename)
}
// TODO: if mem is large enough, write it to a level-0 table and set mem = nil.
batchBuf.Reset()
}
if mem != nil && !mem.Empty() {
meta, err := d.writeLevel0Table(fs, mem)
if err != nil {
return 0, err
}
ve.newFiles = append(ve.newFiles, newFileEntry{level: 0, meta: meta})
// Strictly speaking, it's too early to delete meta.fileNum from d.pendingOutputs,
// but we are replaying the log file, which happens before Open returns, so there
// is no possibility of deleteObsoleteFiles being called concurrently here.
delete(d.pendingOutputs, meta.fileNum)
}
return maxSeqNum, nil
}
func TestVersionEditDecode(t *testing.T) {
testCases := []struct {
filename string
encodedEdits []string
edits []versionEdit
}{
// db-stage-1 and db-stage-2 have the same manifest.
{
filename: "db-stage-1/MANIFEST-000002",
encodedEdits: []string{
"\x01\x1aleveldb.BytewiseComparator",
"\x02\x03\x09\x00\x03\x04\x04\x00",
},
edits: []versionEdit{
{
comparatorName: "leveldb.BytewiseComparator",
},
{
logNumber: 3,
prevLogNumber: 0,
nextFileNumber: 4,
lastSequence: 0,
},
},
},
// db-stage-3 and db-stage-4 have the same manifest.
{
filename: "db-stage-3/MANIFEST-000004",
encodedEdits: []string{
"\x01\x1aleveldb.BytewiseComparator",
"\x02\x06\x09\x00\x03\x07\x04\x05\x07\x00\x05\xa5\x01" +
"\x0bbar\x00\x05\x00\x00\x00\x00\x00\x00" +
"\x0bfoo\x01\x01\x00\x00\x00\x00\x00\x00",
},
edits: []versionEdit{
{
comparatorName: "leveldb.BytewiseComparator",
},
{
logNumber: 6,
prevLogNumber: 0,
nextFileNumber: 7,
lastSequence: 5,
newFiles: []newFileEntry{
{
level: 0,
meta: fileMetadata{
fileNum: 5,
size: 165,
smallest: internalKey("bar\x00\x05\x00\x00\x00\x00\x00\x00"),
largest: internalKey("foo\x01\x01\x00\x00\x00\x00\x00\x00"),
},
},
},
},
},
},
}
loop:
for _, tc := range testCases {
f, err := os.Open("testdata/" + tc.filename)
if err != nil {
t.Errorf("filename=%q: open error: %v", tc.filename, err)
continue
}
defer f.Close()
i, r := 0, record.NewReader(f)
for {
rr, err := r.Next()
if err == io.EOF {
break
}
if err != nil {
t.Errorf("filename=%q i=%d: record reader error: %v", tc.filename, i, err)
continue loop
}
if i >= len(tc.edits) {
t.Errorf("filename=%q i=%d: too many version edits", tc.filename, i+1)
continue loop
}
encodedEdit, err := ioutil.ReadAll(rr)
if err != nil {
t.Errorf("filename=%q i=%d: read error: %v", tc.filename, i, err)
continue loop
}
if s := string(encodedEdit); s != tc.encodedEdits[i] {
t.Errorf("filename=%q i=%d: got encoded %q, want %q", tc.filename, i, s, tc.encodedEdits[i])
continue loop
}
var edit versionEdit
err = edit.decode(bytes.NewReader(encodedEdit))
if err != nil {
t.Errorf("filename=%q i=%d: decode error: %v", tc.filename, i, err)
continue loop
}
if !reflect.DeepEqual(edit, tc.edits[i]) {
t.Errorf("filename=%q i=%d: decode\n\tgot %#v\n\twant %#v", tc.filename, i, edit, tc.edits[i])
continue loop
}
if err := checkRoundTrip(edit); err != nil {
t.Errorf("filename=%q i=%d: round trip: %v", tc.filename, i, err)
continue loop
}
i++
}
if i != len(tc.edits) {
t.Errorf("filename=%q: got %d edits, want %d", tc.filename, i, len(tc.edits))
continue
}
}
}
// load loads the version set from the manifest file.
func (vs *versionSet) load(dirname string, opts *db.Options) error {
vs.dirname = dirname
vs.opts = opts
vs.fs = opts.GetFileSystem()
vs.ucmp = opts.GetComparer()
vs.icmp = internalKeyComparer{vs.ucmp}
vs.dummyVersion.prev = &vs.dummyVersion
vs.dummyVersion.next = &vs.dummyVersion
// For historical reasons, the next file number is initialized to 2.
vs.nextFileNumber = 2
// Read the CURRENT file to find the current manifest file.
current, err := vs.fs.Open(dbFilename(dirname, fileTypeCurrent, 0))
if err != nil {
return fmt.Errorf("leveldb: could not open CURRENT file for DB %q: %v", dirname, err)
}
defer current.Close()
stat, err := current.Stat()
if err != nil {
return err
}
n := stat.Size()
if n == 0 {
return fmt.Errorf("leveldb: CURRENT file for DB %q is empty", dirname)
}
if n > 4096 {
return fmt.Errorf("leveldb: CURRENT file for DB %q is too large", dirname)
}
b := make([]byte, n)
_, err = current.ReadAt(b, 0)
if err != nil {
return err
}
if b[n-1] != '\n' {
return fmt.Errorf("leveldb: CURRENT file for DB %q is malformed", dirname)
}
b = b[:n-1]
// Read the versionEdits in the manifest file.
var bve bulkVersionEdit
manifest, err := vs.fs.Open(dirname + string(os.PathSeparator) + string(b))
if err != nil {
return fmt.Errorf("leveldb: could not open manifest file %q for DB %q: %v", b, dirname, err)
}
defer manifest.Close()
rr := record.NewReader(manifest)
for {
r, err := rr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
var ve versionEdit
err = ve.decode(r)
if err != nil {
return err
}
if ve.comparatorName != "" {
if ve.comparatorName != vs.ucmp.Name() {
return fmt.Errorf("leveldb: manifest file %q for DB %q: "+
"comparer name from file %q != comparer name from db.Options %q",
b, dirname, ve.comparatorName, vs.ucmp.Name())
}
}
bve.accumulate(&ve)
if ve.logNumber != 0 {
vs.logNumber = ve.logNumber
}
if ve.prevLogNumber != 0 {
vs.prevLogNumber = ve.prevLogNumber
}
if ve.nextFileNumber != 0 {
vs.nextFileNumber = ve.nextFileNumber
}
if ve.lastSequence != 0 {
vs.lastSequence = ve.lastSequence
}
}
if vs.logNumber == 0 || vs.nextFileNumber == 0 {
if vs.nextFileNumber == 2 {
// We have a freshly created DB.
} else {
return fmt.Errorf("leveldb: incomplete manifest file %q for DB %q", b, dirname)
}
}
vs.markFileNumUsed(vs.logNumber)
vs.markFileNumUsed(vs.prevLogNumber)
vs.manifestFileNumber = vs.nextFileNum()
newVersion, err := bve.apply(nil, vs.icmp)
if err != nil {
return err
}
vs.append(newVersion)
return nil
}