Beispiel #1
0
// load loads the version set from the manifest file.
func (vs *versionSet) load(dirname string, opts *db.Options) error {
	vs.dirname = dirname
	vs.opts = opts
	vs.fs = opts.GetFileSystem()
	vs.ucmp = opts.GetComparer()
	vs.icmp = internalKeyComparer{vs.ucmp}
	vs.dummyVersion.prev = &vs.dummyVersion
	vs.dummyVersion.next = &vs.dummyVersion
	// For historical reasons, the next file number is initialized to 2.
	vs.nextFileNumber = 2

	// Read the CURRENT file to find the current manifest file.
	current, err := vs.fs.Open(dbFilename(dirname, fileTypeCurrent, 0))
	if err != nil {
		return fmt.Errorf("leveldb: could not open CURRENT file for DB %q: %v", dirname, err)
	}
	defer current.Close()
	stat, err := current.Stat()
	if err != nil {
		return err
	}
	n := stat.Size()
	if n == 0 {
		return fmt.Errorf("leveldb: CURRENT file for DB %q is empty", dirname)
	}
	if n > 4096 {
		return fmt.Errorf("leveldb: CURRENT file for DB %q is too large", dirname)
	}
	b := make([]byte, n)
	_, err = current.ReadAt(b, 0)
	if err != nil {
		return err
	}
	if b[n-1] != '\n' {
		return fmt.Errorf("leveldb: CURRENT file for DB %q is malformed", dirname)
	}
	b = b[:n-1]

	// Read the versionEdits in the manifest file.
	var bve bulkVersionEdit
	manifest, err := vs.fs.Open(dirname + string(os.PathSeparator) + string(b))
	if err != nil {
		return fmt.Errorf("leveldb: could not open manifest file %q for DB %q: %v", b, dirname, err)
	}
	defer manifest.Close()
	rr := record.NewReader(manifest)
	for {
		r, err := rr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		var ve versionEdit
		err = ve.decode(r)
		if err != nil {
			return err
		}
		if ve.comparatorName != "" {
			if ve.comparatorName != vs.ucmp.Name() {
				return fmt.Errorf("leveldb: manifest file %q for DB %q: "+
					"comparer name from file %q != comparer name from db.Options %q",
					b, dirname, ve.comparatorName, vs.ucmp.Name())
			}
		}
		bve.accumulate(&ve)
		if ve.logNumber != 0 {
			vs.logNumber = ve.logNumber
		}
		if ve.prevLogNumber != 0 {
			vs.prevLogNumber = ve.prevLogNumber
		}
		if ve.nextFileNumber != 0 {
			vs.nextFileNumber = ve.nextFileNumber
		}
		if ve.lastSequence != 0 {
			vs.lastSequence = ve.lastSequence
		}
	}
	if vs.logNumber == 0 || vs.nextFileNumber == 0 {
		if vs.nextFileNumber == 2 {
			// We have a freshly created DB.
		} else {
			return fmt.Errorf("leveldb: incomplete manifest file %q for DB %q", b, dirname)
		}
	}
	vs.markFileNumUsed(vs.logNumber)
	vs.markFileNumUsed(vs.prevLogNumber)
	vs.manifestFileNumber = vs.nextFileNum()

	newVersion, err := bve.apply(nil, vs.icmp)
	if err != nil {
		return err
	}
	vs.append(newVersion)
	return nil
}
Beispiel #2
0
// replayLogFile replays the edits in the named log file.
//
// d.mu must be held when calling this, but the mutex may be dropped and
// re-acquired during the course of this method.
func (d *DB) replayLogFile(ve *versionEdit, fs db.FileSystem, filename string) (maxSeqNum uint64, err error) {
	file, err := fs.Open(filename)
	if err != nil {
		return 0, err
	}
	defer file.Close()

	var (
		mem      *memdb.MemDB
		batchBuf = new(bytes.Buffer)
		ikey     = make(internalKey, 512)
		rr       = record.NewReader(file)
	)
	for {
		r, err := rr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return 0, err
		}
		_, err = io.Copy(batchBuf, r)
		if err != nil {
			return 0, err
		}

		if batchBuf.Len() < batchHeaderLen {
			return 0, fmt.Errorf("leveldb: corrupt log file %q", filename)
		}
		b := Batch{batchBuf.Bytes()}
		seqNum := b.seqNum()
		seqNum1 := seqNum + uint64(b.count())
		if maxSeqNum < seqNum1 {
			maxSeqNum = seqNum1
		}

		if mem == nil {
			mem = memdb.New(&d.icmpOpts)
		}

		t := b.iter()
		for ; seqNum != seqNum1; seqNum++ {
			kind, ukey, value, ok := t.next()
			if !ok {
				return 0, fmt.Errorf("leveldb: corrupt log file %q", filename)
			}
			// Convert seqNum, kind and key into an internalKey, and add that ikey/value
			// pair to mem.
			//
			// TODO: instead of copying to an intermediate buffer (ikey), is it worth
			// adding a SetTwoPartKey(db.TwoPartKey{key0, key1}, value, opts) method to
			// memdb.MemDB? What effect does that have on the db.Comparer interface?
			//
			// The C++ LevelDB code does not need an intermediate copy because its memdb
			// implementation is a private implementation detail, and copies each internal
			// key component from the Batch format straight to the skiplist buffer.
			//
			// Go's LevelDB considers the memdb functionality to be useful in its own
			// right, and so leveldb/memdb is a separate package that is usable without
			// having to import the top-level leveldb package. That extra abstraction
			// means that we need to copy to an intermediate buffer here, to reconstruct
			// the complete internal key to pass to the memdb.
			ikey = makeInternalKey(ikey, ukey, kind, seqNum)
			mem.Set(ikey, value, nil)
		}
		if len(t) != 0 {
			return 0, fmt.Errorf("leveldb: corrupt log file %q", filename)
		}

		// TODO: if mem is large enough, write it to a level-0 table and set mem = nil.

		batchBuf.Reset()
	}

	if mem != nil && !mem.Empty() {
		meta, err := d.writeLevel0Table(fs, mem)
		if err != nil {
			return 0, err
		}
		ve.newFiles = append(ve.newFiles, newFileEntry{level: 0, meta: meta})
		// Strictly speaking, it's too early to delete meta.fileNum from d.pendingOutputs,
		// but we are replaying the log file, which happens before Open returns, so there
		// is no possibility of deleteObsoleteFiles being called concurrently here.
		delete(d.pendingOutputs, meta.fileNum)
	}

	return maxSeqNum, nil
}
func TestVersionEditDecode(t *testing.T) {
	testCases := []struct {
		filename     string
		encodedEdits []string
		edits        []versionEdit
	}{
		// db-stage-1 and db-stage-2 have the same manifest.
		{
			filename: "db-stage-1/MANIFEST-000002",
			encodedEdits: []string{
				"\x01\x1aleveldb.BytewiseComparator",
				"\x02\x03\x09\x00\x03\x04\x04\x00",
			},
			edits: []versionEdit{
				{
					comparatorName: "leveldb.BytewiseComparator",
				},
				{
					logNumber:      3,
					prevLogNumber:  0,
					nextFileNumber: 4,
					lastSequence:   0,
				},
			},
		},
		// db-stage-3 and db-stage-4 have the same manifest.
		{
			filename: "db-stage-3/MANIFEST-000004",
			encodedEdits: []string{
				"\x01\x1aleveldb.BytewiseComparator",
				"\x02\x06\x09\x00\x03\x07\x04\x05\x07\x00\x05\xa5\x01" +
					"\x0bbar\x00\x05\x00\x00\x00\x00\x00\x00" +
					"\x0bfoo\x01\x01\x00\x00\x00\x00\x00\x00",
			},
			edits: []versionEdit{
				{
					comparatorName: "leveldb.BytewiseComparator",
				},
				{
					logNumber:      6,
					prevLogNumber:  0,
					nextFileNumber: 7,
					lastSequence:   5,
					newFiles: []newFileEntry{
						{
							level: 0,
							meta: fileMetadata{
								fileNum:  5,
								size:     165,
								smallest: internalKey("bar\x00\x05\x00\x00\x00\x00\x00\x00"),
								largest:  internalKey("foo\x01\x01\x00\x00\x00\x00\x00\x00"),
							},
						},
					},
				},
			},
		},
	}

loop:
	for _, tc := range testCases {
		f, err := os.Open("testdata/" + tc.filename)
		if err != nil {
			t.Errorf("filename=%q: open error: %v", tc.filename, err)
			continue
		}
		defer f.Close()
		i, r := 0, record.NewReader(f)
		for {
			rr, err := r.Next()
			if err == io.EOF {
				break
			}
			if err != nil {
				t.Errorf("filename=%q i=%d: record reader error: %v", tc.filename, i, err)
				continue loop
			}
			if i >= len(tc.edits) {
				t.Errorf("filename=%q i=%d: too many version edits", tc.filename, i+1)
				continue loop
			}

			encodedEdit, err := ioutil.ReadAll(rr)
			if err != nil {
				t.Errorf("filename=%q i=%d: read error: %v", tc.filename, i, err)
				continue loop
			}
			if s := string(encodedEdit); s != tc.encodedEdits[i] {
				t.Errorf("filename=%q i=%d: got encoded %q, want %q", tc.filename, i, s, tc.encodedEdits[i])
				continue loop
			}

			var edit versionEdit
			err = edit.decode(bytes.NewReader(encodedEdit))
			if err != nil {
				t.Errorf("filename=%q i=%d: decode error: %v", tc.filename, i, err)
				continue loop
			}
			if !reflect.DeepEqual(edit, tc.edits[i]) {
				t.Errorf("filename=%q i=%d: decode\n\tgot  %#v\n\twant %#v", tc.filename, i, edit, tc.edits[i])
				continue loop
			}
			if err := checkRoundTrip(edit); err != nil {
				t.Errorf("filename=%q i=%d: round trip: %v", tc.filename, i, err)
				continue loop
			}

			i++
		}
		if i != len(tc.edits) {
			t.Errorf("filename=%q: got %d edits, want %d", tc.filename, i, len(tc.edits))
			continue
		}
	}
}