func (s *session) flushMemdb(rec *sessionRecord, mdb *memdb.DB, level int) (level_ int, err error) {
	// Create sorted table.
	iter := mdb.NewIterator(nil)
	defer iter.Release()
	t, n, err := s.tops.createFrom(iter)
	if err != nil {
		return level, err
	}

	// Pick level and add to record.
	if level < 0 {
		level = s.pickMemdbLevel(t.imin.ukey(), t.imax.ukey())
	}
	rec.addTableFile(level, t)

	s.logf("memdb@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.imin, t.imax)
	return level, nil
}
Beispiel #2
0
func (c *cMem) flush(mem *memdb.DB, level int) error {
	s := c.s

	// Write memdb to table.
	iter := mem.NewIterator(nil)
	defer iter.Release()
	t, n, err := s.tops.createFrom(iter)
	if err != nil {
		return err
	}

	// Pick level.
	if level < 0 {
		v := s.version()
		level = v.pickLevel(t.imin.ukey(), t.imax.ukey())
		v.release()
	}
	c.rec.addTableFile(level, t)

	s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.imin, t.imax)

	c.level = level
	return nil
}
Beispiel #3
0
func (db *DB) recoverJournal() error {
	// Get all tables and sort it by file number.
	journalFiles_, err := db.s.getFiles(storage.TypeJournal)
	if err != nil {
		return err
	}
	journalFiles := files(journalFiles_)
	journalFiles.sort()

	// Discard older journal.
	prev := -1
	for i, file := range journalFiles {
		if file.Num() >= db.s.stJournalNum {
			if prev >= 0 {
				i--
				journalFiles[i] = journalFiles[prev]
			}
			journalFiles = journalFiles[i:]
			break
		} else if file.Num() == db.s.stPrevJournalNum {
			prev = i
		}
	}

	var jr *journal.Reader
	var of storage.File
	var mem *memdb.DB
	batch := new(Batch)
	cm := newCMem(db.s)
	buf := new(util.Buffer)
	// Options.
	strict := db.s.o.GetStrict(opt.StrictJournal)
	checksum := db.s.o.GetStrict(opt.StrictJournalChecksum)
	writeBuffer := db.s.o.GetWriteBuffer()
	recoverJournal := func(file storage.File) error {
		db.logf("journal@recovery recovering @%d", file.Num())
		reader, err := file.Open()
		if err != nil {
			return err
		}
		defer reader.Close()

		// Create/reset journal reader instance.
		if jr == nil {
			jr = journal.NewReader(reader, dropper{db.s, file}, strict, checksum)
		} else {
			jr.Reset(reader, dropper{db.s, file}, strict, checksum)
		}

		// Flush memdb and remove obsolete journal file.
		if of != nil {
			if mem.Len() > 0 {
				if err := cm.flush(mem, 0); err != nil {
					return err
				}
			}
			if err := cm.commit(file.Num(), db.seq); err != nil {
				return err
			}
			cm.reset()
			of.Remove()
			of = nil
		}

		// Replay journal to memdb.
		mem.Reset()
		for {
			r, err := jr.Next()
			if err != nil {
				if err == io.EOF {
					break
				}
				return errors.SetFile(err, file)
			}

			buf.Reset()
			if _, err := buf.ReadFrom(r); err != nil {
				if err == io.ErrUnexpectedEOF {
					// This is error returned due to corruption, with strict == false.
					continue
				} else {
					return errors.SetFile(err, file)
				}
			}
			if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mem); err != nil {
				if strict || !errors.IsCorrupted(err) {
					return errors.SetFile(err, file)
				} else {
					db.s.logf("journal error: %v (skipped)", err)
					// We won't apply sequence number as it might be corrupted.
					continue
				}
			}

			// Save sequence number.
			db.seq = batch.seq + uint64(batch.Len())

			// Flush it if large enough.
			if mem.Size() >= writeBuffer {
				if err := cm.flush(mem, 0); err != nil {
					return err
				}
				mem.Reset()
			}
		}

		of = file
		return nil
	}

	// Recover all journals.
	if len(journalFiles) > 0 {
		db.logf("journal@recovery F·%d", len(journalFiles))

		// Mark file number as used.
		db.s.markFileNum(journalFiles[len(journalFiles)-1].Num())

		mem = memdb.New(db.s.icmp, writeBuffer)
		for _, file := range journalFiles {
			if err := recoverJournal(file); err != nil {
				return err
			}
		}

		// Flush the last journal.
		if mem.Len() > 0 {
			if err := cm.flush(mem, 0); err != nil {
				return err
			}
		}
	}

	// Create a new journal.
	if _, err := db.newMem(0); err != nil {
		return err
	}

	// Commit.
	if err := cm.commit(db.journalFile.Num(), db.seq); err != nil {
		// Close journal.
		if db.journal != nil {
			db.journal.Close()
			db.journalWriter.Close()
		}
		return err
	}

	// Remove the last obsolete journal file.
	if of != nil {
		of.Remove()
	}

	return nil
}
Beispiel #4
0
func isMemOverlaps(icmp *iComparer, mem *memdb.DB, min, max []byte) bool {
	iter := mem.NewIterator(nil)
	defer iter.Release()
	return (max == nil || (iter.First() && icmp.uCompare(max, iKey(iter.Key()).ukey()) >= 0)) &&
		(min == nil || (iter.Last() && icmp.uCompare(min, iKey(iter.Key()).ukey()) <= 0))
}
Beispiel #5
0
func (b *Batch) revertMemReplay(to *memdb.DB) error {
	return b.decodeRec(func(i int, kt kType, key, value []byte) {
		ikey := newIkey(key, b.seq+uint64(i), kt)
		to.Delete(ikey)
	})
}