// NewLevelDBStorage is a constructor of DataStorage. func NewLevelDBStorage(cfg *conf.Config) (*LevelDBStorage, error) { ds := LevelDBStorage{ cfg: cfg, dbName: cfg.DatabasePath, itemCache: make(ItemCache), tmpItemCache: make(ItemCache), closed: false, forceFlushChan: make(chan bool, 1), flushSync: &sync.WaitGroup{}, } // LevelDB write options. opts := new(opt.Options) opts.Compression = opt.NoCompression opts.BlockCacheCapacity = 8 * 1024 * 1024 opts.WriteBuffer = 8 * 1024 * 1024 db, err := leveldb.OpenFile(cfg.DatabasePath, opts) if err != nil { return nil, err } ds.db = db go ds.periodicCacheFlush() return &ds, nil }
func truno(t *testing.T, o *opt.Options, f func(h *dbHarness)) { for i := 0; i < 4; i++ { func() { switch i { case 0: case 1: if o == nil { o = &opt.Options{Filter: _bloom_filter} } else { old := o o = &opt.Options{} *o = *old o.Filter = _bloom_filter } case 2: if o == nil { o = &opt.Options{Compression: opt.NoCompression} } else { old := o o = &opt.Options{} *o = *old o.Compression = opt.NoCompression } } h := newDbHarnessWopt(t, o) defer h.close() switch i { case 3: h.reopenDB() } f(h) }() } }
func (s *session) setOptions(o *opt.Options) { no := dupOptions(o) // Alternative filters. if filters := o.GetAltFilters(); len(filters) > 0 { no.AltFilters = make([]filter.Filter, len(filters)) for i, filter := range filters { no.AltFilters[i] = &iFilter{filter} } } // Block cache. switch o.GetBlockCache() { case nil: no.BlockCache = cache.NewLRUCache(o.GetBlockCacheSize()) case opt.NoCache: no.BlockCache = nil } // Comparer. s.icmp = &iComparer{o.GetComparer()} no.Comparer = s.icmp // Filter. if filter := o.GetFilter(); filter != nil { no.Filter = &iFilter{filter} } s.o = &cachedOptions{Options: no} s.o.cache() }
func newSession(d storage.Storage, o *opt.Options) *session { s := new(session) s.stor = d s.cmp = &iComparer{o.GetComparer()} s.o = newIOptions(s, *o) s.tops = newTableOps(s, s.o.GetMaxOpenFiles()) s.setVersion(&version{s: s}) return s }
func (self *leveldbProvider) createLevelDbOptions(typeName string) *opt.Options { comparer := self.typeName2Comparer[typeName] if comparer == nil { return nil } else { options := new(opt.Options) options.Comparer = comparer return options } }
// OpenFile opens or creates a DB for the given path. // The DB will be created if not exist, unless ErrorIfMissing is true. // Also, if ErrorIfExist is true and the DB exist OpenFile will returns // os.ErrExist error. // // OpenFile uses standard file-system backed storage implementation as // desribed in the leveldb/storage package. // // OpenFile will return an error with type of ErrCorrupted if corruption // detected in the DB. Corrupted DB can be recovered with Recover // function. // // The returned DB instance is goroutine-safe. // The DB must be closed after use, by calling Close method. func OpenFile(path string, o *opt.Options) (db *DB, err error) { stor, err := storage.OpenFile(path, o.GetReadOnly()) if err != nil { return } db, err = Open(stor, o) if err != nil { stor.Close() } else { db.closer = stor } return }
func newSession(d desc.Desc, o *opt.Options) *session { s := new(session) s.desc = d s.o = &iOptions{s, o} s.cmp = &iComparer{o.GetComparer()} filter := o.GetFilter() if filter != nil { s.filter = &iFilter{filter} } s.tops = newTableOps(s, s.o.GetMaxOpenFiles()) s.setVersion(&version{s: s}) return s }
func (s *session) setOptions(o *opt.Options) { s.o = &opt.Options{} if o != nil { *s.o = *o } // Alternative filters. if filters := o.GetAltFilters(); len(filters) > 0 { s.o.AltFilters = make([]filter.Filter, len(filters)) for i, filter := range filters { s.o.AltFilters[i] = &iFilter{filter} } } // Block cache. switch o.GetBlockCache() { case nil: s.o.BlockCache = cache.NewLRUCache(opt.DefaultBlockCacheSize) case opt.NoCache: s.o.BlockCache = nil } // Comparer. s.cmp = &iComparer{o.GetComparer()} s.o.Comparer = s.cmp // Filter. if filter := o.GetFilter(); filter != nil { s.o.Filter = &iFilter{filter} } }
// Open open or create database from given desc. func Open(d desc.Desc, o *opt.Options) (db *DB, err error) { s := newSession(d, o) err = s.recover() if os.IsNotExist(err) && o.HasFlag(opt.OFCreateIfMissing) { err = s.create() } else if err == nil && o.HasFlag(opt.OFErrorIfExist) { err = os.ErrExist } if err != nil { return } return open(s) }
func openSession(stor storage.Storage, o *opt.Options) (s *session, err error) { if stor == nil || o == nil { return nil, os.ErrInvalid } storLock, err := stor.Lock() if err != nil { return } s = new(session) s.stor = stor s.storLock = storLock s.cmp = &iComparer{o.GetComparer()} s.o = newIOptions(s, *o) s.tops = newTableOps(s, s.o.GetMaxOpenFiles()) s.setVersion(&version{s: s}) return }
// Creates new initialized session instance. func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) { if stor == nil { return nil, os.ErrInvalid } storLock, err := stor.Lock() if err != nil { return } s = &session{ stor: stor, storLock: storLock, stCompPtrs: make([]iKey, o.GetNumLevel()), } s.setOptions(o) s.tops = newTableOps(s, s.o.GetCachedOpenFiles()) s.setVersion(newVersion(s)) s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed") return }
func (self *Queue) open() error { self.Lock() defer self.Unlock() if regexp.MustCompile(`[^a-zA-Z0-9_]+`).MatchString(self.Name) { return errors.New("Queue name is not alphanumeric") } if len(self.Name) > 100 { return errors.New("Queue name is too long") } var options opt.Options options.BlockCacher = opt.NoCacher var err error self.db, err = leveldb.OpenFile(self.Path(), &options) if err != nil { return err } self.isOpened = true return self.initialize() }
// NewWriter creates a new initialized table writer for the file. // // Table writer is not safe for concurrent use. func NewWriter(f io.Writer, o *opt.Options) *Writer { w := &Writer{ writer: f, cmp: o.GetComparer(), filter: o.GetFilter(), compression: o.GetCompression(), blockSize: o.GetBlockSize(), comparerScratch: make([]byte, 0), } // data block w.dataBlock.restartInterval = o.GetBlockRestartInterval() // The first 20-bytes are used for encoding block handle. w.dataBlock.scratch = w.scratch[20:] // index block w.indexBlock.restartInterval = 1 w.indexBlock.scratch = w.scratch[20:] // filter block if w.filter != nil { w.filterBlock.generator = w.filter.NewGenerator() w.filterBlock.flush(0) } return w }
func applyConfig(o *opt.Options, config map[string]interface{}) ( *opt.Options, error) { ro, ok := config["read_only"].(bool) if ok { o.ReadOnly = ro } cim, ok := config["create_if_missing"].(bool) if ok { o.ErrorIfMissing = !cim } eie, ok := config["error_if_exists"].(bool) if ok { o.ErrorIfExist = eie } wbs, ok := config["write_buffer_size"].(float64) if ok { o.WriteBuffer = int(wbs) } bs, ok := config["block_size"].(float64) if ok { o.BlockSize = int(bs) } bri, ok := config["block_restart_interval"].(float64) if ok { o.BlockRestartInterval = int(bri) } lcc, ok := config["lru_cache_capacity"].(float64) if ok { o.BlockCacheCapacity = int(lcc) } bfbpk, ok := config["bloom_filter_bits_per_key"].(float64) if ok { bf := filter.NewBloomFilter(int(bfbpk)) o.Filter = bf } return o, nil }
func (s *session) setOptions(o *opt.Options) { no := dupOptions(o) // Alternative filters. if filters := o.GetAltFilters(); len(filters) > 0 { no.AltFilters = make([]filter.Filter, len(filters)) for i, filter := range filters { no.AltFilters[i] = &iFilter{filter} } } // Comparer. s.icmp = &iComparer{o.GetComparer()} no.Comparer = s.icmp // Filter. if filter := o.GetFilter(); filter != nil { no.Filter = &iFilter{filter} } s.o = &cachedOptions{Options: no} s.o.cache() }
// NewReader creates a new initialized table reader for the file. // The cache and bpool is optional and can be nil. // // The returned table reader instance is goroutine-safe. func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, bpool *util.BufferPool, o *opt.Options) *Reader { if bpool == nil { bpool = util.NewBufferPool(o.GetBlockSize() + blockTrailerLen) } r := &Reader{ reader: f, cache: cache, bpool: bpool, cmp: o.GetComparer(), checksum: o.GetStrict(opt.StrictBlockChecksum), strictIter: o.GetStrict(opt.StrictIterator), } if f == nil { r.err = errors.New("leveldb/table: Reader: nil file") return r } if size < footerLen { r.err = errors.New("leveldb/table: Reader: invalid table (file size is too small)") return r } var footer [footerLen]byte if _, err := r.reader.ReadAt(footer[:], size-footerLen); err != nil && err != io.EOF { r.err = fmt.Errorf("leveldb/table: Reader: invalid table (could not read footer): %v", err) } if string(footer[footerLen-len(magic):footerLen]) != magic { r.err = errors.New("leveldb/table: Reader: invalid table (bad magic number)") return r } // Decode the metaindex block handle. metaBH, n := decodeBlockHandle(footer[:]) if n == 0 { r.err = errors.New("leveldb/table: Reader: invalid table (bad metaindex block handle)") return r } // Decode the index block handle. indexBH, n := decodeBlockHandle(footer[n:]) if n == 0 { r.err = errors.New("leveldb/table: Reader: invalid table (bad index block handle)") return r } // Read index block. r.indexBlock, r.err = r.readBlock(indexBH, true) if r.err != nil { return r } // Read metaindex block. metaBlock, err := r.readBlock(metaBH, true) if err != nil { r.err = err return r } // Set data end. r.dataEnd = int64(metaBH.offset) metaIter := metaBlock.newIterator(nil, false, nil) for metaIter.Next() { key := string(metaIter.Key()) if !strings.HasPrefix(key, "filter.") { continue } fn := key[7:] var filter filter.Filter if f0 := o.GetFilter(); f0 != nil && f0.Name() == fn { filter = f0 } else { for _, f0 := range o.GetAltFilters() { if f0.Name() == fn { filter = f0 break } } } if filter != nil { filterBH, n := decodeBlockHandle(metaIter.Value()) if n == 0 { continue } // Update data end. r.dataEnd = int64(filterBH.offset) filterBlock, err := r.readFilterBlock(filterBH, filter) if err != nil { continue } r.filterBlock = filterBlock break } } metaIter.Release() return r }
func newDbHarnessWopt(t *testing.T, o *opt.Options) *dbHarness { o.Flag |= opt.OFCreateIfMissing return newDbHarnessWoptRaw(t, o) }
func recoverTable(s *session, o *opt.Options) error { // Get all tables and sort it by file number. tableFiles_, err := s.getFiles(storage.TypeTable) if err != nil { return err } tableFiles := files(tableFiles_) tableFiles.sort() var mSeq uint64 var good, corrupted int rec := new(sessionRecord) bpool := util.NewBufferPool(o.GetBlockSize() + 5) buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) { tmp = s.newTemp() writer, err := tmp.Create() if err != nil { return } defer func() { writer.Close() if err != nil { tmp.Remove() tmp = nil } }() // Copy entries. tw := table.NewWriter(writer, o) for iter.Next() { key := iter.Key() if validIkey(key) { err = tw.Append(key, iter.Value()) if err != nil { return } } } err = iter.Error() if err != nil { return } err = tw.Close() if err != nil { return } err = writer.Sync() if err != nil { return } size = int64(tw.BytesLen()) return } recoverTable := func(file storage.File) error { s.logf("table@recovery recovering @%d", file.Num()) reader, err := file.Open() if err != nil { return err } defer reader.Close() // Get file size. size, err := reader.Seek(0, 2) if err != nil { return err } var tSeq uint64 var tgood, tcorrupted, blockerr int var imin, imax []byte tr := table.NewReader(reader, size, nil, bpool, o) iter := tr.NewIterator(nil, nil) iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) { s.logf("table@recovery found error @%d %q", file.Num(), err) blockerr++ }) // Scan the table. for iter.Next() { key := iter.Key() _, seq, _, ok := parseIkey(key) if !ok { tcorrupted++ continue } tgood++ if seq > tSeq { tSeq = seq } if imin == nil { imin = append([]byte{}, key...) } imax = append(imax[:0], key...) } if err := iter.Error(); err != nil { iter.Release() return err } iter.Release() if tgood > 0 { if tcorrupted > 0 || blockerr > 0 { // Rebuild the table. s.logf("table@recovery rebuilding @%d", file.Num()) iter := tr.NewIterator(nil, nil) tmp, newSize, err := buildTable(iter) iter.Release() if err != nil { return err } reader.Close() if err := file.Replace(tmp); err != nil { return err } size = newSize } if tSeq > mSeq { mSeq = tSeq } // Add table to level 0. rec.addTable(0, file.Num(), uint64(size), imin, imax) s.logf("table@recovery recovered @%d N·%d C·%d B·%d S·%d Q·%d", file.Num(), tgood, tcorrupted, blockerr, size, tSeq) } else { s.logf("table@recovery unrecoverable @%d C·%d B·%d S·%d", file.Num(), tcorrupted, blockerr, size) } good += tgood corrupted += tcorrupted return nil } // Recover all tables. if len(tableFiles) > 0 { s.logf("table@recovery F·%d", len(tableFiles)) // Mark file number as used. s.markFileNum(tableFiles[len(tableFiles)-1].Num()) for _, file := range tableFiles { if err := recoverTable(file); err != nil { return err } } s.logf("table@recovery recovered F·%d N·%d C·%d Q·%d", len(tableFiles), good, corrupted, mSeq) } // Set sequence number. rec.setSeq(mSeq + 1) // Create new manifest. if err := s.create(); err != nil { return err } // Commit. return s.commit(rec) }
func recoverTable(s *session, o *opt.Options) error { o = dupOptions(o) // Mask StrictReader, lets StrictRecovery doing its job. o.Strict &= ^opt.StrictReader // Get all tables and sort it by file number. tableFiles_, err := s.getFiles(storage.TypeTable) if err != nil { return err } tableFiles := files(tableFiles_) tableFiles.sort() var ( maxSeq uint64 recoveredKey, goodKey, corruptedKey, corruptedBlock, droppedTable int // We will drop corrupted table. strict = o.GetStrict(opt.StrictRecovery) rec = &sessionRecord{numLevel: o.GetNumLevel()} bpool = util.NewBufferPool(o.GetBlockSize() + 5) ) buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) { tmp = s.newTemp() writer, err := tmp.Create() if err != nil { return } defer func() { writer.Close() if err != nil { tmp.Remove() tmp = nil } }() // Copy entries. tw := table.NewWriter(writer, o) for iter.Next() { key := iter.Key() if validIkey(key) { err = tw.Append(key, iter.Value()) if err != nil { return } } } err = iter.Error() if err != nil { return } err = tw.Close() if err != nil { return } err = writer.Sync() if err != nil { return } size = int64(tw.BytesLen()) return } recoverTable := func(file storage.File) error { s.logf("table@recovery recovering @%d", file.Num()) reader, err := file.Open() if err != nil { return err } var closed bool defer func() { if !closed { reader.Close() } }() // Get file size. size, err := reader.Seek(0, 2) if err != nil { return err } var ( tSeq uint64 tgoodKey, tcorruptedKey, tcorruptedBlock int imin, imax []byte ) tr, err := table.NewReader(reader, size, storage.NewFileInfo(file), nil, bpool, o) if err != nil { return err } iter := tr.NewIterator(nil, nil) iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) { if errors.IsCorrupted(err) { s.logf("table@recovery block corruption @%d %q", file.Num(), err) tcorruptedBlock++ } }) // Scan the table. for iter.Next() { key := iter.Key() _, seq, _, kerr := parseIkey(key) if kerr != nil { tcorruptedKey++ continue } tgoodKey++ if seq > tSeq { tSeq = seq } if imin == nil { imin = append([]byte{}, key...) } imax = append(imax[:0], key...) } if err := iter.Error(); err != nil { iter.Release() return err } iter.Release() goodKey += tgoodKey corruptedKey += tcorruptedKey corruptedBlock += tcorruptedBlock if strict && (tcorruptedKey > 0 || tcorruptedBlock > 0) { droppedTable++ s.logf("table@recovery dropped @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq) return nil } if tgoodKey > 0 { if tcorruptedKey > 0 || tcorruptedBlock > 0 { // Rebuild the table. s.logf("table@recovery rebuilding @%d", file.Num()) iter := tr.NewIterator(nil, nil) tmp, newSize, err := buildTable(iter) iter.Release() if err != nil { return err } closed = true reader.Close() if err := file.Replace(tmp); err != nil { return err } size = newSize } if tSeq > maxSeq { maxSeq = tSeq } recoveredKey += tgoodKey // Add table to level 0. rec.addTable(0, file.Num(), uint64(size), imin, imax) s.logf("table@recovery recovered @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq) } else { droppedTable++ s.logf("table@recovery unrecoverable @%d Ck·%d Cb·%d S·%d", file.Num(), tcorruptedKey, tcorruptedBlock, size) } return nil } // Recover all tables. if len(tableFiles) > 0 { s.logf("table@recovery F·%d", len(tableFiles)) // Mark file number as used. s.markFileNum(tableFiles[len(tableFiles)-1].Num()) for _, file := range tableFiles { if err := recoverTable(file); err != nil { return err } } s.logf("table@recovery recovered F·%d N·%d Gk·%d Ck·%d Q·%d", len(tableFiles), recoveredKey, goodKey, corruptedKey, maxSeq) } // Set sequence number. rec.setSeqNum(maxSeq) // Create new manifest. if err := s.create(); err != nil { return err } // Commit. return s.commit(rec) }
// NewReader creates a new initialized table reader for the file. // The fi, cache and bpool is optional and can be nil. // // The returned table reader instance is goroutine-safe. func NewReader(f io.ReaderAt, size int64, fd storage.FileDesc, cache *cache.CacheGetter, bpool *util.BufferPool, o *opt.Options) (*Reader, error) { if f == nil { return nil, errors.New("leveldb/table: nil file") } r := &Reader{ fd: fd, reader: f, cache: cache, bpool: bpool, o: o, cmp: o.GetComparer(), verifyChecksum: o.GetStrict(opt.StrictBlockChecksum), } if size < footerLen { r.err = r.newErrCorrupted(0, size, "table", "too small") return r, nil } footerPos := size - footerLen var footer [footerLen]byte if _, err := r.reader.ReadAt(footer[:], footerPos); err != nil && err != io.EOF { return nil, err } if string(footer[footerLen-len(magic):footerLen]) != magic { r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad magic number") return r, nil } var n int // Decode the metaindex block handle. r.metaBH, n = decodeBlockHandle(footer[:]) if n == 0 { r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad metaindex block handle") return r, nil } // Decode the index block handle. r.indexBH, n = decodeBlockHandle(footer[n:]) if n == 0 { r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad index block handle") return r, nil } // Read metaindex block. metaBlock, err := r.readBlock(r.metaBH, true) if err != nil { if errors.IsCorrupted(err) { r.err = err return r, nil } else { return nil, err } } // Set data end. r.dataEnd = int64(r.metaBH.offset) // Read metaindex. metaIter := r.newBlockIter(metaBlock, nil, nil, true) for metaIter.Next() { key := string(metaIter.Key()) if !strings.HasPrefix(key, "filter.") { continue } fn := key[7:] if f0 := o.GetFilter(); f0 != nil && f0.Name() == fn { r.filter = f0 } else { for _, f0 := range o.GetAltFilters() { if f0.Name() == fn { r.filter = f0 break } } } if r.filter != nil { filterBH, n := decodeBlockHandle(metaIter.Value()) if n == 0 { continue } r.filterBH = filterBH // Update data end. r.dataEnd = int64(filterBH.offset) break } } metaIter.Release() metaBlock.Release() // Cache index and filter block locally, since we don't have global cache. if cache == nil { r.indexBlock, err = r.readBlock(r.indexBH, true) if err != nil { if errors.IsCorrupted(err) { r.err = err return r, nil } else { return nil, err } } if r.filter != nil { r.filterBlock, err = r.readFilterBlock(r.filterBH) if err != nil { if !errors.IsCorrupted(err) { return nil, err } // Don't use filter then. r.filter = nil } } } return r, nil }