// DeleteState delets ALL state keys/values from the DB. This is generally // only used during state synchronization when creating a new state from // a snapshot. func (openchainDB *OpenchainDB) DeleteState() error { err := openchainDB.DB.DropColumnFamily(openchainDB.StateCF) if err != nil { dbLogger.Error("Error dropping state CF", err) return err } err = openchainDB.DB.DropColumnFamily(openchainDB.StateDeltaCF) if err != nil { dbLogger.Error("Error dropping state delta CF", err) return err } opts := gorocksdb.NewDefaultOptions() defer opts.Destroy() openchainDB.StateCF, err = openchainDB.DB.CreateColumnFamily(opts, stateCF) if err != nil { dbLogger.Error("Error creating state CF", err) return err } openchainDB.StateDeltaCF, err = openchainDB.DB.CreateColumnFamily(opts, stateDeltaCF) if err != nil { dbLogger.Error("Error creating state delta CF", err) return err } return nil }
func NewRocksdbStorage(dbfname string, dbtype string, mergeOp gorocksdb.MergeOperator) (*RocksdbStorage, error) { var sto *RocksdbStorage if dbtype != "kv" && dbtype != "json" { return sto, fmt.Errorf("Unkown db type") } opts := gorocksdb.NewDefaultOptions() if mergeOp != nil { opts.SetMergeOperator(mergeOp) } // opts.IncreaseParallelism(runtime.NumCPU()) // opts.OptimizeLevelStyleCompaction(0) opts.SetCreateIfMissing(true) db, err := gorocksdb.OpenDb(opts, dbfname) if err != nil { return sto, err } ro := gorocksdb.NewDefaultReadOptions() wo := gorocksdb.NewDefaultWriteOptions() sto = &RocksdbStorage{ dbfname: dbfname, db: db, ro: ro, wo: wo, } return sto, nil }
func write_multi_cfs() error { dbOpts := gorocksdb.NewDefaultOptions() dbOpts.SetCreateIfMissing(true) if err := os.RemoveAll("/tmp/multicf_db"); err != nil { return err } db, err := gorocksdb.OpenDb(dbOpts, "/tmp/multicf_db") if err != nil { return err } var handles []*gorocksdb.ColumnFamilyHandle for i := 0; i < 4; i++ { handle, err := db.CreateColumnFamily(dbOpts, fmt.Sprint(i)) if err != nil { return err } handles = append(handles, handle) } writeOpts := gorocksdb.NewDefaultWriteOptions() if err := db.Put(writeOpts, []byte("default"), []byte("default")); err != nil { return err } for i := 0; i < 16; i++ { key := []byte(fmt.Sprint(i)) if err := db.PutCF(writeOpts, handles[i%4], key, key); err != nil { return err } } db.Close() return nil }
func read_multi_cfs() error { dbOpts := gorocksdb.NewDefaultOptions() defaultRO := gorocksdb.NewDefaultReadOptions() db, handles, err := gorocksdb.OpenDbColumnFamilies( dbOpts, "/tmp/multicf_db", []string{"default", "0", "1", "2", "3"}, []*gorocksdb.Options{dbOpts, dbOpts, dbOpts, dbOpts, dbOpts}, ) iters, err := gorocksext.NewIterators(defaultRO, db, handles) if err != nil { return err } for i, iter := range iters { fmt.Printf("COUTING FOR ITER: %d\n", i) iter.SeekToFirst() for iter.Valid() { fmt.Println(string(iter.Key().Data())) defer iter.Key().Free() iter.Next() } } db.Close() return nil }
// CreateDB creates a rocks db database func CreateDB() error { dbPath := getDBPath() dbLogger.Debug("Creating DB at [%s]", dbPath) missing, err := dirMissingOrEmpty(dbPath) if err != nil { return err } if !missing { return fmt.Errorf("db dir [%s] already exists", dbPath) } err = os.MkdirAll(path.Dir(dbPath), 0755) if err != nil { dbLogger.Error("Error calling os.MkdirAll for directory path [%s]: %s", dbPath, err) return fmt.Errorf("Error making directory path [%s]: %s", dbPath, err) } opts := gorocksdb.NewDefaultOptions() defer opts.Destroy() opts.SetCreateIfMissing(true) db, err := gorocksdb.OpenDb(opts, dbPath) if err != nil { return err } defer db.Close() dbLogger.Debug("DB created at [%s]", dbPath) return nil }
func NewRocksDB(path string) (*RocksDB, error) { opts := gorocksdb.NewDefaultOptions() //opts.SetBlockCache(gorocksdb.NewLRUCache(3 << 30)) opts.SetCreateIfMissing(true) db, err := gorocksdb.OpenDb(opts, path) return &RocksDB{db: db}, err }
func openDB() (*OpenchainDB, error) { if isOpen { return openchainDB, nil } dbPath := getDBPath() opts := gorocksdb.NewDefaultOptions() defer opts.Destroy() opts.SetCreateIfMissing(false) opts.SetCreateIfMissingColumnFamilies(true) cfNames := []string{"default"} cfNames = append(cfNames, columnfamilies...) var cfOpts []*gorocksdb.Options for _ = range cfNames { cfOpts = append(cfOpts, opts) } db, cfHandlers, err := gorocksdb.OpenDbColumnFamilies(opts, dbPath, cfNames, cfOpts) if err != nil { fmt.Println("Error opening DB", err) return nil, err } isOpen = true // XXX should we close cfHandlers[0]? return &OpenchainDB{db, cfHandlers[1], cfHandlers[2], cfHandlers[3], cfHandlers[4], cfHandlers[5]}, nil }
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { path, ok := config["path"].(string) if !ok { return nil, fmt.Errorf("must specify path") } rv := Store{ path: path, opts: gorocksdb.NewDefaultOptions(), } if mo != nil { rv.opts.SetMergeOperator(mo) } _, err := applyConfig(rv.opts, config) if err != nil { return nil, err } rv.db, err = gorocksdb.OpenDb(rv.opts, rv.path) if err != nil { return nil, err } return &rv, nil }
func main() { dbName := "/data/mydb" cache := gorocksdb.NewLRUCache(512 * 1024 * 1024) filter := gorocksdb.NewBloomFilter(15) to := gorocksdb.NewDefaultBlockBasedTableOptions() to.SetBlockSize(256 * 1024) to.SetBlockCache(cache) to.SetFilterPolicy(filter) options := gorocksdb.NewDefaultOptions() options.SetBlockBasedTableFactory(to) options.SetCreateIfMissing(true) options.SetStatsDumpPeriodSec(60 * 1) // dump stats at 10 minute interval options.SetCompactionStyle(gorocksdb.UniversalCompactionStyle) options.SetWriteBufferSize(512 * 1024 * 1024) options.SetMaxWriteBufferNumber(5) options.SetMinWriteBufferNumberToMerge(2) db, err := gorocksdb.OpenDb(options, dbName) if err != nil { fmt.Println(err) os.Exit(1) } putStatch := doPut(db, 16*1024, 32*1024) for { p := <-putStatch fmt.Println("avgTime ", p.avgTime, "maxTime", p.maxTime) } }
func newRocksDB(dir string) *rocks.DB { opts := gorocksdb.NewDefaultOptions() opts.SetCreateIfMissing(true) rdb, err := gorocksdb.OpenDb(opts, dir) if err != nil { panic(err) } return rocks.New(rdb) }
func newRocksDB(t *testing.T) *gorocksdb.DB { dir, err := ioutil.TempDir("", "rocks") ensure.Nil(t, err) opts := gorocksdb.NewDefaultOptions() opts.SetCreateIfMissing(true) db, err := gorocksdb.OpenDb(opts, dir) ensure.Nil(t, err) return db }
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { path, ok := config["path"].(string) if !ok { return nil, fmt.Errorf("must specify path") } rv := Store{ path: path, config: config, opts: gorocksdb.NewDefaultOptions(), } if mo != nil { rv.opts.SetMergeOperator(mo) } _, err := applyConfig(rv.opts, config) if err != nil { return nil, err } rv.db, err = gorocksdb.OpenDb(rv.opts, rv.path) if err != nil { return nil, err } b, ok := config["readoptions_verify_checksum"].(bool) if ok { rv.roptVerifyChecksums, rv.roptVerifyChecksumsUse = b, true } b, ok = config["readoptions_fill_cache"].(bool) if ok { rv.roptFillCache, rv.roptFillCacheUse = b, true } v, ok := config["readoptions_read_tier"].(float64) if ok { rv.roptReadTier, rv.roptReadTierUse = int(v), true } b, ok = config["writeoptions_sync"].(bool) if ok { rv.woptSync, rv.woptSyncUse = b, true } b, ok = config["writeoptions_disable_WAL"].(bool) if ok { rv.woptDisableWAL, rv.woptDisableWALUse = b, true } return &rv, nil }
func New(path string, config map[string]interface{}) (*Store, error) { rv := Store{ path: path, opts: gorocksdb.NewDefaultOptions(), } _, err := applyConfig(rv.opts, config) if err != nil { return nil, err } return &rv, nil }
// Open open underlying rocksdb func (openchainDB *OpenchainDB) Open() { openchainDB.mux.Lock() if openchainDB.dbState == opened { openchainDB.mux.Unlock() return } defer openchainDB.mux.Unlock() dbPath := getDBPath() missing, err := dirMissingOrEmpty(dbPath) if err != nil { panic(fmt.Sprintf("Error while trying to open DB: %s", err)) } dbLogger.Debugf("Is db path [%s] empty [%t]", dbPath, missing) if missing { err = os.MkdirAll(path.Dir(dbPath), 0755) if err != nil { panic(fmt.Sprintf("Error making directory path [%s]: %s", dbPath, err)) } } opts := gorocksdb.NewDefaultOptions() defer opts.Destroy() opts.SetCreateIfMissing(missing) opts.SetCreateIfMissingColumnFamilies(true) cfNames := []string{"default"} cfNames = append(cfNames, columnfamilies...) var cfOpts []*gorocksdb.Options for range cfNames { cfOpts = append(cfOpts, opts) } db, cfHandlers, err := gorocksdb.OpenDbColumnFamilies(opts, dbPath, cfNames, cfOpts) if err != nil { panic(fmt.Sprintf("Error opening DB: %s", err)) } openchainDB.DB = db openchainDB.BlockchainCF = cfHandlers[1] openchainDB.StateCF = cfHandlers[2] openchainDB.StateDeltaCF = cfHandlers[3] openchainDB.IndexesCF = cfHandlers[4] openchainDB.PersistCF = cfHandlers[5] openchainDB.dbState = opened }
func create_purge_backups() error { dbOpts := gorocksdb.NewDefaultOptions() dbOpts.SetCreateIfMissing(true) if err := os.RemoveAll("/tmp/backups_db"); err != nil { return err } db, err := gorocksdb.OpenDb(dbOpts, "/tmp/backups_db") if err != nil { return err } if err := os.RemoveAll("/tmp/db_backups"); err != nil { return err } be, err := gorocksdb.OpenBackupEngine(dbOpts, "/tmp/db_backups") if err != nil { return err } writeOpts := gorocksdb.NewDefaultWriteOptions() for i := 0; i < 5; i++ { key := []byte(fmt.Sprint(i)) if err := db.Put(writeOpts, key, key); err != nil { return err } if err := be.CreateNewBackup(db); err != nil { return err } } fmt.Println("Num available initially: ", be.GetInfo().GetCount()) if err := gorocksext.PurgeOldBackups(be, 4); err != nil { return err } fmt.Println(be.GetInfo().GetCount()) if err := gorocksext.PurgeOldBackups(be, 2); err != nil { return err } fmt.Println(be.GetInfo().GetCount()) if err := gorocksext.PurgeOldBackups(be, 0); err != nil { return err } fmt.Println(be.GetInfo().GetCount()) db.Close() return nil }
func NewRocksDB(path string) (*RocksDB, error) { opts := gorocksdb.NewDefaultOptions() filter := gorocksdb.NewBloomFilter(14) opts.SetFilterPolicy(filter) opts.SetMaxOpenFiles(10000) db, err := gorocksdb.OpenDbForReadOnly(opts, path, false) if err != nil { return nil, err } return &RocksDB{ db: db, ro: gorocksdb.NewDefaultReadOptions(), hits: metrics.NewMeter(), misses: metrics.NewMeter(), cache: lru.New(1000000), }, nil }
func read_all(db_dir string) error { dbOpts := gorocksdb.NewDefaultOptions() db, err := gorocksdb.OpenDb(dbOpts, db_dir) if err != nil { return err } defaultRO := gorocksdb.NewDefaultReadOptions() iter := db.NewIterator(defaultRO) iter.SeekToFirst() for iter.Valid() { key := iter.Key() defer key.Free() fmt.Println(string(key.Data())) iter.Next() } db.Close() return nil }
func openDB() (*OpenchainDB, error) { if isOpen { return openchainDB, nil } dbPath := getDBPath() opts := gorocksdb.NewDefaultOptions() opts.SetCreateIfMissing(false) db, cfHandlers, err := gorocksdb.OpenDbColumnFamilies(opts, dbPath, []string{"default", blockchainCF, stateCF, stateDeltaCF, indexesCF}, []*gorocksdb.Options{opts, opts, opts, opts, opts}) if err != nil { fmt.Println("Error opening DB", err) return nil, err } isOpen = true return &OpenchainDB{db, cfHandlers[1], cfHandlers[2], cfHandlers[3], cfHandlers[4]}, nil }
func main() { dbName := "/data/mydb" compactionThreads := 4 options := gorocksdb.NewDefaultOptions() options.SetCreateIfMissing(true) options.SetStatsDumpPeriodSec(60 * 10) // dump stats at 10 minute interval options.SetCompactionStyle(gorocksdb.UniversalCompactionStyle) options.IncreaseParallelism(compactionThreads) options.IncreaseParallelism(compactionThreads) db, err := gorocksdb.OpenDb(options, dbName) if err != nil { fmt.Println(err) os.Exit(1) } putStatch := doPut(db, 64*1024, 128*1024) for { p := <-putStatch fmt.Println("avgTime ", p.avgTime, "maxTime", p.maxTime) } }
func check_checkpoints() error { dbOpts := gorocksdb.NewDefaultOptions() dbOpts.SetCreateIfMissing(true) if err := os.RemoveAll("/tmp/checkpoint_db"); err != nil { return err } db, err := gorocksdb.OpenDb(dbOpts, "/tmp/checkpoint_db") if err != nil { return err } writeOpts := gorocksdb.NewDefaultWriteOptions() for i := 0; i < 16; i++ { key := []byte(fmt.Sprint(i)) if err := db.Put(writeOpts, key, key); err != nil { return err } if i == 8 { if err := os.RemoveAll("/tmp/checkpoint_db1"); err != nil { return err } gorocksext.CreateCheckpoint(db, "/tmp/checkpoint_db1") } } db.Close() fmt.Println("Full Db") if err := read_all("/tmp/checkpoint_db"); err != nil { return err } fmt.Println("Checkpointed snapshot") if err := read_all("/tmp/checkpoint_db1"); err != nil { return err } return nil }
} func (hp *HostPort) AsString() string { return hp.Host + ":" + strconv.FormatInt(int64(hp.Port), 10) } func (hp *HostPort) PortString() string { return strconv.FormatInt(int64(hp.Port), 10) } // ------------ LevelDB Storage ------------------ // From reading levigo's and levelDB's docs, it looks like the DB is // safe for concurrent use by multiple goroutines without extra // synchronization. var opts = gorocksdb.NewDefaultOptions() var storeDB *gorocksdb.DB var readOpt = gorocksdb.NewDefaultReadOptions() var writeOpt = gorocksdb.NewDefaultWriteOptions() func storageInit(path string) { //opts.IncreaseParallelism(2) opts.OptimizeForPointLookup(128) // 128mb cache size //opts.SetWriteBufferSize(4*1024*1024) // default 4mb opts.SetCreateIfMissing(true) opts.SetCompression(gorocksdb.NoCompression) opts.SetDisableDataSync(true) opts.SetUseFsync(false) // true:fsync, false:fdatasync db, err := gorocksdb.OpenDb(opts, path) if err != nil { panic("error opening DB at " + path + ": " + err.Error())
// Destroy the rocksdb instance and data files func (s *Store) Destroy() { s.Close() rocks.DestroyDb(s.directory, rocks.NewDefaultOptions()) }
// NewStore returns the Store a rocksdb wrapper func NewStore(options StoreOptions) (*Store, error) { options.SetDefaults() if options.Directory == "" { return nil, fmt.Errorf("Empty directory of store options") } if options.IsDebug { log.EnableDebug() } s := &Store{ directory: options.Directory, useTailing: !options.DisableTailing, cfHandles: make(map[string]*rocks.ColumnFamilyHandle), queues: make(map[string]*Queue), } opts := rocks.NewDefaultOptions() opts.SetCreateIfMissing(true) opts.IncreaseParallelism(options.Parallel) opts.SetMergeOperator(&_CountMerger{}) opts.SetMaxSuccessiveMerges(64) opts.SetWriteBufferSize(options.WriteBufferSize) opts.SetMaxWriteBufferNumber(options.WriteBufferNumber) opts.SetTargetFileSizeBase(options.FileSizeBase) opts.SetLevel0FileNumCompactionTrigger(8) opts.SetLevel0SlowdownWritesTrigger(16) opts.SetLevel0StopWritesTrigger(24) opts.SetNumLevels(4) opts.SetMaxBytesForLevelBase(512 * 1024 * 1024) opts.SetMaxBytesForLevelMultiplier(8) opts.SetCompression(options.Compression) opts.SetDisableAutoCompactions(options.DisableAutoCompaction) bbto := rocks.NewDefaultBlockBasedTableOptions() bbto.SetBlockCache(rocks.NewLRUCache(options.MemorySize)) bbto.SetFilterPolicy(rocks.NewBloomFilter(10)) opts.SetBlockBasedTableFactory(bbto) opts.SetMaxOpenFiles(-1) opts.SetMemtablePrefixBloomBits(8 * 1024 * 1024) var err error if err = os.MkdirAll(options.Directory, 0755); err != nil { log.Errorf("Failed to mkdir %q, %s", options.Directory, err) return nil, err } cfNames, err := rocks.ListColumnFamilies(opts, options.Directory) if err != nil { // FIXME: we need to be sure if this means the db does not exist for now // so that we cannot list the column families log.Errorf("Failed to collect the column family names, %s", err) } else { log.Debugf("Got column family names for the existing db, %+v", cfNames) } if len(cfNames) == 0 { // We create the default column family to get the column family handle cfNames = []string{"default"} } cfOpts := make([]*rocks.Options, len(cfNames)) for i := range cfNames { cfOpts[i] = opts } db, cfHandles, err := rocks.OpenDbColumnFamilies(opts, options.Directory, cfNames, cfOpts) if err != nil { log.Errorf("Failed to open rocks database, %s", err) return nil, err } s.DB = db s.dbOpts = opts s.ro = rocks.NewDefaultReadOptions() s.ro.SetFillCache(false) s.ro.SetTailing(!options.DisableTailing) s.wo = rocks.NewDefaultWriteOptions() s.wo.DisableWAL(options.DisableWAL) s.wo.SetSync(options.Sync) if len(cfNames) > 0 { for i := range cfNames { s.cfHandles[cfNames[i]] = cfHandles[i] } } return s, nil }
func (rh *RocksDBHandler) Init() error { rh.options = rocks.NewDefaultOptions() rh.options.SetBlockCache(rocks.NewLRUCache(rh.cacheSize)) rh.options.SetBlockSize(rh.blockSize) rh.options.SetCreateIfMissing(rh.createIfMissing) if rh.bloomFilter > 0 { rh.options.SetFilterPolicy(rocks.NewBloomFilter(rh.bloomFilter)) } if rh.maxOpenFiles > 0 { rh.options.SetMaxOpenFiles(rh.maxOpenFiles) } switch rh.compression { case "no": rh.options.SetCompression(rocks.NoCompression) case "snappy": rh.options.SetCompression(rocks.SnappyCompression) case "zlib": rh.options.SetCompression(rocks.ZlibCompression) case "bzip2": rh.options.SetCompression(rocks.BZip2Compression) } switch rh.compactionStyle { case "level": rh.options.SetCompactionStyle(rocks.LevelCompactionStyle) case "universal": rh.options.SetCompactionStyle(rocks.UniversalCompactionStyle) } rh.dsMergers = make(map[string]DataStructureMerger) rh.dsMergers[kRedisString] = &StringMerger{} rh.dsMergers[kRedisList] = &ListMerger{} rh.dsMergers[kRedisHash] = &HashMerger{} rh.dsMergers[kRedisSet] = &SetMerger{} if rh.maxMerge > 0 { rh.options.SetMaxSuccessiveMerges(rh.maxMerge) } rh.options.SetMergeOperator(rocks.NewMergeOperator(rh)) db, err := rocks.OpenDb(rh.options, rh.dbDir) if err != nil { rh.Close() return err } rh.db = db infos := []string{ fmt.Sprintf("dbDir=%s", rh.dbDir), fmt.Sprintf("cacheSize=%d", rh.cacheSize), fmt.Sprintf("blockSize=%d", rh.blockSize), fmt.Sprintf("createIfMissing=%v", rh.createIfMissing), fmt.Sprintf("bloomFilter=%d", rh.bloomFilter), fmt.Sprintf("compression=%s", rh.compression), fmt.Sprintf("compactionStyle=%s", rh.compactionStyle), fmt.Sprintf("maxOpenFiles=%d", rh.maxOpenFiles), fmt.Sprintf("maxMerge=%d", rh.maxMerge), } log.Printf("[RocksDBHandler] Inited, %s", strings.Join(infos, ", ")) return nil }
func (db *rocksDB) initialize(path string, conf *config) error { if conf == nil { conf = newDefaultConfig() } // Create path if not exists first if err := os.MkdirAll(path, 0700); err != nil { return errors.Trace(err) } opts := gorocksdb.NewDefaultOptions() opts.SetCreateIfMissing(true) opts.SetErrorIfExists(false) opts.SetCompression(gorocksdb.CompressionType(conf.CompressionType)) opts.SetWriteBufferSize(conf.WriteBufferSize) opts.SetMaxOpenFiles(conf.MaxOpenFiles) opts.SetNumLevels(conf.NumLevels) opts.SetMaxWriteBufferNumber(conf.MaxWriteBufferNumber) opts.SetMinWriteBufferNumberToMerge(conf.MinWriteBufferNumberToMerge) opts.SetLevel0FileNumCompactionTrigger(conf.Level0FileNumCompactionTrigger) opts.SetLevel0SlowdownWritesTrigger(conf.Level0SlowdownWritesTrigger) opts.SetLevel0StopWritesTrigger(conf.Level0StopWritesTrigger) opts.SetTargetFileSizeBase(uint64(conf.TargetFileSizeBase)) opts.SetTargetFileSizeMultiplier(conf.TargetFileSizeMultiplier) opts.SetMaxBytesForLevelBase(uint64(conf.MaxBytesForLevelBase)) opts.SetMaxBytesForLevelMultiplier(conf.MaxBytesForLevelMultiplier) opts.SetDisableAutoCompactions(conf.DisableAutoCompactions) opts.SetDisableDataSync(conf.DisableDataSync) opts.SetUseFsync(conf.UseFsync) opts.SetMaxBackgroundCompactions(conf.MaxBackgroundCompactions) opts.SetMaxBackgroundFlushes(conf.MaxBackgroundFlushes) opts.SetAllowOsBuffer(conf.AllowOSBuffer) topts := gorocksdb.NewDefaultBlockBasedTableOptions() topts.SetBlockSize(conf.BlockSize) cache := gorocksdb.NewLRUCache(conf.CacheSize) topts.SetBlockCache(cache) topts.SetFilterPolicy(gorocksdb.NewBloomFilter(conf.BloomFilterSize)) opts.SetBlockBasedTableFactory(topts) env := gorocksdb.NewDefaultEnv() env.SetBackgroundThreads(conf.BackgroundThreads) env.SetHighPriorityBackgroundThreads(conf.HighPriorityBackgroundThreads) opts.SetEnv(env) db.path = path db.opts = opts db.ropt = gorocksdb.NewDefaultReadOptions() db.wopt = gorocksdb.NewDefaultWriteOptions() db.env = env db.topts = topts db.cache = cache db.snapshotFillCache = conf.SnapshotFillCache var err error if db.rkdb, err = gorocksdb.OpenDb(db.opts, db.path); err != nil { return errors.Trace(err) } return nil }