Exemple #1
0
// DeleteState delets ALL state keys/values from the DB. This is generally
// only used during state synchronization when creating a new state from
// a snapshot.
func (openchainDB *OpenchainDB) DeleteState() error {
	err := openchainDB.DB.DropColumnFamily(openchainDB.StateCF)
	if err != nil {
		dbLogger.Error("Error dropping state CF", err)
		return err
	}
	err = openchainDB.DB.DropColumnFamily(openchainDB.StateDeltaCF)
	if err != nil {
		dbLogger.Error("Error dropping state delta CF", err)
		return err
	}
	opts := gorocksdb.NewDefaultOptions()
	defer opts.Destroy()
	openchainDB.StateCF, err = openchainDB.DB.CreateColumnFamily(opts, stateCF)
	if err != nil {
		dbLogger.Error("Error creating state CF", err)
		return err
	}
	openchainDB.StateDeltaCF, err = openchainDB.DB.CreateColumnFamily(opts, stateDeltaCF)
	if err != nil {
		dbLogger.Error("Error creating state delta CF", err)
		return err
	}
	return nil
}
Exemple #2
0
func NewRocksdbStorage(dbfname string, dbtype string, mergeOp gorocksdb.MergeOperator) (*RocksdbStorage, error) {
	var sto *RocksdbStorage

	if dbtype != "kv" && dbtype != "json" {
		return sto, fmt.Errorf("Unkown db type")
	}

	opts := gorocksdb.NewDefaultOptions()

	if mergeOp != nil {
		opts.SetMergeOperator(mergeOp)
	}
	// opts.IncreaseParallelism(runtime.NumCPU())
	// opts.OptimizeLevelStyleCompaction(0)
	opts.SetCreateIfMissing(true)

	db, err := gorocksdb.OpenDb(opts, dbfname)
	if err != nil {
		return sto, err
	}
	ro := gorocksdb.NewDefaultReadOptions()
	wo := gorocksdb.NewDefaultWriteOptions()

	sto = &RocksdbStorage{
		dbfname: dbfname,
		db:      db,
		ro:      ro,
		wo:      wo,
	}
	return sto, nil
}
func write_multi_cfs() error {
	dbOpts := gorocksdb.NewDefaultOptions()
	dbOpts.SetCreateIfMissing(true)
	if err := os.RemoveAll("/tmp/multicf_db"); err != nil {
		return err
	}

	db, err := gorocksdb.OpenDb(dbOpts, "/tmp/multicf_db")
	if err != nil {
		return err
	}

	var handles []*gorocksdb.ColumnFamilyHandle
	for i := 0; i < 4; i++ {
		handle, err := db.CreateColumnFamily(dbOpts, fmt.Sprint(i))
		if err != nil {
			return err
		}
		handles = append(handles, handle)
	}

	writeOpts := gorocksdb.NewDefaultWriteOptions()
	if err := db.Put(writeOpts, []byte("default"), []byte("default")); err != nil {
		return err
	}
	for i := 0; i < 16; i++ {
		key := []byte(fmt.Sprint(i))
		if err := db.PutCF(writeOpts, handles[i%4], key, key); err != nil {
			return err
		}
	}
	db.Close()
	return nil
}
func read_multi_cfs() error {
	dbOpts := gorocksdb.NewDefaultOptions()
	defaultRO := gorocksdb.NewDefaultReadOptions()
	db, handles, err := gorocksdb.OpenDbColumnFamilies(
		dbOpts,
		"/tmp/multicf_db",
		[]string{"default", "0", "1", "2", "3"},
		[]*gorocksdb.Options{dbOpts, dbOpts, dbOpts, dbOpts, dbOpts},
	)
	iters, err := gorocksext.NewIterators(defaultRO, db, handles)
	if err != nil {
		return err
	}
	for i, iter := range iters {
		fmt.Printf("COUTING FOR ITER: %d\n", i)
		iter.SeekToFirst()
		for iter.Valid() {
			fmt.Println(string(iter.Key().Data()))
			defer iter.Key().Free()
			iter.Next()
		}
	}
	db.Close()
	return nil
}
Exemple #5
0
// CreateDB creates a rocks db database
func CreateDB() error {
	dbPath := getDBPath()
	dbLogger.Debug("Creating DB at [%s]", dbPath)
	missing, err := dirMissingOrEmpty(dbPath)
	if err != nil {
		return err
	}

	if !missing {
		return fmt.Errorf("db dir [%s] already exists", dbPath)
	}
	err = os.MkdirAll(path.Dir(dbPath), 0755)
	if err != nil {
		dbLogger.Error("Error calling  os.MkdirAll for directory path [%s]: %s", dbPath, err)
		return fmt.Errorf("Error making directory path [%s]: %s", dbPath, err)
	}
	opts := gorocksdb.NewDefaultOptions()
	defer opts.Destroy()
	opts.SetCreateIfMissing(true)

	db, err := gorocksdb.OpenDb(opts, dbPath)
	if err != nil {
		return err
	}

	defer db.Close()

	dbLogger.Debug("DB created at [%s]", dbPath)
	return nil
}
Exemple #6
0
func NewRocksDB(path string) (*RocksDB, error) {
	opts := gorocksdb.NewDefaultOptions()
	//opts.SetBlockCache(gorocksdb.NewLRUCache(3 << 30))
	opts.SetCreateIfMissing(true)
	db, err := gorocksdb.OpenDb(opts, path)
	return &RocksDB{db: db}, err
}
Exemple #7
0
func openDB() (*OpenchainDB, error) {
	if isOpen {
		return openchainDB, nil
	}

	dbPath := getDBPath()
	opts := gorocksdb.NewDefaultOptions()
	defer opts.Destroy()

	opts.SetCreateIfMissing(false)
	opts.SetCreateIfMissingColumnFamilies(true)

	cfNames := []string{"default"}
	cfNames = append(cfNames, columnfamilies...)
	var cfOpts []*gorocksdb.Options
	for _ = range cfNames {
		cfOpts = append(cfOpts, opts)
	}

	db, cfHandlers, err := gorocksdb.OpenDbColumnFamilies(opts, dbPath, cfNames, cfOpts)

	if err != nil {
		fmt.Println("Error opening DB", err)
		return nil, err
	}
	isOpen = true
	// XXX should we close cfHandlers[0]?
	return &OpenchainDB{db, cfHandlers[1], cfHandlers[2], cfHandlers[3], cfHandlers[4], cfHandlers[5]}, nil
}
Exemple #8
0
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {

	path, ok := config["path"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify path")
	}

	rv := Store{
		path: path,
		opts: gorocksdb.NewDefaultOptions(),
	}

	if mo != nil {
		rv.opts.SetMergeOperator(mo)
	}

	_, err := applyConfig(rv.opts, config)
	if err != nil {
		return nil, err
	}

	rv.db, err = gorocksdb.OpenDb(rv.opts, rv.path)
	if err != nil {
		return nil, err
	}

	return &rv, nil
}
func main() {
	dbName := "/data/mydb"
	cache := gorocksdb.NewLRUCache(512 * 1024 * 1024)
	filter := gorocksdb.NewBloomFilter(15)
	to := gorocksdb.NewDefaultBlockBasedTableOptions()
	to.SetBlockSize(256 * 1024)
	to.SetBlockCache(cache)
	to.SetFilterPolicy(filter)
	options := gorocksdb.NewDefaultOptions()
	options.SetBlockBasedTableFactory(to)
	options.SetCreateIfMissing(true)
	options.SetStatsDumpPeriodSec(60 * 1) // dump stats at 10 minute interval
	options.SetCompactionStyle(gorocksdb.UniversalCompactionStyle)
	options.SetWriteBufferSize(512 * 1024 * 1024)
	options.SetMaxWriteBufferNumber(5)
	options.SetMinWriteBufferNumberToMerge(2)
	db, err := gorocksdb.OpenDb(options, dbName)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	putStatch := doPut(db, 16*1024, 32*1024)
	for {
		p := <-putStatch
		fmt.Println("avgTime ", p.avgTime, "maxTime", p.maxTime)
	}
}
Exemple #10
0
func newRocksDB(dir string) *rocks.DB {
	opts := gorocksdb.NewDefaultOptions()
	opts.SetCreateIfMissing(true)
	rdb, err := gorocksdb.OpenDb(opts, dir)
	if err != nil {
		panic(err)
	}
	return rocks.New(rdb)
}
Exemple #11
0
func newRocksDB(t *testing.T) *gorocksdb.DB {
	dir, err := ioutil.TempDir("", "rocks")
	ensure.Nil(t, err)

	opts := gorocksdb.NewDefaultOptions()
	opts.SetCreateIfMissing(true)
	db, err := gorocksdb.OpenDb(opts, dir)
	ensure.Nil(t, err)

	return db
}
Exemple #12
0
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {

	path, ok := config["path"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify path")
	}

	rv := Store{
		path:   path,
		config: config,
		opts:   gorocksdb.NewDefaultOptions(),
	}

	if mo != nil {
		rv.opts.SetMergeOperator(mo)
	}

	_, err := applyConfig(rv.opts, config)
	if err != nil {
		return nil, err
	}

	rv.db, err = gorocksdb.OpenDb(rv.opts, rv.path)
	if err != nil {
		return nil, err
	}

	b, ok := config["readoptions_verify_checksum"].(bool)
	if ok {
		rv.roptVerifyChecksums, rv.roptVerifyChecksumsUse = b, true
	}

	b, ok = config["readoptions_fill_cache"].(bool)
	if ok {
		rv.roptFillCache, rv.roptFillCacheUse = b, true
	}

	v, ok := config["readoptions_read_tier"].(float64)
	if ok {
		rv.roptReadTier, rv.roptReadTierUse = int(v), true
	}

	b, ok = config["writeoptions_sync"].(bool)
	if ok {
		rv.woptSync, rv.woptSyncUse = b, true
	}

	b, ok = config["writeoptions_disable_WAL"].(bool)
	if ok {
		rv.woptDisableWAL, rv.woptDisableWALUse = b, true
	}

	return &rv, nil
}
Exemple #13
0
func New(path string, config map[string]interface{}) (*Store, error) {
	rv := Store{
		path: path,
		opts: gorocksdb.NewDefaultOptions(),
	}

	_, err := applyConfig(rv.opts, config)
	if err != nil {
		return nil, err
	}

	return &rv, nil
}
Exemple #14
0
// Open open underlying rocksdb
func (openchainDB *OpenchainDB) Open() {
	openchainDB.mux.Lock()
	if openchainDB.dbState == opened {
		openchainDB.mux.Unlock()
		return
	}

	defer openchainDB.mux.Unlock()

	dbPath := getDBPath()
	missing, err := dirMissingOrEmpty(dbPath)
	if err != nil {
		panic(fmt.Sprintf("Error while trying to open DB: %s", err))
	}
	dbLogger.Debugf("Is db path [%s] empty [%t]", dbPath, missing)

	if missing {
		err = os.MkdirAll(path.Dir(dbPath), 0755)
		if err != nil {
			panic(fmt.Sprintf("Error making directory path [%s]: %s", dbPath, err))
		}
	}

	opts := gorocksdb.NewDefaultOptions()
	defer opts.Destroy()

	opts.SetCreateIfMissing(missing)
	opts.SetCreateIfMissingColumnFamilies(true)

	cfNames := []string{"default"}
	cfNames = append(cfNames, columnfamilies...)
	var cfOpts []*gorocksdb.Options
	for range cfNames {
		cfOpts = append(cfOpts, opts)
	}

	db, cfHandlers, err := gorocksdb.OpenDbColumnFamilies(opts, dbPath, cfNames, cfOpts)

	if err != nil {
		panic(fmt.Sprintf("Error opening DB: %s", err))
	}

	openchainDB.DB = db
	openchainDB.BlockchainCF = cfHandlers[1]
	openchainDB.StateCF = cfHandlers[2]
	openchainDB.StateDeltaCF = cfHandlers[3]
	openchainDB.IndexesCF = cfHandlers[4]
	openchainDB.PersistCF = cfHandlers[5]
	openchainDB.dbState = opened
}
Exemple #15
0
func create_purge_backups() error {
	dbOpts := gorocksdb.NewDefaultOptions()
	dbOpts.SetCreateIfMissing(true)
	if err := os.RemoveAll("/tmp/backups_db"); err != nil {
		return err
	}

	db, err := gorocksdb.OpenDb(dbOpts, "/tmp/backups_db")
	if err != nil {
		return err
	}

	if err := os.RemoveAll("/tmp/db_backups"); err != nil {
		return err
	}
	be, err := gorocksdb.OpenBackupEngine(dbOpts, "/tmp/db_backups")
	if err != nil {
		return err
	}

	writeOpts := gorocksdb.NewDefaultWriteOptions()
	for i := 0; i < 5; i++ {
		key := []byte(fmt.Sprint(i))
		if err := db.Put(writeOpts, key, key); err != nil {
			return err
		}
		if err := be.CreateNewBackup(db); err != nil {
			return err
		}
	}
	fmt.Println("Num available initially: ", be.GetInfo().GetCount())
	if err := gorocksext.PurgeOldBackups(be, 4); err != nil {
		return err
	}
	fmt.Println(be.GetInfo().GetCount())
	if err := gorocksext.PurgeOldBackups(be, 2); err != nil {
		return err
	}
	fmt.Println(be.GetInfo().GetCount())
	if err := gorocksext.PurgeOldBackups(be, 0); err != nil {
		return err
	}
	fmt.Println(be.GetInfo().GetCount())

	db.Close()
	return nil
}
Exemple #16
0
func NewRocksDB(path string) (*RocksDB, error) {
	opts := gorocksdb.NewDefaultOptions()
	filter := gorocksdb.NewBloomFilter(14)
	opts.SetFilterPolicy(filter)
	opts.SetMaxOpenFiles(10000)
	db, err := gorocksdb.OpenDbForReadOnly(opts, path, false)
	if err != nil {
		return nil, err
	}
	return &RocksDB{
		db:     db,
		ro:     gorocksdb.NewDefaultReadOptions(),
		hits:   metrics.NewMeter(),
		misses: metrics.NewMeter(),
		cache:  lru.New(1000000),
	}, nil
}
Exemple #17
0
func read_all(db_dir string) error {
	dbOpts := gorocksdb.NewDefaultOptions()
	db, err := gorocksdb.OpenDb(dbOpts, db_dir)
	if err != nil {
		return err
	}
	defaultRO := gorocksdb.NewDefaultReadOptions()
	iter := db.NewIterator(defaultRO)
	iter.SeekToFirst()
	for iter.Valid() {
		key := iter.Key()
		defer key.Free()
		fmt.Println(string(key.Data()))
		iter.Next()
	}
	db.Close()
	return nil
}
Exemple #18
0
func openDB() (*OpenchainDB, error) {
	if isOpen {
		return openchainDB, nil
	}
	dbPath := getDBPath()
	opts := gorocksdb.NewDefaultOptions()
	opts.SetCreateIfMissing(false)
	db, cfHandlers, err := gorocksdb.OpenDbColumnFamilies(opts, dbPath,
		[]string{"default", blockchainCF, stateCF, stateDeltaCF, indexesCF},
		[]*gorocksdb.Options{opts, opts, opts, opts, opts})

	if err != nil {
		fmt.Println("Error opening DB", err)
		return nil, err
	}
	isOpen = true
	return &OpenchainDB{db, cfHandlers[1], cfHandlers[2], cfHandlers[3], cfHandlers[4]}, nil
}
Exemple #19
0
func main() {
	dbName := "/data/mydb"
	compactionThreads := 4
	options := gorocksdb.NewDefaultOptions()
	options.SetCreateIfMissing(true)
	options.SetStatsDumpPeriodSec(60 * 10) // dump stats at 10 minute interval
	options.SetCompactionStyle(gorocksdb.UniversalCompactionStyle)
	options.IncreaseParallelism(compactionThreads)
	options.IncreaseParallelism(compactionThreads)
	db, err := gorocksdb.OpenDb(options, dbName)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	putStatch := doPut(db, 64*1024, 128*1024)
	for {
		p := <-putStatch
		fmt.Println("avgTime ", p.avgTime, "maxTime", p.maxTime)
	}
}
Exemple #20
0
func check_checkpoints() error {
	dbOpts := gorocksdb.NewDefaultOptions()
	dbOpts.SetCreateIfMissing(true)
	if err := os.RemoveAll("/tmp/checkpoint_db"); err != nil {
		return err
	}

	db, err := gorocksdb.OpenDb(dbOpts, "/tmp/checkpoint_db")
	if err != nil {
		return err
	}
	writeOpts := gorocksdb.NewDefaultWriteOptions()
	for i := 0; i < 16; i++ {
		key := []byte(fmt.Sprint(i))
		if err := db.Put(writeOpts, key, key); err != nil {
			return err
		}
		if i == 8 {
			if err := os.RemoveAll("/tmp/checkpoint_db1"); err != nil {
				return err
			}
			gorocksext.CreateCheckpoint(db, "/tmp/checkpoint_db1")
		}
	}
	db.Close()

	fmt.Println("Full Db")
	if err := read_all("/tmp/checkpoint_db"); err != nil {
		return err
	}
	fmt.Println("Checkpointed snapshot")
	if err := read_all("/tmp/checkpoint_db1"); err != nil {
		return err
	}
	return nil
}
Exemple #21
0
}

func (hp *HostPort) AsString() string {
	return hp.Host + ":" + strconv.FormatInt(int64(hp.Port), 10)
}

func (hp *HostPort) PortString() string {
	return strconv.FormatInt(int64(hp.Port), 10)
}

// ------------ LevelDB Storage ------------------

// From reading levigo's and levelDB's docs, it looks like the DB is
// safe for concurrent use by multiple goroutines without extra
// synchronization.
var opts = gorocksdb.NewDefaultOptions()
var storeDB *gorocksdb.DB
var readOpt = gorocksdb.NewDefaultReadOptions()
var writeOpt = gorocksdb.NewDefaultWriteOptions()

func storageInit(path string) {
	//opts.IncreaseParallelism(2)
	opts.OptimizeForPointLookup(128) // 128mb cache size
	//opts.SetWriteBufferSize(4*1024*1024) // default 4mb
	opts.SetCreateIfMissing(true)
	opts.SetCompression(gorocksdb.NoCompression)
	opts.SetDisableDataSync(true)
	opts.SetUseFsync(false) // true:fsync, false:fdatasync
	db, err := gorocksdb.OpenDb(opts, path)
	if err != nil {
		panic("error opening DB at " + path + ": " + err.Error())
Exemple #22
0
// Destroy the rocksdb instance and data files
func (s *Store) Destroy() {
	s.Close()
	rocks.DestroyDb(s.directory, rocks.NewDefaultOptions())
}
Exemple #23
0
// NewStore returns the Store a rocksdb wrapper
func NewStore(options StoreOptions) (*Store, error) {
	options.SetDefaults()
	if options.Directory == "" {
		return nil, fmt.Errorf("Empty directory of store options")
	}
	if options.IsDebug {
		log.EnableDebug()
	}

	s := &Store{
		directory:  options.Directory,
		useTailing: !options.DisableTailing,
		cfHandles:  make(map[string]*rocks.ColumnFamilyHandle),
		queues:     make(map[string]*Queue),
	}

	opts := rocks.NewDefaultOptions()
	opts.SetCreateIfMissing(true)
	opts.IncreaseParallelism(options.Parallel)
	opts.SetMergeOperator(&_CountMerger{})
	opts.SetMaxSuccessiveMerges(64)

	opts.SetWriteBufferSize(options.WriteBufferSize)
	opts.SetMaxWriteBufferNumber(options.WriteBufferNumber)
	opts.SetTargetFileSizeBase(options.FileSizeBase)
	opts.SetLevel0FileNumCompactionTrigger(8)
	opts.SetLevel0SlowdownWritesTrigger(16)
	opts.SetLevel0StopWritesTrigger(24)
	opts.SetNumLevels(4)
	opts.SetMaxBytesForLevelBase(512 * 1024 * 1024)
	opts.SetMaxBytesForLevelMultiplier(8)
	opts.SetCompression(options.Compression)
	opts.SetDisableAutoCompactions(options.DisableAutoCompaction)

	bbto := rocks.NewDefaultBlockBasedTableOptions()
	bbto.SetBlockCache(rocks.NewLRUCache(options.MemorySize))
	bbto.SetFilterPolicy(rocks.NewBloomFilter(10))
	opts.SetBlockBasedTableFactory(bbto)

	opts.SetMaxOpenFiles(-1)
	opts.SetMemtablePrefixBloomBits(8 * 1024 * 1024)

	var err error
	if err = os.MkdirAll(options.Directory, 0755); err != nil {
		log.Errorf("Failed to mkdir %q, %s", options.Directory, err)
		return nil, err
	}

	cfNames, err := rocks.ListColumnFamilies(opts, options.Directory)
	if err != nil {
		// FIXME: we need to be sure if this means the db does not exist for now
		// so that we cannot list the column families
		log.Errorf("Failed to collect the column family names, %s", err)
	} else {
		log.Debugf("Got column family names for the existing db, %+v", cfNames)
	}

	if len(cfNames) == 0 {
		// We create the default column family to get the column family handle
		cfNames = []string{"default"}
	}
	cfOpts := make([]*rocks.Options, len(cfNames))
	for i := range cfNames {
		cfOpts[i] = opts
	}
	db, cfHandles, err := rocks.OpenDbColumnFamilies(opts, options.Directory, cfNames, cfOpts)
	if err != nil {
		log.Errorf("Failed to open rocks database, %s", err)
		return nil, err
	}

	s.DB = db
	s.dbOpts = opts
	s.ro = rocks.NewDefaultReadOptions()
	s.ro.SetFillCache(false)
	s.ro.SetTailing(!options.DisableTailing)
	s.wo = rocks.NewDefaultWriteOptions()
	s.wo.DisableWAL(options.DisableWAL)
	s.wo.SetSync(options.Sync)

	if len(cfNames) > 0 {
		for i := range cfNames {
			s.cfHandles[cfNames[i]] = cfHandles[i]
		}
	}
	return s, nil
}
Exemple #24
0
func (rh *RocksDBHandler) Init() error {
	rh.options = rocks.NewDefaultOptions()
	rh.options.SetBlockCache(rocks.NewLRUCache(rh.cacheSize))
	rh.options.SetBlockSize(rh.blockSize)
	rh.options.SetCreateIfMissing(rh.createIfMissing)
	if rh.bloomFilter > 0 {
		rh.options.SetFilterPolicy(rocks.NewBloomFilter(rh.bloomFilter))
	}
	if rh.maxOpenFiles > 0 {
		rh.options.SetMaxOpenFiles(rh.maxOpenFiles)
	}

	switch rh.compression {
	case "no":
		rh.options.SetCompression(rocks.NoCompression)
	case "snappy":
		rh.options.SetCompression(rocks.SnappyCompression)
	case "zlib":
		rh.options.SetCompression(rocks.ZlibCompression)
	case "bzip2":
		rh.options.SetCompression(rocks.BZip2Compression)
	}

	switch rh.compactionStyle {
	case "level":
		rh.options.SetCompactionStyle(rocks.LevelCompactionStyle)
	case "universal":
		rh.options.SetCompactionStyle(rocks.UniversalCompactionStyle)
	}

	rh.dsMergers = make(map[string]DataStructureMerger)
	rh.dsMergers[kRedisString] = &StringMerger{}
	rh.dsMergers[kRedisList] = &ListMerger{}
	rh.dsMergers[kRedisHash] = &HashMerger{}
	rh.dsMergers[kRedisSet] = &SetMerger{}

	if rh.maxMerge > 0 {
		rh.options.SetMaxSuccessiveMerges(rh.maxMerge)
	}
	rh.options.SetMergeOperator(rocks.NewMergeOperator(rh))

	db, err := rocks.OpenDb(rh.options, rh.dbDir)
	if err != nil {
		rh.Close()
		return err
	}
	rh.db = db

	infos := []string{
		fmt.Sprintf("dbDir=%s", rh.dbDir),
		fmt.Sprintf("cacheSize=%d", rh.cacheSize),
		fmt.Sprintf("blockSize=%d", rh.blockSize),
		fmt.Sprintf("createIfMissing=%v", rh.createIfMissing),
		fmt.Sprintf("bloomFilter=%d", rh.bloomFilter),
		fmt.Sprintf("compression=%s", rh.compression),
		fmt.Sprintf("compactionStyle=%s", rh.compactionStyle),
		fmt.Sprintf("maxOpenFiles=%d", rh.maxOpenFiles),
		fmt.Sprintf("maxMerge=%d", rh.maxMerge),
	}
	log.Printf("[RocksDBHandler] Inited, %s", strings.Join(infos, ", "))
	return nil
}
Exemple #25
0
func (db *rocksDB) initialize(path string, conf *config) error {
	if conf == nil {
		conf = newDefaultConfig()
	}

	// Create path if not exists first
	if err := os.MkdirAll(path, 0700); err != nil {
		return errors.Trace(err)
	}

	opts := gorocksdb.NewDefaultOptions()
	opts.SetCreateIfMissing(true)
	opts.SetErrorIfExists(false)

	opts.SetCompression(gorocksdb.CompressionType(conf.CompressionType))
	opts.SetWriteBufferSize(conf.WriteBufferSize)
	opts.SetMaxOpenFiles(conf.MaxOpenFiles)
	opts.SetNumLevels(conf.NumLevels)

	opts.SetMaxWriteBufferNumber(conf.MaxWriteBufferNumber)
	opts.SetMinWriteBufferNumberToMerge(conf.MinWriteBufferNumberToMerge)
	opts.SetLevel0FileNumCompactionTrigger(conf.Level0FileNumCompactionTrigger)
	opts.SetLevel0SlowdownWritesTrigger(conf.Level0SlowdownWritesTrigger)
	opts.SetLevel0StopWritesTrigger(conf.Level0StopWritesTrigger)
	opts.SetTargetFileSizeBase(uint64(conf.TargetFileSizeBase))
	opts.SetTargetFileSizeMultiplier(conf.TargetFileSizeMultiplier)
	opts.SetMaxBytesForLevelBase(uint64(conf.MaxBytesForLevelBase))
	opts.SetMaxBytesForLevelMultiplier(conf.MaxBytesForLevelMultiplier)

	opts.SetDisableAutoCompactions(conf.DisableAutoCompactions)
	opts.SetDisableDataSync(conf.DisableDataSync)
	opts.SetUseFsync(conf.UseFsync)
	opts.SetMaxBackgroundCompactions(conf.MaxBackgroundCompactions)
	opts.SetMaxBackgroundFlushes(conf.MaxBackgroundFlushes)
	opts.SetAllowOsBuffer(conf.AllowOSBuffer)

	topts := gorocksdb.NewDefaultBlockBasedTableOptions()
	topts.SetBlockSize(conf.BlockSize)

	cache := gorocksdb.NewLRUCache(conf.CacheSize)
	topts.SetBlockCache(cache)

	topts.SetFilterPolicy(gorocksdb.NewBloomFilter(conf.BloomFilterSize))
	opts.SetBlockBasedTableFactory(topts)

	env := gorocksdb.NewDefaultEnv()
	env.SetBackgroundThreads(conf.BackgroundThreads)
	env.SetHighPriorityBackgroundThreads(conf.HighPriorityBackgroundThreads)
	opts.SetEnv(env)

	db.path = path
	db.opts = opts
	db.ropt = gorocksdb.NewDefaultReadOptions()
	db.wopt = gorocksdb.NewDefaultWriteOptions()
	db.env = env
	db.topts = topts
	db.cache = cache
	db.snapshotFillCache = conf.SnapshotFillCache

	var err error
	if db.rkdb, err = gorocksdb.OpenDb(db.opts, db.path); err != nil {
		return errors.Trace(err)
	}
	return nil
}