// CreateDB creates a rocks db database
func CreateDB() error {
dbPath := getDBPath()
dbLogger.Debug("Creating DB at [%s]", dbPath)
missing, err := dirMissingOrEmpty(dbPath)
if err != nil {
return err
}
if !missing {
return fmt.Errorf("db dir [%s] already exists", dbPath)
}
err = os.MkdirAll(path.Dir(dbPath), 0755)
if err != nil {
dbLogger.Error("Error calling os.MkdirAll for directory path [%s]: %s", dbPath, err)
return fmt.Errorf("Error making directory path [%s]: %s", dbPath, err)
}
opts := gorocksdb.NewDefaultOptions()
defer opts.Destroy()
opts.SetCreateIfMissing(true)
db, err := gorocksdb.OpenDb(opts, dbPath)
if err != nil {
return err
}
defer db.Close()
dbLogger.Debug("DB created at [%s]", dbPath)
return nil
}
func NewRocksdbStorage(dbfname string, dbtype string, mergeOp gorocksdb.MergeOperator) (*RocksdbStorage, error) {
var sto *RocksdbStorage
if dbtype != "kv" && dbtype != "json" {
return sto, fmt.Errorf("Unkown db type")
}
opts := gorocksdb.NewDefaultOptions()
if mergeOp != nil {
opts.SetMergeOperator(mergeOp)
}
// opts.IncreaseParallelism(runtime.NumCPU())
// opts.OptimizeLevelStyleCompaction(0)
opts.SetCreateIfMissing(true)
db, err := gorocksdb.OpenDb(opts, dbfname)
if err != nil {
return sto, err
}
ro := gorocksdb.NewDefaultReadOptions()
wo := gorocksdb.NewDefaultWriteOptions()
sto = &RocksdbStorage{
dbfname: dbfname,
db: db,
ro: ro,
wo: wo,
}
return sto, nil
}
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
rv := Store{
path: path,
opts: gorocksdb.NewDefaultOptions(),
}
if mo != nil {
rv.opts.SetMergeOperator(mo)
}
_, err := applyConfig(rv.opts, config)
if err != nil {
return nil, err
}
rv.db, err = gorocksdb.OpenDb(rv.opts, rv.path)
if err != nil {
return nil, err
}
return &rv, nil
}
func write_multi_cfs() error {
dbOpts := gorocksdb.NewDefaultOptions()
dbOpts.SetCreateIfMissing(true)
if err := os.RemoveAll("/tmp/multicf_db"); err != nil {
return err
}
db, err := gorocksdb.OpenDb(dbOpts, "/tmp/multicf_db")
if err != nil {
return err
}
var handles []*gorocksdb.ColumnFamilyHandle
for i := 0; i < 4; i++ {
handle, err := db.CreateColumnFamily(dbOpts, fmt.Sprint(i))
if err != nil {
return err
}
handles = append(handles, handle)
}
writeOpts := gorocksdb.NewDefaultWriteOptions()
if err := db.Put(writeOpts, []byte("default"), []byte("default")); err != nil {
return err
}
for i := 0; i < 16; i++ {
key := []byte(fmt.Sprint(i))
if err := db.PutCF(writeOpts, handles[i%4], key, key); err != nil {
return err
}
}
db.Close()
return nil
}
func NewRocksDB(path string) (*RocksDB, error) {
opts := gorocksdb.NewDefaultOptions()
//opts.SetBlockCache(gorocksdb.NewLRUCache(3 << 30))
opts.SetCreateIfMissing(true)
db, err := gorocksdb.OpenDb(opts, path)
return &RocksDB{db: db}, err
}
func main() {
dbName := "/data/mydb"
cache := gorocksdb.NewLRUCache(512 * 1024 * 1024)
filter := gorocksdb.NewBloomFilter(15)
to := gorocksdb.NewDefaultBlockBasedTableOptions()
to.SetBlockSize(256 * 1024)
to.SetBlockCache(cache)
to.SetFilterPolicy(filter)
options := gorocksdb.NewDefaultOptions()
options.SetBlockBasedTableFactory(to)
options.SetCreateIfMissing(true)
options.SetStatsDumpPeriodSec(60 * 1) // dump stats at 10 minute interval
options.SetCompactionStyle(gorocksdb.UniversalCompactionStyle)
options.SetWriteBufferSize(512 * 1024 * 1024)
options.SetMaxWriteBufferNumber(5)
options.SetMinWriteBufferNumberToMerge(2)
db, err := gorocksdb.OpenDb(options, dbName)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
putStatch := doPut(db, 16*1024, 32*1024)
for {
p := <-putStatch
fmt.Println("avgTime ", p.avgTime, "maxTime", p.maxTime)
}
}
func (ldbs *Store) Open() error {
var err error
ldbs.db, err = gorocksdb.OpenDb(ldbs.opts, ldbs.path)
if err != nil {
return err
}
return nil
}
func newRocksDB(dir string) *rocks.DB {
opts := gorocksdb.NewDefaultOptions()
opts.SetCreateIfMissing(true)
rdb, err := gorocksdb.OpenDb(opts, dir)
if err != nil {
panic(err)
}
return rocks.New(rdb)
}
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
rv := Store{
path: path,
config: config,
opts: gorocksdb.NewDefaultOptions(),
}
if mo != nil {
rv.opts.SetMergeOperator(mo)
}
_, err := applyConfig(rv.opts, config)
if err != nil {
return nil, err
}
rv.db, err = gorocksdb.OpenDb(rv.opts, rv.path)
if err != nil {
return nil, err
}
b, ok := config["readoptions_verify_checksum"].(bool)
if ok {
rv.roptVerifyChecksums, rv.roptVerifyChecksumsUse = b, true
}
b, ok = config["readoptions_fill_cache"].(bool)
if ok {
rv.roptFillCache, rv.roptFillCacheUse = b, true
}
v, ok := config["readoptions_read_tier"].(float64)
if ok {
rv.roptReadTier, rv.roptReadTierUse = int(v), true
}
b, ok = config["writeoptions_sync"].(bool)
if ok {
rv.woptSync, rv.woptSyncUse = b, true
}
b, ok = config["writeoptions_disable_WAL"].(bool)
if ok {
rv.woptDisableWAL, rv.woptDisableWALUse = b, true
}
return &rv, nil
}
func newRocksDB(t *testing.T) *gorocksdb.DB {
dir, err := ioutil.TempDir("", "rocks")
ensure.Nil(t, err)
opts := gorocksdb.NewDefaultOptions()
opts.SetCreateIfMissing(true)
db, err := gorocksdb.OpenDb(opts, dir)
ensure.Nil(t, err)
return db
}
func storageInit(path string) {
//opts.IncreaseParallelism(2)
opts.OptimizeForPointLookup(128) // 128mb cache size
//opts.SetWriteBufferSize(4*1024*1024) // default 4mb
opts.SetCreateIfMissing(true)
opts.SetCompression(gorocksdb.NoCompression)
opts.SetDisableDataSync(true)
opts.SetUseFsync(false) // true:fsync, false:fdatasync
db, err := gorocksdb.OpenDb(opts, path)
if err != nil {
panic("error opening DB at " + path + ": " + err.Error())
}
storeDB = db
}
func create_purge_backups() error {
dbOpts := gorocksdb.NewDefaultOptions()
dbOpts.SetCreateIfMissing(true)
if err := os.RemoveAll("/tmp/backups_db"); err != nil {
return err
}
db, err := gorocksdb.OpenDb(dbOpts, "/tmp/backups_db")
if err != nil {
return err
}
if err := os.RemoveAll("/tmp/db_backups"); err != nil {
return err
}
be, err := gorocksdb.OpenBackupEngine(dbOpts, "/tmp/db_backups")
if err != nil {
return err
}
writeOpts := gorocksdb.NewDefaultWriteOptions()
for i := 0; i < 5; i++ {
key := []byte(fmt.Sprint(i))
if err := db.Put(writeOpts, key, key); err != nil {
return err
}
if err := be.CreateNewBackup(db); err != nil {
return err
}
}
fmt.Println("Num available initially: ", be.GetInfo().GetCount())
if err := gorocksext.PurgeOldBackups(be, 4); err != nil {
return err
}
fmt.Println(be.GetInfo().GetCount())
if err := gorocksext.PurgeOldBackups(be, 2); err != nil {
return err
}
fmt.Println(be.GetInfo().GetCount())
if err := gorocksext.PurgeOldBackups(be, 0); err != nil {
return err
}
fmt.Println(be.GetInfo().GetCount())
db.Close()
return nil
}
func read_all(db_dir string) error {
dbOpts := gorocksdb.NewDefaultOptions()
db, err := gorocksdb.OpenDb(dbOpts, db_dir)
if err != nil {
return err
}
defaultRO := gorocksdb.NewDefaultReadOptions()
iter := db.NewIterator(defaultRO)
iter.SeekToFirst()
for iter.Valid() {
key := iter.Key()
defer key.Free()
fmt.Println(string(key.Data()))
iter.Next()
}
db.Close()
return nil
}
func main() {
dbName := "/data/mydb"
compactionThreads := 4
options := gorocksdb.NewDefaultOptions()
options.SetCreateIfMissing(true)
options.SetStatsDumpPeriodSec(60 * 10) // dump stats at 10 minute interval
options.SetCompactionStyle(gorocksdb.UniversalCompactionStyle)
options.IncreaseParallelism(compactionThreads)
options.IncreaseParallelism(compactionThreads)
db, err := gorocksdb.OpenDb(options, dbName)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
putStatch := doPut(db, 64*1024, 128*1024)
for {
p := <-putStatch
fmt.Println("avgTime ", p.avgTime, "maxTime", p.maxTime)
}
}
func check_checkpoints() error {
dbOpts := gorocksdb.NewDefaultOptions()
dbOpts.SetCreateIfMissing(true)
if err := os.RemoveAll("/tmp/checkpoint_db"); err != nil {
return err
}
db, err := gorocksdb.OpenDb(dbOpts, "/tmp/checkpoint_db")
if err != nil {
return err
}
writeOpts := gorocksdb.NewDefaultWriteOptions()
for i := 0; i < 16; i++ {
key := []byte(fmt.Sprint(i))
if err := db.Put(writeOpts, key, key); err != nil {
return err
}
if i == 8 {
if err := os.RemoveAll("/tmp/checkpoint_db1"); err != nil {
return err
}
gorocksext.CreateCheckpoint(db, "/tmp/checkpoint_db1")
}
}
db.Close()
fmt.Println("Full Db")
if err := read_all("/tmp/checkpoint_db"); err != nil {
return err
}
fmt.Println("Checkpointed snapshot")
if err := read_all("/tmp/checkpoint_db1"); err != nil {
return err
}
return nil
}
func (rh *RocksDBHandler) Init() error {
rh.options = rocks.NewDefaultOptions()
rh.options.SetBlockCache(rocks.NewLRUCache(rh.cacheSize))
rh.options.SetBlockSize(rh.blockSize)
rh.options.SetCreateIfMissing(rh.createIfMissing)
if rh.bloomFilter > 0 {
rh.options.SetFilterPolicy(rocks.NewBloomFilter(rh.bloomFilter))
}
if rh.maxOpenFiles > 0 {
rh.options.SetMaxOpenFiles(rh.maxOpenFiles)
}
switch rh.compression {
case "no":
rh.options.SetCompression(rocks.NoCompression)
case "snappy":
rh.options.SetCompression(rocks.SnappyCompression)
case "zlib":
rh.options.SetCompression(rocks.ZlibCompression)
case "bzip2":
rh.options.SetCompression(rocks.BZip2Compression)
}
switch rh.compactionStyle {
case "level":
rh.options.SetCompactionStyle(rocks.LevelCompactionStyle)
case "universal":
rh.options.SetCompactionStyle(rocks.UniversalCompactionStyle)
}
rh.dsMergers = make(map[string]DataStructureMerger)
rh.dsMergers[kRedisString] = &StringMerger{}
rh.dsMergers[kRedisList] = &ListMerger{}
rh.dsMergers[kRedisHash] = &HashMerger{}
rh.dsMergers[kRedisSet] = &SetMerger{}
if rh.maxMerge > 0 {
rh.options.SetMaxSuccessiveMerges(rh.maxMerge)
}
rh.options.SetMergeOperator(rocks.NewMergeOperator(rh))
db, err := rocks.OpenDb(rh.options, rh.dbDir)
if err != nil {
rh.Close()
return err
}
rh.db = db
infos := []string{
fmt.Sprintf("dbDir=%s", rh.dbDir),
fmt.Sprintf("cacheSize=%d", rh.cacheSize),
fmt.Sprintf("blockSize=%d", rh.blockSize),
fmt.Sprintf("createIfMissing=%v", rh.createIfMissing),
fmt.Sprintf("bloomFilter=%d", rh.bloomFilter),
fmt.Sprintf("compression=%s", rh.compression),
fmt.Sprintf("compactionStyle=%s", rh.compactionStyle),
fmt.Sprintf("maxOpenFiles=%d", rh.maxOpenFiles),
fmt.Sprintf("maxMerge=%d", rh.maxMerge),
}
log.Printf("[RocksDBHandler] Inited, %s", strings.Join(infos, ", "))
return nil
}
func (db *rocksDB) initialize(path string, conf *config) error {
if conf == nil {
conf = newDefaultConfig()
}
// Create path if not exists first
if err := os.MkdirAll(path, 0700); err != nil {
return errors.Trace(err)
}
opts := gorocksdb.NewDefaultOptions()
opts.SetCreateIfMissing(true)
opts.SetErrorIfExists(false)
opts.SetCompression(gorocksdb.CompressionType(conf.CompressionType))
opts.SetWriteBufferSize(conf.WriteBufferSize)
opts.SetMaxOpenFiles(conf.MaxOpenFiles)
opts.SetNumLevels(conf.NumLevels)
opts.SetMaxWriteBufferNumber(conf.MaxWriteBufferNumber)
opts.SetMinWriteBufferNumberToMerge(conf.MinWriteBufferNumberToMerge)
opts.SetLevel0FileNumCompactionTrigger(conf.Level0FileNumCompactionTrigger)
opts.SetLevel0SlowdownWritesTrigger(conf.Level0SlowdownWritesTrigger)
opts.SetLevel0StopWritesTrigger(conf.Level0StopWritesTrigger)
opts.SetTargetFileSizeBase(uint64(conf.TargetFileSizeBase))
opts.SetTargetFileSizeMultiplier(conf.TargetFileSizeMultiplier)
opts.SetMaxBytesForLevelBase(uint64(conf.MaxBytesForLevelBase))
opts.SetMaxBytesForLevelMultiplier(conf.MaxBytesForLevelMultiplier)
opts.SetDisableAutoCompactions(conf.DisableAutoCompactions)
opts.SetDisableDataSync(conf.DisableDataSync)
opts.SetUseFsync(conf.UseFsync)
opts.SetMaxBackgroundCompactions(conf.MaxBackgroundCompactions)
opts.SetMaxBackgroundFlushes(conf.MaxBackgroundFlushes)
opts.SetAllowOsBuffer(conf.AllowOSBuffer)
topts := gorocksdb.NewDefaultBlockBasedTableOptions()
topts.SetBlockSize(conf.BlockSize)
cache := gorocksdb.NewLRUCache(conf.CacheSize)
topts.SetBlockCache(cache)
topts.SetFilterPolicy(gorocksdb.NewBloomFilter(conf.BloomFilterSize))
opts.SetBlockBasedTableFactory(topts)
env := gorocksdb.NewDefaultEnv()
env.SetBackgroundThreads(conf.BackgroundThreads)
env.SetHighPriorityBackgroundThreads(conf.HighPriorityBackgroundThreads)
opts.SetEnv(env)
db.path = path
db.opts = opts
db.ropt = gorocksdb.NewDefaultReadOptions()
db.wopt = gorocksdb.NewDefaultWriteOptions()
db.env = env
db.topts = topts
db.cache = cache
db.snapshotFillCache = conf.SnapshotFillCache
var err error
if db.rkdb, err = gorocksdb.OpenDb(db.opts, db.path); err != nil {
return errors.Trace(err)
}
return nil
}