Пример #1
0
// formats a new filesystem in the given data dir
func Format(dataDir string, rootUid, rootGid uint32) error {
	// wipe out previous
	err := os.RemoveAll(dataDir)
	if err != nil {
		return err
	}
	// create
	err = os.Mkdir(dataDir, 0755)
	if err != nil {
		return fmt.Errorf("issue creating namenode home dir: %s\n", err.Error())
	}
	err = os.Mkdir(dataDir+"/"+dir_inodb, 0755)
	if err != nil {
		return fmt.Errorf("issue creating inodb parent dir: %s\n", err.Error())
	}
	err = os.Mkdir(dataDir+"/"+dir_counters, 0755)
	if err != nil {
		return fmt.Errorf("issue creating counters parent dir: %s\n", err.Error())
	}

	opts := levigo.NewOptions()
	defer opts.Close()
	opts.SetCreateIfMissing(true)

	// create inodb
	db, err := levigo.Open(dataDir+"/"+dir_inodb, opts)
	if err != nil {
		return err
	}
	// add root node
	ino := maggiefs.NewInode(1, maggiefs.FTYPE_DIR, 0755, rootUid, rootGid)
	binSize := ino.BinSize()
	inoBytes := make([]byte, binSize)
	ino.ToBytes(inoBytes)
	rootNodeId := make([]byte, 8)
	binary.LittleEndian.PutUint64(rootNodeId, 1)
	db.Put(WriteOpts, rootNodeId, inoBytes)
	db.Close()
	db, err = levigo.Open(dataDir+"/"+dir_counters, opts)
	if err != nil {
		return err
	}
	// put 1 for inode counter so other nodes are higher
	key := []byte(COUNTER_INODE)
	val := make([]byte, 8)
	binary.LittleEndian.PutUint64(val, 1)
	err = db.Put(WriteOpts, key, val)
	if err != nil {
		return err
	}
	db.Close()

	return nil
}
Пример #2
0
func New(directory string) (*Level, error) {
	options := levigo.NewOptions()
	options.SetCreateIfMissing(true)
	options.SetCompression(levigo.SnappyCompression)
	db, err := levigo.Open(directory, options)
	return &Level{db: db}, err
}
Пример #3
0
func main() {
	flag.Parse()

	if *showVersion {
		fmt.Printf("gocountme: v%s\n", VERSION)
		return
	}

	if *defaultSize <= 0 {
		fmt.Printf("--default-size must be greater than 0\n")
		return
	}

	if _, err := os.Stat(*dblocation); err != nil {
		if os.IsNotExist(err) {
			fmt.Println("Database location does not exist:", *dblocation)
			return
		}
	}

	log.Println("Opening levelDB")
	Default_KMinValues_Size = *defaultSize
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(*leveldbLRUCache))
	opts.SetCreateIfMissing(true)
	db, err := levigo.Open(*dblocation, opts)
	defer db.Close()

	if err != nil {
		log.Panicln(err)
	}

	RequestChan = make(chan RequestCommand, *nWorkers)
	workerWaitGroup := sync.WaitGroup{}
	log.Printf("Starting %d workers", *nWorkers)
	for i := 0; i < *nWorkers; i++ {
		go func(id int) {
			workerWaitGroup.Add(1)
			levelDBWorker(db, RequestChan)
			workerWaitGroup.Done()
		}(i)
	}

	http.HandleFunc("/get", GetHandler)
	http.HandleFunc("/delete", DeleteHandler)
	http.HandleFunc("/cardinality", CardinalityHandler)
	http.HandleFunc("/jaccard", JaccardHandler)
	http.HandleFunc("/correlation", CorrelationMatrixHandler)
	http.HandleFunc("/add", AddHandler)
	http.HandleFunc("/addhash", AddHashHandler)
	http.HandleFunc("/query", QueryHandler)
	http.HandleFunc("/exit", ExitHandler)

	log.Printf("Starting gocountme HTTP server on %s", *httpAddress)
	go func() {
		log.Fatal(http.ListenAndServe(*httpAddress, nil))
	}()

	workerWaitGroup.Wait()
}
Пример #4
0
func main() {
	dbname := "leveldb"
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(3 << 20))
	opts.SetCreateIfMissing(true)
	_ = levigo.DestroyDatabase(dbname, opts)
	db, _ := levigo.Open(dbname, opts)

	wo := levigo.NewWriteOptions()
	ro := levigo.NewReadOptions()

	start := time.Now()

	for i := 0; i < 10e4; i++ {
		db.Put(wo, []byte(fmt.Sprintf("a%v", i)), []byte(strconv.Itoa(i)))
	}
	for i := 0; i < 10e4; i++ {
		db.Get(ro, []byte(fmt.Sprintf("a%v", i)))
	}
	for i := 0; i < 10e4; i++ {
		db.Delete(wo, []byte(fmt.Sprintf("a%v", i)))
	}
	duration := time.Since(start)
	log.Printf("Elapsed: %v.", duration)
}
Пример #5
0
func TestLevigo(t *testing.T) {
	path := "/tmp/levigo_test_10101"
	os.RemoveAll(path)

	opts := levigo.NewOptions()
	filter := levigo.NewBloomFilter(10)
	opts.SetFilterPolicy(filter)
	opts.SetCache(levigo.NewLRUCache(1024 << 20)) // 1G
	opts.SetCreateIfMissing(true)
	if ldb, err := levigo.Open(path, opts); err == nil {
		key := []byte("test-test hwl0dsfds")
		val := []byte("value")

		if err = ldb.Put(levigo.NewWriteOptions(), key, val); err != nil {
			t.Fail()
		} else {
			ro := levigo.NewReadOptions()
			if data, err := ldb.Get(ro, key); err == nil && reflect.DeepEqual(data, val) {
				ro.SetFillCache(false)
				it := ldb.NewIterator(ro)
				it.Seek([]byte{0})
				for ; it.Valid(); it.Next() {
					log.Printf("%s => %s", it.Key(), it.Value())
				}
			} else {
				t.Fail()
			}
		}
	} else {
		t.Fail()
	}
}
Пример #6
0
func InitDB() (*levigo.DB, error) {
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(1024 * 1024))
	opts.SetCreateIfMissing(true)

	homedir := os.Getenv("HOME")
	dbdir := homedir + "/.overseer/db"
	os.MkdirAll(dbdir, 0700)

	db, err := levigo.Open(dbdir, opts)

	if err != nil {
		fmt.Println("Failed to open database", err)
		return nil, err
	}

	procs, err := ListProcs(db)
	for p, status := range procs {
		log.Printf("Proc ID: %d, status: %d", p, status)
		if isProcAlive(p) == false {
			setProcStatus(db, p, PROC_STOPPED)
		} else {
			setProcStatus(db, p, PROC_ALIVE)
		}
	}
	return db, nil
}
Пример #7
0
func NewLevelDB(path string, config interface{}) (Engine, error) {
	c, ok := config.(*LevelDbConfiguration)
	if !ok {
		return nil, fmt.Errorf("Config is of type %T instead of %T", config, LevelDbConfiguration{})
	}

	// if it wasn't set, set it to 100
	if c.MaxOpenFiles == 0 {
		c.MaxOpenFiles = 100
	}

	// if it wasn't set, set it to 200 MB
	if c.LruCacheSize == 0 {
		c.LruCacheSize = 200 * 1024 * 1024
	}

	// initialize the global cache
	if cache == nil {
		cacheLock.Lock()
		if cache == nil {
			cache = levigo.NewLRUCache(int(c.LruCacheSize))
		}
		cacheLock.Unlock()
	}

	opts := levigo.NewOptions()
	opts.SetCache(cache)
	opts.SetCreateIfMissing(true)
	opts.SetMaxOpenFiles(c.MaxOpenFiles)
	db, err := levigo.Open(path, opts)
	wopts := levigo.NewWriteOptions()
	ropts := levigo.NewReadOptions()
	return LevelDB{db, opts, wopts, ropts, path}, err
}
Пример #8
0
// Open returns a keyvalue DB backed by a LevelDB database at the given
// filepath.  If opts==nil, the DefaultOptions are used.
func Open(path string, opts *Options) (keyvalue.DB, error) {
	if opts == nil {
		opts = DefaultOptions
	}

	options := levigo.NewOptions()
	defer options.Close()
	cache := levigo.NewLRUCache(opts.CacheCapacity)
	options.SetCache(cache)
	options.SetCreateIfMissing(!opts.MustExist)
	if opts.WriteBufferSize > 0 {
		options.SetWriteBufferSize(opts.WriteBufferSize)
	}
	db, err := levigo.Open(path, options)
	if err != nil {
		return nil, fmt.Errorf("could not open LevelDB at %q: %v", path, err)
	}
	largeReadOpts := levigo.NewReadOptions()
	largeReadOpts.SetFillCache(opts.CacheLargeReads)
	return &levelDB{
		db:            db,
		cache:         cache,
		readOpts:      levigo.NewReadOptions(),
		largeReadOpts: largeReadOpts,
		writeOpts:     levigo.NewWriteOptions(),
	}, nil
}
Пример #9
0
func NewStore(path string, sync bool) *Store {
	opts := levigo.NewOptions()
	opts.SetCreateIfMissing(true)

	db, err := levigo.Open(path, opts)
	if err != nil {
		panic(fmt.Sprintf("queued.Store: Unable to open db: %v", err))
	}

	id := 0

	it := db.NewIterator(levigo.NewReadOptions())
	defer it.Close()

	it.SeekToLast()
	if it.Valid() {
		id, err = strconv.Atoi(string(it.Key()))
		if err != nil {
			panic(fmt.Sprintf("queued: Error loading db: %v", err))
		}
	}

	store := &Store{
		id:   id,
		path: path,
		sync: sync,
		db:   db,
	}

	return store
}
Пример #10
0
func NewLevelDBStore(basePath string) LogeStore {

	var opts = levigo.NewOptions()
	opts.SetCreateIfMissing(true)
	db, err := levigo.Open(basePath, opts)

	if err != nil {
		panic(fmt.Sprintf("Can't open DB at %s: %v", basePath, err))
	}

	var store = &levelDBStore{
		basePath: basePath,
		db:       db,
		types:    spack.NewTypeSet(),

		writeQueue: make(chan *levelDBContext),
		flushed:    false,
	}

	store.types.LastTag = ldb_START_TAG
	store.loadTypeMetadata()
	go store.writer()

	return store
}
Пример #11
0
func NewLeveldbCache(dbname string, cacheM int) (*LeveldbCache, error) {
	opts := levigo.NewOptions()
	filter := levigo.NewBloomFilter(10)
	cache := levigo.NewLRUCache(1024 * 1024 * cacheM)
	opts.SetFilterPolicy(filter)
	opts.SetCache(cache)
	opts.SetCreateIfMissing(true)
	opts.SetWriteBufferSize(8 * 1024 * 104) // 8M
	opts.SetCompression(levigo.SnappyCompression)

	if ldb, err := levigo.Open(dbname, opts); err == nil {
		so := levigo.NewReadOptions()
		so.SetFillCache(false)
		return &LeveldbCache{
			db:    ldb,
			fp:    filter,
			cache: cache,
			Ro:    levigo.NewReadOptions(),
			Wo:    levigo.NewWriteOptions(),
			So:    so,
		}, nil
	} else {
		return nil, err
	}
}
Пример #12
0
func NewLevelDbDatastore(dbDir string) (Datastore, error) {
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(ONE_GIGABYTE))
	opts.SetCreateIfMissing(true)
	opts.SetBlockSize(TWO_FIFTY_SIX_KILOBYTES)
	filter := levigo.NewBloomFilter(BLOOM_FILTER_BITS_PER_KEY)
	opts.SetFilterPolicy(filter)
	db, err := levigo.Open(dbDir, opts)
	if err != nil {
		return nil, err
	}

	ro := levigo.NewReadOptions()

	lastIdBytes, err2 := db.Get(ro, NEXT_ID_KEY)
	if err2 != nil {
		return nil, err2
	}

	lastId := uint64(0)
	if lastIdBytes != nil {
		lastId, err2 = binary.ReadUvarint(bytes.NewBuffer(lastIdBytes))
		if err2 != nil {
			return nil, err2
		}
	}

	wo := levigo.NewWriteOptions()

	return &LevelDbDatastore{db: db, lastIdUsed: lastId, readOptions: ro, writeOptions: wo}, nil
}
Пример #13
0
Файл: pt.go Проект: hagna/pt
func NewTree(dbname string) *Tree {
	debug("New tree")
	n := new(Tree)
	n.closed = false
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(3 << 30))
	opts.SetCreateIfMissing(true)
	d, err := levigo.Open(dbname, opts)

	if err != nil {
		Error(err)
	}
	n.DB = d
	if n.Root, err = n.Get(nil, 0); err != nil {
		debug("could not get", err)
		n.Root = &Node{Parent: 0}
		if err := n.Put(nil, n.Root); err != nil {
			Error("could not set", err)
		}
	}
	n.Newid = make(chan int)
	go func() {
		id := n.Root.Parent + 1
		for ; ; id++ {
			n.Newid <- id
			if n.closed {
				break
			}
		}
		log.Println("exiting Newid goroutine")
	}()
	return n
}
Пример #14
0
func (c *cache) open(path string) error {
	opts := levigo.NewOptions()
	opts.SetCreateIfMissing(true)
	if c.options.CacheSizeM > 0 {
		c.cache = levigo.NewLRUCache(c.options.CacheSizeM * 1024 * 1024)
		opts.SetCache(c.cache)
	}
	if c.options.MaxOpenFiles > 0 {
		opts.SetMaxOpenFiles(c.options.MaxOpenFiles)
	}
	if c.options.BlockRestartInterval > 0 {
		opts.SetBlockRestartInterval(c.options.BlockRestartInterval)
	}
	if c.options.WriteBufferSizeM > 0 {
		opts.SetWriteBufferSize(c.options.WriteBufferSizeM * 1024 * 1024)
	}
	if c.options.BlockSizeK > 0 {
		opts.SetBlockSize(c.options.BlockSizeK * 1024)
	}

	db, err := levigo.Open(path, opts)
	if err != nil {
		return err
	}
	c.db = db
	c.wo = levigo.NewWriteOptions()
	c.ro = levigo.NewReadOptions()
	return nil
}
Пример #15
0
func (self *LevelDbShardDatastore) GetOrCreateShard(id uint32) (cluster.LocalShardDb, error) {
	now := time.Now().Unix()
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	db := self.shards[id]
	self.lastAccess[id] = now

	if db != nil {
		self.incrementShardRefCountAndCloseOldestIfNeeded(id)
		return db, nil
	}

	dbDir := self.shardDir(id)

	log.Info("DATASTORE: opening or creating shard %s", dbDir)
	ldb, err := levigo.Open(dbDir, self.levelDbOptions)
	if err != nil {
		return nil, err
	}

	db, err = NewLevelDbShard(ldb, self.pointBatchSize)
	if err != nil {
		return nil, err
	}
	self.shards[id] = db
	self.incrementShardRefCountAndCloseOldestIfNeeded(id)
	return db, nil
}
Пример #16
0
func initDB() (*levigo.DB, error) {
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(3 << 30))
	opts.SetCreateIfMissing(true)
	return levigo.Open(settings.dir+"/db", opts)

}
Пример #17
0
// Will panic if there is a problem with the database.
// Should only be called on server initialization.
func NewDatabase(databaseLocation string) *Database {
	buckets := new(Database)
	buckets.DBMap = make(map[string]*levigo.DB)
	buckets.BaseLocation = databaseLocation

	os.MkdirAll(databaseLocation, 0755)

	files, err := ioutil.ReadDir(databaseLocation)
	if err != nil {
		panic(err)
	}

	for _, file := range files {
		if !file.IsDir() || strings.HasPrefix(file.Name(), "_") {
			continue
		}
		opts := levigo.NewOptions()
		opts.SetCreateIfMissing(true)
		opts.SetCache(levigo.NewLRUCache(4194304))
		buckets.DBMap[file.Name()], err = levigo.Open(path.Join(databaseLocation, file.Name()), opts)
		if err != nil {
			panic(err)
		}
	}
	return buckets
}
Пример #18
0
func (engine *LevelDbEngine) Init(config *proto.DBConfigs) error {
	if config == nil {
		return proto.ErrNoEngineConfig
	}
	if config.LevelDbConfigs == nil {
		config.LevelDbConfigs = DefaultLevelDbConf
	}
	options := levigo.NewOptions()
	// options.SetCreateIfMissing(config.CreateIfMissing)
	options.SetCreateIfMissing(true)
	options.SetParanoidChecks(config.LevelDbConfigs.ParanoidCheck)
	if config.LevelDbConfigs.LRUCacheSize > 0 {
		options.SetCache(levigo.NewLRUCache(config.LevelDbConfigs.LRUCacheSize))
	}
	if config.LevelDbConfigs.BloomFilterLength > 0 {
		options.SetFilterPolicy(levigo.NewBloomFilter(config.LevelDbConfigs.BloomFilterLength))
	}
	engine.config = config
	engine.dbOptions = options
	db, err := levigo.Open(engine.config.DataPath, engine.dbOptions)
	if err != nil {
		return err
	}
	engine.db = db
	return nil
}
Пример #19
0
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
	path, ok := config["path"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify path")
	}
	if path == "" {
		return nil, os.ErrInvalid
	}

	rv := Store{
		path: path,
		opts: levigo.NewOptions(),
		mo:   mo,
	}

	_, err := applyConfig(rv.opts, config)
	if err != nil {
		return nil, err
	}

	rv.db, err = levigo.Open(rv.path, rv.opts)
	if err != nil {
		return nil, err
	}
	return &rv, nil
}
Пример #20
0
func NewLevelDBPersistence(storageRoot string, cacheCapacity, bitsPerBloomFilterEncoded int) (p *LevelDBPersistence, err error) {
	options := levigo.NewOptions()
	options.SetCreateIfMissing(true)
	options.SetParanoidChecks(true)

	cache := levigo.NewLRUCache(cacheCapacity)
	options.SetCache(cache)

	filterPolicy := levigo.NewBloomFilter(bitsPerBloomFilterEncoded)
	options.SetFilterPolicy(filterPolicy)

	storage, err := levigo.Open(storageRoot, options)
	if err != nil {
		return
	}

	readOptions := levigo.NewReadOptions()
	writeOptions := levigo.NewWriteOptions()
	writeOptions.SetSync(true)

	p = &LevelDBPersistence{
		cache:        cache,
		filterPolicy: filterPolicy,
		options:      options,
		readOptions:  readOptions,
		storage:      storage,
		writeOptions: writeOptions,
	}

	return
}
Пример #21
0
func loadVolume(volRoot string) (*volume, error) {
	id, err := getVolId(volRoot)
	if err != nil {
		return nil, err
	}
	dnInfoFile, err := os.Open(volRoot + "/DNINFO")
	defer dnInfoFile.Close()
	if err != nil {
		return nil, err
	}
	d := json.NewDecoder(dnInfoFile)
	dnInfo := maggiefs.DataNodeInfo{}
	d.Decode(dnInfo)

	db, err := levigo.Open(volRoot+"/meta", openOpts)
	if err != nil {
		db.Close()
		return nil, err
	}

	rootFile, err := os.Open(volRoot)
	if err != nil {
		return nil, err
	}
	return &volume{id, volRoot, rootFile, maggiefs.VolumeInfo{id, dnInfo}, db}, nil
}
Пример #22
0
func (ldbs *Store) Open() error {
	var err error
	ldbs.db, err = levigo.Open(ldbs.path, ldbs.opts)
	if err != nil {
		return err
	}
	return nil
}
Пример #23
0
func (dld *DataStoreLoader) Load() error {
	var err error
	// If data.store.clear was set, clear existing data.
	if dld.ClearStored {
		err = dld.clearStored()
		if err != nil {
			return err
		}
	}
	// Make sure the shard directories exist in all cases, with a mkdir -p
	for i := range dld.shards {
		err := os.MkdirAll(dld.shards[i].path, 0777)
		if err != nil {
			return errors.New(fmt.Sprintf("Failed to MkdirAll(%s): %s",
				dld.shards[i].path, err.Error()))
		}
	}
	// Get information about each shard, and verify them.
	dld.LoadShards()
	err = dld.VerifyShardInfos()
	if err != nil {
		return err
	}
	if dld.shards[0].ldb != nil {
		dld.lg.Infof("Loaded %d leveldb instances with "+
			"DaemonId of 0x%016x\n", len(dld.shards),
			dld.shards[0].info.DaemonId)
	} else {
		// Create leveldb instances if needed.
		rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
		daemonId := uint64(rnd.Int63())
		dld.lg.Infof("Initializing %d leveldb instances with a new "+
			"DaemonId of 0x%016x\n", len(dld.shards), daemonId)
		dld.openOpts.SetCreateIfMissing(true)
		for i := range dld.shards {
			shd := dld.shards[i]
			shd.ldb, err = levigo.Open(shd.path, shd.dld.openOpts)
			if err != nil {
				return errors.New(fmt.Sprintf("levigo.Open(%s) failed to "+
					"create the shard: %s", shd.path, err.Error()))
			}
			info := &ShardInfo{
				LayoutVersion: CURRENT_LAYOUT_VERSION,
				DaemonId:      daemonId,
				TotalShards:   uint32(len(dld.shards)),
				ShardIndex:    uint32(i),
			}
			err = shd.writeShardInfo(info)
			if err != nil {
				return errors.New(fmt.Sprintf("levigo.Open(%s) failed to "+
					"write shard info: %s", shd.path, err.Error()))
			}
			dld.lg.Infof("Shard %s initialized with ShardInfo %s \n",
				shd.path, asJson(info))
		}
	}
	return nil
}
Пример #24
0
func TestTrigger(t *testing.T) {
	opts := levigo.NewOptions()
	levigo.DestroyDatabase("test.ldb", opts)
	// opts.SetCache(levigo.NewLRUCache(3<<30))
	opts.SetCreateIfMissing(true)
	db, err := levigo.Open("test.ldb", opts)
	if err != nil {
		t.Fatal(err)
	}

	ro := levigo.NewReadOptions()
	wo := levigo.NewWriteOptions()

	sub1 := sublevel.Sublevel(db, "input")
	index := sublevel.Sublevel(db, "index")
	job := sublevel.Sublevel(db, "job")

	task := Trigger(sub1, job, func(key, value []byte) []byte {
		if strings.HasPrefix(string(key), "Doc_") {
			return key
		}
		return nil
	}, func(key, value []byte) bool {
		doc := make(map[string]string)
		err := json.Unmarshal(value, &doc)
		if err != nil {
			t.Fatal(err)
		}
		index.Put(wo, []byte(doc["id"]), []byte(doc["number"]))
		// Make sure that the next task invocation comes in concurrently to this one
		time.Sleep(300 * time.Millisecond)
		return true
	})

	sub1.Put(wo, []byte("foobar"), []byte("do-not-process"))

	// Two put operations which will both trigger the task for the same taskKey.
	sub1.Put(wo, []byte("Doc_1"), []byte("{\"id\":\"01234\", \"number\": \"42\"}"))
	sub1.Put(wo, []byte("Doc_1"), []byte("{\"id\":\"01234\", \"number\": \"43\"}"))

	val, err := sub1.Get(ro, []byte("Doc_1"))
	if err != nil || string(val) != "{\"id\":\"01234\", \"number\": \"43\"}" {
		t.Fatal(err, string(val))
	}

	time.Sleep(800 * time.Millisecond)

	val, err = index.Get(ro, []byte("01234"))
	if err != nil || string(val) != "43" {
		t.Fatal(err, string(val))
	}

	task.Close()

	ro.Close()
	wo.Close()
	db.Close()
}
Пример #25
0
func NewLevelDbDatastore(dbDir string) (Datastore, error) {
	mainDbDir := filepath.Join(dbDir, DATABASE_DIR)
	requestLogDir := filepath.Join(dbDir, REQUEST_LOG_BASE_DIR)

	err := os.MkdirAll(mainDbDir, 0744)
	if err != nil {
		return nil, err
	}
	previousLog, err := NewRequestLogDb(getRequestLogDirForDate(requestLogDir, time.Now().Add(-time.Hour*24)))
	if err != nil {
		return nil, err
	}
	currentLog, err := NewRequestLogDb(getRequestLogDirForDate(requestLogDir, time.Now()))
	if err != nil {
		return nil, err
	}

	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(ONE_GIGABYTE))
	opts.SetCreateIfMissing(true)
	opts.SetBlockSize(TWO_FIFTY_SIX_KILOBYTES)
	filter := levigo.NewBloomFilter(BLOOM_FILTER_BITS_PER_KEY)
	opts.SetFilterPolicy(filter)
	db, err := levigo.Open(dbDir, opts)
	if err != nil {
		return nil, err
	}

	ro := levigo.NewReadOptions()

	lastIdBytes, err2 := db.Get(ro, NEXT_ID_KEY)
	if err2 != nil {
		return nil, err2
	}

	lastId := uint64(0)
	if lastIdBytes != nil {
		lastId, err2 = binary.ReadUvarint(bytes.NewBuffer(lastIdBytes))
		if err2 != nil {
			return nil, err2
		}
	}

	wo := levigo.NewWriteOptions()

	leveldbStore := &LevelDbDatastore{
		db:                 db,
		lastIdUsed:         lastId,
		readOptions:        ro,
		writeOptions:       wo,
		requestLogDir:      requestLogDir,
		currentRequestLog:  currentLog,
		previousRequestLog: previousLog}

	go leveldbStore.periodicallyRotateRequestLog()

	return leveldbStore, nil
}
Пример #26
0
func (conn *DbConnection) open(path string) error {
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(3 << 10))
	opts.SetCreateIfMissing(true)
	var err error
	conn.db, err = levigo.Open(path, opts)
	conn.ro = levigo.NewReadOptions()
	conn.wo = levigo.NewWriteOptions()
	return err
}
Пример #27
0
// initializes a namedata
func NewNameData(dataDir string) (*NameData, error) {
	opts := OpenOpts
	// todo configure caching
	// todo investigate turning off compression
	inodb, err := levigo.Open(dataDir+"/"+dir_inodb, opts)
	if err != nil {
		return nil, err
	}

	ret := &NameData{}
	ret.inodb = inodb
	ret.inodeStripeLock = make(map[uint64]*sync.Mutex)
	for i := uint64(0); i < STRIPE_SIZE; i++ {
		ret.inodeStripeLock[i] = &sync.Mutex{}
	}
	ret.counterLock = &sync.Mutex{}
	ret.counterdb, err = levigo.Open(dataDir+"/"+dir_counters, opts)
	return ret, err
}
Пример #28
0
func (buckets *Database) GetBucket(name string) (*levigo.DB, error) {
	if db, ok := buckets.DBMap[name]; ok {
		return db, nil
	}
	opts := levigo.NewOptions()
	opts.SetCreateIfMissing(true)
	opts.SetCache(levigo.NewLRUCache(4194304))
	var err error
	buckets.DBMap[name], err = levigo.Open(path.Join(buckets.BaseLocation, name), opts)
	return buckets.DBMap[name], err
}
Пример #29
0
/*
CreateDB creates a db at provided dbpath.
*/
func CreateDB(dbpath string) *levigo.DB {
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(1 << 10))
	opts.SetCreateIfMissing(true)
	db, err := levigo.Open(dbpath, opts)
	if err != nil {
		errMsg := fmt.Sprintf("DB %s Creation failed. %q", dbpath, err)
		golerror.Boohoo(errMsg, true)
	}
	return db
}
Пример #30
0
func storageInit(path string) {
	opts := levigo.NewOptions()
	//opts.SetCache(levigo.NewLRUCache(32*1024*1024))  // 32mb in-mem cache
	//opts.SetBlockSize(4096) // 4096 is the default block size
	//opts.SetCompression(levigo.NoCompression) // default is levigo.SnappyCompression
	opts.SetCreateIfMissing(true)
	db, err := levigo.Open(path, opts)
	if err != nil {
		panic("error opening DB at " + path + ": " + err.Error())
	}
	storeDB = db
}