func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) {
var qs TripleStore
var err error
qs.path = path
cache_size := DefaultCacheSize
if val, ok := options.IntKey("cache_size_mb"); ok {
cache_size = val
}
qs.dbOpts = &opt.Options{
BlockCache: cache.NewLRUCache(cache_size * opt.MiB),
}
qs.dbOpts.ErrorIfMissing = true
write_buffer_mb := DefaultWriteBufferSize
if val, ok := options.IntKey("write_buffer_mb"); ok {
write_buffer_mb = val
}
qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
qs.writeopts = &opt.WriteOptions{
Sync: false,
}
qs.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return nil, err
}
qs.db = db
glog.Infoln(qs.GetStats())
err = qs.getMetadata()
if err != nil {
return nil, err
}
return &qs, nil
}
func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) {
var ts TripleStore
ts.path = path
cache_size := DefaultCacheSize
if val, ok := options.IntKey("cache_size_mb"); ok {
cache_size = val
}
ts.dbOpts = &opt.Options{
BlockCache: cache.NewLRUCache(cache_size * opt.MiB),
}
ts.dbOpts.ErrorIfMissing = true
write_buffer_mb := DefaultWriteBufferSize
if val, ok := options.IntKey("write_buffer_mb"); ok {
write_buffer_mb = val
}
ts.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
ts.hasher = sha1.New()
ts.writeopts = &opt.WriteOptions{
Sync: false,
}
ts.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(ts.path, ts.dbOpts)
if err != nil {
panic("Error, couldn't open! " + err.Error())
}
ts.db = db
glog.Infoln(ts.GetStats())
ts.getSize()
return &ts, nil
}
func (s *session) setOptions(o *opt.Options) {
s.o = &opt.Options{}
if o != nil {
*s.o = *o
}
// Alternative filters.
if filters := o.GetAltFilters(); len(filters) > 0 {
s.o.AltFilters = make([]filter.Filter, len(filters))
for i, filter := range filters {
s.o.AltFilters[i] = &iFilter{filter}
}
}
// Block cache.
switch o.GetBlockCache() {
case nil:
s.o.BlockCache = cache.NewLRUCache(opt.DefaultBlockCacheSize)
case opt.NoCache:
s.o.BlockCache = nil
}
// Comparer.
s.cmp = &iComparer{o.GetComparer()}
s.o.Comparer = s.cmp
// Filter.
if filter := o.GetFilter(); filter != nil {
s.o.Filter = &iFilter{filter}
}
}
func (s *session) setOptions(o *opt.Options) {
no := dupOptions(o)
// Alternative filters.
if filters := o.GetAltFilters(); len(filters) > 0 {
no.AltFilters = make([]filter.Filter, len(filters))
for i, filter := range filters {
no.AltFilters[i] = &iFilter{filter}
}
}
// Block cache.
switch o.GetBlockCache() {
case nil:
no.BlockCache = cache.NewLRUCache(o.GetBlockCacheSize())
case opt.NoCache:
no.BlockCache = nil
}
// Comparer.
s.icmp = &iComparer{o.GetComparer()}
no.Comparer = s.icmp
// Filter.
if filter := o.GetFilter(); filter != nil {
no.Filter = &iFilter{filter}
}
s.o = &cachedOptions{Options: no}
s.o.cache()
}
func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
h := new(dbCorruptHarness)
h.init(t, &opt.Options{
Flag: opt.OFCreateIfMissing,
BlockCache: cache.NewLRUCache(100),
})
return h
}
func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
h := new(dbCorruptHarness)
h.init(t, &opt.Options{
BlockCache: cache.NewLRUCache(100),
Strict: opt.StrictJournalChecksum,
})
return h
}
func (o *iOptions) sanitize() {
if p := o.GetBlockCache(); p == nil {
o.Options.SetBlockCache(cache.NewLRUCache(opt.DefaultBlockCacheSize))
}
if p := o.GetFilter(); p != nil {
o.Options.SetFilter(&iFilter{p})
}
}
// Creates new initialized table ops instance.
func newTableOps(s *session, cacheCap int) *tOps {
c := cache.NewLRUCache(cacheCap)
return &tOps{
s: s,
cache: c,
cacheNS: c.GetNamespace(0),
bpool: util.NewBufferPool(s.o.GetBlockSize() + 5),
}
}
func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
return newDbCorruptHarnessWopt(t, &opt.Options{
<<<<<<< HEAD
BlockCacheCapacity: 100,
Strict: opt.StrictJournalChecksum,
=======
BlockCache: cache.NewLRUCache(100),
Strict: opt.StrictJournalChecksum,
>>>>>>> 9bca75c48d6c31becfbb127702b425e7226052e3
})
func (o *Options) GetBlockCache() cache.Cache {
o.mu.Lock()
defer o.mu.Unlock()
if o == nil {
return nil
}
if o.BlockCache == nil {
o.BlockCache = cache.NewLRUCache(8 << 20)
}
return o.BlockCache
}
func (o *iOptions) sanitize() {
if p := o.GetBlockCache(); p == nil {
o.Options.SetBlockCache(cache.NewLRUCache(opt.DefaultBlockCacheSize))
}
for _, p := range o.GetAltFilters() {
o.InsertAltFilter(p)
}
if p := o.GetFilter(); p != nil {
o.SetFilter(p)
}
}
func newOptions(cfg *config.LevelDBConfig) *opt.Options {
opts := &opt.Options{}
opts.ErrorIfMissing = false
cfg.Adjust()
opts.BlockCache = cache.NewLRUCache(cfg.CacheSize)
//we must use bloomfilter
opts.Filter = filter.NewBloomFilter(defaultFilterBits)
if !cfg.Compression {
opts.Compression = opt.NoCompression
} else {
opts.Compression = opt.SnappyCompression
}
opts.BlockSize = cfg.BlockSize
opts.WriteBuffer = cfg.WriteBufferSize
return opts
}
func TestCorruptDB_MissingManifest(t *testing.T) {
rnd := rand.New(rand.NewSource(0x0badda7a))
h := newDbCorruptHarnessWopt(t, &opt.Options{
BlockCache: cache.NewLRUCache(100),
Strict: opt.StrictJournalChecksum,
WriteBuffer: 1000 * 60,
})
h.build(1000)
h.compactMem()
h.buildShuffled(1000, rnd)
h.compactMem()
h.deleteRand(500, 1000, rnd)
h.compactMem()
h.buildShuffled(1000, rnd)
h.compactMem()
h.deleteRand(500, 1000, rnd)
h.compactMem()
h.buildShuffled(1000, rnd)
h.compactMem()
h.closeDB()
h.stor.SetIgnoreOpenErr(storage.TypeManifest)
h.removeAll(storage.TypeManifest)
h.openAssert(false)
h.stor.SetIgnoreOpenErr(0)
h.recover()
h.check(1000, 1000)
h.build(1000)
h.compactMem()
h.compactRange("", "")
h.closeDB()
h.recover()
h.check(1000, 1000)
h.close()
}
func newTableOps(s *session, cacheCap int) *tOps {
c := cache.NewLRUCache(cacheCap)
ns := c.GetNamespace(0)
return &tOps{s, c, ns}
}
func NewLevelDataStore(conf map[string]string) (DataStore, error) {
encoder := LEVEL_ENCODER_MSGPAK
path := ""
writeBuffer := 16777216
readCache := 16777216
if v, ok := conf["debug_leveldb_encoder"]; ok && v != "" {
switch v {
case LEVEL_ENCODER_GOB, LEVEL_ENCODER_MSGPAK:
encoder = v
}
}
if v, ok := conf["leveldb_path"]; ok && v != "" {
path = v
}
if v, ok := conf["leveldb_cache"]; ok && v != "" {
if n, err := strconv.Atoi(v); err == nil {
if n < 8388608 {
n = 8388608
}
readCache = n
}
}
if v, ok := conf["leveldb_writebuffer"]; ok && v != "" {
if n, err := strconv.Atoi(v); err == nil {
if n < 4194304 {
n = 4194304
}
writeBuffer = n
}
}
o := &opt.Options{
Filter: filter.NewBloomFilter(10),
WriteBuffer: writeBuffer,
BlockCache: cache.NewLRUCache(readCache),
}
lds := new(LevelDataStore)
lds.dbPath = path
if lds.dbPath == "" {
if p, err := ioutil.TempDir("", "tblvl"); err == nil {
lds.dbPath = p
lds.isTemp = true
} else {
return nil, err
}
}
switch encoder {
case LEVEL_ENCODER_MSGPAK:
lds.encoder = msgpackEncoder{}
default:
encoder = LEVEL_ENCODER_GOB
lds.encoder = gobEncoder{}
}
if db, err := leveldb.OpenFile(lds.dbPath, o); err == nil {
if v, e := db.Get([]byte("_VERSION"), nil); e == nil {
if v[0] != LEVELDB_VERSION {
return nil, fmt.Errorf("Invalid LevelDB Version (%d)", v)
}
} else if e != util.ErrNotFound {
return nil, e
}
if v, e := db.Get([]byte("_ENC"), nil); e == nil {
if string(v) != encoder {
return nil, fmt.Errorf("Unexpected LevelDB encoding (Expected '%s', Got '%s')", encoder, v)
}
} else if e != util.ErrNotFound {
return nil, e
}
db.Put([]byte("_VERSION"), []byte{LEVELDB_VERSION}, nil)
db.Put([]byte("_ENC"), []byte(encoder), nil)
lds.db = db
return lds, nil
} else {
return nil, err
}
}
func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
return newDbCorruptHarnessWopt(t, &opt.Options{
BlockCache: cache.NewLRUCache(100),
Strict: opt.StrictJournalChecksum,
})
}
func TestDb_Concurrent(t *testing.T) {
const n, secs, maxkey = 4, 2, 1000
runtime.GOMAXPROCS(n)
runAllOpts(t, func(h *dbHarness) {
var wg sync.WaitGroup
var stop uint32
var cnt [n]uint32
for i := 0; i < n; i++ {
wg.Add(1)
go func(i int) {
var put, get, found uint
defer func() {
t.Logf("goroutine %d stopped after %d ops, put=%d get=%d found=%d missing=%d",
i, cnt[i], put, get, found, get-found)
wg.Done()
}()
rnd := rand.New(rand.NewSource(int64(1000 + i)))
for atomic.LoadUint32(&stop) == 0 {
x := cnt[i]
k := rnd.Intn(maxkey)
kstr := fmt.Sprintf("%016d", k)
if (rnd.Int() % 2) > 0 {
put++
h.put(kstr, fmt.Sprintf("%d.%d.%-1000d", k, i, x))
} else {
get++
v, err := h.db.Get([]byte(kstr), h.ro)
if err == nil {
found++
rk, ri, rx := 0, -1, uint32(0)
fmt.Sscanf(string(v), "%d.%d.%d", &rk, &ri, &rx)
if rk != k {
t.Errorf("invalid key want=%d got=%d", k, rk)
}
if ri < 0 || ri >= n {
t.Error("invalid goroutine number: ", ri)
} else {
tx := atomic.LoadUint32(&(cnt[ri]))
if rx > tx {
t.Errorf("invalid seq number, %d > %d ", rx, tx)
}
}
} else if err != errors.ErrNotFound {
t.Error("Get: got error: ", err)
return
}
}
atomic.AddUint32(&cnt[i], 1)
}
}(i)
}
for i := 0; i < secs; i++ {
h.oo.SetBlockCache(cache.NewLRUCache(rand.Int() % (opt.DefaultBlockCacheSize * 2)))
time.Sleep(time.Second)
}
atomic.StoreUint32(&stop, 1)
wg.Wait()
})
runtime.GOMAXPROCS(1)
}
func (ulevel) NewLRUCache(capacity int) level.UnderlyingCache {
return che{
Cache: C.NewLRUCache(capacity),
}
}