func (db *LevelDB) metadataExists() (bool, error) {
var ctx storage.MetadataContext
keyBeg, keyEnd := ctx.KeyRange()
dvid.StartCgo()
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
it.Seek(keyBeg)
for {
if it.Valid() {
// Did we pass the final key?
if bytes.Compare(it.Key(), keyEnd) > 0 {
break
}
return true, nil
}
break
}
if err := it.GetError(); err != nil {
return false, err
}
dvid.Infof("No metadata found for %s...\n", db)
return false, nil
}
// unversionedRange sends a range of key-value pairs down a channel.
func (db *LevelDB) unversionedRange(ctx storage.Context, begTKey, endTKey storage.TKey, ch chan errorableKV, done <-chan struct{}, keysOnly bool) {
dvid.StartCgo()
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
// Apply context if applicable
begKey := ctx.ConstructKey(begTKey)
endKey := ctx.ConstructKey(endTKey)
// fmt.Printf("unversionedRange():\n")
// fmt.Printf(" index beg: %v\n", kStart)
// fmt.Printf(" index end: %v\n", kEnd)
// fmt.Printf(" key start: %v\n", keyBeg)
// fmt.Printf(" key end: %v\n", keyEnd)
var itValue []byte
it.Seek(begKey)
for {
if it.Valid() {
// fmt.Printf("unversioned found key %v, %d bytes value\n", it.Key(), len(it.Value()))
if !keysOnly {
itValue = it.Value()
storage.StoreValueBytesRead <- len(itValue)
}
itKey := it.Key()
storage.StoreKeyBytesRead <- len(itKey)
// Did we pass the final key?
if bytes.Compare(itKey, endKey) > 0 {
break
}
select {
case <-done:
ch <- errorableKV{nil, nil}
return
case ch <- errorableKV{&storage.KeyValue{K: itKey, V: itValue}, nil}:
it.Next()
}
} else {
break
}
}
if err := it.GetError(); err != nil {
ch <- errorableKV{nil, err}
} else {
ch <- errorableKV{nil, nil}
}
return
}
// RawRangeQuery sends a range of full keys. This is to be used for low-level data
// retrieval like DVID-to-DVID communication and should not be used by data type
// implementations if possible. A nil is sent down the channel when the
// range is complete.
func (db *LevelDB) RawRangeQuery(kStart, kEnd storage.Key, keysOnly bool, out chan *storage.KeyValue, cancel <-chan struct{}) error {
if db == nil {
return fmt.Errorf("Can't call RawRangeQuery on nil LevelDB")
}
dvid.StartCgo()
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
var itValue []byte
it.Seek(kStart)
for {
if it.Valid() {
if !keysOnly {
itValue = it.Value()
storage.StoreValueBytesRead <- len(itValue)
}
itKey := it.Key()
storage.StoreKeyBytesRead <- len(itKey)
// Did we pass the final key?
if bytes.Compare(itKey, kEnd) > 0 {
break
}
kv := storage.KeyValue{itKey, itValue}
select {
case out <- &kv:
case <-cancel:
return nil
}
//out <- &kv
it.Next()
} else {
break
}
}
out <- nil
if err := it.GetError(); err != nil {
return err
}
return nil
}
// getSingleKeyVersions returns all versions of a key. These key-value pairs will be sorted
// in ascending key order and could include a tombstone key.
func (db *LevelDB) getSingleKeyVersions(vctx storage.VersionedCtx, tk []byte) ([]*storage.KeyValue, error) {
dvid.StartCgo()
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
values := []*storage.KeyValue{}
begKey, err := vctx.MinVersionKey(tk)
if err != nil {
return nil, err
}
endKey, err := vctx.MaxVersionKey(tk)
if err != nil {
return nil, err
}
it.Seek(begKey)
for {
if it.Valid() {
itKey := it.Key()
storage.StoreKeyBytesRead <- len(itKey)
if bytes.Compare(itKey, endKey) > 0 {
// log.Printf("key past %v\n", kEnd)
return values, nil
}
itValue := it.Value()
// log.Printf("got value of length %d\n", len(itValue))
storage.StoreValueBytesRead <- len(itValue)
values = append(values, &storage.KeyValue{itKey, itValue})
it.Next()
} else {
err = it.GetError()
// log.Printf("iteration done, err = %v\n", err)
if err == nil {
return values, nil
}
return nil, err
}
}
}
// SendRange sends a range of full keys. This is to be used for low-level data
// retrieval like DVID-to-DVID communication and should not be used by data type
// implementations if possible. A nil is sent down the channel when the
// range is complete.
func (db *LevelDB) SendRange(kStart, kEnd storage.Key, keysOnly bool, out chan *storage.KeyValue) error {
dvid.StartCgo()
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
var itValue []byte
it.Seek(kStart)
for {
if it.Valid() {
if !keysOnly {
itValue = it.Value()
storage.StoreValueBytesRead <- len(itValue)
}
itKey := it.Key()
storage.StoreKeyBytesRead <- len(itKey)
// Did we pass the final key?
if bytes.Compare(itKey, kEnd) > 0 {
break
}
kv := storage.KeyValue{itKey, itValue}
out <- &kv
it.Next()
} else {
break
}
}
out <- nil
if err := it.GetError(); err != nil {
return err
}
return nil
}
// versionedRange sends a range of key-value pairs for a particular version down a channel.
func (db *LevelDB) versionedRange(vctx storage.VersionedCtx, begTKey, endTKey storage.TKey, ch chan errorableKV, done <-chan struct{}, keysOnly bool) {
dvid.StartCgo()
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
minKey, err := vctx.MinVersionKey(begTKey)
if err != nil {
ch <- errorableKV{nil, err}
return
}
maxKey, err := vctx.MaxVersionKey(endTKey)
if err != nil {
ch <- errorableKV{nil, err}
return
}
values := []*storage.KeyValue{}
maxVersionKey, err := vctx.MaxVersionKey(begTKey)
if err != nil {
ch <- errorableKV{nil, err}
return
}
// log.Printf(" minKey %v\n", minKey)
// log.Printf(" maxKey %v\n", maxKey)
// log.Printf(" maxVersionKey %v\n", maxVersionKey)
it.Seek(minKey)
var itValue []byte
for {
select {
case <-done: // only happens if we don't care about rest of data.
ch <- errorableKV{nil, nil}
return
default:
}
if it.Valid() {
if !keysOnly {
itValue = it.Value()
storage.StoreValueBytesRead <- len(itValue)
}
itKey := it.Key()
// log.Printf(" +++valid key %v\n", itKey)
storage.StoreKeyBytesRead <- len(itKey)
// Did we pass all versions for last key read?
if bytes.Compare(itKey, maxVersionKey) > 0 {
indexBytes, err := storage.TKeyFromKey(itKey)
if err != nil {
ch <- errorableKV{nil, err}
return
}
maxVersionKey, err = vctx.MaxVersionKey(indexBytes)
if err != nil {
ch <- errorableKV{nil, err}
return
}
// log.Printf("->maxVersionKey %v (transmitting %d values)\n", maxVersionKey, len(values))
sendKV(vctx, values, ch)
values = []*storage.KeyValue{}
}
// Did we pass the final key?
if bytes.Compare(itKey, maxKey) > 0 {
if len(values) > 0 {
sendKV(vctx, values, ch)
}
ch <- errorableKV{nil, nil}
return
}
// log.Printf("Appending value with key %v\n", itKey)
values = append(values, &storage.KeyValue{K: itKey, V: itValue})
it.Next()
} else {
if err = it.GetError(); err != nil {
ch <- errorableKV{nil, err}
} else {
sendKV(vctx, values, ch)
ch <- errorableKV{nil, nil}
}
return
}
}
}
func getOptions(config dvid.Config) (*leveldbOptions, error) {
opt := &leveldbOptions{
Options: levigo.NewOptions(),
ReadOptions: levigo.NewReadOptions(),
WriteOptions: levigo.NewWriteOptions(),
env: levigo.NewDefaultEnv(),
}
opt.WriteOptions.SetSync(DefaultSync) // Huge performance penalty to set sync to true
// Set flags based on create parameter
opt.SetCreateIfMissing(true)
opt.SetErrorIfExists(false)
// Create associated data structures with default values
bloomBits, found, err := config.GetInt("BloomFilterBitsPerKey")
if err != nil {
return nil, err
}
if !found {
bloomBits = DefaultBloomBits
}
opt.SetBloomFilterBitsPerKey(bloomBits)
cacheSize, found, err := config.GetInt("CacheSize")
if err != nil {
return nil, err
}
if !found {
cacheSize = DefaultCacheSize
} else {
cacheSize *= dvid.Mega
}
dvid.Infof("leveldb cache size: %s\n",
humanize.Bytes(uint64(cacheSize)))
opt.SetLRUCacheSize(cacheSize)
writeBufferSize, found, err := config.GetInt("WriteBufferSize")
if err != nil {
return nil, err
}
if !found {
writeBufferSize = DefaultWriteBufferSize
} else {
writeBufferSize *= dvid.Mega
}
dvid.Infof("leveldb write buffer size: %s\n",
humanize.Bytes(uint64(writeBufferSize)))
opt.SetWriteBufferSize(writeBufferSize)
maxOpenFiles, found, err := config.GetInt("MaxOpenFiles")
if err != nil {
return nil, err
}
if !found {
maxOpenFiles = DefaultMaxOpenFiles
}
opt.SetMaxOpenFiles(maxOpenFiles)
blockSize, found, err := config.GetInt("BlockSize")
if err != nil {
return nil, err
}
if !found {
blockSize = DefaultBlockSize
}
opt.SetBlockSize(blockSize)
opt.SetInfoLog(nil)
opt.SetParanoidChecks(false)
//opt.SetBlockRestartInterval(8)
// Don't bother with compression on leveldb side because it will be
// selectively applied on DVID side. We may return and then transmit
// Snappy-compressed data without ever decompressing on server-side.
opt.SetCompression(levigo.NoCompression) // (levigo.SnappyCompression)
return opt, nil
}
func (db *LevelDB) deleteAllVersions(ctx storage.Context) error {
dvid.StartCgo()
var err error
var minKey, maxKey storage.Key
vctx, versioned := ctx.(storage.VersionedCtx)
if versioned {
// Don't have to worry about tombstones. Delete all keys from all versions for this instance id.
minTKey := storage.MinTKey(storage.TKeyMinClass)
maxTKey := storage.MaxTKey(storage.TKeyMaxClass)
minKey, err = vctx.MinVersionKey(minTKey)
if err != nil {
return err
}
maxKey, err = vctx.MaxVersionKey(maxTKey)
if err != nil {
return err
}
} else {
minKey, maxKey = ctx.KeyRange()
}
const BATCH_SIZE = 10000
batch := db.NewBatch(ctx).(*goBatch)
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
numKV := 0
it.Seek(minKey)
for {
if err := it.GetError(); err != nil {
return fmt.Errorf("Error iterating during DeleteAll for %s: %v", ctx, err)
}
if it.Valid() {
itKey := it.Key()
storage.StoreKeyBytesRead <- len(itKey)
// Did we pass the final key?
if bytes.Compare(itKey, maxKey) > 0 {
break
}
batch.WriteBatch.Delete(itKey)
if (numKV+1)%BATCH_SIZE == 0 {
if err := batch.Commit(); err != nil {
dvid.Criticalf("Error on batch commit of DeleteAll at key-value pair %d: %v\n", numKV, err)
return fmt.Errorf("Error on batch commit of DeleteAll at key-value pair %d: %v", numKV, err)
}
batch = db.NewBatch(ctx).(*goBatch)
dvid.Debugf("Deleted %d key-value pairs in ongoing DELETE ALL for %s.\n", numKV+1, ctx)
}
numKV++
it.Next()
} else {
break
}
}
if numKV%BATCH_SIZE != 0 {
if err := batch.Commit(); err != nil {
dvid.Criticalf("Error on last batch commit of DeleteAll: %v\n", err)
return fmt.Errorf("Error on last batch commit of DeleteAll: %v", err)
}
}
dvid.Debugf("Deleted %d key-value pairs via DELETE ALL for %s.\n", numKV, ctx)
return nil
}
func (db *LevelDB) deleteSingleVersion(vctx storage.VersionedCtx) error {
dvid.StartCgo()
minTKey := storage.MinTKey(storage.TKeyMinClass)
maxTKey := storage.MaxTKey(storage.TKeyMaxClass)
minKey, err := vctx.MinVersionKey(minTKey)
if err != nil {
return err
}
maxKey, err := vctx.MaxVersionKey(maxTKey)
if err != nil {
return err
}
const BATCH_SIZE = 10000
batch := db.NewBatch(vctx).(*goBatch)
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
numKV := 0
it.Seek(minKey)
deleteVersion := vctx.VersionID()
for {
if err := it.GetError(); err != nil {
return fmt.Errorf("Error iterating during DeleteAll for %s: %v", vctx, err)
}
if it.Valid() {
itKey := it.Key()
storage.StoreKeyBytesRead <- len(itKey)
// Did we pass the final key?
if bytes.Compare(itKey, maxKey) > 0 {
break
}
_, v, _, err := storage.DataKeyToLocalIDs(itKey)
if err != nil {
return fmt.Errorf("Error on DELETE ALL for version %d: %v", vctx.VersionID(), err)
}
if v == deleteVersion {
batch.WriteBatch.Delete(itKey)
if (numKV+1)%BATCH_SIZE == 0 {
if err := batch.Commit(); err != nil {
dvid.Criticalf("Error on batch commit of DeleteAll at key-value pair %d: %v\n", numKV, err)
return fmt.Errorf("Error on batch commit of DeleteAll at key-value pair %d: %v", numKV, err)
}
batch = db.NewBatch(vctx).(*goBatch)
dvid.Debugf("Deleted %d key-value pairs in ongoing DELETE ALL for %s.\n", numKV+1, vctx)
}
numKV++
}
it.Next()
} else {
break
}
}
if numKV%BATCH_SIZE != 0 {
if err := batch.Commit(); err != nil {
dvid.Criticalf("Error on last batch commit of DeleteAll: %v\n", err)
return fmt.Errorf("Error on last batch commit of DeleteAll: %v", err)
}
}
dvid.Debugf("Deleted %d key-value pairs via DELETE ALL for %s.\n", numKV, vctx)
return nil
}
// DeleteAll deletes all key-value associated with a context (data instance and version).
func (db *LevelDB) DeleteAll(ctx storage.Context, allVersions bool) error {
if ctx == nil {
return fmt.Errorf("Received nil context in Delete()")
}
dvid.StartCgo()
// Don't have to worry about tombstones. Delete all keys from all versions for this instance id.
minTKey := storage.MinTKey(storage.TKeyMinClass)
maxTKey := storage.MaxTKey(storage.TKeyMaxClass)
vctx, ok := ctx.(storage.VersionedCtx)
if !ok {
return fmt.Errorf("Non-versioned context passed to DELETE ALL VERSIONS in basholeveldb driver: %v", ctx)
}
minKey, err := vctx.MinVersionKey(minTKey)
if err != nil {
return err
}
maxKey, err := vctx.MaxVersionKey(maxTKey)
if err != nil {
return err
}
const BATCH_SIZE = 10000
batch := db.NewBatch(ctx).(*goBatch)
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
numKV := 0
it.Seek(minKey)
deleteVersion := ctx.VersionID()
for {
if it.Valid() {
itKey := it.Key()
storage.StoreKeyBytesRead <- len(itKey)
// Did we pass the final key?
if bytes.Compare(itKey, maxKey) > 0 {
break
}
if !allVersions {
_, v, _, err := storage.DataKeyToLocalIDs(itKey)
if err != nil {
return fmt.Errorf("Error on DELETE ALL for version %d: %v", ctx.VersionID(), err)
}
if v != deleteVersion {
it.Next()
continue
}
}
batch.WriteBatch.Delete(itKey)
if (numKV+1)%BATCH_SIZE == 0 {
if err := batch.Commit(); err != nil {
batch.Close()
return fmt.Errorf("Error on DELETE ALL at key-value pair %d: %v", numKV, err)
}
batch = db.NewBatch(ctx).(*goBatch)
}
numKV++
it.Next()
} else {
break
}
}
if numKV%BATCH_SIZE != 0 {
if err := batch.Commit(); err != nil {
batch.Close()
return fmt.Errorf("Error on last batch DELETE: %v\n", err)
}
}
dvid.Debugf("Deleted %d key-value pairs via DELETE ALL for %s.\n", numKV, ctx)
return nil
}