Esempio n. 1
0
func (db *LevelDB) metadataExists() (bool, error) {
	var ctx storage.MetadataContext
	keyBeg, keyEnd := ctx.KeyRange()
	dvid.StartCgo()
	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	it.Seek(keyBeg)
	for {
		if it.Valid() {
			// Did we pass the final key?
			if bytes.Compare(it.Key(), keyEnd) > 0 {
				break
			}
			return true, nil
		}
		break
	}
	if err := it.GetError(); err != nil {
		return false, err
	}
	dvid.Infof("No metadata found for %s...\n", db)
	return false, nil
}
Esempio n. 2
0
// unversionedRange sends a range of key-value pairs down a channel.
func (db *LevelDB) unversionedRange(ctx storage.Context, begTKey, endTKey storage.TKey, ch chan errorableKV, done <-chan struct{}, keysOnly bool) {
	dvid.StartCgo()
	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	// Apply context if applicable
	begKey := ctx.ConstructKey(begTKey)
	endKey := ctx.ConstructKey(endTKey)

	// fmt.Printf("unversionedRange():\n")
	// fmt.Printf("    index beg: %v\n", kStart)
	// fmt.Printf("    index end: %v\n", kEnd)
	// fmt.Printf("    key start: %v\n", keyBeg)
	// fmt.Printf("      key end: %v\n", keyEnd)

	var itValue []byte
	it.Seek(begKey)
	for {
		if it.Valid() {
			// fmt.Printf("unversioned found key %v, %d bytes value\n", it.Key(), len(it.Value()))
			if !keysOnly {
				itValue = it.Value()
				storage.StoreValueBytesRead <- len(itValue)
			}
			itKey := it.Key()
			storage.StoreKeyBytesRead <- len(itKey)
			// Did we pass the final key?
			if bytes.Compare(itKey, endKey) > 0 {
				break
			}
			select {
			case <-done:
				ch <- errorableKV{nil, nil}
				return
			case ch <- errorableKV{&storage.KeyValue{K: itKey, V: itValue}, nil}:
				it.Next()
			}
		} else {
			break
		}
	}
	if err := it.GetError(); err != nil {
		ch <- errorableKV{nil, err}
	} else {
		ch <- errorableKV{nil, nil}
	}
	return
}
Esempio n. 3
0
// RawRangeQuery sends a range of full keys.  This is to be used for low-level data
// retrieval like DVID-to-DVID communication and should not be used by data type
// implementations if possible.  A nil is sent down the channel when the
// range is complete.
func (db *LevelDB) RawRangeQuery(kStart, kEnd storage.Key, keysOnly bool, out chan *storage.KeyValue, cancel <-chan struct{}) error {
	if db == nil {
		return fmt.Errorf("Can't call RawRangeQuery on nil LevelDB")
	}
	dvid.StartCgo()
	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	var itValue []byte
	it.Seek(kStart)
	for {
		if it.Valid() {
			if !keysOnly {
				itValue = it.Value()
				storage.StoreValueBytesRead <- len(itValue)
			}
			itKey := it.Key()
			storage.StoreKeyBytesRead <- len(itKey)
			// Did we pass the final key?
			if bytes.Compare(itKey, kEnd) > 0 {
				break
			}
			kv := storage.KeyValue{itKey, itValue}
			select {
			case out <- &kv:
			case <-cancel:
				return nil
			}
			//out <- &kv
			it.Next()
		} else {
			break
		}
	}
	out <- nil
	if err := it.GetError(); err != nil {
		return err
	}
	return nil
}
Esempio n. 4
0
// getSingleKeyVersions returns all versions of a key.  These key-value pairs will be sorted
// in ascending key order and could include a tombstone key.
func (db *LevelDB) getSingleKeyVersions(vctx storage.VersionedCtx, tk []byte) ([]*storage.KeyValue, error) {
	dvid.StartCgo()
	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	values := []*storage.KeyValue{}
	begKey, err := vctx.MinVersionKey(tk)
	if err != nil {
		return nil, err
	}
	endKey, err := vctx.MaxVersionKey(tk)
	if err != nil {
		return nil, err
	}

	it.Seek(begKey)
	for {
		if it.Valid() {
			itKey := it.Key()
			storage.StoreKeyBytesRead <- len(itKey)
			if bytes.Compare(itKey, endKey) > 0 {
				// log.Printf("key past %v\n", kEnd)
				return values, nil
			}
			itValue := it.Value()
			// log.Printf("got value of length %d\n", len(itValue))
			storage.StoreValueBytesRead <- len(itValue)
			values = append(values, &storage.KeyValue{itKey, itValue})
			it.Next()
		} else {
			err = it.GetError()
			// log.Printf("iteration done, err = %v\n", err)
			if err == nil {
				return values, nil
			}
			return nil, err
		}
	}
}
Esempio n. 5
0
// SendRange sends a range of full keys.  This is to be used for low-level data
// retrieval like DVID-to-DVID communication and should not be used by data type
// implementations if possible.  A nil is sent down the channel when the
// range is complete.
func (db *LevelDB) SendRange(kStart, kEnd storage.Key, keysOnly bool, out chan *storage.KeyValue) error {
	dvid.StartCgo()
	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	var itValue []byte
	it.Seek(kStart)
	for {
		if it.Valid() {
			if !keysOnly {
				itValue = it.Value()
				storage.StoreValueBytesRead <- len(itValue)
			}
			itKey := it.Key()
			storage.StoreKeyBytesRead <- len(itKey)
			// Did we pass the final key?
			if bytes.Compare(itKey, kEnd) > 0 {
				break
			}
			kv := storage.KeyValue{itKey, itValue}
			out <- &kv
			it.Next()
		} else {
			break
		}
	}
	out <- nil
	if err := it.GetError(); err != nil {
		return err
	}
	return nil
}
Esempio n. 6
0
// versionedRange sends a range of key-value pairs for a particular version down a channel.
func (db *LevelDB) versionedRange(vctx storage.VersionedCtx, begTKey, endTKey storage.TKey, ch chan errorableKV, done <-chan struct{}, keysOnly bool) {
	dvid.StartCgo()
	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	minKey, err := vctx.MinVersionKey(begTKey)
	if err != nil {
		ch <- errorableKV{nil, err}
		return
	}
	maxKey, err := vctx.MaxVersionKey(endTKey)
	if err != nil {
		ch <- errorableKV{nil, err}
		return
	}

	values := []*storage.KeyValue{}
	maxVersionKey, err := vctx.MaxVersionKey(begTKey)
	if err != nil {
		ch <- errorableKV{nil, err}
		return
	}
	// log.Printf("         minKey %v\n", minKey)
	// log.Printf("         maxKey %v\n", maxKey)
	// log.Printf("  maxVersionKey %v\n", maxVersionKey)

	it.Seek(minKey)
	var itValue []byte
	for {
		select {
		case <-done: // only happens if we don't care about rest of data.
			ch <- errorableKV{nil, nil}
			return
		default:
		}
		if it.Valid() {
			if !keysOnly {
				itValue = it.Value()
				storage.StoreValueBytesRead <- len(itValue)
			}
			itKey := it.Key()
			// log.Printf("   +++valid key %v\n", itKey)
			storage.StoreKeyBytesRead <- len(itKey)

			// Did we pass all versions for last key read?
			if bytes.Compare(itKey, maxVersionKey) > 0 {
				indexBytes, err := storage.TKeyFromKey(itKey)
				if err != nil {
					ch <- errorableKV{nil, err}
					return
				}
				maxVersionKey, err = vctx.MaxVersionKey(indexBytes)
				if err != nil {
					ch <- errorableKV{nil, err}
					return
				}
				// log.Printf("->maxVersionKey %v (transmitting %d values)\n", maxVersionKey, len(values))
				sendKV(vctx, values, ch)
				values = []*storage.KeyValue{}
			}
			// Did we pass the final key?
			if bytes.Compare(itKey, maxKey) > 0 {
				if len(values) > 0 {
					sendKV(vctx, values, ch)
				}
				ch <- errorableKV{nil, nil}
				return
			}
			// log.Printf("Appending value with key %v\n", itKey)
			values = append(values, &storage.KeyValue{K: itKey, V: itValue})
			it.Next()
		} else {
			if err = it.GetError(); err != nil {
				ch <- errorableKV{nil, err}
			} else {
				sendKV(vctx, values, ch)
				ch <- errorableKV{nil, nil}
			}
			return
		}
	}
}
Esempio n. 7
0
func getOptions(config dvid.Config) (*leveldbOptions, error) {
	opt := &leveldbOptions{
		Options:      levigo.NewOptions(),
		ReadOptions:  levigo.NewReadOptions(),
		WriteOptions: levigo.NewWriteOptions(),
		env:          levigo.NewDefaultEnv(),
	}
	opt.WriteOptions.SetSync(DefaultSync) // Huge performance penalty to set sync to true

	// Set flags based on create parameter
	opt.SetCreateIfMissing(true)
	opt.SetErrorIfExists(false)

	// Create associated data structures with default values
	bloomBits, found, err := config.GetInt("BloomFilterBitsPerKey")
	if err != nil {
		return nil, err
	}
	if !found {
		bloomBits = DefaultBloomBits
	}
	opt.SetBloomFilterBitsPerKey(bloomBits)

	cacheSize, found, err := config.GetInt("CacheSize")
	if err != nil {
		return nil, err
	}
	if !found {
		cacheSize = DefaultCacheSize
	} else {
		cacheSize *= dvid.Mega
	}
	dvid.Infof("leveldb cache size: %s\n",
		humanize.Bytes(uint64(cacheSize)))
	opt.SetLRUCacheSize(cacheSize)

	writeBufferSize, found, err := config.GetInt("WriteBufferSize")
	if err != nil {
		return nil, err
	}
	if !found {
		writeBufferSize = DefaultWriteBufferSize
	} else {
		writeBufferSize *= dvid.Mega
	}
	dvid.Infof("leveldb write buffer size: %s\n",
		humanize.Bytes(uint64(writeBufferSize)))
	opt.SetWriteBufferSize(writeBufferSize)

	maxOpenFiles, found, err := config.GetInt("MaxOpenFiles")
	if err != nil {
		return nil, err
	}
	if !found {
		maxOpenFiles = DefaultMaxOpenFiles
	}
	opt.SetMaxOpenFiles(maxOpenFiles)

	blockSize, found, err := config.GetInt("BlockSize")
	if err != nil {
		return nil, err
	}
	if !found {
		blockSize = DefaultBlockSize
	}
	opt.SetBlockSize(blockSize)
	opt.SetInfoLog(nil)
	opt.SetParanoidChecks(false)
	//opt.SetBlockRestartInterval(8)

	// Don't bother with compression on leveldb side because it will be
	// selectively applied on DVID side.  We may return and then transmit
	// Snappy-compressed data without ever decompressing on server-side.
	opt.SetCompression(levigo.NoCompression) // (levigo.SnappyCompression)

	return opt, nil
}
Esempio n. 8
0
func (db *LevelDB) deleteAllVersions(ctx storage.Context) error {
	dvid.StartCgo()

	var err error
	var minKey, maxKey storage.Key

	vctx, versioned := ctx.(storage.VersionedCtx)
	if versioned {
		// Don't have to worry about tombstones.  Delete all keys from all versions for this instance id.
		minTKey := storage.MinTKey(storage.TKeyMinClass)
		maxTKey := storage.MaxTKey(storage.TKeyMaxClass)
		minKey, err = vctx.MinVersionKey(minTKey)
		if err != nil {
			return err
		}
		maxKey, err = vctx.MaxVersionKey(maxTKey)
		if err != nil {
			return err
		}
	} else {
		minKey, maxKey = ctx.KeyRange()
	}

	const BATCH_SIZE = 10000
	batch := db.NewBatch(ctx).(*goBatch)

	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	numKV := 0
	it.Seek(minKey)
	for {
		if err := it.GetError(); err != nil {
			return fmt.Errorf("Error iterating during DeleteAll for %s: %v", ctx, err)
		}
		if it.Valid() {
			itKey := it.Key()
			storage.StoreKeyBytesRead <- len(itKey)
			// Did we pass the final key?
			if bytes.Compare(itKey, maxKey) > 0 {
				break
			}

			batch.WriteBatch.Delete(itKey)
			if (numKV+1)%BATCH_SIZE == 0 {
				if err := batch.Commit(); err != nil {
					dvid.Criticalf("Error on batch commit of DeleteAll at key-value pair %d: %v\n", numKV, err)
					return fmt.Errorf("Error on batch commit of DeleteAll at key-value pair %d: %v", numKV, err)
				}
				batch = db.NewBatch(ctx).(*goBatch)
				dvid.Debugf("Deleted %d key-value pairs in ongoing DELETE ALL for %s.\n", numKV+1, ctx)
			}
			numKV++
			it.Next()
		} else {
			break
		}
	}
	if numKV%BATCH_SIZE != 0 {
		if err := batch.Commit(); err != nil {
			dvid.Criticalf("Error on last batch commit of DeleteAll: %v\n", err)
			return fmt.Errorf("Error on last batch commit of DeleteAll: %v", err)
		}
	}
	dvid.Debugf("Deleted %d key-value pairs via DELETE ALL for %s.\n", numKV, ctx)
	return nil
}
Esempio n. 9
0
func (db *LevelDB) deleteSingleVersion(vctx storage.VersionedCtx) error {
	dvid.StartCgo()

	minTKey := storage.MinTKey(storage.TKeyMinClass)
	maxTKey := storage.MaxTKey(storage.TKeyMaxClass)
	minKey, err := vctx.MinVersionKey(minTKey)
	if err != nil {
		return err
	}
	maxKey, err := vctx.MaxVersionKey(maxTKey)
	if err != nil {
		return err
	}

	const BATCH_SIZE = 10000
	batch := db.NewBatch(vctx).(*goBatch)

	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	numKV := 0
	it.Seek(minKey)
	deleteVersion := vctx.VersionID()
	for {
		if err := it.GetError(); err != nil {
			return fmt.Errorf("Error iterating during DeleteAll for %s: %v", vctx, err)
		}
		if it.Valid() {
			itKey := it.Key()
			storage.StoreKeyBytesRead <- len(itKey)
			// Did we pass the final key?
			if bytes.Compare(itKey, maxKey) > 0 {
				break
			}
			_, v, _, err := storage.DataKeyToLocalIDs(itKey)
			if err != nil {
				return fmt.Errorf("Error on DELETE ALL for version %d: %v", vctx.VersionID(), err)
			}
			if v == deleteVersion {
				batch.WriteBatch.Delete(itKey)
				if (numKV+1)%BATCH_SIZE == 0 {
					if err := batch.Commit(); err != nil {
						dvid.Criticalf("Error on batch commit of DeleteAll at key-value pair %d: %v\n", numKV, err)
						return fmt.Errorf("Error on batch commit of DeleteAll at key-value pair %d: %v", numKV, err)
					}
					batch = db.NewBatch(vctx).(*goBatch)
					dvid.Debugf("Deleted %d key-value pairs in ongoing DELETE ALL for %s.\n", numKV+1, vctx)
				}
				numKV++
			}

			it.Next()
		} else {
			break
		}
	}
	if numKV%BATCH_SIZE != 0 {
		if err := batch.Commit(); err != nil {
			dvid.Criticalf("Error on last batch commit of DeleteAll: %v\n", err)
			return fmt.Errorf("Error on last batch commit of DeleteAll: %v", err)
		}
	}
	dvid.Debugf("Deleted %d key-value pairs via DELETE ALL for %s.\n", numKV, vctx)
	return nil
}
Esempio n. 10
0
// DeleteAll deletes all key-value associated with a context (data instance and version).
func (db *LevelDB) DeleteAll(ctx storage.Context, allVersions bool) error {
	if ctx == nil {
		return fmt.Errorf("Received nil context in Delete()")
	}
	dvid.StartCgo()

	// Don't have to worry about tombstones.  Delete all keys from all versions for this instance id.
	minTKey := storage.MinTKey(storage.TKeyMinClass)
	maxTKey := storage.MaxTKey(storage.TKeyMaxClass)
	vctx, ok := ctx.(storage.VersionedCtx)
	if !ok {
		return fmt.Errorf("Non-versioned context passed to DELETE ALL VERSIONS in basholeveldb driver: %v", ctx)
	}
	minKey, err := vctx.MinVersionKey(minTKey)
	if err != nil {
		return err
	}
	maxKey, err := vctx.MaxVersionKey(maxTKey)
	if err != nil {
		return err
	}

	const BATCH_SIZE = 10000
	batch := db.NewBatch(ctx).(*goBatch)

	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	numKV := 0
	it.Seek(minKey)
	deleteVersion := ctx.VersionID()
	for {
		if it.Valid() {
			itKey := it.Key()
			storage.StoreKeyBytesRead <- len(itKey)
			// Did we pass the final key?
			if bytes.Compare(itKey, maxKey) > 0 {
				break
			}
			if !allVersions {
				_, v, _, err := storage.DataKeyToLocalIDs(itKey)
				if err != nil {
					return fmt.Errorf("Error on DELETE ALL for version %d: %v", ctx.VersionID(), err)
				}
				if v != deleteVersion {
					it.Next()
					continue
				}
			}
			batch.WriteBatch.Delete(itKey)
			if (numKV+1)%BATCH_SIZE == 0 {
				if err := batch.Commit(); err != nil {
					batch.Close()
					return fmt.Errorf("Error on DELETE ALL at key-value pair %d: %v", numKV, err)
				}
				batch = db.NewBatch(ctx).(*goBatch)
			}
			numKV++

			it.Next()
		} else {
			break
		}
	}
	if numKV%BATCH_SIZE != 0 {
		if err := batch.Commit(); err != nil {
			batch.Close()
			return fmt.Errorf("Error on last batch DELETE: %v\n", err)
		}
	}
	dvid.Debugf("Deleted %d key-value pairs via DELETE ALL for %s.\n", numKV, ctx)
	return nil
}