Beispiel #1
0
// Get returns a value given a key.
func (db *BigTable) Get(ctx storage.Context, tk storage.TKey) ([]byte, error) {
	if db == nil {
		return nil, fmt.Errorf("Can't call Get() on nil BigTable")
	}
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in Get()")
	}

	unvKey, verKey, err := ctx.SplitKey(tk)
	if err != nil {
		dvid.Errorf("Error in Get(): %v\n", err)
	}

	r, err := tbl.ReadRow(db.ctx, encodeKey(unvKey))

	//A missing row will return a zero-length map and a nil error
	if len(r) == 0 {
		return nil, err
	}

	if err != nil {
		return nil, err
	}

	value, err := getValue(r, verKey)
	if err != nil {
		return nil, err
	}

	return value, nil
}
Beispiel #2
0
// GetRange returns a range of values spanning (kStart, kEnd) keys.  These key-value
// pairs will be sorted in ascending key order.  If the keys are versioned, all key-value
// pairs for the particular version will be returned.
func (db *LevelDB) GetRange(ctx storage.Context, kStart, kEnd storage.TKey) ([]*storage.TKeyValue, error) {
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in GetRange()")
	}
	ch := make(chan errorableKV)

	// Run the range query on a potentially versioned key in a goroutine.
	go func() {
		if ctx == nil || !ctx.Versioned() {
			db.unversionedRange(ctx, kStart, kEnd, ch, false)
		} else {
			db.versionedRange(ctx.(storage.VersionedCtx), kStart, kEnd, ch, false)
		}
	}()

	// Consume the key-value pairs.
	values := []*storage.TKeyValue{}
	for {
		result := <-ch
		if result.KeyValue == nil {
			return values, nil
		}
		if result.error != nil {
			return nil, result.error
		}
		tk, err := ctx.TKeyFromKey(result.KeyValue.K)
		if err != nil {
			return nil, err
		}
		tkv := storage.TKeyValue{tk, result.KeyValue.V}
		values = append(values, &tkv)
	}
}
Beispiel #3
0
// Get returns a value given a key.
func (db *LevelDB) Get(ctx storage.Context, tk storage.TKey) ([]byte, error) {
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in Get()")
	}
	if ctx.Versioned() {
		vctx, ok := ctx.(storage.VersionedCtx)
		if !ok {
			return nil, fmt.Errorf("Bad Get(): context is versioned but doesn't fulfill interface: %v", ctx)
		}

		// Get all versions of this key and return the most recent
		// log.Printf("  basholeveldb versioned get of key %v\n", k)
		values, err := db.getSingleKeyVersions(vctx, tk)
		// log.Printf("            got back %v\n", values)
		if err != nil {
			return nil, err
		}
		kv, err := vctx.VersionedKeyValue(values)
		// log.Printf("  after deversioning: %v\n", kv)
		if kv != nil {
			return kv.V, err
		}
		return nil, err
	} else {
		key := ctx.ConstructKey(tk)
		ro := db.options.ReadOptions
		// log.Printf("  basholeveldb unversioned get of key %v\n", key)
		dvid.StartCgo()
		v, err := db.ldb.Get(ro, key)
		dvid.StopCgo()
		storage.StoreValueBytesRead <- len(v)
		return v, err
	}
}
Beispiel #4
0
// Delete removes a value with given key.
func (db *KVAutobus) Delete(ctx storage.Context, tk storage.TKey) error {
	if ctx == nil {
		return fmt.Errorf("Received nil context in Delete()")
	}
	key := ctx.ConstructKey(tk)
	return db.RawDelete(key)
}
Beispiel #5
0
// Put writes a value with given key.
func (db *LevelDB) Put(ctx storage.Context, tk storage.TKey, v []byte) error {
	if ctx == nil {
		return fmt.Errorf("Received nil context in Put()")
	}
	wo := db.options.WriteOptions

	var err error
	key := ctx.ConstructKey(tk)
	if !ctx.Versioned() {
		dvid.StartCgo()
		err = db.ldb.Put(wo, key, v)
		dvid.StopCgo()
	} else {
		vctx, ok := ctx.(storage.VersionedCtx)
		if !ok {
			return fmt.Errorf("Non-versioned context that says it's versioned received in Put(): %v", ctx)
		}
		tombstoneKey := vctx.TombstoneKey(tk)
		batch := db.NewBatch(vctx).(*goBatch)
		batch.WriteBatch.Delete(tombstoneKey)
		batch.WriteBatch.Put(key, v)
		if err = batch.Commit(); err != nil {
			batch.Close()
			err = fmt.Errorf("Error on PUT: %v\n", err)
		}
	}

	storage.StoreKeyBytesWritten <- len(key)
	storage.StoreValueBytesWritten <- len(v)
	return err
}
Beispiel #6
0
// SendKeysInRange sends a range of keys spanning (kStart, kEnd).  Values
// associated with the keys are not read.   If the keys are versioned, only keys
// in the ancestor path of the current context's version will be returned.
// End of range is marked by a nil key.
func (db *LevelDB) SendKeysInRange(ctx storage.Context, kStart, kEnd storage.TKey, kch storage.KeyChan) error {
	if db == nil {
		return fmt.Errorf("Can't call SendKeysInRange on nil LevelDB")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in SendKeysInRange()")
	}
	ch := make(chan errorableKV)
	done := make(chan struct{})
	defer close(done)

	// Run the range query on a potentially versioned key in a goroutine.
	go func() {
		if !ctx.Versioned() {
			db.unversionedRange(ctx, kStart, kEnd, ch, done, true)
		} else {
			db.versionedRange(ctx.(storage.VersionedCtx), kStart, kEnd, ch, done, true)
		}
	}()

	// Consume the keys.
	for {
		result := <-ch
		if result.KeyValue == nil {
			kch <- nil
			return nil
		}
		if result.error != nil {
			kch <- nil
			return result.error
		}
		kch <- result.KeyValue.K
	}
}
Beispiel #7
0
// Delete removes a value with given key.
func (db *LevelDB) Delete(ctx storage.Context, tk storage.TKey) error {
	if db == nil {
		return fmt.Errorf("Can't call Delete on nil LevelDB")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in Delete()")
	}
	wo := db.options.WriteOptions

	var err error
	key := ctx.ConstructKey(tk)
	if !ctx.Versioned() {
		dvid.StartCgo()
		err = db.ldb.Delete(wo, key)
		dvid.StopCgo()
	} else {
		vctx, ok := ctx.(storage.VersionedCtx)
		if !ok {
			return fmt.Errorf("Non-versioned context that says it's versioned received in Delete(): %v", ctx)
		}
		tombstoneKey := vctx.TombstoneKey(tk)
		batch := db.NewBatch(vctx).(*goBatch)
		batch.WriteBatch.Delete(key)
		batch.WriteBatch.Put(tombstoneKey, dvid.EmptyValue())
		if err = batch.Commit(); err != nil {
			dvid.Criticalf("Error on batch commit of Delete: %v\n", err)
			err = fmt.Errorf("Error on batch commit of Delete: %v", err)
		}
	}

	return err
}
Beispiel #8
0
// KeysInRange returns a range of present keys spanning (kStart, kEnd).  Values
// associated with the keys are not read.   If the keys are versioned, only keys
// in the ancestor path of the current context's version will be returned.
func (db *LevelDB) KeysInRange(ctx storage.Context, kStart, kEnd storage.TKey) ([]storage.TKey, error) {
	if db == nil {
		return nil, fmt.Errorf("Can't call KeysInRange on nil LevelDB")
	}
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in KeysInRange()")
	}
	ch := make(chan errorableKV)

	// Run the range query on a potentially versioned key in a goroutine.
	go func() {
		if !ctx.Versioned() {
			db.unversionedRange(ctx, kStart, kEnd, ch, true)
		} else {
			db.versionedRange(ctx.(storage.VersionedCtx), kStart, kEnd, ch, true)
		}
	}()

	// Consume the keys.
	values := []storage.TKey{}
	for {
		result := <-ch
		if result.KeyValue == nil {
			return values, nil
		}
		if result.error != nil {
			return nil, result.error
		}
		tk, err := ctx.TKeyFromKey(result.KeyValue.K)
		if err != nil {
			return nil, err
		}
		values = append(values, tk)
	}
}
Beispiel #9
0
// Put key-value pairs.  Note that it could be more efficient to use the Batcher
// interface so you don't have to create and keep a slice of KeyValue.  Some
// databases like leveldb will copy on batch put anyway.
func (db *BigTable) PutRange(ctx storage.Context, TKeyValue []storage.TKeyValue) error {
	if db == nil {
		return fmt.Errorf("Can't call PutRange() on nil BigTable")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in PutRange()")
	}

	for _, tkeyvalue := range TKeyValue {

		unvKey, verKey, err := ctx.SplitKey(tkeyvalue.K)
		if err != nil {
			dvid.Errorf("Error in PutRange(): %v\n", err)
		}

		mut := api.NewMutation()

		mut.Set(familyName, encodeKey(verKey), 0, tkeyvalue.V)
		err = tbl.Apply(db.ctx, encodeKey(unvKey), mut)
		if err != nil {
			dvid.Errorf("Failed to Put value in PutRange()")
		}
	}

	return nil
}
Beispiel #10
0
// DeleteRange removes all key-value pairs with keys in the given range.
func (db *LevelDB) DeleteRange(ctx storage.Context, kStart, kEnd storage.TKey) error {
	if ctx == nil {
		return fmt.Errorf("Received nil context in DeleteRange()")
	}

	// For leveldb, we just iterate over keys in range and delete each one using batch.
	const BATCH_SIZE = 10000
	batch := db.NewBatch(ctx).(*goBatch)

	ch := make(chan errorableKV)

	// Run the keys-only range query in a goroutine.
	go func() {
		if ctx == nil || !ctx.Versioned() {
			db.unversionedRange(ctx, kStart, kEnd, ch, true)
		} else {
			db.versionedRange(ctx.(storage.VersionedCtx), kStart, kEnd, ch, true)
		}
	}()

	// Consume the key-value pairs.
	numKV := 0
	for {
		result := <-ch
		if result.KeyValue == nil {
			break
		}
		if result.error != nil {
			return result.error
		}

		// The key coming down channel is not index but full key, so no need to construct key using context.
		// If versioned, write a tombstone using current version id since we don't want to delete locked ancestors.
		// If unversioned, just delete.
		tk, err := ctx.TKeyFromKey(result.KeyValue.K)
		if err != nil {
			return err
		}
		batch.Delete(tk)

		if (numKV+1)%BATCH_SIZE == 0 {
			if err := batch.Commit(); err != nil {
				batch.Close()
				return fmt.Errorf("Error on batch DELETE at key-value pair %d: %v\n", numKV, err)
			}
			batch = db.NewBatch(ctx).(*goBatch)
		}
		numKV++
	}
	if numKV%BATCH_SIZE != 0 {
		if err := batch.Commit(); err != nil {
			batch.Close()
			return fmt.Errorf("Error on last batch DELETE: %v\n", err)
		}
	}
	dvid.Debugf("Deleted %d key-value pairs via delete range for %s.\n", numKV, ctx)
	return nil
}
Beispiel #11
0
// GetTileKey returns the internal key as a hexadecimal string
func (d *Data) GetTileKey(ctx storage.Context, w http.ResponseWriter, r *http.Request, parts []string) (string, error) {
	req, err := d.ParseTileReq(r, parts)
	if err != nil {
		return "", err
	}
	tk := NewTKeyByTileReq(req)
	key := ctx.ConstructKey(tk)
	return fmt.Sprintf("%x", key), nil
}
Beispiel #12
0
// PutRange puts type key-value pairs that have been sorted in sequential key order.
func (db *KVAutobus) PutRange(ctx storage.Context, tkvs []storage.TKeyValue) error {
	if ctx == nil {
		return fmt.Errorf("Received nil context in PutRange()")
	}
	kvs := make([]storage.KeyValue, len(tkvs))
	for i, tkv := range tkvs {
		kvs[i] = storage.KeyValue{ctx.ConstructKey(tkv.K), tkv.V}
	}
	return db.putRange(kvs)
}
Beispiel #13
0
// DeleteAll deletes all key-value associated with a context (data instance and version).
func (db *KVAutobus) DeleteAll(ctx storage.Context, allVersions bool) error {
	if ctx == nil {
		return fmt.Errorf("Received nil context in DeleteAll()")
	}
	if !allVersions {
		return fmt.Errorf("DeleteAll() not supported for immutable store unless for all versions")
	}
	keyBeg, keyEnd := ctx.KeyRange()
	return db.deleteRange(keyBeg, keyEnd)
}
Beispiel #14
0
// KeysInRange returns a range of type-specific key components spanning (TkBeg, TkEnd).
func (db *BigTable) KeysInRange(ctx storage.Context, TkBeg, TkEnd storage.TKey) ([]storage.TKey, error) {
	if db == nil {
		return nil, fmt.Errorf("Can't call KeysInRange() on nil BigTable")
	}
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in KeysInRange()")
	}

	tKeys := make([]storage.TKey, 0)

	unvKeyBeg, _, err := ctx.SplitKey(TkBeg)
	if err != nil {
		dvid.Errorf("Error in KeysInRange(): %v\n", err)
	}

	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		dvid.Errorf("Error in KeysInRange(): %v\n", err)
	}

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))

	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		if len(r[familyName]) == 0 {
			dvid.Errorf("Error in KeysInRange(): row has no columns")
			return false
		}

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in KeysInRange(): %v\n", err)
			return false
		}

		verKeyRow, err := decodeKey(r[familyName][0].Column)
		if err != nil {
			dvid.Errorf("Error in KeysInRange(): %v\n", err)
			return false
		}

		fullKey := storage.MergeKey(unvKeyRow, verKeyRow)
		tkey, err := storage.TKeyFromKey(fullKey)
		if err != nil {
			dvid.Errorf("Error in KeysInRange(): %v\n", err)
			return false
		}

		tKeys = append(tKeys, tkey)

		return true // keep going
	}, api.RowFilter(api.StripValueFilter()))

	return tKeys, err
}
Beispiel #15
0
// GetRange returns a range of values spanning (TkBeg, kEnd) keys.
func (db *BigTable) GetRange(ctx storage.Context, TkBeg, TkEnd storage.TKey) ([]*storage.TKeyValue, error) {
	if db == nil {
		return nil, fmt.Errorf("Can't call GetRange() on nil BigTable")
	}
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in GetRange()")
	}

	unvKeyBeg, _, err := ctx.SplitKey(TkBeg)
	if err != nil {
		dvid.Errorf("Error in GetRange(): %v\n", err)
	}
	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		dvid.Errorf("Error in GetRange(): %v\n", err)
	}

	tKeyValues := make([]*storage.TKeyValue, 0)

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))
	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in GetRange() decodeKey(r.Key()): %v\n", err)
			return false
		}

		// dvid.Infof("GetRange() with row key= %v", r.Key())
		for _, readItem := range r[familyName] {

			verKey, err := decodeKey(readItem.Column)
			if err != nil {
				dvid.Errorf("Error in GetRange() decodeKey(readItem.Column): %v\n", err)
				return false
			}

			fullKey := storage.MergeKey(unvKeyRow, verKey)
			// dvid.Infof("colum key= %v , timestamp = %v", verKey, readItem.Timestamp)
			tkey, err := storage.TKeyFromKey(fullKey)
			if err != nil {
				dvid.Errorf("Error in GetRange() storage.TKeyFromKey(fullKey): %v\n", err)
				return false
			}

			kv := storage.TKeyValue{tkey, readItem.Value}
			tKeyValues = append(tKeyValues, &kv)
		}

		return true // keep going
	})

	return tKeyValues, err
}
Beispiel #16
0
// Put writes a value with given key.  Since KVAutobus is immutable, we do not
// have to worry about a PUT on a previously deleted key.
func (db *KVAutobus) Put(ctx storage.Context, tk storage.TKey, v []byte) error {
	if ctx == nil {
		return fmt.Errorf("Received nil context in Put()")
	}

	key := ctx.ConstructKey(tk)
	err := db.RawPut(key, v)
	if err != nil {
		return err
	}
	return nil
}
Beispiel #17
0
// unversionedRange sends a range of key-value pairs down a channel.
func (db *LevelDB) unversionedRange(ctx storage.Context, begTKey, endTKey storage.TKey, ch chan errorableKV, done <-chan struct{}, keysOnly bool) {
	dvid.StartCgo()
	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	// Apply context if applicable
	begKey := ctx.ConstructKey(begTKey)
	endKey := ctx.ConstructKey(endTKey)

	// fmt.Printf("unversionedRange():\n")
	// fmt.Printf("    index beg: %v\n", kStart)
	// fmt.Printf("    index end: %v\n", kEnd)
	// fmt.Printf("    key start: %v\n", keyBeg)
	// fmt.Printf("      key end: %v\n", keyEnd)

	var itValue []byte
	it.Seek(begKey)
	for {
		if it.Valid() {
			// fmt.Printf("unversioned found key %v, %d bytes value\n", it.Key(), len(it.Value()))
			if !keysOnly {
				itValue = it.Value()
				storage.StoreValueBytesRead <- len(itValue)
			}
			itKey := it.Key()
			storage.StoreKeyBytesRead <- len(itKey)
			// Did we pass the final key?
			if bytes.Compare(itKey, endKey) > 0 {
				break
			}
			select {
			case <-done:
				ch <- errorableKV{nil, nil}
				return
			case ch <- errorableKV{&storage.KeyValue{K: itKey, V: itValue}, nil}:
				it.Next()
			}
		} else {
			break
		}
	}
	if err := it.GetError(); err != nil {
		ch <- errorableKV{nil, err}
	} else {
		ch <- errorableKV{nil, nil}
	}
	return
}
Beispiel #18
0
// unversionedRange sends a range of key-value pairs down a channel.
func (db *KVAutobus) unversionedRange(ctx storage.Context, kStart, kEnd storage.TKey, ch chan errorableKV, keysOnly bool) {
	keyBeg := ctx.ConstructKey(kStart)
	keyEnd := ctx.ConstructKey(kEnd)

	kvs, err := db.getRange(ctx, keyBeg, keyEnd)
	if err != nil {
		ch <- errorableKV{nil, err}
	}
	for _, kv := range kvs {
		ch <- errorableKV{kv, nil}
	}
	ch <- errorableKV{nil, nil}
}
Beispiel #19
0
// Put writes a value with given key.  Since KVAutobus is immutable, we do not
// have to worry about a PUT on a previously deleted key.
func (db *KVAutobus) Put(ctx storage.Context, tk storage.TKey, v []byte) error {
	if ctx == nil {
		return fmt.Errorf("Received nil context in Put()")
	}

	key := ctx.ConstructKey(tk)
	err := db.RawPut(key, v)
	if err != nil {
		return err
	}
	storage.StoreKeyBytesWritten <- len(key)
	storage.StoreValueBytesWritten <- len(v)
	return nil
}
Beispiel #20
0
// SendKeysInRange sends a range of full keys down a key channel.
func (db *BigTable) SendKeysInRange(ctx storage.Context, TkBeg, TkEnd storage.TKey, ch storage.KeyChan) error {
	if db == nil {
		return fmt.Errorf("Can't call SendKeysInRange() on nil BigTable")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in SendKeysInRange()")
	}

	unvKeyBeg, _, err := ctx.SplitKey(TkBeg)
	if err != nil {
		dvid.Errorf("Error in SendKeysInRange(): %v\n", err)
	}

	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		dvid.Errorf("Error in SendKeysInRange(): %v\n", err)
	}

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))

	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in SendKeysInRange(): %v\n", err)
			return false
		}

		//I need the versioned key to merged it with the unversioned
		// and send it throu the channel
		for _, readItem := range r[familyName] {

			verKey, err := decodeKey(readItem.Column)
			if err != nil {
				dvid.Errorf("Error in SendKeysInRange(): %v\n", err)
				return false
			}

			fullKey := storage.MergeKey(unvKeyRow, verKey)

			ch <- fullKey
		}

		return true // keep going
	}, api.RowFilter(api.StripValueFilter()))

	return err
}
Beispiel #21
0
// Delete deletes a key-value pair so that subsequent Get on the key returns nil.
func (db *BigTable) Delete(ctx storage.Context, tkey storage.TKey) error {
	if db == nil {
		return fmt.Errorf("Can't call Delete() on nil BigTable")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in Delete()")
	}

	unvKey, verKey, err := ctx.SplitKey(tkey)
	if err != nil {
		dvid.Errorf("Error in Delete(): %v\n", err)
	}

	r, err := tbl.ReadRow(db.ctx, encodeKey(unvKey), api.RowFilter(api.StripValueFilter()))

	//A missing row will return a zero-length map and a nil error
	if len(r) == 0 {
		return fmt.Errorf("Error in Delete(): This unvKey doesn't exists")
	}

	if err != nil {
		return err
	}
	if len(r[familyName]) == 0 {
		return fmt.Errorf("Error in Delete(): This row is empty")
	}
	_, err = getValue(r, verKey)
	if err != nil {
		return fmt.Errorf("Error in Delete(): The version to be deleted doesn't exist")
	}

	mut := api.NewMutation()

	//There is only one version left, and is the one we are trying to delete\
	//remove the whole row
	if len(r[familyName]) == 1 {
		mut.DeleteRow()
	} else {
		mut.DeleteCellsInColumn(familyName, encodeKey(verKey))
	}

	err = tbl.Apply(db.ctx, encodeKey(unvKey), mut)
	if err != nil {
		return fmt.Errorf("Error in Delete(): %v\n", err)
	}

	return err
}
Beispiel #22
0
// unversionedRange sends a range of key-value pairs down a channel.
func (db *KVAutobus) unversionedRange(ctx storage.Context, kStart, kEnd storage.TKey, ch chan errorableKV, keysOnly bool) {
	keyBeg := ctx.ConstructKey(kStart)
	keyEnd := ctx.ConstructKey(kEnd)

	kvs, err := db.getRange(keyBeg, keyEnd)
	if err != nil {
		ch <- errorableKV{nil, err}
	}
	for _, kv := range kvs {
		if !keysOnly {
			storage.StoreValueBytesRead <- len(kv.V)
		}
		storage.StoreKeyBytesRead <- len(kv.K)
		ch <- errorableKV{kv, nil}
	}
	ch <- errorableKV{nil, nil}
	return
}
Beispiel #23
0
// ProcessRange sends a range of key-value pairs to chunk handlers.  If the keys are versioned,
// only key-value pairs for kStart's version will be transmitted.  If f returns an error, the
// function is immediately terminated and returns an error.
func (db *LevelDB) ProcessRange(ctx storage.Context, kStart, kEnd storage.TKey, op *storage.ChunkOp, f storage.ChunkFunc) error {
	if db == nil {
		return fmt.Errorf("Can't call ProcessRange on nil LevelDB")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in ProcessRange()")
	}
	ch := make(chan errorableKV)
	done := make(chan struct{})
	defer close(done)

	// Run the range query on a potentially versioned key in a goroutine.
	go func() {
		if ctx == nil || !ctx.Versioned() {
			db.unversionedRange(ctx, kStart, kEnd, ch, done, false)
		} else {
			db.versionedRange(ctx.(storage.VersionedCtx), kStart, kEnd, ch, done, false)
		}
	}()

	// Consume the key-value pairs.
	for {
		result := <-ch
		if result.KeyValue == nil {
			return nil
		}
		if result.error != nil {
			return result.error
		}
		if op != nil && op.Wg != nil {
			op.Wg.Add(1)
		}
		tk, err := storage.TKeyFromKey(result.KeyValue.K)
		if err != nil {
			return err
		}
		tkv := storage.TKeyValue{tk, result.KeyValue.V}
		chunk := &storage.Chunk{op, &tkv}
		if err := f(chunk); err != nil {
			return err
		}
	}
}
Beispiel #24
0
// Get returns a value given a key.
func (db *KVAutobus) Get(ctx storage.Context, tk storage.TKey) ([]byte, error) {
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in Get()")
	}
	if ctx.Versioned() {
		vctx, ok := ctx.(storage.VersionedCtx)
		if !ok {
			return nil, fmt.Errorf("Bad Get(): context is versioned but doesn't fulfill interface: %v", ctx)
		}

		// Get all versions of this key and return the most recent
		// log.Printf("  kvautobus versioned get of key %v\n", k)
		key := ctx.ConstructKey(tk)
		dvid.Infof("   Get on key: %s\n", hex.EncodeToString(key))
		values, err := db.getSingleKeyVersions(vctx, tk)
		// log.Printf("            got back %v\n", values)
		if err != nil {
			return nil, err
		}
		kv, err := vctx.VersionedKeyValue(values)
		// log.Printf("  after deversioning: %v\n", kv)
		if kv != nil {
			return kv.V, err
		}
		return nil, err
	} else {
		key := ctx.ConstructKey(tk)
		// log.Printf("  kvautobus unversioned get of key %v\n", key)
		v, err := db.RawGet(key)
		storage.StoreValueBytesRead <- len(v)
		return v, err
	}
}
Beispiel #25
0
// DeleteRange removes all key-value pairs with keys in the given range.
// For all versions
func (db *BigTable) DeleteRange(ctx storage.Context, TkBeg, TkEnd storage.TKey) error {
	if db == nil {
		return fmt.Errorf("Can't call DeleteRange() on nil BigTable")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in DeleteRange()")
	}

	unvKeyBeg, _, err := ctx.SplitKey(TkBeg)
	if err != nil {
		return fmt.Errorf("Error in DeleteRange(): %v\n", err)
	}
	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		return fmt.Errorf("Error in DeleteRange(): %v\n", err)
	}

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))

	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in DeleteRange(): %v\n", err)
			return false
		}

		mut := api.NewMutation()
		mut.DeleteRow()
		err = tbl.Apply(db.ctx, encodeKey(unvKeyRow), mut)
		if err != nil {
			dvid.Errorf("Failed to delete row in DeleteRange()")
		}

		return true // keep going
	}, api.RowFilter(api.StripValueFilter()))

	return err
}
Beispiel #26
0
// Put writes a value with given key in a possibly versioned context.
func (db *BigTable) Put(ctx storage.Context, tkey storage.TKey, value []byte) error {
	if db == nil {
		return fmt.Errorf("Can't call Put() on nil BigTable")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in Put()")
	}

	unvKey, verKey, err := ctx.SplitKey(tkey)
	if err != nil {
		dvid.Errorf("Error in Put(): %v\n", err)
	}

	mut := api.NewMutation()

	// dvid.Infof("Putting value %s\n", string(value))
	mut.Set(familyName, encodeKey(verKey), 0, value)
	err = tbl.Apply(db.ctx, encodeKey(unvKey), mut)
	if err != nil {
		return fmt.Errorf("Error in Put(): %v\n", err)
	}

	return err
}
Beispiel #27
0
// DeleteRange removes all key-value pairs with keys in the given range.  Versioned
// contexts cannot use immutable stores to delete ranges.
func (db *KVAutobus) DeleteRange(ctx storage.Context, kStart, kEnd storage.TKey) error {
	if ctx == nil {
		return fmt.Errorf("Received nil context in DeleteRange()")
	}
	if ctx.Versioned() {
		return fmt.Errorf("DeleteRange() not supported for versioned contexts in immutable store")
	}
	keyBeg := ctx.ConstructKey(kStart)
	keyEnd := ctx.ConstructKey(kEnd)
	return db.deleteRange(keyBeg, keyEnd)
}
Beispiel #28
0
func (db *LevelDB) deleteAllVersions(ctx storage.Context) error {
	dvid.StartCgo()

	var err error
	var minKey, maxKey storage.Key

	vctx, versioned := ctx.(storage.VersionedCtx)
	if versioned {
		// Don't have to worry about tombstones.  Delete all keys from all versions for this instance id.
		minTKey := storage.MinTKey(storage.TKeyMinClass)
		maxTKey := storage.MaxTKey(storage.TKeyMaxClass)
		minKey, err = vctx.MinVersionKey(minTKey)
		if err != nil {
			return err
		}
		maxKey, err = vctx.MaxVersionKey(maxTKey)
		if err != nil {
			return err
		}
	} else {
		minKey, maxKey = ctx.KeyRange()
	}

	const BATCH_SIZE = 10000
	batch := db.NewBatch(ctx).(*goBatch)

	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	numKV := 0
	it.Seek(minKey)
	for {
		if err := it.GetError(); err != nil {
			return fmt.Errorf("Error iterating during DeleteAll for %s: %v", ctx, err)
		}
		if it.Valid() {
			itKey := it.Key()
			storage.StoreKeyBytesRead <- len(itKey)
			// Did we pass the final key?
			if bytes.Compare(itKey, maxKey) > 0 {
				break
			}

			batch.WriteBatch.Delete(itKey)
			if (numKV+1)%BATCH_SIZE == 0 {
				if err := batch.Commit(); err != nil {
					dvid.Criticalf("Error on batch commit of DeleteAll at key-value pair %d: %v\n", numKV, err)
					return fmt.Errorf("Error on batch commit of DeleteAll at key-value pair %d: %v", numKV, err)
				}
				batch = db.NewBatch(ctx).(*goBatch)
				dvid.Debugf("Deleted %d key-value pairs in ongoing DELETE ALL for %s.\n", numKV+1, ctx)
			}
			numKV++
			it.Next()
		} else {
			break
		}
	}
	if numKV%BATCH_SIZE != 0 {
		if err := batch.Commit(); err != nil {
			dvid.Criticalf("Error on last batch commit of DeleteAll: %v\n", err)
			return fmt.Errorf("Error on last batch commit of DeleteAll: %v", err)
		}
	}
	dvid.Debugf("Deleted %d key-value pairs via DELETE ALL for %s.\n", numKV, ctx)
	return nil
}
Beispiel #29
0
// DeleteAll deletes all key-value associated with a context (data instance and version).
func (db *LevelDB) DeleteAll(ctx storage.Context, allVersions bool) error {
	if ctx == nil {
		return fmt.Errorf("Received nil context in Delete()")
	}
	dvid.StartCgo()

	// Don't have to worry about tombstones.  Delete all keys from all versions for this instance id.
	minTKey := storage.MinTKey(storage.TKeyMinClass)
	maxTKey := storage.MaxTKey(storage.TKeyMaxClass)
	vctx, ok := ctx.(storage.VersionedCtx)
	if !ok {
		return fmt.Errorf("Non-versioned context passed to DELETE ALL VERSIONS in basholeveldb driver: %v", ctx)
	}
	minKey, err := vctx.MinVersionKey(minTKey)
	if err != nil {
		return err
	}
	maxKey, err := vctx.MaxVersionKey(maxTKey)
	if err != nil {
		return err
	}

	const BATCH_SIZE = 10000
	batch := db.NewBatch(ctx).(*goBatch)

	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	numKV := 0
	it.Seek(minKey)
	deleteVersion := ctx.VersionID()
	for {
		if it.Valid() {
			itKey := it.Key()
			storage.StoreKeyBytesRead <- len(itKey)
			// Did we pass the final key?
			if bytes.Compare(itKey, maxKey) > 0 {
				break
			}
			if !allVersions {
				_, v, _, err := storage.DataKeyToLocalIDs(itKey)
				if err != nil {
					return fmt.Errorf("Error on DELETE ALL for version %d: %v", ctx.VersionID(), err)
				}
				if v != deleteVersion {
					it.Next()
					continue
				}
			}
			batch.WriteBatch.Delete(itKey)
			if (numKV+1)%BATCH_SIZE == 0 {
				if err := batch.Commit(); err != nil {
					batch.Close()
					return fmt.Errorf("Error on DELETE ALL at key-value pair %d: %v", numKV, err)
				}
				batch = db.NewBatch(ctx).(*goBatch)
			}
			numKV++

			it.Next()
		} else {
			break
		}
	}
	if numKV%BATCH_SIZE != 0 {
		if err := batch.Commit(); err != nil {
			batch.Close()
			return fmt.Errorf("Error on last batch DELETE: %v\n", err)
		}
	}
	dvid.Debugf("Deleted %d key-value pairs via DELETE ALL for %s.\n", numKV, ctx)
	return nil
}
Beispiel #30
0
// DeleteAll removes all key-value pairs for the context.  If allVersions is true,
// then all versions of the data instance are deleted.
func (db *BigTable) DeleteAll(ctx storage.Context, allVersions bool) error {
	if db == nil {
		return fmt.Errorf("Can't call DeleteAll() on nil BigTable")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in DeleteAll()")
	}

	//Row range corresponde to all keys corresponding to this data instace.
	min, max := ctx.KeyRange()
	rr := api.NewRange(encodeKey(min), encodeKey(max))

	err := tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in DeleteAll(): %v\n", err)
			return false
		}

		if allVersions {

			mut := api.NewMutation()
			mut.DeleteRow()
			err := tbl.Apply(db.ctx, encodeKey(unvKeyRow), mut)
			if err != nil {
				dvid.Errorf("Failed to delete row")
			}
		} else {

			emptyTkey := make([]byte, 0)
			_, versionToDelete, err := ctx.SplitKey(emptyTkey)
			if err != nil {
				dvid.Errorf("Error in DeleteAll(): %v\n", err)
				return false
			}

			for _, readItem := range r[familyName] {

				verKey, err := decodeKey(readItem.Column)
				if err != nil {
					dvid.Errorf("Error in DeleteAll(): %v\n", err)
					return false
				}

				if bytes.Equal(verKey, versionToDelete) {
					mut := api.NewMutation()
					mut.DeleteCellsInColumn(familyName, encodeKey(verKey))
					err := tbl.Apply(db.ctx, encodeKey(unvKeyRow), mut)
					if err != nil {
						dvid.Errorf("Failed to DeleteCellsInColumn in DeleteAll()")
					}
					return true // One I found the version I don't have to keep serching for it.
				}
			}
		}

		return true // keep going
	}, api.RowFilter(api.StripValueFilter()))

	return err
}