// versionedRange sends a range of key-value pairs for a particular version down a channel.
func (db *KVAutobus) versionedRange(vctx storage.VersionedCtx, kStart, kEnd storage.TKey, ch chan errorableKV, keysOnly bool) {
minKey, err := vctx.MinVersionKey(kStart)
if err != nil {
ch <- errorableKV{nil, err}
return
}
maxKey, err := vctx.MaxVersionKey(kEnd)
if err != nil {
ch <- errorableKV{nil, err}
return
}
maxVersionKey, err := vctx.MaxVersionKey(kStart)
if err != nil {
ch <- errorableKV{nil, err}
return
}
kvs, err := db.getRange(minKey, maxKey)
if err != nil {
ch <- errorableKV{nil, err}
return
}
versions := []*storage.KeyValue{}
for _, kv := range kvs {
if !keysOnly {
storage.StoreValueBytesRead <- len(kv.V)
}
storage.StoreKeyBytesRead <- len(kv.K)
// Did we pass all versions for last key read?
if bytes.Compare(kv.K, maxVersionKey) > 0 {
indexBytes, err := vctx.TKeyFromKey(kv.K)
if err != nil {
ch <- errorableKV{nil, err}
return
}
maxVersionKey, err = vctx.MaxVersionKey(indexBytes)
if err != nil {
ch <- errorableKV{nil, err}
return
}
// log.Printf("->maxVersionKey %v (transmitting %d values)\n", maxVersionKey, len(values))
sendKV(vctx, versions, ch)
versions = []*storage.KeyValue{}
}
// Did we pass the final key?
if bytes.Compare(kv.K, maxKey) > 0 {
if len(versions) > 0 {
sendKV(vctx, versions, ch)
}
ch <- errorableKV{nil, nil}
return
}
// log.Printf("Appending value with key %v\n", itKey)
versions = append(versions, kv)
}
}
// getSingleKeyVersions returns all versions of a key. These key-value pairs will be sorted
// in ascending key order and could include a tombstone key.
func (db *KVAutobus) getSingleKeyVersions(vctx storage.VersionedCtx, k []byte) ([]*storage.KeyValue, error) {
kStart, err := vctx.MinVersionKey(k)
if err != nil {
return nil, err
}
kEnd, err := vctx.MaxVersionKey(k)
if err != nil {
return nil, err
}
kvs, err := db.getRange(vctx, kStart, kEnd)
if err != nil {
return nil, err
}
return kvs, nil
}
// getSingleKeyVersions returns all versions of a key. These key-value pairs will be sorted
// in ascending key order and could include a tombstone key.
func (db *LevelDB) getSingleKeyVersions(vctx storage.VersionedCtx, tk []byte) ([]*storage.KeyValue, error) {
dvid.StartCgo()
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
values := []*storage.KeyValue{}
begKey, err := vctx.MinVersionKey(tk)
if err != nil {
return nil, err
}
endKey, err := vctx.MaxVersionKey(tk)
if err != nil {
return nil, err
}
it.Seek(begKey)
for {
if it.Valid() {
itKey := it.Key()
storage.StoreKeyBytesRead <- len(itKey)
if bytes.Compare(itKey, endKey) > 0 {
// log.Printf("key past %v\n", kEnd)
return values, nil
}
itValue := it.Value()
// log.Printf("got value of length %d\n", len(itValue))
storage.StoreValueBytesRead <- len(itValue)
values = append(values, &storage.KeyValue{itKey, itValue})
it.Next()
} else {
err = it.GetError()
// log.Printf("iteration done, err = %v\n", err)
if err == nil {
return values, nil
}
return nil, err
}
}
}
// versionedRange sends a range of key-value pairs for a particular version down a channel.
func (db *LevelDB) versionedRange(vctx storage.VersionedCtx, begTKey, endTKey storage.TKey, ch chan errorableKV, done <-chan struct{}, keysOnly bool) {
dvid.StartCgo()
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
minKey, err := vctx.MinVersionKey(begTKey)
if err != nil {
ch <- errorableKV{nil, err}
return
}
maxKey, err := vctx.MaxVersionKey(endTKey)
if err != nil {
ch <- errorableKV{nil, err}
return
}
values := []*storage.KeyValue{}
maxVersionKey, err := vctx.MaxVersionKey(begTKey)
if err != nil {
ch <- errorableKV{nil, err}
return
}
// log.Printf(" minKey %v\n", minKey)
// log.Printf(" maxKey %v\n", maxKey)
// log.Printf(" maxVersionKey %v\n", maxVersionKey)
it.Seek(minKey)
var itValue []byte
for {
select {
case <-done: // only happens if we don't care about rest of data.
ch <- errorableKV{nil, nil}
return
default:
}
if it.Valid() {
if !keysOnly {
itValue = it.Value()
storage.StoreValueBytesRead <- len(itValue)
}
itKey := it.Key()
// log.Printf(" +++valid key %v\n", itKey)
storage.StoreKeyBytesRead <- len(itKey)
// Did we pass all versions for last key read?
if bytes.Compare(itKey, maxVersionKey) > 0 {
indexBytes, err := storage.TKeyFromKey(itKey)
if err != nil {
ch <- errorableKV{nil, err}
return
}
maxVersionKey, err = vctx.MaxVersionKey(indexBytes)
if err != nil {
ch <- errorableKV{nil, err}
return
}
// log.Printf("->maxVersionKey %v (transmitting %d values)\n", maxVersionKey, len(values))
sendKV(vctx, values, ch)
values = []*storage.KeyValue{}
}
// Did we pass the final key?
if bytes.Compare(itKey, maxKey) > 0 {
if len(values) > 0 {
sendKV(vctx, values, ch)
}
ch <- errorableKV{nil, nil}
return
}
// log.Printf("Appending value with key %v\n", itKey)
values = append(values, &storage.KeyValue{K: itKey, V: itValue})
it.Next()
} else {
if err = it.GetError(); err != nil {
ch <- errorableKV{nil, err}
} else {
sendKV(vctx, values, ch)
ch <- errorableKV{nil, nil}
}
return
}
}
}
func (db *LevelDB) deleteSingleVersion(vctx storage.VersionedCtx) error {
dvid.StartCgo()
minTKey := storage.MinTKey(storage.TKeyMinClass)
maxTKey := storage.MaxTKey(storage.TKeyMaxClass)
minKey, err := vctx.MinVersionKey(minTKey)
if err != nil {
return err
}
maxKey, err := vctx.MaxVersionKey(maxTKey)
if err != nil {
return err
}
const BATCH_SIZE = 10000
batch := db.NewBatch(vctx).(*goBatch)
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
numKV := 0
it.Seek(minKey)
deleteVersion := vctx.VersionID()
for {
if err := it.GetError(); err != nil {
return fmt.Errorf("Error iterating during DeleteAll for %s: %v", vctx, err)
}
if it.Valid() {
itKey := it.Key()
storage.StoreKeyBytesRead <- len(itKey)
// Did we pass the final key?
if bytes.Compare(itKey, maxKey) > 0 {
break
}
_, v, _, err := storage.DataKeyToLocalIDs(itKey)
if err != nil {
return fmt.Errorf("Error on DELETE ALL for version %d: %v", vctx.VersionID(), err)
}
if v == deleteVersion {
batch.WriteBatch.Delete(itKey)
if (numKV+1)%BATCH_SIZE == 0 {
if err := batch.Commit(); err != nil {
dvid.Criticalf("Error on batch commit of DeleteAll at key-value pair %d: %v\n", numKV, err)
return fmt.Errorf("Error on batch commit of DeleteAll at key-value pair %d: %v", numKV, err)
}
batch = db.NewBatch(vctx).(*goBatch)
dvid.Debugf("Deleted %d key-value pairs in ongoing DELETE ALL for %s.\n", numKV+1, vctx)
}
numKV++
}
it.Next()
} else {
break
}
}
if numKV%BATCH_SIZE != 0 {
if err := batch.Commit(); err != nil {
dvid.Criticalf("Error on last batch commit of DeleteAll: %v\n", err)
return fmt.Errorf("Error on last batch commit of DeleteAll: %v", err)
}
}
dvid.Debugf("Deleted %d key-value pairs via DELETE ALL for %s.\n", numKV, vctx)
return nil
}