func (p *pusher) putData(kvmsg *KVMessage) error {
// If this is a termination token
if kvmsg.Terminate {
p.printStats()
return nil
}
// Process the key-value pair
kv := &kvmsg.KV
oldInstance, oldVersion, _, err := storage.DataKeyToLocalIDs(kv.K)
if err != nil {
return err
}
// Modify the transmitted key-value to have local instance and version ids.
newInstanceID, found := p.instanceMap[oldInstance]
if !found {
return fmt.Errorf("Received key with instance id (%d) not present in repo: %v", oldInstance, p.instanceMap)
}
newVersionID, found := p.versionMap[oldVersion]
if !found {
return fmt.Errorf("Received key with version id (%d) not present in repo: %v", oldVersion, p.versionMap)
}
// Compute the updated key-value
// TODO: When client IDs are used, need to transmit list of pertinent clients and their IDs or just use 0 as here.
if err := storage.UpdateDataKey(kv.K, newInstanceID, newVersionID, 0); err != nil {
return fmt.Errorf("Unable to update data key %v: %v", kv.K, err)
}
p.stats.addKV(kv.K, kv.V)
p.store.RawPut(kv.K, kv.V)
p.received += uint64(len(kv.V) + len(kv.K))
return nil
}
func (db *LevelDB) deleteSingleVersion(vctx storage.VersionedCtx) error {
dvid.StartCgo()
minTKey := storage.MinTKey(storage.TKeyMinClass)
maxTKey := storage.MaxTKey(storage.TKeyMaxClass)
minKey, err := vctx.MinVersionKey(minTKey)
if err != nil {
return err
}
maxKey, err := vctx.MaxVersionKey(maxTKey)
if err != nil {
return err
}
const BATCH_SIZE = 10000
batch := db.NewBatch(vctx).(*goBatch)
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
numKV := 0
it.Seek(minKey)
deleteVersion := vctx.VersionID()
for {
if err := it.GetError(); err != nil {
return fmt.Errorf("Error iterating during DeleteAll for %s: %v", vctx, err)
}
if it.Valid() {
itKey := it.Key()
storage.StoreKeyBytesRead <- len(itKey)
// Did we pass the final key?
if bytes.Compare(itKey, maxKey) > 0 {
break
}
_, v, _, err := storage.DataKeyToLocalIDs(itKey)
if err != nil {
return fmt.Errorf("Error on DELETE ALL for version %d: %v", vctx.VersionID(), err)
}
if v == deleteVersion {
batch.WriteBatch.Delete(itKey)
if (numKV+1)%BATCH_SIZE == 0 {
if err := batch.Commit(); err != nil {
dvid.Criticalf("Error on batch commit of DeleteAll at key-value pair %d: %v\n", numKV, err)
return fmt.Errorf("Error on batch commit of DeleteAll at key-value pair %d: %v", numKV, err)
}
batch = db.NewBatch(vctx).(*goBatch)
dvid.Debugf("Deleted %d key-value pairs in ongoing DELETE ALL for %s.\n", numKV+1, vctx)
}
numKV++
}
it.Next()
} else {
break
}
}
if numKV%BATCH_SIZE != 0 {
if err := batch.Commit(); err != nil {
dvid.Criticalf("Error on last batch commit of DeleteAll: %v\n", err)
return fmt.Errorf("Error on last batch commit of DeleteAll: %v", err)
}
}
dvid.Debugf("Deleted %d key-value pairs via DELETE ALL for %s.\n", numKV, vctx)
return nil
}
// DeleteAll deletes all key-value associated with a context (data instance and version).
func (db *LevelDB) DeleteAll(ctx storage.Context, allVersions bool) error {
if ctx == nil {
return fmt.Errorf("Received nil context in Delete()")
}
dvid.StartCgo()
// Don't have to worry about tombstones. Delete all keys from all versions for this instance id.
minTKey := storage.MinTKey(storage.TKeyMinClass)
maxTKey := storage.MaxTKey(storage.TKeyMaxClass)
vctx, ok := ctx.(storage.VersionedCtx)
if !ok {
return fmt.Errorf("Non-versioned context passed to DELETE ALL VERSIONS in basholeveldb driver: %v", ctx)
}
minKey, err := vctx.MinVersionKey(minTKey)
if err != nil {
return err
}
maxKey, err := vctx.MaxVersionKey(maxTKey)
if err != nil {
return err
}
const BATCH_SIZE = 10000
batch := db.NewBatch(ctx).(*goBatch)
ro := levigo.NewReadOptions()
it := db.ldb.NewIterator(ro)
defer func() {
it.Close()
dvid.StopCgo()
}()
numKV := 0
it.Seek(minKey)
deleteVersion := ctx.VersionID()
for {
if it.Valid() {
itKey := it.Key()
storage.StoreKeyBytesRead <- len(itKey)
// Did we pass the final key?
if bytes.Compare(itKey, maxKey) > 0 {
break
}
if !allVersions {
_, v, _, err := storage.DataKeyToLocalIDs(itKey)
if err != nil {
return fmt.Errorf("Error on DELETE ALL for version %d: %v", ctx.VersionID(), err)
}
if v != deleteVersion {
it.Next()
continue
}
}
batch.WriteBatch.Delete(itKey)
if (numKV+1)%BATCH_SIZE == 0 {
if err := batch.Commit(); err != nil {
batch.Close()
return fmt.Errorf("Error on DELETE ALL at key-value pair %d: %v", numKV, err)
}
batch = db.NewBatch(ctx).(*goBatch)
}
numKV++
it.Next()
} else {
break
}
}
if numKV%BATCH_SIZE != 0 {
if err := batch.Commit(); err != nil {
batch.Close()
return fmt.Errorf("Error on last batch DELETE: %v\n", err)
}
}
dvid.Debugf("Deleted %d key-value pairs via DELETE ALL for %s.\n", numKV, ctx)
return nil
}