// returned map will only include labels that had previously been seen (has key)
func (d *Data) getCounts(ctx *datastore.VersionedCtx, labels map[indexedLabel]int32) (counts map[indexedLabel]uint32, err error) {
var store storage.OrderedKeyValueDB
store, err = d.GetOrderedKeyValueDB()
if err != nil {
return
}
counts = make(map[indexedLabel]uint32, len(labels))
var i IndexType
var label uint64
var val []byte
for il := range labels {
i, label, err = decodeIndexedLabel(il)
if err != nil {
return
}
val, err = store.Get(ctx, NewTypeLabelTKey(i, label))
if err != nil {
return
}
if val == nil {
continue
}
if len(val) != 4 {
err = fmt.Errorf("bad size in value for index type %s, label %d: value has length %d", i, label, len(val))
return
}
counts[il] = binary.LittleEndian.Uint32(val)
}
return
}
// copyData copies all key-value pairs pertinent to the given data instance d2. If d2 is nil,
// the destination data instance is d1, useful for migration of data to a new store.
// Each datatype can implement filters that can restrict the transmitted key-value pairs
// based on the given FilterSpec.
func copyData(oldKV, newKV storage.OrderedKeyValueDB, d1, d2 dvid.Data, uuid dvid.UUID, f storage.Filter, flatten bool) error {
// Get data context for this UUID.
v, err := VersionFromUUID(uuid)
if err != nil {
return err
}
srcCtx := NewVersionedCtx(d1, v)
var dstCtx *VersionedCtx
if d2 == nil {
d2 = d1
dstCtx = srcCtx
} else {
dstCtx = NewVersionedCtx(d2, v)
}
// Send this instance's key-value pairs
var wg sync.WaitGroup
wg.Add(1)
stats := new(txStats)
stats.lastTime = time.Now()
var kvTotal, kvSent int
var bytesTotal, bytesSent uint64
keysOnly := false
if flatten {
// Start goroutine to receive flattened key-value pairs and store them.
ch := make(chan *storage.TKeyValue, 1000)
go func() {
for {
tkv := <-ch
if tkv == nil {
wg.Done()
dvid.Infof("Copied %d %q key-value pairs (%s, out of %d kv pairs, %s) [flattened]\n",
kvSent, d1.DataName(), humanize.Bytes(bytesSent), kvTotal, humanize.Bytes(bytesTotal))
stats.printStats()
return
}
kvTotal++
curBytes := uint64(len(tkv.V) + len(tkv.K))
bytesTotal += curBytes
if f != nil {
skip, err := f.Check(tkv)
if err != nil {
dvid.Errorf("problem applying filter on data %q: %v\n", d1.DataName(), err)
continue
}
if skip {
continue
}
}
kvSent++
bytesSent += curBytes
if err := newKV.Put(dstCtx, tkv.K, tkv.V); err != nil {
dvid.Errorf("can't put k/v pair to destination instance %q: %v\n", d2.DataName(), err)
}
stats.addKV(tkv.K, tkv.V)
}
}()
begKey, endKey := srcCtx.TKeyRange()
err := oldKV.ProcessRange(srcCtx, begKey, endKey, &storage.ChunkOp{}, func(c *storage.Chunk) error {
if c == nil {
return fmt.Errorf("received nil chunk in flatten push for data %s", d1.DataName())
}
ch <- c.TKeyValue
return nil
})
ch <- nil
if err != nil {
return fmt.Errorf("error in flatten push for data %q: %v", d1.DataName(), err)
}
} else {
// Start goroutine to receive all key-value pairs and store them.
ch := make(chan *storage.KeyValue, 1000)
go func() {
for {
kv := <-ch
if kv == nil {
wg.Done()
dvid.Infof("Sent %d %q key-value pairs (%s, out of %d kv pairs, %s)\n",
kvSent, d1.DataName(), humanize.Bytes(bytesSent), kvTotal, humanize.Bytes(bytesTotal))
stats.printStats()
return
}
tkey, err := storage.TKeyFromKey(kv.K)
if err != nil {
dvid.Errorf("couldn't get %q TKey from Key %v: %v\n", d1.DataName(), kv.K, err)
continue
}
kvTotal++
curBytes := uint64(len(kv.V) + len(kv.K))
bytesTotal += curBytes
if f != nil {
skip, err := f.Check(&storage.TKeyValue{K: tkey, V: kv.V})
if err != nil {
dvid.Errorf("problem applying filter on data %q: %v\n", d1.DataName(), err)
continue
}
if skip {
continue
}
}
kvSent++
bytesSent += curBytes
if dstCtx != nil {
err := dstCtx.UpdateInstance(kv.K)
if err != nil {
dvid.Errorf("can't update raw key to new data instance %q: %v\n", d2.DataName(), err)
}
}
if err := newKV.RawPut(kv.K, kv.V); err != nil {
dvid.Errorf("can't put k/v pair to destination instance %q: %v\n", d2.DataName(), err)
}
stats.addKV(kv.K, kv.V)
}
}()
begKey, endKey := srcCtx.KeyRange()
if err = oldKV.RawRangeQuery(begKey, endKey, keysOnly, ch, nil); err != nil {
return fmt.Errorf("push voxels %q range query: %v", d1.DataName(), err)
}
}
wg.Wait()
return nil
}